openclaw/Peekaboo
openclaw/Peekaboo
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 17 Wiki Activity

fix(cli): harden e2e error output and aliases

Browse Source
This commit is contained in:
Peter Steinberger 2026-05-09 06:42:26 -04:00
parent b79a2a2c68
commit 268ea1df98
11 changed files with 189 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Apps/CLI/Sources/PeekabooCLI/CLI/Output/JSONOutput.swift
View File

@ -87,6 +87,7 @@ enum ErrorCode: String, Codable {
case ELEMENT_NOT_FOUND
case SESSION_NOT_FOUND
case SNAPSHOT_NOT_FOUND
case SNAPSHOT_STALE
case APPLICATION_NOT_FOUND
case NO_POINT_SPECIFIED
case INVALID_COORDINATES

192
Apps/CLI/Sources/PeekabooCLI/Commands/AI/SeeCommand.swift
View File

@ -147,113 +147,105 @@ struct SeeCommand: ApplicationResolvable, ErrorHandlingCommand, RuntimeOptionsCo
"error": error.localizedDescription,
]
)
throw error
self.handleError(error)
throw ExitCode.failure
}
}
private func runImpl(startTime: Date, logger: Logger) async throws {
do {
// ScreenCaptureService performs the authoritative permission check inside each capture path.
// Avoid duplicating that TCC probe here; `see` is often called in latency-sensitive loops.
// ScreenCaptureService performs the authoritative permission check inside each capture path.
// Avoid duplicating that TCC probe here; `see` is often called in latency-sensitive loops.
// Perform capture and element detection
logger.verbose("Starting capture and detection phase", category: "Capture")
let captureResult = try await performCaptureWithDetection()
logger.verbose("Capture completed successfully", category: "Capture", metadata: [
"snapshotId": captureResult.snapshotId,
"elementCount": captureResult.elements.all.count,
"screenshotSize": self.getFileSize(captureResult.screenshotPath) ?? 0,
])
// Perform capture and element detection
logger.verbose("Starting capture and detection phase", category: "Capture")
let captureResult = try await performCaptureWithDetection()
logger.verbose("Capture completed successfully", category: "Capture", metadata: [
"snapshotId": captureResult.snapshotId,
"elementCount": captureResult.elements.all.count,
"screenshotSize": self.getFileSize(captureResult.screenshotPath) ?? 0,
])
// Generate annotated screenshot if requested
var annotatedPath = captureResult.annotatedPath
let annotationsAllowed = self.allowsAnnotationForCurrentCapture()
if self.annotate, !annotationsAllowed {
self.logger.info("Annotation is disabled for full screen captures due to performance constraints")
}
if self.annotate, annotatedPath == nil, annotationsAllowed {
logger.operationStart("generate_annotations")
annotatedPath = try await self.generateAnnotatedScreenshot(
snapshotId: captureResult.snapshotId,
originalPath: captureResult.screenshotPath
)
if let annotatedPath,
annotatedPath != captureResult.screenshotPath {
try await self.services.snapshots.storeAnnotatedScreenshot(
snapshotId: captureResult.snapshotId,
annotatedScreenshotPath: annotatedPath
)
}
logger.operationComplete("generate_annotations", metadata: [
"annotatedPath": annotatedPath ?? "none",
])
}
if self.annotate, annotationsAllowed, annotatedPath == nil, !self.jsonOutput {
print("\(AgentDisplayTokens.Status.warning) No interactive UI elements found to annotate")
} else if self.annotate, annotationsAllowed, let annotatedPath, !self.jsonOutput {
let interactableElements = captureResult.elements.all.filter(\.isEnabled)
print("📝 Created annotated screenshot with \(interactableElements.count) interactive elements")
self.logger.verbose("Annotated screenshot path: \(annotatedPath)")
}
// Perform AI analysis if requested
var analysisResult: SeeAnalysisData?
if let prompt = analyze {
// Pre-analysis diagnostics
let fileSize = (try? FileManager.default
.attributesOfItem(atPath: captureResult.screenshotPath)[.size] as? Int) ?? 0
logger.verbose(
"Starting AI analysis",
category: "AI",
metadata: [
"imagePath": captureResult.screenshotPath,
"imageSizeBytes": fileSize,
"promptLength": prompt.count
]
)
logger.operationStart("ai_analysis", metadata: ["promptPreview": String(prompt.prefix(80))])
logger.startTimer("ai_generate")
analysisResult = try await self.performAnalysisDetailed(
imagePath: captureResult.screenshotPath,
prompt: prompt
)
logger.stopTimer("ai_generate")
logger.operationComplete(
"ai_analysis",
success: analysisResult != nil,
metadata: [
"provider": analysisResult?.provider ?? "unknown",
"model": analysisResult?.model ?? "unknown"
]
)
}
// Output results
let executionTime = Date().timeIntervalSince(startTime)
logger.operationComplete("see_command", metadata: [
"executionTimeMs": Int(executionTime * 1000),
"success": true,
])
let context = SeeCommandRenderContext(
snapshotId: captureResult.snapshotId,
screenshotPath: captureResult.screenshotPath,
annotatedPath: annotatedPath,
metadata: captureResult.metadata,
elements: captureResult.elements,
analysis: analysisResult,
executionTime: executionTime,
observation: captureResult.observation
)
await self.renderResults(context: context)
} catch {
logger.operationComplete("see_command", success: false, metadata: [
"error": error.localizedDescription,
])
self.handleError(error) // Use protocol's error handling
throw ExitCode.failure
// Generate annotated screenshot if requested
var annotatedPath = captureResult.annotatedPath
let annotationsAllowed = self.allowsAnnotationForCurrentCapture()
if self.annotate, !annotationsAllowed {
self.logger.info("Annotation is disabled for full screen captures due to performance constraints")
}
if self.annotate, annotatedPath == nil, annotationsAllowed {
logger.operationStart("generate_annotations")
annotatedPath = try await self.generateAnnotatedScreenshot(
snapshotId: captureResult.snapshotId,
originalPath: captureResult.screenshotPath
)
if let annotatedPath,
annotatedPath != captureResult.screenshotPath {
try await self.services.snapshots.storeAnnotatedScreenshot(
snapshotId: captureResult.snapshotId,
annotatedScreenshotPath: annotatedPath
)
}
logger.operationComplete("generate_annotations", metadata: [
"annotatedPath": annotatedPath ?? "none",
])
}
if self.annotate, annotationsAllowed, annotatedPath == nil, !self.jsonOutput {
print("\(AgentDisplayTokens.Status.warning) No interactive UI elements found to annotate")
} else if self.annotate, annotationsAllowed, let annotatedPath, !self.jsonOutput {
let interactableElements = captureResult.elements.all.filter(\.isEnabled)
print("📝 Created annotated screenshot with \(interactableElements.count) interactive elements")
self.logger.verbose("Annotated screenshot path: \(annotatedPath)")
}
// Perform AI analysis if requested
var analysisResult: SeeAnalysisData?
if let prompt = analyze {
// Pre-analysis diagnostics
let fileSize = (try? FileManager.default
.attributesOfItem(atPath: captureResult.screenshotPath)[.size] as? Int) ?? 0
logger.verbose(
"Starting AI analysis",
category: "AI",
metadata: [
"imagePath": captureResult.screenshotPath,
"imageSizeBytes": fileSize,
"promptLength": prompt.count
]
)
logger.operationStart("ai_analysis", metadata: ["promptPreview": String(prompt.prefix(80))])
logger.startTimer("ai_generate")
analysisResult = try await self.performAnalysisDetailed(
imagePath: captureResult.screenshotPath,
prompt: prompt
)
logger.stopTimer("ai_generate")
logger.operationComplete(
"ai_analysis",
success: analysisResult != nil,
metadata: [
"provider": analysisResult?.provider ?? "unknown",
"model": analysisResult?.model ?? "unknown"
]
)
}
// Output results
let executionTime = Date().timeIntervalSince(startTime)
logger.operationComplete("see_command", metadata: [
"executionTimeMs": Int(executionTime * 1000),
"success": true,
])
let context = SeeCommandRenderContext(
snapshotId: captureResult.snapshotId,
screenshotPath: captureResult.screenshotPath,
annotatedPath: annotatedPath,
metadata: captureResult.metadata,
elements: captureResult.elements,
analysis: analysisResult,
executionTime: executionTime,
observation: captureResult.observation
)
await self.renderResults(context: context)
}
func getFileSize(_ path: String) -> Int? {

25
Apps/CLI/Sources/PeekabooCLI/Commands/Base/CommandErrorHandling.swift
View File

@ -1,5 +1,6 @@
import Commander
import Foundation
import PeekabooBridge
import PeekabooCore
import PeekabooFoundation
@ -49,6 +50,10 @@ extension ErrorHandlingCommand {
self.mapCaptureErrorToCode(captureError)
case let observationError as DesktopObservationError:
self.mapObservationErrorToCode(observationError)
case let bridgeError as PeekabooBridgeErrorEnvelope:
errorCode(for: bridgeError)
case let posixError as POSIXError:
errorCode(for: posixError)
case is Commander.ValidationError:
.VALIDATION_ERROR
default:
@ -101,6 +106,8 @@ extension ErrorHandlingCommand {
.SESSION_NOT_FOUND
case .snapshotNotFound:
.SNAPSHOT_NOT_FOUND
case .snapshotStale:
.SNAPSHOT_STALE
case .menuNotFound:
.MENU_BAR_NOT_FOUND
case .menuItemNotFound:
@ -223,3 +230,21 @@ func errorCode(for focusError: FocusError) -> ErrorCode {
.WINDOW_NOT_FOUND
}
}
func errorCode(for bridgeError: PeekabooBridgeErrorEnvelope) -> ErrorCode {
switch bridgeError.code {
case .timeout:
.TIMEOUT
default:
.INTERNAL_SWIFT_ERROR
}
}
func errorCode(for posixError: POSIXError) -> ErrorCode {
switch posixError.code {
case .ETIMEDOUT:
.TIMEOUT
default:
.INTERNAL_SWIFT_ERROR
}
}

7
Apps/CLI/Sources/PeekabooCLI/Commands/Interaction/PressCommand+CommanderMetadata.swift
View File

@ -7,10 +7,15 @@ extension PressCommand: CommanderSignatureProviding {
.make(
label: "keys",
help: "Key(s) to press",
isOptional: false
isOptional: true
),
],
options: [
.commandOption(
"key",
help: "Key to press (alternative to positional argument)",
long: "key"
),
.commandOption(
"count",
help: "Repeat count for all keys",

9
Apps/CLI/Sources/PeekabooCLI/Commands/Interaction/PressCommand.swift
View File

@ -218,10 +218,15 @@ extension PressCommand: AsyncRuntimeCommand {}
@MainActor
extension PressCommand: CommanderBindableCommand {
mutating func applyCommanderValues(_ values: CommanderBindableValues) throws {
guard !values.positional.isEmpty else {
let resolvedKeys = if values.positional.isEmpty {
values.singleOption("key").map { [$0] } ?? []
} else {
values.positional
}
guard !resolvedKeys.isEmpty else {
throw CommanderBindingError.missingArgument(label: "keys")
}
self.keys = values.positional
self.keys = resolvedKeys
self.target = try values.makeInteractionTargetOptions()
if let count: Int = try values.decodeOption("count", as: Int.self) {
self.count = count

3
Apps/CLI/Sources/PeekabooCLI/Commands/Interaction/SetValueCommand.swift
View File

@ -132,7 +132,7 @@ extension SetValueCommand: AsyncRuntimeCommand {}
@MainActor
extension SetValueCommand: CommanderBindableCommand {
mutating func applyCommanderValues(_ values: CommanderBindableValues) throws {
self.value = try values.decodeOptionalPositional(0, label: "value")
self.value = try values.decodeOptionalPositional(0, label: "value") ?? values.singleOption("value")
self.on = values.singleOption("on")
self.snapshot = values.singleOption("snapshot")
}
@ -145,6 +145,7 @@ extension SetValueCommand: CommanderSignatureProviding {
.make(label: "value", help: "Value to set", isOptional: true),
],
options: [
.commandOption("value", help: "Value to set (alternative to positional argument)", long: "value"),
.commandOption("on", help: "Element ID or query to set", long: "on"),
.commandOption("snapshot", help: "Snapshot ID (uses latest if not specified)", long: "snapshot"),
]

5
Apps/CLI/Sources/PeekabooCLI/Commands/Interaction/TypeCommand+CommanderMetadata.swift
View File

@ -11,6 +11,11 @@ extension TypeCommand: CommanderSignatureProviding {
),
],
options: [
.commandOption(
"textOption",
help: "Text to type (alternative to positional argument)",
long: "text"
),
.commandOption(
"snapshot",
help: "Snapshot ID (uses latest if not specified)",

1
Apps/CLI/Tests/CLIAutomationTests/DragCommandTests.swift
View File

@ -83,6 +83,7 @@ struct DragCommandTests {
#expect(ErrorCode.NO_POINT_SPECIFIED.rawValue == "NO_POINT_SPECIFIED")
#expect(ErrorCode.INVALID_COORDINATES.rawValue == "INVALID_COORDINATES")
#expect(ErrorCode.SNAPSHOT_NOT_FOUND.rawValue == "SNAPSHOT_NOT_FOUND")
#expect(ErrorCode.SNAPSHOT_STALE.rawValue == "SNAPSHOT_STALE")
}
@Test

32
Apps/CLI/Tests/CoreCLITests/CommanderBinderInteractionAliasTests.swift Normal file
View File

@ -0,0 +1,32 @@
import Commander
import Testing
@testable import PeekabooCLI
struct CommanderBinderInteractionAliasTests {
@Test
func `Type command accepts text option alias`() throws {
let parsed = ParsedValues(positional: [], options: ["textOption": ["Hello option"]], flags: [])
let command = try CommanderCLIBinder.instantiateCommand(ofType: TypeCommand.self, parsedValues: parsed)
#expect(command.text == nil)
#expect(command.textOption == "Hello option")
}
@Test
func `Press command accepts key option alias`() throws {
let parsed = ParsedValues(positional: [], options: ["key": ["return"]], flags: [])
let command = try CommanderCLIBinder.instantiateCommand(ofType: PressCommand.self, parsedValues: parsed)
#expect(command.keys == ["return"])
}
@Test
func `Set value command accepts value option alias`() throws {
let parsed = ParsedValues(
positional: [],
options: ["value": ["Hello value"], "on": ["elem_2"]],
flags: []
)
let command = try CommanderCLIBinder.instantiateCommand(ofType: SetValueCommand.self, parsedValues: parsed)
#expect(command.value == "Hello value")
#expect(command.on == "elem_2")
}
}

14
Apps/CLI/Tests/CoreCLITests/ErrorHandlingTests.swift
View File

@ -3,7 +3,9 @@
// PeekabooCLI
//
import Foundation
import Testing
@testable import PeekabooBridge
@testable import PeekabooCLI
@testable import PeekabooCore
@ -32,4 +34,16 @@ struct FocusErrorMappingTests {
let code = errorCode(for: .timeoutWaitingForCondition)
#expect(code == .TIMEOUT)
}
@Test
func `bridge timeout maps to TIMEOUT`() {
let code = errorCode(for: PeekabooBridgeErrorEnvelope(code: .timeout, message: "Timed out"))
#expect(code == .TIMEOUT)
}
@Test
func `POSIX timeout maps to TIMEOUT`() {
let code = errorCode(for: POSIXError(.ETIMEDOUT))
#expect(code == .TIMEOUT)
}
}

4
CHANGELOG.md
View File

@ -175,6 +175,10 @@
- `peekaboo set-value` now reports unsupported direct value writes as `INVALID_INPUT` with the target element named instead of surfacing an internal Swift error.
- `peekaboo config add-provider --dry-run` and `remove-provider --dry-run` now preserve the config file when invoked through the Commander CLI path.
- `peekaboo config add` now exits nonzero when credential validation fails or times out, matching its JSON `success: false` response.
- Explicit stale snapshots now report the JSON error code `SNAPSHOT_STALE` instead of falling through to `UNKNOWN_ERROR`.
- Bridge transport timeouts now report the JSON error code `TIMEOUT` instead of `INTERNAL_SWIFT_ERROR`.
- `peekaboo see --json` now emits a single structured error response for capture and detection failures instead of occasionally printing two JSON objects.
- `peekaboo type --text`, `peekaboo press --key`, and `peekaboo set-value --value` now work as aliases for their positional arguments.
- Peekaboo.app no longer crashes at launch on macOS 26 when the hidden Settings helper window is created.
- `peekaboo hotkey` now accepts plus-separated shortcuts such as `cmd+s`, matching common CLI shorthand and the help text while still supporting comma and space separators.
- `peekaboo type` is more reliable in VM and headless launch paths because printable ASCII input now uses physical key events instead of Unicode-only events.