fix(cli): keep implicit see screenshots private (#200)
Some checks are pending
macOS CI / PeekabooCore build & tests (push) Waiting to run
macOS CI / Peekaboo CLI build & tests (push) Blocked by required conditions
macOS CI / Tachikoma build & tests (push) Blocked by required conditions
macOS CI / Build macOS apps (Peekaboo + Inspector) (push) Blocked by required conditions
macOS CI / SwiftLint (core + CLI) (push) Blocked by required conditions
Website (GitHub Pages) / build (push) Waiting to run
Website (GitHub Pages) / deploy (push) Blocked by required conditions

This commit is contained in:
Peter Steinberger 2026-06-24 00:58:16 +01:00 committed by GitHub
parent db5192bb37
commit 4085f18ddc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 220 additions and 8 deletions

View File

@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
## [3.5.3] - 2026-06-13
### Fixed
- JSON-only `peekaboo see` runs without `--path` now keep required screenshots in snapshot storage instead of leaving files on Desktop or exposing their temporary paths. Thanks @coygeek for #196.
- Background element/query/coordinate clicks now pin actions to the requested process and exact window, reject mismatched window/PID selectors and unverifiable snapshots, invalidate implicit latest snapshots without deleting history, and no longer require Event Synthesizing when Accessibility completes the click.
- App launch, open, and inventory commands now use the selected runtime host, fixing sandboxed LaunchServices failures; launch/open preserve `--no-focus` and caller-relative app paths, relaunch preflights and keeps quit/wait/launch in one daemon-held transaction, build-scoped fallback daemons remain reusable and controllable across native/Rosetta execution and executable upgrades, incompatible legacy hosts no longer force sandboxed local fallback, and inventory ignores unrelated input overrides.
- Agent, MCP, script, CLI, and bridge mutations now advance implicit-snapshot watermarks at host-confirmed completion or observation boundaries, keep durable pending barriers across client timeouts/disconnects without hiding the acting command's own snapshot, carry remote script observation certificates, recover safely from PID reuse, ignore unavailable alternate hosts after protecting the selected/local stores, and preserve explicit snapshot history.

View File

@ -5,7 +5,17 @@ import PeekabooFoundation
@MainActor
extension SeeCommand {
func screenshotOutputPath() -> String {
var usesTemporaryScreenshotOutput: Bool {
self.jsonOutput && self.path == nil
}
func screenshotOutputPath(snapshotID: String? = nil) -> String {
if self.usesTemporaryScreenshotOutput {
return self.temporaryScreenshotDirectory(snapshotID: snapshotID)
.appendingPathComponent("raw.png")
.path
}
let timestamp = Date().timeIntervalSince1970
let filename = "peekaboo_see_\(Int(timestamp)).png"
return ObservationCommandSupport.outputPath(
@ -16,8 +26,8 @@ extension SeeCommand {
)
}
func saveScreenshot(_ imageData: Data) throws -> String {
let outputPath = self.screenshotOutputPath()
func saveScreenshot(_ imageData: Data, snapshotID: String) throws -> String {
let outputPath = self.screenshotOutputPath(snapshotID: snapshotID)
let directory = (outputPath as NSString).deletingLastPathComponent
try FileManager.default.createDirectory(
@ -31,6 +41,17 @@ extension SeeCommand {
return outputPath
}
func cleanupTemporaryScreenshotOutput(snapshotID: String) {
guard self.usesTemporaryScreenshotOutput else { return }
try? FileManager.default.removeItem(at: self.temporaryScreenshotDirectory(snapshotID: snapshotID))
}
private func temporaryScreenshotDirectory(snapshotID: String?) -> URL {
FileManager.default.temporaryDirectory
.appendingPathComponent("peekaboo-see", isDirectory: true)
.appendingPathComponent(snapshotID ?? UUID().uuidString, isDirectory: true)
}
func resolveSeeWindowIndex(appIdentifier: String, titleFragment: String?) async throws -> Int? {
guard let fragment = titleFragment, !fragment.isEmpty else {
return nil

View File

@ -15,7 +15,7 @@ extension SeeCommand {
let captureResult = captureContext.captureResult
self.logger.startTimer("file_write")
let outputPath = try saveScreenshot(captureResult.imageData)
let outputPath = try saveScreenshot(captureResult.imageData, snapshotID: snapshotID)
self.logger.stopTimer("file_write")
let windowContext = WindowContext(

View File

@ -89,7 +89,7 @@ extension SeeCommand {
),
detection: self.observationDetectionOptions(for: target),
output: DesktopObservationOutputOptions(
path: self.screenshotOutputPath(),
path: self.screenshotOutputPath(snapshotID: snapshotID),
saveRawScreenshot: true,
saveAnnotatedScreenshot: self.annotate && self.allowsAnnotation(for: target),
saveSnapshot: true,

View File

@ -176,9 +176,10 @@ extension SeeCommand {
}
private func snapshotPaths(for context: SeeCommandRenderContext) -> SnapshotPaths {
SnapshotPaths(
raw: context.screenshotPath,
annotated: context.annotatedPath ?? "",
let publishesScreenshotPaths = !self.usesTemporaryScreenshotOutput
return SnapshotPaths(
raw: publishesScreenshotPaths ? context.screenshotPath : "",
annotated: publishesScreenshotPaths ? context.annotatedPath ?? "" : "",
map: self.services.snapshots.getSnapshotStoragePath() + "/\(context.snapshotId)/snapshot.json"
)
}

View File

@ -151,6 +151,11 @@ struct SeeCommand: ApplicationResolvable, ErrorHandlingCommand, RuntimeOptionsCo
) {
try await snapshotManager.createSnapshot(pendingAt: observationStartedAt)
}
defer {
if snapshotManager.copiesScreenshotArtifactsIntoStorage {
commandCopy.cleanupTemporaryScreenshotOutput(snapshotID: snapshotID)
}
}
var observationCompleted = false
do {
let preparationTimeout = try Self.remainingObservationTimeout(

View File

@ -263,6 +263,92 @@ struct SeeCommandRuntimeTests {
}
}
@Test
func `JSON See without path keeps screenshot private to snapshot storage`() async throws {
try await self.withTempConfigEnv { _ in
let fixture = Self.makeSeeCommandRuntimeFixture()
let automation = StubAutomationService()
automation.nextDetectionResult = fixture.detectionResult
let (context, _) = Self.makeSeeCommandRuntimeContext(
automation: automation,
screenCapture: fixture.screenCapture,
applicationInfo: fixture.applicationInfo,
windowInfo: fixture.windowInfo
)
context.snapshots.copiesScreenshotArtifactsIntoStorage = true
let result = try await InProcessCommandRunner.run(
[
"see",
"--mode", "frontmost",
"--no-web-focus",
"--json",
],
services: context.services
)
let data = try #require(result.stdout.data(using: .utf8))
let response = try JSONDecoder().decode(
CodableJSONResponse<SeeResult>.self,
from: data
)
let storedScreenshot = try #require(
context.snapshots.storedScreenshots.values.flatMap(\.self).first
)
#expect(result.exitStatus == 0)
#expect(response.data.screenshot_raw.isEmpty)
#expect(response.data.screenshot_annotated.isEmpty)
#expect(storedScreenshot.path.hasPrefix(FileManager.default.temporaryDirectory.path))
#expect(!FileManager.default.fileExists(atPath: storedScreenshot.path))
}
}
@Test
func `JSON See retains temporary screenshot for borrowing snapshot backend`() async throws {
try await self.withTempConfigEnv { _ in
let fixture = Self.makeSeeCommandRuntimeFixture()
let automation = StubAutomationService()
automation.nextDetectionResult = fixture.detectionResult
let (context, _) = Self.makeSeeCommandRuntimeContext(
automation: automation,
screenCapture: fixture.screenCapture,
applicationInfo: fixture.applicationInfo,
windowInfo: fixture.windowInfo
)
let result = try await InProcessCommandRunner.run(
[
"see",
"--mode", "frontmost",
"--no-web-focus",
"--json",
],
services: context.services
)
let data = try #require(result.stdout.data(using: .utf8))
let response = try JSONDecoder().decode(
CodableJSONResponse<SeeResult>.self,
from: data
)
let storedScreenshot = try #require(
context.snapshots.storedScreenshots.values.flatMap(\.self).first
)
defer {
try? FileManager.default.removeItem(
at: URL(fileURLWithPath: storedScreenshot.path).deletingLastPathComponent()
)
}
#expect(result.exitStatus == 0)
#expect(response.data.screenshot_raw.isEmpty)
#expect(FileManager.default.fileExists(atPath: storedScreenshot.path))
}
}
@Test
func `See suppresses success output when snapshot publication fails`() async throws {
try await self.withTempConfigEnv { _ in

View File

@ -590,6 +590,7 @@ final class StubApplicationService: ApplicationServiceProtocol {
final class StubSnapshotManager: SnapshotManagerProtocol, @unchecked Sendable {
let supportsImplicitLatestSnapshotInvalidation = true
var copiesScreenshotArtifactsIntoStorage = false
var effectiveImplicitLatestInvalidationWatermark: Date?
private(set) var detectionResults: [String: ElementDetectionResult] = [:]
private(set) var snapshotInfos: [String: SnapshotInfo] = [:]

View File

@ -233,6 +233,19 @@ struct SeeCommandAnnotationTests {
)
}
@Test
@MainActor
func `JSON output without path uses snapshot-scoped temporary screenshot`() throws {
let snapshotID = "snapshot-test"
let command = try SeeCommand.parse(["--json"])
let output = URL(fileURLWithPath: command.screenshotOutputPath(snapshotID: snapshotID))
#expect(command.usesTemporaryScreenshotOutput)
#expect(output.lastPathComponent == "raw.png")
#expect(output.deletingLastPathComponent().lastPathComponent == snapshotID)
#expect(output.path.hasPrefix(FileManager.default.temporaryDirectory.path))
}
@Test
func `Coordinate system conversion for NSGraphicsContext`() {
// Given a window-relative element bounds with top-left origin

View File

@ -3,6 +3,7 @@
## [3.5.3] - 2026-06-13
### Fixed
- JSON-only `peekaboo see` runs without `--path` now keep required screenshots in snapshot storage instead of leaving files on Desktop or exposing their temporary paths. Thanks @coygeek for #196.
- Watch captures now honor stop requests during transient ScreenCaptureKit retry backoff instead of waiting out the full delay. Thanks @SebTardif for #193.
- Peekaboo agent skill install and usage guidance now uses the current `skills/peekaboo` path, treats observed element IDs as opaque, and keeps screenshot artifacts in explicit temporary paths. Thanks @coygeek for #197.
- `peekaboo app list` now excludes accessory/background processes by default, while `--include-background` restores them as documented.

View File

@ -35,6 +35,9 @@ public protocol SnapshotManagerProtocol: Sendable {
/// Whether this manager applies cutoff-aware, non-destructive implicit-latest invalidation.
var supportsImplicitLatestSnapshotInvalidation: Bool { get }
/// Whether `storeScreenshot` copies source artifacts into independently managed storage.
var copiesScreenshotArtifactsIntoStorage: Bool { get }
/// Effective desktop-wide cutoff applied to implicit latest-snapshot lookup.
/// Managers without a shared watermark can rely on the default `nil` implementation.
var effectiveImplicitLatestInvalidationWatermark: Date? { get }
@ -135,6 +138,10 @@ public protocol SnapshotManagerProtocol: Sendable {
}
extension SnapshotManagerProtocol {
public var copiesScreenshotArtifactsIntoStorage: Bool {
false
}
public var supportsImplicitLatestSnapshotInvalidation: Bool {
false
}

View File

@ -219,6 +219,10 @@ extension InMemorySnapshotManager {
for entry in self.entries.values {
self.deleteArtifacts(for: entry.snapshotData)
}
} else {
for entry in self.entries.values {
self.deleteManagedTemporaryArtifacts(for: entry.snapshotData)
}
}
self.entries.removeAll()
self.implicitLatestInvalidatedAt = nil

View File

@ -24,6 +24,8 @@ extension InMemorySnapshotManager {
}
if self.options.deleteArtifactsOnCleanup {
self.deleteArtifacts(for: entry.snapshotData)
} else {
self.deleteManagedTemporaryArtifacts(for: entry.snapshotData)
}
}
@ -43,4 +45,25 @@ extension InMemorySnapshotManager {
try? fm.removeItem(atPath: annotatedPath)
}
}
func deleteManagedTemporaryArtifacts(for snapshotData: UIAutomationSnapshot) {
let paths = [snapshotData.screenshotPath, snapshotData.annotatedPath]
.compactMap(\.self)
.filter(Self.isManagedTemporaryArtifact)
let directories = Set(paths.map { URL(fileURLWithPath: $0).deletingLastPathComponent() })
for path in paths {
try? FileManager.default.removeItem(atPath: path)
}
for directory in directories {
try? FileManager.default.removeItem(at: directory)
}
}
private static func isManagedTemporaryArtifact(_ path: String) -> Bool {
let root = FileManager.default.temporaryDirectory
.appendingPathComponent("peekaboo-see", isDirectory: true)
.standardizedFileURL.path + "/"
return URL(fileURLWithPath: path).standardizedFileURL.path.hasPrefix(root)
}
}

View File

@ -8,6 +8,7 @@ import PeekabooFoundation
@MainActor
public final class SnapshotManager: SnapshotManagerProtocol {
public let supportsImplicitLatestSnapshotInvalidation = true
public let copiesScreenshotArtifactsIntoStorage = true
public var effectiveImplicitLatestInvalidationWatermark: Date? {
let shared = self.desktopMutationWatermarkStore?.effectiveWatermark()

View File

@ -344,6 +344,42 @@ struct InMemorySnapshotManagerTests {
#expect(snapshots.contains { $0.id == first } == false)
}
@Test
func `snapshot cleanup removes managed temporary artifacts`() async throws {
let snapshotId = "managed-temp"
let directory = FileManager.default.temporaryDirectory
.appendingPathComponent("peekaboo-see/\(snapshotId)", isDirectory: true)
let raw = directory.appendingPathComponent("raw.png")
let annotated = directory.appendingPathComponent("raw_annotated.png")
try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true)
try Data("raw".utf8).write(to: raw)
try Data("annotated".utf8).write(to: annotated)
let manager = InMemorySnapshotManager()
_ = try await manager.createSnapshot()
try await manager.storeScreenshot(Self.screenshotRequest(snapshotId: snapshotId, path: raw.path))
try await manager.storeAnnotatedScreenshot(snapshotId: snapshotId, annotatedScreenshotPath: annotated.path)
try await manager.cleanSnapshot(snapshotId: snapshotId)
#expect(!FileManager.default.fileExists(atPath: raw.path))
#expect(!FileManager.default.fileExists(atPath: annotated.path))
#expect(!FileManager.default.fileExists(atPath: directory.path))
}
@Test
func `snapshot cleanup preserves borrowed screenshot artifacts`() async throws {
let artifact = try Self.createTemporaryArtifact(named: "borrowed-screenshot.png")
defer { try? FileManager.default.removeItem(at: artifact) }
let manager = InMemorySnapshotManager()
let snapshotId = try await manager.createSnapshot()
try await manager.storeScreenshot(Self.screenshotRequest(snapshotId: snapshotId, path: artifact.path))
try await manager.cleanSnapshot(snapshotId: snapshotId)
#expect(FileManager.default.fileExists(atPath: artifact.path))
}
@Test
func `getDetectionResult preserves window context for action re-resolution`() async throws {
let manager = InMemorySnapshotManager()

View File

@ -7,6 +7,7 @@ import PeekabooFoundation
@MainActor
public final class RemoteSnapshotManager: SnapshotManagerProtocol {
public let copiesScreenshotArtifactsIntoStorage = true
public let supportsImplicitLatestSnapshotInvalidation: Bool
public var effectiveImplicitLatestInvalidationWatermark: Date? {

View File

@ -5,6 +5,15 @@ import PeekabooCore
import Testing
struct RemoteSnapshotManagerTests {
@Test
@MainActor
func `remote snapshot storage owns copied screenshot artifacts`() {
let remote = RemoteSnapshotManager(
client: PeekabooBridgeClient(socketPath: "/tmp/unused.sock", requestTimeoutSec: 1))
#expect(remote.copiesScreenshotArtifactsIntoStorage)
}
@Test
func `bridge invalidation payload preserves subsecond cutoff`() throws {
let cutoff = Date(timeIntervalSinceReferenceDate: 123_456_789.123_456)

View File

@ -43,6 +43,8 @@ Note: `--app menubar` captures only the menu bar strip; `--menubar` attempts to
For agent and automation runs, pass `--path` to a known temporary file when using `see` so capture artifacts land where expected. Use `peekaboo inspect-ui --json` when you need AX metadata and no screenshot artifact.
When `--json` is used without `--path`, Peekaboo retains the raw image only in managed snapshot storage and returns empty `screenshot_raw` and `screenshot_annotated` fields. Pass `--path` when the caller needs a directly accessible image file.
## Automatic web focus fallback (Nov 2025)
Modern browsers sometimes keep keyboard focus in the omnibox, which means embedded login forms (Instagram, Facebook, etc.) never expose their `AXTextField` nodes to accessibility clients. Starting November 2025: