diff --git a/package.json b/package.json index 418a73e..bb682c6 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ } }, "scripts": { - "check": "npm run sync:surface -- --check && node scripts/check-sdk-surface.mjs && node scripts/check-kitchen-runtime.mjs && npm run plugin:inspect", + "check": "npm run sync:surface -- --check && node scripts/check-sdk-surface.mjs && node scripts/check-kitchen-runtime.mjs && node scripts/check-kitchen-contract-probes.mjs && npm run plugin:inspect", "pack:check": "node scripts/check-pack-payload.mjs", "plugin:inspect": "plugin-inspector check --config plugin-inspector.config.json --no-openclaw", "plugin:inspect:runtime": "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector check --config plugin-inspector.config.json --no-openclaw --runtime --mock-sdk", diff --git a/reports/kitchen-contract-probes.json b/reports/kitchen-contract-probes.json new file mode 100644 index 0000000..1e9f98f --- /dev/null +++ b/reports/kitchen-contract-probes.json @@ -0,0 +1,501 @@ +{ + "beforeToolCall": { + "allow": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "hook": "before_tool_call", + "route": "hook:before_tool_call", + "matchedKitchen": true, + "scenarioId": "image.generate", + "observedEventKeys": [ + "toolId", + "args" + ], + "observedContextKeys": [ + "providerId" + ], + "params": { + "args": { + "prompt": "generate a kitchen image", + "kitchenSinkScenario": "image.generate", + "kitchenSinkPluginId": "openclaw-kitchen-sink-fixture" + } + }, + "decision": "allow" + }, + "block": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "hook": "before_tool_call", + "route": "hook:before_tool_call", + "matchedKitchen": true, + "scenarioId": "image.generate", + "observedEventKeys": [ + "toolId", + "args" + ], + "observedContextKeys": [ + "providerId" + ], + "params": { + "args": { + "prompt": "kitchen block image generation", + "kitchenSinkScenario": "image.generate", + "kitchenSinkPluginId": "openclaw-kitchen-sink-fixture" + } + }, + "block": true, + "blockReason": "Kitchen Sink fixture blocked kitchen_sink_image_job for image.generate.", + "terminal": true, + "decision": "block" + }, + "approval": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "hook": "before_tool_call", + "route": "hook:before_tool_call", + "matchedKitchen": true, + "scenarioId": "image.generate", + "observedEventKeys": [ + "toolId", + "args" + ], + "observedContextKeys": [ + "providerId" + ], + "params": { + "args": { + "prompt": "kitchen image generation needs approval", + "kitchenSinkScenario": "image.generate", + "kitchenSinkPluginId": "openclaw-kitchen-sink-fixture" + } + }, + "requireApproval": { + "id": "ks_approval_9863b78c", + "title": "Kitchen Sink tool approval", + "reason": "Kitchen Sink fixture requires approval before kitchen_sink_image_job runs.", + "summary": "Approve deterministic image.generate fixture execution.", + "scenarioId": "image.generate", + "pluginId": "openclaw-kitchen-sink-fixture" + }, + "decision": "approval" + } + }, + "conversationPrivacy": { + "input": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "hook": "llm_input", + "route": "hook:llm_input", + "matchedKitchen": true, + "scenarioId": "text.reply", + "observedEventKeys": [ + "prompt", + "apiKey", + "token" + ], + "observedContextKeys": [ + "providerId", + "authorization" + ], + "privacy": { + "boundary": "conversation-observer", + "promptHash": "21ed2705", + "promptLength": 55, + "redactedFields": [ + "event.apiKey", + "event.token", + "context.authorization" + ], + "secretPatternCount": 3, + "storesRawPayload": false, + "exposesRawPayload": false + } + }, + "output": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "hook": "llm_output", + "route": "hook:llm_output", + "matchedKitchen": true, + "scenarioId": "text.reply", + "observedEventKeys": [ + "prompt", + "apiKey", + "token" + ], + "observedContextKeys": [ + "providerId", + "authorization" + ], + "privacy": { + "boundary": "conversation-observer", + "promptHash": "a3e6f809", + "promptLength": 41, + "redactedFields": [ + "event.apiKey", + "event.token", + "context.authorization" + ], + "secretPatternCount": 3, + "storesRawPayload": false, + "exposesRawPayload": false + } + }, + "end": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "hook": "agent_end", + "route": "hook:agent_end", + "matchedKitchen": true, + "scenarioId": "text.reply", + "observedEventKeys": [ + "prompt", + "apiKey", + "token" + ], + "observedContextKeys": [ + "providerId", + "authorization" + ], + "privacy": { + "boundary": "conversation-observer", + "promptHash": "8bf533dd", + "promptLength": 41, + "redactedFields": [ + "event.apiKey", + "event.token", + "context.authorization" + ], + "secretPatternCount": 3, + "storesRawPayload": false, + "exposesRawPayload": false + } + } + }, + "channel": { + "account": { + "accountId": "local", + "name": "Kitchen Sink Local", + "enabled": true, + "configured": true, + "statusState": "ready", + "linked": true, + "running": true, + "connected": true, + "mode": "local", + "health": { + "ok": true, + "checkedAt": "2026-04-28T00:00:00.000Z", + "message": "Kitchen Sink local fixture account is ready." + }, + "capabilities": [ + "text", + "media", + "threads", + "dry-run" + ] + }, + "delivery": { + "channel": "kitchen-sink-channel", + "messageId": "ks_channel_d813aa04", + "conversationId": "kitchen-demo", + "channelId": "kitchen-demo", + "timestamp": 1777494231851, + "deliveryStatus": "sent", + "transport": "kitchen-sink-local", + "meta": { + "kitchenSink": true, + "pluginId": "openclaw-kitchen-sink-fixture", + "scenarioId": "image.generate", + "kind": "text" + } + }, + "route": { + "sessionKey": "kitchen:fixture-agent:kitchen-demo", + "baseSessionKey": "kitchen:fixture-agent:kitchen-demo", + "peer": { + "kind": "direct", + "id": "kitchen-demo" + }, + "chatType": "direct", + "from": "local", + "to": "kitchen-demo", + "threadId": "thread-1" + } + }, + "runtimeRegistrations": { + "registerAgentHarness": { + "count": 1, + "ids": [ + "kitchen-sink-agent-harness" + ] + }, + "registerAgentToolResultMiddleware": { + "count": 2, + "ids": [ + "kitchen-sink-agent-tool-result-middleware", + "kitchen-sink-agent-tool-result-middleware" + ] + }, + "registerAutoEnableProbe": { + "count": 1, + "ids": [ + "kitchen-sink-auto-enable-probe" + ] + }, + "registerChannel": { + "count": 2, + "ids": [ + "kitchen-sink-channel", + "kitchen-sink-channel-probe" + ] + }, + "registerCli": { + "count": 2, + "ids": [ + "kitchen-sink", + "kitchen-sink-cli" + ] + }, + "registerCliBackend": { + "count": 1, + "ids": [ + "kitchen-sink-cli-backend" + ] + }, + "registerCodexAppServerExtensionFactory": { + "count": 1, + "ids": [ + "kitchen-sink-codex-app-server-extension-factory" + ] + }, + "registerCommand": { + "count": 3, + "ids": [ + "kitchen", + "kitchen-sink", + "kitchen-sink-command" + ] + }, + "registerCompactionProvider": { + "count": 2, + "ids": [ + "kitchen-sink-compaction", + "kitchen-sink-compaction-provider" + ] + }, + "registerConfigMigration": { + "count": 1, + "ids": [ + "kitchen-sink-config-migration" + ] + }, + "registerContextEngine": { + "count": 1, + "ids": [ + "kitchen-sink-context-engine" + ] + }, + "registerDetachedTaskRuntime": { + "count": 1, + "ids": [ + "kitchen-sink-detached-task-runtime" + ] + }, + "registerGatewayDiscoveryService": { + "count": 1, + "ids": [ + "kitchen-sink-gateway-discovery-service" + ] + }, + "registerGatewayMethod": { + "count": 2, + "ids": [ + "kitchen-sink-gateway-method", + "kitchen.status" + ] + }, + "registerHook": { + "count": 1, + "ids": [ + "kitchen-sink-hook" + ] + }, + "registerHttpRoute": { + "count": 2, + "ids": [ + "kitchen-sink-http-route", + "kitchen-sink-http-status" + ] + }, + "registerImageGenerationProvider": { + "count": 2, + "ids": [ + "kitchen-sink-image", + "kitchen-sink-image-generation-provider" + ] + }, + "registerInteractiveHandler": { + "count": 2, + "ids": [ + "kitchen-sink-interactive-handler", + "kitchen-sink-interactive-handler" + ] + }, + "registerMediaUnderstandingProvider": { + "count": 2, + "ids": [ + "kitchen-sink-media", + "kitchen-sink-media-understanding-provider" + ] + }, + "registerMemoryCapability": { + "count": 1, + "ids": [ + "kitchen-sink-memory-capability" + ] + }, + "registerMemoryCorpusSupplement": { + "count": 2, + "ids": [ + "kitchen-sink-memory-corpus", + "kitchen-sink-memory-corpus-supplement" + ] + }, + "registerMemoryEmbeddingProvider": { + "count": 2, + "ids": [ + "kitchen-sink-memory-embedding", + "kitchen-sink-memory-embedding-provider" + ] + }, + "registerMemoryFlushPlan": { + "count": 1, + "ids": [ + "kitchen-sink-memory-flush-plan" + ] + }, + "registerMemoryPromptSection": { + "count": 1, + "ids": [ + "kitchen-sink-memory-prompt-section" + ] + }, + "registerMemoryPromptSupplement": { + "count": 2, + "ids": [ + "kitchen-sink-memory-prompt-supplement", + "kitchen-sink-memory-prompt-supplement" + ] + }, + "registerMemoryRuntime": { + "count": 1, + "ids": [ + "kitchen-sink-memory-runtime" + ] + }, + "registerMigrationProvider": { + "count": 1, + "ids": [ + "kitchen-sink-migration-provider" + ] + }, + "registerMusicGenerationProvider": { + "count": 2, + "ids": [ + "kitchen-sink-music", + "kitchen-sink-music-generation-provider" + ] + }, + "registerNodeHostCommand": { + "count": 1, + "ids": [ + "kitchen-sink-node-host-command" + ] + }, + "registerProvider": { + "count": 2, + "ids": [ + "kitchen-sink-llm", + "kitchen-sink-provider" + ] + }, + "registerRealtimeTranscriptionProvider": { + "count": 2, + "ids": [ + "kitchen-sink-realtime-transcription", + "kitchen-sink-realtime-transcription-provider" + ] + }, + "registerRealtimeVoiceProvider": { + "count": 2, + "ids": [ + "kitchen-sink-realtime-voice", + "kitchen-sink-realtime-voice-provider" + ] + }, + "registerReload": { + "count": 1, + "ids": [ + "kitchen-sink-reload" + ] + }, + "registerSecurityAuditCollector": { + "count": 1, + "ids": [ + "kitchen-sink-security-audit-collector" + ] + }, + "registerService": { + "count": 2, + "ids": [ + "kitchen-sink-service", + "kitchen-sink-service" + ] + }, + "registerSpeechProvider": { + "count": 2, + "ids": [ + "kitchen-sink-speech", + "kitchen-sink-speech-provider" + ] + }, + "registerTextTransforms": { + "count": 1, + "ids": [ + "kitchen-sink-text-transforms" + ] + }, + "registerTool": { + "count": 4, + "ids": [ + "kitchen-sink-tool", + "kitchen_sink_image_job", + "kitchen_sink_search", + "kitchen_sink_text" + ] + }, + "registerVideoGenerationProvider": { + "count": 2, + "ids": [ + "kitchen-sink-video", + "kitchen-sink-video-generation-provider" + ] + }, + "registerWebFetchProvider": { + "count": 2, + "ids": [ + "kitchen-sink-fetch", + "kitchen-sink-web-fetch-provider" + ] + }, + "registerWebSearchProvider": { + "count": 2, + "ids": [ + "kitchen-sink-search", + "kitchen-sink-web-search-provider" + ] + } + } +} diff --git a/reports/kitchen-contract-probes.md b/reports/kitchen-contract-probes.md new file mode 100644 index 0000000..1f67625 --- /dev/null +++ b/reports/kitchen-contract-probes.md @@ -0,0 +1,57 @@ +# Kitchen Sink Contract Probes + +Generated: deterministic +Status: PASS + +## Covered Inspector Gaps + +- before_tool_call allow/block/approval semantics +- llm_input, llm_output, and agent_end privacy-boundary probes +- runtime registrar capture for service, route, gateway, command, interactive handler, and channel surfaces +- channel account, envelope, and outbound route probes + +## Runtime Registrations + +| Method | Count | IDs | +| ------ | ----- | --- | +| registerAgentHarness | 1 | kitchen-sink-agent-harness | +| registerAgentToolResultMiddleware | 2 | kitchen-sink-agent-tool-result-middleware, kitchen-sink-agent-tool-result-middleware | +| registerAutoEnableProbe | 1 | kitchen-sink-auto-enable-probe | +| registerChannel | 2 | kitchen-sink-channel, kitchen-sink-channel-probe | +| registerCli | 2 | kitchen-sink, kitchen-sink-cli | +| registerCliBackend | 1 | kitchen-sink-cli-backend | +| registerCodexAppServerExtensionFactory | 1 | kitchen-sink-codex-app-server-extension-factory | +| registerCommand | 3 | kitchen, kitchen-sink, kitchen-sink-command | +| registerCompactionProvider | 2 | kitchen-sink-compaction, kitchen-sink-compaction-provider | +| registerConfigMigration | 1 | kitchen-sink-config-migration | +| registerContextEngine | 1 | kitchen-sink-context-engine | +| registerDetachedTaskRuntime | 1 | kitchen-sink-detached-task-runtime | +| registerGatewayDiscoveryService | 1 | kitchen-sink-gateway-discovery-service | +| registerGatewayMethod | 2 | kitchen-sink-gateway-method, kitchen.status | +| registerHook | 1 | kitchen-sink-hook | +| registerHttpRoute | 2 | kitchen-sink-http-route, kitchen-sink-http-status | +| registerImageGenerationProvider | 2 | kitchen-sink-image, kitchen-sink-image-generation-provider | +| registerInteractiveHandler | 2 | kitchen-sink-interactive-handler, kitchen-sink-interactive-handler | +| registerMediaUnderstandingProvider | 2 | kitchen-sink-media, kitchen-sink-media-understanding-provider | +| registerMemoryCapability | 1 | kitchen-sink-memory-capability | +| registerMemoryCorpusSupplement | 2 | kitchen-sink-memory-corpus, kitchen-sink-memory-corpus-supplement | +| registerMemoryEmbeddingProvider | 2 | kitchen-sink-memory-embedding, kitchen-sink-memory-embedding-provider | +| registerMemoryFlushPlan | 1 | kitchen-sink-memory-flush-plan | +| registerMemoryPromptSection | 1 | kitchen-sink-memory-prompt-section | +| registerMemoryPromptSupplement | 2 | kitchen-sink-memory-prompt-supplement, kitchen-sink-memory-prompt-supplement | +| registerMemoryRuntime | 1 | kitchen-sink-memory-runtime | +| registerMigrationProvider | 1 | kitchen-sink-migration-provider | +| registerMusicGenerationProvider | 2 | kitchen-sink-music, kitchen-sink-music-generation-provider | +| registerNodeHostCommand | 1 | kitchen-sink-node-host-command | +| registerProvider | 2 | kitchen-sink-llm, kitchen-sink-provider | +| registerRealtimeTranscriptionProvider | 2 | kitchen-sink-realtime-transcription, kitchen-sink-realtime-transcription-provider | +| registerRealtimeVoiceProvider | 2 | kitchen-sink-realtime-voice, kitchen-sink-realtime-voice-provider | +| registerReload | 1 | kitchen-sink-reload | +| registerSecurityAuditCollector | 1 | kitchen-sink-security-audit-collector | +| registerService | 2 | kitchen-sink-service, kitchen-sink-service | +| registerSpeechProvider | 2 | kitchen-sink-speech, kitchen-sink-speech-provider | +| registerTextTransforms | 1 | kitchen-sink-text-transforms | +| registerTool | 4 | kitchen-sink-tool, kitchen_sink_image_job, kitchen_sink_search, kitchen_sink_text | +| registerVideoGenerationProvider | 2 | kitchen-sink-video, kitchen-sink-video-generation-provider | +| registerWebFetchProvider | 2 | kitchen-sink-fetch, kitchen-sink-web-fetch-provider | +| registerWebSearchProvider | 2 | kitchen-sink-search, kitchen-sink-web-search-provider | diff --git a/scripts/check-kitchen-contract-probes.mjs b/scripts/check-kitchen-contract-probes.mjs new file mode 100644 index 0000000..a2b94e0 --- /dev/null +++ b/scripts/check-kitchen-contract-probes.mjs @@ -0,0 +1,197 @@ +#!/usr/bin/env node + +import assert from "node:assert/strict"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { plugin } from "../src/index.js"; + +const registrations = {}; +const api = new Proxy( + { + id: "openclaw-kitchen-sink-fixture", + registrationMode: "full", + config: {}, + logger: console, + }, + { + get(target, property) { + if (property in target) { + return target[property]; + } + if (property === "on") { + return (...args) => capture("on", args); + } + if (typeof property !== "string" || !property.startsWith("register")) { + return undefined; + } + return (...args) => capture(property, args); + }, + }, +); + +plugin.register(api); + +const beforeToolCall = findHook("before_tool_call"); +const llmInput = findHook("llm_input"); +const llmOutput = findHook("llm_output"); +const agentEnd = findHook("agent_end"); + +const probes = { + beforeToolCall: { + allow: await beforeToolCall(toolEvent("generate a kitchen image"), { providerId: "kitchen-sink-image" }), + block: await beforeToolCall(toolEvent("kitchen block image generation"), { providerId: "kitchen-sink-image" }), + approval: await beforeToolCall(toolEvent("kitchen image generation needs approval"), { + providerId: "kitchen-sink-image", + }), + }, + conversationPrivacy: { + input: await llmInput(secretEvent("kitchen explain the fixture"), secretContext()), + output: await llmOutput(secretEvent("kitchen image result"), secretContext()), + end: await agentEnd(secretEvent("kitchen final answer"), secretContext()), + }, + channel: await captureChannelProbe(), + runtimeRegistrations: registrationSummary(), +}; + +assert.equal(probes.beforeToolCall.allow.decision, "allow"); +assert.equal(probes.beforeToolCall.allow.params.args.kitchenSinkScenario, "image.generate"); +assert.equal(probes.beforeToolCall.block.block, true); +assert.equal(probes.beforeToolCall.block.terminal, true); +assert.equal(probes.beforeToolCall.approval.decision, "approval"); +assert.equal(probes.beforeToolCall.approval.requireApproval.pluginId, "openclaw-kitchen-sink-fixture"); + +for (const result of Object.values(probes.conversationPrivacy)) { + assert.equal(result.privacy.boundary, "conversation-observer"); + assert.equal(result.privacy.storesRawPayload, false); + assert.equal(result.privacy.exposesRawPayload, false); + assert.ok(result.privacy.redactedFields.length >= 2); + assert.ok(result.privacy.secretPatternCount >= 1); +} + +for (const method of [ + "registerChannel", + "registerCommand", + "registerGatewayMethod", + "registerHttpRoute", + "registerInteractiveHandler", + "registerService", +]) { + assert.ok(probes.runtimeRegistrations[method]?.count > 0, `${method} was not captured`); +} + +assert.equal(probes.channel.account.statusState, "ready"); +assert.equal(probes.channel.delivery.deliveryStatus, "sent"); +assert.equal(probes.channel.route.peer.kind, "direct"); + +mkdirSync("reports", { recursive: true }); +writeFileSync("reports/kitchen-contract-probes.json", `${JSON.stringify(probes, null, 2)}\n`); +writeFileSync("reports/kitchen-contract-probes.md", renderMarkdown(probes)); + +console.log( + `Kitchen contract probes OK: ${Object.keys(probes.runtimeRegistrations).length} registration methods, before_tool_call allow/block/approval, conversation privacy, channel envelope`, +); + +function capture(method, args) { + registrations[method] ??= []; + registrations[method].push(args); +} + +function findHook(name) { + const entry = registrations.on?.find(([hookName]) => hookName === name); + assert.ok(entry, `hook ${name} registered`); + return entry[1]; +} + +function registrationSummary() { + return Object.fromEntries( + Object.entries(registrations) + .filter(([method]) => method !== "on") + .sort(([a], [b]) => a.localeCompare(b)) + .map(([method, entries]) => [ + method, + { + count: entries.length, + ids: entries.map((args) => idForRegistration(method, args)).filter(Boolean).sort(), + }, + ]), + ); +} + +function idForRegistration(method, args) { + const [value, second] = args; + if (method === "registerGatewayMethod" && typeof value === "string") { + return value; + } + if (method === "registerCli" && second?.descriptors?.length > 0) { + return second.descriptors.map((descriptor) => descriptor.name).join(", "); + } + if (value?.id || value?.name) { + return value.id || value.name; + } + if (typeof second === "string") { + return second; + } + const slug = method.slice("register".length).replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase(); + return `kitchen-sink-${slug}`; +} + +async function captureChannelProbe() { + const channel = registrations.registerChannel?.map(([value]) => value).find((value) => value.id === "kitchen-sink-channel"); + assert.ok(channel, "kitchen-sink-channel registered"); + return { + account: channel.config.resolveAccount({}, "local"), + delivery: await channel.outbound.sendText({ to: "kitchen demo", text: "kitchen generate image" }), + route: await channel.messaging.resolveOutboundSessionRoute({ + agentId: "fixture-agent", + target: "kitchen demo", + threadId: "thread-1", + }), + }; +} + +function toolEvent(prompt) { + return { + toolId: "kitchen_sink_image_job", + args: { prompt }, + }; +} + +function secretEvent(prompt) { + return { + prompt, + apiKey: "sk-local-secret-not-stored", + token: "fixture-token-not-stored", + }; +} + +function secretContext() { + return { + providerId: "kitchen-sink-llm", + authorization: "Bearer fixture-secret-not-stored", + }; +} + +function renderMarkdown(report) { + const methods = Object.entries(report.runtimeRegistrations) + .map(([method, summary]) => `| ${method} | ${summary.count} | ${summary.ids.join(", ")} |`) + .join("\n"); + return [ + "# Kitchen Sink Contract Probes", + "", + "Generated: deterministic", + "Status: PASS", + "", + "## Covered Inspector Gaps", + "", + "- before_tool_call allow/block/approval semantics", + "- llm_input, llm_output, and agent_end privacy-boundary probes", + "- runtime registrar capture for service, route, gateway, command, interactive handler, and channel surfaces", + "- channel account, envelope, and outbound route probes", + "", + "## Runtime Registrations", + "", + "| Method | Count | IDs |", + "| ------ | ----- | --- |", + methods, + "", + ].join("\n"); +} diff --git a/scripts/check-kitchen-runtime.mjs b/scripts/check-kitchen-runtime.mjs index 17810d6..c7cfd63 100644 --- a/scripts/check-kitchen-runtime.mjs +++ b/scripts/check-kitchen-runtime.mjs @@ -48,6 +48,40 @@ assert.equal(hookResult.pluginId, "openclaw-kitchen-sink-fixture"); assert.equal(hookResult.route, "hook:before_tool_call"); assert.equal(hookResult.scenarioId, "image.generate"); assert.equal(hookResult.matchedKitchen, true); +assert.equal(hookResult.decision, "allow"); +assert.equal(hookResult.params.args.kitchenSinkScenario, "image.generate"); + +const blockedToolHookResult = await beforeToolHook( + { toolId: "kitchen_sink_image_job", args: { prompt: "kitchen block this image" } }, + { providerId: "kitchen-sink-image" }, +); +assert.equal(blockedToolHookResult.block, true); +assert.equal(blockedToolHookResult.terminal, true); +assert.equal(blockedToolHookResult.decision, "block"); +assert.match(blockedToolHookResult.blockReason, /blocked kitchen_sink_image_job/); + +const approvalToolHookResult = await beforeToolHook( + { toolId: "kitchen_sink_image_job", args: { prompt: "kitchen image needs approval" } }, + { providerId: "kitchen-sink-image" }, +); +assert.equal(approvalToolHookResult.decision, "approval"); +assert.equal(approvalToolHookResult.requireApproval.pluginId, "openclaw-kitchen-sink-fixture"); +assert.equal(approvalToolHookResult.requireApproval.scenarioId, "image.generate"); + +const llmInputHook = findHook("llm_input"); +const llmInputResult = await llmInputHook( + { + prompt: "kitchen explain image routing with api_key sk-test-redacted", + apiKey: "sk-real-secret-not-stored", + }, + { providerId: "kitchen-sink-llm", authorization: "Bearer local-secret" }, +); +assert.equal(llmInputResult.scenarioId, "text.reply"); +assert.equal(llmInputResult.privacy.boundary, "conversation-observer"); +assert.equal(llmInputResult.privacy.storesRawPayload, false); +assert.equal(llmInputResult.privacy.exposesRawPayload, false); +assert.deepEqual(llmInputResult.privacy.redactedFields, ["event.apiKey", "context.authorization"]); +assert.ok(llmInputResult.privacy.secretPatternCount >= 2); const channel = findRegistration("registerChannel", "kitchen-sink-channel"); const channelAccount = channel.config.resolveAccount({}, "local"); diff --git a/src/scenarios.js b/src/scenarios.js index 1eca287..4644667 100644 --- a/src/scenarios.js +++ b/src/scenarios.js @@ -754,8 +754,7 @@ export function observeKitchenHook(name, event, context) { const url = firstHookString(event, ["url"]) || firstHookString(event?.args, ["url"]); const text = extractHookText(event) || extractHookText(context); const scenarioId = inferKitchenScenario({ providerId, text, toolId, url }); - - return { + const observation = { kitchenSink: true, pluginId: PLUGIN_ID, hook: name, @@ -765,6 +764,22 @@ export function observeKitchenHook(name, event, context) { observedEventKeys: Object.keys(event ?? {}), observedContextKeys: Object.keys(context ?? {}), }; + + if (name === "before_tool_call") { + return { + ...observation, + ...createBeforeToolCallDecision({ event, scenarioId, text, toolId }), + }; + } + + if (name === "llm_input" || name === "llm_output" || name === "agent_end") { + return { + ...observation, + privacy: createConversationPrivacyProbe({ event, context, text }), + }; + } + + return observation; } function createKitchenJob(kind, prompt, date, delayMs, scenarioId, route) { @@ -1073,6 +1088,90 @@ function extractHookText(value) { ); } +function createBeforeToolCallDecision({ event, scenarioId, text, toolId }) { + const params = createToolCallParams(event, scenarioId); + const lowerText = String(text ?? "").toLowerCase(); + if (/\b(block|deny|forbid)\b/.test(lowerText)) { + return { + params, + block: true, + blockReason: `Kitchen Sink fixture blocked ${toolId || "tool"} for ${scenarioId}.`, + terminal: true, + decision: "block", + }; + } + if (/\b(approval|approve|permission)\b/.test(lowerText)) { + const approvalId = `ks_approval_${stableHash(`${toolId}:${text}:${scenarioId}`).slice(0, 10)}`; + return { + params, + requireApproval: { + id: approvalId, + title: "Kitchen Sink tool approval", + reason: `Kitchen Sink fixture requires approval before ${toolId || "tool"} runs.`, + summary: `Approve deterministic ${scenarioId} fixture execution.`, + scenarioId, + pluginId: PLUGIN_ID, + }, + decision: "approval", + }; + } + return { + params, + decision: scenarioId === "observe" ? "observe" : "allow", + }; +} + +function createToolCallParams(event, scenarioId) { + const rawParams = event?.params && typeof event.params === "object" ? event.params : {}; + const rawArgs = event?.args && typeof event.args === "object" ? event.args : {}; + return { + ...rawParams, + args: { + ...rawArgs, + kitchenSinkScenario: scenarioId, + kitchenSinkPluginId: PLUGIN_ID, + }, + }; +} + +function createConversationPrivacyProbe({ event, context, text }) { + const eventText = extractHookText(event); + const contextText = extractHookText(context); + const combined = [text, eventText, contextText].filter(Boolean).join("\n"); + const redactedFields = []; + for (const [label, value] of [ + ["event.apiKey", event?.apiKey], + ["event.authorization", event?.authorization], + ["event.token", event?.token], + ["context.apiKey", context?.apiKey], + ["context.authorization", context?.authorization], + ["context.token", context?.token], + ]) { + if (typeof value === "string" && value.trim()) { + redactedFields.push(label); + } + } + const secretText = [ + combined, + event?.apiKey, + event?.authorization, + event?.token, + context?.apiKey, + context?.authorization, + context?.token, + ].filter(Boolean).join("\n"); + const secretPatternHits = secretText.match(/\b(?:sk-[a-z0-9_-]+|api[_-]?key|authorization|bearer\s+[a-z0-9._-]+|fixture-token-[a-z0-9_-]+)\b/gi) ?? []; + return { + boundary: "conversation-observer", + promptHash: stableHash(combined || "empty"), + promptLength: combined.length, + redactedFields, + secretPatternCount: secretPatternHits.length, + storesRawPayload: false, + exposesRawPayload: false, + }; +} + function firstHookString(source, keys) { if (!source || typeof source !== "object") { return "";