test: add kitchen sink contract probes
This commit is contained in:
Vincent Koc
parent
dcfec9ecef
commit
7ca1d2c53e
@ -51,7 +51,7 @@
|
||||
}
|
||||
},
|
||||
"scripts": {
|
||||
"check": "npm run sync:surface -- --check && node scripts/check-sdk-surface.mjs && node scripts/check-kitchen-runtime.mjs && npm run plugin:inspect",
|
||||
"check": "npm run sync:surface -- --check && node scripts/check-sdk-surface.mjs && node scripts/check-kitchen-runtime.mjs && node scripts/check-kitchen-contract-probes.mjs && npm run plugin:inspect",
|
||||
"pack:check": "node scripts/check-pack-payload.mjs",
|
||||
"plugin:inspect": "plugin-inspector check --config plugin-inspector.config.json --no-openclaw",
|
||||
"plugin:inspect:runtime": "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector check --config plugin-inspector.config.json --no-openclaw --runtime --mock-sdk",
|
||||
|
||||
501
reports/kitchen-contract-probes.json
Normal file
501
reports/kitchen-contract-probes.json
Normal file
@ -0,0 +1,501 @@
|
||||
{
|
||||
"beforeToolCall": {
|
||||
"allow": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"hook": "before_tool_call",
|
||||
"route": "hook:before_tool_call",
|
||||
"matchedKitchen": true,
|
||||
"scenarioId": "image.generate",
|
||||
"observedEventKeys": [
|
||||
"toolId",
|
||||
"args"
|
||||
],
|
||||
"observedContextKeys": [
|
||||
"providerId"
|
||||
],
|
||||
"params": {
|
||||
"args": {
|
||||
"prompt": "generate a kitchen image",
|
||||
"kitchenSinkScenario": "image.generate",
|
||||
"kitchenSinkPluginId": "openclaw-kitchen-sink-fixture"
|
||||
}
|
||||
},
|
||||
"decision": "allow"
|
||||
},
|
||||
"block": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"hook": "before_tool_call",
|
||||
"route": "hook:before_tool_call",
|
||||
"matchedKitchen": true,
|
||||
"scenarioId": "image.generate",
|
||||
"observedEventKeys": [
|
||||
"toolId",
|
||||
"args"
|
||||
],
|
||||
"observedContextKeys": [
|
||||
"providerId"
|
||||
],
|
||||
"params": {
|
||||
"args": {
|
||||
"prompt": "kitchen block image generation",
|
||||
"kitchenSinkScenario": "image.generate",
|
||||
"kitchenSinkPluginId": "openclaw-kitchen-sink-fixture"
|
||||
}
|
||||
},
|
||||
"block": true,
|
||||
"blockReason": "Kitchen Sink fixture blocked kitchen_sink_image_job for image.generate.",
|
||||
"terminal": true,
|
||||
"decision": "block"
|
||||
},
|
||||
"approval": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"hook": "before_tool_call",
|
||||
"route": "hook:before_tool_call",
|
||||
"matchedKitchen": true,
|
||||
"scenarioId": "image.generate",
|
||||
"observedEventKeys": [
|
||||
"toolId",
|
||||
"args"
|
||||
],
|
||||
"observedContextKeys": [
|
||||
"providerId"
|
||||
],
|
||||
"params": {
|
||||
"args": {
|
||||
"prompt": "kitchen image generation needs approval",
|
||||
"kitchenSinkScenario": "image.generate",
|
||||
"kitchenSinkPluginId": "openclaw-kitchen-sink-fixture"
|
||||
}
|
||||
},
|
||||
"requireApproval": {
|
||||
"id": "ks_approval_9863b78c",
|
||||
"title": "Kitchen Sink tool approval",
|
||||
"reason": "Kitchen Sink fixture requires approval before kitchen_sink_image_job runs.",
|
||||
"summary": "Approve deterministic image.generate fixture execution.",
|
||||
"scenarioId": "image.generate",
|
||||
"pluginId": "openclaw-kitchen-sink-fixture"
|
||||
},
|
||||
"decision": "approval"
|
||||
}
|
||||
},
|
||||
"conversationPrivacy": {
|
||||
"input": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"hook": "llm_input",
|
||||
"route": "hook:llm_input",
|
||||
"matchedKitchen": true,
|
||||
"scenarioId": "text.reply",
|
||||
"observedEventKeys": [
|
||||
"prompt",
|
||||
"apiKey",
|
||||
"token"
|
||||
],
|
||||
"observedContextKeys": [
|
||||
"providerId",
|
||||
"authorization"
|
||||
],
|
||||
"privacy": {
|
||||
"boundary": "conversation-observer",
|
||||
"promptHash": "21ed2705",
|
||||
"promptLength": 55,
|
||||
"redactedFields": [
|
||||
"event.apiKey",
|
||||
"event.token",
|
||||
"context.authorization"
|
||||
],
|
||||
"secretPatternCount": 3,
|
||||
"storesRawPayload": false,
|
||||
"exposesRawPayload": false
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"hook": "llm_output",
|
||||
"route": "hook:llm_output",
|
||||
"matchedKitchen": true,
|
||||
"scenarioId": "text.reply",
|
||||
"observedEventKeys": [
|
||||
"prompt",
|
||||
"apiKey",
|
||||
"token"
|
||||
],
|
||||
"observedContextKeys": [
|
||||
"providerId",
|
||||
"authorization"
|
||||
],
|
||||
"privacy": {
|
||||
"boundary": "conversation-observer",
|
||||
"promptHash": "a3e6f809",
|
||||
"promptLength": 41,
|
||||
"redactedFields": [
|
||||
"event.apiKey",
|
||||
"event.token",
|
||||
"context.authorization"
|
||||
],
|
||||
"secretPatternCount": 3,
|
||||
"storesRawPayload": false,
|
||||
"exposesRawPayload": false
|
||||
}
|
||||
},
|
||||
"end": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"hook": "agent_end",
|
||||
"route": "hook:agent_end",
|
||||
"matchedKitchen": true,
|
||||
"scenarioId": "text.reply",
|
||||
"observedEventKeys": [
|
||||
"prompt",
|
||||
"apiKey",
|
||||
"token"
|
||||
],
|
||||
"observedContextKeys": [
|
||||
"providerId",
|
||||
"authorization"
|
||||
],
|
||||
"privacy": {
|
||||
"boundary": "conversation-observer",
|
||||
"promptHash": "8bf533dd",
|
||||
"promptLength": 41,
|
||||
"redactedFields": [
|
||||
"event.apiKey",
|
||||
"event.token",
|
||||
"context.authorization"
|
||||
],
|
||||
"secretPatternCount": 3,
|
||||
"storesRawPayload": false,
|
||||
"exposesRawPayload": false
|
||||
}
|
||||
}
|
||||
},
|
||||
"channel": {
|
||||
"account": {
|
||||
"accountId": "local",
|
||||
"name": "Kitchen Sink Local",
|
||||
"enabled": true,
|
||||
"configured": true,
|
||||
"statusState": "ready",
|
||||
"linked": true,
|
||||
"running": true,
|
||||
"connected": true,
|
||||
"mode": "local",
|
||||
"health": {
|
||||
"ok": true,
|
||||
"checkedAt": "2026-04-28T00:00:00.000Z",
|
||||
"message": "Kitchen Sink local fixture account is ready."
|
||||
},
|
||||
"capabilities": [
|
||||
"text",
|
||||
"media",
|
||||
"threads",
|
||||
"dry-run"
|
||||
]
|
||||
},
|
||||
"delivery": {
|
||||
"channel": "kitchen-sink-channel",
|
||||
"messageId": "ks_channel_d813aa04",
|
||||
"conversationId": "kitchen-demo",
|
||||
"channelId": "kitchen-demo",
|
||||
"timestamp": 1777494231851,
|
||||
"deliveryStatus": "sent",
|
||||
"transport": "kitchen-sink-local",
|
||||
"meta": {
|
||||
"kitchenSink": true,
|
||||
"pluginId": "openclaw-kitchen-sink-fixture",
|
||||
"scenarioId": "image.generate",
|
||||
"kind": "text"
|
||||
}
|
||||
},
|
||||
"route": {
|
||||
"sessionKey": "kitchen:fixture-agent:kitchen-demo",
|
||||
"baseSessionKey": "kitchen:fixture-agent:kitchen-demo",
|
||||
"peer": {
|
||||
"kind": "direct",
|
||||
"id": "kitchen-demo"
|
||||
},
|
||||
"chatType": "direct",
|
||||
"from": "local",
|
||||
"to": "kitchen-demo",
|
||||
"threadId": "thread-1"
|
||||
}
|
||||
},
|
||||
"runtimeRegistrations": {
|
||||
"registerAgentHarness": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-agent-harness"
|
||||
]
|
||||
},
|
||||
"registerAgentToolResultMiddleware": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-agent-tool-result-middleware",
|
||||
"kitchen-sink-agent-tool-result-middleware"
|
||||
]
|
||||
},
|
||||
"registerAutoEnableProbe": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-auto-enable-probe"
|
||||
]
|
||||
},
|
||||
"registerChannel": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-channel",
|
||||
"kitchen-sink-channel-probe"
|
||||
]
|
||||
},
|
||||
"registerCli": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink",
|
||||
"kitchen-sink-cli"
|
||||
]
|
||||
},
|
||||
"registerCliBackend": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-cli-backend"
|
||||
]
|
||||
},
|
||||
"registerCodexAppServerExtensionFactory": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-codex-app-server-extension-factory"
|
||||
]
|
||||
},
|
||||
"registerCommand": {
|
||||
"count": 3,
|
||||
"ids": [
|
||||
"kitchen",
|
||||
"kitchen-sink",
|
||||
"kitchen-sink-command"
|
||||
]
|
||||
},
|
||||
"registerCompactionProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-compaction",
|
||||
"kitchen-sink-compaction-provider"
|
||||
]
|
||||
},
|
||||
"registerConfigMigration": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-config-migration"
|
||||
]
|
||||
},
|
||||
"registerContextEngine": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-context-engine"
|
||||
]
|
||||
},
|
||||
"registerDetachedTaskRuntime": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-detached-task-runtime"
|
||||
]
|
||||
},
|
||||
"registerGatewayDiscoveryService": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-gateway-discovery-service"
|
||||
]
|
||||
},
|
||||
"registerGatewayMethod": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-gateway-method",
|
||||
"kitchen.status"
|
||||
]
|
||||
},
|
||||
"registerHook": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-hook"
|
||||
]
|
||||
},
|
||||
"registerHttpRoute": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-http-route",
|
||||
"kitchen-sink-http-status"
|
||||
]
|
||||
},
|
||||
"registerImageGenerationProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-image",
|
||||
"kitchen-sink-image-generation-provider"
|
||||
]
|
||||
},
|
||||
"registerInteractiveHandler": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-interactive-handler",
|
||||
"kitchen-sink-interactive-handler"
|
||||
]
|
||||
},
|
||||
"registerMediaUnderstandingProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-media",
|
||||
"kitchen-sink-media-understanding-provider"
|
||||
]
|
||||
},
|
||||
"registerMemoryCapability": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-capability"
|
||||
]
|
||||
},
|
||||
"registerMemoryCorpusSupplement": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-corpus",
|
||||
"kitchen-sink-memory-corpus-supplement"
|
||||
]
|
||||
},
|
||||
"registerMemoryEmbeddingProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-embedding",
|
||||
"kitchen-sink-memory-embedding-provider"
|
||||
]
|
||||
},
|
||||
"registerMemoryFlushPlan": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-flush-plan"
|
||||
]
|
||||
},
|
||||
"registerMemoryPromptSection": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-prompt-section"
|
||||
]
|
||||
},
|
||||
"registerMemoryPromptSupplement": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-prompt-supplement",
|
||||
"kitchen-sink-memory-prompt-supplement"
|
||||
]
|
||||
},
|
||||
"registerMemoryRuntime": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-memory-runtime"
|
||||
]
|
||||
},
|
||||
"registerMigrationProvider": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-migration-provider"
|
||||
]
|
||||
},
|
||||
"registerMusicGenerationProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-music",
|
||||
"kitchen-sink-music-generation-provider"
|
||||
]
|
||||
},
|
||||
"registerNodeHostCommand": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-node-host-command"
|
||||
]
|
||||
},
|
||||
"registerProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-llm",
|
||||
"kitchen-sink-provider"
|
||||
]
|
||||
},
|
||||
"registerRealtimeTranscriptionProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-realtime-transcription",
|
||||
"kitchen-sink-realtime-transcription-provider"
|
||||
]
|
||||
},
|
||||
"registerRealtimeVoiceProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-realtime-voice",
|
||||
"kitchen-sink-realtime-voice-provider"
|
||||
]
|
||||
},
|
||||
"registerReload": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-reload"
|
||||
]
|
||||
},
|
||||
"registerSecurityAuditCollector": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-security-audit-collector"
|
||||
]
|
||||
},
|
||||
"registerService": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-service",
|
||||
"kitchen-sink-service"
|
||||
]
|
||||
},
|
||||
"registerSpeechProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-speech",
|
||||
"kitchen-sink-speech-provider"
|
||||
]
|
||||
},
|
||||
"registerTextTransforms": {
|
||||
"count": 1,
|
||||
"ids": [
|
||||
"kitchen-sink-text-transforms"
|
||||
]
|
||||
},
|
||||
"registerTool": {
|
||||
"count": 4,
|
||||
"ids": [
|
||||
"kitchen-sink-tool",
|
||||
"kitchen_sink_image_job",
|
||||
"kitchen_sink_search",
|
||||
"kitchen_sink_text"
|
||||
]
|
||||
},
|
||||
"registerVideoGenerationProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-video",
|
||||
"kitchen-sink-video-generation-provider"
|
||||
]
|
||||
},
|
||||
"registerWebFetchProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-fetch",
|
||||
"kitchen-sink-web-fetch-provider"
|
||||
]
|
||||
},
|
||||
"registerWebSearchProvider": {
|
||||
"count": 2,
|
||||
"ids": [
|
||||
"kitchen-sink-search",
|
||||
"kitchen-sink-web-search-provider"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
57
reports/kitchen-contract-probes.md
Normal file
57
reports/kitchen-contract-probes.md
Normal file
@ -0,0 +1,57 @@
|
||||
# Kitchen Sink Contract Probes
|
||||
|
||||
Generated: deterministic
|
||||
Status: PASS
|
||||
|
||||
## Covered Inspector Gaps
|
||||
|
||||
- before_tool_call allow/block/approval semantics
|
||||
- llm_input, llm_output, and agent_end privacy-boundary probes
|
||||
- runtime registrar capture for service, route, gateway, command, interactive handler, and channel surfaces
|
||||
- channel account, envelope, and outbound route probes
|
||||
|
||||
## Runtime Registrations
|
||||
|
||||
| Method | Count | IDs |
|
||||
| ------ | ----- | --- |
|
||||
| registerAgentHarness | 1 | kitchen-sink-agent-harness |
|
||||
| registerAgentToolResultMiddleware | 2 | kitchen-sink-agent-tool-result-middleware, kitchen-sink-agent-tool-result-middleware |
|
||||
| registerAutoEnableProbe | 1 | kitchen-sink-auto-enable-probe |
|
||||
| registerChannel | 2 | kitchen-sink-channel, kitchen-sink-channel-probe |
|
||||
| registerCli | 2 | kitchen-sink, kitchen-sink-cli |
|
||||
| registerCliBackend | 1 | kitchen-sink-cli-backend |
|
||||
| registerCodexAppServerExtensionFactory | 1 | kitchen-sink-codex-app-server-extension-factory |
|
||||
| registerCommand | 3 | kitchen, kitchen-sink, kitchen-sink-command |
|
||||
| registerCompactionProvider | 2 | kitchen-sink-compaction, kitchen-sink-compaction-provider |
|
||||
| registerConfigMigration | 1 | kitchen-sink-config-migration |
|
||||
| registerContextEngine | 1 | kitchen-sink-context-engine |
|
||||
| registerDetachedTaskRuntime | 1 | kitchen-sink-detached-task-runtime |
|
||||
| registerGatewayDiscoveryService | 1 | kitchen-sink-gateway-discovery-service |
|
||||
| registerGatewayMethod | 2 | kitchen-sink-gateway-method, kitchen.status |
|
||||
| registerHook | 1 | kitchen-sink-hook |
|
||||
| registerHttpRoute | 2 | kitchen-sink-http-route, kitchen-sink-http-status |
|
||||
| registerImageGenerationProvider | 2 | kitchen-sink-image, kitchen-sink-image-generation-provider |
|
||||
| registerInteractiveHandler | 2 | kitchen-sink-interactive-handler, kitchen-sink-interactive-handler |
|
||||
| registerMediaUnderstandingProvider | 2 | kitchen-sink-media, kitchen-sink-media-understanding-provider |
|
||||
| registerMemoryCapability | 1 | kitchen-sink-memory-capability |
|
||||
| registerMemoryCorpusSupplement | 2 | kitchen-sink-memory-corpus, kitchen-sink-memory-corpus-supplement |
|
||||
| registerMemoryEmbeddingProvider | 2 | kitchen-sink-memory-embedding, kitchen-sink-memory-embedding-provider |
|
||||
| registerMemoryFlushPlan | 1 | kitchen-sink-memory-flush-plan |
|
||||
| registerMemoryPromptSection | 1 | kitchen-sink-memory-prompt-section |
|
||||
| registerMemoryPromptSupplement | 2 | kitchen-sink-memory-prompt-supplement, kitchen-sink-memory-prompt-supplement |
|
||||
| registerMemoryRuntime | 1 | kitchen-sink-memory-runtime |
|
||||
| registerMigrationProvider | 1 | kitchen-sink-migration-provider |
|
||||
| registerMusicGenerationProvider | 2 | kitchen-sink-music, kitchen-sink-music-generation-provider |
|
||||
| registerNodeHostCommand | 1 | kitchen-sink-node-host-command |
|
||||
| registerProvider | 2 | kitchen-sink-llm, kitchen-sink-provider |
|
||||
| registerRealtimeTranscriptionProvider | 2 | kitchen-sink-realtime-transcription, kitchen-sink-realtime-transcription-provider |
|
||||
| registerRealtimeVoiceProvider | 2 | kitchen-sink-realtime-voice, kitchen-sink-realtime-voice-provider |
|
||||
| registerReload | 1 | kitchen-sink-reload |
|
||||
| registerSecurityAuditCollector | 1 | kitchen-sink-security-audit-collector |
|
||||
| registerService | 2 | kitchen-sink-service, kitchen-sink-service |
|
||||
| registerSpeechProvider | 2 | kitchen-sink-speech, kitchen-sink-speech-provider |
|
||||
| registerTextTransforms | 1 | kitchen-sink-text-transforms |
|
||||
| registerTool | 4 | kitchen-sink-tool, kitchen_sink_image_job, kitchen_sink_search, kitchen_sink_text |
|
||||
| registerVideoGenerationProvider | 2 | kitchen-sink-video, kitchen-sink-video-generation-provider |
|
||||
| registerWebFetchProvider | 2 | kitchen-sink-fetch, kitchen-sink-web-fetch-provider |
|
||||
| registerWebSearchProvider | 2 | kitchen-sink-search, kitchen-sink-web-search-provider |
|
||||
197
scripts/check-kitchen-contract-probes.mjs
Normal file
197
scripts/check-kitchen-contract-probes.mjs
Normal file
@ -0,0 +1,197 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { plugin } from "../src/index.js";
|
||||
|
||||
const registrations = {};
|
||||
const api = new Proxy(
|
||||
{
|
||||
id: "openclaw-kitchen-sink-fixture",
|
||||
registrationMode: "full",
|
||||
config: {},
|
||||
logger: console,
|
||||
},
|
||||
{
|
||||
get(target, property) {
|
||||
if (property in target) {
|
||||
return target[property];
|
||||
}
|
||||
if (property === "on") {
|
||||
return (...args) => capture("on", args);
|
||||
}
|
||||
if (typeof property !== "string" || !property.startsWith("register")) {
|
||||
return undefined;
|
||||
}
|
||||
return (...args) => capture(property, args);
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
plugin.register(api);
|
||||
|
||||
const beforeToolCall = findHook("before_tool_call");
|
||||
const llmInput = findHook("llm_input");
|
||||
const llmOutput = findHook("llm_output");
|
||||
const agentEnd = findHook("agent_end");
|
||||
|
||||
const probes = {
|
||||
beforeToolCall: {
|
||||
allow: await beforeToolCall(toolEvent("generate a kitchen image"), { providerId: "kitchen-sink-image" }),
|
||||
block: await beforeToolCall(toolEvent("kitchen block image generation"), { providerId: "kitchen-sink-image" }),
|
||||
approval: await beforeToolCall(toolEvent("kitchen image generation needs approval"), {
|
||||
providerId: "kitchen-sink-image",
|
||||
}),
|
||||
},
|
||||
conversationPrivacy: {
|
||||
input: await llmInput(secretEvent("kitchen explain the fixture"), secretContext()),
|
||||
output: await llmOutput(secretEvent("kitchen image result"), secretContext()),
|
||||
end: await agentEnd(secretEvent("kitchen final answer"), secretContext()),
|
||||
},
|
||||
channel: await captureChannelProbe(),
|
||||
runtimeRegistrations: registrationSummary(),
|
||||
};
|
||||
|
||||
assert.equal(probes.beforeToolCall.allow.decision, "allow");
|
||||
assert.equal(probes.beforeToolCall.allow.params.args.kitchenSinkScenario, "image.generate");
|
||||
assert.equal(probes.beforeToolCall.block.block, true);
|
||||
assert.equal(probes.beforeToolCall.block.terminal, true);
|
||||
assert.equal(probes.beforeToolCall.approval.decision, "approval");
|
||||
assert.equal(probes.beforeToolCall.approval.requireApproval.pluginId, "openclaw-kitchen-sink-fixture");
|
||||
|
||||
for (const result of Object.values(probes.conversationPrivacy)) {
|
||||
assert.equal(result.privacy.boundary, "conversation-observer");
|
||||
assert.equal(result.privacy.storesRawPayload, false);
|
||||
assert.equal(result.privacy.exposesRawPayload, false);
|
||||
assert.ok(result.privacy.redactedFields.length >= 2);
|
||||
assert.ok(result.privacy.secretPatternCount >= 1);
|
||||
}
|
||||
|
||||
for (const method of [
|
||||
"registerChannel",
|
||||
"registerCommand",
|
||||
"registerGatewayMethod",
|
||||
"registerHttpRoute",
|
||||
"registerInteractiveHandler",
|
||||
"registerService",
|
||||
]) {
|
||||
assert.ok(probes.runtimeRegistrations[method]?.count > 0, `${method} was not captured`);
|
||||
}
|
||||
|
||||
assert.equal(probes.channel.account.statusState, "ready");
|
||||
assert.equal(probes.channel.delivery.deliveryStatus, "sent");
|
||||
assert.equal(probes.channel.route.peer.kind, "direct");
|
||||
|
||||
mkdirSync("reports", { recursive: true });
|
||||
writeFileSync("reports/kitchen-contract-probes.json", `${JSON.stringify(probes, null, 2)}\n`);
|
||||
writeFileSync("reports/kitchen-contract-probes.md", renderMarkdown(probes));
|
||||
|
||||
console.log(
|
||||
`Kitchen contract probes OK: ${Object.keys(probes.runtimeRegistrations).length} registration methods, before_tool_call allow/block/approval, conversation privacy, channel envelope`,
|
||||
);
|
||||
|
||||
function capture(method, args) {
|
||||
registrations[method] ??= [];
|
||||
registrations[method].push(args);
|
||||
}
|
||||
|
||||
function findHook(name) {
|
||||
const entry = registrations.on?.find(([hookName]) => hookName === name);
|
||||
assert.ok(entry, `hook ${name} registered`);
|
||||
return entry[1];
|
||||
}
|
||||
|
||||
function registrationSummary() {
|
||||
return Object.fromEntries(
|
||||
Object.entries(registrations)
|
||||
.filter(([method]) => method !== "on")
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([method, entries]) => [
|
||||
method,
|
||||
{
|
||||
count: entries.length,
|
||||
ids: entries.map((args) => idForRegistration(method, args)).filter(Boolean).sort(),
|
||||
},
|
||||
]),
|
||||
);
|
||||
}
|
||||
|
||||
function idForRegistration(method, args) {
|
||||
const [value, second] = args;
|
||||
if (method === "registerGatewayMethod" && typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
if (method === "registerCli" && second?.descriptors?.length > 0) {
|
||||
return second.descriptors.map((descriptor) => descriptor.name).join(", ");
|
||||
}
|
||||
if (value?.id || value?.name) {
|
||||
return value.id || value.name;
|
||||
}
|
||||
if (typeof second === "string") {
|
||||
return second;
|
||||
}
|
||||
const slug = method.slice("register".length).replace(/([a-z0-9])([A-Z])/g, "$1-$2").toLowerCase();
|
||||
return `kitchen-sink-${slug}`;
|
||||
}
|
||||
|
||||
async function captureChannelProbe() {
|
||||
const channel = registrations.registerChannel?.map(([value]) => value).find((value) => value.id === "kitchen-sink-channel");
|
||||
assert.ok(channel, "kitchen-sink-channel registered");
|
||||
return {
|
||||
account: channel.config.resolveAccount({}, "local"),
|
||||
delivery: await channel.outbound.sendText({ to: "kitchen demo", text: "kitchen generate image" }),
|
||||
route: await channel.messaging.resolveOutboundSessionRoute({
|
||||
agentId: "fixture-agent",
|
||||
target: "kitchen demo",
|
||||
threadId: "thread-1",
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
function toolEvent(prompt) {
|
||||
return {
|
||||
toolId: "kitchen_sink_image_job",
|
||||
args: { prompt },
|
||||
};
|
||||
}
|
||||
|
||||
function secretEvent(prompt) {
|
||||
return {
|
||||
prompt,
|
||||
apiKey: "sk-local-secret-not-stored",
|
||||
token: "fixture-token-not-stored",
|
||||
};
|
||||
}
|
||||
|
||||
function secretContext() {
|
||||
return {
|
||||
providerId: "kitchen-sink-llm",
|
||||
authorization: "Bearer fixture-secret-not-stored",
|
||||
};
|
||||
}
|
||||
|
||||
function renderMarkdown(report) {
|
||||
const methods = Object.entries(report.runtimeRegistrations)
|
||||
.map(([method, summary]) => `| ${method} | ${summary.count} | ${summary.ids.join(", ")} |`)
|
||||
.join("\n");
|
||||
return [
|
||||
"# Kitchen Sink Contract Probes",
|
||||
"",
|
||||
"Generated: deterministic",
|
||||
"Status: PASS",
|
||||
"",
|
||||
"## Covered Inspector Gaps",
|
||||
"",
|
||||
"- before_tool_call allow/block/approval semantics",
|
||||
"- llm_input, llm_output, and agent_end privacy-boundary probes",
|
||||
"- runtime registrar capture for service, route, gateway, command, interactive handler, and channel surfaces",
|
||||
"- channel account, envelope, and outbound route probes",
|
||||
"",
|
||||
"## Runtime Registrations",
|
||||
"",
|
||||
"| Method | Count | IDs |",
|
||||
"| ------ | ----- | --- |",
|
||||
methods,
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
@ -48,6 +48,40 @@ assert.equal(hookResult.pluginId, "openclaw-kitchen-sink-fixture");
|
||||
assert.equal(hookResult.route, "hook:before_tool_call");
|
||||
assert.equal(hookResult.scenarioId, "image.generate");
|
||||
assert.equal(hookResult.matchedKitchen, true);
|
||||
assert.equal(hookResult.decision, "allow");
|
||||
assert.equal(hookResult.params.args.kitchenSinkScenario, "image.generate");
|
||||
|
||||
const blockedToolHookResult = await beforeToolHook(
|
||||
{ toolId: "kitchen_sink_image_job", args: { prompt: "kitchen block this image" } },
|
||||
{ providerId: "kitchen-sink-image" },
|
||||
);
|
||||
assert.equal(blockedToolHookResult.block, true);
|
||||
assert.equal(blockedToolHookResult.terminal, true);
|
||||
assert.equal(blockedToolHookResult.decision, "block");
|
||||
assert.match(blockedToolHookResult.blockReason, /blocked kitchen_sink_image_job/);
|
||||
|
||||
const approvalToolHookResult = await beforeToolHook(
|
||||
{ toolId: "kitchen_sink_image_job", args: { prompt: "kitchen image needs approval" } },
|
||||
{ providerId: "kitchen-sink-image" },
|
||||
);
|
||||
assert.equal(approvalToolHookResult.decision, "approval");
|
||||
assert.equal(approvalToolHookResult.requireApproval.pluginId, "openclaw-kitchen-sink-fixture");
|
||||
assert.equal(approvalToolHookResult.requireApproval.scenarioId, "image.generate");
|
||||
|
||||
const llmInputHook = findHook("llm_input");
|
||||
const llmInputResult = await llmInputHook(
|
||||
{
|
||||
prompt: "kitchen explain image routing with api_key sk-test-redacted",
|
||||
apiKey: "sk-real-secret-not-stored",
|
||||
},
|
||||
{ providerId: "kitchen-sink-llm", authorization: "Bearer local-secret" },
|
||||
);
|
||||
assert.equal(llmInputResult.scenarioId, "text.reply");
|
||||
assert.equal(llmInputResult.privacy.boundary, "conversation-observer");
|
||||
assert.equal(llmInputResult.privacy.storesRawPayload, false);
|
||||
assert.equal(llmInputResult.privacy.exposesRawPayload, false);
|
||||
assert.deepEqual(llmInputResult.privacy.redactedFields, ["event.apiKey", "context.authorization"]);
|
||||
assert.ok(llmInputResult.privacy.secretPatternCount >= 2);
|
||||
|
||||
const channel = findRegistration("registerChannel", "kitchen-sink-channel");
|
||||
const channelAccount = channel.config.resolveAccount({}, "local");
|
||||
|
||||
103
src/scenarios.js
103
src/scenarios.js
@ -754,8 +754,7 @@ export function observeKitchenHook(name, event, context) {
|
||||
const url = firstHookString(event, ["url"]) || firstHookString(event?.args, ["url"]);
|
||||
const text = extractHookText(event) || extractHookText(context);
|
||||
const scenarioId = inferKitchenScenario({ providerId, text, toolId, url });
|
||||
|
||||
return {
|
||||
const observation = {
|
||||
kitchenSink: true,
|
||||
pluginId: PLUGIN_ID,
|
||||
hook: name,
|
||||
@ -765,6 +764,22 @@ export function observeKitchenHook(name, event, context) {
|
||||
observedEventKeys: Object.keys(event ?? {}),
|
||||
observedContextKeys: Object.keys(context ?? {}),
|
||||
};
|
||||
|
||||
if (name === "before_tool_call") {
|
||||
return {
|
||||
...observation,
|
||||
...createBeforeToolCallDecision({ event, scenarioId, text, toolId }),
|
||||
};
|
||||
}
|
||||
|
||||
if (name === "llm_input" || name === "llm_output" || name === "agent_end") {
|
||||
return {
|
||||
...observation,
|
||||
privacy: createConversationPrivacyProbe({ event, context, text }),
|
||||
};
|
||||
}
|
||||
|
||||
return observation;
|
||||
}
|
||||
|
||||
function createKitchenJob(kind, prompt, date, delayMs, scenarioId, route) {
|
||||
@ -1073,6 +1088,90 @@ function extractHookText(value) {
|
||||
);
|
||||
}
|
||||
|
||||
function createBeforeToolCallDecision({ event, scenarioId, text, toolId }) {
|
||||
const params = createToolCallParams(event, scenarioId);
|
||||
const lowerText = String(text ?? "").toLowerCase();
|
||||
if (/\b(block|deny|forbid)\b/.test(lowerText)) {
|
||||
return {
|
||||
params,
|
||||
block: true,
|
||||
blockReason: `Kitchen Sink fixture blocked ${toolId || "tool"} for ${scenarioId}.`,
|
||||
terminal: true,
|
||||
decision: "block",
|
||||
};
|
||||
}
|
||||
if (/\b(approval|approve|permission)\b/.test(lowerText)) {
|
||||
const approvalId = `ks_approval_${stableHash(`${toolId}:${text}:${scenarioId}`).slice(0, 10)}`;
|
||||
return {
|
||||
params,
|
||||
requireApproval: {
|
||||
id: approvalId,
|
||||
title: "Kitchen Sink tool approval",
|
||||
reason: `Kitchen Sink fixture requires approval before ${toolId || "tool"} runs.`,
|
||||
summary: `Approve deterministic ${scenarioId} fixture execution.`,
|
||||
scenarioId,
|
||||
pluginId: PLUGIN_ID,
|
||||
},
|
||||
decision: "approval",
|
||||
};
|
||||
}
|
||||
return {
|
||||
params,
|
||||
decision: scenarioId === "observe" ? "observe" : "allow",
|
||||
};
|
||||
}
|
||||
|
||||
function createToolCallParams(event, scenarioId) {
|
||||
const rawParams = event?.params && typeof event.params === "object" ? event.params : {};
|
||||
const rawArgs = event?.args && typeof event.args === "object" ? event.args : {};
|
||||
return {
|
||||
...rawParams,
|
||||
args: {
|
||||
...rawArgs,
|
||||
kitchenSinkScenario: scenarioId,
|
||||
kitchenSinkPluginId: PLUGIN_ID,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function createConversationPrivacyProbe({ event, context, text }) {
|
||||
const eventText = extractHookText(event);
|
||||
const contextText = extractHookText(context);
|
||||
const combined = [text, eventText, contextText].filter(Boolean).join("\n");
|
||||
const redactedFields = [];
|
||||
for (const [label, value] of [
|
||||
["event.apiKey", event?.apiKey],
|
||||
["event.authorization", event?.authorization],
|
||||
["event.token", event?.token],
|
||||
["context.apiKey", context?.apiKey],
|
||||
["context.authorization", context?.authorization],
|
||||
["context.token", context?.token],
|
||||
]) {
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
redactedFields.push(label);
|
||||
}
|
||||
}
|
||||
const secretText = [
|
||||
combined,
|
||||
event?.apiKey,
|
||||
event?.authorization,
|
||||
event?.token,
|
||||
context?.apiKey,
|
||||
context?.authorization,
|
||||
context?.token,
|
||||
].filter(Boolean).join("\n");
|
||||
const secretPatternHits = secretText.match(/\b(?:sk-[a-z0-9_-]+|api[_-]?key|authorization|bearer\s+[a-z0-9._-]+|fixture-token-[a-z0-9_-]+)\b/gi) ?? [];
|
||||
return {
|
||||
boundary: "conversation-observer",
|
||||
promptHash: stableHash(combined || "empty"),
|
||||
promptLength: combined.length,
|
||||
redactedFields,
|
||||
secretPatternCount: secretPatternHits.length,
|
||||
storesRawPayload: false,
|
||||
exposesRawPayload: false,
|
||||
};
|
||||
}
|
||||
|
||||
function firstHookString(source, keys) {
|
||||
if (!source || typeof source !== "object") {
|
||||
return "";
|
||||
|
||||
Loading…
Reference in New Issue
Block a user