From 2254df72f9781314f3c9be28de2f68dee47cfc8a Mon Sep 17 00:00:00 2001 From: Shakker Date: Fri, 1 May 2026 16:23:11 +0100 Subject: [PATCH] feat: split agent message ingress surfaces --- process-roles/agent-cli.json | 2 +- process-roles/dashboard-cli.json | 4 +- process-roles/openai-compatible-client.json | 7 ++ process-roles/tui-cli.json | 4 +- profiles/diagnostic.json | 74 +++++++++++- profiles/exhaustive.json | 4 + profiles/release.json | 76 +++++++++++- profiles/smoke.json | 15 +++ profiles/soak.json | 10 ++ scenarios/agent-auth-missing.json | 2 +- scenarios/agent-cold-warm-message.json | 6 +- scenarios/agent-gateway-rpc-turn.json | 47 ++++++++ scenarios/agent-long-session.json | 2 +- scenarios/agent-provider-concurrent.json | 2 +- scenarios/agent-provider-malformed.json | 2 +- scenarios/agent-provider-recovery.json | 2 +- scenarios/agent-provider-slow.json | 2 +- scenarios/agent-provider-streaming-stall.json | 2 +- scenarios/agent-provider-timeout.json | 2 +- scenarios/dashboard-session-send-turn.json | 47 ++++++++ scenarios/openai-compatible-turn.json | 47 ++++++++ scenarios/tui-message-turn.json | 47 ++++++++ src/evaluator.mjs | 17 ++- src/runner.mjs | 5 +- src/selfcheck.mjs | 18 ++- states/agent-auth-missing.json | 2 +- states/mock-openai-provider.json | 6 +- support/openclaw-runtime.mjs | 77 ++++++++++++ support/run-dashboard-session-send-turn.mjs | 111 ++++++++++++++++++ support/run-openai-compatible-turn.mjs | 80 +++++++++++++ support/run-tui-message-turn.mjs | 106 +++++++++++++++++ ...message.json => agent-cli-local-turn.json} | 6 +- surfaces/agent-gateway-rpc-turn.json | 20 ++++ surfaces/dashboard-session-send-turn.json | 20 ++++ surfaces/openai-compatible-turn.json | 20 ++++ surfaces/tui-message-turn.json | 20 ++++ 36 files changed, 882 insertions(+), 32 deletions(-) create mode 100644 process-roles/openai-compatible-client.json create mode 100644 scenarios/agent-gateway-rpc-turn.json create mode 100644 scenarios/dashboard-session-send-turn.json create mode 100644 scenarios/openai-compatible-turn.json create mode 100644 scenarios/tui-message-turn.json create mode 100644 support/openclaw-runtime.mjs create mode 100755 support/run-dashboard-session-send-turn.mjs create mode 100755 support/run-openai-compatible-turn.mjs create mode 100755 support/run-tui-message-turn.mjs rename surfaces/{agent-message.json => agent-cli-local-turn.json} (82%) create mode 100644 surfaces/agent-gateway-rpc-turn.json create mode 100644 surfaces/dashboard-session-send-turn.json create mode 100644 surfaces/openai-compatible-turn.json create mode 100644 surfaces/tui-message-turn.json diff --git a/process-roles/agent-cli.json b/process-roles/agent-cli.json index fd211fb..c4cbda8 100644 --- a/process-roles/agent-cli.json +++ b/process-roles/agent-cli.json @@ -2,6 +2,6 @@ "id": "agent-cli", "title": "Agent CLI", "description": "User-facing OpenClaw agent command invocations that send messages or manage local sessions.", - "commandPatterns": ["agent --local", "agent --session-id"], + "commandPatterns": ["agent --local", "agent --session-id", " -- agent ", "run-concurrent-agent-turns.mjs"], "processPatterns": ["(^|\\s|/)openclaw\\s+.*\\bagent\\b", "(^|\\s|/)openclaw-agent(\\s|$)"] } diff --git a/process-roles/dashboard-cli.json b/process-roles/dashboard-cli.json index 886c023..6bd8aa1 100644 --- a/process-roles/dashboard-cli.json +++ b/process-roles/dashboard-cli.json @@ -2,6 +2,6 @@ "id": "dashboard-cli", "title": "Dashboard CLI", "description": "OpenClaw dashboard command paths that produce or validate dashboard URLs and websocket configuration.", - "commandPatterns": ["dashboard", "dashboard --no-open"], - "processPatterns": ["openclaw.*dashboard"] + "commandPatterns": ["dashboard", "dashboard --no-open", "run-dashboard-session-send-turn.mjs", "sessions.send"], + "processPatterns": ["openclaw.*dashboard", "node\\s+.*run-dashboard-session-send-turn\\.mjs"] } diff --git a/process-roles/openai-compatible-client.json b/process-roles/openai-compatible-client.json new file mode 100644 index 0000000..baea725 --- /dev/null +++ b/process-roles/openai-compatible-client.json @@ -0,0 +1,7 @@ +{ + "id": "openai-compatible-client", + "title": "OpenAI-Compatible Client", + "description": "Kova client process that drives OpenClaw's OpenAI-compatible HTTP API.", + "commandPatterns": ["run-openai-compatible-turn.mjs", "/v1/chat/completions"], + "processPatterns": ["node\\s+.*run-openai-compatible-turn\\.mjs"] +} diff --git a/process-roles/tui-cli.json b/process-roles/tui-cli.json index 837c511..96a5b53 100644 --- a/process-roles/tui-cli.json +++ b/process-roles/tui-cli.json @@ -2,6 +2,6 @@ "id": "tui-cli", "title": "TUI CLI", "description": "OpenClaw terminal UI attachment, rendering, input, and shutdown paths.", - "commandPatterns": ["tui", "support/tui-smoke.mjs"], - "processPatterns": ["openclaw.*tui"] + "commandPatterns": ["tui", "support/tui-smoke.mjs", "run-tui-message-turn.mjs"], + "processPatterns": ["openclaw.*tui", "node\\s+.*run-tui-message-turn\\.mjs"] } diff --git a/profiles/diagnostic.json b/profiles/diagnostic.json index 9d9bc52..9d2a91a 100644 --- a/profiles/diagnostic.json +++ b/profiles/diagnostic.json @@ -37,7 +37,7 @@ "openclawSlowestSpanMs": 45000 } }, - "agent-message": { + "agent-cli-local-turn": { "thresholds": { "coldAgentTurnMs": 60000, "warmAgentTurnMs": 15000, @@ -47,6 +47,42 @@ "openclawSlowestSpanMs": 45000 } }, + "agent-gateway-rpc-turn": { + "thresholds": { + "agentTurnMs": 60000, + "preProviderMs": 15000, + "providerFinalMs": 3000, + "agentCleanupMs": 5000, + "openclawSlowestSpanMs": 45000 + } + }, + "dashboard-session-send-turn": { + "thresholds": { + "agentTurnMs": 60000, + "preProviderMs": 15000, + "providerFinalMs": 3000, + "agentCleanupMs": 5000, + "openclawSlowestSpanMs": 45000 + } + }, + "tui-message-turn": { + "thresholds": { + "agentTurnMs": 60000, + "preProviderMs": 15000, + "providerFinalMs": 3000, + "agentCleanupMs": 5000, + "openclawSlowestSpanMs": 45000 + } + }, + "openai-compatible-turn": { + "thresholds": { + "agentTurnMs": 60000, + "preProviderMs": 15000, + "providerFinalMs": 3000, + "agentCleanupMs": 5000, + "openclawSlowestSpanMs": 45000 + } + }, "bundled-runtime-deps": { "thresholds": { "runtimeDepsStagingMs": 45000, @@ -68,7 +104,7 @@ "id": "openclaw-diagnostic", "coverage": { "surfaces": { - "blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-message"] + "blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-cli-local-turn", "agent-gateway-rpc-turn", "dashboard-session-send-turn", "tui-message-turn", "openai-compatible-turn"] }, "states": { "blocking": ["fresh", "missing-plugin-index", "many-bundled-plugins", "mock-openai-provider"] @@ -78,18 +114,26 @@ "release-runtime-startup:fresh", "gateway-performance:many-bundled-plugins", "bundled-runtime-deps:missing-plugin-index", - "agent-message:mock-openai-provider" + "agent-cli-local-turn:mock-openai-provider", + "agent-gateway-rpc-turn:mock-openai-provider", + "dashboard-session-send-turn:mock-openai-provider", + "tui-message-turn:mock-openai-provider", + "openai-compatible-turn:mock-openai-provider" ] }, "scenarios": { - "blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-cold-warm-message"] + "blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-cold-warm-message", "agent-gateway-rpc-turn", "dashboard-session-send-turn", "tui-message-turn", "openai-compatible-turn"] } }, "blocking": [ { "scenario": "release-runtime-startup", "state": "fresh" }, { "scenario": "gateway-performance", "state": "many-bundled-plugins" }, { "scenario": "bundled-runtime-deps", "state": "missing-plugin-index" }, - { "scenario": "agent-cold-warm-message", "state": "mock-openai-provider" } + { "scenario": "agent-cold-warm-message", "state": "mock-openai-provider" }, + { "scenario": "agent-gateway-rpc-turn", "state": "mock-openai-provider" }, + { "scenario": "dashboard-session-send-turn", "state": "mock-openai-provider" }, + { "scenario": "tui-message-turn", "state": "mock-openai-provider" }, + { "scenario": "openai-compatible-turn", "state": "mock-openai-provider" } ] }, "entries": [ @@ -112,6 +156,26 @@ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider", "timeoutMs": 240000 + }, + { + "scenario": "agent-gateway-rpc-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "dashboard-session-send-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "tui-message-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "openai-compatible-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 } ] } diff --git a/profiles/exhaustive.json b/profiles/exhaustive.json index a4ba889..a371562 100644 --- a/profiles/exhaustive.json +++ b/profiles/exhaustive.json @@ -26,6 +26,10 @@ { "scenario": "provider-models", "state": "model-auth-configured" }, { "scenario": "provider-models", "state": "model-auth-missing" }, { "scenario": "agent-cold-warm-message", "state": "mock-openai-provider", "timeoutMs": 180000 }, + { "scenario": "agent-gateway-rpc-turn", "state": "mock-openai-provider", "timeoutMs": 180000 }, + { "scenario": "dashboard-session-send-turn", "state": "mock-openai-provider", "timeoutMs": 180000 }, + { "scenario": "tui-message-turn", "state": "mock-openai-provider", "timeoutMs": 180000 }, + { "scenario": "openai-compatible-turn", "state": "mock-openai-provider", "timeoutMs": 180000 }, { "scenario": "agent-auth-missing", "state": "agent-auth-missing", "timeoutMs": 180000 }, { "scenario": "agent-long-session", "state": "mock-openai-provider", "timeoutMs": 360000 }, { "scenario": "agent-provider-slow", "state": "mock-openai-provider", "timeoutMs": 180000 }, diff --git a/profiles/release.json b/profiles/release.json index f4f3d61..23205f5 100644 --- a/profiles/release.json +++ b/profiles/release.json @@ -15,6 +15,7 @@ "doctor-cli": { "peakRssMb": 700, "maxCpuPercent": 300 }, "tui-cli": { "peakRssMb": 650, "maxCpuPercent": 250 }, "dashboard-cli": { "peakRssMb": 650, "maxCpuPercent": 250 }, + "openai-compatible-client": { "peakRssMb": 650, "maxCpuPercent": 250 }, "mcp-runtime": { "peakRssMb": 500, "maxCpuPercent": 200 }, "browser-sidecar": { "peakRssMb": 700, "maxCpuPercent": 250 }, "mock-provider": { "peakRssMb": 300, "maxCpuPercent": 150 } @@ -39,7 +40,7 @@ "pluginLoadFailures": 0 } }, - "agent-message": { + "agent-cli-local-turn": { "thresholds": { "coldAgentTurnMs": 45000, "warmAgentTurnMs": 15000, @@ -50,6 +51,38 @@ "agentProcessLeaks": 0 } }, + "agent-gateway-rpc-turn": { + "thresholds": { + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "agentProcessLeaks": 0 + } + }, + "dashboard-session-send-turn": { + "thresholds": { + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "agentProcessLeaks": 0 + } + }, + "tui-message-turn": { + "thresholds": { + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "agentProcessLeaks": 0 + } + }, + "openai-compatible-turn": { + "thresholds": { + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "agentProcessLeaks": 0 + } + }, "bundled-runtime-deps": { "thresholds": { "runtimeDepsStagingMs": 30000, @@ -159,7 +192,11 @@ "plugin-lifecycle:plugin-index", "plugin-lifecycle:external-plugin", "gateway-performance:many-bundled-plugins", - "agent-message:mock-openai-provider", + "agent-cli-local-turn:mock-openai-provider", + "agent-gateway-rpc-turn:mock-openai-provider", + "dashboard-session-send-turn:mock-openai-provider", + "tui-message-turn:mock-openai-provider", + "openai-compatible-turn:mock-openai-provider", "provider-models:model-auth-missing", "dashboard:fresh", "tui:fresh" @@ -189,7 +226,11 @@ "plugin-missing-runtime-deps", "bundled-plugin-startup", "provider-models", - "agent-message", + "agent-cli-local-turn", + "agent-gateway-rpc-turn", + "dashboard-session-send-turn", + "tui-message-turn", + "openai-compatible-turn", "dashboard", "tui", "gateway-performance" @@ -206,6 +247,11 @@ "bundled-runtime-deps", "plugin-lifecycle", "provider-models", + "agent-cold-warm-message", + "agent-gateway-rpc-turn", + "dashboard-session-send-turn", + "tui-message-turn", + "openai-compatible-turn", "dashboard-readiness", "tui-responsiveness", "gateway-performance" @@ -237,6 +283,10 @@ { "scenario": "plugin-lifecycle", "state": "external-plugin" }, { "scenario": "provider-models", "state": "model-auth-missing" }, { "scenario": "agent-cold-warm-message", "state": "mock-openai-provider" }, + { "scenario": "agent-gateway-rpc-turn", "state": "mock-openai-provider" }, + { "scenario": "dashboard-session-send-turn", "state": "mock-openai-provider" }, + { "scenario": "tui-message-turn", "state": "mock-openai-provider" }, + { "scenario": "openai-compatible-turn", "state": "mock-openai-provider" }, { "scenario": "dashboard-readiness", "state": "fresh" }, { "scenario": "tui-responsiveness", "state": "fresh" }, { "scenario": "gateway-performance", "state": "many-bundled-plugins" } @@ -333,6 +383,26 @@ "state": "mock-openai-provider", "timeoutMs": 180000 }, + { + "scenario": "agent-gateway-rpc-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "dashboard-session-send-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "tui-message-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "openai-compatible-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, { "scenario": "agent-provider-slow", "state": "mock-openai-provider", diff --git a/profiles/smoke.json b/profiles/smoke.json index c6b7193..b161ee1 100644 --- a/profiles/smoke.json +++ b/profiles/smoke.json @@ -31,6 +31,21 @@ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider", "timeoutMs": 180000 + }, + { + "scenario": "agent-gateway-rpc-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "dashboard-session-send-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "openai-compatible-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 } ] } diff --git a/profiles/soak.json b/profiles/soak.json index 151262f..b6ef7b2 100644 --- a/profiles/soak.json +++ b/profiles/soak.json @@ -38,6 +38,16 @@ "state": "mock-openai-provider", "timeoutMs": 180000 }, + { + "scenario": "agent-gateway-rpc-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, + { + "scenario": "dashboard-session-send-turn", + "state": "mock-openai-provider", + "timeoutMs": 180000 + }, { "scenario": "agent-long-session", "state": "mock-openai-provider", diff --git a/scenarios/agent-auth-missing.json b/scenarios/agent-auth-missing.json index 1ba39cd..eb9c3ce 100644 --- a/scenarios/agent-auth-missing.json +++ b/scenarios/agent-auth-missing.json @@ -1,6 +1,6 @@ { "id": "agent-auth-missing", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Auth Missing", "objective": "Prove OpenClaw fails a local agent turn clearly when model auth is missing, keeps the gateway responsive, and leaves no leaked child processes.", "tags": ["agent", "message", "auth", "failure", "containment"], diff --git a/scenarios/agent-cold-warm-message.json b/scenarios/agent-cold-warm-message.json index 39f8818..9b4e0f1 100644 --- a/scenarios/agent-cold-warm-message.json +++ b/scenarios/agent-cold-warm-message.json @@ -1,8 +1,8 @@ { "id": "agent-cold-warm-message", - "surface": "agent-message", - "title": "Agent Cold/Warm Message", - "objective": "Send cold and warm simple messages through OpenClaw's real local agent path, verify mock-provider responses, and attribute latency before, during, and after provider work.", + "surface": "agent-cli-local-turn", + "title": "Agent CLI Local Cold/Warm Message", + "objective": "Send cold and warm simple messages through `openclaw agent --local`, verify mock-provider responses, and attribute latency before, during, and after provider work.", "tags": ["agent", "message", "latency", "providers", "gateway", "cold-warm"], "timeoutMs": 240000, "agent": { diff --git a/scenarios/agent-gateway-rpc-turn.json b/scenarios/agent-gateway-rpc-turn.json new file mode 100644 index 0000000..46c3d0e --- /dev/null +++ b/scenarios/agent-gateway-rpc-turn.json @@ -0,0 +1,47 @@ +{ + "id": "agent-gateway-rpc-turn", + "surface": "agent-gateway-rpc-turn", + "title": "Agent Gateway RPC Turn", + "objective": "Send a simple user message through `openclaw agent` without `--local`, forcing the CLI to cross the Gateway agent RPC boundary and prove final response, provider timing, gateway health, and process containment.", + "tags": ["agent", "message", "gateway", "rpc", "providers"], + "timeoutMs": 180000, + "agent": { + "expectedText": "KOVA_AGENT_OK" + }, + "thresholds": { + "gatewayReadyMs": 30000, + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "preProviderDominanceRatio": 0.8, + "statusMs": 10000, + "peakRssMb": 900, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "provision", + "title": "Provision Gateway Env", + "intent": "Start a disposable OpenClaw gateway before sending the Gateway RPC agent turn.", + "commands": ["ocm start {env} {startSelector} --json"], + "evidence": ["gateway port", "runtime binding", "startup readiness"] + }, + { + "id": "gateway-agent-turn", + "title": "Gateway Agent Turn", + "intent": "Send a user message through the Gateway-backed OpenClaw agent CLI path.", + "commands": [ + "ocm @{env} -- agent --agent main --session-id kova-agent-gateway-rpc --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --thinking off --timeout 120 --json" + ], + "evidence": ["command duration", "final assistant text", "mock provider request timing", "gateway health after turn", "role resource samples"] + }, + { + "id": "post-agent-health", + "title": "Post-Agent Gateway Health", + "intent": "Verify the gateway remains responsive after the Gateway RPC agent turn.", + "commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"], + "evidence": ["gateway status", "provider logs", "plugin errors", "memory after agent turn"] + } + ] +} diff --git a/scenarios/agent-long-session.json b/scenarios/agent-long-session.json index 1046a91..49d33a4 100644 --- a/scenarios/agent-long-session.json +++ b/scenarios/agent-long-session.json @@ -1,6 +1,6 @@ { "id": "agent-long-session", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Long Session", "objective": "Send repeated simple messages through one OpenClaw session to catch latency drift, provider routing drift, resource growth, health degradation, and child-process leaks during normal assistant use.", "tags": ["agent", "message", "latency", "providers", "soak", "long-session"], diff --git a/scenarios/agent-provider-concurrent.json b/scenarios/agent-provider-concurrent.json index ec296ab..bff1770 100644 --- a/scenarios/agent-provider-concurrent.json +++ b/scenarios/agent-provider-concurrent.json @@ -1,6 +1,6 @@ { "id": "agent-provider-concurrent", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Provider Concurrent Pressure", "objective": "Prove OpenClaw can process multiple overlapping local agent turns through the provider path, keep the gateway healthy, return correct responses, and leave no leaked child processes.", "tags": ["agent", "message", "provider-failure", "concurrency", "containment"], diff --git a/scenarios/agent-provider-malformed.json b/scenarios/agent-provider-malformed.json index 45888b4..86fde80 100644 --- a/scenarios/agent-provider-malformed.json +++ b/scenarios/agent-provider-malformed.json @@ -1,6 +1,6 @@ { "id": "agent-provider-malformed", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Provider Malformed", "objective": "Prove OpenClaw handles malformed provider responses as contained agent failures without taking down the gateway.", "tags": ["agent", "message", "provider-failure", "malformed", "containment"], diff --git a/scenarios/agent-provider-recovery.json b/scenarios/agent-provider-recovery.json index 0da0135..a58198b 100644 --- a/scenarios/agent-provider-recovery.json +++ b/scenarios/agent-provider-recovery.json @@ -1,6 +1,6 @@ { "id": "agent-provider-recovery", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Provider Recovery", "objective": "Prove a transient provider failure is contained and provider recovery produces a normal assistant response in the same OpenClaw env/session lifecycle.", "tags": ["agent", "message", "provider-failure", "recovery", "containment"], diff --git a/scenarios/agent-provider-slow.json b/scenarios/agent-provider-slow.json index f73cc01..1523e67 100644 --- a/scenarios/agent-provider-slow.json +++ b/scenarios/agent-provider-slow.json @@ -1,6 +1,6 @@ { "id": "agent-provider-slow", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Provider Slow", "objective": "Prove Kova can attribute a slow assistant turn to provider latency while confirming OpenClaw still returns a valid response and keeps the gateway healthy.", "tags": ["agent", "message", "provider-failure", "latency", "containment"], diff --git a/scenarios/agent-provider-streaming-stall.json b/scenarios/agent-provider-streaming-stall.json index a63fba0..0937091 100644 --- a/scenarios/agent-provider-streaming-stall.json +++ b/scenarios/agent-provider-streaming-stall.json @@ -1,6 +1,6 @@ { "id": "agent-provider-streaming-stall", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Provider Streaming Stall", "objective": "Prove OpenClaw contains a provider stream that starts but stalls, keeps the gateway responsive, and leaves no leaked agent/plugin child processes.", "tags": ["agent", "message", "provider-failure", "streaming-stall", "containment"], diff --git a/scenarios/agent-provider-timeout.json b/scenarios/agent-provider-timeout.json index 57ec772..49a835d 100644 --- a/scenarios/agent-provider-timeout.json +++ b/scenarios/agent-provider-timeout.json @@ -1,6 +1,6 @@ { "id": "agent-provider-timeout", - "surface": "agent-message", + "surface": "agent-cli-local-turn", "title": "Agent Provider Timeout", "objective": "Prove OpenClaw surfaces provider timeout failure clearly, keeps the gateway healthy, and leaves no retained child process state after a failed agent turn.", "tags": ["agent", "message", "provider-failure", "timeout", "containment"], diff --git a/scenarios/dashboard-session-send-turn.json b/scenarios/dashboard-session-send-turn.json new file mode 100644 index 0000000..48819c4 --- /dev/null +++ b/scenarios/dashboard-session-send-turn.json @@ -0,0 +1,47 @@ +{ + "id": "dashboard-session-send-turn", + "surface": "dashboard-session-send-turn", + "title": "Dashboard Session Send Turn", + "objective": "Create a dashboard session, send a user message through Gateway `sessions.send`, and wait for final assistant text in chat history.", + "tags": ["agent", "message", "dashboard", "sessions", "gateway", "providers"], + "timeoutMs": 180000, + "agent": { + "expectedText": "KOVA_AGENT_OK" + }, + "thresholds": { + "gatewayReadyMs": 30000, + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "preProviderDominanceRatio": 0.8, + "statusMs": 10000, + "peakRssMb": 900, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "provision", + "title": "Provision Dashboard Env", + "intent": "Start a disposable OpenClaw gateway before sending a dashboard session message.", + "commands": ["ocm start {env} {startSelector} --json"], + "evidence": ["gateway port", "runtime binding", "startup readiness"] + }, + { + "id": "dashboard-session-turn", + "title": "Dashboard Session Message", + "intent": "Exercise Gateway `sessions.send` and verify the final assistant response is present in chat history.", + "commands": [ + "ocm @{env} -- node {kovaRoot}/support/run-dashboard-session-send-turn.mjs --session-key kova-dashboard-session-send --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000" + ], + "evidence": ["sessions.send command duration", "chat history final assistant text", "mock provider request timing", "gateway health after turn", "role resource samples"] + }, + { + "id": "post-dashboard-health", + "title": "Post-Dashboard Gateway Health", + "intent": "Verify the gateway remains responsive after the dashboard-style message turn.", + "commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"], + "evidence": ["gateway status", "provider logs", "plugin errors", "memory after dashboard turn"] + } + ] +} diff --git a/scenarios/openai-compatible-turn.json b/scenarios/openai-compatible-turn.json new file mode 100644 index 0000000..26c5ea5 --- /dev/null +++ b/scenarios/openai-compatible-turn.json @@ -0,0 +1,47 @@ +{ + "id": "openai-compatible-turn", + "surface": "openai-compatible-turn", + "title": "OpenAI-Compatible Turn", + "objective": "Send a chat-completions request to OpenClaw's OpenAI-compatible HTTP endpoint and verify the final assistant response, provider timing, auth handling, and gateway health.", + "tags": ["agent", "message", "openai-compatible", "http", "providers"], + "timeoutMs": 180000, + "agent": { + "expectedText": "KOVA_AGENT_OK" + }, + "thresholds": { + "gatewayReadyMs": 30000, + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "preProviderDominanceRatio": 0.8, + "statusMs": 10000, + "peakRssMb": 900, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "provision", + "title": "Provision HTTP Env", + "intent": "Start a disposable OpenClaw gateway before sending an OpenAI-compatible HTTP request.", + "commands": ["ocm start {env} {startSelector} --json"], + "evidence": ["gateway port", "runtime binding", "startup readiness"] + }, + { + "id": "openai-compatible-turn", + "title": "OpenAI-Compatible Message", + "intent": "Exercise the `/v1/chat/completions` user API and verify final assistant output.", + "commands": [ + "ocm @{env} -- node {kovaRoot}/support/run-openai-compatible-turn.mjs --model openai/gpt-5.5 --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000" + ], + "evidence": ["HTTP status", "final assistant text", "mock provider request timing", "gateway health after turn", "role resource samples"] + }, + { + "id": "post-http-health", + "title": "Post-HTTP Gateway Health", + "intent": "Verify the gateway remains responsive after the OpenAI-compatible request.", + "commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"], + "evidence": ["gateway status", "provider logs", "plugin errors", "memory after HTTP turn"] + } + ] +} diff --git a/scenarios/tui-message-turn.json b/scenarios/tui-message-turn.json new file mode 100644 index 0000000..58f3f97 --- /dev/null +++ b/scenarios/tui-message-turn.json @@ -0,0 +1,47 @@ +{ + "id": "tui-message-turn", + "surface": "tui-message-turn", + "title": "TUI Message Turn", + "objective": "Launch the TUI, send a real user message through stdin, and require the expected assistant response to appear in the TUI output.", + "tags": ["agent", "message", "tui", "input", "providers"], + "timeoutMs": 180000, + "agent": { + "expectedText": "KOVA_AGENT_OK" + }, + "thresholds": { + "gatewayReadyMs": 30000, + "agentTurnMs": 45000, + "preProviderMs": 10000, + "providerFinalMs": 3000, + "preProviderDominanceRatio": 0.8, + "statusMs": 10000, + "peakRssMb": 900, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "provision", + "title": "Provision TUI Env", + "intent": "Start a disposable OpenClaw gateway before attaching the TUI.", + "commands": ["ocm start {env} {startSelector} --json"], + "evidence": ["gateway port", "runtime binding", "startup readiness"] + }, + { + "id": "tui-message-turn", + "title": "TUI Message", + "intent": "Exercise the user-visible terminal input path and verify final assistant output.", + "commands": [ + "node {kovaRoot}/support/run-tui-message-turn.mjs --env {env} --session kova-tui-message --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000" + ], + "evidence": ["TUI input accepted", "final assistant text rendered", "mock provider request timing", "gateway health after turn", "role resource samples"] + }, + { + "id": "post-tui-health", + "title": "Post-TUI Gateway Health", + "intent": "Verify the gateway remains responsive after the TUI message turn.", + "commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"], + "evidence": ["gateway status", "provider logs", "plugin errors", "memory after TUI turn"] + } + ] +} diff --git a/src/evaluator.mjs b/src/evaluator.mjs index 33077db..1ef4cb3 100644 --- a/src/evaluator.mjs +++ b/src/evaluator.mjs @@ -1625,6 +1625,18 @@ function agentTurnLabel(phaseId, index) { if (phaseId?.includes("warm")) { return "warm"; } + if (phaseId?.includes("gateway")) { + return "gateway-rpc"; + } + if (phaseId?.includes("dashboard")) { + return "dashboard-session"; + } + if (phaseId?.includes("tui")) { + return "tui"; + } + if (phaseId?.includes("openai")) { + return "openai-compatible"; + } return `turn-${index}`; } @@ -2779,7 +2791,10 @@ function countDiagnosticMetric(record, key) { function isAgentMessageCommand(command) { return (command.includes(" -- agent ") && command.includes("--message")) || - command.includes("run-concurrent-agent-turns.mjs"); + command.includes("run-concurrent-agent-turns.mjs") || + command.includes("run-dashboard-session-send-turn.mjs") || + command.includes("run-tui-message-turn.mjs") || + command.includes("run-openai-compatible-turn.mjs"); } function extractAgentResponse(result) { diff --git a/src/runner.mjs b/src/runner.mjs index ddd7cc9..13e8242 100644 --- a/src/runner.mjs +++ b/src/runner.mjs @@ -632,7 +632,10 @@ async function runScenarioCommand(command, context, envName, artifactDir, phaseI function isAgentMessageCommand(command) { return (command.includes(" -- agent ") && command.includes("--message")) || - command.includes("run-concurrent-agent-turns.mjs"); + command.includes("run-concurrent-agent-turns.mjs") || + command.includes("run-dashboard-session-send-turn.mjs") || + command.includes("run-tui-message-turn.mjs") || + command.includes("run-openai-compatible-turn.mjs"); } function agentLeakRoles() { diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index e308dfa..775e2b5 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -182,6 +182,22 @@ export async function runSelfCheck(flags = {}) { throw new Error(`missing auth override should not inject auth phases: ${phaseIds.join(", ")}`); } })); + for (const item of [ + ["agent-gateway-rpc-turn", "agent-gateway-rpc-turn", "ocm @"], + ["dashboard-session-send-turn", "dashboard-session-send-turn", "run-dashboard-session-send-turn.mjs"], + ["tui-message-turn", "tui-message-turn", "run-tui-message-turn.mjs"], + ["openai-compatible-turn", "openai-compatible-turn", "run-openai-compatible-turn.mjs"] + ]) { + const [scenarioId, surfaceId, expectedCommand] = item; + checks.push(await jsonCommandCheck(`message-ingress-${scenarioId}-dry-run-json`, `node bin/kova.mjs run --target runtime:stable --scenario ${scenarioId} --state mock-openai-provider --report-dir ${quoteShell(tmp)} --json`, async (data) => { + const report = JSON.parse(await readFile(data.jsonPath, "utf8")); + const record = report.records?.[0]; + assertEqual(record?.surface, surfaceId, `${scenarioId} surface`); + assertEqual(record?.auth?.mode, "mock", `${scenarioId} mock auth mode`); + const commands = record?.phases?.flatMap((phase) => phase.commands ?? []) ?? []; + assertEqual(commands.some((command) => command.includes(expectedCommand)), true, `${scenarioId} ingress command`); + })); + } checks.push(await jsonCommandCheck("run-profiling-dry-run-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --node-profile --report-dir ${quoteShell(tmp)} --json`, async (data) => { const report = JSON.parse(await readFile(data.jsonPath, "utf8")); assertEqual(report.records?.[0]?.profiling?.enabled, true, "profiling marker"); @@ -2985,7 +3001,7 @@ function syntheticCompareReport({ runId, target, timelineAvailable, preProviderM summary: { statuses: { PASS: 1 } }, records: [{ scenario: "agent-cold-warm-message", - surface: "agent-message", + surface: "agent-cli-local-turn", state: { id: "mock-openai-provider" }, status: "PASS", measurements: { diff --git a/states/agent-auth-missing.json b/states/agent-auth-missing.json index a5a8e8b..430b895 100644 --- a/states/agent-auth-missing.json +++ b/states/agent-auth-missing.json @@ -20,7 +20,7 @@ "reason": "This state intentionally bypasses Kova's default mock provider auth to test missing model credentials." }, "compatibleSurfaces": [ - "agent-message" + "agent-cli-local-turn" ], "incompatibleSurfaces": [], "riskArea": "agent-provider-auth", diff --git a/states/mock-openai-provider.json b/states/mock-openai-provider.json index 02a795d..ef9cb69 100644 --- a/states/mock-openai-provider.json +++ b/states/mock-openai-provider.json @@ -20,7 +20,11 @@ "agent-state" ], "compatibleSurfaces": [ - "agent-message" + "agent-cli-local-turn", + "agent-gateway-rpc-turn", + "dashboard-session-send-turn", + "tui-message-turn", + "openai-compatible-turn" ], "incompatibleSurfaces": [], "riskArea": "agent-provider-latency", diff --git a/support/openclaw-runtime.mjs b/support/openclaw-runtime.mjs new file mode 100644 index 0000000..e8ef980 --- /dev/null +++ b/support/openclaw-runtime.mjs @@ -0,0 +1,77 @@ +import { pathToFileURL } from "node:url"; +import path from "node:path"; + +export async function importOpenClawDistModule(relativePath) { + const packageRoot = process.cwd(); + const absolutePath = path.join(packageRoot, "dist", ...relativePath.split("/")); + try { + return await import(pathToFileURL(absolutePath).href); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error( + `failed to import OpenClaw runtime module ${relativePath} from ${packageRoot}: ${message}. ` + + "Kova user-message scenarios require a built/release-shaped OpenClaw runtime." + ); + } +} + +export function parseSupportArgs(argv) { + const parsed = {}; + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (!arg.startsWith("--")) { + throw new Error(`unexpected argument: ${arg}`); + } + const key = arg.slice(2); + const value = argv[index + 1]; + if (!value || value.startsWith("--")) { + throw new Error(`${arg} requires a value`); + } + parsed[key] = value; + index += 1; + } + return parsed; +} + +export function readTimeoutMs(value, fallbackMs) { + if (value === undefined) { + return fallbackMs; + } + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`invalid timeout: ${value}`); + } + return parsed; +} + +export function extractText(value) { + if (typeof value === "string") { + return value; + } + if (!value || typeof value !== "object") { + return ""; + } + if (Array.isArray(value)) { + return value.map(extractText).filter(Boolean).join("\n"); + } + for (const key of ["finalAssistantVisibleText", "finalAssistantRawText", "text", "content", "reply"]) { + if (typeof value[key] === "string") { + return value[key]; + } + } + return Object.values(value).map(extractText).filter(Boolean).join("\n"); +} + +export async function sleep(ms) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +export function finishJson(payload) { + process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`); +} + +export function failJson(error, extra = {}) { + const message = error instanceof Error ? error.message : String(error); + process.stdout.write(`${JSON.stringify({ ok: false, error: message, ...extra }, null, 2)}\n`); + process.exit(1); +} diff --git a/support/run-dashboard-session-send-turn.mjs b/support/run-dashboard-session-send-turn.mjs new file mode 100755 index 0000000..34124b3 --- /dev/null +++ b/support/run-dashboard-session-send-turn.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node + +import { randomUUID } from "node:crypto"; +import { + extractText, + failJson, + finishJson, + importOpenClawDistModule, + parseSupportArgs, + readTimeoutMs, + sleep +} from "./openclaw-runtime.mjs"; + +const startedAtEpochMs = Date.now(); + +try { + const args = parseSupportArgs(process.argv.slice(2)); + const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only."; + const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK"; + const timeoutMs = readTimeoutMs(args.timeout, 120000); + const sessionKey = args["session-key"] ?? `kova-dashboard-${randomUUID()}`; + const { callGateway } = await importOpenClawDistModule("gateway/call.js"); + + const created = await callGateway({ + method: "sessions.create", + params: { + agentId: "main", + key: sessionKey, + label: "Kova Dashboard Session Send" + }, + timeoutMs: Math.min(timeoutMs, 30000) + }); + const canonicalKey = created?.key ?? sessionKey; + const sendStartedAtEpochMs = Date.now(); + const sent = await callGateway({ + method: "sessions.send", + params: { + key: canonicalKey, + message, + thinking: "off", + timeoutMs, + idempotencyKey: `kova-dashboard-${randomUUID()}` + }, + timeoutMs: Math.min(timeoutMs, 30000) + }); + const runId = typeof sent?.runId === "string" ? sent.runId : null; + + const history = await waitForAssistantText({ + callGateway, + sessionKey: canonicalKey, + expectedText, + timeoutMs, + minAssistantCount: 1 + }); + + finishJson({ + ok: true, + surface: "dashboard-session-send-turn", + method: "sessions.send", + sessionKey: canonicalKey, + runId, + startedAtEpochMs, + sendStartedAtEpochMs, + finishedAtEpochMs: Date.now(), + finalAssistantVisibleText: history.matchedAssistantText, + finalAssistantRawText: history.lastAssistantText, + assistantMessageCount: history.assistantTexts.length, + expectedTextPresent: history.matchedAssistantText.includes(expectedText) + }); +} catch (error) { + failJson(error, { surface: "dashboard-session-send-turn", finishedAtEpochMs: Date.now() }); +} + +async function waitForAssistantText({ callGateway, sessionKey, expectedText, timeoutMs, minAssistantCount }) { + const deadline = Date.now() + timeoutMs; + let lastAssistantText = ""; + let assistantTexts = []; + while (Date.now() < deadline) { + const history = await callGateway({ + method: "chat.history", + params: { sessionKey, limit: 16 }, + timeoutMs: 15000 + }); + assistantTexts = extractAssistantTexts(history?.messages ?? []); + lastAssistantText = assistantTexts.at(-1) ?? ""; + const matchedAssistantText = assistantTexts + .slice(Math.max(0, minAssistantCount - 1)) + .find((text) => text.includes(expectedText)); + if (matchedAssistantText) { + return { assistantTexts, lastAssistantText, matchedAssistantText }; + } + await sleep(500); + } + throw new Error( + `timed out waiting for dashboard assistant text ${JSON.stringify(expectedText)}; last=${JSON.stringify(lastAssistantText)}` + ); +} + +function extractAssistantTexts(messages) { + if (!Array.isArray(messages)) { + return []; + } + return messages + .filter((message) => { + const role = String(message?.role ?? message?.sender ?? message?.type ?? "").toLowerCase(); + return role.includes("assistant") || role.includes("agent"); + }) + .map((message) => extractText(message)) + .map((text) => text.trim()) + .filter(Boolean); +} diff --git a/support/run-openai-compatible-turn.mjs b/support/run-openai-compatible-turn.mjs new file mode 100755 index 0000000..ef7118f --- /dev/null +++ b/support/run-openai-compatible-turn.mjs @@ -0,0 +1,80 @@ +#!/usr/bin/env node + +import { + extractText, + failJson, + finishJson, + importOpenClawDistModule, + parseSupportArgs, + readTimeoutMs +} from "./openclaw-runtime.mjs"; + +const startedAtEpochMs = Date.now(); + +try { + const args = parseSupportArgs(process.argv.slice(2)); + const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only."; + const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK"; + const timeoutMs = readTimeoutMs(args.timeout, 120000); + const model = args.model ?? "openai/gpt-5.5"; + const { getRuntimeConfig } = await importOpenClawDistModule("config/io.js"); + const { resolveGatewayPort } = await importOpenClawDistModule("config/paths.js"); + const cfg = getRuntimeConfig(); + const port = resolveGatewayPort(cfg, process.env); + const token = readGatewayToken(cfg); + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(new Error(`OpenAI-compatible request timed out after ${timeoutMs}ms`)), timeoutMs); + const requestStartedAtEpochMs = Date.now(); + try { + const response = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, { + method: "POST", + headers: { + "content-type": "application/json", + ...(token ? { authorization: `Bearer ${token}` } : {}) + }, + body: JSON.stringify({ + model, + messages: [{ role: "user", content: message }], + stream: false + }), + signal: controller.signal + }); + const bodyText = await response.text(); + let body = {}; + try { + body = bodyText ? JSON.parse(bodyText) : {}; + } catch { + body = { raw: bodyText }; + } + if (!response.ok) { + throw new Error(`OpenAI-compatible HTTP ${response.status}: ${bodyText.slice(0, 1000)}`); + } + const finalText = extractText(body?.choices?.[0]?.message ?? body); + finishJson({ + ok: true, + surface: "openai-compatible-turn", + method: "POST /v1/chat/completions", + model, + startedAtEpochMs, + requestStartedAtEpochMs, + finishedAtEpochMs: Date.now(), + status: response.status, + finalAssistantVisibleText: finalText, + finalAssistantRawText: finalText, + expectedTextPresent: finalText.includes(expectedText) + }); + } finally { + clearTimeout(timer); + } +} catch (error) { + failJson(error, { surface: "openai-compatible-turn", finishedAtEpochMs: Date.now() }); +} + +function readGatewayToken(cfg) { + const candidates = [ + process.env.OPENCLAW_GATEWAY_TOKEN, + cfg?.gateway?.auth?.token, + cfg?.gateway?.token + ]; + return candidates.find((value) => typeof value === "string" && value.trim().length > 0)?.trim() ?? ""; +} diff --git a/support/run-tui-message-turn.mjs b/support/run-tui-message-turn.mjs new file mode 100755 index 0000000..2072e02 --- /dev/null +++ b/support/run-tui-message-turn.mjs @@ -0,0 +1,106 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { failJson, finishJson, parseSupportArgs, readTimeoutMs } from "./openclaw-runtime.mjs"; + +const startedAtEpochMs = Date.now(); + +try { + const args = parseSupportArgs(process.argv.slice(2)); + const envName = args.env; + if (!envName) { + throw new Error("--env is required"); + } + const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only."; + const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK"; + const timeoutMs = readTimeoutMs(args.timeout, 120000); + const session = args.session ?? "kova-tui-message"; + const result = await runTuiTurn({ envName, message, expectedText, timeoutMs, session }); + finishJson({ + ok: true, + surface: "tui-message-turn", + method: "tui stdin/stdout", + session, + startedAtEpochMs, + inputAcceptedAtEpochMs: result.inputAcceptedAtEpochMs, + finishedAtEpochMs: Date.now(), + finalAssistantVisibleText: result.finalAssistantText, + finalAssistantRawText: result.finalAssistantText, + expectedTextPresent: result.finalAssistantText.includes(expectedText), + outputTail: result.outputTail + }); +} catch (error) { + failJson(error, { surface: "tui-message-turn", finishedAtEpochMs: Date.now() }); +} + +function runTuiTurn({ envName, message, expectedText, timeoutMs, session }) { + return new Promise((resolve, reject) => { + const child = spawn("ocm", [ + `@${envName}`, + "--", + "tui", + "--session", + session, + "--history-limit", + "5" + ], { + stdio: ["pipe", "pipe", "pipe"], + shell: false + }); + + let output = ""; + let inputSent = false; + let settled = false; + let inputAcceptedAtEpochMs = null; + + const timer = setTimeout(() => { + finish(new Error(`TUI turn did not complete within ${timeoutMs}ms`)); + }, timeoutMs); + + child.stdout.on("data", onData); + child.stderr.on("data", onData); + child.on("error", finish); + child.on("exit", (code, signal) => { + if (!settled) { + finish(new Error(`TUI exited before message turn completed (code=${code}, signal=${signal ?? "none"})`)); + } + }); + + function onData(chunk) { + output += chunk.toString("utf8"); + if (!inputSent && /openclaw tui|local ready|agent\s+main|session\s+/i.test(output)) { + inputSent = true; + inputAcceptedAtEpochMs = Date.now(); + child.stdin.write(`${message}\n`); + } + if (output.includes(expectedText)) { + finish(null, { + inputAcceptedAtEpochMs, + finalAssistantText: expectedText, + outputTail: output.slice(-4000) + }); + } + } + + function finish(error, value) { + if (settled) { + return; + } + settled = true; + clearTimeout(timer); + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGINT"); + setTimeout(() => { + if (child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + } + }, 1000).unref?.(); + } + if (error) { + reject(new Error(`${error.message}; outputTail=${JSON.stringify(output.slice(-4000))}`)); + } else { + resolve(value); + } + } + }); +} diff --git a/surfaces/agent-message.json b/surfaces/agent-cli-local-turn.json similarity index 82% rename from surfaces/agent-message.json rename to surfaces/agent-cli-local-turn.json index abca5be..43439ce 100644 --- a/surfaces/agent-message.json +++ b/surfaces/agent-cli-local-turn.json @@ -1,8 +1,8 @@ { - "id": "agent-message", - "title": "Agent Message", + "id": "agent-cli-local-turn", + "title": "Agent CLI Local Turn", "ownerArea": "agent-runtime", - "description": "Send cold, warm, and repeated local OpenClaw agent messages and verify response latency, provider routing, gateway health, memory, and logs.", + "description": "Send cold, warm, repeated, and failure-mode messages through `openclaw agent --local`, then verify response latency, provider routing, gateway health, memory, and process containment.", "requiredStates": ["mock-openai-provider"], "targetKinds": ["npm", "channel", "runtime", "local-build"], "requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "agentTurnMaxMs", "coldAgentTurnMs", "warmAgentTurnMs", "agentColdWarmDeltaMs", "coldPreProviderMs", "warmPreProviderMs", "agentPreProviderP95Ms", "agentCleanupMaxMs", "healthP95Ms", "peakRssMb", "providerTimeoutMentions", "pluginLoadFailures"], diff --git a/surfaces/agent-gateway-rpc-turn.json b/surfaces/agent-gateway-rpc-turn.json new file mode 100644 index 0000000..45ac9d5 --- /dev/null +++ b/surfaces/agent-gateway-rpc-turn.json @@ -0,0 +1,20 @@ +{ + "id": "agent-gateway-rpc-turn", + "title": "Agent Gateway RPC Turn", + "ownerArea": "gateway-agent-runtime", + "description": "Send messages through `openclaw agent` without `--local`, forcing the CLI to cross the Gateway agent RPC boundary before the agent turn runs.", + "requiredStates": ["mock-openai-provider"], + "targetKinds": ["npm", "channel", "runtime", "local-build"], + "requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "agentTurnMaxMs", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"], + "processRoles": ["gateway", "command-tree", "agent-cli", "agent-process", "mock-provider"], + "thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 }, + "roleThresholds": { + "gateway": { "peakRssMb": 700, "maxCpuPercent": 250 }, + "agent-cli": { "peakRssMb": 900, "maxCpuPercent": 300 }, + "mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 } + }, + "diagnostics": { + "timelineRequiredForSourceBuild": true, + "expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"] + } +} diff --git a/surfaces/dashboard-session-send-turn.json b/surfaces/dashboard-session-send-turn.json new file mode 100644 index 0000000..d32a117 --- /dev/null +++ b/surfaces/dashboard-session-send-turn.json @@ -0,0 +1,20 @@ +{ + "id": "dashboard-session-send-turn", + "title": "Dashboard Session Send Turn", + "ownerArea": "gateway-chat-session-runtime", + "description": "Create a dashboard session, call Gateway `sessions.send`, and wait for the assistant response in chat history to validate the same path dashboard users exercise.", + "requiredStates": ["mock-openai-provider"], + "targetKinds": ["npm", "channel", "runtime", "local-build"], + "requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"], + "processRoles": ["gateway", "command-tree", "dashboard-cli", "agent-process", "mock-provider"], + "thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 }, + "roleThresholds": { + "gateway": { "peakRssMb": 700, "maxCpuPercent": 250 }, + "dashboard-cli": { "peakRssMb": 700, "maxCpuPercent": 250 }, + "mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 } + }, + "diagnostics": { + "timelineRequiredForSourceBuild": true, + "expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"] + } +} diff --git a/surfaces/openai-compatible-turn.json b/surfaces/openai-compatible-turn.json new file mode 100644 index 0000000..36dfcb8 --- /dev/null +++ b/surfaces/openai-compatible-turn.json @@ -0,0 +1,20 @@ +{ + "id": "openai-compatible-turn", + "title": "OpenAI-Compatible Turn", + "ownerArea": "gateway-openai-compatible-runtime", + "description": "POST a user message to the OpenAI-compatible chat completions endpoint and verify the final response, provider timing, auth behavior, and gateway health.", + "requiredStates": ["mock-openai-provider"], + "targetKinds": ["npm", "channel", "runtime", "local-build"], + "requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"], + "processRoles": ["gateway", "command-tree", "openai-compatible-client", "agent-process", "mock-provider"], + "thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 }, + "roleThresholds": { + "gateway": { "peakRssMb": 700, "maxCpuPercent": 250 }, + "openai-compatible-client": { "peakRssMb": 700, "maxCpuPercent": 250 }, + "mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 } + }, + "diagnostics": { + "timelineRequiredForSourceBuild": true, + "expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"] + } +} diff --git a/surfaces/tui-message-turn.json b/surfaces/tui-message-turn.json new file mode 100644 index 0000000..4544390 --- /dev/null +++ b/surfaces/tui-message-turn.json @@ -0,0 +1,20 @@ +{ + "id": "tui-message-turn", + "title": "TUI Message Turn", + "ownerArea": "tui-agent-runtime", + "description": "Launch the OpenClaw TUI, send a real stdin message, and require visible assistant output so Kova can catch TUI input freezes and delayed user-visible responses.", + "requiredStates": ["mock-openai-provider"], + "targetKinds": ["npm", "channel", "runtime", "local-build"], + "requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"], + "processRoles": ["gateway", "command-tree", "tui-cli", "agent-process", "mock-provider"], + "thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 }, + "roleThresholds": { + "gateway": { "peakRssMb": 700, "maxCpuPercent": 250 }, + "tui-cli": { "peakRssMb": 900, "maxCpuPercent": 300 }, + "mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 } + }, + "diagnostics": { + "timelineRequiredForSourceBuild": true, + "expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"] + } +}