feat: split agent message ingress surfaces
This commit is contained in:
Shakker
parent
82ff7591be
commit
2254df72f9
@ -2,6 +2,6 @@
|
||||
"id": "agent-cli",
|
||||
"title": "Agent CLI",
|
||||
"description": "User-facing OpenClaw agent command invocations that send messages or manage local sessions.",
|
||||
"commandPatterns": ["agent --local", "agent --session-id"],
|
||||
"commandPatterns": ["agent --local", "agent --session-id", " -- agent ", "run-concurrent-agent-turns.mjs"],
|
||||
"processPatterns": ["(^|\\s|/)openclaw\\s+.*\\bagent\\b", "(^|\\s|/)openclaw-agent(\\s|$)"]
|
||||
}
|
||||
|
||||
@ -2,6 +2,6 @@
|
||||
"id": "dashboard-cli",
|
||||
"title": "Dashboard CLI",
|
||||
"description": "OpenClaw dashboard command paths that produce or validate dashboard URLs and websocket configuration.",
|
||||
"commandPatterns": ["dashboard", "dashboard --no-open"],
|
||||
"processPatterns": ["openclaw.*dashboard"]
|
||||
"commandPatterns": ["dashboard", "dashboard --no-open", "run-dashboard-session-send-turn.mjs", "sessions.send"],
|
||||
"processPatterns": ["openclaw.*dashboard", "node\\s+.*run-dashboard-session-send-turn\\.mjs"]
|
||||
}
|
||||
|
||||
7
process-roles/openai-compatible-client.json
Normal file
7
process-roles/openai-compatible-client.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": "openai-compatible-client",
|
||||
"title": "OpenAI-Compatible Client",
|
||||
"description": "Kova client process that drives OpenClaw's OpenAI-compatible HTTP API.",
|
||||
"commandPatterns": ["run-openai-compatible-turn.mjs", "/v1/chat/completions"],
|
||||
"processPatterns": ["node\\s+.*run-openai-compatible-turn\\.mjs"]
|
||||
}
|
||||
@ -2,6 +2,6 @@
|
||||
"id": "tui-cli",
|
||||
"title": "TUI CLI",
|
||||
"description": "OpenClaw terminal UI attachment, rendering, input, and shutdown paths.",
|
||||
"commandPatterns": ["tui", "support/tui-smoke.mjs"],
|
||||
"processPatterns": ["openclaw.*tui"]
|
||||
"commandPatterns": ["tui", "support/tui-smoke.mjs", "run-tui-message-turn.mjs"],
|
||||
"processPatterns": ["openclaw.*tui", "node\\s+.*run-tui-message-turn\\.mjs"]
|
||||
}
|
||||
|
||||
@ -37,7 +37,7 @@
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"agent-message": {
|
||||
"agent-cli-local-turn": {
|
||||
"thresholds": {
|
||||
"coldAgentTurnMs": 60000,
|
||||
"warmAgentTurnMs": 15000,
|
||||
@ -47,6 +47,42 @@
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"agent-gateway-rpc-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 60000,
|
||||
"preProviderMs": 15000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentCleanupMs": 5000,
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"dashboard-session-send-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 60000,
|
||||
"preProviderMs": 15000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentCleanupMs": 5000,
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"tui-message-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 60000,
|
||||
"preProviderMs": 15000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentCleanupMs": 5000,
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"openai-compatible-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 60000,
|
||||
"preProviderMs": 15000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentCleanupMs": 5000,
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"bundled-runtime-deps": {
|
||||
"thresholds": {
|
||||
"runtimeDepsStagingMs": 45000,
|
||||
@ -68,7 +104,7 @@
|
||||
"id": "openclaw-diagnostic",
|
||||
"coverage": {
|
||||
"surfaces": {
|
||||
"blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-message"]
|
||||
"blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-cli-local-turn", "agent-gateway-rpc-turn", "dashboard-session-send-turn", "tui-message-turn", "openai-compatible-turn"]
|
||||
},
|
||||
"states": {
|
||||
"blocking": ["fresh", "missing-plugin-index", "many-bundled-plugins", "mock-openai-provider"]
|
||||
@ -78,18 +114,26 @@
|
||||
"release-runtime-startup:fresh",
|
||||
"gateway-performance:many-bundled-plugins",
|
||||
"bundled-runtime-deps:missing-plugin-index",
|
||||
"agent-message:mock-openai-provider"
|
||||
"agent-cli-local-turn:mock-openai-provider",
|
||||
"agent-gateway-rpc-turn:mock-openai-provider",
|
||||
"dashboard-session-send-turn:mock-openai-provider",
|
||||
"tui-message-turn:mock-openai-provider",
|
||||
"openai-compatible-turn:mock-openai-provider"
|
||||
]
|
||||
},
|
||||
"scenarios": {
|
||||
"blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-cold-warm-message"]
|
||||
"blocking": ["release-runtime-startup", "gateway-performance", "bundled-runtime-deps", "agent-cold-warm-message", "agent-gateway-rpc-turn", "dashboard-session-send-turn", "tui-message-turn", "openai-compatible-turn"]
|
||||
}
|
||||
},
|
||||
"blocking": [
|
||||
{ "scenario": "release-runtime-startup", "state": "fresh" },
|
||||
{ "scenario": "gateway-performance", "state": "many-bundled-plugins" },
|
||||
{ "scenario": "bundled-runtime-deps", "state": "missing-plugin-index" },
|
||||
{ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider" }
|
||||
{ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider" },
|
||||
{ "scenario": "agent-gateway-rpc-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "dashboard-session-send-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "tui-message-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "openai-compatible-turn", "state": "mock-openai-provider" }
|
||||
]
|
||||
},
|
||||
"entries": [
|
||||
@ -112,6 +156,26 @@
|
||||
"scenario": "agent-cold-warm-message",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 240000
|
||||
},
|
||||
{
|
||||
"scenario": "agent-gateway-rpc-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "tui-message-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "openai-compatible-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -26,6 +26,10 @@
|
||||
{ "scenario": "provider-models", "state": "model-auth-configured" },
|
||||
{ "scenario": "provider-models", "state": "model-auth-missing" },
|
||||
{ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider", "timeoutMs": 180000 },
|
||||
{ "scenario": "agent-gateway-rpc-turn", "state": "mock-openai-provider", "timeoutMs": 180000 },
|
||||
{ "scenario": "dashboard-session-send-turn", "state": "mock-openai-provider", "timeoutMs": 180000 },
|
||||
{ "scenario": "tui-message-turn", "state": "mock-openai-provider", "timeoutMs": 180000 },
|
||||
{ "scenario": "openai-compatible-turn", "state": "mock-openai-provider", "timeoutMs": 180000 },
|
||||
{ "scenario": "agent-auth-missing", "state": "agent-auth-missing", "timeoutMs": 180000 },
|
||||
{ "scenario": "agent-long-session", "state": "mock-openai-provider", "timeoutMs": 360000 },
|
||||
{ "scenario": "agent-provider-slow", "state": "mock-openai-provider", "timeoutMs": 180000 },
|
||||
|
||||
@ -15,6 +15,7 @@
|
||||
"doctor-cli": { "peakRssMb": 700, "maxCpuPercent": 300 },
|
||||
"tui-cli": { "peakRssMb": 650, "maxCpuPercent": 250 },
|
||||
"dashboard-cli": { "peakRssMb": 650, "maxCpuPercent": 250 },
|
||||
"openai-compatible-client": { "peakRssMb": 650, "maxCpuPercent": 250 },
|
||||
"mcp-runtime": { "peakRssMb": 500, "maxCpuPercent": 200 },
|
||||
"browser-sidecar": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"mock-provider": { "peakRssMb": 300, "maxCpuPercent": 150 }
|
||||
@ -39,7 +40,7 @@
|
||||
"pluginLoadFailures": 0
|
||||
}
|
||||
},
|
||||
"agent-message": {
|
||||
"agent-cli-local-turn": {
|
||||
"thresholds": {
|
||||
"coldAgentTurnMs": 45000,
|
||||
"warmAgentTurnMs": 15000,
|
||||
@ -50,6 +51,38 @@
|
||||
"agentProcessLeaks": 0
|
||||
}
|
||||
},
|
||||
"agent-gateway-rpc-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentProcessLeaks": 0
|
||||
}
|
||||
},
|
||||
"dashboard-session-send-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentProcessLeaks": 0
|
||||
}
|
||||
},
|
||||
"tui-message-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentProcessLeaks": 0
|
||||
}
|
||||
},
|
||||
"openai-compatible-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"agentProcessLeaks": 0
|
||||
}
|
||||
},
|
||||
"bundled-runtime-deps": {
|
||||
"thresholds": {
|
||||
"runtimeDepsStagingMs": 30000,
|
||||
@ -159,7 +192,11 @@
|
||||
"plugin-lifecycle:plugin-index",
|
||||
"plugin-lifecycle:external-plugin",
|
||||
"gateway-performance:many-bundled-plugins",
|
||||
"agent-message:mock-openai-provider",
|
||||
"agent-cli-local-turn:mock-openai-provider",
|
||||
"agent-gateway-rpc-turn:mock-openai-provider",
|
||||
"dashboard-session-send-turn:mock-openai-provider",
|
||||
"tui-message-turn:mock-openai-provider",
|
||||
"openai-compatible-turn:mock-openai-provider",
|
||||
"provider-models:model-auth-missing",
|
||||
"dashboard:fresh",
|
||||
"tui:fresh"
|
||||
@ -189,7 +226,11 @@
|
||||
"plugin-missing-runtime-deps",
|
||||
"bundled-plugin-startup",
|
||||
"provider-models",
|
||||
"agent-message",
|
||||
"agent-cli-local-turn",
|
||||
"agent-gateway-rpc-turn",
|
||||
"dashboard-session-send-turn",
|
||||
"tui-message-turn",
|
||||
"openai-compatible-turn",
|
||||
"dashboard",
|
||||
"tui",
|
||||
"gateway-performance"
|
||||
@ -206,6 +247,11 @@
|
||||
"bundled-runtime-deps",
|
||||
"plugin-lifecycle",
|
||||
"provider-models",
|
||||
"agent-cold-warm-message",
|
||||
"agent-gateway-rpc-turn",
|
||||
"dashboard-session-send-turn",
|
||||
"tui-message-turn",
|
||||
"openai-compatible-turn",
|
||||
"dashboard-readiness",
|
||||
"tui-responsiveness",
|
||||
"gateway-performance"
|
||||
@ -237,6 +283,10 @@
|
||||
{ "scenario": "plugin-lifecycle", "state": "external-plugin" },
|
||||
{ "scenario": "provider-models", "state": "model-auth-missing" },
|
||||
{ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider" },
|
||||
{ "scenario": "agent-gateway-rpc-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "dashboard-session-send-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "tui-message-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "openai-compatible-turn", "state": "mock-openai-provider" },
|
||||
{ "scenario": "dashboard-readiness", "state": "fresh" },
|
||||
{ "scenario": "tui-responsiveness", "state": "fresh" },
|
||||
{ "scenario": "gateway-performance", "state": "many-bundled-plugins" }
|
||||
@ -333,6 +383,26 @@
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "agent-gateway-rpc-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "tui-message-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "openai-compatible-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "agent-provider-slow",
|
||||
"state": "mock-openai-provider",
|
||||
|
||||
@ -31,6 +31,21 @@
|
||||
"scenario": "agent-cold-warm-message",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "agent-gateway-rpc-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "openai-compatible-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@ -38,6 +38,16 @@
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "agent-gateway-rpc-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "agent-long-session",
|
||||
"state": "mock-openai-provider",
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-auth-missing",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Auth Missing",
|
||||
"objective": "Prove OpenClaw fails a local agent turn clearly when model auth is missing, keeps the gateway responsive, and leaves no leaked child processes.",
|
||||
"tags": ["agent", "message", "auth", "failure", "containment"],
|
||||
|
||||
@ -1,8 +1,8 @@
|
||||
{
|
||||
"id": "agent-cold-warm-message",
|
||||
"surface": "agent-message",
|
||||
"title": "Agent Cold/Warm Message",
|
||||
"objective": "Send cold and warm simple messages through OpenClaw's real local agent path, verify mock-provider responses, and attribute latency before, during, and after provider work.",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent CLI Local Cold/Warm Message",
|
||||
"objective": "Send cold and warm simple messages through `openclaw agent --local`, verify mock-provider responses, and attribute latency before, during, and after provider work.",
|
||||
"tags": ["agent", "message", "latency", "providers", "gateway", "cold-warm"],
|
||||
"timeoutMs": 240000,
|
||||
"agent": {
|
||||
|
||||
47
scenarios/agent-gateway-rpc-turn.json
Normal file
47
scenarios/agent-gateway-rpc-turn.json
Normal file
@ -0,0 +1,47 @@
|
||||
{
|
||||
"id": "agent-gateway-rpc-turn",
|
||||
"surface": "agent-gateway-rpc-turn",
|
||||
"title": "Agent Gateway RPC Turn",
|
||||
"objective": "Send a simple user message through `openclaw agent` without `--local`, forcing the CLI to cross the Gateway agent RPC boundary and prove final response, provider timing, gateway health, and process containment.",
|
||||
"tags": ["agent", "message", "gateway", "rpc", "providers"],
|
||||
"timeoutMs": 180000,
|
||||
"agent": {
|
||||
"expectedText": "KOVA_AGENT_OK"
|
||||
},
|
||||
"thresholds": {
|
||||
"gatewayReadyMs": 30000,
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"preProviderDominanceRatio": 0.8,
|
||||
"statusMs": 10000,
|
||||
"peakRssMb": 900,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "provision",
|
||||
"title": "Provision Gateway Env",
|
||||
"intent": "Start a disposable OpenClaw gateway before sending the Gateway RPC agent turn.",
|
||||
"commands": ["ocm start {env} {startSelector} --json"],
|
||||
"evidence": ["gateway port", "runtime binding", "startup readiness"]
|
||||
},
|
||||
{
|
||||
"id": "gateway-agent-turn",
|
||||
"title": "Gateway Agent Turn",
|
||||
"intent": "Send a user message through the Gateway-backed OpenClaw agent CLI path.",
|
||||
"commands": [
|
||||
"ocm @{env} -- agent --agent main --session-id kova-agent-gateway-rpc --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --thinking off --timeout 120 --json"
|
||||
],
|
||||
"evidence": ["command duration", "final assistant text", "mock provider request timing", "gateway health after turn", "role resource samples"]
|
||||
},
|
||||
{
|
||||
"id": "post-agent-health",
|
||||
"title": "Post-Agent Gateway Health",
|
||||
"intent": "Verify the gateway remains responsive after the Gateway RPC agent turn.",
|
||||
"commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"],
|
||||
"evidence": ["gateway status", "provider logs", "plugin errors", "memory after agent turn"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-long-session",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Long Session",
|
||||
"objective": "Send repeated simple messages through one OpenClaw session to catch latency drift, provider routing drift, resource growth, health degradation, and child-process leaks during normal assistant use.",
|
||||
"tags": ["agent", "message", "latency", "providers", "soak", "long-session"],
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-provider-concurrent",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Provider Concurrent Pressure",
|
||||
"objective": "Prove OpenClaw can process multiple overlapping local agent turns through the provider path, keep the gateway healthy, return correct responses, and leave no leaked child processes.",
|
||||
"tags": ["agent", "message", "provider-failure", "concurrency", "containment"],
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-provider-malformed",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Provider Malformed",
|
||||
"objective": "Prove OpenClaw handles malformed provider responses as contained agent failures without taking down the gateway.",
|
||||
"tags": ["agent", "message", "provider-failure", "malformed", "containment"],
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-provider-recovery",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Provider Recovery",
|
||||
"objective": "Prove a transient provider failure is contained and provider recovery produces a normal assistant response in the same OpenClaw env/session lifecycle.",
|
||||
"tags": ["agent", "message", "provider-failure", "recovery", "containment"],
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-provider-slow",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Provider Slow",
|
||||
"objective": "Prove Kova can attribute a slow assistant turn to provider latency while confirming OpenClaw still returns a valid response and keeps the gateway healthy.",
|
||||
"tags": ["agent", "message", "provider-failure", "latency", "containment"],
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-provider-streaming-stall",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Provider Streaming Stall",
|
||||
"objective": "Prove OpenClaw contains a provider stream that starts but stalls, keeps the gateway responsive, and leaves no leaked agent/plugin child processes.",
|
||||
"tags": ["agent", "message", "provider-failure", "streaming-stall", "containment"],
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"id": "agent-provider-timeout",
|
||||
"surface": "agent-message",
|
||||
"surface": "agent-cli-local-turn",
|
||||
"title": "Agent Provider Timeout",
|
||||
"objective": "Prove OpenClaw surfaces provider timeout failure clearly, keeps the gateway healthy, and leaves no retained child process state after a failed agent turn.",
|
||||
"tags": ["agent", "message", "provider-failure", "timeout", "containment"],
|
||||
|
||||
47
scenarios/dashboard-session-send-turn.json
Normal file
47
scenarios/dashboard-session-send-turn.json
Normal file
@ -0,0 +1,47 @@
|
||||
{
|
||||
"id": "dashboard-session-send-turn",
|
||||
"surface": "dashboard-session-send-turn",
|
||||
"title": "Dashboard Session Send Turn",
|
||||
"objective": "Create a dashboard session, send a user message through Gateway `sessions.send`, and wait for final assistant text in chat history.",
|
||||
"tags": ["agent", "message", "dashboard", "sessions", "gateway", "providers"],
|
||||
"timeoutMs": 180000,
|
||||
"agent": {
|
||||
"expectedText": "KOVA_AGENT_OK"
|
||||
},
|
||||
"thresholds": {
|
||||
"gatewayReadyMs": 30000,
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"preProviderDominanceRatio": 0.8,
|
||||
"statusMs": 10000,
|
||||
"peakRssMb": 900,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "provision",
|
||||
"title": "Provision Dashboard Env",
|
||||
"intent": "Start a disposable OpenClaw gateway before sending a dashboard session message.",
|
||||
"commands": ["ocm start {env} {startSelector} --json"],
|
||||
"evidence": ["gateway port", "runtime binding", "startup readiness"]
|
||||
},
|
||||
{
|
||||
"id": "dashboard-session-turn",
|
||||
"title": "Dashboard Session Message",
|
||||
"intent": "Exercise Gateway `sessions.send` and verify the final assistant response is present in chat history.",
|
||||
"commands": [
|
||||
"ocm @{env} -- node {kovaRoot}/support/run-dashboard-session-send-turn.mjs --session-key kova-dashboard-session-send --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
],
|
||||
"evidence": ["sessions.send command duration", "chat history final assistant text", "mock provider request timing", "gateway health after turn", "role resource samples"]
|
||||
},
|
||||
{
|
||||
"id": "post-dashboard-health",
|
||||
"title": "Post-Dashboard Gateway Health",
|
||||
"intent": "Verify the gateway remains responsive after the dashboard-style message turn.",
|
||||
"commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"],
|
||||
"evidence": ["gateway status", "provider logs", "plugin errors", "memory after dashboard turn"]
|
||||
}
|
||||
]
|
||||
}
|
||||
47
scenarios/openai-compatible-turn.json
Normal file
47
scenarios/openai-compatible-turn.json
Normal file
@ -0,0 +1,47 @@
|
||||
{
|
||||
"id": "openai-compatible-turn",
|
||||
"surface": "openai-compatible-turn",
|
||||
"title": "OpenAI-Compatible Turn",
|
||||
"objective": "Send a chat-completions request to OpenClaw's OpenAI-compatible HTTP endpoint and verify the final assistant response, provider timing, auth handling, and gateway health.",
|
||||
"tags": ["agent", "message", "openai-compatible", "http", "providers"],
|
||||
"timeoutMs": 180000,
|
||||
"agent": {
|
||||
"expectedText": "KOVA_AGENT_OK"
|
||||
},
|
||||
"thresholds": {
|
||||
"gatewayReadyMs": 30000,
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"preProviderDominanceRatio": 0.8,
|
||||
"statusMs": 10000,
|
||||
"peakRssMb": 900,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "provision",
|
||||
"title": "Provision HTTP Env",
|
||||
"intent": "Start a disposable OpenClaw gateway before sending an OpenAI-compatible HTTP request.",
|
||||
"commands": ["ocm start {env} {startSelector} --json"],
|
||||
"evidence": ["gateway port", "runtime binding", "startup readiness"]
|
||||
},
|
||||
{
|
||||
"id": "openai-compatible-turn",
|
||||
"title": "OpenAI-Compatible Message",
|
||||
"intent": "Exercise the `/v1/chat/completions` user API and verify final assistant output.",
|
||||
"commands": [
|
||||
"ocm @{env} -- node {kovaRoot}/support/run-openai-compatible-turn.mjs --model openai/gpt-5.5 --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
],
|
||||
"evidence": ["HTTP status", "final assistant text", "mock provider request timing", "gateway health after turn", "role resource samples"]
|
||||
},
|
||||
{
|
||||
"id": "post-http-health",
|
||||
"title": "Post-HTTP Gateway Health",
|
||||
"intent": "Verify the gateway remains responsive after the OpenAI-compatible request.",
|
||||
"commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"],
|
||||
"evidence": ["gateway status", "provider logs", "plugin errors", "memory after HTTP turn"]
|
||||
}
|
||||
]
|
||||
}
|
||||
47
scenarios/tui-message-turn.json
Normal file
47
scenarios/tui-message-turn.json
Normal file
@ -0,0 +1,47 @@
|
||||
{
|
||||
"id": "tui-message-turn",
|
||||
"surface": "tui-message-turn",
|
||||
"title": "TUI Message Turn",
|
||||
"objective": "Launch the TUI, send a real user message through stdin, and require the expected assistant response to appear in the TUI output.",
|
||||
"tags": ["agent", "message", "tui", "input", "providers"],
|
||||
"timeoutMs": 180000,
|
||||
"agent": {
|
||||
"expectedText": "KOVA_AGENT_OK"
|
||||
},
|
||||
"thresholds": {
|
||||
"gatewayReadyMs": 30000,
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
"providerFinalMs": 3000,
|
||||
"preProviderDominanceRatio": 0.8,
|
||||
"statusMs": 10000,
|
||||
"peakRssMb": 900,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "provision",
|
||||
"title": "Provision TUI Env",
|
||||
"intent": "Start a disposable OpenClaw gateway before attaching the TUI.",
|
||||
"commands": ["ocm start {env} {startSelector} --json"],
|
||||
"evidence": ["gateway port", "runtime binding", "startup readiness"]
|
||||
},
|
||||
{
|
||||
"id": "tui-message-turn",
|
||||
"title": "TUI Message",
|
||||
"intent": "Exercise the user-visible terminal input path and verify final assistant output.",
|
||||
"commands": [
|
||||
"node {kovaRoot}/support/run-tui-message-turn.mjs --env {env} --session kova-tui-message --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
],
|
||||
"evidence": ["TUI input accepted", "final assistant text rendered", "mock provider request timing", "gateway health after turn", "role resource samples"]
|
||||
},
|
||||
{
|
||||
"id": "post-tui-health",
|
||||
"title": "Post-TUI Gateway Health",
|
||||
"intent": "Verify the gateway remains responsive after the TUI message turn.",
|
||||
"commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"],
|
||||
"evidence": ["gateway status", "provider logs", "plugin errors", "memory after TUI turn"]
|
||||
}
|
||||
]
|
||||
}
|
||||
@ -1625,6 +1625,18 @@ function agentTurnLabel(phaseId, index) {
|
||||
if (phaseId?.includes("warm")) {
|
||||
return "warm";
|
||||
}
|
||||
if (phaseId?.includes("gateway")) {
|
||||
return "gateway-rpc";
|
||||
}
|
||||
if (phaseId?.includes("dashboard")) {
|
||||
return "dashboard-session";
|
||||
}
|
||||
if (phaseId?.includes("tui")) {
|
||||
return "tui";
|
||||
}
|
||||
if (phaseId?.includes("openai")) {
|
||||
return "openai-compatible";
|
||||
}
|
||||
return `turn-${index}`;
|
||||
}
|
||||
|
||||
@ -2779,7 +2791,10 @@ function countDiagnosticMetric(record, key) {
|
||||
|
||||
function isAgentMessageCommand(command) {
|
||||
return (command.includes(" -- agent ") && command.includes("--message")) ||
|
||||
command.includes("run-concurrent-agent-turns.mjs");
|
||||
command.includes("run-concurrent-agent-turns.mjs") ||
|
||||
command.includes("run-dashboard-session-send-turn.mjs") ||
|
||||
command.includes("run-tui-message-turn.mjs") ||
|
||||
command.includes("run-openai-compatible-turn.mjs");
|
||||
}
|
||||
|
||||
function extractAgentResponse(result) {
|
||||
|
||||
@ -632,7 +632,10 @@ async function runScenarioCommand(command, context, envName, artifactDir, phaseI
|
||||
|
||||
function isAgentMessageCommand(command) {
|
||||
return (command.includes(" -- agent ") && command.includes("--message")) ||
|
||||
command.includes("run-concurrent-agent-turns.mjs");
|
||||
command.includes("run-concurrent-agent-turns.mjs") ||
|
||||
command.includes("run-dashboard-session-send-turn.mjs") ||
|
||||
command.includes("run-tui-message-turn.mjs") ||
|
||||
command.includes("run-openai-compatible-turn.mjs");
|
||||
}
|
||||
|
||||
function agentLeakRoles() {
|
||||
|
||||
@ -182,6 +182,22 @@ export async function runSelfCheck(flags = {}) {
|
||||
throw new Error(`missing auth override should not inject auth phases: ${phaseIds.join(", ")}`);
|
||||
}
|
||||
}));
|
||||
for (const item of [
|
||||
["agent-gateway-rpc-turn", "agent-gateway-rpc-turn", "ocm @"],
|
||||
["dashboard-session-send-turn", "dashboard-session-send-turn", "run-dashboard-session-send-turn.mjs"],
|
||||
["tui-message-turn", "tui-message-turn", "run-tui-message-turn.mjs"],
|
||||
["openai-compatible-turn", "openai-compatible-turn", "run-openai-compatible-turn.mjs"]
|
||||
]) {
|
||||
const [scenarioId, surfaceId, expectedCommand] = item;
|
||||
checks.push(await jsonCommandCheck(`message-ingress-${scenarioId}-dry-run-json`, `node bin/kova.mjs run --target runtime:stable --scenario ${scenarioId} --state mock-openai-provider --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
|
||||
const record = report.records?.[0];
|
||||
assertEqual(record?.surface, surfaceId, `${scenarioId} surface`);
|
||||
assertEqual(record?.auth?.mode, "mock", `${scenarioId} mock auth mode`);
|
||||
const commands = record?.phases?.flatMap((phase) => phase.commands ?? []) ?? [];
|
||||
assertEqual(commands.some((command) => command.includes(expectedCommand)), true, `${scenarioId} ingress command`);
|
||||
}));
|
||||
}
|
||||
checks.push(await jsonCommandCheck("run-profiling-dry-run-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --node-profile --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
|
||||
assertEqual(report.records?.[0]?.profiling?.enabled, true, "profiling marker");
|
||||
@ -2985,7 +3001,7 @@ function syntheticCompareReport({ runId, target, timelineAvailable, preProviderM
|
||||
summary: { statuses: { PASS: 1 } },
|
||||
records: [{
|
||||
scenario: "agent-cold-warm-message",
|
||||
surface: "agent-message",
|
||||
surface: "agent-cli-local-turn",
|
||||
state: { id: "mock-openai-provider" },
|
||||
status: "PASS",
|
||||
measurements: {
|
||||
|
||||
@ -20,7 +20,7 @@
|
||||
"reason": "This state intentionally bypasses Kova's default mock provider auth to test missing model credentials."
|
||||
},
|
||||
"compatibleSurfaces": [
|
||||
"agent-message"
|
||||
"agent-cli-local-turn"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "agent-provider-auth",
|
||||
|
||||
@ -20,7 +20,11 @@
|
||||
"agent-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"agent-message"
|
||||
"agent-cli-local-turn",
|
||||
"agent-gateway-rpc-turn",
|
||||
"dashboard-session-send-turn",
|
||||
"tui-message-turn",
|
||||
"openai-compatible-turn"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "agent-provider-latency",
|
||||
|
||||
77
support/openclaw-runtime.mjs
Normal file
77
support/openclaw-runtime.mjs
Normal file
@ -0,0 +1,77 @@
|
||||
import { pathToFileURL } from "node:url";
|
||||
import path from "node:path";
|
||||
|
||||
export async function importOpenClawDistModule(relativePath) {
|
||||
const packageRoot = process.cwd();
|
||||
const absolutePath = path.join(packageRoot, "dist", ...relativePath.split("/"));
|
||||
try {
|
||||
return await import(pathToFileURL(absolutePath).href);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
throw new Error(
|
||||
`failed to import OpenClaw runtime module ${relativePath} from ${packageRoot}: ${message}. ` +
|
||||
"Kova user-message scenarios require a built/release-shaped OpenClaw runtime."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export function parseSupportArgs(argv) {
|
||||
const parsed = {};
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (!arg.startsWith("--")) {
|
||||
throw new Error(`unexpected argument: ${arg}`);
|
||||
}
|
||||
const key = arg.slice(2);
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith("--")) {
|
||||
throw new Error(`${arg} requires a value`);
|
||||
}
|
||||
parsed[key] = value;
|
||||
index += 1;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function readTimeoutMs(value, fallbackMs) {
|
||||
if (value === undefined) {
|
||||
return fallbackMs;
|
||||
}
|
||||
const parsed = Number(value);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error(`invalid timeout: ${value}`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
|
||||
export function extractText(value) {
|
||||
if (typeof value === "string") {
|
||||
return value;
|
||||
}
|
||||
if (!value || typeof value !== "object") {
|
||||
return "";
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(extractText).filter(Boolean).join("\n");
|
||||
}
|
||||
for (const key of ["finalAssistantVisibleText", "finalAssistantRawText", "text", "content", "reply"]) {
|
||||
if (typeof value[key] === "string") {
|
||||
return value[key];
|
||||
}
|
||||
}
|
||||
return Object.values(value).map(extractText).filter(Boolean).join("\n");
|
||||
}
|
||||
|
||||
export async function sleep(ms) {
|
||||
await new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
export function finishJson(payload) {
|
||||
process.stdout.write(`${JSON.stringify(payload, null, 2)}\n`);
|
||||
}
|
||||
|
||||
export function failJson(error, extra = {}) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
process.stdout.write(`${JSON.stringify({ ok: false, error: message, ...extra }, null, 2)}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
111
support/run-dashboard-session-send-turn.mjs
Executable file
111
support/run-dashboard-session-send-turn.mjs
Executable file
@ -0,0 +1,111 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { randomUUID } from "node:crypto";
|
||||
import {
|
||||
extractText,
|
||||
failJson,
|
||||
finishJson,
|
||||
importOpenClawDistModule,
|
||||
parseSupportArgs,
|
||||
readTimeoutMs,
|
||||
sleep
|
||||
} from "./openclaw-runtime.mjs";
|
||||
|
||||
const startedAtEpochMs = Date.now();
|
||||
|
||||
try {
|
||||
const args = parseSupportArgs(process.argv.slice(2));
|
||||
const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only.";
|
||||
const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK";
|
||||
const timeoutMs = readTimeoutMs(args.timeout, 120000);
|
||||
const sessionKey = args["session-key"] ?? `kova-dashboard-${randomUUID()}`;
|
||||
const { callGateway } = await importOpenClawDistModule("gateway/call.js");
|
||||
|
||||
const created = await callGateway({
|
||||
method: "sessions.create",
|
||||
params: {
|
||||
agentId: "main",
|
||||
key: sessionKey,
|
||||
label: "Kova Dashboard Session Send"
|
||||
},
|
||||
timeoutMs: Math.min(timeoutMs, 30000)
|
||||
});
|
||||
const canonicalKey = created?.key ?? sessionKey;
|
||||
const sendStartedAtEpochMs = Date.now();
|
||||
const sent = await callGateway({
|
||||
method: "sessions.send",
|
||||
params: {
|
||||
key: canonicalKey,
|
||||
message,
|
||||
thinking: "off",
|
||||
timeoutMs,
|
||||
idempotencyKey: `kova-dashboard-${randomUUID()}`
|
||||
},
|
||||
timeoutMs: Math.min(timeoutMs, 30000)
|
||||
});
|
||||
const runId = typeof sent?.runId === "string" ? sent.runId : null;
|
||||
|
||||
const history = await waitForAssistantText({
|
||||
callGateway,
|
||||
sessionKey: canonicalKey,
|
||||
expectedText,
|
||||
timeoutMs,
|
||||
minAssistantCount: 1
|
||||
});
|
||||
|
||||
finishJson({
|
||||
ok: true,
|
||||
surface: "dashboard-session-send-turn",
|
||||
method: "sessions.send",
|
||||
sessionKey: canonicalKey,
|
||||
runId,
|
||||
startedAtEpochMs,
|
||||
sendStartedAtEpochMs,
|
||||
finishedAtEpochMs: Date.now(),
|
||||
finalAssistantVisibleText: history.matchedAssistantText,
|
||||
finalAssistantRawText: history.lastAssistantText,
|
||||
assistantMessageCount: history.assistantTexts.length,
|
||||
expectedTextPresent: history.matchedAssistantText.includes(expectedText)
|
||||
});
|
||||
} catch (error) {
|
||||
failJson(error, { surface: "dashboard-session-send-turn", finishedAtEpochMs: Date.now() });
|
||||
}
|
||||
|
||||
async function waitForAssistantText({ callGateway, sessionKey, expectedText, timeoutMs, minAssistantCount }) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let lastAssistantText = "";
|
||||
let assistantTexts = [];
|
||||
while (Date.now() < deadline) {
|
||||
const history = await callGateway({
|
||||
method: "chat.history",
|
||||
params: { sessionKey, limit: 16 },
|
||||
timeoutMs: 15000
|
||||
});
|
||||
assistantTexts = extractAssistantTexts(history?.messages ?? []);
|
||||
lastAssistantText = assistantTexts.at(-1) ?? "";
|
||||
const matchedAssistantText = assistantTexts
|
||||
.slice(Math.max(0, minAssistantCount - 1))
|
||||
.find((text) => text.includes(expectedText));
|
||||
if (matchedAssistantText) {
|
||||
return { assistantTexts, lastAssistantText, matchedAssistantText };
|
||||
}
|
||||
await sleep(500);
|
||||
}
|
||||
throw new Error(
|
||||
`timed out waiting for dashboard assistant text ${JSON.stringify(expectedText)}; last=${JSON.stringify(lastAssistantText)}`
|
||||
);
|
||||
}
|
||||
|
||||
function extractAssistantTexts(messages) {
|
||||
if (!Array.isArray(messages)) {
|
||||
return [];
|
||||
}
|
||||
return messages
|
||||
.filter((message) => {
|
||||
const role = String(message?.role ?? message?.sender ?? message?.type ?? "").toLowerCase();
|
||||
return role.includes("assistant") || role.includes("agent");
|
||||
})
|
||||
.map((message) => extractText(message))
|
||||
.map((text) => text.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
80
support/run-openai-compatible-turn.mjs
Executable file
80
support/run-openai-compatible-turn.mjs
Executable file
@ -0,0 +1,80 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import {
|
||||
extractText,
|
||||
failJson,
|
||||
finishJson,
|
||||
importOpenClawDistModule,
|
||||
parseSupportArgs,
|
||||
readTimeoutMs
|
||||
} from "./openclaw-runtime.mjs";
|
||||
|
||||
const startedAtEpochMs = Date.now();
|
||||
|
||||
try {
|
||||
const args = parseSupportArgs(process.argv.slice(2));
|
||||
const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only.";
|
||||
const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK";
|
||||
const timeoutMs = readTimeoutMs(args.timeout, 120000);
|
||||
const model = args.model ?? "openai/gpt-5.5";
|
||||
const { getRuntimeConfig } = await importOpenClawDistModule("config/io.js");
|
||||
const { resolveGatewayPort } = await importOpenClawDistModule("config/paths.js");
|
||||
const cfg = getRuntimeConfig();
|
||||
const port = resolveGatewayPort(cfg, process.env);
|
||||
const token = readGatewayToken(cfg);
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(new Error(`OpenAI-compatible request timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
const requestStartedAtEpochMs = Date.now();
|
||||
try {
|
||||
const response = await fetch(`http://127.0.0.1:${port}/v1/chat/completions`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
"content-type": "application/json",
|
||||
...(token ? { authorization: `Bearer ${token}` } : {})
|
||||
},
|
||||
body: JSON.stringify({
|
||||
model,
|
||||
messages: [{ role: "user", content: message }],
|
||||
stream: false
|
||||
}),
|
||||
signal: controller.signal
|
||||
});
|
||||
const bodyText = await response.text();
|
||||
let body = {};
|
||||
try {
|
||||
body = bodyText ? JSON.parse(bodyText) : {};
|
||||
} catch {
|
||||
body = { raw: bodyText };
|
||||
}
|
||||
if (!response.ok) {
|
||||
throw new Error(`OpenAI-compatible HTTP ${response.status}: ${bodyText.slice(0, 1000)}`);
|
||||
}
|
||||
const finalText = extractText(body?.choices?.[0]?.message ?? body);
|
||||
finishJson({
|
||||
ok: true,
|
||||
surface: "openai-compatible-turn",
|
||||
method: "POST /v1/chat/completions",
|
||||
model,
|
||||
startedAtEpochMs,
|
||||
requestStartedAtEpochMs,
|
||||
finishedAtEpochMs: Date.now(),
|
||||
status: response.status,
|
||||
finalAssistantVisibleText: finalText,
|
||||
finalAssistantRawText: finalText,
|
||||
expectedTextPresent: finalText.includes(expectedText)
|
||||
});
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
} catch (error) {
|
||||
failJson(error, { surface: "openai-compatible-turn", finishedAtEpochMs: Date.now() });
|
||||
}
|
||||
|
||||
function readGatewayToken(cfg) {
|
||||
const candidates = [
|
||||
process.env.OPENCLAW_GATEWAY_TOKEN,
|
||||
cfg?.gateway?.auth?.token,
|
||||
cfg?.gateway?.token
|
||||
];
|
||||
return candidates.find((value) => typeof value === "string" && value.trim().length > 0)?.trim() ?? "";
|
||||
}
|
||||
106
support/run-tui-message-turn.mjs
Executable file
106
support/run-tui-message-turn.mjs
Executable file
@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import { failJson, finishJson, parseSupportArgs, readTimeoutMs } from "./openclaw-runtime.mjs";
|
||||
|
||||
const startedAtEpochMs = Date.now();
|
||||
|
||||
try {
|
||||
const args = parseSupportArgs(process.argv.slice(2));
|
||||
const envName = args.env;
|
||||
if (!envName) {
|
||||
throw new Error("--env is required");
|
||||
}
|
||||
const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only.";
|
||||
const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK";
|
||||
const timeoutMs = readTimeoutMs(args.timeout, 120000);
|
||||
const session = args.session ?? "kova-tui-message";
|
||||
const result = await runTuiTurn({ envName, message, expectedText, timeoutMs, session });
|
||||
finishJson({
|
||||
ok: true,
|
||||
surface: "tui-message-turn",
|
||||
method: "tui stdin/stdout",
|
||||
session,
|
||||
startedAtEpochMs,
|
||||
inputAcceptedAtEpochMs: result.inputAcceptedAtEpochMs,
|
||||
finishedAtEpochMs: Date.now(),
|
||||
finalAssistantVisibleText: result.finalAssistantText,
|
||||
finalAssistantRawText: result.finalAssistantText,
|
||||
expectedTextPresent: result.finalAssistantText.includes(expectedText),
|
||||
outputTail: result.outputTail
|
||||
});
|
||||
} catch (error) {
|
||||
failJson(error, { surface: "tui-message-turn", finishedAtEpochMs: Date.now() });
|
||||
}
|
||||
|
||||
function runTuiTurn({ envName, message, expectedText, timeoutMs, session }) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn("ocm", [
|
||||
`@${envName}`,
|
||||
"--",
|
||||
"tui",
|
||||
"--session",
|
||||
session,
|
||||
"--history-limit",
|
||||
"5"
|
||||
], {
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
shell: false
|
||||
});
|
||||
|
||||
let output = "";
|
||||
let inputSent = false;
|
||||
let settled = false;
|
||||
let inputAcceptedAtEpochMs = null;
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
finish(new Error(`TUI turn did not complete within ${timeoutMs}ms`));
|
||||
}, timeoutMs);
|
||||
|
||||
child.stdout.on("data", onData);
|
||||
child.stderr.on("data", onData);
|
||||
child.on("error", finish);
|
||||
child.on("exit", (code, signal) => {
|
||||
if (!settled) {
|
||||
finish(new Error(`TUI exited before message turn completed (code=${code}, signal=${signal ?? "none"})`));
|
||||
}
|
||||
});
|
||||
|
||||
function onData(chunk) {
|
||||
output += chunk.toString("utf8");
|
||||
if (!inputSent && /openclaw tui|local ready|agent\s+main|session\s+/i.test(output)) {
|
||||
inputSent = true;
|
||||
inputAcceptedAtEpochMs = Date.now();
|
||||
child.stdin.write(`${message}\n`);
|
||||
}
|
||||
if (output.includes(expectedText)) {
|
||||
finish(null, {
|
||||
inputAcceptedAtEpochMs,
|
||||
finalAssistantText: expectedText,
|
||||
outputTail: output.slice(-4000)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function finish(error, value) {
|
||||
if (settled) {
|
||||
return;
|
||||
}
|
||||
settled = true;
|
||||
clearTimeout(timer);
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGINT");
|
||||
setTimeout(() => {
|
||||
if (child.exitCode === null && child.signalCode === null) {
|
||||
child.kill("SIGKILL");
|
||||
}
|
||||
}, 1000).unref?.();
|
||||
}
|
||||
if (error) {
|
||||
reject(new Error(`${error.message}; outputTail=${JSON.stringify(output.slice(-4000))}`));
|
||||
} else {
|
||||
resolve(value);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
@ -1,8 +1,8 @@
|
||||
{
|
||||
"id": "agent-message",
|
||||
"title": "Agent Message",
|
||||
"id": "agent-cli-local-turn",
|
||||
"title": "Agent CLI Local Turn",
|
||||
"ownerArea": "agent-runtime",
|
||||
"description": "Send cold, warm, and repeated local OpenClaw agent messages and verify response latency, provider routing, gateway health, memory, and logs.",
|
||||
"description": "Send cold, warm, repeated, and failure-mode messages through `openclaw agent --local`, then verify response latency, provider routing, gateway health, memory, and process containment.",
|
||||
"requiredStates": ["mock-openai-provider"],
|
||||
"targetKinds": ["npm", "channel", "runtime", "local-build"],
|
||||
"requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "agentTurnMaxMs", "coldAgentTurnMs", "warmAgentTurnMs", "agentColdWarmDeltaMs", "coldPreProviderMs", "warmPreProviderMs", "agentPreProviderP95Ms", "agentCleanupMaxMs", "healthP95Ms", "peakRssMb", "providerTimeoutMentions", "pluginLoadFailures"],
|
||||
20
surfaces/agent-gateway-rpc-turn.json
Normal file
20
surfaces/agent-gateway-rpc-turn.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "agent-gateway-rpc-turn",
|
||||
"title": "Agent Gateway RPC Turn",
|
||||
"ownerArea": "gateway-agent-runtime",
|
||||
"description": "Send messages through `openclaw agent` without `--local`, forcing the CLI to cross the Gateway agent RPC boundary before the agent turn runs.",
|
||||
"requiredStates": ["mock-openai-provider"],
|
||||
"targetKinds": ["npm", "channel", "runtime", "local-build"],
|
||||
"requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "agentTurnMaxMs", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"],
|
||||
"processRoles": ["gateway", "command-tree", "agent-cli", "agent-process", "mock-provider"],
|
||||
"thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 },
|
||||
"roleThresholds": {
|
||||
"gateway": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"agent-cli": { "peakRssMb": 900, "maxCpuPercent": 300 },
|
||||
"mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 }
|
||||
},
|
||||
"diagnostics": {
|
||||
"timelineRequiredForSourceBuild": true,
|
||||
"expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"]
|
||||
}
|
||||
}
|
||||
20
surfaces/dashboard-session-send-turn.json
Normal file
20
surfaces/dashboard-session-send-turn.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "dashboard-session-send-turn",
|
||||
"title": "Dashboard Session Send Turn",
|
||||
"ownerArea": "gateway-chat-session-runtime",
|
||||
"description": "Create a dashboard session, call Gateway `sessions.send`, and wait for the assistant response in chat history to validate the same path dashboard users exercise.",
|
||||
"requiredStates": ["mock-openai-provider"],
|
||||
"targetKinds": ["npm", "channel", "runtime", "local-build"],
|
||||
"requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"],
|
||||
"processRoles": ["gateway", "command-tree", "dashboard-cli", "agent-process", "mock-provider"],
|
||||
"thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 },
|
||||
"roleThresholds": {
|
||||
"gateway": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"dashboard-cli": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 }
|
||||
},
|
||||
"diagnostics": {
|
||||
"timelineRequiredForSourceBuild": true,
|
||||
"expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"]
|
||||
}
|
||||
}
|
||||
20
surfaces/openai-compatible-turn.json
Normal file
20
surfaces/openai-compatible-turn.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "openai-compatible-turn",
|
||||
"title": "OpenAI-Compatible Turn",
|
||||
"ownerArea": "gateway-openai-compatible-runtime",
|
||||
"description": "POST a user message to the OpenAI-compatible chat completions endpoint and verify the final response, provider timing, auth behavior, and gateway health.",
|
||||
"requiredStates": ["mock-openai-provider"],
|
||||
"targetKinds": ["npm", "channel", "runtime", "local-build"],
|
||||
"requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"],
|
||||
"processRoles": ["gateway", "command-tree", "openai-compatible-client", "agent-process", "mock-provider"],
|
||||
"thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 },
|
||||
"roleThresholds": {
|
||||
"gateway": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"openai-compatible-client": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 }
|
||||
},
|
||||
"diagnostics": {
|
||||
"timelineRequiredForSourceBuild": true,
|
||||
"expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"]
|
||||
}
|
||||
}
|
||||
20
surfaces/tui-message-turn.json
Normal file
20
surfaces/tui-message-turn.json
Normal file
@ -0,0 +1,20 @@
|
||||
{
|
||||
"id": "tui-message-turn",
|
||||
"title": "TUI Message Turn",
|
||||
"ownerArea": "tui-agent-runtime",
|
||||
"description": "Launch the OpenClaw TUI, send a real stdin message, and require visible assistant output so Kova can catch TUI input freezes and delayed user-visible responses.",
|
||||
"requiredStates": ["mock-openai-provider"],
|
||||
"targetKinds": ["npm", "channel", "runtime", "local-build"],
|
||||
"requiredMetrics": ["agentTurnMs", "agentTurnP95Ms", "coldPreProviderMs", "healthP95Ms", "peakRssMb", "pluginLoadFailures"],
|
||||
"processRoles": ["gateway", "command-tree", "tui-cli", "agent-process", "mock-provider"],
|
||||
"thresholds": { "agentTurnMs": 45000, "preProviderMs": 10000, "providerFinalMs": 3000, "healthP95Ms": 1000, "peakRssMb": 900 },
|
||||
"roleThresholds": {
|
||||
"gateway": { "peakRssMb": 700, "maxCpuPercent": 250 },
|
||||
"tui-cli": { "peakRssMb": 900, "maxCpuPercent": 300 },
|
||||
"mock-provider": { "peakRssMb": 250, "maxCpuPercent": 150 }
|
||||
},
|
||||
"diagnostics": {
|
||||
"timelineRequiredForSourceBuild": true,
|
||||
"expectedSpans": ["agent.turn", "agent.prepare", "models.catalog", "provider.request", "agent.cleanup"]
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user