diff --git a/metrics/known.json b/metrics/known.json index 2d13125..ee0e6d6 100644 --- a/metrics/known.json +++ b/metrics/known.json @@ -51,6 +51,11 @@ "mcpShutdownMs", "mcpToolCountMin", "mcpToolsListMs", + "networkCommandTimedOut", + "networkFailureObserved", + "networkGatewayStatusWorks", + "networkStatusAfterFailureMs", + "networkTurnMs", "openclawSlowestSpanMs", "openclawTimelineParseErrors", "peakRssMb", diff --git a/process-roles/agent-cli.json b/process-roles/agent-cli.json index 35b23d4..fd211fb 100644 --- a/process-roles/agent-cli.json +++ b/process-roles/agent-cli.json @@ -3,5 +3,5 @@ "title": "Agent CLI", "description": "User-facing OpenClaw agent command invocations that send messages or manage local sessions.", "commandPatterns": ["agent --local", "agent --session-id"], - "processPatterns": ["(^|\\s|/)openclaw\\s+.*\\bagent\\b"] + "processPatterns": ["(^|\\s|/)openclaw\\s+.*\\bagent\\b", "(^|\\s|/)openclaw-agent(\\s|$)"] } diff --git a/process-roles/agent-process.json b/process-roles/agent-process.json index 5ccca7b..b665769 100644 --- a/process-roles/agent-process.json +++ b/process-roles/agent-process.json @@ -4,6 +4,7 @@ "description": "Longer-running agent execution work spawned by OpenClaw while processing a turn.", "commandPatterns": ["(^|\\s)agent(\\s|$)"], "processPatterns": [ + "(^|\\s|/)openclaw-agent(\\s|$)", "(^|\\s|/)openclaw\\s+.*\\bsession\\b", "(^|\\s|/)openclaw\\s+.*\\bagent\\b" ] diff --git a/profiles/exhaustive.json b/profiles/exhaustive.json index bda3fa4..a4ba889 100644 --- a/profiles/exhaustive.json +++ b/profiles/exhaustive.json @@ -39,6 +39,7 @@ { "scenario": "mcp-runtime-start-stop", "state": "fresh" }, { "scenario": "browser-automation-smoke", "state": "fresh", "timeoutMs": 180000 }, { "scenario": "media-understanding-timeout", "state": "fresh", "timeoutMs": 180000 }, + { "scenario": "agent-network-offline", "state": "fresh", "timeoutMs": 180000 }, { "scenario": "gateway-performance", "state": "many-bundled-plugins" }, { "scenario": "gateway-performance", "state": "gateway-already-running" }, { "scenario": "gateway-performance", "state": "stale-service-state" }, diff --git a/profiles/release.json b/profiles/release.json index 3d7bce6..f4f3d61 100644 --- a/profiles/release.json +++ b/profiles/release.json @@ -114,6 +114,14 @@ "statusMs": 10000, "pluginLoadFailures": 0 } + }, + "network-offline": { + "thresholds": { + "networkFailureObserved": 1, + "networkStatusAfterFailureMs": 10000, + "statusMs": 10000, + "pluginLoadFailures": 0 + } } } }, @@ -163,6 +171,7 @@ "mcp-runtime:fresh", "browser-automation:fresh", "media-understanding:fresh", + "network-offline:fresh", "cross-platform-smoke:slow-filesystem" ] }, @@ -185,7 +194,7 @@ "tui", "gateway-performance" ], - "warning": ["failure-containment", "soak", "workspace-scan", "mcp-runtime", "browser-automation", "media-understanding", "cross-platform-smoke"] + "warning": ["failure-containment", "soak", "workspace-scan", "mcp-runtime", "browser-automation", "media-understanding", "network-offline", "cross-platform-smoke"] }, "scenarios": { "blocking": [ @@ -205,7 +214,8 @@ "workspace-scan-pressure", "mcp-runtime-start-stop", "browser-automation-smoke", - "media-understanding-timeout" + "media-understanding-timeout", + "agent-network-offline" ] } }, @@ -246,6 +256,7 @@ { "scenario": "mcp-runtime-start-stop", "state": "fresh" }, { "scenario": "browser-automation-smoke", "state": "fresh", "timeoutMs": 180000 }, { "scenario": "media-understanding-timeout", "state": "fresh", "timeoutMs": 180000 }, + { "scenario": "agent-network-offline", "state": "fresh", "timeoutMs": 180000 }, { "scenario": "cross-platform-smoke", "state": "slow-filesystem" } ] }, @@ -402,6 +413,11 @@ "state": "fresh", "timeoutMs": 180000 }, + { + "scenario": "agent-network-offline", + "state": "fresh", + "timeoutMs": 180000 + }, { "scenario": "cross-platform-smoke", "state": "slow-filesystem" diff --git a/scenarios/agent-network-offline.json b/scenarios/agent-network-offline.json new file mode 100644 index 0000000..a1d3767 --- /dev/null +++ b/scenarios/agent-network-offline.json @@ -0,0 +1,43 @@ +{ + "id": "agent-network-offline", + "surface": "network-offline", + "title": "Agent Network Offline", + "objective": "Run a local OpenClaw agent turn against an unreachable provider endpoint and prove OpenClaw fails clearly without taking down the gateway.", + "tags": ["agent", "message", "provider", "network", "offline", "containment"], + "timeoutMs": 180000, + "auth": { "mode": "none" }, + "thresholds": { + "gatewayReadyMs": 30000, + "networkFailureObserved": 1, + "networkStatusAfterFailureMs": 10000, + "statusMs": 10000, + "peakRssMb": 900, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "provision", + "title": "Provision Network Offline Env", + "intent": "Start a disposable OpenClaw gateway before configuring an unreachable provider endpoint.", + "commands": ["ocm start {env} {startSelector} --json"], + "evidence": ["gateway port", "runtime binding", "startup readiness"] + }, + { + "id": "network-offline-turn", + "title": "Network Offline Agent Turn", + "intent": "Send a simple local agent message that must fail because the configured provider endpoint is unreachable.", + "commands": [ + "node {kovaRoot}/support/agent-network-offline.mjs --env {env} --artifact-dir {artifactDir} --timeout-seconds 20 --max-command-ms 45000" + ], + "evidence": ["bounded network failure", "gateway status after failure", "role resource samples"] + }, + { + "id": "post-network-health", + "title": "Post-Network Gateway Health", + "intent": "Verify the gateway remains responsive and collect logs after the network failure path.", + "commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"], + "evidence": ["gateway status", "network/provider failure logs", "plugin errors", "memory after network failure"] + } + ] +} diff --git a/src/evaluator.mjs b/src/evaluator.mjs index 6caec2c..e241e69 100644 --- a/src/evaluator.mjs +++ b/src/evaluator.mjs @@ -80,6 +80,7 @@ export function evaluateRecord(record, scenario, options = {}) { const mcpBridgeEvidence = collectMcpBridgeEvidence(allResults); const browserAutomationEvidence = collectBrowserAutomationEvidence(allResults); const mediaUnderstandingEvidence = collectMediaUnderstandingEvidence(allResults); + const networkOfflineEvidence = collectNetworkOfflineEvidence(allResults); const listeningFailures = countListeningFailures(record); const tcpConnectMaxMs = collectTcpConnectMax(record); const timeToListeningMs = collectTimeToListening(record); @@ -391,6 +392,51 @@ export function evaluateRecord(record, scenario, options = {}) { } } + if (networkOfflineEvidence.available) { + checkEvidenceThreshold(violations, "network-offline", "networkTurnMs", networkOfflineEvidence.networkTurnMs, thresholds.networkTurnMs, "Network offline agent turn"); + checkEvidenceThreshold(violations, "network-offline", "networkStatusAfterFailureMs", networkOfflineEvidence.networkStatusAfterFailureMs, thresholds.networkStatusAfterFailureMs, "Post-network status"); + + if (typeof thresholds.networkFailureObserved === "number" && networkOfflineEvidence.networkFailureObserved !== true) { + violations.push({ + kind: "network-offline", + metric: "networkFailureObserved", + expected: true, + actual: networkOfflineEvidence.networkFailureObserved, + message: "Network/provider failure was not observed as a bounded command failure" + }); + } + + if (networkOfflineEvidence.networkCommandTimedOut === true) { + violations.push({ + kind: "network-offline", + metric: "networkCommandTimedOut", + expected: false, + actual: true, + message: "Network offline command hit Kova's outer timeout instead of OpenClaw surfacing the provider failure" + }); + } + + if (networkOfflineEvidence.gatewayStatusWorks === false) { + violations.push({ + kind: "network-offline", + metric: "networkGatewayStatusWorks", + expected: true, + actual: false, + message: "Gateway status did not work after network/provider failure" + }); + } + + if (networkOfflineEvidence.errors.length > 0) { + violations.push({ + kind: "network-offline", + metric: "networkOfflineErrors", + expected: "0", + actual: networkOfflineEvidence.errors.length, + message: `Network offline smoke reported ${networkOfflineEvidence.errors.length} error(s): ${networkOfflineEvidence.errors[0]}` + }); + } + } + if (typeof thresholds.providerRequestCountMin === "number") { const requestCount = record.providerEvidence?.requestCount ?? 0; if (requestCount < thresholds.providerRequestCountMin) { @@ -697,6 +743,13 @@ export function evaluateRecord(record, scenario, options = {}) { mediaStatusAfterTimeoutMs: mediaUnderstandingEvidence.mediaStatusAfterTimeoutMs, mediaGatewayStatusWorks: mediaUnderstandingEvidence.gatewayStatusWorks, mediaErrors: mediaUnderstandingEvidence.errors, + networkOfflineEvidence, + networkTurnMs: networkOfflineEvidence.networkTurnMs, + networkFailureObserved: networkOfflineEvidence.networkFailureObserved, + networkCommandTimedOut: networkOfflineEvidence.networkCommandTimedOut, + networkStatusAfterFailureMs: networkOfflineEvidence.networkStatusAfterFailureMs, + networkGatewayStatusWorks: networkOfflineEvidence.gatewayStatusWorks, + networkErrors: networkOfflineEvidence.errors, soakDurationMs: soakEvidence.durationMs, soakIterations: soakEvidence.iterations, soakCommandP95Ms: soakEvidence.commandP95Ms, @@ -1986,6 +2039,62 @@ function parseMediaUnderstandingTimeoutOutput(result) { } } +function collectNetworkOfflineEvidence(results) { + const smokes = results + .filter((result) => result.command?.includes("agent-network-offline.mjs")) + .map((result) => parseNetworkOfflineOutput(result)) + .filter(Boolean); + + if (smokes.length === 0) { + return { + schemaVersion: "kova.networkOfflineEvidence.v1", + available: false, + networkTurnMs: null, + networkFailureObserved: null, + networkCommandTimedOut: null, + networkStatusAfterFailureMs: null, + gatewayStatusWorks: null, + errors: [], + smokes: [] + }; + } + + return { + schemaVersion: "kova.networkOfflineEvidence.v1", + available: true, + networkTurnMs: maxNullable(...smokes.map((smoke) => smoke.networkTurnMs)), + networkFailureObserved: smokes.every((smoke) => smoke.networkFailureObserved === true), + networkCommandTimedOut: smokes.some((smoke) => smoke.networkCommandTimedOut === true), + networkStatusAfterFailureMs: maxNullable(...smokes.map((smoke) => smoke.networkStatusAfterFailureMs)), + gatewayStatusWorks: smokes.every((smoke) => smoke.gatewayStatusWorks === true), + errors: smokes.flatMap((smoke) => smoke.errors ?? []), + smokes: smokes.map((smoke) => ({ + durationMs: smoke.durationMs ?? null, + networkTurnMs: smoke.networkTurnMs ?? null, + networkFailureObserved: smoke.networkFailureObserved ?? null, + networkCommandTimedOut: smoke.networkCommandTimedOut ?? null, + networkCommandStatus: smoke.networkCommandStatus ?? null, + networkStatusAfterFailureMs: smoke.networkStatusAfterFailureMs ?? null, + gatewayStatusWorks: smoke.gatewayStatusWorks ?? null, + errors: smoke.errors ?? [] + })) + }; +} + +function parseNetworkOfflineOutput(result) { + const text = result.stdout ?? ""; + const jsonStart = text.indexOf("{"); + if (jsonStart < 0) { + return null; + } + try { + const parsed = JSON.parse(text.slice(jsonStart)); + return parsed?.schemaVersion === "kova.agentNetworkOffline.v1" ? parsed : null; + } catch { + return null; + } +} + function checkEvidenceThreshold(violations, kind, metric, actual, threshold, label) { if (typeof threshold !== "number" || actual === null) { return; diff --git a/src/report.mjs b/src/report.mjs index 557675b..02b6da9 100644 --- a/src/report.mjs +++ b/src/report.mjs @@ -155,6 +155,9 @@ export function renderMarkdownReport(report) { if (record.measurements.mediaUnderstandingEvidence?.available) { lines.push(`- Media understanding: describe ${record.measurements.mediaDescribeMs ?? "unknown"} ms; timeout observed ${record.measurements.mediaTimeoutObserved ?? "unknown"}; command outer timeout ${record.measurements.mediaCommandTimedOut ?? "unknown"}; status after timeout ${record.measurements.mediaStatusAfterTimeoutMs ?? "unknown"} ms; gateway status ${record.measurements.mediaGatewayStatusWorks ?? "unknown"}`); } + if (record.measurements.networkOfflineEvidence?.available) { + lines.push(`- Network offline: turn ${record.measurements.networkTurnMs ?? "unknown"} ms; failure observed ${record.measurements.networkFailureObserved ?? "unknown"}; command outer timeout ${record.measurements.networkCommandTimedOut ?? "unknown"}; status after failure ${record.measurements.networkStatusAfterFailureMs ?? "unknown"} ms; gateway status ${record.measurements.networkGatewayStatusWorks ?? "unknown"}`); + } lines.push(`- Provider/model timing: ${record.measurements.providerModelTimingMs ?? "unknown"} ms`); lines.push(`- Agent turn: ${record.measurements.agentTurnMs ?? "unknown"} ms (${record.measurements.agentResponseOk ?? "not-run"})`); if (record.measurements.agentTurnCount > 0) { @@ -711,6 +714,12 @@ function summarizeMeasurements(measurements) { mediaCommandTimedOut: measurements.mediaCommandTimedOut ?? null, mediaStatusAfterTimeoutMs: measurements.mediaStatusAfterTimeoutMs ?? null, mediaGatewayStatusWorks: measurements.mediaGatewayStatusWorks ?? null, + networkOfflineEvidence: measurements.networkOfflineEvidence ?? null, + networkTurnMs: measurements.networkTurnMs ?? null, + networkFailureObserved: measurements.networkFailureObserved ?? null, + networkCommandTimedOut: measurements.networkCommandTimedOut ?? null, + networkStatusAfterFailureMs: measurements.networkStatusAfterFailureMs ?? null, + networkGatewayStatusWorks: measurements.networkGatewayStatusWorks ?? null, resourceTrend: measurements.resourceTrend ?? null, profilingEnabled: measurements.profilingEnabled ?? null, profilingResourceInterpretation: measurements.profilingResourceInterpretation ?? null, @@ -864,6 +873,12 @@ export function renderPasteSummary(report) { } else if (record.violations?.length > 0) { if (record.measurements) { lines.push(`Measurements: agent turn ${record.measurements.agentTurnMs ?? "not-run"}ms; cold/warm ${record.measurements.coldAgentTurnMs ?? "unknown"}/${record.measurements.warmAgentTurnMs ?? "unknown"}ms; cold-warm delta ${record.measurements.agentColdWarmDeltaMs ?? "unknown"}ms; pre-provider ${record.measurements.agentPreProviderMs ?? "unknown"}ms; provider work ${record.measurements.agentProviderFinalMs ?? "unknown"}ms; cleanup max ${record.measurements.agentCleanupMaxMs ?? "unknown"}ms; diagnosis ${record.measurements.agentLatencyDiagnosis?.kind ?? "unknown"}; cleanup diagnosis ${record.measurements.agentCleanupDiagnosis?.kind ?? "none"}; provider simulation ${record.measurements.agentProviderMode ?? "normal"}/${record.measurements.agentProviderIssue ?? "none"} containment ${record.measurements.agentProviderContainmentOk ?? "n/a"} recovery ${record.measurements.agentProviderRecoveryOk ?? "n/a"}; agent process leaks ${record.measurements.agentProcessLeakCount ?? "unknown"}.`); + if (record.measurements.mediaUnderstandingEvidence?.available) { + lines.push(`Media: describe ${record.measurements.mediaDescribeMs ?? "unknown"}ms; timeout ${record.measurements.mediaTimeoutObserved ?? "unknown"}; status ${record.measurements.mediaStatusAfterTimeoutMs ?? "unknown"}ms.`); + } + if (record.measurements.networkOfflineEvidence?.available) { + lines.push(`Network offline: turn ${record.measurements.networkTurnMs ?? "unknown"}ms; failure ${record.measurements.networkFailureObserved ?? "unknown"}; status ${record.measurements.networkStatusAfterFailureMs ?? "unknown"}ms.`); + } } lines.push("Violations:"); for (const violation of record.violations) { diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index 1788a92..93d8c55 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -215,6 +215,21 @@ export async function runSelfCheck(flags = {}) { assertEqual(record?.thresholds?.mediaTimeoutObserved, 1, "media timeout threshold"); assertEqual(record?.thresholds?.providerRequestCountMin, 1, "media provider request threshold"); })); + checks.push(await jsonCommandCheck("network-offline-dry-run-json", `node bin/kova.mjs run --target runtime:stable --scenario agent-network-offline --state fresh --report-dir ${quoteShell(tmp)} --json`, async (data) => { + const report = JSON.parse(await readFile(data.jsonPath, "utf8")); + const record = report.records?.[0]; + assertEqual(record?.surface, "network-offline", "network offline surface"); + assertEqual(record?.auth?.mode, "none", "network offline opts out of default mock auth"); + const phaseIds = record?.phases?.map((phase) => phase.id) ?? []; + if (phaseIds.includes("auth-prepare") || phaseIds.includes("auth-setup")) { + throw new Error(`network offline must not start mock auth phases: ${phaseIds.join(", ")}`); + } + const commands = record?.phases?.flatMap((phase) => phase.commands ?? []) ?? []; + const networkCommand = commands.find((command) => command.includes("agent-network-offline.mjs")) ?? ""; + assertEqual(networkCommand.includes("--artifact-dir '"), true, "network helper receives quoted artifact dir"); + assertEqual(networkCommand.includes("--max-command-ms 45000"), true, "network helper allows cold CLI evidence before outer timeout"); + assertEqual(record?.thresholds?.networkFailureObserved, 1, "network failure threshold"); + })); checks.push(await jsonCommandCheck("diagnostic-profile-plan-json", "node bin/kova.mjs matrix plan --profile diagnostic --target local-build:/tmp/openclaw --include scenario:release-runtime-startup --json", (data) => { assertEqual(data.schemaVersion, "kova.matrix.plan.v1", "diagnostic matrix plan schema"); assertEqual(data.profile?.id, "diagnostic", "diagnostic profile id"); @@ -292,6 +307,7 @@ export async function runSelfCheck(flags = {}) { checks.push(mcpBridgeEvidenceEvaluationCheck()); checks.push(browserAutomationEvidenceEvaluationCheck()); checks.push(mediaUnderstandingEvidenceEvaluationCheck()); + checks.push(networkOfflineEvidenceEvaluationCheck()); checks.push(await jsonCommandCheck( "dry-run-state-lifecycle-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --state missing-plugin-index --report-dir ${quoteShell(tmp)} --json`, @@ -2567,6 +2583,111 @@ function mediaUnderstandingEvidenceEvaluationCheck() { } } +function networkOfflineEvidenceEvaluationCheck() { + try { + const smoke = { + schemaVersion: "kova.agentNetworkOffline.v1", + ok: true, + durationMs: 1800, + networkTurnMs: 1400, + networkFailureObserved: true, + networkCommandTimedOut: false, + networkCommandStatus: 1, + networkStatusAfterFailureMs: 190, + gatewayStatusWorks: true, + errors: [] + }; + const record = { + scenario: "agent-network-offline", + status: "PASS", + phases: [{ + id: "network-offline-turn", + results: [{ + command: "node support/agent-network-offline.mjs --env kova-self-check --artifact-dir /tmp/kova", + status: 0, + timedOut: false, + durationMs: 1800, + stdout: JSON.stringify(smoke), + stderr: "" + }], + metrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() } + }], + finalMetrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() } + }; + evaluateRecord(record, { + id: "agent-network-offline", + thresholds: { + networkFailureObserved: 1, + networkStatusAfterFailureMs: 10000 + } + }, { surface: { thresholds: {} }, targetPlan: { kind: "npm" } }); + + assertEqual(record.status, "PASS", "network offline record status"); + assertEqual(record.measurements.networkTurnMs, 1400, "network turn ms"); + assertEqual(record.measurements.networkFailureObserved, true, "network failure observed"); + assertEqual(record.measurements.networkCommandTimedOut, false, "network command did not hit outer timeout"); + assertEqual(record.measurements.networkStatusAfterFailureMs, 190, "post-network status ms"); + assertEqual(record.measurements.networkGatewayStatusWorks, true, "gateway status after network failure"); + + const failed = { + ...record, + status: "PASS", + violations: [], + measurements: undefined, + phases: [{ + id: "network-offline-turn", + results: [{ + command: "node support/agent-network-offline.mjs --env kova-self-check --artifact-dir /tmp/kova", + status: 0, + timedOut: false, + durationMs: 1800, + stdout: JSON.stringify({ + ...smoke, + ok: false, + networkFailureObserved: false, + gatewayStatusWorks: false, + errors: ["network failure not observed"] + }), + stderr: "" + }], + metrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() } + }] + }; + evaluateRecord(failed, { + id: "agent-network-offline", + thresholds: { + networkFailureObserved: 1 + } + }, { surface: { thresholds: {} }, targetPlan: { kind: "npm" } }); + assertEqual(failed.status, "FAIL", "network failure status"); + assertEqual( + failed.violations.some((violation) => violation.metric === "networkFailureObserved"), + true, + "network failure observed violation" + ); + assertEqual( + failed.violations.some((violation) => violation.metric === "networkGatewayStatusWorks"), + true, + "network gateway status violation" + ); + + return { + id: "network-offline-evidence-evaluation", + status: "PASS", + command: "evaluate synthetic network offline evidence", + durationMs: 0 + }; + } catch (error) { + return { + id: "network-offline-evidence-evaluation", + status: "FAIL", + command: "evaluate synthetic network offline evidence", + durationMs: 0, + message: error.message + }; + } +} + function agentColdWarmEvaluationCheck() { try { const coldCommand = "ocm @kova -- agent --local --agent main --session-id kova-agent-cold-warm --message hi --json"; @@ -3339,6 +3460,14 @@ async function resourceRolePollutionCheck() { existingRoles: ["command-tree"] } ); + const openclawAgentRoles = classifyRegistryRolesForProcess( + { command: "openclaw-agent" }, + { + processRoles, + rootCommand: "ocm @kova -- agent --local --message hi", + existingRoles: ["command-tree"] + } + ); assertEqual(mockProviderRoles.includes("mock-provider"), true, "mock provider helper remains classified"); assertEqual(mockProviderRoles.includes("agent-cli"), false, "KOVA_AGENT_OK marker must not imply agent-cli"); @@ -3346,6 +3475,8 @@ async function resourceRolePollutionCheck() { assertEqual(mockProviderRoles.includes("browser-sidecar"), false, "browser env name must not imply browser-sidecar"); assertEqual(envNameRoles.includes("runtime-management"), false, "mcp-runtime env name must not imply runtime-management"); assertEqual(envNameRoles.includes("model-cli"), false, "configure-openclaw fixture helper must not imply model-cli"); + assertEqual(openclawAgentRoles.includes("agent-cli"), true, "openclaw-agent process must imply agent-cli"); + assertEqual(openclawAgentRoles.includes("agent-process"), true, "openclaw-agent process must imply agent-process"); return { id: "resource-role-pollution-boundary", status: "PASS", diff --git a/states/fresh.json b/states/fresh.json index b580afa..dd7eca6 100644 --- a/states/fresh.json +++ b/states/fresh.json @@ -24,7 +24,8 @@ "tui", "mcp-runtime", "browser-automation", - "media-understanding" + "media-understanding", + "network-offline" ], "incompatibleSurfaces": [ "upgrade-existing-user" diff --git a/support/agent-network-offline.mjs b/support/agent-network-offline.mjs new file mode 100644 index 0000000..31daeaa --- /dev/null +++ b/support/agent-network-offline.mjs @@ -0,0 +1,213 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +const options = parseArgs(process.argv.slice(2)); +const envName = requiredString(options.env, "--env"); +const artifactDir = requiredString(options.artifactDir, "--artifact-dir"); +const timeoutSeconds = positiveInteger(options.timeoutSeconds ?? "20", "--timeout-seconds"); +const maxCommandMs = positiveInteger(options.maxCommandMs ?? "45000", "--max-command-ms"); +const networkFailurePattern = /ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|fetch failed|network|connection refused|provider.*fail|failed to fetch|socket/i; + +if (!/^kova-[a-z0-9][a-z0-9-]*$/i.test(envName)) { + failUsage(`refusing to run network offline smoke against non-Kova env: ${JSON.stringify(envName)}`); +} + +await fs.mkdir(artifactDir, { recursive: true }); +const networkDir = path.join(artifactDir, "network-offline"); +await fs.mkdir(networkDir, { recursive: true }); + +const configure = await run("ocm", [ + "env", + "exec", + envName, + "--", + "node", + supportPath("configure-openclaw-provider-blackhole.mjs") +], { timeoutMs: 15000 }); + +const startedAtEpochMs = Date.now(); +const startedAt = new Date(startedAtEpochMs).toISOString(); +const agent = await run("ocm", [ + `@${envName}`, + "--", + "agent", + "--local", + "--agent", + "main", + "--session-id", + "kova-agent-network-offline", + "--message", + "Reply with exact ASCII text KOVA_AGENT_OK only.", + "--thinking", + "off", + "--timeout", + String(timeoutSeconds), + "--json" +], { timeoutMs: maxCommandMs + 3000 }); +const finishedAtEpochMs = Date.now(); + +const status = await run("ocm", [`@${envName}`, "--", "status"], { timeoutMs: 15000 }); +const combinedOutput = `${agent.stdout}\n${agent.stderr}`; +const networkFailureObserved = agent.status !== 0 && + networkFailurePattern.test(combinedOutput); +const statusWorks = status.status === 0 && status.timedOut !== true; +const ok = configure.status === 0 && networkFailureObserved && agent.timedOut !== true && statusWorks; +const summary = { + schemaVersion: "kova.agentNetworkOffline.v1", + ok, + env: envName, + timeoutSeconds, + maxCommandMs, + startedAt, + startedAtEpochMs, + finishedAt: new Date(finishedAtEpochMs).toISOString(), + finishedAtEpochMs, + durationMs: finishedAtEpochMs - startedAtEpochMs, + networkTurnMs: agent.durationMs, + networkFailureObserved, + networkCommandTimedOut: agent.timedOut === true, + networkCommandStatus: agent.status, + networkStatusAfterFailureMs: status.durationMs, + gatewayStatusWorks: statusWorks, + configureStatus: configure.status, + errors: [ + ...(configure.status === 0 ? [] : [`configure failed: ${snippet(configure.stderr || configure.stdout)}`]), + ...(networkFailureObserved ? [] : [`network failure not observed: status=${agent.status} timedOut=${agent.timedOut} duration=${agent.durationMs} stderr=${snippet(agent.stderr)}`]), + ...(agent.timedOut ? [`network failure command did not exit before ${maxCommandMs}ms; stderr=${snippet(agent.stderr)}`] : []), + ...(statusWorks ? [] : [`status after network failure failed: status=${status.status} stderr=${snippet(status.stderr || status.stdout)}`]) + ], + commands: { + configure: compactCommand(configure), + agent: compactCommand(agent), + status: compactCommand(status) + } +}; + +await fs.writeFile(path.join(networkDir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8"); +process.stdout.write(`${JSON.stringify(summary)}\n`); +process.exit(ok ? 0 : 1); + +function run(command, args, options = {}) { + const started = Date.now(); + return new Promise((resolve) => { + const child = spawn(command, args, { + env: process.env, + shell: false, + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + setTimeout(() => child.kill("SIGKILL"), 3000).unref(); + }, options.timeoutMs ?? 30000); + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => { + clearTimeout(timer); + resolve({ + command: [command, ...args].join(" "), + status: 127, + signal: null, + timedOut, + durationMs: Date.now() - started, + stdout: "", + stderr: error.message + }); + }); + child.on("close", (status, signal) => { + clearTimeout(timer); + resolve({ + command: [command, ...args].join(" "), + status: timedOut ? 124 : (status ?? 1), + signal, + timedOut, + durationMs: Date.now() - started, + stdout: truncate(stdout), + stderr: truncate(stderr) + }); + }); + }); +} + +function compactCommand(result) { + return { + command: result.command, + status: result.status, + signal: result.signal, + timedOut: result.timedOut, + durationMs: result.durationMs, + stdout: truncate(result.stdout, 1200), + stderr: truncate(result.stderr, 1200) + }; +} + +function supportPath(file) { + return path.join(path.dirname(fileURLToPath(import.meta.url)), file); +} + +function parseArgs(args) { + const parsed = {}; + for (let index = 0; index < args.length; index += 1) { + const arg = args[index]; + if (!arg.startsWith("--")) { + failUsage(`unexpected positional argument ${JSON.stringify(arg)}`); + } + const key = arg.slice(2).replaceAll("-", "_"); + const value = args[index + 1]; + if (value === undefined || value.startsWith("--")) { + failUsage(`${arg} requires a value`); + } + parsed[key] = value; + index += 1; + } + return { + env: parsed.env, + artifactDir: parsed.artifact_dir, + timeoutSeconds: parsed.timeout_seconds, + maxCommandMs: parsed.max_command_ms + }; +} + +function requiredString(value, flag) { + if (typeof value !== "string" || value.length === 0) { + failUsage(`${flag} is required`); + } + return value; +} + +function positiveInteger(value, flag) { + const number = Number(value); + if (!Number.isInteger(number) || number <= 0) { + failUsage(`${flag} must be a positive integer`); + } + return number; +} + +function truncate(value, limit = 4000) { + const text = String(value ?? ""); + if (text.length <= limit) { + return text; + } + return `${text.slice(0, limit)}\n[truncated ${text.length - limit} chars]`; +} + +function snippet(value) { + return truncate(String(value ?? "").replace(/\s+/g, " ").trim(), 500); +} + +function failUsage(message) { + process.stderr.write(`${message}\n`); + process.stderr.write("usage: agent-network-offline.mjs --env --artifact-dir [--timeout-seconds ] [--max-command-ms ]\n"); + process.exit(2); +} diff --git a/support/configure-openclaw-provider-blackhole.mjs b/support/configure-openclaw-provider-blackhole.mjs new file mode 100644 index 0000000..8c274a1 --- /dev/null +++ b/support/configure-openclaw-provider-blackhole.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node + +import fs from "node:fs"; +import net from "node:net"; +import path from "node:path"; + +const port = await reserveClosedPort(); +const stateDir = process.env.OPENCLAW_STATE_DIR || path.join(requiredEnv("OPENCLAW_HOME"), ".openclaw"); +const configPath = process.env.OPENCLAW_CONFIG_PATH || path.join(stateDir, "openclaw.json"); +fs.mkdirSync(path.dirname(configPath), { recursive: true }); +fs.mkdirSync(stateDir, { recursive: true }); +fs.writeFileSync(path.join(stateDir, ".env"), "OPENAI_API_KEY=kova-network-offline-key\n", "utf8"); + +let config = {}; +try { + config = JSON.parse(fs.readFileSync(configPath, "utf8")); +} catch { + config = {}; +} + +const modelRef = "openai/gpt-5.5"; +const cost = { + input: 0, + output: 0, + cacheRead: 0, + cacheWrite: 0 +}; + +config.models = { + ...(config.models || {}), + mode: "merge", + providers: { + ...(config.models?.providers || {}), + openai: { + ...(config.models?.providers?.openai || {}), + baseUrl: `http://127.0.0.1:${port}/v1`, + apiKey: { + source: "env", + provider: "default", + id: "OPENAI_API_KEY" + }, + api: "openai-responses", + request: { + ...(config.models?.providers?.openai?.request || {}), + allowPrivateNetwork: true + }, + models: [ + { + id: "gpt-5.5", + name: "gpt-5.5", + api: "openai-responses", + reasoning: false, + input: ["text"], + cost, + contextWindow: 128000, + contextTokens: 96000, + maxTokens: 4096 + } + ] + } + } +}; + +config.agents = { + ...(config.agents || {}), + defaults: { + ...(config.agents?.defaults || {}), + model: { + ...(config.agents?.defaults?.model || {}), + primary: modelRef + }, + models: { + ...(config.agents?.defaults?.models || {}), + [modelRef]: { + params: { + ...(config.agents?.defaults?.models?.[modelRef]?.params || {}), + transport: "sse", + openaiWsWarmup: false + } + } + } + } +}; + +fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8"); +console.log(JSON.stringify({ + schemaVersion: "kova.openclawProviderBlackhole.v1", + configPath, + baseUrl: `http://127.0.0.1:${port}/v1` +})); + +async function reserveClosedPort() { + const server = net.createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + const selected = address?.port; + await new Promise((resolve, reject) => { + server.close((error) => error ? reject(error) : resolve()); + }); + if (!Number.isInteger(selected) || selected <= 0) { + throw new Error("failed to reserve a closed localhost port"); + } + return selected; +} + +function requiredEnv(name) { + const value = process.env[name]; + if (!value) { + throw new Error(`${name} is required`); + } + return value; +} diff --git a/surfaces/network-offline.json b/surfaces/network-offline.json new file mode 100644 index 0000000..85cd1f7 --- /dev/null +++ b/surfaces/network-offline.json @@ -0,0 +1,31 @@ +{ + "id": "network-offline", + "title": "Network Offline", + "ownerArea": "provider-network", + "description": "Exercise an OpenClaw agent turn when the configured provider endpoint is unreachable and verify the failure is bounded and contained.", + "requiredStates": ["fresh"], + "targetKinds": ["npm", "channel", "runtime", "local-build"], + "requiredMetrics": [ + "networkFailureObserved", + "networkCommandTimedOut", + "networkStatusAfterFailureMs", + "networkGatewayStatusWorks", + "peakRssMb" + ], + "processRoles": ["gateway", "command-tree", "agent-cli", "agent-process"], + "thresholds": { + "networkFailureObserved": 1, + "networkStatusAfterFailureMs": 10000, + "peakRssMb": 900 + }, + "roleThresholds": { + "gateway": { "peakRssMb": 850, "maxCpuPercent": 250 }, + "command-tree": { "peakRssMb": 1000, "maxCpuPercent": 350 }, + "agent-cli": { "peakRssMb": 900, "maxCpuPercent": 300 }, + "agent-process": { "peakRssMb": 900, "maxCpuPercent": 300 } + }, + "diagnostics": { + "timelineRequiredForSourceBuild": true, + "expectedSpans": ["agent.turn", "provider.request"] + } +}