openclaw/Kova
openclaw/Kova
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add network offline agent scenario

Browse Source
This commit is contained in:
Shakker 2026-05-01 12:52:48 +01:00
parent acadaf2bad
commit 730a332358
No known key found for this signature in database
13 changed files with 685 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
metrics/known.json
View File

@ -51,6 +51,11 @@
"mcpShutdownMs",
"mcpToolCountMin",
"mcpToolsListMs",
"networkCommandTimedOut",
"networkFailureObserved",
"networkGatewayStatusWorks",
"networkStatusAfterFailureMs",
"networkTurnMs",
"openclawSlowestSpanMs",
"openclawTimelineParseErrors",
"peakRssMb",

2
process-roles/agent-cli.json
View File

@ -3,5 +3,5 @@
"title": "Agent CLI",
"description": "User-facing OpenClaw agent command invocations that send messages or manage local sessions.",
"commandPatterns": ["agent --local", "agent --session-id"],
"processPatterns": ["(^|\\s|/)openclaw\\s+.*\\bagent\\b"]
"processPatterns": ["(^|\\s|/)openclaw\\s+.*\\bagent\\b", "(^|\\s|/)openclaw-agent(\\s|$)"]
}

1
process-roles/agent-process.json
View File

@ -4,6 +4,7 @@
"description": "Longer-running agent execution work spawned by OpenClaw while processing a turn.",
"commandPatterns": ["(^|\\s)agent(\\s|$)"],
"processPatterns": [
"(^|\\s|/)openclaw-agent(\\s|$)",
"(^|\\s|/)openclaw\\s+.*\\bsession\\b",
"(^|\\s|/)openclaw\\s+.*\\bagent\\b"
]

1
profiles/exhaustive.json
View File

@ -39,6 +39,7 @@
{ "scenario": "mcp-runtime-start-stop", "state": "fresh" },
{ "scenario": "browser-automation-smoke", "state": "fresh", "timeoutMs": 180000 },
{ "scenario": "media-understanding-timeout", "state": "fresh", "timeoutMs": 180000 },
{ "scenario": "agent-network-offline", "state": "fresh", "timeoutMs": 180000 },
{ "scenario": "gateway-performance", "state": "many-bundled-plugins" },
{ "scenario": "gateway-performance", "state": "gateway-already-running" },
{ "scenario": "gateway-performance", "state": "stale-service-state" },

20
profiles/release.json
View File

@ -114,6 +114,14 @@
"statusMs": 10000,
"pluginLoadFailures": 0
}
},
"network-offline": {
"thresholds": {
"networkFailureObserved": 1,
"networkStatusAfterFailureMs": 10000,
"statusMs": 10000,
"pluginLoadFailures": 0
}
}
}
},
@ -163,6 +171,7 @@
"mcp-runtime:fresh",
"browser-automation:fresh",
"media-understanding:fresh",
"network-offline:fresh",
"cross-platform-smoke:slow-filesystem"
]
},
@ -185,7 +194,7 @@
"tui",
"gateway-performance"
],
"warning": ["failure-containment", "soak", "workspace-scan", "mcp-runtime", "browser-automation", "media-understanding", "cross-platform-smoke"]
"warning": ["failure-containment", "soak", "workspace-scan", "mcp-runtime", "browser-automation", "media-understanding", "network-offline", "cross-platform-smoke"]
},
"scenarios": {
"blocking": [
@ -205,7 +214,8 @@
"workspace-scan-pressure",
"mcp-runtime-start-stop",
"browser-automation-smoke",
"media-understanding-timeout"
"media-understanding-timeout",
"agent-network-offline"
]
}
},
@ -246,6 +256,7 @@
{ "scenario": "mcp-runtime-start-stop", "state": "fresh" },
{ "scenario": "browser-automation-smoke", "state": "fresh", "timeoutMs": 180000 },
{ "scenario": "media-understanding-timeout", "state": "fresh", "timeoutMs": 180000 },
{ "scenario": "agent-network-offline", "state": "fresh", "timeoutMs": 180000 },
{ "scenario": "cross-platform-smoke", "state": "slow-filesystem" }
]
},
@ -402,6 +413,11 @@
"state": "fresh",
"timeoutMs": 180000
},
{
"scenario": "agent-network-offline",
"state": "fresh",
"timeoutMs": 180000
},
{
"scenario": "cross-platform-smoke",
"state": "slow-filesystem"

43
scenarios/agent-network-offline.json Normal file
View File

@ -0,0 +1,43 @@
{
"id": "agent-network-offline",
"surface": "network-offline",
"title": "Agent Network Offline",
"objective": "Run a local OpenClaw agent turn against an unreachable provider endpoint and prove OpenClaw fails clearly without taking down the gateway.",
"tags": ["agent", "message", "provider", "network", "offline", "containment"],
"timeoutMs": 180000,
"auth": { "mode": "none" },
"thresholds": {
"gatewayReadyMs": 30000,
"networkFailureObserved": 1,
"networkStatusAfterFailureMs": 10000,
"statusMs": 10000,
"peakRssMb": 900,
"missingDependencyErrors": 0,
"pluginLoadFailures": 0
},
"phases": [
{
"id": "provision",
"title": "Provision Network Offline Env",
"intent": "Start a disposable OpenClaw gateway before configuring an unreachable provider endpoint.",
"commands": ["ocm start {env} {startSelector} --json"],
"evidence": ["gateway port", "runtime binding", "startup readiness"]
},
{
"id": "network-offline-turn",
"title": "Network Offline Agent Turn",
"intent": "Send a simple local agent message that must fail because the configured provider endpoint is unreachable.",
"commands": [
"node {kovaRoot}/support/agent-network-offline.mjs --env {env} --artifact-dir {artifactDir} --timeout-seconds 20 --max-command-ms 45000"
],
"evidence": ["bounded network failure", "gateway status after failure", "role resource samples"]
},
{
"id": "post-network-health",
"title": "Post-Network Gateway Health",
"intent": "Verify the gateway remains responsive and collect logs after the network failure path.",
"commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"],
"evidence": ["gateway status", "network/provider failure logs", "plugin errors", "memory after network failure"]
}
]
}

109
src/evaluator.mjs
View File

@ -80,6 +80,7 @@ export function evaluateRecord(record, scenario, options = {}) {
const mcpBridgeEvidence = collectMcpBridgeEvidence(allResults);
const browserAutomationEvidence = collectBrowserAutomationEvidence(allResults);
const mediaUnderstandingEvidence = collectMediaUnderstandingEvidence(allResults);
const networkOfflineEvidence = collectNetworkOfflineEvidence(allResults);
const listeningFailures = countListeningFailures(record);
const tcpConnectMaxMs = collectTcpConnectMax(record);
const timeToListeningMs = collectTimeToListening(record);
@ -391,6 +392,51 @@ export function evaluateRecord(record, scenario, options = {}) {
}
}
if (networkOfflineEvidence.available) {
checkEvidenceThreshold(violations, "network-offline", "networkTurnMs", networkOfflineEvidence.networkTurnMs, thresholds.networkTurnMs, "Network offline agent turn");
checkEvidenceThreshold(violations, "network-offline", "networkStatusAfterFailureMs", networkOfflineEvidence.networkStatusAfterFailureMs, thresholds.networkStatusAfterFailureMs, "Post-network status");
if (typeof thresholds.networkFailureObserved === "number" && networkOfflineEvidence.networkFailureObserved !== true) {
violations.push({
kind: "network-offline",
metric: "networkFailureObserved",
expected: true,
actual: networkOfflineEvidence.networkFailureObserved,
message: "Network/provider failure was not observed as a bounded command failure"
});
}
if (networkOfflineEvidence.networkCommandTimedOut === true) {
violations.push({
kind: "network-offline",
metric: "networkCommandTimedOut",
expected: false,
actual: true,
message: "Network offline command hit Kova's outer timeout instead of OpenClaw surfacing the provider failure"
});
}
if (networkOfflineEvidence.gatewayStatusWorks === false) {
violations.push({
kind: "network-offline",
metric: "networkGatewayStatusWorks",
expected: true,
actual: false,
message: "Gateway status did not work after network/provider failure"
});
}
if (networkOfflineEvidence.errors.length > 0) {
violations.push({
kind: "network-offline",
metric: "networkOfflineErrors",
expected: "0",
actual: networkOfflineEvidence.errors.length,
message: `Network offline smoke reported ${networkOfflineEvidence.errors.length} error(s): ${networkOfflineEvidence.errors[0]}`
});
}
}
if (typeof thresholds.providerRequestCountMin === "number") {
const requestCount = record.providerEvidence?.requestCount ?? 0;
if (requestCount < thresholds.providerRequestCountMin) {
@ -697,6 +743,13 @@ export function evaluateRecord(record, scenario, options = {}) {
mediaStatusAfterTimeoutMs: mediaUnderstandingEvidence.mediaStatusAfterTimeoutMs,
mediaGatewayStatusWorks: mediaUnderstandingEvidence.gatewayStatusWorks,
mediaErrors: mediaUnderstandingEvidence.errors,
networkOfflineEvidence,
networkTurnMs: networkOfflineEvidence.networkTurnMs,
networkFailureObserved: networkOfflineEvidence.networkFailureObserved,
networkCommandTimedOut: networkOfflineEvidence.networkCommandTimedOut,
networkStatusAfterFailureMs: networkOfflineEvidence.networkStatusAfterFailureMs,
networkGatewayStatusWorks: networkOfflineEvidence.gatewayStatusWorks,
networkErrors: networkOfflineEvidence.errors,
soakDurationMs: soakEvidence.durationMs,
soakIterations: soakEvidence.iterations,
soakCommandP95Ms: soakEvidence.commandP95Ms,
@ -1986,6 +2039,62 @@ function parseMediaUnderstandingTimeoutOutput(result) {
}
}
function collectNetworkOfflineEvidence(results) {
const smokes = results
.filter((result) => result.command?.includes("agent-network-offline.mjs"))
.map((result) => parseNetworkOfflineOutput(result))
.filter(Boolean);
if (smokes.length === 0) {
return {
schemaVersion: "kova.networkOfflineEvidence.v1",
available: false,
networkTurnMs: null,
networkFailureObserved: null,
networkCommandTimedOut: null,
networkStatusAfterFailureMs: null,
gatewayStatusWorks: null,
errors: [],
smokes: []
};
}
return {
schemaVersion: "kova.networkOfflineEvidence.v1",
available: true,
networkTurnMs: maxNullable(...smokes.map((smoke) => smoke.networkTurnMs)),
networkFailureObserved: smokes.every((smoke) => smoke.networkFailureObserved === true),
networkCommandTimedOut: smokes.some((smoke) => smoke.networkCommandTimedOut === true),
networkStatusAfterFailureMs: maxNullable(...smokes.map((smoke) => smoke.networkStatusAfterFailureMs)),
gatewayStatusWorks: smokes.every((smoke) => smoke.gatewayStatusWorks === true),
errors: smokes.flatMap((smoke) => smoke.errors ?? []),
smokes: smokes.map((smoke) => ({
durationMs: smoke.durationMs ?? null,
networkTurnMs: smoke.networkTurnMs ?? null,
networkFailureObserved: smoke.networkFailureObserved ?? null,
networkCommandTimedOut: smoke.networkCommandTimedOut ?? null,
networkCommandStatus: smoke.networkCommandStatus ?? null,
networkStatusAfterFailureMs: smoke.networkStatusAfterFailureMs ?? null,
gatewayStatusWorks: smoke.gatewayStatusWorks ?? null,
errors: smoke.errors ?? []
}))
};
}
function parseNetworkOfflineOutput(result) {
const text = result.stdout ?? "";
const jsonStart = text.indexOf("{");
if (jsonStart < 0) {
return null;
}
try {
const parsed = JSON.parse(text.slice(jsonStart));
return parsed?.schemaVersion === "kova.agentNetworkOffline.v1" ? parsed : null;
} catch {
return null;
}
}
function checkEvidenceThreshold(violations, kind, metric, actual, threshold, label) {
if (typeof threshold !== "number" || actual === null) {
return;

15
src/report.mjs
View File

@ -155,6 +155,9 @@ export function renderMarkdownReport(report) {
if (record.measurements.mediaUnderstandingEvidence?.available) {
lines.push(`- Media understanding: describe ${record.measurements.mediaDescribeMs ?? "unknown"} ms; timeout observed ${record.measurements.mediaTimeoutObserved ?? "unknown"}; command outer timeout ${record.measurements.mediaCommandTimedOut ?? "unknown"}; status after timeout ${record.measurements.mediaStatusAfterTimeoutMs ?? "unknown"} ms; gateway status ${record.measurements.mediaGatewayStatusWorks ?? "unknown"}`);
}
if (record.measurements.networkOfflineEvidence?.available) {
lines.push(`- Network offline: turn ${record.measurements.networkTurnMs ?? "unknown"} ms; failure observed ${record.measurements.networkFailureObserved ?? "unknown"}; command outer timeout ${record.measurements.networkCommandTimedOut ?? "unknown"}; status after failure ${record.measurements.networkStatusAfterFailureMs ?? "unknown"} ms; gateway status ${record.measurements.networkGatewayStatusWorks ?? "unknown"}`);
}
lines.push(`- Provider/model timing: ${record.measurements.providerModelTimingMs ?? "unknown"} ms`);
lines.push(`- Agent turn: ${record.measurements.agentTurnMs ?? "unknown"} ms (${record.measurements.agentResponseOk ?? "not-run"})`);
if (record.measurements.agentTurnCount > 0) {
@ -711,6 +714,12 @@ function summarizeMeasurements(measurements) {
mediaCommandTimedOut: measurements.mediaCommandTimedOut ?? null,
mediaStatusAfterTimeoutMs: measurements.mediaStatusAfterTimeoutMs ?? null,
mediaGatewayStatusWorks: measurements.mediaGatewayStatusWorks ?? null,
networkOfflineEvidence: measurements.networkOfflineEvidence ?? null,
networkTurnMs: measurements.networkTurnMs ?? null,
networkFailureObserved: measurements.networkFailureObserved ?? null,
networkCommandTimedOut: measurements.networkCommandTimedOut ?? null,
networkStatusAfterFailureMs: measurements.networkStatusAfterFailureMs ?? null,
networkGatewayStatusWorks: measurements.networkGatewayStatusWorks ?? null,
resourceTrend: measurements.resourceTrend ?? null,
profilingEnabled: measurements.profilingEnabled ?? null,
profilingResourceInterpretation: measurements.profilingResourceInterpretation ?? null,
@ -864,6 +873,12 @@ export function renderPasteSummary(report) {
} else if (record.violations?.length > 0) {
if (record.measurements) {
lines.push(`Measurements: agent turn ${record.measurements.agentTurnMs ?? "not-run"}ms; cold/warm ${record.measurements.coldAgentTurnMs ?? "unknown"}/${record.measurements.warmAgentTurnMs ?? "unknown"}ms; cold-warm delta ${record.measurements.agentColdWarmDeltaMs ?? "unknown"}ms; pre-provider ${record.measurements.agentPreProviderMs ?? "unknown"}ms; provider work ${record.measurements.agentProviderFinalMs ?? "unknown"}ms; cleanup max ${record.measurements.agentCleanupMaxMs ?? "unknown"}ms; diagnosis ${record.measurements.agentLatencyDiagnosis?.kind ?? "unknown"}; cleanup diagnosis ${record.measurements.agentCleanupDiagnosis?.kind ?? "none"}; provider simulation ${record.measurements.agentProviderMode ?? "normal"}/${record.measurements.agentProviderIssue ?? "none"} containment ${record.measurements.agentProviderContainmentOk ?? "n/a"} recovery ${record.measurements.agentProviderRecoveryOk ?? "n/a"}; agent process leaks ${record.measurements.agentProcessLeakCount ?? "unknown"}.`);
if (record.measurements.mediaUnderstandingEvidence?.available) {
lines.push(`Media: describe ${record.measurements.mediaDescribeMs ?? "unknown"}ms; timeout ${record.measurements.mediaTimeoutObserved ?? "unknown"}; status ${record.measurements.mediaStatusAfterTimeoutMs ?? "unknown"}ms.`);
}
if (record.measurements.networkOfflineEvidence?.available) {
lines.push(`Network offline: turn ${record.measurements.networkTurnMs ?? "unknown"}ms; failure ${record.measurements.networkFailureObserved ?? "unknown"}; status ${record.measurements.networkStatusAfterFailureMs ?? "unknown"}ms.`);
}
}
lines.push("Violations:");
for (const violation of record.violations) {

131
src/selfcheck.mjs
View File

@ -215,6 +215,21 @@ export async function runSelfCheck(flags = {}) {
assertEqual(record?.thresholds?.mediaTimeoutObserved, 1, "media timeout threshold");
assertEqual(record?.thresholds?.providerRequestCountMin, 1, "media provider request threshold");
}));
checks.push(await jsonCommandCheck("network-offline-dry-run-json", `node bin/kova.mjs run --target runtime:stable --scenario agent-network-offline --state fresh --report-dir ${quoteShell(tmp)} --json`, async (data) => {
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
const record = report.records?.[0];
assertEqual(record?.surface, "network-offline", "network offline surface");
assertEqual(record?.auth?.mode, "none", "network offline opts out of default mock auth");
const phaseIds = record?.phases?.map((phase) => phase.id) ?? [];
if (phaseIds.includes("auth-prepare") || phaseIds.includes("auth-setup")) {
throw new Error(`network offline must not start mock auth phases: ${phaseIds.join(", ")}`);
}
const commands = record?.phases?.flatMap((phase) => phase.commands ?? []) ?? [];
const networkCommand = commands.find((command) => command.includes("agent-network-offline.mjs")) ?? "";
assertEqual(networkCommand.includes("--artifact-dir '"), true, "network helper receives quoted artifact dir");
assertEqual(networkCommand.includes("--max-command-ms 45000"), true, "network helper allows cold CLI evidence before outer timeout");
assertEqual(record?.thresholds?.networkFailureObserved, 1, "network failure threshold");
}));
checks.push(await jsonCommandCheck("diagnostic-profile-plan-json", "node bin/kova.mjs matrix plan --profile diagnostic --target local-build:/tmp/openclaw --include scenario:release-runtime-startup --json", (data) => {
assertEqual(data.schemaVersion, "kova.matrix.plan.v1", "diagnostic matrix plan schema");
assertEqual(data.profile?.id, "diagnostic", "diagnostic profile id");
@ -292,6 +307,7 @@ export async function runSelfCheck(flags = {}) {
checks.push(mcpBridgeEvidenceEvaluationCheck());
checks.push(browserAutomationEvidenceEvaluationCheck());
checks.push(mediaUnderstandingEvidenceEvaluationCheck());
checks.push(networkOfflineEvidenceEvaluationCheck());
checks.push(await jsonCommandCheck(
"dry-run-state-lifecycle-json",
`node bin/kova.mjs run --target runtime:stable --scenario fresh-install --state missing-plugin-index --report-dir ${quoteShell(tmp)} --json`,
@ -2567,6 +2583,111 @@ function mediaUnderstandingEvidenceEvaluationCheck() {
}
}
function networkOfflineEvidenceEvaluationCheck() {
try {
const smoke = {
schemaVersion: "kova.agentNetworkOffline.v1",
ok: true,
durationMs: 1800,
networkTurnMs: 1400,
networkFailureObserved: true,
networkCommandTimedOut: false,
networkCommandStatus: 1,
networkStatusAfterFailureMs: 190,
gatewayStatusWorks: true,
errors: []
};
const record = {
scenario: "agent-network-offline",
status: "PASS",
phases: [{
id: "network-offline-turn",
results: [{
command: "node support/agent-network-offline.mjs --env kova-self-check --artifact-dir /tmp/kova",
status: 0,
timedOut: false,
durationMs: 1800,
stdout: JSON.stringify(smoke),
stderr: ""
}],
metrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() }
}],
finalMetrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() }
};
evaluateRecord(record, {
id: "agent-network-offline",
thresholds: {
networkFailureObserved: 1,
networkStatusAfterFailureMs: 10000
}
}, { surface: { thresholds: {} }, targetPlan: { kind: "npm" } });
assertEqual(record.status, "PASS", "network offline record status");
assertEqual(record.measurements.networkTurnMs, 1400, "network turn ms");
assertEqual(record.measurements.networkFailureObserved, true, "network failure observed");
assertEqual(record.measurements.networkCommandTimedOut, false, "network command did not hit outer timeout");
assertEqual(record.measurements.networkStatusAfterFailureMs, 190, "post-network status ms");
assertEqual(record.measurements.networkGatewayStatusWorks, true, "gateway status after network failure");
const failed = {
...record,
status: "PASS",
violations: [],
measurements: undefined,
phases: [{
id: "network-offline-turn",
results: [{
command: "node support/agent-network-offline.mjs --env kova-self-check --artifact-dir /tmp/kova",
status: 0,
timedOut: false,
durationMs: 1800,
stdout: JSON.stringify({
...smoke,
ok: false,
networkFailureObserved: false,
gatewayStatusWorks: false,
errors: ["network failure not observed"]
}),
stderr: ""
}],
metrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() }
}]
};
evaluateRecord(failed, {
id: "agent-network-offline",
thresholds: {
networkFailureObserved: 1
}
}, { surface: { thresholds: {} }, targetPlan: { kind: "npm" } });
assertEqual(failed.status, "FAIL", "network failure status");
assertEqual(
failed.violations.some((violation) => violation.metric === "networkFailureObserved"),
true,
"network failure observed violation"
);
assertEqual(
failed.violations.some((violation) => violation.metric === "networkGatewayStatusWorks"),
true,
"network gateway status violation"
);
return {
id: "network-offline-evidence-evaluation",
status: "PASS",
command: "evaluate synthetic network offline evidence",
durationMs: 0
};
} catch (error) {
return {
id: "network-offline-evidence-evaluation",
status: "FAIL",
command: "evaluate synthetic network offline evidence",
durationMs: 0,
message: error.message
};
}
}
function agentColdWarmEvaluationCheck() {
try {
const coldCommand = "ocm @kova -- agent --local --agent main --session-id kova-agent-cold-warm --message hi --json";
@ -3339,6 +3460,14 @@ async function resourceRolePollutionCheck() {
existingRoles: ["command-tree"]
}
);
const openclawAgentRoles = classifyRegistryRolesForProcess(
{ command: "openclaw-agent" },
{
processRoles,
rootCommand: "ocm @kova -- agent --local --message hi",
existingRoles: ["command-tree"]
}
);
assertEqual(mockProviderRoles.includes("mock-provider"), true, "mock provider helper remains classified");
assertEqual(mockProviderRoles.includes("agent-cli"), false, "KOVA_AGENT_OK marker must not imply agent-cli");
@ -3346,6 +3475,8 @@ async function resourceRolePollutionCheck() {
assertEqual(mockProviderRoles.includes("browser-sidecar"), false, "browser env name must not imply browser-sidecar");
assertEqual(envNameRoles.includes("runtime-management"), false, "mcp-runtime env name must not imply runtime-management");
assertEqual(envNameRoles.includes("model-cli"), false, "configure-openclaw fixture helper must not imply model-cli");
assertEqual(openclawAgentRoles.includes("agent-cli"), true, "openclaw-agent process must imply agent-cli");
assertEqual(openclawAgentRoles.includes("agent-process"), true, "openclaw-agent process must imply agent-process");
return {
id: "resource-role-pollution-boundary",
status: "PASS",

3
states/fresh.json
View File

@ -24,7 +24,8 @@
"tui",
"mcp-runtime",
"browser-automation",
"media-understanding"
"media-understanding",
"network-offline"
],
"incompatibleSurfaces": [
"upgrade-existing-user"

213
support/agent-network-offline.mjs Normal file
View File

@ -0,0 +1,213 @@
#!/usr/bin/env node
import { spawn } from "node:child_process";
import fs from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
const options = parseArgs(process.argv.slice(2));
const envName = requiredString(options.env, "--env");
const artifactDir = requiredString(options.artifactDir, "--artifact-dir");
const timeoutSeconds = positiveInteger(options.timeoutSeconds ?? "20", "--timeout-seconds");
const maxCommandMs = positiveInteger(options.maxCommandMs ?? "45000", "--max-command-ms");
const networkFailurePattern = /ECONNREFUSED|ENOTFOUND|ETIMEDOUT|EAI_AGAIN|fetch failed|network|connection refused|provider.*fail|failed to fetch|socket/i;
if (!/^kova-[a-z0-9][a-z0-9-]*$/i.test(envName)) {
failUsage(`refusing to run network offline smoke against non-Kova env: ${JSON.stringify(envName)}`);
}
await fs.mkdir(artifactDir, { recursive: true });
const networkDir = path.join(artifactDir, "network-offline");
await fs.mkdir(networkDir, { recursive: true });
const configure = await run("ocm", [
"env",
"exec",
envName,
"--",
"node",
supportPath("configure-openclaw-provider-blackhole.mjs")
], { timeoutMs: 15000 });
const startedAtEpochMs = Date.now();
const startedAt = new Date(startedAtEpochMs).toISOString();
const agent = await run("ocm", [
`@${envName}`,
"--",
"agent",
"--local",
"--agent",
"main",
"--session-id",
"kova-agent-network-offline",
"--message",
"Reply with exact ASCII text KOVA_AGENT_OK only.",
"--thinking",
"off",
"--timeout",
String(timeoutSeconds),
"--json"
], { timeoutMs: maxCommandMs + 3000 });
const finishedAtEpochMs = Date.now();
const status = await run("ocm", [`@${envName}`, "--", "status"], { timeoutMs: 15000 });
const combinedOutput = `${agent.stdout}\n${agent.stderr}`;
const networkFailureObserved = agent.status !== 0 &&
networkFailurePattern.test(combinedOutput);
const statusWorks = status.status === 0 && status.timedOut !== true;
const ok = configure.status === 0 && networkFailureObserved && agent.timedOut !== true && statusWorks;
const summary = {
schemaVersion: "kova.agentNetworkOffline.v1",
ok,
env: envName,
timeoutSeconds,
maxCommandMs,
startedAt,
startedAtEpochMs,
finishedAt: new Date(finishedAtEpochMs).toISOString(),
finishedAtEpochMs,
durationMs: finishedAtEpochMs - startedAtEpochMs,
networkTurnMs: agent.durationMs,
networkFailureObserved,
networkCommandTimedOut: agent.timedOut === true,
networkCommandStatus: agent.status,
networkStatusAfterFailureMs: status.durationMs,
gatewayStatusWorks: statusWorks,
configureStatus: configure.status,
errors: [
...(configure.status === 0 ? [] : [`configure failed: ${snippet(configure.stderr || configure.stdout)}`]),
...(networkFailureObserved ? [] : [`network failure not observed: status=${agent.status} timedOut=${agent.timedOut} duration=${agent.durationMs} stderr=${snippet(agent.stderr)}`]),
...(agent.timedOut ? [`network failure command did not exit before ${maxCommandMs}ms; stderr=${snippet(agent.stderr)}`] : []),
...(statusWorks ? [] : [`status after network failure failed: status=${status.status} stderr=${snippet(status.stderr || status.stdout)}`])
],
commands: {
configure: compactCommand(configure),
agent: compactCommand(agent),
status: compactCommand(status)
}
};
await fs.writeFile(path.join(networkDir, "summary.json"), `${JSON.stringify(summary, null, 2)}\n`, "utf8");
process.stdout.write(`${JSON.stringify(summary)}\n`);
process.exit(ok ? 0 : 1);
function run(command, args, options = {}) {
const started = Date.now();
return new Promise((resolve) => {
const child = spawn(command, args, {
env: process.env,
shell: false,
stdio: ["ignore", "pipe", "pipe"]
});
let stdout = "";
let stderr = "";
let timedOut = false;
const timer = setTimeout(() => {
timedOut = true;
child.kill("SIGTERM");
setTimeout(() => child.kill("SIGKILL"), 3000).unref();
}, options.timeoutMs ?? 30000);
child.stdout.on("data", (chunk) => {
stdout += chunk.toString();
});
child.stderr.on("data", (chunk) => {
stderr += chunk.toString();
});
child.on("error", (error) => {
clearTimeout(timer);
resolve({
command: [command, ...args].join(" "),
status: 127,
signal: null,
timedOut,
durationMs: Date.now() - started,
stdout: "",
stderr: error.message
});
});
child.on("close", (status, signal) => {
clearTimeout(timer);
resolve({
command: [command, ...args].join(" "),
status: timedOut ? 124 : (status ?? 1),
signal,
timedOut,
durationMs: Date.now() - started,
stdout: truncate(stdout),
stderr: truncate(stderr)
});
});
});
}
function compactCommand(result) {
return {
command: result.command,
status: result.status,
signal: result.signal,
timedOut: result.timedOut,
durationMs: result.durationMs,
stdout: truncate(result.stdout, 1200),
stderr: truncate(result.stderr, 1200)
};
}
function supportPath(file) {
return path.join(path.dirname(fileURLToPath(import.meta.url)), file);
}
function parseArgs(args) {
const parsed = {};
for (let index = 0; index < args.length; index += 1) {
const arg = args[index];
if (!arg.startsWith("--")) {
failUsage(`unexpected positional argument ${JSON.stringify(arg)}`);
}
const key = arg.slice(2).replaceAll("-", "_");
const value = args[index + 1];
if (value === undefined || value.startsWith("--")) {
failUsage(`${arg} requires a value`);
}
parsed[key] = value;
index += 1;
}
return {
env: parsed.env,
artifactDir: parsed.artifact_dir,
timeoutSeconds: parsed.timeout_seconds,
maxCommandMs: parsed.max_command_ms
};
}
function requiredString(value, flag) {
if (typeof value !== "string" || value.length === 0) {
failUsage(`${flag} is required`);
}
return value;
}
function positiveInteger(value, flag) {
const number = Number(value);
if (!Number.isInteger(number) || number <= 0) {
failUsage(`${flag} must be a positive integer`);
}
return number;
}
function truncate(value, limit = 4000) {
const text = String(value ?? "");
if (text.length <= limit) {
return text;
}
return `${text.slice(0, limit)}\n[truncated ${text.length - limit} chars]`;
}
function snippet(value) {
return truncate(String(value ?? "").replace(/\s+/g, " ").trim(), 500);
}
function failUsage(message) {
process.stderr.write(`${message}\n`);
process.stderr.write("usage: agent-network-offline.mjs --env <kova-env> --artifact-dir <dir> [--timeout-seconds <seconds>] [--max-command-ms <ms>]\n");
process.exit(2);
}

115
support/configure-openclaw-provider-blackhole.mjs Normal file
View File

@ -0,0 +1,115 @@
#!/usr/bin/env node
import fs from "node:fs";
import net from "node:net";
import path from "node:path";
const port = await reserveClosedPort();
const stateDir = process.env.OPENCLAW_STATE_DIR || path.join(requiredEnv("OPENCLAW_HOME"), ".openclaw");
const configPath = process.env.OPENCLAW_CONFIG_PATH || path.join(stateDir, "openclaw.json");
fs.mkdirSync(path.dirname(configPath), { recursive: true });
fs.mkdirSync(stateDir, { recursive: true });
fs.writeFileSync(path.join(stateDir, ".env"), "OPENAI_API_KEY=kova-network-offline-key\n", "utf8");
let config = {};
try {
config = JSON.parse(fs.readFileSync(configPath, "utf8"));
} catch {
config = {};
}
const modelRef = "openai/gpt-5.5";
const cost = {
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0
};
config.models = {
...(config.models || {}),
mode: "merge",
providers: {
...(config.models?.providers || {}),
openai: {
...(config.models?.providers?.openai || {}),
baseUrl: `http://127.0.0.1:${port}/v1`,
apiKey: {
source: "env",
provider: "default",
id: "OPENAI_API_KEY"
},
api: "openai-responses",
request: {
...(config.models?.providers?.openai?.request || {}),
allowPrivateNetwork: true
},
models: [
{
id: "gpt-5.5",
name: "gpt-5.5",
api: "openai-responses",
reasoning: false,
input: ["text"],
cost,
contextWindow: 128000,
contextTokens: 96000,
maxTokens: 4096
}
]
}
}
};
config.agents = {
...(config.agents || {}),
defaults: {
...(config.agents?.defaults || {}),
model: {
...(config.agents?.defaults?.model || {}),
primary: modelRef
},
models: {
...(config.agents?.defaults?.models || {}),
[modelRef]: {
params: {
...(config.agents?.defaults?.models?.[modelRef]?.params || {}),
transport: "sse",
openaiWsWarmup: false
}
}
}
}
};
fs.writeFileSync(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
console.log(JSON.stringify({
schemaVersion: "kova.openclawProviderBlackhole.v1",
configPath,
baseUrl: `http://127.0.0.1:${port}/v1`
}));
async function reserveClosedPort() {
const server = net.createServer();
await new Promise((resolve, reject) => {
server.once("error", reject);
server.listen(0, "127.0.0.1", resolve);
});
const address = server.address();
const selected = address?.port;
await new Promise((resolve, reject) => {
server.close((error) => error ? reject(error) : resolve());
});
if (!Number.isInteger(selected) || selected <= 0) {
throw new Error("failed to reserve a closed localhost port");
}
return selected;
}
function requiredEnv(name) {
const value = process.env[name];
if (!value) {
throw new Error(`${name} is required`);
}
return value;
}

31
surfaces/network-offline.json Normal file
View File

@ -0,0 +1,31 @@
{
"id": "network-offline",
"title": "Network Offline",
"ownerArea": "provider-network",
"description": "Exercise an OpenClaw agent turn when the configured provider endpoint is unreachable and verify the failure is bounded and contained.",
"requiredStates": ["fresh"],
"targetKinds": ["npm", "channel", "runtime", "local-build"],
"requiredMetrics": [
"networkFailureObserved",
"networkCommandTimedOut",
"networkStatusAfterFailureMs",
"networkGatewayStatusWorks",
"peakRssMb"
],
"processRoles": ["gateway", "command-tree", "agent-cli", "agent-process"],
"thresholds": {
"networkFailureObserved": 1,
"networkStatusAfterFailureMs": 10000,
"peakRssMb": 900
},
"roleThresholds": {
"gateway": { "peakRssMb": 850, "maxCpuPercent": 250 },
"command-tree": { "peakRssMb": 1000, "maxCpuPercent": 350 },
"agent-cli": { "peakRssMb": 900, "maxCpuPercent": 300 },
"agent-process": { "peakRssMb": 900, "maxCpuPercent": 300 }
},
"diagnostics": {
"timelineRequiredForSourceBuild": true,
"expectedSpans": ["agent.turn", "provider.request"]
}
}