diff --git a/profiles/exhaustive.json b/profiles/exhaustive.json index add4d26..1095221 100644 --- a/profiles/exhaustive.json +++ b/profiles/exhaustive.json @@ -26,6 +26,7 @@ { "scenario": "provider-models", "state": "model-auth-configured" }, { "scenario": "provider-models", "state": "model-auth-missing" }, { "scenario": "agent-cold-warm-message", "state": "mock-openai-provider", "timeoutMs": 180000 }, + { "scenario": "agent-auth-missing", "state": "agent-auth-missing", "timeoutMs": 180000 }, { "scenario": "agent-long-session", "state": "mock-openai-provider", "timeoutMs": 360000 }, { "scenario": "agent-provider-slow", "state": "mock-openai-provider", "timeoutMs": 180000 }, { "scenario": "agent-provider-timeout", "state": "mock-openai-provider", "timeoutMs": 180000 }, diff --git a/profiles/release.json b/profiles/release.json index c6b7017..1350296 100644 --- a/profiles/release.json +++ b/profiles/release.json @@ -99,6 +99,7 @@ { "scenario": "agent-provider-streaming-stall", "state": "mock-openai-provider" }, { "scenario": "agent-provider-concurrent", "state": "mock-openai-provider" }, { "scenario": "agent-provider-recovery", "state": "mock-openai-provider" }, + { "scenario": "agent-auth-missing", "state": "agent-auth-missing" }, { "scenario": "agent-long-session", "state": "mock-openai-provider" }, { "scenario": "failure-injection", "state": "broken-plugin-deps" }, { "scenario": "soak", "state": "large-workspace" }, @@ -198,6 +199,11 @@ "state": "mock-openai-provider", "timeoutMs": 240000 }, + { + "scenario": "agent-auth-missing", + "state": "agent-auth-missing", + "timeoutMs": 180000 + }, { "scenario": "agent-long-session", "state": "mock-openai-provider", diff --git a/scenarios/agent-auth-missing.json b/scenarios/agent-auth-missing.json new file mode 100644 index 0000000..1ba39cd --- /dev/null +++ b/scenarios/agent-auth-missing.json @@ -0,0 +1,48 @@ +{ + "id": "agent-auth-missing", + "surface": "agent-message", + "title": "Agent Auth Missing", + "objective": "Prove OpenClaw fails a local agent turn clearly when model auth is missing, keeps the gateway responsive, and leaves no leaked child processes.", + "tags": ["agent", "message", "auth", "failure", "containment"], + "states": ["agent-auth-missing"], + "timeoutMs": 180000, + "auth": { "mode": "missing" }, + "agent": { + "expectedFailure": true + }, + "thresholds": { + "gatewayReadyMs": 30000, + "agentTurnMs": 60000, + "agentContainmentHealthFailures": 0, + "agentProcessLeaks": 0, + "peakRssMb": 900, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "provision", + "title": "Provision Missing-Auth Env", + "intent": "Start a disposable OpenClaw gateway without Kova mock or live model credentials.", + "commands": ["ocm start {env} {startSelector} --json"], + "evidence": ["gateway port", "runtime binding", "startup readiness", "no Kova auth setup phase"] + }, + { + "id": "missing-auth-agent-turn", + "title": "Missing Auth Agent Turn", + "intent": "Send a local agent message that must fail quickly and clearly because no provider credentials are configured.", + "expectedAgentFailure": true, + "commands": [ + "node {kovaRoot}/support/expect-command-fails.mjs -- ocm @{env} -- agent --local --agent main --session-id kova-agent-auth-missing --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --thinking off --timeout 30 --json" + ], + "evidence": ["clear auth failure", "no provider request", "process leak snapshot", "role resource samples"] + }, + { + "id": "post-auth-failure-health", + "title": "Post-Auth-Failure Gateway Health", + "intent": "Verify the gateway remains responsive after the missing-auth agent failure.", + "commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"], + "evidence": ["gateway status", "auth failure logs", "plugin errors", "memory after auth failure"] + } + ] +} diff --git a/src/evaluator.mjs b/src/evaluator.mjs index 521e5b9..6957cae 100644 --- a/src/evaluator.mjs +++ b/src/evaluator.mjs @@ -55,6 +55,7 @@ export function evaluateRecord(record, scenario, options = {}) { providerTurn, thresholds, timelineSummary, + authMode: record.auth?.mode ?? null, expectedProviderMode: scenario.mockProvider?.mode ?? "normal", providerSimulation: agentProviderSimulation }); @@ -902,6 +903,13 @@ function buildAgentFailureFixerSummary(latencyDiagnosis, providerSimulation, con likelyOwner: latencyDiagnosis.likelyOwner }); } + if (latencyDiagnosis?.kind === "auth-failure") { + items.push({ + kind: "auth-failure", + summary: "Agent turn failed before provider work because model/provider auth was missing; verify OpenClaw reports credential setup guidance and keeps the gateway usable.", + likelyOwner: latencyDiagnosis.likelyOwner + }); + } if ((containment?.processLeakCount ?? 0) > 0) { const first = containment.leakedProcesses?.[0]; items.push({ @@ -1016,11 +1024,19 @@ function checkTurnThreshold(violations, turn, metric, threshold, message) { }); } -function diagnoseAgentLatency({ coldAgentTurn, warmAgentTurn, providerTurn, thresholds, timelineSummary, expectedProviderMode = "normal", providerSimulation = null }) { +function diagnoseAgentLatency({ coldAgentTurn, warmAgentTurn, providerTurn, thresholds, timelineSummary, authMode = null, expectedProviderMode = "normal", providerSimulation = null }) { if (!providerTurn) { return null; } if (providerTurn.missingProviderRequest === true) { + if (providerTurn.expectedFailure === true && ["missing", "broken", "none", "skip"].includes(authMode)) { + return { + kind: "auth-failure", + severity: "info", + summary: `Agent turn failed before provider work because auth mode is ${authMode}.`, + likelyOwner: "agent-runtime/auth" + }; + } return { kind: "no-provider-request", severity: "fail", diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index 6b3396e..ee66d0c 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -170,6 +170,7 @@ export async function runSelfCheck(flags = {}) { checks.push(agentColdWarmEvaluationCheck()); checks.push(await concurrentAgentRunnerCheck(tmp)); checks.push(providerConcurrentEvaluationCheck()); + checks.push(agentAuthFailureEvaluationCheck()); checks.push(await jsonCommandCheck( "dry-run-state-lifecycle-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --state missing-plugin-index --report-dir ${quoteShell(tmp)} --json`, @@ -1602,6 +1603,102 @@ function providerConcurrentEvaluationCheck() { } } +function agentAuthFailureEvaluationCheck() { + try { + const command = "node support/expect-command-fails.mjs -- ocm @kova-self-check -- agent --local --agent main --session-id kova-agent-auth-missing --message hi --json"; + const record = { + scenario: "agent-auth-missing", + status: "PASS", + auth: { mode: "missing", source: "override:missing", providerId: null }, + phases: [ + { + id: "missing-auth-agent-turn", + expectedAgentFailure: true, + results: [{ + command, + status: 0, + timedOut: false, + startedAt: "2026-04-30T10:00:01.000Z", + startedAtEpochMs: 1777543201000, + finishedAt: "2026-04-30T10:00:02.000Z", + finishedAtEpochMs: 1777543202000, + durationMs: 1000, + stdout: "", + stderr: "missing OpenAI credentials", + processSnapshots: { + leaks: { + schemaVersion: "kova.processLeakSummary.v1", + leakCount: 0, + leakedProcesses: [], + leaksByRole: {} + } + } + }], + metrics: { logs: zeroLogMetrics(), health: { ok: true } } + }, + { + id: "post-auth-failure-health", + results: [{ + command: "ocm @kova-self-check -- status", + status: 0, + timedOut: false, + durationMs: 100, + stdout: "status ok", + stderr: "" + }], + metrics: { logs: zeroLogMetrics(), health: { ok: true } } + } + ], + providerEvidence: { + available: false, + requestCount: 0, + requests: [], + errors: [], + error: "provider request log not found" + }, + finalMetrics: { + service: { gatewayState: "running" }, + logs: zeroLogMetrics() + } + }; + + evaluateRecord(record, { + id: "agent-auth-missing", + auth: { mode: "missing" }, + agent: { expectedFailure: true }, + thresholds: { + agentContainmentHealthFailures: 0, + agentProcessLeaks: 0 + } + }, { surface: { thresholds: {} }, targetPlan: { kind: "npm" } }); + + assertEqual(record.status, "PASS", "agent auth failure scenario status"); + assertEqual(record.measurements.agentTurnCount, 1, "auth failure agent turn count"); + assertEqual(record.measurements.agentTurns[0].expectedFailureObserved, true, "auth failure observed"); + assertEqual(record.measurements.agentLatencyDiagnosis.kind, "auth-failure", "auth failure diagnosis"); + assertEqual(record.measurements.agentFailureContainment.gatewayHealthy, true, "auth failure gateway healthy"); + assertEqual( + record.measurements.agentFailureFixerSummary.items.some((item) => item.kind === "auth-failure"), + true, + "auth failure fixer evidence" + ); + return { + id: "agent-auth-failure-evaluation", + status: "PASS", + command: "evaluate synthetic missing-auth agent failure containment", + durationMs: 0 + }; + } catch (error) { + return { + id: "agent-auth-failure-evaluation", + status: "FAIL", + command: "evaluate synthetic missing-auth agent failure containment", + durationMs: 0, + message: error.message + }; + } +} + function agentColdWarmEvaluationCheck() { try { const coldCommand = "ocm @kova -- agent --local --agent main --session-id kova-agent-cold-warm --message hi --json"; diff --git a/states/agent-auth-missing.json b/states/agent-auth-missing.json new file mode 100644 index 0000000..a5a8e8b --- /dev/null +++ b/states/agent-auth-missing.json @@ -0,0 +1,39 @@ +{ + "id": "agent-auth-missing", + "title": "Agent Auth Missing", + "objective": "A disposable OpenClaw env with no model/provider credentials so Kova can verify agent turns fail clearly and do not degrade the gateway.", + "tags": [ + "agent", + "auth", + "providers", + "failure" + ], + "setup": [], + "traits": [ + "agent-state", + "provider-pressure", + "missing-auth", + "failure-state" + ], + "auth": { + "mode": "missing", + "reason": "This state intentionally bypasses Kova's default mock provider auth to test missing model credentials." + }, + "compatibleSurfaces": [ + "agent-message" + ], + "incompatibleSurfaces": [], + "riskArea": "agent-provider-auth", + "ownerArea": "agent-runtime", + "setupEvidence": [ + "run-level auth policy is missing", + "no mock provider is injected" + ], + "cleanupGuarantees": [ + "disposable env cleanup removes the unauthenticated OpenClaw home" + ], + "source": { + "kind": "auth-policy-fixture", + "mode": "missing" + } +}