openclaw/Kova
openclaw/Kova
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add missing auth agent containment

Browse Source
This commit is contained in:
Shakker 2026-05-01 08:45:00 +01:00
parent 3f649687f5
commit 01422f4ea8
No known key found for this signature in database
6 changed files with 208 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
profiles/exhaustive.json
View File

@ -26,6 +26,7 @@
{ "scenario": "provider-models", "state": "model-auth-configured" },
{ "scenario": "provider-models", "state": "model-auth-missing" },
{ "scenario": "agent-cold-warm-message", "state": "mock-openai-provider", "timeoutMs": 180000 },
{ "scenario": "agent-auth-missing", "state": "agent-auth-missing", "timeoutMs": 180000 },
{ "scenario": "agent-long-session", "state": "mock-openai-provider", "timeoutMs": 360000 },
{ "scenario": "agent-provider-slow", "state": "mock-openai-provider", "timeoutMs": 180000 },
{ "scenario": "agent-provider-timeout", "state": "mock-openai-provider", "timeoutMs": 180000 },

6
profiles/release.json
View File

@ -99,6 +99,7 @@
{ "scenario": "agent-provider-streaming-stall", "state": "mock-openai-provider" },
{ "scenario": "agent-provider-concurrent", "state": "mock-openai-provider" },
{ "scenario": "agent-provider-recovery", "state": "mock-openai-provider" },
{ "scenario": "agent-auth-missing", "state": "agent-auth-missing" },
{ "scenario": "agent-long-session", "state": "mock-openai-provider" },
{ "scenario": "failure-injection", "state": "broken-plugin-deps" },
{ "scenario": "soak", "state": "large-workspace" },
@ -198,6 +199,11 @@
"state": "mock-openai-provider",
"timeoutMs": 240000
},
{
"scenario": "agent-auth-missing",
"state": "agent-auth-missing",
"timeoutMs": 180000
},
{
"scenario": "agent-long-session",
"state": "mock-openai-provider",

48
scenarios/agent-auth-missing.json Normal file
View File

@ -0,0 +1,48 @@
{
"id": "agent-auth-missing",
"surface": "agent-message",
"title": "Agent Auth Missing",
"objective": "Prove OpenClaw fails a local agent turn clearly when model auth is missing, keeps the gateway responsive, and leaves no leaked child processes.",
"tags": ["agent", "message", "auth", "failure", "containment"],
"states": ["agent-auth-missing"],
"timeoutMs": 180000,
"auth": { "mode": "missing" },
"agent": {
"expectedFailure": true
},
"thresholds": {
"gatewayReadyMs": 30000,
"agentTurnMs": 60000,
"agentContainmentHealthFailures": 0,
"agentProcessLeaks": 0,
"peakRssMb": 900,
"missingDependencyErrors": 0,
"pluginLoadFailures": 0
},
"phases": [
{
"id": "provision",
"title": "Provision Missing-Auth Env",
"intent": "Start a disposable OpenClaw gateway without Kova mock or live model credentials.",
"commands": ["ocm start {env} {startSelector} --json"],
"evidence": ["gateway port", "runtime binding", "startup readiness", "no Kova auth setup phase"]
},
{
"id": "missing-auth-agent-turn",
"title": "Missing Auth Agent Turn",
"intent": "Send a local agent message that must fail quickly and clearly because no provider credentials are configured.",
"expectedAgentFailure": true,
"commands": [
"node {kovaRoot}/support/expect-command-fails.mjs -- ocm @{env} -- agent --local --agent main --session-id kova-agent-auth-missing --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --thinking off --timeout 30 --json"
],
"evidence": ["clear auth failure", "no provider request", "process leak snapshot", "role resource samples"]
},
{
"id": "post-auth-failure-health",
"title": "Post-Auth-Failure Gateway Health",
"intent": "Verify the gateway remains responsive after the missing-auth agent failure.",
"commands": ["ocm @{env} -- status", "ocm logs {env} --tail 300 --raw"],
"evidence": ["gateway status", "auth failure logs", "plugin errors", "memory after auth failure"]
}
]
}

18
src/evaluator.mjs
View File

@ -55,6 +55,7 @@ export function evaluateRecord(record, scenario, options = {}) {
providerTurn,
thresholds,
timelineSummary,
authMode: record.auth?.mode ?? null,
expectedProviderMode: scenario.mockProvider?.mode ?? "normal",
providerSimulation: agentProviderSimulation
});
@ -902,6 +903,13 @@ function buildAgentFailureFixerSummary(latencyDiagnosis, providerSimulation, con
likelyOwner: latencyDiagnosis.likelyOwner
});
}
if (latencyDiagnosis?.kind === "auth-failure") {
items.push({
kind: "auth-failure",
summary: "Agent turn failed before provider work because model/provider auth was missing; verify OpenClaw reports credential setup guidance and keeps the gateway usable.",
likelyOwner: latencyDiagnosis.likelyOwner
});
}
if ((containment?.processLeakCount ?? 0) > 0) {
const first = containment.leakedProcesses?.[0];
items.push({
@ -1016,11 +1024,19 @@ function checkTurnThreshold(violations, turn, metric, threshold, message) {
});
}
function diagnoseAgentLatency({ coldAgentTurn, warmAgentTurn, providerTurn, thresholds, timelineSummary, expectedProviderMode = "normal", providerSimulation = null }) {
function diagnoseAgentLatency({ coldAgentTurn, warmAgentTurn, providerTurn, thresholds, timelineSummary, authMode = null, expectedProviderMode = "normal", providerSimulation = null }) {
if (!providerTurn) {
return null;
}
if (providerTurn.missingProviderRequest === true) {
if (providerTurn.expectedFailure === true && ["missing", "broken", "none", "skip"].includes(authMode)) {
return {
kind: "auth-failure",
severity: "info",
summary: `Agent turn failed before provider work because auth mode is ${authMode}.`,
likelyOwner: "agent-runtime/auth"
};
}
return {
kind: "no-provider-request",
severity: "fail",

97
src/selfcheck.mjs
View File

@ -170,6 +170,7 @@ export async function runSelfCheck(flags = {}) {
checks.push(agentColdWarmEvaluationCheck());
checks.push(await concurrentAgentRunnerCheck(tmp));
checks.push(providerConcurrentEvaluationCheck());
checks.push(agentAuthFailureEvaluationCheck());
checks.push(await jsonCommandCheck(
"dry-run-state-lifecycle-json",
`node bin/kova.mjs run --target runtime:stable --scenario fresh-install --state missing-plugin-index --report-dir ${quoteShell(tmp)} --json`,
@ -1602,6 +1603,102 @@ function providerConcurrentEvaluationCheck() {
}
}
function agentAuthFailureEvaluationCheck() {
try {
const command = "node support/expect-command-fails.mjs -- ocm @kova-self-check -- agent --local --agent main --session-id kova-agent-auth-missing --message hi --json";
const record = {
scenario: "agent-auth-missing",
status: "PASS",
auth: { mode: "missing", source: "override:missing", providerId: null },
phases: [
{
id: "missing-auth-agent-turn",
expectedAgentFailure: true,
results: [{
command,
status: 0,
timedOut: false,
startedAt: "2026-04-30T10:00:01.000Z",
startedAtEpochMs: 1777543201000,
finishedAt: "2026-04-30T10:00:02.000Z",
finishedAtEpochMs: 1777543202000,
durationMs: 1000,
stdout: "",
stderr: "missing OpenAI credentials",
processSnapshots: {
leaks: {
schemaVersion: "kova.processLeakSummary.v1",
leakCount: 0,
leakedProcesses: [],
leaksByRole: {}
}
}
}],
metrics: { logs: zeroLogMetrics(), health: { ok: true } }
},
{
id: "post-auth-failure-health",
results: [{
command: "ocm @kova-self-check -- status",
status: 0,
timedOut: false,
durationMs: 100,
stdout: "status ok",
stderr: ""
}],
metrics: { logs: zeroLogMetrics(), health: { ok: true } }
}
],
providerEvidence: {
available: false,
requestCount: 0,
requests: [],
errors: [],
error: "provider request log not found"
},
finalMetrics: {
service: { gatewayState: "running" },
logs: zeroLogMetrics()
}
};
evaluateRecord(record, {
id: "agent-auth-missing",
auth: { mode: "missing" },
agent: { expectedFailure: true },
thresholds: {
agentContainmentHealthFailures: 0,
agentProcessLeaks: 0
}
}, { surface: { thresholds: {} }, targetPlan: { kind: "npm" } });
assertEqual(record.status, "PASS", "agent auth failure scenario status");
assertEqual(record.measurements.agentTurnCount, 1, "auth failure agent turn count");
assertEqual(record.measurements.agentTurns[0].expectedFailureObserved, true, "auth failure observed");
assertEqual(record.measurements.agentLatencyDiagnosis.kind, "auth-failure", "auth failure diagnosis");
assertEqual(record.measurements.agentFailureContainment.gatewayHealthy, true, "auth failure gateway healthy");
assertEqual(
record.measurements.agentFailureFixerSummary.items.some((item) => item.kind === "auth-failure"),
true,
"auth failure fixer evidence"
);
return {
id: "agent-auth-failure-evaluation",
status: "PASS",
command: "evaluate synthetic missing-auth agent failure containment",
durationMs: 0
};
} catch (error) {
return {
id: "agent-auth-failure-evaluation",
status: "FAIL",
command: "evaluate synthetic missing-auth agent failure containment",
durationMs: 0,
message: error.message
};
}
}
function agentColdWarmEvaluationCheck() {
try {
const coldCommand = "ocm @kova -- agent --local --agent main --session-id kova-agent-cold-warm --message hi --json";

39
states/agent-auth-missing.json Normal file
View File

@ -0,0 +1,39 @@
{
"id": "agent-auth-missing",
"title": "Agent Auth Missing",
"objective": "A disposable OpenClaw env with no model/provider credentials so Kova can verify agent turns fail clearly and do not degrade the gateway.",
"tags": [
"agent",
"auth",
"providers",
"failure"
],
"setup": [],
"traits": [
"agent-state",
"provider-pressure",
"missing-auth",
"failure-state"
],
"auth": {
"mode": "missing",
"reason": "This state intentionally bypasses Kova's default mock provider auth to test missing model credentials."
},
"compatibleSurfaces": [
"agent-message"
],
"incompatibleSurfaces": [],
"riskArea": "agent-provider-auth",
"ownerArea": "agent-runtime",
"setupEvidence": [
"run-level auth policy is missing",
"no mock provider is injected"
],
"cleanupGuarantees": [
"disposable env cleanup removes the unauthenticated OpenClaw home"
],
"source": {
"kind": "auth-policy-fixture",
"mode": "missing"
}
}