From 97454f3f99cd680b4bffc849b2b9ce3c0c777df6 Mon Sep 17 00:00:00 2001 From: Shakker Date: Fri, 1 May 2026 08:18:50 +0100 Subject: [PATCH] feat: complete live provider auth attribution --- README.md | 8 +- docs/AGENT_USAGE.md | 6 +- docs/REPORT_SCHEMA.md | 5 + fixtures/provider/mock-requests.jsonl | 2 +- src/auth.mjs | 71 +++++++++++++- src/cli.mjs | 5 +- src/collectors/provider.mjs | 57 +++++++++++ src/report.mjs | 8 +- src/selfcheck.mjs | 136 ++++++++++++++++++++++++++ support/mock-openai-server.mjs | 32 ++++-- 10 files changed, 311 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 3af9cf5..fc6c00e 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,7 @@ codex skills install https://github.com/shakkernerd/ocm/tree/main/skills/ocm-ope node bin/kova.mjs version node bin/kova.mjs setup node bin/kova.mjs setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY +node bin/kova.mjs setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY --fallback-policy external-cli node bin/kova.mjs setup --ci --json node bin/kova.mjs self-check node bin/kova.mjs plan @@ -77,7 +78,9 @@ answers accept either the displayed number or the name, for example `2` or External CLI auth is strict: Kova verifies the selected CLI binary and local auth evidence before setup can pass. `openai + external-cli` uses Codex CLI; `anthropic + external-cli` uses Claude CLI. `custom-openai` should use API-key -or env-only auth. +or env-only auth. External CLI fallback is not automatic; set +`--fallback-policy external-cli` when a live API-key/env-only run may use the +selected local CLI if the live env var is missing. `run` is dry-run by default. It writes Markdown and JSON reports showing the planned OpenClaw scenario. @@ -87,7 +90,8 @@ the scenario/state explicitly tests missing or broken auth. `--auth mock` is the default and uses Kova's deterministic local OpenAI-compatible provider. `--auth live` requires credentials configured through `kova setup`; live results are marked environment-dependent and should be compared separately from mock -baselines. +baselines. Kova's live auth setup patches disposable env config as fixture setup +for runtime validation; it is not proof that OpenClaw onboarding/auth UX passed. `plan --json` is coverage-aware: scenarios map to declared OpenClaw surfaces, surfaces declare process roles and required metrics, and profile coverage gaps diff --git a/docs/AGENT_USAGE.md b/docs/AGENT_USAGE.md index f8638bd..b9e6799 100644 --- a/docs/AGENT_USAGE.md +++ b/docs/AGENT_USAGE.md @@ -66,7 +66,11 @@ Interactive `kova setup` asks provider first and then auth method. Both prompts accept either the displayed number or the name. For `external-cli`, Kova must verify the selected CLI and auth evidence before setup passes. `openai + external-cli` uses Codex CLI; `anthropic + external-cli` -uses Claude CLI. Use API-key or env-only auth for `custom-openai`. +uses Claude CLI. External CLI fallback is only valid when setup explicitly +selected `--fallback-policy external-cli`. Use API-key or env-only auth for +`custom-openai`. +Live auth setup patches disposable env config as fixture setup for runtime +validation; do not cite it as proof that OpenClaw onboarding/auth UX passed. 4. Execute one scenario explicitly: diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index d4a22cb..e8ac09e 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -47,6 +47,8 @@ kova.report.v1 "available": false, "providerId": "openai", "method": "mock", + "fallbackFrom": null, + "fallbackPolicy": "mock", "envVars": ["OPENAI_API_KEY"], "reason": "no live provider configured", "environmentDependent": false @@ -109,6 +111,9 @@ Important fields: - `collectorArtifactDirs`: stable per-record artifact directories used by collectors - `measurements`: evaluated measurements +- `providerEvidence`: provider request timing, route/model/status summaries, + optional token-like usage totals, and whether evidence came from mock-provider + logs or OpenClaw timeline events - `violations`: threshold or behavior violations - `phases`: commands, results, and metrics by phase - `finalMetrics`: service/process snapshot before cleanup diff --git a/fixtures/provider/mock-requests.jsonl b/fixtures/provider/mock-requests.jsonl index 162e7ab..6c9ee71 100644 --- a/fixtures/provider/mock-requests.jsonl +++ b/fixtures/provider/mock-requests.jsonl @@ -1,2 +1,2 @@ {"schemaVersion":"kova.mockProvider.request.v1","requestId":"req_models","receivedAt":"2026-04-30T10:00:00.100Z","receivedAtEpochMs":1777543200100,"respondedAt":"2026-04-30T10:00:00.120Z","respondedAtEpochMs":1777543200120,"durationMs":20,"firstByteAt":"2026-04-30T10:00:00.115Z","firstByteAtEpochMs":1777543200115,"firstByteLatencyMs":15,"firstChunkAt":"2026-04-30T10:00:00.115Z","firstChunkAtEpochMs":1777543200115,"firstChunkLatencyMs":15,"method":"GET","route":"/v1/models","path":"/v1/models","model":null,"stream":false,"status":200,"statusClass":"2xx","bodyBytes":0,"parseError":null} -{"schemaVersion":"kova.mockProvider.request.v1","requestId":"req_agent","receivedAt":"2026-04-30T10:00:06.000Z","receivedAtEpochMs":1777543206000,"respondedAt":"2026-04-30T10:00:06.800Z","respondedAtEpochMs":1777543206800,"durationMs":800,"firstByteAt":"2026-04-30T10:00:06.050Z","firstByteAtEpochMs":1777543206050,"firstByteLatencyMs":50,"firstChunkAt":"2026-04-30T10:00:06.050Z","firstChunkAtEpochMs":1777543206050,"firstChunkLatencyMs":50,"method":"POST","route":"/v1/responses","path":"/v1/responses","model":"gpt-5.5","stream":true,"status":200,"statusClass":"2xx","bodyBytes":42,"parseError":null} +{"schemaVersion":"kova.mockProvider.request.v1","requestId":"req_agent","receivedAt":"2026-04-30T10:00:06.000Z","receivedAtEpochMs":1777543206000,"respondedAt":"2026-04-30T10:00:06.800Z","respondedAtEpochMs":1777543206800,"durationMs":800,"firstByteAt":"2026-04-30T10:00:06.050Z","firstByteAtEpochMs":1777543206050,"firstByteLatencyMs":50,"firstChunkAt":"2026-04-30T10:00:06.050Z","firstChunkAtEpochMs":1777543206050,"firstChunkLatencyMs":50,"method":"POST","route":"/v1/responses","path":"/v1/responses","model":"gpt-5.5","stream":true,"status":200,"usage":{"input_tokens":9,"output_tokens":3,"total_tokens":12},"statusClass":"2xx","bodyBytes":42,"parseError":null} diff --git a/src/auth.mjs b/src/auth.mjs index 356506e..df0e154 100644 --- a/src/auth.mjs +++ b/src/auth.mjs @@ -3,11 +3,16 @@ import { constants } from "node:fs"; import { join } from "node:path"; import { credentialsDir, liveEnvPath, providersPath, repoRoot } from "./paths.mjs"; import { quoteShell } from "./commands.mjs"; -import { externalCliVerificationSummary, verifyExternalCliAuth } from "./external-cli-auth.mjs"; +import { + externalCliVerificationSummary, + resolveExternalCliName, + verifyExternalCliAuth +} from "./external-cli-auth.mjs"; export const authModes = ["mock", "live", "skip"]; export const credentialMethods = ["mock", "api-key", "env-only", "external-cli", "oauth", "skip"]; export const authOverrideModes = ["default", "mock", "live", "skip", "missing", "broken", "none"]; +export const fallbackPolicies = ["mock", "external-cli", "none"]; const defaultProviderId = "openai"; const mockApiKey = "kova-mock-key"; @@ -35,6 +40,7 @@ export async function configureCredentialProvider(options = {}) { const metadata = await readProvidersMetadata(); const envVar = options.envVar ?? defaultEnvVarForProvider(providerId); + const fallbackPolicy = normalizeFallbackPolicy(options.fallbackPolicy ?? "mock"); metadata.defaultProvider = providerId; metadata.providers = { ...(metadata.providers ?? {}), @@ -43,7 +49,7 @@ export async function configureCredentialProvider(options = {}) { method, envVars: method === "api-key" || method === "env-only" ? [envVar] : [], externalCli: method === "external-cli" ? (options.externalCli ?? providerId) : null, - fallbackPolicy: options.fallbackPolicy ?? "mock", + fallbackPolicy, configuredAt: new Date().toISOString() } }; @@ -120,7 +126,10 @@ export function scenarioAuthPolicy(context, scenario, state) { providerId, source: live.method, externalCli: live.externalCli ?? null, + fallbackFrom: live.fallbackFrom ?? null, + fallbackPolicy: live.fallbackPolicy ?? null, setup: true, + setupKind: "fixture-config-patch", commandEnv: env, redactionValues: [...(context.auth?.redactionValues ?? []), ...secretValues(env)], summary: authDisplay({ @@ -128,7 +137,10 @@ export function scenarioAuthPolicy(context, scenario, state) { providerId, source: live.method, externalCli: live.externalCli ?? null, + fallbackFrom: live.fallbackFrom ?? null, + fallbackPolicy: live.fallbackPolicy ?? null, setup: true, + setupKind: "fixture-config-patch", envVars: live.envVars }) }; @@ -187,9 +199,9 @@ export function buildAuthSetupPhase(authPolicy, envName, artifactDir) { return { id: "auth-setup", title: "Auth Setup", - intent: "Configure the disposable OpenClaw env with the selected live provider auth.", + intent: "Patch the disposable OpenClaw env with fixture live auth config; this proves runtime behavior, not OpenClaw onboarding/auth UX.", commands: [configureLiveAuthCommand(authPolicy, envName)], - evidence: ["OpenClaw config references live auth env vars", "live auth env vars available to OpenClaw runtime"] + evidence: ["fixture auth config applied", "OpenClaw config references live auth env vars or selected external CLI", "live auth is environment-dependent"] }; } @@ -214,7 +226,10 @@ export function authDisplay(policy) { providerId: policy.providerId ?? null, source: policy.source, externalCli: policy.externalCli ?? null, + fallbackFrom: policy.fallbackFrom ?? null, + fallbackPolicy: policy.fallbackPolicy ?? null, setup: policy.setup === true, + setupKind: policy.setupKind ?? null, deterministic: policy.mode === "mock", environmentDependent: policy.mode === "live", envVars: policy.envVars ?? [], @@ -233,6 +248,8 @@ export function authReportSummary(authContext) { providerId: authContext.live.providerId, method: authContext.live.method, externalCli: authContext.live.externalCli ?? null, + fallbackFrom: authContext.live.fallbackFrom ?? null, + fallbackPolicy: authContext.live.fallbackPolicy ?? null, verification: authContext.live.verification ?? null, envVars: authContext.live.envVars, reason: authContext.live.reason, @@ -301,6 +318,9 @@ function validateProvidersMetadata(metadata) { if (!credentialMethods.includes(provider.method)) { throw new Error(`providers.${id}.method must be one of ${credentialMethods.join(", ")}`); } + if (provider.fallbackPolicy !== undefined && !fallbackPolicies.includes(provider.fallbackPolicy)) { + throw new Error(`providers.${id}.fallbackPolicy must be one of ${fallbackPolicies.join(", ")}`); + } if (provider.envVars !== undefined && !Array.isArray(provider.envVars)) { throw new Error(`providers.${id}.envVars must be an array`); } @@ -379,6 +399,7 @@ function liveCredentialStatus(store) { available: false, providerId: provider.id, method: provider.method, + fallbackPolicy: provider.fallbackPolicy ?? "mock", envVars, reason: `missing env var(s): ${missing.join(", ")}` }; @@ -389,6 +410,7 @@ function liveCredentialStatus(store) { providerId: provider.id, method: provider.method, externalCli: provider.externalCli ?? null, + fallbackPolicy: provider.fallbackPolicy ?? "mock", envVars, reason: "configured" }; @@ -398,12 +420,35 @@ function liveCredentialStatus(store) { providerId: defaultId, method: providers[defaultId]?.method ?? "mock", externalCli: providers[defaultId]?.externalCli ?? null, + fallbackPolicy: providers[defaultId]?.fallbackPolicy ?? "mock", envVars: providers[defaultId]?.envVars ?? [], reason: "no live provider configured" }; } async function verifyLiveCredentialStatus(status) { + if (status.available === false && status.fallbackPolicy === "external-cli") { + try { + const externalCli = resolveExternalCliName(status.providerId); + const verification = await verifyExternalCliAuth(externalCli); + return { + ...status, + available: verification.verified, + method: "external-cli", + externalCli, + fallbackFrom: status.method, + envVars: [], + reason: verification.verified ? "configured via external-cli fallback" : `external-cli ${externalCli} is not usable: ${verification.reason}`, + verification: externalCliVerificationSummary(verification) + }; + } catch (error) { + return { + ...status, + available: false, + reason: `${status.reason}; external-cli fallback unavailable: ${error.message}` + }; + } + } if (status.method !== "external-cli") { return status; } @@ -454,6 +499,24 @@ function normalizeAuthOverride(value) { return mode; } +function normalizeFallbackPolicy(value) { + const normalized = String(value ?? "mock").trim().toLowerCase().replaceAll("_", "-"); + const aliases = { + mock: "mock", + default: "mock", + "external-cli": "external-cli", + external: "external-cli", + cli: "external-cli", + none: "none", + skip: "none", + disabled: "none" + }; + if (aliases[normalized]) { + return aliases[normalized]; + } + throw new Error(`fallbackPolicy must be one of ${fallbackPolicies.join(", ")}`); +} + function mockDir(artifactDir) { return join(artifactDir, "mock-openai"); } diff --git a/src/cli.mjs b/src/cli.mjs index 11dbe3e..a9b7cd8 100644 --- a/src/cli.mjs +++ b/src/cli.mjs @@ -35,8 +35,8 @@ export function printHelp() { Usage: kova version [--json] kova --version - kova setup [--ci|--non-interactive] [--auth ] [--provider ] [--env-var ] [--value ] [--json] - kova setup auth [--provider ] [--method ] [--env-var ] [--value ] [--json] + kova setup [--ci|--non-interactive] [--auth ] [--provider ] [--env-var ] [--value ] [--fallback-policy ] [--json] + kova setup auth [--provider ] [--method ] [--env-var ] [--value ] [--fallback-policy ] [--json] kova self-check [--json] kova plan [--scenario ] [--json] kova run --target [--from ] [--scenario ] [--state ] [--auth ] [--repeat ] [--baseline [path]] [--save-baseline [path]] [--regression-thresholds ] [--report-dir ] [--health-samples ] [--readiness-interval-ms ] [--resource-sample-interval-ms ] [--deep-profile] [--node-profile] [--heap-snapshot] [--profile-on-failure] [--execute] [--keep-env] [--retain-on-failure] [--json] @@ -65,6 +65,7 @@ Notes: --auth defaults to mock so every disposable env has deliberate model auth unless a scenario opts out. setup provider/auth choices accept either numbers from the prompt or names such as openai, anthropic, env-only, api-key. external-cli setup derives Codex for OpenAI and Claude CLI for Anthropic, then verifies the CLI and auth evidence. + external CLI fallback is only used when explicitly configured with --fallback-policy external-cli. --baseline compares executed aggregates against a Kova baseline store; without a path it uses the default store. --save-baseline writes executed aggregates into the selected baseline store. --deep-profile enables Node CPU/heap/trace profiling, OpenClaw timeline envs, diff --git a/src/collectors/provider.mjs b/src/collectors/provider.mjs index a321e3d..259f990 100644 --- a/src/collectors/provider.mjs +++ b/src/collectors/provider.mjs @@ -33,6 +33,7 @@ export async function collectProviderEvidence(artifactDir, options = {}) { routes: [], models: [], statuses: [], + usage: null, errors: [], requests: [], artifacts: [], @@ -146,6 +147,7 @@ export function parseProviderRequestLog(text) { outcomes: summarizeBy(requests, "outcome"), errorClasses: summarizeBy(requests, "errorClass"), statuses: summarizeBy(requests, "status"), + usage: summarizeUsage(requests), errors: [...parseErrors, ...requestErrors(requests)], requests }; @@ -226,6 +228,7 @@ export function computeProviderTurnAttribution(result, providerEvidence) { modes: [], outcomes: [], errorClasses: [], + usage: null, errors: [], providerDominates: null, preProviderDominates: null, @@ -263,6 +266,7 @@ export function computeProviderTurnAttribution(result, providerEvidence) { modes: summarizeBy(requests, "mode"), outcomes: summarizeBy(requests, "outcome"), errorClasses: summarizeBy(requests, "errorClass"), + usage: summarizeUsage(requests), errors: requestErrors(requests), providerDominates: dominanceRatio(Math.max(0, lastProviderResponseAt - firstProviderRequestAt), Math.max(0, commandFinishedAt - commandStartedAt)), preProviderDominates: dominanceRatio(Math.max(0, firstProviderRequestAt - commandStartedAt), Math.max(0, commandFinishedAt - commandStartedAt)), @@ -322,6 +326,7 @@ function normalizeTimelineProviderRequest(event, line) { stream: typeof event.stream === "boolean" ? event.stream : null, status: numberOrNull(event.status), statusClass: typeof event.status === "number" ? `${Math.floor(event.status / 100)}xx` : null, + usage: normalizeUsage(event.usage ?? event.attributes?.usage), bodyBytes: null, parseError: null }; @@ -358,6 +363,7 @@ function summarizeProviderRequests(requests) { outcomes: summarizeBy(requests, "outcome"), errorClasses: summarizeBy(requests, "errorClass"), statuses: summarizeBy(requests, "status"), + usage: summarizeUsage(requests), errors: requestErrors(requests) }; } @@ -394,6 +400,7 @@ function normalizeProviderRequest(raw, line) { model: raw.model ?? modelFromBody(raw.body), stream: typeof raw.stream === "boolean" ? raw.stream : streamFromBody(raw.body), status: numberOrNull(raw.status), + usage: normalizeUsage(raw.usage), statusClass: raw.statusClass ?? (typeof raw.status === "number" ? `${Math.floor(raw.status / 100)}xx` : null), bodyBytes: numberOrNull(raw.bodyBytes) ?? (typeof raw.body === "string" ? Buffer.byteLength(raw.body) : null), parseError: raw.parseError ?? null @@ -458,6 +465,56 @@ function requestErrors(requests) { return errors; } +function summarizeUsage(requests) { + const usages = requests + .map((request) => request.usage) + .filter((usage) => usage && typeof usage === "object"); + if (usages.length === 0) { + return { + schemaVersion: "kova.providerUsageSummary.v1", + available: false, + requestCount: requests.length, + requestsWithUsage: 0, + inputTokens: null, + outputTokens: null, + totalTokens: null + }; + } + return { + schemaVersion: "kova.providerUsageSummary.v1", + available: true, + requestCount: requests.length, + requestsWithUsage: usages.length, + inputTokens: sumNullable(usages.map((usage) => usage.inputTokens)), + outputTokens: sumNullable(usages.map((usage) => usage.outputTokens)), + totalTokens: sumNullable(usages.map((usage) => usage.totalTokens)) + }; +} + +function normalizeUsage(value) { + if (!value || typeof value !== "object" || Array.isArray(value)) { + return null; + } + const inputTokens = numberOrNull(value.input_tokens ?? value.inputTokens ?? value.prompt_tokens ?? value.promptTokens); + const outputTokens = numberOrNull(value.output_tokens ?? value.outputTokens ?? value.completion_tokens ?? value.completionTokens); + const totalTokens = numberOrNull(value.total_tokens ?? value.totalTokens) ?? + (typeof inputTokens === "number" && typeof outputTokens === "number" ? inputTokens + outputTokens : null); + if (![inputTokens, outputTokens, totalTokens].some((item) => typeof item === "number")) { + return null; + } + return { + schemaVersion: "kova.providerUsage.v1", + inputTokens, + outputTokens, + totalTokens + }; +} + +function sumNullable(values) { + const numbers = values.filter((value) => typeof value === "number"); + return numbers.length > 0 ? numbers.reduce((sum, value) => sum + value, 0) : null; +} + function modelFromBody(body) { const parsed = parseBody(body); return typeof parsed?.model === "string" ? parsed.model : null; diff --git a/src/report.mjs b/src/report.mjs index d61bb72..86f97ef 100644 --- a/src/report.mjs +++ b/src/report.mjs @@ -74,6 +74,9 @@ export function renderMarkdownReport(report) { } if (record.auth) { lines.push(`- Auth: ${record.auth.mode} (${record.auth.source}; provider ${record.auth.providerId ?? "none"})`); + if (record.auth.fallbackFrom) { + lines.push(`- Auth fallback: ${record.auth.fallbackFrom} -> ${record.auth.source}`); + } if (record.auth.environmentDependent) { lines.push("- Live provider lane: environment-dependent; compare separately from deterministic mock baselines."); } @@ -145,7 +148,10 @@ export function renderMarkdownReport(report) { if (record.measurements.agentProviderAttribution) { lines.push(`- Provider evidence: ${record.measurements.agentProviderRequestCount ?? 0} request(s); provider work ${record.measurements.agentProviderFinalMs ?? "unknown"} ms; pre-provider ${record.measurements.agentPreProviderMs ?? "unknown"} ms; post-provider ${record.measurements.agentPostProviderMs ?? "unknown"} ms`); } else if (record.providerEvidence?.available) { - lines.push(`- Provider evidence: ${record.providerEvidence.requestCount ?? 0} request(s); provider duration ${record.providerEvidence.providerDurationMs ?? "unknown"} ms`); + const usage = record.providerEvidence.usage?.available + ? `; tokens ${record.providerEvidence.usage.totalTokens ?? "unknown"}` + : ""; + lines.push(`- Provider evidence: ${record.providerEvidence.requestCount ?? 0} request(s); provider duration ${record.providerEvidence.providerDurationMs ?? "unknown"} ms${usage}`); } else if (record.auth?.mode === "live") { lines.push(`- Provider evidence: unavailable for live lane (${record.providerEvidence?.error ?? "no provider events captured"})`); } diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index 53b66f6..1dbbd15 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -63,8 +63,11 @@ export async function runSelfCheck(flags = {}) { checks.push(await interactiveSetupChoiceCheck(tmp)); checks.push(await externalCliSetupCheck(tmp)); checks.push(await externalCliOpenClawConfigCheck(tmp)); + checks.push(await anthropicApiKeyOpenClawConfigCheck(tmp)); + checks.push(await claudeCliOpenClawConfigCheck(tmp)); checks.push(await liveApiKeyExecutionCheck(tmp)); checks.push(await liveExternalCliDryRunCheck(tmp)); + checks.push(await liveExternalCliFallbackCheck(tmp)); checks.push(await failingCommandCheck( "setup-custom-provider-rejects-external-cli", `KOVA_HOME=${quoteShell(join(tmp, "custom-external-cli-home"))} node bin/kova.mjs setup --non-interactive --provider custom-openai --auth external-cli --json`, @@ -840,6 +843,8 @@ async function providerEvidenceParserCheck() { assertEqual(attribution.preProviderMs, 5000, "pre-provider latency"); assertEqual(attribution.providerFinalMs, 800, "provider final latency"); assertEqual(attribution.postProviderMs, 200, "post-provider latency"); + assertEqual(evidence.usage?.available, true, "provider usage availability"); + assertEqual(evidence.usage?.totalTokens, 12, "provider usage total tokens"); return { id: "provider-evidence-parser", status: "PASS", @@ -1011,6 +1016,71 @@ async function liveExternalCliDryRunCheck(tmp) { } } +async function liveExternalCliFallbackCheck(tmp) { + const home = join(tmp, "live-external-cli-fallback-home"); + const kovaHome = join(tmp, "live-external-cli-fallback-kova-home"); + const fakeBin = join(tmp, "live-external-cli-fallback-bin"); + const reportDir = join(tmp, "live-external-cli-fallback-report"); + await mkdir(join(home, ".codex"), { recursive: true }); + await mkdir(join(kovaHome, "credentials"), { recursive: true }); + await mkdir(fakeBin, { recursive: true }); + await writeFile(join(home, ".codex", "auth.json"), "{\"tokens\":{\"access_token\":\"redacted\"}}\n", "utf8"); + await writeFile(join(fakeBin, "codex"), "#!/bin/sh\necho codex-selfcheck\n", "utf8"); + await chmod(join(fakeBin, "codex"), 0o755); + await writeFile(join(kovaHome, "credentials", "providers.json"), `${JSON.stringify({ + schemaVersion: "kova.credentials.providers.v1", + defaultProvider: "openai", + providers: { + openai: { + id: "openai", + method: "env-only", + envVars: ["OPENAI_API_KEY"], + externalCli: null, + fallbackPolicy: "external-cli", + configuredAt: new Date().toISOString() + } + } + }, null, 2)}\n`, "utf8"); + await writeFile(join(kovaHome, "credentials", "live.env"), "", { encoding: "utf8", mode: 0o600 }); + + const command = [ + `HOME=${quoteShell(home)}`, + `PATH=${quoteShell(`${fakeBin}:${process.env.PATH}`)}`, + `KOVA_HOME=${quoteShell(kovaHome)}`, + `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --auth live --report-dir ${quoteShell(reportDir)} --json` + ].join(" "); + const result = await runCommand(command, { timeoutMs: 30000, maxOutputChars: 1000000 }); + + try { + if (result.status !== 0) { + throw new Error(result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`); + } + const receipt = JSON.parse(result.stdout); + const report = JSON.parse(await readFile(receipt.jsonPath, "utf8")); + const record = report.records?.[0]; + assertEqual(report.auth?.live?.method, "external-cli", "fallback live method"); + assertEqual(report.auth?.live?.fallbackFrom, "env-only", "fallback source method"); + assertEqual(report.auth?.live?.fallbackPolicy, "external-cli", "fallback policy"); + assertEqual(record?.auth?.source, "external-cli", "record fallback source"); + assertEqual(record?.auth?.fallbackFrom, "env-only", "record fallback from"); + assertEqual(record?.auth?.externalCli, "codex", "record fallback CLI"); + return { + id: "live-external-cli-fallback", + status: "PASS", + command, + durationMs: result.durationMs + }; + } catch (error) { + return { + id: "live-external-cli-fallback", + status: "FAIL", + command, + durationMs: result.durationMs, + message: error.message + }; + } +} + function fakeOcmScript() { return `#!/bin/sh printf '%s\\n' "$*" >> "$KOVA_MOCK_OCM_LOG" @@ -2310,6 +2380,72 @@ async function externalCliOpenClawConfigCheck(tmp) { } } +async function anthropicApiKeyOpenClawConfigCheck(tmp) { + const home = join(tmp, "anthropic-api-key-config-home"); + const command = [ + `OPENCLAW_HOME=${quoteShell(home)}`, + "node support/configure-openclaw-live-auth.mjs --provider anthropic --env-var ANTHROPIC_API_KEY" + ].join(" "); + const result = await runCommand(command, { timeoutMs: 30000, maxOutputChars: 1000000 }); + try { + if (result.status !== 0) { + throw new Error(result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`); + } + const config = JSON.parse(await readFile(join(home, ".openclaw", "openclaw.json"), "utf8")); + assertEqual(config.models?.providers?.anthropic?.apiKey?.id, "ANTHROPIC_API_KEY", "anthropic env ref"); + assertEqual(config.agents?.defaults?.model?.primary, "anthropic/claude-sonnet-4-5", "anthropic default model"); + return { + id: "anthropic-api-key-openclaw-config", + status: "PASS", + command, + durationMs: result.durationMs + }; + } catch (error) { + return { + id: "anthropic-api-key-openclaw-config", + status: "FAIL", + command, + durationMs: result.durationMs, + message: error.message + }; + } +} + +async function claudeCliOpenClawConfigCheck(tmp) { + const home = join(tmp, "claude-cli-config-home"); + const command = [ + `OPENCLAW_HOME=${quoteShell(home)}`, + "node support/configure-openclaw-live-auth.mjs --provider anthropic --auth-method external-cli --external-cli claude" + ].join(" "); + const result = await runCommand(command, { timeoutMs: 30000, maxOutputChars: 1000000 }); + try { + if (result.status !== 0) { + throw new Error(result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`); + } + const config = JSON.parse(await readFile(join(home, ".openclaw", "openclaw.json"), "utf8")); + assertEqual(config.agents?.defaults?.model?.primary, "anthropic/claude-sonnet-4-5", "claude cli model ref"); + assertEqual(config.agents?.defaults?.agentRuntime?.id, "claude-cli", "claude cli runtime id"); + assertEqual(config.agents?.defaults?.agentRuntime?.fallback, "none", "claude cli runtime fallback"); + if (config.models?.providers?.anthropic?.apiKey !== undefined) { + throw new Error("Claude CLI config must not write env apiKey config"); + } + return { + id: "claude-cli-openclaw-config", + status: "PASS", + command, + durationMs: result.durationMs + }; + } catch (error) { + return { + id: "claude-cli-openclaw-config", + status: "FAIL", + command, + durationMs: result.durationMs, + message: error.message + }; + } +} + async function externalCliRunAuthVerificationCheck(tmp) { const home = join(tmp, "stale-external-cli-home"); const kovaHome = join(tmp, "stale-external-cli-kova-home"); diff --git a/support/mock-openai-server.mjs b/support/mock-openai-server.mjs index e245f2d..125288c 100644 --- a/support/mock-openai-server.mjs +++ b/support/mock-openai-server.mjs @@ -84,6 +84,7 @@ async function writeStreamingStall(res, stream, call) { } function responseEvents(text) { + const usage = mockUsage(); return [ { type: "response.output_item.added", @@ -103,17 +104,25 @@ function responseEvents(text) { type: "response.completed", response: { status: "completed", - usage: { - input_tokens: 9, - output_tokens: 3, - total_tokens: 12, - input_tokens_details: { cached_tokens: 0 } - } + usage } } ]; } +function mockUsage() { + return { + input_tokens: 9, + output_tokens: 3, + total_tokens: 12, + input_tokens_details: { cached_tokens: 0 } + }; +} + +function chatUsage() { + return { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }; +} + function writeChatCompletion(res, stream) { if (stream) { writeSse(res, [ @@ -135,7 +144,7 @@ function writeChatCompletion(res, stream) { id: "chatcmpl_kova", object: "chat.completion", choices: [{ index: 0, message: { role: "assistant", content: marker }, finish_reason: "stop" }], - usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 } + usage: chatUsage() }); } @@ -169,6 +178,7 @@ const server = http.createServer(async (req, res) => { let loggable = false; let stream = false; let model = null; + let usage = null; let behavior = { mode: providerMode, outcome: null, @@ -212,6 +222,7 @@ const server = http.createServer(async (req, res) => { model, stream, status, + usage, statusClass: typeof status === "number" ? `${Math.floor(status / 100)}xx` : null, bodyBytes: Buffer.byteLength(bodyText), parseError @@ -259,6 +270,7 @@ const server = http.createServer(async (req, res) => { return; } if (body.stream === false) { + usage = mockUsage(); writeJson(res, 200, { id: "resp_kova", object: "response", @@ -272,10 +284,11 @@ const server = http.createServer(async (req, res) => { content: [{ type: "output_text", text: marker, annotations: [] }] } ], - usage: { input_tokens: 9, output_tokens: 3, total_tokens: 12 } + usage }); return; } + usage = mockUsage(); writeSse(res, responseEvents(marker)); return; } @@ -286,6 +299,9 @@ const server = http.createServer(async (req, res) => { if (await maybeWriteFailureBehavior(res, behavior, body.stream !== false)) { return; } + if (body.stream === false) { + usage = chatUsage(); + } writeChatCompletion(res, body.stream !== false); return; }