feat: complete live provider auth attribution
This commit is contained in:
Shakker
parent
8a0bf5d080
commit
97454f3f99
@ -44,6 +44,7 @@ codex skills install https://github.com/shakkernerd/ocm/tree/main/skills/ocm-ope
|
||||
node bin/kova.mjs version
|
||||
node bin/kova.mjs setup
|
||||
node bin/kova.mjs setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY
|
||||
node bin/kova.mjs setup --non-interactive --auth env-only --provider openai --env-var OPENAI_API_KEY --fallback-policy external-cli
|
||||
node bin/kova.mjs setup --ci --json
|
||||
node bin/kova.mjs self-check
|
||||
node bin/kova.mjs plan
|
||||
@ -77,7 +78,9 @@ answers accept either the displayed number or the name, for example `2` or
|
||||
External CLI auth is strict: Kova verifies the selected CLI binary and local
|
||||
auth evidence before setup can pass. `openai + external-cli` uses Codex CLI;
|
||||
`anthropic + external-cli` uses Claude CLI. `custom-openai` should use API-key
|
||||
or env-only auth.
|
||||
or env-only auth. External CLI fallback is not automatic; set
|
||||
`--fallback-policy external-cli` when a live API-key/env-only run may use the
|
||||
selected local CLI if the live env var is missing.
|
||||
|
||||
`run` is dry-run by default. It writes Markdown and JSON reports showing the
|
||||
planned OpenClaw scenario.
|
||||
@ -87,7 +90,8 @@ the scenario/state explicitly tests missing or broken auth. `--auth mock` is the
|
||||
default and uses Kova's deterministic local OpenAI-compatible provider.
|
||||
`--auth live` requires credentials configured through `kova setup`; live results
|
||||
are marked environment-dependent and should be compared separately from mock
|
||||
baselines.
|
||||
baselines. Kova's live auth setup patches disposable env config as fixture setup
|
||||
for runtime validation; it is not proof that OpenClaw onboarding/auth UX passed.
|
||||
|
||||
`plan --json` is coverage-aware: scenarios map to declared OpenClaw surfaces,
|
||||
surfaces declare process roles and required metrics, and profile coverage gaps
|
||||
|
||||
@ -66,7 +66,11 @@ Interactive `kova setup` asks provider first and then auth method. Both prompts
|
||||
accept either the displayed number or the name.
|
||||
For `external-cli`, Kova must verify the selected CLI and auth evidence before
|
||||
setup passes. `openai + external-cli` uses Codex CLI; `anthropic + external-cli`
|
||||
uses Claude CLI. Use API-key or env-only auth for `custom-openai`.
|
||||
uses Claude CLI. External CLI fallback is only valid when setup explicitly
|
||||
selected `--fallback-policy external-cli`. Use API-key or env-only auth for
|
||||
`custom-openai`.
|
||||
Live auth setup patches disposable env config as fixture setup for runtime
|
||||
validation; do not cite it as proof that OpenClaw onboarding/auth UX passed.
|
||||
|
||||
4. Execute one scenario explicitly:
|
||||
|
||||
|
||||
@ -47,6 +47,8 @@ kova.report.v1
|
||||
"available": false,
|
||||
"providerId": "openai",
|
||||
"method": "mock",
|
||||
"fallbackFrom": null,
|
||||
"fallbackPolicy": "mock",
|
||||
"envVars": ["OPENAI_API_KEY"],
|
||||
"reason": "no live provider configured",
|
||||
"environmentDependent": false
|
||||
@ -109,6 +111,9 @@ Important fields:
|
||||
- `collectorArtifactDirs`: stable per-record artifact directories used by
|
||||
collectors
|
||||
- `measurements`: evaluated measurements
|
||||
- `providerEvidence`: provider request timing, route/model/status summaries,
|
||||
optional token-like usage totals, and whether evidence came from mock-provider
|
||||
logs or OpenClaw timeline events
|
||||
- `violations`: threshold or behavior violations
|
||||
- `phases`: commands, results, and metrics by phase
|
||||
- `finalMetrics`: service/process snapshot before cleanup
|
||||
|
||||
@ -1,2 +1,2 @@
|
||||
{"schemaVersion":"kova.mockProvider.request.v1","requestId":"req_models","receivedAt":"2026-04-30T10:00:00.100Z","receivedAtEpochMs":1777543200100,"respondedAt":"2026-04-30T10:00:00.120Z","respondedAtEpochMs":1777543200120,"durationMs":20,"firstByteAt":"2026-04-30T10:00:00.115Z","firstByteAtEpochMs":1777543200115,"firstByteLatencyMs":15,"firstChunkAt":"2026-04-30T10:00:00.115Z","firstChunkAtEpochMs":1777543200115,"firstChunkLatencyMs":15,"method":"GET","route":"/v1/models","path":"/v1/models","model":null,"stream":false,"status":200,"statusClass":"2xx","bodyBytes":0,"parseError":null}
|
||||
{"schemaVersion":"kova.mockProvider.request.v1","requestId":"req_agent","receivedAt":"2026-04-30T10:00:06.000Z","receivedAtEpochMs":1777543206000,"respondedAt":"2026-04-30T10:00:06.800Z","respondedAtEpochMs":1777543206800,"durationMs":800,"firstByteAt":"2026-04-30T10:00:06.050Z","firstByteAtEpochMs":1777543206050,"firstByteLatencyMs":50,"firstChunkAt":"2026-04-30T10:00:06.050Z","firstChunkAtEpochMs":1777543206050,"firstChunkLatencyMs":50,"method":"POST","route":"/v1/responses","path":"/v1/responses","model":"gpt-5.5","stream":true,"status":200,"statusClass":"2xx","bodyBytes":42,"parseError":null}
|
||||
{"schemaVersion":"kova.mockProvider.request.v1","requestId":"req_agent","receivedAt":"2026-04-30T10:00:06.000Z","receivedAtEpochMs":1777543206000,"respondedAt":"2026-04-30T10:00:06.800Z","respondedAtEpochMs":1777543206800,"durationMs":800,"firstByteAt":"2026-04-30T10:00:06.050Z","firstByteAtEpochMs":1777543206050,"firstByteLatencyMs":50,"firstChunkAt":"2026-04-30T10:00:06.050Z","firstChunkAtEpochMs":1777543206050,"firstChunkLatencyMs":50,"method":"POST","route":"/v1/responses","path":"/v1/responses","model":"gpt-5.5","stream":true,"status":200,"usage":{"input_tokens":9,"output_tokens":3,"total_tokens":12},"statusClass":"2xx","bodyBytes":42,"parseError":null}
|
||||
|
||||
71
src/auth.mjs
71
src/auth.mjs
@ -3,11 +3,16 @@ import { constants } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { credentialsDir, liveEnvPath, providersPath, repoRoot } from "./paths.mjs";
|
||||
import { quoteShell } from "./commands.mjs";
|
||||
import { externalCliVerificationSummary, verifyExternalCliAuth } from "./external-cli-auth.mjs";
|
||||
import {
|
||||
externalCliVerificationSummary,
|
||||
resolveExternalCliName,
|
||||
verifyExternalCliAuth
|
||||
} from "./external-cli-auth.mjs";
|
||||
|
||||
export const authModes = ["mock", "live", "skip"];
|
||||
export const credentialMethods = ["mock", "api-key", "env-only", "external-cli", "oauth", "skip"];
|
||||
export const authOverrideModes = ["default", "mock", "live", "skip", "missing", "broken", "none"];
|
||||
export const fallbackPolicies = ["mock", "external-cli", "none"];
|
||||
|
||||
const defaultProviderId = "openai";
|
||||
const mockApiKey = "kova-mock-key";
|
||||
@ -35,6 +40,7 @@ export async function configureCredentialProvider(options = {}) {
|
||||
|
||||
const metadata = await readProvidersMetadata();
|
||||
const envVar = options.envVar ?? defaultEnvVarForProvider(providerId);
|
||||
const fallbackPolicy = normalizeFallbackPolicy(options.fallbackPolicy ?? "mock");
|
||||
metadata.defaultProvider = providerId;
|
||||
metadata.providers = {
|
||||
...(metadata.providers ?? {}),
|
||||
@ -43,7 +49,7 @@ export async function configureCredentialProvider(options = {}) {
|
||||
method,
|
||||
envVars: method === "api-key" || method === "env-only" ? [envVar] : [],
|
||||
externalCli: method === "external-cli" ? (options.externalCli ?? providerId) : null,
|
||||
fallbackPolicy: options.fallbackPolicy ?? "mock",
|
||||
fallbackPolicy,
|
||||
configuredAt: new Date().toISOString()
|
||||
}
|
||||
};
|
||||
@ -120,7 +126,10 @@ export function scenarioAuthPolicy(context, scenario, state) {
|
||||
providerId,
|
||||
source: live.method,
|
||||
externalCli: live.externalCli ?? null,
|
||||
fallbackFrom: live.fallbackFrom ?? null,
|
||||
fallbackPolicy: live.fallbackPolicy ?? null,
|
||||
setup: true,
|
||||
setupKind: "fixture-config-patch",
|
||||
commandEnv: env,
|
||||
redactionValues: [...(context.auth?.redactionValues ?? []), ...secretValues(env)],
|
||||
summary: authDisplay({
|
||||
@ -128,7 +137,10 @@ export function scenarioAuthPolicy(context, scenario, state) {
|
||||
providerId,
|
||||
source: live.method,
|
||||
externalCli: live.externalCli ?? null,
|
||||
fallbackFrom: live.fallbackFrom ?? null,
|
||||
fallbackPolicy: live.fallbackPolicy ?? null,
|
||||
setup: true,
|
||||
setupKind: "fixture-config-patch",
|
||||
envVars: live.envVars
|
||||
})
|
||||
};
|
||||
@ -187,9 +199,9 @@ export function buildAuthSetupPhase(authPolicy, envName, artifactDir) {
|
||||
return {
|
||||
id: "auth-setup",
|
||||
title: "Auth Setup",
|
||||
intent: "Configure the disposable OpenClaw env with the selected live provider auth.",
|
||||
intent: "Patch the disposable OpenClaw env with fixture live auth config; this proves runtime behavior, not OpenClaw onboarding/auth UX.",
|
||||
commands: [configureLiveAuthCommand(authPolicy, envName)],
|
||||
evidence: ["OpenClaw config references live auth env vars", "live auth env vars available to OpenClaw runtime"]
|
||||
evidence: ["fixture auth config applied", "OpenClaw config references live auth env vars or selected external CLI", "live auth is environment-dependent"]
|
||||
};
|
||||
}
|
||||
|
||||
@ -214,7 +226,10 @@ export function authDisplay(policy) {
|
||||
providerId: policy.providerId ?? null,
|
||||
source: policy.source,
|
||||
externalCli: policy.externalCli ?? null,
|
||||
fallbackFrom: policy.fallbackFrom ?? null,
|
||||
fallbackPolicy: policy.fallbackPolicy ?? null,
|
||||
setup: policy.setup === true,
|
||||
setupKind: policy.setupKind ?? null,
|
||||
deterministic: policy.mode === "mock",
|
||||
environmentDependent: policy.mode === "live",
|
||||
envVars: policy.envVars ?? [],
|
||||
@ -233,6 +248,8 @@ export function authReportSummary(authContext) {
|
||||
providerId: authContext.live.providerId,
|
||||
method: authContext.live.method,
|
||||
externalCli: authContext.live.externalCli ?? null,
|
||||
fallbackFrom: authContext.live.fallbackFrom ?? null,
|
||||
fallbackPolicy: authContext.live.fallbackPolicy ?? null,
|
||||
verification: authContext.live.verification ?? null,
|
||||
envVars: authContext.live.envVars,
|
||||
reason: authContext.live.reason,
|
||||
@ -301,6 +318,9 @@ function validateProvidersMetadata(metadata) {
|
||||
if (!credentialMethods.includes(provider.method)) {
|
||||
throw new Error(`providers.${id}.method must be one of ${credentialMethods.join(", ")}`);
|
||||
}
|
||||
if (provider.fallbackPolicy !== undefined && !fallbackPolicies.includes(provider.fallbackPolicy)) {
|
||||
throw new Error(`providers.${id}.fallbackPolicy must be one of ${fallbackPolicies.join(", ")}`);
|
||||
}
|
||||
if (provider.envVars !== undefined && !Array.isArray(provider.envVars)) {
|
||||
throw new Error(`providers.${id}.envVars must be an array`);
|
||||
}
|
||||
@ -379,6 +399,7 @@ function liveCredentialStatus(store) {
|
||||
available: false,
|
||||
providerId: provider.id,
|
||||
method: provider.method,
|
||||
fallbackPolicy: provider.fallbackPolicy ?? "mock",
|
||||
envVars,
|
||||
reason: `missing env var(s): ${missing.join(", ")}`
|
||||
};
|
||||
@ -389,6 +410,7 @@ function liveCredentialStatus(store) {
|
||||
providerId: provider.id,
|
||||
method: provider.method,
|
||||
externalCli: provider.externalCli ?? null,
|
||||
fallbackPolicy: provider.fallbackPolicy ?? "mock",
|
||||
envVars,
|
||||
reason: "configured"
|
||||
};
|
||||
@ -398,12 +420,35 @@ function liveCredentialStatus(store) {
|
||||
providerId: defaultId,
|
||||
method: providers[defaultId]?.method ?? "mock",
|
||||
externalCli: providers[defaultId]?.externalCli ?? null,
|
||||
fallbackPolicy: providers[defaultId]?.fallbackPolicy ?? "mock",
|
||||
envVars: providers[defaultId]?.envVars ?? [],
|
||||
reason: "no live provider configured"
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyLiveCredentialStatus(status) {
|
||||
if (status.available === false && status.fallbackPolicy === "external-cli") {
|
||||
try {
|
||||
const externalCli = resolveExternalCliName(status.providerId);
|
||||
const verification = await verifyExternalCliAuth(externalCli);
|
||||
return {
|
||||
...status,
|
||||
available: verification.verified,
|
||||
method: "external-cli",
|
||||
externalCli,
|
||||
fallbackFrom: status.method,
|
||||
envVars: [],
|
||||
reason: verification.verified ? "configured via external-cli fallback" : `external-cli ${externalCli} is not usable: ${verification.reason}`,
|
||||
verification: externalCliVerificationSummary(verification)
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
...status,
|
||||
available: false,
|
||||
reason: `${status.reason}; external-cli fallback unavailable: ${error.message}`
|
||||
};
|
||||
}
|
||||
}
|
||||
if (status.method !== "external-cli") {
|
||||
return status;
|
||||
}
|
||||
@ -454,6 +499,24 @@ function normalizeAuthOverride(value) {
|
||||
return mode;
|
||||
}
|
||||
|
||||
function normalizeFallbackPolicy(value) {
|
||||
const normalized = String(value ?? "mock").trim().toLowerCase().replaceAll("_", "-");
|
||||
const aliases = {
|
||||
mock: "mock",
|
||||
default: "mock",
|
||||
"external-cli": "external-cli",
|
||||
external: "external-cli",
|
||||
cli: "external-cli",
|
||||
none: "none",
|
||||
skip: "none",
|
||||
disabled: "none"
|
||||
};
|
||||
if (aliases[normalized]) {
|
||||
return aliases[normalized];
|
||||
}
|
||||
throw new Error(`fallbackPolicy must be one of ${fallbackPolicies.join(", ")}`);
|
||||
}
|
||||
|
||||
function mockDir(artifactDir) {
|
||||
return join(artifactDir, "mock-openai");
|
||||
}
|
||||
|
||||
@ -35,8 +35,8 @@ export function printHelp() {
|
||||
Usage:
|
||||
kova version [--json]
|
||||
kova --version
|
||||
kova setup [--ci|--non-interactive] [--auth <mock|api-key|env-only|external-cli|oauth|skip>] [--provider <id>] [--env-var <name>] [--value <secret>] [--json]
|
||||
kova setup auth [--provider <id>] [--method <mock|api-key|env-only|external-cli|oauth|skip>] [--env-var <name>] [--value <secret>] [--json]
|
||||
kova setup [--ci|--non-interactive] [--auth <mock|api-key|env-only|external-cli|oauth|skip>] [--provider <id>] [--env-var <name>] [--value <secret>] [--fallback-policy <mock|external-cli|none>] [--json]
|
||||
kova setup auth [--provider <id>] [--method <mock|api-key|env-only|external-cli|oauth|skip>] [--env-var <name>] [--value <secret>] [--fallback-policy <mock|external-cli|none>] [--json]
|
||||
kova self-check [--json]
|
||||
kova plan [--scenario <id>] [--json]
|
||||
kova run --target <selector> [--from <selector>] [--scenario <id>] [--state <id>] [--auth <mock|live|skip>] [--repeat <n>] [--baseline [path]] [--save-baseline [path]] [--regression-thresholds <json>] [--report-dir <path>] [--health-samples <n>] [--readiness-interval-ms <n>] [--resource-sample-interval-ms <n>] [--deep-profile] [--node-profile] [--heap-snapshot] [--profile-on-failure] [--execute] [--keep-env] [--retain-on-failure] [--json]
|
||||
@ -65,6 +65,7 @@ Notes:
|
||||
--auth defaults to mock so every disposable env has deliberate model auth unless a scenario opts out.
|
||||
setup provider/auth choices accept either numbers from the prompt or names such as openai, anthropic, env-only, api-key.
|
||||
external-cli setup derives Codex for OpenAI and Claude CLI for Anthropic, then verifies the CLI and auth evidence.
|
||||
external CLI fallback is only used when explicitly configured with --fallback-policy external-cli.
|
||||
--baseline compares executed aggregates against a Kova baseline store; without a path it uses the default store.
|
||||
--save-baseline writes executed aggregates into the selected baseline store.
|
||||
--deep-profile enables Node CPU/heap/trace profiling, OpenClaw timeline envs,
|
||||
|
||||
@ -33,6 +33,7 @@ export async function collectProviderEvidence(artifactDir, options = {}) {
|
||||
routes: [],
|
||||
models: [],
|
||||
statuses: [],
|
||||
usage: null,
|
||||
errors: [],
|
||||
requests: [],
|
||||
artifacts: [],
|
||||
@ -146,6 +147,7 @@ export function parseProviderRequestLog(text) {
|
||||
outcomes: summarizeBy(requests, "outcome"),
|
||||
errorClasses: summarizeBy(requests, "errorClass"),
|
||||
statuses: summarizeBy(requests, "status"),
|
||||
usage: summarizeUsage(requests),
|
||||
errors: [...parseErrors, ...requestErrors(requests)],
|
||||
requests
|
||||
};
|
||||
@ -226,6 +228,7 @@ export function computeProviderTurnAttribution(result, providerEvidence) {
|
||||
modes: [],
|
||||
outcomes: [],
|
||||
errorClasses: [],
|
||||
usage: null,
|
||||
errors: [],
|
||||
providerDominates: null,
|
||||
preProviderDominates: null,
|
||||
@ -263,6 +266,7 @@ export function computeProviderTurnAttribution(result, providerEvidence) {
|
||||
modes: summarizeBy(requests, "mode"),
|
||||
outcomes: summarizeBy(requests, "outcome"),
|
||||
errorClasses: summarizeBy(requests, "errorClass"),
|
||||
usage: summarizeUsage(requests),
|
||||
errors: requestErrors(requests),
|
||||
providerDominates: dominanceRatio(Math.max(0, lastProviderResponseAt - firstProviderRequestAt), Math.max(0, commandFinishedAt - commandStartedAt)),
|
||||
preProviderDominates: dominanceRatio(Math.max(0, firstProviderRequestAt - commandStartedAt), Math.max(0, commandFinishedAt - commandStartedAt)),
|
||||
@ -322,6 +326,7 @@ function normalizeTimelineProviderRequest(event, line) {
|
||||
stream: typeof event.stream === "boolean" ? event.stream : null,
|
||||
status: numberOrNull(event.status),
|
||||
statusClass: typeof event.status === "number" ? `${Math.floor(event.status / 100)}xx` : null,
|
||||
usage: normalizeUsage(event.usage ?? event.attributes?.usage),
|
||||
bodyBytes: null,
|
||||
parseError: null
|
||||
};
|
||||
@ -358,6 +363,7 @@ function summarizeProviderRequests(requests) {
|
||||
outcomes: summarizeBy(requests, "outcome"),
|
||||
errorClasses: summarizeBy(requests, "errorClass"),
|
||||
statuses: summarizeBy(requests, "status"),
|
||||
usage: summarizeUsage(requests),
|
||||
errors: requestErrors(requests)
|
||||
};
|
||||
}
|
||||
@ -394,6 +400,7 @@ function normalizeProviderRequest(raw, line) {
|
||||
model: raw.model ?? modelFromBody(raw.body),
|
||||
stream: typeof raw.stream === "boolean" ? raw.stream : streamFromBody(raw.body),
|
||||
status: numberOrNull(raw.status),
|
||||
usage: normalizeUsage(raw.usage),
|
||||
statusClass: raw.statusClass ?? (typeof raw.status === "number" ? `${Math.floor(raw.status / 100)}xx` : null),
|
||||
bodyBytes: numberOrNull(raw.bodyBytes) ?? (typeof raw.body === "string" ? Buffer.byteLength(raw.body) : null),
|
||||
parseError: raw.parseError ?? null
|
||||
@ -458,6 +465,56 @@ function requestErrors(requests) {
|
||||
return errors;
|
||||
}
|
||||
|
||||
function summarizeUsage(requests) {
|
||||
const usages = requests
|
||||
.map((request) => request.usage)
|
||||
.filter((usage) => usage && typeof usage === "object");
|
||||
if (usages.length === 0) {
|
||||
return {
|
||||
schemaVersion: "kova.providerUsageSummary.v1",
|
||||
available: false,
|
||||
requestCount: requests.length,
|
||||
requestsWithUsage: 0,
|
||||
inputTokens: null,
|
||||
outputTokens: null,
|
||||
totalTokens: null
|
||||
};
|
||||
}
|
||||
return {
|
||||
schemaVersion: "kova.providerUsageSummary.v1",
|
||||
available: true,
|
||||
requestCount: requests.length,
|
||||
requestsWithUsage: usages.length,
|
||||
inputTokens: sumNullable(usages.map((usage) => usage.inputTokens)),
|
||||
outputTokens: sumNullable(usages.map((usage) => usage.outputTokens)),
|
||||
totalTokens: sumNullable(usages.map((usage) => usage.totalTokens))
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeUsage(value) {
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return null;
|
||||
}
|
||||
const inputTokens = numberOrNull(value.input_tokens ?? value.inputTokens ?? value.prompt_tokens ?? value.promptTokens);
|
||||
const outputTokens = numberOrNull(value.output_tokens ?? value.outputTokens ?? value.completion_tokens ?? value.completionTokens);
|
||||
const totalTokens = numberOrNull(value.total_tokens ?? value.totalTokens) ??
|
||||
(typeof inputTokens === "number" && typeof outputTokens === "number" ? inputTokens + outputTokens : null);
|
||||
if (![inputTokens, outputTokens, totalTokens].some((item) => typeof item === "number")) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
schemaVersion: "kova.providerUsage.v1",
|
||||
inputTokens,
|
||||
outputTokens,
|
||||
totalTokens
|
||||
};
|
||||
}
|
||||
|
||||
function sumNullable(values) {
|
||||
const numbers = values.filter((value) => typeof value === "number");
|
||||
return numbers.length > 0 ? numbers.reduce((sum, value) => sum + value, 0) : null;
|
||||
}
|
||||
|
||||
function modelFromBody(body) {
|
||||
const parsed = parseBody(body);
|
||||
return typeof parsed?.model === "string" ? parsed.model : null;
|
||||
|
||||
@ -74,6 +74,9 @@ export function renderMarkdownReport(report) {
|
||||
}
|
||||
if (record.auth) {
|
||||
lines.push(`- Auth: ${record.auth.mode} (${record.auth.source}; provider ${record.auth.providerId ?? "none"})`);
|
||||
if (record.auth.fallbackFrom) {
|
||||
lines.push(`- Auth fallback: ${record.auth.fallbackFrom} -> ${record.auth.source}`);
|
||||
}
|
||||
if (record.auth.environmentDependent) {
|
||||
lines.push("- Live provider lane: environment-dependent; compare separately from deterministic mock baselines.");
|
||||
}
|
||||
@ -145,7 +148,10 @@ export function renderMarkdownReport(report) {
|
||||
if (record.measurements.agentProviderAttribution) {
|
||||
lines.push(`- Provider evidence: ${record.measurements.agentProviderRequestCount ?? 0} request(s); provider work ${record.measurements.agentProviderFinalMs ?? "unknown"} ms; pre-provider ${record.measurements.agentPreProviderMs ?? "unknown"} ms; post-provider ${record.measurements.agentPostProviderMs ?? "unknown"} ms`);
|
||||
} else if (record.providerEvidence?.available) {
|
||||
lines.push(`- Provider evidence: ${record.providerEvidence.requestCount ?? 0} request(s); provider duration ${record.providerEvidence.providerDurationMs ?? "unknown"} ms`);
|
||||
const usage = record.providerEvidence.usage?.available
|
||||
? `; tokens ${record.providerEvidence.usage.totalTokens ?? "unknown"}`
|
||||
: "";
|
||||
lines.push(`- Provider evidence: ${record.providerEvidence.requestCount ?? 0} request(s); provider duration ${record.providerEvidence.providerDurationMs ?? "unknown"} ms${usage}`);
|
||||
} else if (record.auth?.mode === "live") {
|
||||
lines.push(`- Provider evidence: unavailable for live lane (${record.providerEvidence?.error ?? "no provider events captured"})`);
|
||||
}
|
||||
|
||||
@ -63,8 +63,11 @@ export async function runSelfCheck(flags = {}) {
|
||||
checks.push(await interactiveSetupChoiceCheck(tmp));
|
||||
checks.push(await externalCliSetupCheck(tmp));
|
||||
checks.push(await externalCliOpenClawConfigCheck(tmp));
|
||||
checks.push(await anthropicApiKeyOpenClawConfigCheck(tmp));
|
||||
checks.push(await claudeCliOpenClawConfigCheck(tmp));
|
||||
checks.push(await liveApiKeyExecutionCheck(tmp));
|
||||
checks.push(await liveExternalCliDryRunCheck(tmp));
|
||||
checks.push(await liveExternalCliFallbackCheck(tmp));
|
||||
checks.push(await failingCommandCheck(
|
||||
"setup-custom-provider-rejects-external-cli",
|
||||
`KOVA_HOME=${quoteShell(join(tmp, "custom-external-cli-home"))} node bin/kova.mjs setup --non-interactive --provider custom-openai --auth external-cli --json`,
|
||||
@ -840,6 +843,8 @@ async function providerEvidenceParserCheck() {
|
||||
assertEqual(attribution.preProviderMs, 5000, "pre-provider latency");
|
||||
assertEqual(attribution.providerFinalMs, 800, "provider final latency");
|
||||
assertEqual(attribution.postProviderMs, 200, "post-provider latency");
|
||||
assertEqual(evidence.usage?.available, true, "provider usage availability");
|
||||
assertEqual(evidence.usage?.totalTokens, 12, "provider usage total tokens");
|
||||
return {
|
||||
id: "provider-evidence-parser",
|
||||
status: "PASS",
|
||||
@ -1011,6 +1016,71 @@ async function liveExternalCliDryRunCheck(tmp) {
|
||||
}
|
||||
}
|
||||
|
||||
async function liveExternalCliFallbackCheck(tmp) {
|
||||
const home = join(tmp, "live-external-cli-fallback-home");
|
||||
const kovaHome = join(tmp, "live-external-cli-fallback-kova-home");
|
||||
const fakeBin = join(tmp, "live-external-cli-fallback-bin");
|
||||
const reportDir = join(tmp, "live-external-cli-fallback-report");
|
||||
await mkdir(join(home, ".codex"), { recursive: true });
|
||||
await mkdir(join(kovaHome, "credentials"), { recursive: true });
|
||||
await mkdir(fakeBin, { recursive: true });
|
||||
await writeFile(join(home, ".codex", "auth.json"), "{\"tokens\":{\"access_token\":\"redacted\"}}\n", "utf8");
|
||||
await writeFile(join(fakeBin, "codex"), "#!/bin/sh\necho codex-selfcheck\n", "utf8");
|
||||
await chmod(join(fakeBin, "codex"), 0o755);
|
||||
await writeFile(join(kovaHome, "credentials", "providers.json"), `${JSON.stringify({
|
||||
schemaVersion: "kova.credentials.providers.v1",
|
||||
defaultProvider: "openai",
|
||||
providers: {
|
||||
openai: {
|
||||
id: "openai",
|
||||
method: "env-only",
|
||||
envVars: ["OPENAI_API_KEY"],
|
||||
externalCli: null,
|
||||
fallbackPolicy: "external-cli",
|
||||
configuredAt: new Date().toISOString()
|
||||
}
|
||||
}
|
||||
}, null, 2)}\n`, "utf8");
|
||||
await writeFile(join(kovaHome, "credentials", "live.env"), "", { encoding: "utf8", mode: 0o600 });
|
||||
|
||||
const command = [
|
||||
`HOME=${quoteShell(home)}`,
|
||||
`PATH=${quoteShell(`${fakeBin}:${process.env.PATH}`)}`,
|
||||
`KOVA_HOME=${quoteShell(kovaHome)}`,
|
||||
`node bin/kova.mjs run --target runtime:stable --scenario fresh-install --auth live --report-dir ${quoteShell(reportDir)} --json`
|
||||
].join(" ");
|
||||
const result = await runCommand(command, { timeoutMs: 30000, maxOutputChars: 1000000 });
|
||||
|
||||
try {
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`);
|
||||
}
|
||||
const receipt = JSON.parse(result.stdout);
|
||||
const report = JSON.parse(await readFile(receipt.jsonPath, "utf8"));
|
||||
const record = report.records?.[0];
|
||||
assertEqual(report.auth?.live?.method, "external-cli", "fallback live method");
|
||||
assertEqual(report.auth?.live?.fallbackFrom, "env-only", "fallback source method");
|
||||
assertEqual(report.auth?.live?.fallbackPolicy, "external-cli", "fallback policy");
|
||||
assertEqual(record?.auth?.source, "external-cli", "record fallback source");
|
||||
assertEqual(record?.auth?.fallbackFrom, "env-only", "record fallback from");
|
||||
assertEqual(record?.auth?.externalCli, "codex", "record fallback CLI");
|
||||
return {
|
||||
id: "live-external-cli-fallback",
|
||||
status: "PASS",
|
||||
command,
|
||||
durationMs: result.durationMs
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "live-external-cli-fallback",
|
||||
status: "FAIL",
|
||||
command,
|
||||
durationMs: result.durationMs,
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function fakeOcmScript() {
|
||||
return `#!/bin/sh
|
||||
printf '%s\\n' "$*" >> "$KOVA_MOCK_OCM_LOG"
|
||||
@ -2310,6 +2380,72 @@ async function externalCliOpenClawConfigCheck(tmp) {
|
||||
}
|
||||
}
|
||||
|
||||
async function anthropicApiKeyOpenClawConfigCheck(tmp) {
|
||||
const home = join(tmp, "anthropic-api-key-config-home");
|
||||
const command = [
|
||||
`OPENCLAW_HOME=${quoteShell(home)}`,
|
||||
"node support/configure-openclaw-live-auth.mjs --provider anthropic --env-var ANTHROPIC_API_KEY"
|
||||
].join(" ");
|
||||
const result = await runCommand(command, { timeoutMs: 30000, maxOutputChars: 1000000 });
|
||||
try {
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`);
|
||||
}
|
||||
const config = JSON.parse(await readFile(join(home, ".openclaw", "openclaw.json"), "utf8"));
|
||||
assertEqual(config.models?.providers?.anthropic?.apiKey?.id, "ANTHROPIC_API_KEY", "anthropic env ref");
|
||||
assertEqual(config.agents?.defaults?.model?.primary, "anthropic/claude-sonnet-4-5", "anthropic default model");
|
||||
return {
|
||||
id: "anthropic-api-key-openclaw-config",
|
||||
status: "PASS",
|
||||
command,
|
||||
durationMs: result.durationMs
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "anthropic-api-key-openclaw-config",
|
||||
status: "FAIL",
|
||||
command,
|
||||
durationMs: result.durationMs,
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function claudeCliOpenClawConfigCheck(tmp) {
|
||||
const home = join(tmp, "claude-cli-config-home");
|
||||
const command = [
|
||||
`OPENCLAW_HOME=${quoteShell(home)}`,
|
||||
"node support/configure-openclaw-live-auth.mjs --provider anthropic --auth-method external-cli --external-cli claude"
|
||||
].join(" ");
|
||||
const result = await runCommand(command, { timeoutMs: 30000, maxOutputChars: 1000000 });
|
||||
try {
|
||||
if (result.status !== 0) {
|
||||
throw new Error(result.stderr.trim() || result.stdout.trim() || `exit ${result.status}`);
|
||||
}
|
||||
const config = JSON.parse(await readFile(join(home, ".openclaw", "openclaw.json"), "utf8"));
|
||||
assertEqual(config.agents?.defaults?.model?.primary, "anthropic/claude-sonnet-4-5", "claude cli model ref");
|
||||
assertEqual(config.agents?.defaults?.agentRuntime?.id, "claude-cli", "claude cli runtime id");
|
||||
assertEqual(config.agents?.defaults?.agentRuntime?.fallback, "none", "claude cli runtime fallback");
|
||||
if (config.models?.providers?.anthropic?.apiKey !== undefined) {
|
||||
throw new Error("Claude CLI config must not write env apiKey config");
|
||||
}
|
||||
return {
|
||||
id: "claude-cli-openclaw-config",
|
||||
status: "PASS",
|
||||
command,
|
||||
durationMs: result.durationMs
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "claude-cli-openclaw-config",
|
||||
status: "FAIL",
|
||||
command,
|
||||
durationMs: result.durationMs,
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function externalCliRunAuthVerificationCheck(tmp) {
|
||||
const home = join(tmp, "stale-external-cli-home");
|
||||
const kovaHome = join(tmp, "stale-external-cli-kova-home");
|
||||
|
||||
@ -84,6 +84,7 @@ async function writeStreamingStall(res, stream, call) {
|
||||
}
|
||||
|
||||
function responseEvents(text) {
|
||||
const usage = mockUsage();
|
||||
return [
|
||||
{
|
||||
type: "response.output_item.added",
|
||||
@ -103,17 +104,25 @@ function responseEvents(text) {
|
||||
type: "response.completed",
|
||||
response: {
|
||||
status: "completed",
|
||||
usage: {
|
||||
input_tokens: 9,
|
||||
output_tokens: 3,
|
||||
total_tokens: 12,
|
||||
input_tokens_details: { cached_tokens: 0 }
|
||||
}
|
||||
usage
|
||||
}
|
||||
}
|
||||
];
|
||||
}
|
||||
|
||||
function mockUsage() {
|
||||
return {
|
||||
input_tokens: 9,
|
||||
output_tokens: 3,
|
||||
total_tokens: 12,
|
||||
input_tokens_details: { cached_tokens: 0 }
|
||||
};
|
||||
}
|
||||
|
||||
function chatUsage() {
|
||||
return { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 };
|
||||
}
|
||||
|
||||
function writeChatCompletion(res, stream) {
|
||||
if (stream) {
|
||||
writeSse(res, [
|
||||
@ -135,7 +144,7 @@ function writeChatCompletion(res, stream) {
|
||||
id: "chatcmpl_kova",
|
||||
object: "chat.completion",
|
||||
choices: [{ index: 0, message: { role: "assistant", content: marker }, finish_reason: "stop" }],
|
||||
usage: { prompt_tokens: 9, completion_tokens: 3, total_tokens: 12 }
|
||||
usage: chatUsage()
|
||||
});
|
||||
}
|
||||
|
||||
@ -169,6 +178,7 @@ const server = http.createServer(async (req, res) => {
|
||||
let loggable = false;
|
||||
let stream = false;
|
||||
let model = null;
|
||||
let usage = null;
|
||||
let behavior = {
|
||||
mode: providerMode,
|
||||
outcome: null,
|
||||
@ -212,6 +222,7 @@ const server = http.createServer(async (req, res) => {
|
||||
model,
|
||||
stream,
|
||||
status,
|
||||
usage,
|
||||
statusClass: typeof status === "number" ? `${Math.floor(status / 100)}xx` : null,
|
||||
bodyBytes: Buffer.byteLength(bodyText),
|
||||
parseError
|
||||
@ -259,6 +270,7 @@ const server = http.createServer(async (req, res) => {
|
||||
return;
|
||||
}
|
||||
if (body.stream === false) {
|
||||
usage = mockUsage();
|
||||
writeJson(res, 200, {
|
||||
id: "resp_kova",
|
||||
object: "response",
|
||||
@ -272,10 +284,11 @@ const server = http.createServer(async (req, res) => {
|
||||
content: [{ type: "output_text", text: marker, annotations: [] }]
|
||||
}
|
||||
],
|
||||
usage: { input_tokens: 9, output_tokens: 3, total_tokens: 12 }
|
||||
usage
|
||||
});
|
||||
return;
|
||||
}
|
||||
usage = mockUsage();
|
||||
writeSse(res, responseEvents(marker));
|
||||
return;
|
||||
}
|
||||
@ -286,6 +299,9 @@ const server = http.createServer(async (req, res) => {
|
||||
if (await maybeWriteFailureBehavior(res, behavior, body.stream !== false)) {
|
||||
return;
|
||||
}
|
||||
if (body.stream === false) {
|
||||
usage = chatUsage();
|
||||
}
|
||||
writeChatCompletion(res, body.stream !== false);
|
||||
return;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user