diff --git a/docs/AGENT_USAGE.md b/docs/AGENT_USAGE.md index 16e94f5..18b720d 100644 --- a/docs/AGENT_USAGE.md +++ b/docs/AGENT_USAGE.md @@ -158,6 +158,18 @@ node bin/kova.mjs run \ --execute ``` +Focused upgrade lanes are target-specific and Kova validates the selector: + +```sh +node bin/kova.mjs matrix run --profile channel-upgrade --target channel:beta --execute --json +node bin/kova.mjs matrix run --profile local-build-upgrade --target local-build:/path/to/openclaw --source-env Violet --execute --json +``` + +`channel-upgrade` is specifically stable-to-beta. Running it with +`channel:stable` is rejected instead of producing misleading evidence. +`local-build-upgrade` exercises stable-channel and cloned existing-user upgrades +against the release-shaped local build. + If a user wants to retain a failed env: ```sh diff --git a/profiles/channel-upgrade.json b/profiles/channel-upgrade.json new file mode 100644 index 0000000..632a0fb --- /dev/null +++ b/profiles/channel-upgrade.json @@ -0,0 +1,9 @@ +{ + "id": "channel-upgrade", + "title": "Channel Upgrade Matrix", + "objective": "Focused OpenClaw channel upgrade coverage for stable-to-beta transitions through real OCM-managed envs.", + "targetKinds": ["channel"], + "entries": [ + { "scenario": "upgrade-stable-channel-to-beta", "state": "stable-channel-user", "timeoutMs": 240000 } + ] +} diff --git a/profiles/local-build-upgrade.json b/profiles/local-build-upgrade.json new file mode 100644 index 0000000..edb00d9 --- /dev/null +++ b/profiles/local-build-upgrade.json @@ -0,0 +1,11 @@ +{ + "id": "local-build-upgrade", + "title": "Local Build Upgrade Matrix", + "objective": "Focused OpenClaw upgrade coverage for release-shaped local builds against stable-channel and cloned existing-user state.", + "targetKinds": ["local-build"], + "entries": [ + { "scenario": "upgrade-stable-channel-to-local-build", "state": "stable-channel-user", "timeoutMs": 300000 }, + { "scenario": "upgrade-durable-clone-to-local-build", "state": "onboarded-user", "timeoutMs": 300000 }, + { "scenario": "upgrade-durable-clone-to-local-build", "state": "plugin-index", "timeoutMs": 300000 } + ] +} diff --git a/scenarios/upgrade-durable-clone-to-local-build.json b/scenarios/upgrade-durable-clone-to-local-build.json new file mode 100644 index 0000000..a5071bd --- /dev/null +++ b/scenarios/upgrade-durable-clone-to-local-build.json @@ -0,0 +1,40 @@ +{ + "id": "upgrade-durable-clone-to-local-build", + "surface": "upgrade-existing-user", + "title": "Durable Clone To Local Build Upgrade", + "objective": "Clone a real existing OpenClaw env into disposable state, upgrade the clone to a release-shaped local build, and verify user-state migration, plugin indexes, bundled runtime deps, and gateway health without mutating the source env.", + "tags": ["existing-user", "upgrade", "local-build", "plugins", "migration", "gateway"], + "states": ["onboarded-user", "plugin-index", "failed-upgrade"], + "targetKinds": ["local-build"], + "timeoutMs": 300000, + "thresholds": { + "upgradeMs": 180000, + "gatewayReadyMs": 60000, + "statusMs": 10000, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "clone", + "title": "Clone Existing Env", + "intent": "Clone a durable existing env into a disposable Kova env before any local-build upgrade action.", + "commands": ["ocm env clone {sourceEnv} {env} --json", "ocm service status {env} --json"], + "evidence": ["clone result", "source env", "clone root", "pre-upgrade service status"] + }, + { + "id": "upgrade", + "title": "Upgrade Clone To Local Build", + "intent": "Run the real OpenClaw upgrade path against the cloned user state using the release-shaped local build runtime.", + "commands": ["ocm upgrade {env} {upgradeSelector} --json"], + "evidence": ["local-build upgrade JSON", "snapshot id", "doctor/update output", "rollback status"] + }, + { + "id": "post-upgrade", + "title": "Post-Upgrade Clone Checks", + "intent": "Verify cloned user state survives local-build upgrade, including plugin index migration and gateway readiness.", + "commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"], + "evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"] + } + ] +} diff --git a/scenarios/upgrade-stable-channel-to-beta.json b/scenarios/upgrade-stable-channel-to-beta.json new file mode 100644 index 0000000..79bbc55 --- /dev/null +++ b/scenarios/upgrade-stable-channel-to-beta.json @@ -0,0 +1,41 @@ +{ + "id": "upgrade-stable-channel-to-beta", + "surface": "upgrade-existing-user", + "title": "Stable Channel To Beta Upgrade", + "objective": "Start a disposable OpenClaw env on the stable channel, run the real upgrade path to beta, and verify channel upgrade behavior, plugin index migration, bundled runtime deps, and gateway health.", + "tags": ["upgrade", "channel", "stable", "beta", "plugins", "gateway"], + "states": ["stable-channel-user"], + "targetKinds": ["channel"], + "targetValues": ["beta"], + "timeoutMs": 240000, + "thresholds": { + "upgradeMs": 180000, + "gatewayReadyMs": 60000, + "statusMs": 10000, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "start", + "title": "Start Stable Channel Env", + "intent": "Create a disposable env through the real stable channel before switching to beta.", + "commands": ["ocm start {env} --channel stable --json", "ocm service status {env} --json", "ocm @{env} -- status"], + "evidence": ["stable channel start output", "pre-upgrade gateway status", "pre-upgrade OpenClaw status"] + }, + { + "id": "upgrade", + "title": "Upgrade To Beta", + "intent": "Run the real OpenClaw upgrade path to the beta channel through OCM.", + "commands": ["ocm upgrade {env} {upgradeSelector} --json"], + "evidence": ["beta upgrade JSON", "snapshot id", "doctor/update output", "rollback status"] + }, + { + "id": "post-upgrade", + "title": "Post-Beta Upgrade Checks", + "intent": "Verify OpenClaw is usable after the beta channel upgrade and capture plugin/runtime dependency evidence.", + "commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"], + "evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"] + } + ] +} diff --git a/scenarios/upgrade-stable-channel-to-local-build.json b/scenarios/upgrade-stable-channel-to-local-build.json new file mode 100644 index 0000000..4a1c589 --- /dev/null +++ b/scenarios/upgrade-stable-channel-to-local-build.json @@ -0,0 +1,40 @@ +{ + "id": "upgrade-stable-channel-to-local-build", + "surface": "upgrade-existing-user", + "title": "Stable Channel To Local Build Upgrade", + "objective": "Start a disposable OpenClaw env on the stable channel, upgrade it to a release-shaped local build, and verify the same migration/runtime behavior users would see when updating to the upcoming release.", + "tags": ["upgrade", "channel", "stable", "local-build", "plugins", "gateway"], + "states": ["stable-channel-user"], + "targetKinds": ["local-build"], + "timeoutMs": 300000, + "thresholds": { + "upgradeMs": 180000, + "gatewayReadyMs": 60000, + "statusMs": 10000, + "missingDependencyErrors": 0, + "pluginLoadFailures": 0 + }, + "phases": [ + { + "id": "start", + "title": "Start Stable Channel Env", + "intent": "Create a disposable env through the real stable channel before upgrading to the local release-shaped runtime.", + "commands": ["ocm start {env} --channel stable --json", "ocm service status {env} --json", "ocm @{env} -- status"], + "evidence": ["stable channel start output", "pre-upgrade gateway status", "pre-upgrade OpenClaw status"] + }, + { + "id": "upgrade", + "title": "Upgrade To Local Build", + "intent": "Run the real OpenClaw upgrade path to the release-shaped local build runtime.", + "commands": ["ocm upgrade {env} {upgradeSelector} --json"], + "evidence": ["local-build upgrade JSON", "snapshot id", "doctor/update output", "rollback status"] + }, + { + "id": "post-upgrade", + "title": "Post-Local-Build Upgrade Checks", + "intent": "Verify the local build handles plugin indexes, runtime deps, doctor recovery, and gateway readiness after upgrade.", + "commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"], + "evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"] + } + ] +} diff --git a/src/main.mjs b/src/main.mjs index 22f042f..a37633c 100644 --- a/src/main.mjs +++ b/src/main.mjs @@ -169,11 +169,12 @@ async function matrixCommand(flags) { const target = required(flags.target, "--target"); const targetPlan = resolveTarget(target, "target"); validateProfileTarget(profile, targetPlan); - if (flags.from) { - resolveTarget(flags.from, "from"); - } + const fromPlan = flags.from ? resolveTarget(flags.from, "from") : null; const platform = platformInfo(); const entries = applyMatrixControls(await expandProfile(profile), flags, platform); + for (const entry of entries.filter((item) => !item.skipReason)) { + validateScenarioRun(entry.scenario, flags, { targetPlan, fromPlan }); + } const response = { schemaVersion: "kova.matrix.plan.v1", generatedAt: new Date().toISOString(), @@ -305,7 +306,7 @@ async function matrixRun(flags) { const baselineStore = baselinePath ? await loadBaselineStore(baselinePath) : null; preflightGateRun({ entries, flags }); for (const entry of entries.filter((item) => !item.skipReason)) { - validateScenarioRun(entry.scenario, flags); + validateScenarioRun(entry.scenario, flags, { targetPlan, fromPlan }); } const reportRoot = flags.report_dir ? resolveFromCwd(flags.report_dir) : reportsDir; const runId = createRunId(); @@ -816,7 +817,7 @@ async function run(flags) { const state = await loadState(flags.state ?? "fresh"); const scenarios = await loadScenarios(flags.scenario); for (const scenario of scenarios) { - validateScenarioRun(scenario, flags); + validateScenarioRun(scenario, flags, { targetPlan, fromPlan }); } const reportRoot = flags.report_dir ? resolveFromCwd(flags.report_dir) : reportsDir; diff --git a/src/registries/scenarios.mjs b/src/registries/scenarios.mjs index e6c53c9..243fe05 100644 --- a/src/registries/scenarios.mjs +++ b/src/registries/scenarios.mjs @@ -39,6 +39,12 @@ export function validateScenarioShape(scenario, sourceName = "scenario") { validateStringArray(scenario.tags, "tags", errors); validateStringArray(scenario.states, "states", errors, { optional: true }); validateStringArray(scenario.targetKinds, "targetKinds", errors, { optional: true }); + validateStringArray(scenario.targetValues, "targetValues", errors, { optional: true }); + validateStringArray(scenario.fromKinds, "fromKinds", errors, { optional: true }); + validateStringArray(scenario.fromValues, "fromValues", errors, { optional: true }); + if (scenario.requiresFrom !== undefined && typeof scenario.requiresFrom !== "boolean") { + errors.push("requiresFrom must be a boolean when set"); + } validatePhases(scenario.phases, errors); assertNoShapeErrors(errors, sourceName); @@ -143,11 +149,18 @@ function validateStringArray(values, key, errors, options = {}) { } } -export function validateScenarioRun(scenario, flags) { +export function validateScenarioRun(scenario, flags, context = {}) { const needsSourceEnv = scenarioUsesSourceEnv(scenario); if (needsSourceEnv && flags.execute === true && !flags.source_env) { throw new Error(`${scenario.id} execution requires --source-env `); } + validateTargetContract(scenario, context.targetPlan, "target", "targetKinds", "targetValues"); + if (scenario.requiresFrom === true && !context.fromPlan) { + throw new Error(`${scenario.id} requires --from `); + } + if (context.fromPlan) { + validateTargetContract(scenario, context.fromPlan, "from", "fromKinds", "fromValues"); + } } function scenarioUsesSourceEnv(scenario) { @@ -156,6 +169,20 @@ function scenarioUsesSourceEnv(scenario) { ); } +function validateTargetContract(scenario, plan, role, kindKey, valueKey) { + if (!plan) { + return; + } + const allowedKinds = scenario[kindKey] ?? []; + if (allowedKinds.length > 0 && !allowedKinds.includes(plan.kind)) { + throw new Error(`${scenario.id} supports ${role} kind ${allowedKinds.join(", ")}, got ${plan.kind}`); + } + const allowedValues = scenario[valueKey] ?? []; + if (allowedValues.length > 0 && !allowedValues.includes(plan.value)) { + throw new Error(`${scenario.id} supports ${role} value ${allowedValues.join(", ")}, got ${plan.value}`); + } +} + export function materializeCommands(commands, values) { return commands.map((command) => command diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index b359a39..dc2953b 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -113,6 +113,34 @@ export async function runSelfCheck(flags = {}) { checks.push(await jsonCommandCheck("matrix-plan-repeat-json", "node bin/kova.mjs matrix plan --profile smoke --target runtime:stable --include scenario:fresh-install --repeat 3 --json", (data) => { assertEqual(data.controls?.repeat, 3, "matrix repeat control"); })); + checks.push(await jsonCommandCheck("channel-upgrade-plan-json", "node bin/kova.mjs matrix plan --profile channel-upgrade --target channel:beta --json", (data) => { + assertEqual(data.profile?.id, "channel-upgrade", "channel upgrade profile id"); + assertEqual(data.target, "channel:beta", "channel upgrade target"); + assertEqual(data.entries?.[0]?.scenario?.id, "upgrade-stable-channel-to-beta", "channel upgrade scenario"); + })); + checks.push(await failingCommandCheck( + "channel-upgrade-rejects-wrong-target-value", + "node bin/kova.mjs matrix plan --profile channel-upgrade --target channel:stable --json", + "upgrade-stable-channel-to-beta supports target value beta, got stable" + )); + checks.push(await jsonCommandCheck("local-build-upgrade-plan-json", "node bin/kova.mjs matrix plan --profile local-build-upgrade --target local-build:/tmp/openclaw --include scenario:upgrade-stable-channel-to-local-build --json", (data) => { + assertEqual(data.profile?.id, "local-build-upgrade", "local-build upgrade profile id"); + assertEqual(data.entries?.[0]?.scenario?.id, "upgrade-stable-channel-to-local-build", "local-build stable upgrade scenario"); + })); + checks.push(await jsonCommandCheck("channel-upgrade-dry-run-json", `node bin/kova.mjs run --target channel:beta --scenario upgrade-stable-channel-to-beta --state stable-channel-user --report-dir ${quoteShell(tmp)} --json`, async (data) => { + const report = JSON.parse(await readFile(data.jsonPath, "utf8")); + const record = report.records?.[0]; + const commands = (record?.phases ?? []).flatMap((phase) => phase.commands ?? []); + assertEqual(commands.some((command) => command.includes("ocm start") && command.includes("--channel stable")), true, "stable start command present"); + assertEqual(commands.some((command) => command.includes("ocm upgrade") && /--channel '?beta'?/.test(command)), true, "beta upgrade command present"); + })); + checks.push(await jsonCommandCheck("durable-clone-local-build-dry-run-json", `node bin/kova.mjs run --target local-build:/tmp/openclaw --scenario upgrade-durable-clone-to-local-build --state plugin-index --source-env 'Team Env' --report-dir ${quoteShell(tmp)} --json`, async (data) => { + const report = JSON.parse(await readFile(data.jsonPath, "utf8")); + const record = report.records?.[0]; + const commands = (record?.phases ?? []).flatMap((phase) => phase.commands ?? []); + assertEqual(commands.some((command) => command.includes("ocm env clone 'Team Env'")), true, "quoted source env clone command present"); + assertEqual(commands.some((command) => command.includes("ocm upgrade") && /--runtime '?kova-local-/.test(command)), true, "local-build runtime upgrade command present"); + })); checks.push(await jsonCommandCheck("run-auth-default-mock-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --report-dir ${quoteShell(tmp)} --json`, async (data) => { const report = JSON.parse(await readFile(data.jsonPath, "utf8")); const record = report.records?.[0]; diff --git a/states/onboarded-user.json b/states/onboarded-user.json index 797b89a..98d622c 100644 --- a/states/onboarded-user.json +++ b/states/onboarded-user.json @@ -35,7 +35,8 @@ "config-state" ], "compatibleSurfaces": [ - "fresh-install" + "fresh-install", + "upgrade-existing-user" ], "incompatibleSurfaces": [], "riskArea": "onboarding-migration", diff --git a/states/plugin-index.json b/states/plugin-index.json index 710ca95..81beadb 100644 --- a/states/plugin-index.json +++ b/states/plugin-index.json @@ -35,7 +35,8 @@ ], "compatibleSurfaces": [ "fresh-install", - "plugin-lifecycle" + "plugin-lifecycle", + "upgrade-existing-user" ], "incompatibleSurfaces": [], "riskArea": "plugin-index-migration", diff --git a/states/stable-channel-user.json b/states/stable-channel-user.json new file mode 100644 index 0000000..506e317 --- /dev/null +++ b/states/stable-channel-user.json @@ -0,0 +1,29 @@ +{ + "id": "stable-channel-user", + "title": "Stable Channel User", + "objective": "A disposable OpenClaw env that has been started on the stable release channel before testing channel or local-build upgrade behavior.", + "tags": ["upgrade", "channel", "stable", "existing-user"], + "setup": [ + { + "id": "write-stable-channel-marker", + "title": "Write Stable Channel Marker", + "intent": "Persist stable-channel metadata after the env is first started so reports can prove the upgrade source shape.", + "afterPhases": ["start", "clone"], + "commands": [ + "ocm env exec {env} -- node -e 'const fs=require(\"fs\"), path=require(\"path\"); const home=process.env.OPENCLAW_HOME; fs.mkdirSync(path.join(home,\"config\"),{recursive:true}); fs.writeFileSync(path.join(home,\"config\",\"channel.json\"),JSON.stringify({schemaVersion:\"kova.fixture.channel.v1\",channel:\"stable\"},null,2)); fs.writeFileSync(path.join(home,\".openclaw-channel\"),\"stable\\n\");'" + ], + "evidence": ["stable channel marker exists"] + } + ], + "traits": ["existing-user", "channel-state"], + "compatibleSurfaces": ["upgrade-existing-user"], + "incompatibleSurfaces": [], + "riskArea": "release-channel-upgrade", + "ownerArea": "upgrade", + "setupEvidence": ["stable channel marker exists"], + "cleanupGuarantees": ["disposable env cleanup removes stable channel fixture files"], + "source": { + "kind": "generated", + "note": "The scenario starts the env through the real stable channel before upgrading it." + } +}