feat: add target-specific upgrade coverage

This commit is contained in:
Shakker 2026-05-01 10:36:41 +01:00
parent b35ebb7934
commit 8a5517cdc2
No known key found for this signature in database
12 changed files with 248 additions and 8 deletions

View File

@ -158,6 +158,18 @@ node bin/kova.mjs run \
--execute
```
Focused upgrade lanes are target-specific and Kova validates the selector:
```sh
node bin/kova.mjs matrix run --profile channel-upgrade --target channel:beta --execute --json
node bin/kova.mjs matrix run --profile local-build-upgrade --target local-build:/path/to/openclaw --source-env Violet --execute --json
```
`channel-upgrade` is specifically stable-to-beta. Running it with
`channel:stable` is rejected instead of producing misleading evidence.
`local-build-upgrade` exercises stable-channel and cloned existing-user upgrades
against the release-shaped local build.
If a user wants to retain a failed env:
```sh

View File

@ -0,0 +1,9 @@
{
"id": "channel-upgrade",
"title": "Channel Upgrade Matrix",
"objective": "Focused OpenClaw channel upgrade coverage for stable-to-beta transitions through real OCM-managed envs.",
"targetKinds": ["channel"],
"entries": [
{ "scenario": "upgrade-stable-channel-to-beta", "state": "stable-channel-user", "timeoutMs": 240000 }
]
}

View File

@ -0,0 +1,11 @@
{
"id": "local-build-upgrade",
"title": "Local Build Upgrade Matrix",
"objective": "Focused OpenClaw upgrade coverage for release-shaped local builds against stable-channel and cloned existing-user state.",
"targetKinds": ["local-build"],
"entries": [
{ "scenario": "upgrade-stable-channel-to-local-build", "state": "stable-channel-user", "timeoutMs": 300000 },
{ "scenario": "upgrade-durable-clone-to-local-build", "state": "onboarded-user", "timeoutMs": 300000 },
{ "scenario": "upgrade-durable-clone-to-local-build", "state": "plugin-index", "timeoutMs": 300000 }
]
}

View File

@ -0,0 +1,40 @@
{
"id": "upgrade-durable-clone-to-local-build",
"surface": "upgrade-existing-user",
"title": "Durable Clone To Local Build Upgrade",
"objective": "Clone a real existing OpenClaw env into disposable state, upgrade the clone to a release-shaped local build, and verify user-state migration, plugin indexes, bundled runtime deps, and gateway health without mutating the source env.",
"tags": ["existing-user", "upgrade", "local-build", "plugins", "migration", "gateway"],
"states": ["onboarded-user", "plugin-index", "failed-upgrade"],
"targetKinds": ["local-build"],
"timeoutMs": 300000,
"thresholds": {
"upgradeMs": 180000,
"gatewayReadyMs": 60000,
"statusMs": 10000,
"missingDependencyErrors": 0,
"pluginLoadFailures": 0
},
"phases": [
{
"id": "clone",
"title": "Clone Existing Env",
"intent": "Clone a durable existing env into a disposable Kova env before any local-build upgrade action.",
"commands": ["ocm env clone {sourceEnv} {env} --json", "ocm service status {env} --json"],
"evidence": ["clone result", "source env", "clone root", "pre-upgrade service status"]
},
{
"id": "upgrade",
"title": "Upgrade Clone To Local Build",
"intent": "Run the real OpenClaw upgrade path against the cloned user state using the release-shaped local build runtime.",
"commands": ["ocm upgrade {env} {upgradeSelector} --json"],
"evidence": ["local-build upgrade JSON", "snapshot id", "doctor/update output", "rollback status"]
},
{
"id": "post-upgrade",
"title": "Post-Upgrade Clone Checks",
"intent": "Verify cloned user state survives local-build upgrade, including plugin index migration and gateway readiness.",
"commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"],
"evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"]
}
]
}

View File

@ -0,0 +1,41 @@
{
"id": "upgrade-stable-channel-to-beta",
"surface": "upgrade-existing-user",
"title": "Stable Channel To Beta Upgrade",
"objective": "Start a disposable OpenClaw env on the stable channel, run the real upgrade path to beta, and verify channel upgrade behavior, plugin index migration, bundled runtime deps, and gateway health.",
"tags": ["upgrade", "channel", "stable", "beta", "plugins", "gateway"],
"states": ["stable-channel-user"],
"targetKinds": ["channel"],
"targetValues": ["beta"],
"timeoutMs": 240000,
"thresholds": {
"upgradeMs": 180000,
"gatewayReadyMs": 60000,
"statusMs": 10000,
"missingDependencyErrors": 0,
"pluginLoadFailures": 0
},
"phases": [
{
"id": "start",
"title": "Start Stable Channel Env",
"intent": "Create a disposable env through the real stable channel before switching to beta.",
"commands": ["ocm start {env} --channel stable --json", "ocm service status {env} --json", "ocm @{env} -- status"],
"evidence": ["stable channel start output", "pre-upgrade gateway status", "pre-upgrade OpenClaw status"]
},
{
"id": "upgrade",
"title": "Upgrade To Beta",
"intent": "Run the real OpenClaw upgrade path to the beta channel through OCM.",
"commands": ["ocm upgrade {env} {upgradeSelector} --json"],
"evidence": ["beta upgrade JSON", "snapshot id", "doctor/update output", "rollback status"]
},
{
"id": "post-upgrade",
"title": "Post-Beta Upgrade Checks",
"intent": "Verify OpenClaw is usable after the beta channel upgrade and capture plugin/runtime dependency evidence.",
"commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"],
"evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"]
}
]
}

View File

@ -0,0 +1,40 @@
{
"id": "upgrade-stable-channel-to-local-build",
"surface": "upgrade-existing-user",
"title": "Stable Channel To Local Build Upgrade",
"objective": "Start a disposable OpenClaw env on the stable channel, upgrade it to a release-shaped local build, and verify the same migration/runtime behavior users would see when updating to the upcoming release.",
"tags": ["upgrade", "channel", "stable", "local-build", "plugins", "gateway"],
"states": ["stable-channel-user"],
"targetKinds": ["local-build"],
"timeoutMs": 300000,
"thresholds": {
"upgradeMs": 180000,
"gatewayReadyMs": 60000,
"statusMs": 10000,
"missingDependencyErrors": 0,
"pluginLoadFailures": 0
},
"phases": [
{
"id": "start",
"title": "Start Stable Channel Env",
"intent": "Create a disposable env through the real stable channel before upgrading to the local release-shaped runtime.",
"commands": ["ocm start {env} --channel stable --json", "ocm service status {env} --json", "ocm @{env} -- status"],
"evidence": ["stable channel start output", "pre-upgrade gateway status", "pre-upgrade OpenClaw status"]
},
{
"id": "upgrade",
"title": "Upgrade To Local Build",
"intent": "Run the real OpenClaw upgrade path to the release-shaped local build runtime.",
"commands": ["ocm upgrade {env} {upgradeSelector} --json"],
"evidence": ["local-build upgrade JSON", "snapshot id", "doctor/update output", "rollback status"]
},
{
"id": "post-upgrade",
"title": "Post-Local-Build Upgrade Checks",
"intent": "Verify the local build handles plugin indexes, runtime deps, doctor recovery, and gateway readiness after upgrade.",
"commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"],
"evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"]
}
]
}

View File

@ -169,11 +169,12 @@ async function matrixCommand(flags) {
const target = required(flags.target, "--target");
const targetPlan = resolveTarget(target, "target");
validateProfileTarget(profile, targetPlan);
if (flags.from) {
resolveTarget(flags.from, "from");
}
const fromPlan = flags.from ? resolveTarget(flags.from, "from") : null;
const platform = platformInfo();
const entries = applyMatrixControls(await expandProfile(profile), flags, platform);
for (const entry of entries.filter((item) => !item.skipReason)) {
validateScenarioRun(entry.scenario, flags, { targetPlan, fromPlan });
}
const response = {
schemaVersion: "kova.matrix.plan.v1",
generatedAt: new Date().toISOString(),
@ -305,7 +306,7 @@ async function matrixRun(flags) {
const baselineStore = baselinePath ? await loadBaselineStore(baselinePath) : null;
preflightGateRun({ entries, flags });
for (const entry of entries.filter((item) => !item.skipReason)) {
validateScenarioRun(entry.scenario, flags);
validateScenarioRun(entry.scenario, flags, { targetPlan, fromPlan });
}
const reportRoot = flags.report_dir ? resolveFromCwd(flags.report_dir) : reportsDir;
const runId = createRunId();
@ -816,7 +817,7 @@ async function run(flags) {
const state = await loadState(flags.state ?? "fresh");
const scenarios = await loadScenarios(flags.scenario);
for (const scenario of scenarios) {
validateScenarioRun(scenario, flags);
validateScenarioRun(scenario, flags, { targetPlan, fromPlan });
}
const reportRoot = flags.report_dir ? resolveFromCwd(flags.report_dir) : reportsDir;

View File

@ -39,6 +39,12 @@ export function validateScenarioShape(scenario, sourceName = "scenario") {
validateStringArray(scenario.tags, "tags", errors);
validateStringArray(scenario.states, "states", errors, { optional: true });
validateStringArray(scenario.targetKinds, "targetKinds", errors, { optional: true });
validateStringArray(scenario.targetValues, "targetValues", errors, { optional: true });
validateStringArray(scenario.fromKinds, "fromKinds", errors, { optional: true });
validateStringArray(scenario.fromValues, "fromValues", errors, { optional: true });
if (scenario.requiresFrom !== undefined && typeof scenario.requiresFrom !== "boolean") {
errors.push("requiresFrom must be a boolean when set");
}
validatePhases(scenario.phases, errors);
assertNoShapeErrors(errors, sourceName);
@ -143,11 +149,18 @@ function validateStringArray(values, key, errors, options = {}) {
}
}
export function validateScenarioRun(scenario, flags) {
export function validateScenarioRun(scenario, flags, context = {}) {
const needsSourceEnv = scenarioUsesSourceEnv(scenario);
if (needsSourceEnv && flags.execute === true && !flags.source_env) {
throw new Error(`${scenario.id} execution requires --source-env <env>`);
}
validateTargetContract(scenario, context.targetPlan, "target", "targetKinds", "targetValues");
if (scenario.requiresFrom === true && !context.fromPlan) {
throw new Error(`${scenario.id} requires --from <selector>`);
}
if (context.fromPlan) {
validateTargetContract(scenario, context.fromPlan, "from", "fromKinds", "fromValues");
}
}
function scenarioUsesSourceEnv(scenario) {
@ -156,6 +169,20 @@ function scenarioUsesSourceEnv(scenario) {
);
}
function validateTargetContract(scenario, plan, role, kindKey, valueKey) {
if (!plan) {
return;
}
const allowedKinds = scenario[kindKey] ?? [];
if (allowedKinds.length > 0 && !allowedKinds.includes(plan.kind)) {
throw new Error(`${scenario.id} supports ${role} kind ${allowedKinds.join(", ")}, got ${plan.kind}`);
}
const allowedValues = scenario[valueKey] ?? [];
if (allowedValues.length > 0 && !allowedValues.includes(plan.value)) {
throw new Error(`${scenario.id} supports ${role} value ${allowedValues.join(", ")}, got ${plan.value}`);
}
}
export function materializeCommands(commands, values) {
return commands.map((command) =>
command

View File

@ -113,6 +113,34 @@ export async function runSelfCheck(flags = {}) {
checks.push(await jsonCommandCheck("matrix-plan-repeat-json", "node bin/kova.mjs matrix plan --profile smoke --target runtime:stable --include scenario:fresh-install --repeat 3 --json", (data) => {
assertEqual(data.controls?.repeat, 3, "matrix repeat control");
}));
checks.push(await jsonCommandCheck("channel-upgrade-plan-json", "node bin/kova.mjs matrix plan --profile channel-upgrade --target channel:beta --json", (data) => {
assertEqual(data.profile?.id, "channel-upgrade", "channel upgrade profile id");
assertEqual(data.target, "channel:beta", "channel upgrade target");
assertEqual(data.entries?.[0]?.scenario?.id, "upgrade-stable-channel-to-beta", "channel upgrade scenario");
}));
checks.push(await failingCommandCheck(
"channel-upgrade-rejects-wrong-target-value",
"node bin/kova.mjs matrix plan --profile channel-upgrade --target channel:stable --json",
"upgrade-stable-channel-to-beta supports target value beta, got stable"
));
checks.push(await jsonCommandCheck("local-build-upgrade-plan-json", "node bin/kova.mjs matrix plan --profile local-build-upgrade --target local-build:/tmp/openclaw --include scenario:upgrade-stable-channel-to-local-build --json", (data) => {
assertEqual(data.profile?.id, "local-build-upgrade", "local-build upgrade profile id");
assertEqual(data.entries?.[0]?.scenario?.id, "upgrade-stable-channel-to-local-build", "local-build stable upgrade scenario");
}));
checks.push(await jsonCommandCheck("channel-upgrade-dry-run-json", `node bin/kova.mjs run --target channel:beta --scenario upgrade-stable-channel-to-beta --state stable-channel-user --report-dir ${quoteShell(tmp)} --json`, async (data) => {
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
const record = report.records?.[0];
const commands = (record?.phases ?? []).flatMap((phase) => phase.commands ?? []);
assertEqual(commands.some((command) => command.includes("ocm start") && command.includes("--channel stable")), true, "stable start command present");
assertEqual(commands.some((command) => command.includes("ocm upgrade") && /--channel '?beta'?/.test(command)), true, "beta upgrade command present");
}));
checks.push(await jsonCommandCheck("durable-clone-local-build-dry-run-json", `node bin/kova.mjs run --target local-build:/tmp/openclaw --scenario upgrade-durable-clone-to-local-build --state plugin-index --source-env 'Team Env' --report-dir ${quoteShell(tmp)} --json`, async (data) => {
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
const record = report.records?.[0];
const commands = (record?.phases ?? []).flatMap((phase) => phase.commands ?? []);
assertEqual(commands.some((command) => command.includes("ocm env clone 'Team Env'")), true, "quoted source env clone command present");
assertEqual(commands.some((command) => command.includes("ocm upgrade") && /--runtime '?kova-local-/.test(command)), true, "local-build runtime upgrade command present");
}));
checks.push(await jsonCommandCheck("run-auth-default-mock-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --report-dir ${quoteShell(tmp)} --json`, async (data) => {
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
const record = report.records?.[0];

View File

@ -35,7 +35,8 @@
"config-state"
],
"compatibleSurfaces": [
"fresh-install"
"fresh-install",
"upgrade-existing-user"
],
"incompatibleSurfaces": [],
"riskArea": "onboarding-migration",

View File

@ -35,7 +35,8 @@
],
"compatibleSurfaces": [
"fresh-install",
"plugin-lifecycle"
"plugin-lifecycle",
"upgrade-existing-user"
],
"incompatibleSurfaces": [],
"riskArea": "plugin-index-migration",

View File

@ -0,0 +1,29 @@
{
"id": "stable-channel-user",
"title": "Stable Channel User",
"objective": "A disposable OpenClaw env that has been started on the stable release channel before testing channel or local-build upgrade behavior.",
"tags": ["upgrade", "channel", "stable", "existing-user"],
"setup": [
{
"id": "write-stable-channel-marker",
"title": "Write Stable Channel Marker",
"intent": "Persist stable-channel metadata after the env is first started so reports can prove the upgrade source shape.",
"afterPhases": ["start", "clone"],
"commands": [
"ocm env exec {env} -- node -e 'const fs=require(\"fs\"), path=require(\"path\"); const home=process.env.OPENCLAW_HOME; fs.mkdirSync(path.join(home,\"config\"),{recursive:true}); fs.writeFileSync(path.join(home,\"config\",\"channel.json\"),JSON.stringify({schemaVersion:\"kova.fixture.channel.v1\",channel:\"stable\"},null,2)); fs.writeFileSync(path.join(home,\".openclaw-channel\"),\"stable\\n\");'"
],
"evidence": ["stable channel marker exists"]
}
],
"traits": ["existing-user", "channel-state"],
"compatibleSurfaces": ["upgrade-existing-user"],
"incompatibleSurfaces": [],
"riskArea": "release-channel-upgrade",
"ownerArea": "upgrade",
"setupEvidence": ["stable channel marker exists"],
"cleanupGuarantees": ["disposable env cleanup removes stable channel fixture files"],
"source": {
"kind": "generated",
"note": "The scenario starts the env through the real stable channel before upgrading it."
}
}