feat: add target-specific upgrade coverage
This commit is contained in:
parent
b35ebb7934
commit
8a5517cdc2
@ -158,6 +158,18 @@ node bin/kova.mjs run \
|
||||
--execute
|
||||
```
|
||||
|
||||
Focused upgrade lanes are target-specific and Kova validates the selector:
|
||||
|
||||
```sh
|
||||
node bin/kova.mjs matrix run --profile channel-upgrade --target channel:beta --execute --json
|
||||
node bin/kova.mjs matrix run --profile local-build-upgrade --target local-build:/path/to/openclaw --source-env Violet --execute --json
|
||||
```
|
||||
|
||||
`channel-upgrade` is specifically stable-to-beta. Running it with
|
||||
`channel:stable` is rejected instead of producing misleading evidence.
|
||||
`local-build-upgrade` exercises stable-channel and cloned existing-user upgrades
|
||||
against the release-shaped local build.
|
||||
|
||||
If a user wants to retain a failed env:
|
||||
|
||||
```sh
|
||||
|
||||
9
profiles/channel-upgrade.json
Normal file
9
profiles/channel-upgrade.json
Normal file
@ -0,0 +1,9 @@
|
||||
{
|
||||
"id": "channel-upgrade",
|
||||
"title": "Channel Upgrade Matrix",
|
||||
"objective": "Focused OpenClaw channel upgrade coverage for stable-to-beta transitions through real OCM-managed envs.",
|
||||
"targetKinds": ["channel"],
|
||||
"entries": [
|
||||
{ "scenario": "upgrade-stable-channel-to-beta", "state": "stable-channel-user", "timeoutMs": 240000 }
|
||||
]
|
||||
}
|
||||
11
profiles/local-build-upgrade.json
Normal file
11
profiles/local-build-upgrade.json
Normal file
@ -0,0 +1,11 @@
|
||||
{
|
||||
"id": "local-build-upgrade",
|
||||
"title": "Local Build Upgrade Matrix",
|
||||
"objective": "Focused OpenClaw upgrade coverage for release-shaped local builds against stable-channel and cloned existing-user state.",
|
||||
"targetKinds": ["local-build"],
|
||||
"entries": [
|
||||
{ "scenario": "upgrade-stable-channel-to-local-build", "state": "stable-channel-user", "timeoutMs": 300000 },
|
||||
{ "scenario": "upgrade-durable-clone-to-local-build", "state": "onboarded-user", "timeoutMs": 300000 },
|
||||
{ "scenario": "upgrade-durable-clone-to-local-build", "state": "plugin-index", "timeoutMs": 300000 }
|
||||
]
|
||||
}
|
||||
40
scenarios/upgrade-durable-clone-to-local-build.json
Normal file
40
scenarios/upgrade-durable-clone-to-local-build.json
Normal file
@ -0,0 +1,40 @@
|
||||
{
|
||||
"id": "upgrade-durable-clone-to-local-build",
|
||||
"surface": "upgrade-existing-user",
|
||||
"title": "Durable Clone To Local Build Upgrade",
|
||||
"objective": "Clone a real existing OpenClaw env into disposable state, upgrade the clone to a release-shaped local build, and verify user-state migration, plugin indexes, bundled runtime deps, and gateway health without mutating the source env.",
|
||||
"tags": ["existing-user", "upgrade", "local-build", "plugins", "migration", "gateway"],
|
||||
"states": ["onboarded-user", "plugin-index", "failed-upgrade"],
|
||||
"targetKinds": ["local-build"],
|
||||
"timeoutMs": 300000,
|
||||
"thresholds": {
|
||||
"upgradeMs": 180000,
|
||||
"gatewayReadyMs": 60000,
|
||||
"statusMs": 10000,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "clone",
|
||||
"title": "Clone Existing Env",
|
||||
"intent": "Clone a durable existing env into a disposable Kova env before any local-build upgrade action.",
|
||||
"commands": ["ocm env clone {sourceEnv} {env} --json", "ocm service status {env} --json"],
|
||||
"evidence": ["clone result", "source env", "clone root", "pre-upgrade service status"]
|
||||
},
|
||||
{
|
||||
"id": "upgrade",
|
||||
"title": "Upgrade Clone To Local Build",
|
||||
"intent": "Run the real OpenClaw upgrade path against the cloned user state using the release-shaped local build runtime.",
|
||||
"commands": ["ocm upgrade {env} {upgradeSelector} --json"],
|
||||
"evidence": ["local-build upgrade JSON", "snapshot id", "doctor/update output", "rollback status"]
|
||||
},
|
||||
{
|
||||
"id": "post-upgrade",
|
||||
"title": "Post-Upgrade Clone Checks",
|
||||
"intent": "Verify cloned user state survives local-build upgrade, including plugin index migration and gateway readiness.",
|
||||
"commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"],
|
||||
"evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"]
|
||||
}
|
||||
]
|
||||
}
|
||||
41
scenarios/upgrade-stable-channel-to-beta.json
Normal file
41
scenarios/upgrade-stable-channel-to-beta.json
Normal file
@ -0,0 +1,41 @@
|
||||
{
|
||||
"id": "upgrade-stable-channel-to-beta",
|
||||
"surface": "upgrade-existing-user",
|
||||
"title": "Stable Channel To Beta Upgrade",
|
||||
"objective": "Start a disposable OpenClaw env on the stable channel, run the real upgrade path to beta, and verify channel upgrade behavior, plugin index migration, bundled runtime deps, and gateway health.",
|
||||
"tags": ["upgrade", "channel", "stable", "beta", "plugins", "gateway"],
|
||||
"states": ["stable-channel-user"],
|
||||
"targetKinds": ["channel"],
|
||||
"targetValues": ["beta"],
|
||||
"timeoutMs": 240000,
|
||||
"thresholds": {
|
||||
"upgradeMs": 180000,
|
||||
"gatewayReadyMs": 60000,
|
||||
"statusMs": 10000,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "start",
|
||||
"title": "Start Stable Channel Env",
|
||||
"intent": "Create a disposable env through the real stable channel before switching to beta.",
|
||||
"commands": ["ocm start {env} --channel stable --json", "ocm service status {env} --json", "ocm @{env} -- status"],
|
||||
"evidence": ["stable channel start output", "pre-upgrade gateway status", "pre-upgrade OpenClaw status"]
|
||||
},
|
||||
{
|
||||
"id": "upgrade",
|
||||
"title": "Upgrade To Beta",
|
||||
"intent": "Run the real OpenClaw upgrade path to the beta channel through OCM.",
|
||||
"commands": ["ocm upgrade {env} {upgradeSelector} --json"],
|
||||
"evidence": ["beta upgrade JSON", "snapshot id", "doctor/update output", "rollback status"]
|
||||
},
|
||||
{
|
||||
"id": "post-upgrade",
|
||||
"title": "Post-Beta Upgrade Checks",
|
||||
"intent": "Verify OpenClaw is usable after the beta channel upgrade and capture plugin/runtime dependency evidence.",
|
||||
"commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"],
|
||||
"evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"]
|
||||
}
|
||||
]
|
||||
}
|
||||
40
scenarios/upgrade-stable-channel-to-local-build.json
Normal file
40
scenarios/upgrade-stable-channel-to-local-build.json
Normal file
@ -0,0 +1,40 @@
|
||||
{
|
||||
"id": "upgrade-stable-channel-to-local-build",
|
||||
"surface": "upgrade-existing-user",
|
||||
"title": "Stable Channel To Local Build Upgrade",
|
||||
"objective": "Start a disposable OpenClaw env on the stable channel, upgrade it to a release-shaped local build, and verify the same migration/runtime behavior users would see when updating to the upcoming release.",
|
||||
"tags": ["upgrade", "channel", "stable", "local-build", "plugins", "gateway"],
|
||||
"states": ["stable-channel-user"],
|
||||
"targetKinds": ["local-build"],
|
||||
"timeoutMs": 300000,
|
||||
"thresholds": {
|
||||
"upgradeMs": 180000,
|
||||
"gatewayReadyMs": 60000,
|
||||
"statusMs": 10000,
|
||||
"missingDependencyErrors": 0,
|
||||
"pluginLoadFailures": 0
|
||||
},
|
||||
"phases": [
|
||||
{
|
||||
"id": "start",
|
||||
"title": "Start Stable Channel Env",
|
||||
"intent": "Create a disposable env through the real stable channel before upgrading to the local release-shaped runtime.",
|
||||
"commands": ["ocm start {env} --channel stable --json", "ocm service status {env} --json", "ocm @{env} -- status"],
|
||||
"evidence": ["stable channel start output", "pre-upgrade gateway status", "pre-upgrade OpenClaw status"]
|
||||
},
|
||||
{
|
||||
"id": "upgrade",
|
||||
"title": "Upgrade To Local Build",
|
||||
"intent": "Run the real OpenClaw upgrade path to the release-shaped local build runtime.",
|
||||
"commands": ["ocm upgrade {env} {upgradeSelector} --json"],
|
||||
"evidence": ["local-build upgrade JSON", "snapshot id", "doctor/update output", "rollback status"]
|
||||
},
|
||||
{
|
||||
"id": "post-upgrade",
|
||||
"title": "Post-Local-Build Upgrade Checks",
|
||||
"intent": "Verify the local build handles plugin indexes, runtime deps, doctor recovery, and gateway readiness after upgrade.",
|
||||
"commands": ["ocm service status {env} --json", "ocm @{env} -- status", "ocm @{env} -- plugins list", "ocm @{env} -- doctor --fix", "ocm logs {env} --tail 400 --raw"],
|
||||
"evidence": ["gateway state", "status output", "plugins install index", "doctor output", "gateway logs without missing dependency/plugin load failures"]
|
||||
}
|
||||
]
|
||||
}
|
||||
11
src/main.mjs
11
src/main.mjs
@ -169,11 +169,12 @@ async function matrixCommand(flags) {
|
||||
const target = required(flags.target, "--target");
|
||||
const targetPlan = resolveTarget(target, "target");
|
||||
validateProfileTarget(profile, targetPlan);
|
||||
if (flags.from) {
|
||||
resolveTarget(flags.from, "from");
|
||||
}
|
||||
const fromPlan = flags.from ? resolveTarget(flags.from, "from") : null;
|
||||
const platform = platformInfo();
|
||||
const entries = applyMatrixControls(await expandProfile(profile), flags, platform);
|
||||
for (const entry of entries.filter((item) => !item.skipReason)) {
|
||||
validateScenarioRun(entry.scenario, flags, { targetPlan, fromPlan });
|
||||
}
|
||||
const response = {
|
||||
schemaVersion: "kova.matrix.plan.v1",
|
||||
generatedAt: new Date().toISOString(),
|
||||
@ -305,7 +306,7 @@ async function matrixRun(flags) {
|
||||
const baselineStore = baselinePath ? await loadBaselineStore(baselinePath) : null;
|
||||
preflightGateRun({ entries, flags });
|
||||
for (const entry of entries.filter((item) => !item.skipReason)) {
|
||||
validateScenarioRun(entry.scenario, flags);
|
||||
validateScenarioRun(entry.scenario, flags, { targetPlan, fromPlan });
|
||||
}
|
||||
const reportRoot = flags.report_dir ? resolveFromCwd(flags.report_dir) : reportsDir;
|
||||
const runId = createRunId();
|
||||
@ -816,7 +817,7 @@ async function run(flags) {
|
||||
const state = await loadState(flags.state ?? "fresh");
|
||||
const scenarios = await loadScenarios(flags.scenario);
|
||||
for (const scenario of scenarios) {
|
||||
validateScenarioRun(scenario, flags);
|
||||
validateScenarioRun(scenario, flags, { targetPlan, fromPlan });
|
||||
}
|
||||
|
||||
const reportRoot = flags.report_dir ? resolveFromCwd(flags.report_dir) : reportsDir;
|
||||
|
||||
@ -39,6 +39,12 @@ export function validateScenarioShape(scenario, sourceName = "scenario") {
|
||||
validateStringArray(scenario.tags, "tags", errors);
|
||||
validateStringArray(scenario.states, "states", errors, { optional: true });
|
||||
validateStringArray(scenario.targetKinds, "targetKinds", errors, { optional: true });
|
||||
validateStringArray(scenario.targetValues, "targetValues", errors, { optional: true });
|
||||
validateStringArray(scenario.fromKinds, "fromKinds", errors, { optional: true });
|
||||
validateStringArray(scenario.fromValues, "fromValues", errors, { optional: true });
|
||||
if (scenario.requiresFrom !== undefined && typeof scenario.requiresFrom !== "boolean") {
|
||||
errors.push("requiresFrom must be a boolean when set");
|
||||
}
|
||||
validatePhases(scenario.phases, errors);
|
||||
|
||||
assertNoShapeErrors(errors, sourceName);
|
||||
@ -143,11 +149,18 @@ function validateStringArray(values, key, errors, options = {}) {
|
||||
}
|
||||
}
|
||||
|
||||
export function validateScenarioRun(scenario, flags) {
|
||||
export function validateScenarioRun(scenario, flags, context = {}) {
|
||||
const needsSourceEnv = scenarioUsesSourceEnv(scenario);
|
||||
if (needsSourceEnv && flags.execute === true && !flags.source_env) {
|
||||
throw new Error(`${scenario.id} execution requires --source-env <env>`);
|
||||
}
|
||||
validateTargetContract(scenario, context.targetPlan, "target", "targetKinds", "targetValues");
|
||||
if (scenario.requiresFrom === true && !context.fromPlan) {
|
||||
throw new Error(`${scenario.id} requires --from <selector>`);
|
||||
}
|
||||
if (context.fromPlan) {
|
||||
validateTargetContract(scenario, context.fromPlan, "from", "fromKinds", "fromValues");
|
||||
}
|
||||
}
|
||||
|
||||
function scenarioUsesSourceEnv(scenario) {
|
||||
@ -156,6 +169,20 @@ function scenarioUsesSourceEnv(scenario) {
|
||||
);
|
||||
}
|
||||
|
||||
function validateTargetContract(scenario, plan, role, kindKey, valueKey) {
|
||||
if (!plan) {
|
||||
return;
|
||||
}
|
||||
const allowedKinds = scenario[kindKey] ?? [];
|
||||
if (allowedKinds.length > 0 && !allowedKinds.includes(plan.kind)) {
|
||||
throw new Error(`${scenario.id} supports ${role} kind ${allowedKinds.join(", ")}, got ${plan.kind}`);
|
||||
}
|
||||
const allowedValues = scenario[valueKey] ?? [];
|
||||
if (allowedValues.length > 0 && !allowedValues.includes(plan.value)) {
|
||||
throw new Error(`${scenario.id} supports ${role} value ${allowedValues.join(", ")}, got ${plan.value}`);
|
||||
}
|
||||
}
|
||||
|
||||
export function materializeCommands(commands, values) {
|
||||
return commands.map((command) =>
|
||||
command
|
||||
|
||||
@ -113,6 +113,34 @@ export async function runSelfCheck(flags = {}) {
|
||||
checks.push(await jsonCommandCheck("matrix-plan-repeat-json", "node bin/kova.mjs matrix plan --profile smoke --target runtime:stable --include scenario:fresh-install --repeat 3 --json", (data) => {
|
||||
assertEqual(data.controls?.repeat, 3, "matrix repeat control");
|
||||
}));
|
||||
checks.push(await jsonCommandCheck("channel-upgrade-plan-json", "node bin/kova.mjs matrix plan --profile channel-upgrade --target channel:beta --json", (data) => {
|
||||
assertEqual(data.profile?.id, "channel-upgrade", "channel upgrade profile id");
|
||||
assertEqual(data.target, "channel:beta", "channel upgrade target");
|
||||
assertEqual(data.entries?.[0]?.scenario?.id, "upgrade-stable-channel-to-beta", "channel upgrade scenario");
|
||||
}));
|
||||
checks.push(await failingCommandCheck(
|
||||
"channel-upgrade-rejects-wrong-target-value",
|
||||
"node bin/kova.mjs matrix plan --profile channel-upgrade --target channel:stable --json",
|
||||
"upgrade-stable-channel-to-beta supports target value beta, got stable"
|
||||
));
|
||||
checks.push(await jsonCommandCheck("local-build-upgrade-plan-json", "node bin/kova.mjs matrix plan --profile local-build-upgrade --target local-build:/tmp/openclaw --include scenario:upgrade-stable-channel-to-local-build --json", (data) => {
|
||||
assertEqual(data.profile?.id, "local-build-upgrade", "local-build upgrade profile id");
|
||||
assertEqual(data.entries?.[0]?.scenario?.id, "upgrade-stable-channel-to-local-build", "local-build stable upgrade scenario");
|
||||
}));
|
||||
checks.push(await jsonCommandCheck("channel-upgrade-dry-run-json", `node bin/kova.mjs run --target channel:beta --scenario upgrade-stable-channel-to-beta --state stable-channel-user --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
|
||||
const record = report.records?.[0];
|
||||
const commands = (record?.phases ?? []).flatMap((phase) => phase.commands ?? []);
|
||||
assertEqual(commands.some((command) => command.includes("ocm start") && command.includes("--channel stable")), true, "stable start command present");
|
||||
assertEqual(commands.some((command) => command.includes("ocm upgrade") && /--channel '?beta'?/.test(command)), true, "beta upgrade command present");
|
||||
}));
|
||||
checks.push(await jsonCommandCheck("durable-clone-local-build-dry-run-json", `node bin/kova.mjs run --target local-build:/tmp/openclaw --scenario upgrade-durable-clone-to-local-build --state plugin-index --source-env 'Team Env' --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
|
||||
const record = report.records?.[0];
|
||||
const commands = (record?.phases ?? []).flatMap((phase) => phase.commands ?? []);
|
||||
assertEqual(commands.some((command) => command.includes("ocm env clone 'Team Env'")), true, "quoted source env clone command present");
|
||||
assertEqual(commands.some((command) => command.includes("ocm upgrade") && /--runtime '?kova-local-/.test(command)), true, "local-build runtime upgrade command present");
|
||||
}));
|
||||
checks.push(await jsonCommandCheck("run-auth-default-mock-json", `node bin/kova.mjs run --target runtime:stable --scenario fresh-install --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
|
||||
const record = report.records?.[0];
|
||||
|
||||
@ -35,7 +35,8 @@
|
||||
"config-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install"
|
||||
"fresh-install",
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "onboarding-migration",
|
||||
|
||||
@ -35,7 +35,8 @@
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install",
|
||||
"plugin-lifecycle"
|
||||
"plugin-lifecycle",
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "plugin-index-migration",
|
||||
|
||||
29
states/stable-channel-user.json
Normal file
29
states/stable-channel-user.json
Normal file
@ -0,0 +1,29 @@
|
||||
{
|
||||
"id": "stable-channel-user",
|
||||
"title": "Stable Channel User",
|
||||
"objective": "A disposable OpenClaw env that has been started on the stable release channel before testing channel or local-build upgrade behavior.",
|
||||
"tags": ["upgrade", "channel", "stable", "existing-user"],
|
||||
"setup": [
|
||||
{
|
||||
"id": "write-stable-channel-marker",
|
||||
"title": "Write Stable Channel Marker",
|
||||
"intent": "Persist stable-channel metadata after the env is first started so reports can prove the upgrade source shape.",
|
||||
"afterPhases": ["start", "clone"],
|
||||
"commands": [
|
||||
"ocm env exec {env} -- node -e 'const fs=require(\"fs\"), path=require(\"path\"); const home=process.env.OPENCLAW_HOME; fs.mkdirSync(path.join(home,\"config\"),{recursive:true}); fs.writeFileSync(path.join(home,\"config\",\"channel.json\"),JSON.stringify({schemaVersion:\"kova.fixture.channel.v1\",channel:\"stable\"},null,2)); fs.writeFileSync(path.join(home,\".openclaw-channel\"),\"stable\\n\");'"
|
||||
],
|
||||
"evidence": ["stable channel marker exists"]
|
||||
}
|
||||
],
|
||||
"traits": ["existing-user", "channel-state"],
|
||||
"compatibleSurfaces": ["upgrade-existing-user"],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "release-channel-upgrade",
|
||||
"ownerArea": "upgrade",
|
||||
"setupEvidence": ["stable channel marker exists"],
|
||||
"cleanupGuarantees": ["disposable env cleanup removes stable channel fixture files"],
|
||||
"source": {
|
||||
"kind": "generated",
|
||||
"note": "The scenario starts the env through the real stable channel before upgrading it."
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user