refactor: make requirements own surface contracts
This commit is contained in:
parent
03717c9d1c
commit
dc011f4305
@ -17,14 +17,11 @@ Required fields:
|
||||
- `purposes`: Kova purposes this surface is relevant to, such as `release`,
|
||||
`regression`, `diagnostic`, `performance`, `upgrade`, `plugin`, `provider`,
|
||||
or `soak`.
|
||||
- `requiredStates`: state ids or traits expected for meaningful coverage.
|
||||
- `targetKinds`: target kinds that can run the surface.
|
||||
- `requiredMetrics`: metric ids from `metrics/known.json`.
|
||||
- `processRoles`: role ids from `process-roles/*.json`.
|
||||
- `thresholds`: default pass/fail thresholds for the surface.
|
||||
- `diagnostics`: source-build timeline expectations when available.
|
||||
- `requirements`: stable requirement ids for this surface. Each requirement can
|
||||
narrow the states, state traits, target kinds, and metrics that prove part of
|
||||
- `requirements`: stable requirement ids for this surface. Each requirement owns
|
||||
the states or state traits, target kinds, and metrics that prove that part of
|
||||
the surface contract.
|
||||
|
||||
Then:
|
||||
@ -53,8 +50,6 @@ Required fields:
|
||||
- `title`: short human name.
|
||||
- `objective`: what user history or degraded condition this state models.
|
||||
- `traits`: known traits validated by Kova.
|
||||
- `compatibleSurfaces`: surface ids this state can be paired with.
|
||||
- `incompatibleSurfaces`: surface ids this state must not be paired with.
|
||||
- `riskArea`: what can break when this state is used.
|
||||
- `ownerArea`: OpenClaw subsystem most likely to own state-specific failures.
|
||||
- `setupEvidence`: what proves setup happened.
|
||||
@ -65,10 +60,10 @@ inside disposable Kova envs and make the evidence explicit. Existing user state
|
||||
must be represented through clone/import metadata, not direct mutation of a
|
||||
durable env.
|
||||
|
||||
Prefer requirement-level state ids or state traits on the surface over broad
|
||||
state compatibility lists. Keep `incompatibleSurfaces` for hard safety blocks.
|
||||
Treat broad compatibility metadata as migration support until the resolver owns
|
||||
the pairing decision.
|
||||
Put positive state compatibility on surface requirements through `states` or
|
||||
`stateTraits`. Add `incompatibleSurfaces` only for hard safety blocks where a
|
||||
fixture must never run against a surface; do not store empty compatibility
|
||||
lists.
|
||||
|
||||
Then:
|
||||
|
||||
@ -91,7 +86,8 @@ Self-check and plan validation must fail for:
|
||||
metrics
|
||||
- invalid state traits
|
||||
- malformed lifecycle phases
|
||||
- scenario/state pairs that violate compatibility
|
||||
- scenario/state pairs that violate requirement state contracts or hard
|
||||
incompatibility blocks
|
||||
- profile entries that require unknown surfaces or states
|
||||
|
||||
If a new surface or state needs exceptions to these rules, the contract is too
|
||||
|
||||
@ -1,3 +1,5 @@
|
||||
import { stateSatisfiesRequirement } from "../registries/surface-requirements.mjs";
|
||||
|
||||
export const RESOLVED_COVERAGE_SCHEMA = "kova.resolvedCoverage.v1";
|
||||
|
||||
export function resolveCoverageObligations({ profile, entries, surfaces, targetPlan }) {
|
||||
@ -48,7 +50,6 @@ export function resolveCoverageObligations({ profile, entries, surfaces, targetP
|
||||
|
||||
const stateResult = stateSatisfiesRequirement(state, requirement);
|
||||
const targetResult = targetSatisfiesRequirement(targetPlan, requirement);
|
||||
warnings.push(...legacyCompatibilityWarnings({ entry, surface, requirement }));
|
||||
const status = entry.skipReason
|
||||
? "skipped"
|
||||
: !stateResult.ok
|
||||
@ -81,49 +82,6 @@ export function resolveCoverageObligations({ profile, entries, surfaces, targetP
|
||||
};
|
||||
}
|
||||
|
||||
function legacyCompatibilityWarnings({ entry, surface, requirement }) {
|
||||
const warnings = [];
|
||||
const scenario = entry.scenario;
|
||||
const state = entry.state;
|
||||
if (!scenario || !state || !surface || !requirement) {
|
||||
return warnings;
|
||||
}
|
||||
|
||||
if ((state.compatibleSurfaces ?? []).length > 0 && !state.compatibleSurfaces.includes(surface.id)) {
|
||||
warnings.push({
|
||||
kind: "legacy-compatibility-disagreement",
|
||||
surface: surface.id,
|
||||
scenario: scenario.id,
|
||||
state: state.id,
|
||||
requirement: requirement.id,
|
||||
message: `state '${state.id}' does not list surface '${surface.id}' in compatibleSurfaces`
|
||||
});
|
||||
}
|
||||
if ((state.incompatibleSurfaces ?? []).includes(surface.id)) {
|
||||
warnings.push({
|
||||
kind: "legacy-compatibility-disagreement",
|
||||
surface: surface.id,
|
||||
scenario: scenario.id,
|
||||
state: state.id,
|
||||
requirement: requirement.id,
|
||||
message: `state '${state.id}' lists surface '${surface.id}' in incompatibleSurfaces`
|
||||
});
|
||||
}
|
||||
if ((surface.requiredStates ?? []).length > 0 &&
|
||||
(scenario.states ?? []).length === 0 &&
|
||||
!surface.requiredStates.includes(state.id)) {
|
||||
warnings.push({
|
||||
kind: "legacy-compatibility-disagreement",
|
||||
surface: surface.id,
|
||||
scenario: scenario.id,
|
||||
state: state.id,
|
||||
requirement: requirement.id,
|
||||
message: `surface '${surface.id}' requiredStates does not include state '${state.id}'`
|
||||
});
|
||||
}
|
||||
return warnings;
|
||||
}
|
||||
|
||||
export function assertResolvedCoverageIsRunnable(resolved) {
|
||||
const invalid = (resolved?.obligations ?? []).filter((obligation) =>
|
||||
["invalid", "missing-proof", "unsupported-state", "unsupported-target"].includes(obligation.status)
|
||||
@ -155,25 +113,6 @@ function obligationFor(entry, options) {
|
||||
};
|
||||
}
|
||||
|
||||
function stateSatisfiesRequirement(state, requirement) {
|
||||
const states = requirement.states ?? [];
|
||||
const traits = requirement.stateTraits ?? [];
|
||||
if (states.length === 0 && traits.length === 0) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
if (state?.id && states.includes(state.id)) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
const stateTraits = new Set(state?.traits ?? []);
|
||||
if (traits.some((trait) => stateTraits.has(trait))) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: `state '${state?.id ?? "unknown"}' does not satisfy requirement state ids or traits`
|
||||
};
|
||||
}
|
||||
|
||||
function targetSatisfiesRequirement(targetPlan, requirement) {
|
||||
const targetKinds = requirement.targetKinds ?? [];
|
||||
if (targetKinds.length === 0 || targetKinds.includes(targetPlan?.kind)) {
|
||||
|
||||
@ -42,7 +42,7 @@ export function validateScenarioShape(scenario, sourceName = "scenario") {
|
||||
validateStringArray(scenario.targetValues, "targetValues", errors, { optional: true });
|
||||
validateStringArray(scenario.fromKinds, "fromKinds", errors, { optional: true });
|
||||
validateStringArray(scenario.fromValues, "fromValues", errors, { optional: true });
|
||||
validateStringArray(scenario.proves, "proves", errors, { optional: true });
|
||||
validateStringArray(scenario.proves, "proves", errors);
|
||||
if (scenario.requiresFrom !== undefined && typeof scenario.requiresFrom !== "boolean") {
|
||||
errors.push("requiresFrom must be a boolean when set");
|
||||
}
|
||||
|
||||
@ -52,8 +52,6 @@ export function validateStateShape(state, sourceName = "state") {
|
||||
requireString(state, "objective", errors);
|
||||
requireArray(state, "tags", errors);
|
||||
requireArray(state, "traits", errors);
|
||||
requireArray(state, "compatibleSurfaces", errors);
|
||||
requireArray(state, "incompatibleSurfaces", errors);
|
||||
requireString(state, "riskArea", errors);
|
||||
requireString(state, "ownerArea", errors);
|
||||
requireArray(state, "setupEvidence", errors);
|
||||
@ -69,7 +67,9 @@ export function validateStateShape(state, sourceName = "state") {
|
||||
validateSteps(state.prepare, "prepare", errors, { phaseBinding: false });
|
||||
validateSteps(state.setup, "setup", errors, { phaseBinding: true });
|
||||
validateSteps(state.cleanup, "cleanup", errors, { phaseBinding: false });
|
||||
validateStringArray(state.compatibleSurfaces, "compatibleSurfaces", errors, { optional: true });
|
||||
if (state.compatibleSurfaces !== undefined) {
|
||||
errors.push("compatibleSurfaces is not supported; surface requirements own positive state compatibility");
|
||||
}
|
||||
validateStringArray(state.incompatibleSurfaces, "incompatibleSurfaces", errors, { optional: true });
|
||||
validateStringArray(state.traits, "traits", errors);
|
||||
validateStringArray(state.setupEvidence, "setupEvidence", errors, { nonEmpty: true });
|
||||
|
||||
80
src/registries/surface-requirements.mjs
Normal file
80
src/registries/surface-requirements.mjs
Normal file
@ -0,0 +1,80 @@
|
||||
export const knownTargetKinds = ["npm", "channel", "runtime", "local-build"];
|
||||
|
||||
export function requirementsForScenario(surface, scenario) {
|
||||
return requirementsForIds(surface, scenario?.proves ?? []);
|
||||
}
|
||||
|
||||
export function requirementsForIds(surface, ids) {
|
||||
const requirements = surface?.requirements ?? [];
|
||||
if (!Array.isArray(ids) || ids.length === 0) {
|
||||
return [];
|
||||
}
|
||||
const byId = new Map(requirements.map((requirement) => [requirement.id, requirement]));
|
||||
return ids.map((id) => byId.get(id)).filter(Boolean);
|
||||
}
|
||||
|
||||
export function targetKindsForRequirements(requirements) {
|
||||
return [...new Set((requirements ?? []).flatMap((requirement) => requirement.targetKinds ?? []))].sort();
|
||||
}
|
||||
|
||||
export function stateSatisfiesRequirement(state, requirement) {
|
||||
const states = requirement?.states ?? [];
|
||||
const traits = requirement?.stateTraits ?? [];
|
||||
if (states.length === 0 && traits.length === 0) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
if (state?.id && states.includes(state.id)) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
const stateTraits = new Set(state?.traits ?? []);
|
||||
if (traits.some((trait) => stateTraits.has(trait))) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: `state '${state?.id ?? "unknown"}' does not satisfy requirement state ids or traits`
|
||||
};
|
||||
}
|
||||
|
||||
export function scenarioSupportsState({ scenario, surface, state }) {
|
||||
if ((scenario?.states ?? []).length > 0) {
|
||||
return {
|
||||
ok: scenario.states.includes(state?.id),
|
||||
reason: scenario.states.includes(state?.id)
|
||||
? null
|
||||
: `scenario '${scenario.id}' supports only states: ${scenario.states.join(", ")}`
|
||||
};
|
||||
}
|
||||
|
||||
const requirements = requirementsForScenario(surface, scenario);
|
||||
if (requirements.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `scenario '${scenario?.id ?? "unknown"}' has no known requirements for surface '${surface?.id ?? "unknown"}'`
|
||||
};
|
||||
}
|
||||
if (requirements.some((requirement) => stateSatisfiesRequirement(state, requirement).ok)) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: `state '${state?.id ?? "unknown"}' does not satisfy scenario '${scenario.id}' requirement state ids or traits`
|
||||
};
|
||||
}
|
||||
|
||||
export function surfaceSupportsState({ surface, state }) {
|
||||
const requirements = surface?.requirements ?? [];
|
||||
if (requirements.length === 0) {
|
||||
return {
|
||||
ok: false,
|
||||
reason: `surface '${surface?.id ?? "unknown"}' has no requirements`
|
||||
};
|
||||
}
|
||||
if (requirements.some((requirement) => stateSatisfiesRequirement(state, requirement).ok)) {
|
||||
return { ok: true, reason: null };
|
||||
}
|
||||
return {
|
||||
ok: false,
|
||||
reason: `state '${state?.id ?? "unknown"}' does not satisfy surface '${surface.id}' requirement state ids or traits`
|
||||
};
|
||||
}
|
||||
@ -1,6 +1,7 @@
|
||||
import { surfacesDir } from "../paths.mjs";
|
||||
import { validatePurposes } from "./purposes.mjs";
|
||||
import { assertNoShapeErrors, loadJsonRegistry, requireArray, requireKebabId, requireObject, requireString } from "./validate.mjs";
|
||||
import { knownTargetKinds } from "./surface-requirements.mjs";
|
||||
|
||||
export async function loadSurfaces(selectedId) {
|
||||
return loadJsonRegistry({
|
||||
@ -17,13 +18,19 @@ export function validateSurfaceShape(surface, sourceName = "surface") {
|
||||
requireString(surface, "title", errors);
|
||||
requireString(surface, "ownerArea", errors);
|
||||
requireString(surface, "description", errors);
|
||||
requireArray(surface, "requiredMetrics", errors);
|
||||
requireArray(surface, "processRoles", errors);
|
||||
requireObject(surface, "thresholds", errors);
|
||||
requireObject(surface, "diagnostics", errors);
|
||||
requireArray(surface, "requirements", errors);
|
||||
validatePurposes(surface.purposes, "purposes", errors, { optional: true });
|
||||
|
||||
for (const key of ["requiredMetrics", "processRoles", "requiredStates", "targetKinds"]) {
|
||||
for (const key of ["requiredStates", "targetKinds", "requiredMetrics"]) {
|
||||
if (surface[key] !== undefined) {
|
||||
errors.push(`${key} is not supported on surfaces; put requirement-specific contract data in requirements[]`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of ["processRoles"]) {
|
||||
if (surface[key] === undefined) {
|
||||
continue;
|
||||
}
|
||||
@ -54,15 +61,11 @@ export function validateSurfaceShape(surface, sourceName = "surface") {
|
||||
}
|
||||
|
||||
function validateRequirements(requirements, errors) {
|
||||
if (requirements === undefined) {
|
||||
return;
|
||||
}
|
||||
if (!Array.isArray(requirements)) {
|
||||
errors.push("requirements must be an array when set");
|
||||
return;
|
||||
}
|
||||
if (requirements.length === 0) {
|
||||
errors.push("requirements must not be empty when set");
|
||||
errors.push("requirements must not be empty");
|
||||
}
|
||||
|
||||
const ids = new Set();
|
||||
@ -77,12 +80,27 @@ function validateRequirements(requirements, errors) {
|
||||
}
|
||||
validateStringArray(requirement?.states, `${prefix}.states`, errors, { optional: true });
|
||||
validateStringArray(requirement?.stateTraits, `${prefix}.stateTraits`, errors, { optional: true });
|
||||
validateStringArray(requirement?.targetKinds, `${prefix}.targetKinds`, errors, { optional: true });
|
||||
validateStringArray(requirement?.metrics, `${prefix}.metrics`, errors, { optional: true });
|
||||
if (!Array.isArray(requirement?.states) && !Array.isArray(requirement?.stateTraits)) {
|
||||
errors.push(`${prefix} must define states or stateTraits`);
|
||||
}
|
||||
validateStringArray(requirement?.targetKinds, `${prefix}.targetKinds`, errors);
|
||||
validateKnownTargetKinds(requirement?.targetKinds, `${prefix}.targetKinds`, errors);
|
||||
validateStringArray(requirement?.metrics, `${prefix}.metrics`, errors);
|
||||
validatePurposes(requirement?.purposes, `${prefix}.purposes`, errors, { optional: true });
|
||||
}
|
||||
}
|
||||
|
||||
function validateKnownTargetKinds(values, prefix, errors) {
|
||||
if (!Array.isArray(values)) {
|
||||
return;
|
||||
}
|
||||
for (const [index, value] of values.entries()) {
|
||||
if (typeof value === "string" && !knownTargetKinds.includes(value)) {
|
||||
errors.push(`${prefix}[${index}] references unknown target kind '${value}'`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function validateRoleThresholds(value, prefix, errors) {
|
||||
if (value === undefined) {
|
||||
return;
|
||||
|
||||
@ -1,6 +1,13 @@
|
||||
import { readFile, readdir } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { isKnownPlatformCoverageKey } from "../platform.mjs";
|
||||
import {
|
||||
knownTargetKinds,
|
||||
requirementsForScenario,
|
||||
scenarioSupportsState,
|
||||
surfaceSupportsState,
|
||||
targetKindsForRequirements
|
||||
} from "./surface-requirements.mjs";
|
||||
|
||||
export async function loadJsonRegistry({ dir, kind, selectedId, validate }) {
|
||||
const names = await readdir(dir);
|
||||
@ -47,11 +54,6 @@ export function validateRegistryReferences({ scenarios, states, profiles, surfac
|
||||
}
|
||||
|
||||
for (const state of states) {
|
||||
for (const surface of state.compatibleSurfaces ?? []) {
|
||||
if (!surfaceIds.has(surface)) {
|
||||
errors.push(`state '${state.id}' compatibleSurfaces references unknown surface '${surface}'`);
|
||||
}
|
||||
}
|
||||
for (const surface of state.incompatibleSurfaces ?? []) {
|
||||
if (!surfaceIds.has(surface)) {
|
||||
errors.push(`state '${state.id}' incompatibleSurfaces references unknown surface '${surface}'`);
|
||||
@ -70,13 +72,7 @@ export function validateRegistryReferences({ scenarios, states, profiles, surfac
|
||||
errors.push(`surface '${surface.id}' roleThresholds references unknown process role '${role}'`);
|
||||
}
|
||||
}
|
||||
for (const state of surface.requiredStates ?? []) {
|
||||
if (!stateIds.has(state)) {
|
||||
errors.push(`surface '${surface.id}' references unknown required state '${state}'`);
|
||||
}
|
||||
}
|
||||
validateSurfaceRequirements(surface, { stateIds, traitIds, metricIds }, errors);
|
||||
validateMetricList(surface.requiredMetrics ?? [], metricIds, errors, `surface '${surface.id}' requiredMetrics`);
|
||||
validateThresholdMetrics(surface.thresholds ?? {}, metricIds, errors, `surface '${surface.id}' thresholds`);
|
||||
for (const [role, thresholds] of Object.entries(surface.roleThresholds ?? {})) {
|
||||
validateThresholdMetrics(thresholds, metricIds, errors, `surface '${surface.id}' roleThresholds.${role}`);
|
||||
@ -103,13 +99,17 @@ function validateScenarioContract(scenario, surface, refs, errors) {
|
||||
errors.push(`scenario '${scenario.id}' processRoles references unknown process role '${role}'`);
|
||||
}
|
||||
}
|
||||
const surfaceTargetKinds = new Set(surface.targetKinds ?? []);
|
||||
const scenarioRequirements = requirementsForScenario(surface, scenario);
|
||||
const surfaceTargetKinds = new Set(targetKindsForRequirements(scenarioRequirements));
|
||||
for (const targetKind of scenario.targetKinds ?? []) {
|
||||
if (surfaceTargetKinds.size > 0 && !surfaceTargetKinds.has(targetKind)) {
|
||||
errors.push(`scenario '${scenario.id}' targetKinds references '${targetKind}' which is not supported by surface '${surface.id}'`);
|
||||
errors.push(`scenario '${scenario.id}' targetKinds references '${targetKind}' which is not supported by proved requirements on surface '${surface.id}'`);
|
||||
}
|
||||
}
|
||||
const requirementIds = new Set((surface.requirements ?? []).map((requirement) => requirement.id));
|
||||
if ((scenario.proves ?? []).length === 0) {
|
||||
errors.push(`scenario '${scenario.id}' must prove at least one requirement for surface '${surface.id}'`);
|
||||
}
|
||||
for (const requirement of scenario.proves ?? []) {
|
||||
if (requirementIds.size > 0 && !requirementIds.has(requirement)) {
|
||||
errors.push(`scenario '${scenario.id}' proves unknown surface requirement '${surface.id}.${requirement}'`);
|
||||
@ -119,7 +119,6 @@ function validateScenarioContract(scenario, surface, refs, errors) {
|
||||
}
|
||||
|
||||
function validateSurfaceRequirements(surface, refs, errors) {
|
||||
const surfaceTargetKinds = new Set(surface.targetKinds ?? []);
|
||||
for (const requirement of surface.requirements ?? []) {
|
||||
const prefix = `surface '${surface.id}' requirement '${requirement.id}'`;
|
||||
for (const state of requirement.states ?? []) {
|
||||
@ -133,8 +132,8 @@ function validateSurfaceRequirements(surface, refs, errors) {
|
||||
}
|
||||
}
|
||||
for (const targetKind of requirement.targetKinds ?? []) {
|
||||
if (surfaceTargetKinds.size > 0 && !surfaceTargetKinds.has(targetKind)) {
|
||||
errors.push(`${prefix} targetKinds references '${targetKind}' which is not supported by surface '${surface.id}'`);
|
||||
if (!knownTargetKinds.includes(targetKind)) {
|
||||
errors.push(`${prefix} targetKinds references unknown target kind '${targetKind}'`);
|
||||
}
|
||||
}
|
||||
validateMetricList(requirement.metrics ?? [], refs.metricIds, errors, `${prefix} metrics`);
|
||||
@ -264,12 +263,9 @@ function validateScenarioStatePair({ profileId, location, scenarioId, stateId, r
|
||||
if (!surface) {
|
||||
return;
|
||||
}
|
||||
const allowedStates = scenario.states?.length > 0 ? scenario.states : surface.requiredStates ?? [];
|
||||
if (allowedStates.length > 0 && !allowedStates.includes(state.id)) {
|
||||
errors.push(`profile '${profileId}' ${location} pairs scenario '${scenario.id}' with state '${state.id}', but surface/scenario allows only: ${allowedStates.join(", ")}`);
|
||||
}
|
||||
if ((state.compatibleSurfaces ?? []).length > 0 && !state.compatibleSurfaces.includes(scenario.surface)) {
|
||||
errors.push(`profile '${profileId}' ${location} pairs state '${state.id}' with incompatible surface '${scenario.surface}'; compatible surfaces: ${state.compatibleSurfaces.join(", ")}`);
|
||||
const stateResult = scenarioSupportsState({ scenario, surface, state });
|
||||
if (!stateResult.ok) {
|
||||
errors.push(`profile '${profileId}' ${location} pairs scenario '${scenario.id}' with state '${state.id}', but ${stateResult.reason}`);
|
||||
}
|
||||
if ((state.incompatibleSurfaces ?? []).includes(scenario.surface)) {
|
||||
errors.push(`profile '${profileId}' ${location} pairs state '${state.id}' with explicitly incompatible surface '${scenario.surface}'`);
|
||||
@ -312,11 +308,9 @@ function validateStateSurfacePair({ profileId, location, surfaceId, stateId, ref
|
||||
if (!surface || !state) {
|
||||
return;
|
||||
}
|
||||
if ((surface.requiredStates ?? []).length > 0 && !surface.requiredStates.includes(state.id)) {
|
||||
errors.push(`profile '${profileId}' ${location} requires '${surface.id}:${state.id}', but surface allows only: ${surface.requiredStates.join(", ")}`);
|
||||
}
|
||||
if ((state.compatibleSurfaces ?? []).length > 0 && !state.compatibleSurfaces.includes(surface.id)) {
|
||||
errors.push(`profile '${profileId}' ${location} requires '${surface.id}:${state.id}', but state compatible surfaces are: ${state.compatibleSurfaces.join(", ")}`);
|
||||
const stateResult = surfaceSupportsState({ surface, state });
|
||||
if (!stateResult.ok) {
|
||||
errors.push(`profile '${profileId}' ${location} requires '${surface.id}:${state.id}', but ${stateResult.reason}`);
|
||||
}
|
||||
if ((state.incompatibleSurfaces ?? []).includes(surface.id)) {
|
||||
errors.push(`profile '${profileId}' ${location} requires explicitly incompatible state/surface pair '${surface.id}:${state.id}'`);
|
||||
|
||||
@ -4385,8 +4385,6 @@ function stateRegistryValidationCheck() {
|
||||
objective: "Invalid state fixture",
|
||||
tags: [],
|
||||
traits: ["not-a-real-trait"],
|
||||
compatibleSurfaces: [],
|
||||
incompatibleSurfaces: [],
|
||||
riskArea: "test",
|
||||
ownerArea: "test",
|
||||
setupEvidence: ["evidence"],
|
||||
@ -4406,8 +4404,6 @@ function stateRegistryValidationCheck() {
|
||||
objective: "Invalid state fixture evidence",
|
||||
tags: [],
|
||||
traits: ["fresh-user"],
|
||||
compatibleSurfaces: [],
|
||||
incompatibleSurfaces: [],
|
||||
riskArea: "test",
|
||||
ownerArea: "test",
|
||||
setupEvidence: [],
|
||||
@ -4426,6 +4422,7 @@ function stateRegistryValidationCheck() {
|
||||
scenarios: [{
|
||||
id: "scenario",
|
||||
surface: "known-surface",
|
||||
proves: ["baseline"],
|
||||
states: [],
|
||||
targetKinds: [],
|
||||
processRoles: []
|
||||
@ -4433,22 +4430,25 @@ function stateRegistryValidationCheck() {
|
||||
states: [{
|
||||
id: "state",
|
||||
traits: ["fresh-user"],
|
||||
compatibleSurfaces: ["missing-surface"],
|
||||
incompatibleSurfaces: []
|
||||
incompatibleSurfaces: ["missing-surface"]
|
||||
}],
|
||||
profiles: [],
|
||||
surfaces: [{
|
||||
id: "known-surface",
|
||||
processRoles: [],
|
||||
requiredStates: [],
|
||||
targetKinds: []
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["state"],
|
||||
targetKinds: ["runtime"],
|
||||
metrics: []
|
||||
}]
|
||||
}],
|
||||
processRoles: []
|
||||
});
|
||||
} catch (error) {
|
||||
rejectedSurface = /compatibleSurfaces references unknown surface/.test(error.message);
|
||||
rejectedSurface = /incompatibleSurfaces references unknown surface/.test(error.message);
|
||||
}
|
||||
assertEqual(rejectedSurface, true, "unknown compatible surface rejected");
|
||||
assertEqual(rejectedSurface, true, "unknown incompatible surface rejected");
|
||||
|
||||
let rejectedPurpose = false;
|
||||
try {
|
||||
@ -4478,17 +4478,12 @@ function stateRegistryValidationCheck() {
|
||||
states: [{
|
||||
id: "state",
|
||||
traits: ["fresh-user"],
|
||||
compatibleSurfaces: ["known-surface"],
|
||||
incompatibleSurfaces: []
|
||||
}],
|
||||
profiles: [],
|
||||
surfaces: [{
|
||||
id: "known-surface",
|
||||
processRoles: [],
|
||||
requiredStates: ["state"],
|
||||
requiredMetrics: ["knownMetric"],
|
||||
thresholds: { knownMetric: 1 },
|
||||
targetKinds: ["runtime"],
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["missing-state"],
|
||||
@ -4522,7 +4517,6 @@ function stateRegistryValidationCheck() {
|
||||
states: [{
|
||||
id: "state",
|
||||
traits: ["fresh-user"],
|
||||
compatibleSurfaces: ["other-surface"],
|
||||
incompatibleSurfaces: ["known-surface"]
|
||||
}],
|
||||
profiles: [{
|
||||
@ -4540,21 +4534,28 @@ function stateRegistryValidationCheck() {
|
||||
{
|
||||
id: "known-surface",
|
||||
processRoles: [],
|
||||
requiredStates: [],
|
||||
targetKinds: []
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["state"],
|
||||
targetKinds: ["runtime"],
|
||||
metrics: []
|
||||
}]
|
||||
},
|
||||
{
|
||||
id: "other-surface",
|
||||
processRoles: [],
|
||||
requiredStates: [],
|
||||
targetKinds: []
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["state"],
|
||||
targetKinds: ["runtime"],
|
||||
metrics: []
|
||||
}]
|
||||
}
|
||||
],
|
||||
processRoles: []
|
||||
});
|
||||
} catch (error) {
|
||||
rejectedCoveragePair = /explicitly incompatible state\/surface pair/.test(error.message) ||
|
||||
/state compatible surfaces/.test(error.message);
|
||||
rejectedCoveragePair = /explicitly incompatible state\/surface pair/.test(error.message);
|
||||
}
|
||||
assertEqual(rejectedCoveragePair, true, "invalid coverage state/surface pair rejected");
|
||||
|
||||
@ -4564,6 +4565,7 @@ function stateRegistryValidationCheck() {
|
||||
scenarios: [{
|
||||
id: "scenario",
|
||||
surface: "known-surface",
|
||||
proves: ["baseline"],
|
||||
thresholds: { madeUpMetric: 1 },
|
||||
states: [],
|
||||
targetKinds: [],
|
||||
@ -4574,10 +4576,13 @@ function stateRegistryValidationCheck() {
|
||||
surfaces: [{
|
||||
id: "known-surface",
|
||||
processRoles: [],
|
||||
requiredStates: [],
|
||||
requiredMetrics: ["knownMetric"],
|
||||
thresholds: { knownMetric: 1 },
|
||||
targetKinds: []
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["state"],
|
||||
targetKinds: ["runtime"],
|
||||
metrics: ["knownMetric"]
|
||||
}]
|
||||
}],
|
||||
processRoles: [],
|
||||
metrics: [{ id: "knownMetric" }]
|
||||
@ -4615,8 +4620,12 @@ function stateRegistryValidationCheck() {
|
||||
surfaces: [{
|
||||
id: "knownSurface",
|
||||
processRoles: [],
|
||||
requiredStates: [],
|
||||
targetKinds: []
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["state"],
|
||||
targetKinds: ["runtime"],
|
||||
metrics: []
|
||||
}]
|
||||
}],
|
||||
processRoles: [{ id: "knownRole" }],
|
||||
metrics: [{ id: "peakRssMb" }]
|
||||
@ -4680,6 +4689,7 @@ function scenarioCloneFirstValidationCheck() {
|
||||
title: "Bad Existing User",
|
||||
objective: "Touches source env without clone-first protection.",
|
||||
tags: ["upgrade"],
|
||||
proves: ["baseline"],
|
||||
thresholds: {},
|
||||
phases: [{
|
||||
id: "status",
|
||||
@ -4702,6 +4712,7 @@ function scenarioCloneFirstValidationCheck() {
|
||||
title: "Bad Existing User Second Source",
|
||||
objective: "References source env after clone.",
|
||||
tags: ["upgrade"],
|
||||
proves: ["baseline"],
|
||||
thresholds: {},
|
||||
phases: [{
|
||||
id: "clone",
|
||||
@ -4722,6 +4733,7 @@ function scenarioCloneFirstValidationCheck() {
|
||||
title: "Good Existing User",
|
||||
objective: "Clone first, then operate only on the disposable env.",
|
||||
tags: ["upgrade"],
|
||||
proves: ["baseline"],
|
||||
thresholds: {},
|
||||
phases: [{
|
||||
id: "clone",
|
||||
@ -4763,6 +4775,7 @@ function scenarioStateCompatibilityCheck() {
|
||||
scenarios: [{
|
||||
id: "upgrade-existing-user",
|
||||
surface: "upgrade-existing-user",
|
||||
proves: ["baseline"],
|
||||
states: [],
|
||||
targetKinds: [],
|
||||
processRoles: []
|
||||
@ -4770,7 +4783,6 @@ function scenarioStateCompatibilityCheck() {
|
||||
states: [{
|
||||
id: "fresh",
|
||||
traits: ["fresh-user"],
|
||||
compatibleSurfaces: ["fresh-install"],
|
||||
incompatibleSurfaces: ["upgrade-existing-user"]
|
||||
}],
|
||||
profiles: [{
|
||||
@ -4780,8 +4792,12 @@ function scenarioStateCompatibilityCheck() {
|
||||
surfaces: [{
|
||||
id: "upgrade-existing-user",
|
||||
processRoles: [],
|
||||
requiredStates: ["old-release-user"],
|
||||
targetKinds: []
|
||||
requirements: [{
|
||||
id: "baseline",
|
||||
states: ["old-release-user"],
|
||||
targetKinds: ["runtime"],
|
||||
metrics: []
|
||||
}]
|
||||
}],
|
||||
processRoles: []
|
||||
});
|
||||
|
||||
@ -19,10 +19,6 @@
|
||||
"mode": "missing",
|
||||
"reason": "This state intentionally bypasses Kova's default mock provider auth to test missing model credentials."
|
||||
},
|
||||
"compatibleSurfaces": [
|
||||
"agent-cli-local-turn"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "agent-provider-auth",
|
||||
"ownerArea": "agent-runtime",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -34,11 +34,6 @@
|
||||
"plugin-pressure",
|
||||
"runtime-deps"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"plugin-lifecycle",
|
||||
"failure-containment"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "plugin-runtime-deps",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -32,10 +32,6 @@
|
||||
"existing-user",
|
||||
"channel-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"cross-platform-smoke"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "release-channel",
|
||||
"ownerArea": "runtime-startup",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -32,11 +32,6 @@
|
||||
"failure-state",
|
||||
"config-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install",
|
||||
"failure-containment"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "config-normalization",
|
||||
"ownerArea": "gateway",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -33,12 +33,6 @@
|
||||
"external-plugin",
|
||||
"plugin-pressure"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"plugin-lifecycle",
|
||||
"plugin-external-install",
|
||||
"plugin-update"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "external-plugin-lifecycle",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -36,10 +36,6 @@
|
||||
"failure-state",
|
||||
"old-release"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "upgrade-recovery",
|
||||
"ownerArea": "upgrade",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -11,22 +11,6 @@
|
||||
"fresh-user",
|
||||
"baseline"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"release-runtime-startup",
|
||||
"fresh-install",
|
||||
"bundled-plugin-startup",
|
||||
"plugin-external-install",
|
||||
"plugin-remove",
|
||||
"plugin-update",
|
||||
"plugin-bad-manifest",
|
||||
"plugin-missing-runtime-deps",
|
||||
"dashboard",
|
||||
"tui",
|
||||
"mcp-runtime",
|
||||
"browser-automation",
|
||||
"media-understanding",
|
||||
"network-offline"
|
||||
],
|
||||
"incompatibleSurfaces": [
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
|
||||
@ -33,10 +33,6 @@
|
||||
"service-state",
|
||||
"existing-user"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"gateway-performance"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "gateway-restart",
|
||||
"ownerArea": "gateway",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -61,11 +61,6 @@
|
||||
"session-state",
|
||||
"performance-pressure"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install",
|
||||
"soak"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "memory-session-load",
|
||||
"ownerArea": "agent-runtime",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -33,11 +33,6 @@
|
||||
"performance-pressure",
|
||||
"workspace-pressure"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"soak",
|
||||
"workspace-scan"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "workspace-scan",
|
||||
"ownerArea": "gateway",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -33,10 +33,6 @@
|
||||
"provider-pressure",
|
||||
"performance-pressure"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"gateway-performance"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "plugin-metadata-scan",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -32,11 +32,6 @@
|
||||
"plugin-pressure",
|
||||
"migration-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install",
|
||||
"bundled-runtime-deps"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "plugin-index-migration",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -19,14 +19,6 @@
|
||||
"provider-pressure",
|
||||
"agent-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"agent-cli-local-turn",
|
||||
"agent-gateway-rpc-turn",
|
||||
"dashboard-session-send-turn",
|
||||
"tui-message-turn",
|
||||
"openai-compatible-turn"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "agent-provider-latency",
|
||||
"ownerArea": "agent-runtime",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -33,10 +33,6 @@
|
||||
"provider-pressure",
|
||||
"configured-auth"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"provider-models"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "provider-discovery",
|
||||
"ownerArea": "providers",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -38,10 +38,6 @@
|
||||
"mode": "missing",
|
||||
"reason": "This state intentionally tests missing model credentials."
|
||||
},
|
||||
"compatibleSurfaces": [
|
||||
"provider-models"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "provider-auth",
|
||||
"ownerArea": "providers",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -2,14 +2,27 @@
|
||||
"id": "official-plugins",
|
||||
"title": "Official Plugins",
|
||||
"objective": "Exercise real published official OpenClaw plugin install paths from the state-owned official plugin list.",
|
||||
"tags": ["plugins", "official-plugin", "install", "security-scan"],
|
||||
"traits": ["fresh-user", "external-plugin", "official-plugin", "plugin-pressure"],
|
||||
"compatibleSurfaces": ["official-plugin-install"],
|
||||
"incompatibleSurfaces": [],
|
||||
"tags": [
|
||||
"plugins",
|
||||
"official-plugin",
|
||||
"install",
|
||||
"security-scan"
|
||||
],
|
||||
"traits": [
|
||||
"fresh-user",
|
||||
"external-plugin",
|
||||
"official-plugin",
|
||||
"plugin-pressure"
|
||||
],
|
||||
"riskArea": "official-plugin-install",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": ["official plugin list declared", "fresh disposable env receives real plugin install commands"],
|
||||
"cleanupGuarantees": ["temporary Kova env is destroyed after execution"],
|
||||
"setupEvidence": [
|
||||
"official plugin list declared",
|
||||
"fresh disposable env receives real plugin install commands"
|
||||
],
|
||||
"cleanupGuarantees": [
|
||||
"temporary Kova env is destroyed after execution"
|
||||
],
|
||||
"officialPlugins": [
|
||||
{
|
||||
"id": "discord",
|
||||
|
||||
@ -33,10 +33,6 @@
|
||||
"migration-state",
|
||||
"config-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "config-migration",
|
||||
"ownerArea": "gateway",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -2,26 +2,41 @@
|
||||
"id": "old-release-2026-4-20-user",
|
||||
"title": "OpenClaw 2026.4.20 User",
|
||||
"objective": "A cloned existing user env that is deliberately moved through OpenClaw 2026.4.20 before testing target upgrade behavior.",
|
||||
"tags": ["existing-user", "upgrade", "old-release", "2026-4-20"],
|
||||
"tags": [
|
||||
"existing-user",
|
||||
"upgrade",
|
||||
"old-release",
|
||||
"2026-4-20"
|
||||
],
|
||||
"setup": [
|
||||
{
|
||||
"id": "write-2026-4-20-markers",
|
||||
"title": "Write 2026.4.20 Fixture Markers",
|
||||
"intent": "Persist non-invasive markers so reports can prove the disposable clone was shaped as a 2026.4.20 upgrade source.",
|
||||
"afterPhases": ["source-runtime"],
|
||||
"afterPhases": [
|
||||
"source-runtime"
|
||||
],
|
||||
"commands": [
|
||||
"ocm env exec {env} -- node -e 'const fs=require(\"fs\"), path=require(\"path\"); const home=process.env.OPENCLAW_HOME; fs.mkdirSync(path.join(home,\"config\"),{recursive:true}); fs.writeFileSync(path.join(home,\"config\",\"kova-source-release.json\"),JSON.stringify({schemaVersion:\"kova.fixture.source-release.v1\",release:\"2026.4.20\",surface:\"upgrade-existing-user\"},null,2));'"
|
||||
],
|
||||
"evidence": ["2026.4.20 source marker exists"]
|
||||
"evidence": [
|
||||
"2026.4.20 source marker exists"
|
||||
]
|
||||
}
|
||||
],
|
||||
"traits": ["existing-user", "old-release", "migration-state"],
|
||||
"compatibleSurfaces": ["upgrade-existing-user"],
|
||||
"incompatibleSurfaces": [],
|
||||
"traits": [
|
||||
"existing-user",
|
||||
"old-release",
|
||||
"migration-state"
|
||||
],
|
||||
"riskArea": "release-upgrade",
|
||||
"ownerArea": "upgrade",
|
||||
"setupEvidence": ["2026.4.20 source marker exists"],
|
||||
"cleanupGuarantees": ["disposable env cleanup removes 2026.4.20 fixture markers"],
|
||||
"setupEvidence": [
|
||||
"2026.4.20 source marker exists"
|
||||
],
|
||||
"cleanupGuarantees": [
|
||||
"disposable env cleanup removes 2026.4.20 fixture markers"
|
||||
],
|
||||
"source": {
|
||||
"kind": "real-runtime-downgrade",
|
||||
"release": "2026.4.20",
|
||||
|
||||
@ -2,26 +2,46 @@
|
||||
"id": "old-release-2026-4-24-user",
|
||||
"title": "OpenClaw 2026.4.24 Plugin Architecture User",
|
||||
"objective": "A cloned existing user env that is deliberately moved through OpenClaw 2026.4.24 before testing target upgrade behavior around plugin install indexes and bundled runtime deps.",
|
||||
"tags": ["existing-user", "upgrade", "old-release", "plugin-architecture", "2026-4-24"],
|
||||
"tags": [
|
||||
"existing-user",
|
||||
"upgrade",
|
||||
"old-release",
|
||||
"plugin-architecture",
|
||||
"2026-4-24"
|
||||
],
|
||||
"setup": [
|
||||
{
|
||||
"id": "write-2026-4-24-markers",
|
||||
"title": "Write 2026.4.24 Fixture Markers",
|
||||
"intent": "Persist non-invasive markers so reports can prove the disposable clone was shaped as a 2026.4.24 plugin-architecture upgrade source.",
|
||||
"afterPhases": ["source-runtime"],
|
||||
"afterPhases": [
|
||||
"source-runtime"
|
||||
],
|
||||
"commands": [
|
||||
"ocm env exec {env} -- node -e 'const fs=require(\"fs\"), path=require(\"path\"); const home=process.env.OPENCLAW_HOME; fs.mkdirSync(path.join(home,\"config\"),{recursive:true}); fs.mkdirSync(path.join(home,\"plugins\"),{recursive:true}); fs.writeFileSync(path.join(home,\"config\",\"kova-source-release.json\"),JSON.stringify({schemaVersion:\"kova.fixture.source-release.v1\",release:\"2026.4.24\",surface:\"upgrade-existing-user\",risk:\"plugin-runtime-deps\"},null,2)); if(!fs.existsSync(path.join(home,\"plugins\",\"installs.json\"))) fs.writeFileSync(path.join(home,\"plugins\",\"kova-missing-installs-index-marker.json\"),JSON.stringify({schemaVersion:\"kova.fixture.plugin-index-marker.v1\",release:\"2026.4.24\"},null,2));'"
|
||||
],
|
||||
"evidence": ["2026.4.24 source marker exists", "plugin install index marker exists when installs.json is missing"]
|
||||
"evidence": [
|
||||
"2026.4.24 source marker exists",
|
||||
"plugin install index marker exists when installs.json is missing"
|
||||
]
|
||||
}
|
||||
],
|
||||
"traits": ["existing-user", "old-release", "migration-state", "plugin-pressure", "runtime-deps"],
|
||||
"compatibleSurfaces": ["upgrade-existing-user"],
|
||||
"incompatibleSurfaces": [],
|
||||
"traits": [
|
||||
"existing-user",
|
||||
"old-release",
|
||||
"migration-state",
|
||||
"plugin-pressure",
|
||||
"runtime-deps"
|
||||
],
|
||||
"riskArea": "plugin-runtime-deps",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": ["2026.4.24 source marker exists", "plugin install index marker exists when installs.json is missing"],
|
||||
"cleanupGuarantees": ["disposable env cleanup removes 2026.4.24 fixture markers"],
|
||||
"setupEvidence": [
|
||||
"2026.4.24 source marker exists",
|
||||
"plugin install index marker exists when installs.json is missing"
|
||||
],
|
||||
"cleanupGuarantees": [
|
||||
"disposable env cleanup removes 2026.4.24 fixture markers"
|
||||
],
|
||||
"source": {
|
||||
"kind": "real-runtime-downgrade",
|
||||
"release": "2026.4.24",
|
||||
|
||||
@ -35,10 +35,6 @@
|
||||
"old-release",
|
||||
"migration-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "release-upgrade",
|
||||
"ownerArea": "upgrade",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -34,11 +34,6 @@
|
||||
"onboarded-user",
|
||||
"config-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install",
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "onboarding-migration",
|
||||
"ownerArea": "runtime-startup",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -33,12 +33,6 @@
|
||||
"plugin-pressure",
|
||||
"migration-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"fresh-install",
|
||||
"plugin-lifecycle",
|
||||
"upgrade-existing-user"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "plugin-index-migration",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -35,11 +35,6 @@
|
||||
"plugin-pressure",
|
||||
"performance-pressure"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"cross-platform-smoke",
|
||||
"gateway-performance"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "sync-filesystem-scan",
|
||||
"ownerArea": "gateway",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -2,26 +2,41 @@
|
||||
"id": "stable-channel-user",
|
||||
"title": "Stable Channel User",
|
||||
"objective": "A disposable OpenClaw env that has been started on the stable release channel before testing channel or local-build upgrade behavior.",
|
||||
"tags": ["upgrade", "channel", "stable", "existing-user"],
|
||||
"tags": [
|
||||
"upgrade",
|
||||
"channel",
|
||||
"stable",
|
||||
"existing-user"
|
||||
],
|
||||
"setup": [
|
||||
{
|
||||
"id": "write-stable-channel-marker",
|
||||
"title": "Write Stable Channel Marker",
|
||||
"intent": "Persist stable-channel metadata after the env is first started so reports can prove the upgrade source shape.",
|
||||
"afterPhases": ["start", "clone"],
|
||||
"afterPhases": [
|
||||
"start",
|
||||
"clone"
|
||||
],
|
||||
"commands": [
|
||||
"ocm env exec {env} -- node -e 'const fs=require(\"fs\"), path=require(\"path\"); const home=process.env.OPENCLAW_HOME; fs.mkdirSync(path.join(home,\"config\"),{recursive:true}); fs.writeFileSync(path.join(home,\"config\",\"channel.json\"),JSON.stringify({schemaVersion:\"kova.fixture.channel.v1\",channel:\"stable\"},null,2)); fs.writeFileSync(path.join(home,\".openclaw-channel\"),\"stable\\n\");'"
|
||||
],
|
||||
"evidence": ["stable channel marker exists"]
|
||||
"evidence": [
|
||||
"stable channel marker exists"
|
||||
]
|
||||
}
|
||||
],
|
||||
"traits": ["existing-user", "channel-state"],
|
||||
"compatibleSurfaces": ["upgrade-existing-user"],
|
||||
"incompatibleSurfaces": [],
|
||||
"traits": [
|
||||
"existing-user",
|
||||
"channel-state"
|
||||
],
|
||||
"riskArea": "release-channel-upgrade",
|
||||
"ownerArea": "upgrade",
|
||||
"setupEvidence": ["stable channel marker exists"],
|
||||
"cleanupGuarantees": ["disposable env cleanup removes stable channel fixture files"],
|
||||
"setupEvidence": [
|
||||
"stable channel marker exists"
|
||||
],
|
||||
"cleanupGuarantees": [
|
||||
"disposable env cleanup removes stable channel fixture files"
|
||||
],
|
||||
"source": {
|
||||
"kind": "generated",
|
||||
"note": "The scenario starts the env through the real stable channel before upgrading it."
|
||||
|
||||
@ -33,11 +33,6 @@
|
||||
"runtime-deps",
|
||||
"migration-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"bundled-runtime-deps",
|
||||
"plugin-lifecycle"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "runtime-deps-restage",
|
||||
"ownerArea": "plugins",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -33,10 +33,6 @@
|
||||
"service-state",
|
||||
"failure-state"
|
||||
],
|
||||
"compatibleSurfaces": [
|
||||
"gateway-performance"
|
||||
],
|
||||
"incompatibleSurfaces": [],
|
||||
"riskArea": "service-recovery",
|
||||
"ownerArea": "gateway",
|
||||
"setupEvidence": [
|
||||
|
||||
@ -3,31 +3,6 @@
|
||||
"title": "Agent CLI Local Turn",
|
||||
"ownerArea": "agent-runtime",
|
||||
"description": "Send cold, warm, repeated, and failure-mode messages through `openclaw agent --local`, then verify response latency, provider routing, gateway health, memory, and process containment.",
|
||||
"requiredStates": [
|
||||
"mock-openai-provider"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"agentTurnMs",
|
||||
"agentTurnP95Ms",
|
||||
"agentTurnMaxMs",
|
||||
"coldAgentTurnMs",
|
||||
"warmAgentTurnMs",
|
||||
"agentColdWarmDeltaMs",
|
||||
"coldPreProviderMs",
|
||||
"warmPreProviderMs",
|
||||
"agentPreProviderP95Ms",
|
||||
"agentCleanupMaxMs",
|
||||
"healthP95Ms",
|
||||
"peakRssMb",
|
||||
"providerTimeoutMentions",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,24 +3,6 @@
|
||||
"title": "Agent Gateway RPC Turn",
|
||||
"ownerArea": "gateway-agent-runtime",
|
||||
"description": "Send messages through `openclaw agent` without `--local`, forcing the CLI to cross the Gateway agent RPC boundary before the agent turn runs.",
|
||||
"requiredStates": [
|
||||
"mock-openai-provider"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"agentTurnMs",
|
||||
"agentTurnP95Ms",
|
||||
"agentTurnMaxMs",
|
||||
"coldPreProviderMs",
|
||||
"healthP95Ms",
|
||||
"peakRssMb",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,28 +3,6 @@
|
||||
"title": "Browser Automation",
|
||||
"ownerArea": "browser-runtime",
|
||||
"description": "Start OpenClaw's browser control surface, open a tab, inspect browser state, and shut the browser profile down cleanly.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"browserDoctorMs",
|
||||
"browserStartMs",
|
||||
"browserTabsMs",
|
||||
"browserOpenMs",
|
||||
"browserSnapshotMs",
|
||||
"browserStopMs",
|
||||
"browserTabCount",
|
||||
"browserProcessLeaks",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"gateway-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "Bundled Plugin Startup",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Start OpenClaw with bundled plugins and prove plugin loading does not degrade the gateway.",
|
||||
"requiredStates": [
|
||||
"fresh",
|
||||
"many-bundled-plugins"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"pluginLoadFailures",
|
||||
"missingDependencyErrors",
|
||||
"runtimeDepsStagingMs"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,26 +3,6 @@
|
||||
"title": "Bundled Runtime Dependencies",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Validate cold and warm bundled plugin runtime dependency staging behavior.",
|
||||
"requiredStates": [
|
||||
"missing-plugin-index",
|
||||
"stale-runtime-deps"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"coldReadyMs",
|
||||
"warmReadyMs",
|
||||
"runtimeDepsStagingMs",
|
||||
"coldRuntimeDepsStagingMs",
|
||||
"warmRuntimeDepsStagingMs",
|
||||
"warmRuntimeDepsRestageCount",
|
||||
"runtimeDepsWarmReuseOk",
|
||||
"missingDependencyErrors"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "Cross-Platform Smoke",
|
||||
"ownerArea": "platform-runtime",
|
||||
"description": "Run core OpenClaw runtime checks across supported operating systems and filesystem conditions.",
|
||||
"requiredStates": [
|
||||
"slow-filesystem",
|
||||
"channel-configured"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"healthMs",
|
||||
"syncFsStallDetected",
|
||||
"peakRssMb"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,23 +3,6 @@
|
||||
"title": "Dashboard Session Send Turn",
|
||||
"ownerArea": "gateway-chat-session-runtime",
|
||||
"description": "Create a dashboard session, call Gateway `sessions.send`, and wait for the assistant response in chat history to validate the same path dashboard users exercise.",
|
||||
"requiredStates": [
|
||||
"mock-openai-provider"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"agentTurnMs",
|
||||
"agentTurnP95Ms",
|
||||
"coldPreProviderMs",
|
||||
"healthP95Ms",
|
||||
"peakRssMb",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,21 +3,6 @@
|
||||
"title": "Dashboard",
|
||||
"ownerArea": "control-ui",
|
||||
"description": "Verify dashboard command output, websocket entry, and post-dashboard gateway health.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"dashboardConnectMs",
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"websocketDisconnects"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "Failure Containment",
|
||||
"ownerArea": "diagnostics",
|
||||
"description": "Inject degraded OpenClaw state and verify diagnostics are useful without taking down the gateway.",
|
||||
"requiredStates": [
|
||||
"broken-plugin-deps",
|
||||
"corrupted-config"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewaySurvives",
|
||||
"diagnosticPresent",
|
||||
"statusAfterFailureMs",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,28 +3,6 @@
|
||||
"title": "Fresh Install",
|
||||
"ownerArea": "runtime-startup",
|
||||
"description": "Create a disposable fresh OpenClaw home and verify gateway, plugins, models, logs, and baseline resource use.",
|
||||
"requiredStates": [
|
||||
"fresh",
|
||||
"onboarded-user",
|
||||
"plugin-index",
|
||||
"old-config-keys",
|
||||
"large-memory-session",
|
||||
"corrupted-config",
|
||||
"missing-plugin-index"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"pluginsListMs",
|
||||
"modelsListMs",
|
||||
"peakRssMb"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,25 +3,6 @@
|
||||
"title": "Gateway Performance",
|
||||
"ownerArea": "gateway",
|
||||
"description": "Measure cold start, warm start, health latency, memory, CPU, and event-loop behavior for OpenClaw gateway usage.",
|
||||
"requiredStates": [
|
||||
"many-bundled-plugins",
|
||||
"gateway-already-running",
|
||||
"stale-service-state",
|
||||
"slow-filesystem"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"coldReadyMs",
|
||||
"warmReadyMs",
|
||||
"healthP95Ms",
|
||||
"peakRssMb",
|
||||
"eventLoopMaxMs"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"gateway-tree",
|
||||
|
||||
@ -3,26 +3,6 @@
|
||||
"title": "MCP Runtime",
|
||||
"ownerArea": "mcp-runtime",
|
||||
"description": "Start OpenClaw's MCP stdio bridge against a running gateway, list exposed tools, and prove the bridge process stops cleanly.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"mcpInitializeMs",
|
||||
"mcpToolsListMs",
|
||||
"mcpShutdownMs",
|
||||
"mcpToolCountMin",
|
||||
"mcpProcessLeaks",
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"pluginLoadFailures",
|
||||
"peakRssMb"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,23 +3,6 @@
|
||||
"title": "Media Understanding",
|
||||
"ownerArea": "media-understanding",
|
||||
"description": "Exercise OpenClaw media understanding through the packaged capability CLI and verify provider timeouts do not stall the gateway or command path.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"mediaDescribeMs",
|
||||
"mediaTimeoutObserved",
|
||||
"mediaStatusAfterTimeoutMs",
|
||||
"mediaGatewayStatusWorks",
|
||||
"providerRequestCountMin",
|
||||
"peakRssMb"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "Network Offline",
|
||||
"ownerArea": "provider-network",
|
||||
"description": "Exercise an OpenClaw agent turn when the configured provider endpoint is unreachable and verify the failure is bounded and contained.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"networkFailureObserved",
|
||||
"networkCommandTimedOut",
|
||||
"networkStatusAfterFailureMs",
|
||||
"networkGatewayStatusWorks",
|
||||
"peakRssMb"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,23 +3,6 @@
|
||||
"title": "Official Plugin Install",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Install a published official OpenClaw plugin through the exact user command path, then verify persisted registry/list state and gateway health.",
|
||||
"requiredStates": [
|
||||
"official-plugins"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"pluginInstallMs",
|
||||
"officialPluginInstallOk",
|
||||
"officialPluginSecurityBlocks",
|
||||
"pluginsListMs",
|
||||
"missingDependencyErrors",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,23 +3,6 @@
|
||||
"title": "OpenAI-Compatible Turn",
|
||||
"ownerArea": "gateway-openai-compatible-runtime",
|
||||
"description": "POST a user message to the OpenAI-compatible chat completions endpoint and verify the final response, provider timing, auth behavior, and gateway health.",
|
||||
"requiredStates": [
|
||||
"mock-openai-provider"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"agentTurnMs",
|
||||
"agentTurnP95Ms",
|
||||
"coldPreProviderMs",
|
||||
"healthP95Ms",
|
||||
"peakRssMb",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,21 +3,6 @@
|
||||
"title": "Plugin Bad Manifest",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Install an invalid plugin fixture and verify OpenClaw rejects it cleanly without corrupting gateway state.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"pluginLoadFailures",
|
||||
"diagnosticPresent"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "External Plugin Install",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Install a real local external plugin and verify registry state, restart health, and dependency behavior.",
|
||||
"requiredStates": [
|
||||
"fresh",
|
||||
"external-plugin"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"pluginsListMs",
|
||||
"missingDependencyErrors",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,24 +3,6 @@
|
||||
"title": "Plugin Lifecycle",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Exercise plugin list, update planning, restart, runtime dependency diagnostics, and registry consistency.",
|
||||
"requiredStates": [
|
||||
"plugin-index",
|
||||
"external-plugin",
|
||||
"broken-plugin-deps",
|
||||
"stale-runtime-deps"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"pluginsListMs",
|
||||
"pluginUpdateDryRunMs",
|
||||
"restartReadyMs",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,21 +3,6 @@
|
||||
"title": "Plugin Missing Runtime Dependencies",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Install a plugin with undeclared runtime imports and verify OpenClaw reports the failure while keeping the gateway usable.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"missingDependencyErrors",
|
||||
"pluginLoadFailures",
|
||||
"statusMs",
|
||||
"gatewayReadyMs"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,21 +3,6 @@
|
||||
"title": "Plugin Remove",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Install and remove an external plugin, then verify install index cleanup and restart health.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"pluginsListMs",
|
||||
"missingDependencyErrors",
|
||||
"pluginLoadFailures",
|
||||
"restartReadyMs"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "Plugin Update",
|
||||
"ownerArea": "plugins",
|
||||
"description": "Run external plugin update planning and registry refresh paths without mutating durable state.",
|
||||
"requiredStates": [
|
||||
"fresh",
|
||||
"external-plugin"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"pluginUpdateDryRunMs",
|
||||
"pluginsListMs",
|
||||
"missingDependencyErrors",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "Provider And Models",
|
||||
"ownerArea": "providers",
|
||||
"description": "Exercise model/provider discovery and verify slow or unauthenticated providers do not stall the gateway.",
|
||||
"requiredStates": [
|
||||
"model-auth-configured",
|
||||
"model-auth-missing"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"modelsListMs",
|
||||
"statusAfterModelsMs",
|
||||
"gatewayResponsive",
|
||||
"providerTimeoutMentions"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,24 +3,6 @@
|
||||
"title": "Release Runtime Startup",
|
||||
"ownerArea": "release-runtime",
|
||||
"description": "Start a release-shaped OpenClaw runtime and measure cold readiness, runtime dependency staging, plugin load health, and resource use.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"local-build",
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"gatewayReadyMs",
|
||||
"gatewayReadyHardTimeoutMs",
|
||||
"statusMs",
|
||||
"pluginsListMs",
|
||||
"peakRssMb",
|
||||
"runtimeDepsStagingMs",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"gateway-tree",
|
||||
|
||||
@ -3,28 +3,6 @@
|
||||
"title": "Gateway Soak",
|
||||
"ownerArea": "gateway",
|
||||
"description": "Run longer OpenClaw gateway usage loops to detect memory growth, CPU spikes, event-loop stalls, and degraded command latency.",
|
||||
"requiredStates": [
|
||||
"large-workspace",
|
||||
"large-memory-session"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"soakDurationMs",
|
||||
"soakIterations",
|
||||
"soakCommandP95Ms",
|
||||
"soakCommandFailures",
|
||||
"soakHealthP95Ms",
|
||||
"soakHealthFailures",
|
||||
"rssGrowthMb",
|
||||
"gatewayRssGrowthMb",
|
||||
"peakRssMb",
|
||||
"restartCount"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"gateway-tree",
|
||||
|
||||
@ -3,23 +3,6 @@
|
||||
"title": "TUI Message Turn",
|
||||
"ownerArea": "tui-agent-runtime",
|
||||
"description": "Launch the OpenClaw TUI, send a real stdin message, and require visible assistant output so Kova can catch TUI input freezes and delayed user-visible responses.",
|
||||
"requiredStates": [
|
||||
"mock-openai-provider"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"agentTurnMs",
|
||||
"agentTurnP95Ms",
|
||||
"coldPreProviderMs",
|
||||
"healthP95Ms",
|
||||
"peakRssMb",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,22 +3,6 @@
|
||||
"title": "TUI",
|
||||
"ownerArea": "terminal-ui",
|
||||
"description": "Attach the terminal UI to a running OpenClaw gateway and verify render, input responsiveness, and clean shutdown.",
|
||||
"requiredStates": [
|
||||
"fresh"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"tuiSmokeMs",
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"inputLagMs",
|
||||
"pluginLoadFailures"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,25 +3,6 @@
|
||||
"title": "Existing User Upgrade",
|
||||
"ownerArea": "upgrade",
|
||||
"description": "Clone existing OpenClaw state, run the real upgrade path, and verify migrations, plugin indexes, runtime deps, gateway readiness, and diagnostics.",
|
||||
"requiredStates": [
|
||||
"old-release-user",
|
||||
"old-release-2026-4-20-user",
|
||||
"old-release-2026-4-24-user",
|
||||
"failed-upgrade"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"upgradeMs",
|
||||
"gatewayReadyMs",
|
||||
"statusMs",
|
||||
"pluginIndexPresent",
|
||||
"doctorFixMs"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -3,26 +3,6 @@
|
||||
"title": "Workspace Scan Pressure",
|
||||
"ownerArea": "workspace-runtime",
|
||||
"description": "Exercise OpenClaw with a large workspace tree to catch synchronous filesystem scans, slow command paths, memory growth, and event-loop stalls.",
|
||||
"requiredStates": [
|
||||
"large-workspace"
|
||||
],
|
||||
"targetKinds": [
|
||||
"npm",
|
||||
"channel",
|
||||
"runtime",
|
||||
"local-build"
|
||||
],
|
||||
"requiredMetrics": [
|
||||
"warmReadyMs",
|
||||
"statusMs",
|
||||
"pluginsListMs",
|
||||
"modelsListMs",
|
||||
"soakCommandP95Ms",
|
||||
"soakHealthP95Ms",
|
||||
"peakRssMb",
|
||||
"healthP95Ms",
|
||||
"eventLoopMaxMs"
|
||||
],
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"gateway-tree",
|
||||
|
||||
Loading…
Reference in New Issue
Block a user