From d950420642e3440ff94743309b55ae2ccb240efe Mon Sep 17 00:00:00 2001 From: Shakker Date: Mon, 4 May 2026 22:57:26 +0100 Subject: [PATCH] feat: gate on resolved requirement coverage --- src/main.mjs | 9 +++++- src/matrix/coverage.mjs | 8 +++++ src/matrix/gate.mjs | 22 ++++++++++--- src/selfcheck.mjs | 68 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+), 5 deletions(-) diff --git a/src/main.mjs b/src/main.mjs index a461b4f..a93f9b2 100644 --- a/src/main.mjs +++ b/src/main.mjs @@ -310,6 +310,13 @@ async function matrixRun(flags) { validateProfileTarget(profile, targetPlan); const fromPlan = flags.from ? resolveTarget(flags.from, "from") : null; const entries = applyMatrixControls(await expandProfile(profile), flags, platformInfo()); + const resolvedCoverage = resolveCoverageObligations({ + profile, + entries, + surfaces: registry.surfaces, + targetPlan + }); + assertResolvedCoverageIsRunnable(resolvedCoverage); const controls = matrixControlSummary(flags, targetPlan); const auth = await resolveRunAuthContext(flags); const regressionThresholds = await loadRegressionThresholds(flags); @@ -416,7 +423,7 @@ async function matrixRun(flags) { baseline: reportBase.baseline, platform: reportBase.platform, records - }, profile) + }, profile, { resolvedCoverage }) : null; await mkdir(reportRoot, { recursive: true }); diff --git a/src/matrix/coverage.mjs b/src/matrix/coverage.mjs index a9796ae..24fe229 100644 --- a/src/matrix/coverage.mjs +++ b/src/matrix/coverage.mjs @@ -36,6 +36,7 @@ function profileCoverage(profile, { scenarios, states, platform }) { const entryScenarios = new Set(); const entryStates = new Set(); const entrySurfaces = new Set(); + const entryRequirements = new Set(); const entryStateSurfaces = new Set(); for (const entry of profile.entries ?? []) { @@ -45,6 +46,9 @@ function profileCoverage(profile, { scenarios, states, platform }) { if (scenario?.surface) { entrySurfaces.add(scenario.surface); entryStateSurfaces.add(`${scenario.surface}:${entry.state}`); + for (const requirement of scenario.proves ?? []) { + entryRequirements.add(`${scenario.surface}:${requirement}`); + } } } @@ -53,6 +57,7 @@ function profileCoverage(profile, { scenarios, states, platform }) { const requiredStates = coverageIds(profile, "states"); const requiredTraits = coverageIds(profile, "traits"); const requiredStateSurfaces = coverageIds(profile, "stateSurfaces"); + const requiredRequirements = coverageIds(profile, "requirements"); const requiredPlatforms = coverageIds(profile, "platforms"); const coveredTraits = coveredStateTraits(profile, states); const currentPlatformKeys = platformCoverageKeys(platform); @@ -63,6 +68,7 @@ function profileCoverage(profile, { scenarios, states, platform }) { surfaces: [...entrySurfaces].sort(), states: [...entryStates].sort(), scenarios: [...entryScenarios].sort(), + requirements: [...entryRequirements].sort(), stateSurfaces: [...entryStateSurfaces].sort(), required: { surfaces: requiredSurfaces, @@ -70,6 +76,7 @@ function profileCoverage(profile, { scenarios, states, platform }) { states: requiredStates, traits: requiredTraits, platforms: requiredPlatforms, + requirements: requiredRequirements, stateSurfaces: requiredStateSurfaces }, gaps: { @@ -78,6 +85,7 @@ function profileCoverage(profile, { scenarios, states, platform }) { states: requiredStates.filter((id) => !entryStates.has(id)), traits: requiredTraits.filter((id) => !coveredTraits.has(id)), platforms: requiredPlatforms.filter((id) => !currentPlatformKeys.has(id)), + requirements: requiredRequirements.filter((id) => !entryRequirements.has(id)), stateSurfaces: requiredStateSurfaces.filter((id) => !entryStateSurfaces.has(id)) }, currentPlatformKeys: [...currentPlatformKeys].sort(), diff --git a/src/matrix/gate.mjs b/src/matrix/gate.mjs index fc443dc..ab3662f 100644 --- a/src/matrix/gate.mjs +++ b/src/matrix/gate.mjs @@ -21,7 +21,7 @@ function scenarioUsesSourceEnv(scenario) { ); } -export function evaluateGate(report, profile) { +export function evaluateGate(report, profile, options = {}) { const policy = normalizeGatePolicy(profile); const records = report.records ?? []; const cards = []; @@ -68,7 +68,7 @@ export function evaluateGate(report, profile) { } } - for (const card of buildCoverageCards(report, policy, partial)) { + for (const card of buildCoverageCards(report, policy, partial, options)) { cards.push(card); if (card.severity === "blocking" || card.severity === "info") { missingRequired.push({ @@ -162,7 +162,8 @@ function normalizeCoveragePolicy(coverage) { states: normalizeCoverageSet(input.states), traits: normalizeCoverageSet(input.traits), scenarios: normalizeCoverageSet(input.scenarios), - stateSurfaces: normalizeCoverageSet(input.stateSurfaces) + stateSurfaces: normalizeCoverageSet(input.stateSurfaces), + requirements: normalizeCoverageSet(input.requirements) }; } @@ -178,9 +179,10 @@ function normalizeStringList(value) { return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.length > 0) : []; } -function buildCoverageCards(report, policy, partial) { +function buildCoverageCards(report, policy, partial, options = {}) { const cards = []; const records = report.records ?? []; + const resolvedCoverage = options.resolvedCoverage ?? null; const platformKeys = platformCoverageKeys(report.platform); const scenarioKeys = new Set(records.map((record) => record.scenario).filter(Boolean)); const stateKeys = new Set(records.map((record) => record.state?.id).filter(Boolean)); @@ -193,6 +195,10 @@ function buildCoverageCards(report, policy, partial) { return surface && state ? `${surface}:${state}` : null; }) .filter(Boolean)); + const requirementKeys = new Set((resolvedCoverage?.obligations ?? []) + .filter((obligation) => obligation.status === "planned") + .map((obligation) => `${obligation.surface}:${obligation.requirement}`) + .filter((value) => !value.endsWith(":null"))); addCoverageCards(cards, { kind: "surface", @@ -236,6 +242,13 @@ function buildCoverageCards(report, policy, partial) { partial, statusText: `${stateSurfaceKeys.size} state/surface pair(s) present` }); + addCoverageCards(cards, { + kind: "requirement", + expected: policy.coverage.requirements, + observed: requirementKeys, + partial, + statusText: `${requirementKeys.size} requirement obligation(s) present` + }); return cards; } @@ -263,6 +276,7 @@ function coverageCard({ severity, kind, value, partial, statusText }) { coverage: kind, scenario: kind === "scenario" ? value : null, state: kind === "state" ? value : stateFromCoverage(kind, value), + requirement: kind === "requirement" ? value : null, status: "MISSING", title: `Required ${kind} Coverage Missing`, summary: filtered diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index 6d4f5d4..eed82ba 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -449,6 +449,7 @@ export async function runSelfCheck(flags = {}) { checks.push(gatePartialFailureCheck()); checks.push(gatePartialPassCheck()); checks.push(gatePlatformCoverageCheck()); + checks.push(gateRequirementCoverageCheck()); checks.push(gateSubsystemSummaryCheck()); checks.push(safetyGuardCheck()); checks.push(await failingCommandCheck( @@ -831,6 +832,73 @@ function gatePlatformCoverageCheck() { } } +function gateRequirementCoverageCheck() { + try { + const profile = { + id: "release", + gate: { + id: "test-release-gate", + coverage: { + requirements: { + blocking: ["release-runtime-startup:baseline"], + warning: ["fresh-install:baseline"] + } + }, + blocking: [ + { scenario: "release-runtime-startup", state: "fresh" } + ] + } + }; + const report = { + mode: "execution", + controls: { + include: [], + exclude: [] + }, + records: [ + { + scenario: "release-runtime-startup", + surface: "release-runtime-startup", + state: { id: "fresh" }, + status: "PASS", + title: "Release Runtime Startup", + likelyOwner: "OpenClaw", + phases: [] + } + ] + }; + const gate = evaluateGate(report, profile, { + resolvedCoverage: { + obligations: [{ + surface: "release-runtime-startup", + requirement: "baseline", + scenario: "release-runtime-startup", + state: "fresh", + status: "planned" + }] + } + }); + + assertEqual(gate.verdict, "SHIP", "required requirement coverage should pass"); + assertEqual(gate.cards.some((card) => card.coverage === "requirement" && card.expected === "requirement coverage release-runtime-startup:baseline"), false, "covered requirement should not be missing"); + assertEqual(gate.cards.some((card) => card.coverage === "requirement" && card.expected === "requirement coverage fresh-install:baseline" && card.severity === "warning"), true, "missing warning requirement should remain visible"); + return { + id: "gate-requirement-coverage", + status: "PASS", + command: "evaluate synthetic release gate requirement coverage", + durationMs: 0 + }; + } catch (error) { + return { + id: "gate-requirement-coverage", + status: "FAIL", + command: "evaluate synthetic release gate requirement coverage", + durationMs: 0, + message: error.message + }; + } +} + function gateSubsystemSummaryCheck() { try { const gate = evaluateGate({