feat: gate on resolved requirement coverage

This commit is contained in:
Shakker 2026-05-04 22:57:26 +01:00
parent 2aa9a75a89
commit d950420642
No known key found for this signature in database
4 changed files with 102 additions and 5 deletions

View File

@ -310,6 +310,13 @@ async function matrixRun(flags) {
validateProfileTarget(profile, targetPlan);
const fromPlan = flags.from ? resolveTarget(flags.from, "from") : null;
const entries = applyMatrixControls(await expandProfile(profile), flags, platformInfo());
const resolvedCoverage = resolveCoverageObligations({
profile,
entries,
surfaces: registry.surfaces,
targetPlan
});
assertResolvedCoverageIsRunnable(resolvedCoverage);
const controls = matrixControlSummary(flags, targetPlan);
const auth = await resolveRunAuthContext(flags);
const regressionThresholds = await loadRegressionThresholds(flags);
@ -416,7 +423,7 @@ async function matrixRun(flags) {
baseline: reportBase.baseline,
platform: reportBase.platform,
records
}, profile)
}, profile, { resolvedCoverage })
: null;
await mkdir(reportRoot, { recursive: true });

View File

@ -36,6 +36,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
const entryScenarios = new Set();
const entryStates = new Set();
const entrySurfaces = new Set();
const entryRequirements = new Set();
const entryStateSurfaces = new Set();
for (const entry of profile.entries ?? []) {
@ -45,6 +46,9 @@ function profileCoverage(profile, { scenarios, states, platform }) {
if (scenario?.surface) {
entrySurfaces.add(scenario.surface);
entryStateSurfaces.add(`${scenario.surface}:${entry.state}`);
for (const requirement of scenario.proves ?? []) {
entryRequirements.add(`${scenario.surface}:${requirement}`);
}
}
}
@ -53,6 +57,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
const requiredStates = coverageIds(profile, "states");
const requiredTraits = coverageIds(profile, "traits");
const requiredStateSurfaces = coverageIds(profile, "stateSurfaces");
const requiredRequirements = coverageIds(profile, "requirements");
const requiredPlatforms = coverageIds(profile, "platforms");
const coveredTraits = coveredStateTraits(profile, states);
const currentPlatformKeys = platformCoverageKeys(platform);
@ -63,6 +68,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
surfaces: [...entrySurfaces].sort(),
states: [...entryStates].sort(),
scenarios: [...entryScenarios].sort(),
requirements: [...entryRequirements].sort(),
stateSurfaces: [...entryStateSurfaces].sort(),
required: {
surfaces: requiredSurfaces,
@ -70,6 +76,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
states: requiredStates,
traits: requiredTraits,
platforms: requiredPlatforms,
requirements: requiredRequirements,
stateSurfaces: requiredStateSurfaces
},
gaps: {
@ -78,6 +85,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
states: requiredStates.filter((id) => !entryStates.has(id)),
traits: requiredTraits.filter((id) => !coveredTraits.has(id)),
platforms: requiredPlatforms.filter((id) => !currentPlatformKeys.has(id)),
requirements: requiredRequirements.filter((id) => !entryRequirements.has(id)),
stateSurfaces: requiredStateSurfaces.filter((id) => !entryStateSurfaces.has(id))
},
currentPlatformKeys: [...currentPlatformKeys].sort(),

View File

@ -21,7 +21,7 @@ function scenarioUsesSourceEnv(scenario) {
);
}
export function evaluateGate(report, profile) {
export function evaluateGate(report, profile, options = {}) {
const policy = normalizeGatePolicy(profile);
const records = report.records ?? [];
const cards = [];
@ -68,7 +68,7 @@ export function evaluateGate(report, profile) {
}
}
for (const card of buildCoverageCards(report, policy, partial)) {
for (const card of buildCoverageCards(report, policy, partial, options)) {
cards.push(card);
if (card.severity === "blocking" || card.severity === "info") {
missingRequired.push({
@ -162,7 +162,8 @@ function normalizeCoveragePolicy(coverage) {
states: normalizeCoverageSet(input.states),
traits: normalizeCoverageSet(input.traits),
scenarios: normalizeCoverageSet(input.scenarios),
stateSurfaces: normalizeCoverageSet(input.stateSurfaces)
stateSurfaces: normalizeCoverageSet(input.stateSurfaces),
requirements: normalizeCoverageSet(input.requirements)
};
}
@ -178,9 +179,10 @@ function normalizeStringList(value) {
return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.length > 0) : [];
}
function buildCoverageCards(report, policy, partial) {
function buildCoverageCards(report, policy, partial, options = {}) {
const cards = [];
const records = report.records ?? [];
const resolvedCoverage = options.resolvedCoverage ?? null;
const platformKeys = platformCoverageKeys(report.platform);
const scenarioKeys = new Set(records.map((record) => record.scenario).filter(Boolean));
const stateKeys = new Set(records.map((record) => record.state?.id).filter(Boolean));
@ -193,6 +195,10 @@ function buildCoverageCards(report, policy, partial) {
return surface && state ? `${surface}:${state}` : null;
})
.filter(Boolean));
const requirementKeys = new Set((resolvedCoverage?.obligations ?? [])
.filter((obligation) => obligation.status === "planned")
.map((obligation) => `${obligation.surface}:${obligation.requirement}`)
.filter((value) => !value.endsWith(":null")));
addCoverageCards(cards, {
kind: "surface",
@ -236,6 +242,13 @@ function buildCoverageCards(report, policy, partial) {
partial,
statusText: `${stateSurfaceKeys.size} state/surface pair(s) present`
});
addCoverageCards(cards, {
kind: "requirement",
expected: policy.coverage.requirements,
observed: requirementKeys,
partial,
statusText: `${requirementKeys.size} requirement obligation(s) present`
});
return cards;
}
@ -263,6 +276,7 @@ function coverageCard({ severity, kind, value, partial, statusText }) {
coverage: kind,
scenario: kind === "scenario" ? value : null,
state: kind === "state" ? value : stateFromCoverage(kind, value),
requirement: kind === "requirement" ? value : null,
status: "MISSING",
title: `Required ${kind} Coverage Missing`,
summary: filtered

View File

@ -449,6 +449,7 @@ export async function runSelfCheck(flags = {}) {
checks.push(gatePartialFailureCheck());
checks.push(gatePartialPassCheck());
checks.push(gatePlatformCoverageCheck());
checks.push(gateRequirementCoverageCheck());
checks.push(gateSubsystemSummaryCheck());
checks.push(safetyGuardCheck());
checks.push(await failingCommandCheck(
@ -831,6 +832,73 @@ function gatePlatformCoverageCheck() {
}
}
function gateRequirementCoverageCheck() {
try {
const profile = {
id: "release",
gate: {
id: "test-release-gate",
coverage: {
requirements: {
blocking: ["release-runtime-startup:baseline"],
warning: ["fresh-install:baseline"]
}
},
blocking: [
{ scenario: "release-runtime-startup", state: "fresh" }
]
}
};
const report = {
mode: "execution",
controls: {
include: [],
exclude: []
},
records: [
{
scenario: "release-runtime-startup",
surface: "release-runtime-startup",
state: { id: "fresh" },
status: "PASS",
title: "Release Runtime Startup",
likelyOwner: "OpenClaw",
phases: []
}
]
};
const gate = evaluateGate(report, profile, {
resolvedCoverage: {
obligations: [{
surface: "release-runtime-startup",
requirement: "baseline",
scenario: "release-runtime-startup",
state: "fresh",
status: "planned"
}]
}
});
assertEqual(gate.verdict, "SHIP", "required requirement coverage should pass");
assertEqual(gate.cards.some((card) => card.coverage === "requirement" && card.expected === "requirement coverage release-runtime-startup:baseline"), false, "covered requirement should not be missing");
assertEqual(gate.cards.some((card) => card.coverage === "requirement" && card.expected === "requirement coverage fresh-install:baseline" && card.severity === "warning"), true, "missing warning requirement should remain visible");
return {
id: "gate-requirement-coverage",
status: "PASS",
command: "evaluate synthetic release gate requirement coverage",
durationMs: 0
};
} catch (error) {
return {
id: "gate-requirement-coverage",
status: "FAIL",
command: "evaluate synthetic release gate requirement coverage",
durationMs: 0,
message: error.message
};
}
}
function gateSubsystemSummaryCheck() {
try {
const gate = evaluateGate({