feat: gate on resolved requirement coverage
This commit is contained in:
parent
2aa9a75a89
commit
d950420642
@ -310,6 +310,13 @@ async function matrixRun(flags) {
|
||||
validateProfileTarget(profile, targetPlan);
|
||||
const fromPlan = flags.from ? resolveTarget(flags.from, "from") : null;
|
||||
const entries = applyMatrixControls(await expandProfile(profile), flags, platformInfo());
|
||||
const resolvedCoverage = resolveCoverageObligations({
|
||||
profile,
|
||||
entries,
|
||||
surfaces: registry.surfaces,
|
||||
targetPlan
|
||||
});
|
||||
assertResolvedCoverageIsRunnable(resolvedCoverage);
|
||||
const controls = matrixControlSummary(flags, targetPlan);
|
||||
const auth = await resolveRunAuthContext(flags);
|
||||
const regressionThresholds = await loadRegressionThresholds(flags);
|
||||
@ -416,7 +423,7 @@ async function matrixRun(flags) {
|
||||
baseline: reportBase.baseline,
|
||||
platform: reportBase.platform,
|
||||
records
|
||||
}, profile)
|
||||
}, profile, { resolvedCoverage })
|
||||
: null;
|
||||
|
||||
await mkdir(reportRoot, { recursive: true });
|
||||
|
||||
@ -36,6 +36,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
|
||||
const entryScenarios = new Set();
|
||||
const entryStates = new Set();
|
||||
const entrySurfaces = new Set();
|
||||
const entryRequirements = new Set();
|
||||
const entryStateSurfaces = new Set();
|
||||
|
||||
for (const entry of profile.entries ?? []) {
|
||||
@ -45,6 +46,9 @@ function profileCoverage(profile, { scenarios, states, platform }) {
|
||||
if (scenario?.surface) {
|
||||
entrySurfaces.add(scenario.surface);
|
||||
entryStateSurfaces.add(`${scenario.surface}:${entry.state}`);
|
||||
for (const requirement of scenario.proves ?? []) {
|
||||
entryRequirements.add(`${scenario.surface}:${requirement}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -53,6 +57,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
|
||||
const requiredStates = coverageIds(profile, "states");
|
||||
const requiredTraits = coverageIds(profile, "traits");
|
||||
const requiredStateSurfaces = coverageIds(profile, "stateSurfaces");
|
||||
const requiredRequirements = coverageIds(profile, "requirements");
|
||||
const requiredPlatforms = coverageIds(profile, "platforms");
|
||||
const coveredTraits = coveredStateTraits(profile, states);
|
||||
const currentPlatformKeys = platformCoverageKeys(platform);
|
||||
@ -63,6 +68,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
|
||||
surfaces: [...entrySurfaces].sort(),
|
||||
states: [...entryStates].sort(),
|
||||
scenarios: [...entryScenarios].sort(),
|
||||
requirements: [...entryRequirements].sort(),
|
||||
stateSurfaces: [...entryStateSurfaces].sort(),
|
||||
required: {
|
||||
surfaces: requiredSurfaces,
|
||||
@ -70,6 +76,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
|
||||
states: requiredStates,
|
||||
traits: requiredTraits,
|
||||
platforms: requiredPlatforms,
|
||||
requirements: requiredRequirements,
|
||||
stateSurfaces: requiredStateSurfaces
|
||||
},
|
||||
gaps: {
|
||||
@ -78,6 +85,7 @@ function profileCoverage(profile, { scenarios, states, platform }) {
|
||||
states: requiredStates.filter((id) => !entryStates.has(id)),
|
||||
traits: requiredTraits.filter((id) => !coveredTraits.has(id)),
|
||||
platforms: requiredPlatforms.filter((id) => !currentPlatformKeys.has(id)),
|
||||
requirements: requiredRequirements.filter((id) => !entryRequirements.has(id)),
|
||||
stateSurfaces: requiredStateSurfaces.filter((id) => !entryStateSurfaces.has(id))
|
||||
},
|
||||
currentPlatformKeys: [...currentPlatformKeys].sort(),
|
||||
|
||||
@ -21,7 +21,7 @@ function scenarioUsesSourceEnv(scenario) {
|
||||
);
|
||||
}
|
||||
|
||||
export function evaluateGate(report, profile) {
|
||||
export function evaluateGate(report, profile, options = {}) {
|
||||
const policy = normalizeGatePolicy(profile);
|
||||
const records = report.records ?? [];
|
||||
const cards = [];
|
||||
@ -68,7 +68,7 @@ export function evaluateGate(report, profile) {
|
||||
}
|
||||
}
|
||||
|
||||
for (const card of buildCoverageCards(report, policy, partial)) {
|
||||
for (const card of buildCoverageCards(report, policy, partial, options)) {
|
||||
cards.push(card);
|
||||
if (card.severity === "blocking" || card.severity === "info") {
|
||||
missingRequired.push({
|
||||
@ -162,7 +162,8 @@ function normalizeCoveragePolicy(coverage) {
|
||||
states: normalizeCoverageSet(input.states),
|
||||
traits: normalizeCoverageSet(input.traits),
|
||||
scenarios: normalizeCoverageSet(input.scenarios),
|
||||
stateSurfaces: normalizeCoverageSet(input.stateSurfaces)
|
||||
stateSurfaces: normalizeCoverageSet(input.stateSurfaces),
|
||||
requirements: normalizeCoverageSet(input.requirements)
|
||||
};
|
||||
}
|
||||
|
||||
@ -178,9 +179,10 @@ function normalizeStringList(value) {
|
||||
return Array.isArray(value) ? value.filter((item) => typeof item === "string" && item.length > 0) : [];
|
||||
}
|
||||
|
||||
function buildCoverageCards(report, policy, partial) {
|
||||
function buildCoverageCards(report, policy, partial, options = {}) {
|
||||
const cards = [];
|
||||
const records = report.records ?? [];
|
||||
const resolvedCoverage = options.resolvedCoverage ?? null;
|
||||
const platformKeys = platformCoverageKeys(report.platform);
|
||||
const scenarioKeys = new Set(records.map((record) => record.scenario).filter(Boolean));
|
||||
const stateKeys = new Set(records.map((record) => record.state?.id).filter(Boolean));
|
||||
@ -193,6 +195,10 @@ function buildCoverageCards(report, policy, partial) {
|
||||
return surface && state ? `${surface}:${state}` : null;
|
||||
})
|
||||
.filter(Boolean));
|
||||
const requirementKeys = new Set((resolvedCoverage?.obligations ?? [])
|
||||
.filter((obligation) => obligation.status === "planned")
|
||||
.map((obligation) => `${obligation.surface}:${obligation.requirement}`)
|
||||
.filter((value) => !value.endsWith(":null")));
|
||||
|
||||
addCoverageCards(cards, {
|
||||
kind: "surface",
|
||||
@ -236,6 +242,13 @@ function buildCoverageCards(report, policy, partial) {
|
||||
partial,
|
||||
statusText: `${stateSurfaceKeys.size} state/surface pair(s) present`
|
||||
});
|
||||
addCoverageCards(cards, {
|
||||
kind: "requirement",
|
||||
expected: policy.coverage.requirements,
|
||||
observed: requirementKeys,
|
||||
partial,
|
||||
statusText: `${requirementKeys.size} requirement obligation(s) present`
|
||||
});
|
||||
|
||||
return cards;
|
||||
}
|
||||
@ -263,6 +276,7 @@ function coverageCard({ severity, kind, value, partial, statusText }) {
|
||||
coverage: kind,
|
||||
scenario: kind === "scenario" ? value : null,
|
||||
state: kind === "state" ? value : stateFromCoverage(kind, value),
|
||||
requirement: kind === "requirement" ? value : null,
|
||||
status: "MISSING",
|
||||
title: `Required ${kind} Coverage Missing`,
|
||||
summary: filtered
|
||||
|
||||
@ -449,6 +449,7 @@ export async function runSelfCheck(flags = {}) {
|
||||
checks.push(gatePartialFailureCheck());
|
||||
checks.push(gatePartialPassCheck());
|
||||
checks.push(gatePlatformCoverageCheck());
|
||||
checks.push(gateRequirementCoverageCheck());
|
||||
checks.push(gateSubsystemSummaryCheck());
|
||||
checks.push(safetyGuardCheck());
|
||||
checks.push(await failingCommandCheck(
|
||||
@ -831,6 +832,73 @@ function gatePlatformCoverageCheck() {
|
||||
}
|
||||
}
|
||||
|
||||
function gateRequirementCoverageCheck() {
|
||||
try {
|
||||
const profile = {
|
||||
id: "release",
|
||||
gate: {
|
||||
id: "test-release-gate",
|
||||
coverage: {
|
||||
requirements: {
|
||||
blocking: ["release-runtime-startup:baseline"],
|
||||
warning: ["fresh-install:baseline"]
|
||||
}
|
||||
},
|
||||
blocking: [
|
||||
{ scenario: "release-runtime-startup", state: "fresh" }
|
||||
]
|
||||
}
|
||||
};
|
||||
const report = {
|
||||
mode: "execution",
|
||||
controls: {
|
||||
include: [],
|
||||
exclude: []
|
||||
},
|
||||
records: [
|
||||
{
|
||||
scenario: "release-runtime-startup",
|
||||
surface: "release-runtime-startup",
|
||||
state: { id: "fresh" },
|
||||
status: "PASS",
|
||||
title: "Release Runtime Startup",
|
||||
likelyOwner: "OpenClaw",
|
||||
phases: []
|
||||
}
|
||||
]
|
||||
};
|
||||
const gate = evaluateGate(report, profile, {
|
||||
resolvedCoverage: {
|
||||
obligations: [{
|
||||
surface: "release-runtime-startup",
|
||||
requirement: "baseline",
|
||||
scenario: "release-runtime-startup",
|
||||
state: "fresh",
|
||||
status: "planned"
|
||||
}]
|
||||
}
|
||||
});
|
||||
|
||||
assertEqual(gate.verdict, "SHIP", "required requirement coverage should pass");
|
||||
assertEqual(gate.cards.some((card) => card.coverage === "requirement" && card.expected === "requirement coverage release-runtime-startup:baseline"), false, "covered requirement should not be missing");
|
||||
assertEqual(gate.cards.some((card) => card.coverage === "requirement" && card.expected === "requirement coverage fresh-install:baseline" && card.severity === "warning"), true, "missing warning requirement should remain visible");
|
||||
return {
|
||||
id: "gate-requirement-coverage",
|
||||
status: "PASS",
|
||||
command: "evaluate synthetic release gate requirement coverage",
|
||||
durationMs: 0
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "gate-requirement-coverage",
|
||||
status: "FAIL",
|
||||
command: "evaluate synthetic release gate requirement coverage",
|
||||
durationMs: 0,
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function gateSubsystemSummaryCheck() {
|
||||
try {
|
||||
const gate = evaluateGate({
|
||||
|
||||
Loading…
Reference in New Issue
Block a user