diff --git a/README.md b/README.md index c106b71..7f028a1 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,7 @@ PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector capture ./dist/index.js ```js import { buildCiSummary, + buildCiPolicyReport, buildColdImportReadiness, buildContractCapture, buildExecutionResultsReport, @@ -58,6 +59,7 @@ import { inspectFixtureSet, loadInspectorConfig, readOpenClawTargetSurface, + renderCiPolicyMarkdown, renderColdImportReadinessMarkdown, renderContractCaptureMarkdown, renderExecutionResultsMarkdown, @@ -68,8 +70,10 @@ import { renderRuntimeProfileMarkdown, renderWorkspacePlanMarkdown, renderMarkdownReport, + validateCiPolicyReport, validateContractCoverage, writeCiSummary, + writeCiPolicyReport, writeColdImportReadiness, writeContractCapture, writeExecutionResultsReport, @@ -89,6 +93,9 @@ await writeReport(report, { outDir: "reports" }); const summary = await buildCiSummary({ reportsDir: "reports" }); await writeCiSummary(summary); +const policyReport = buildCiPolicyReport({ policy, compatibilityReport: report }); +await writeCiPolicyReport(policyReport); + const capture = buildContractCapture({ report }); await writeContractCapture(capture); const coverageErrors = validateContractCoverage(report); diff --git a/package.json b/package.json index f2a3df0..5ff41f5 100644 --- a/package.json +++ b/package.json @@ -23,6 +23,7 @@ "exports": { ".": "./src/index.js", "./capture-api": "./src/capture-api.js", + "./ci-policy": "./src/ci-policy.js", "./cold-import-readiness": "./src/cold-import-readiness.js", "./contract-capture": "./src/contract-capture.js", "./contract-coverage": "./src/contract-coverage.js", diff --git a/src/ci-policy.js b/src/ci-policy.js new file mode 100644 index 0000000..94c1180 --- /dev/null +++ b/src/ci-policy.js @@ -0,0 +1,262 @@ +import path from "node:path"; +import { renderMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js"; + +export const defaultCiPolicyReportOptions = { + generatedAt: "deterministic", + jsonPath: "reports/plugin-inspector-ci-policy.json", + markdownPath: "reports/plugin-inspector-ci-policy.md", + reportTitle: "Plugin Inspector CI Policy", +}; + +export function buildCiPolicyReport(options = {}) { + const policy = options.policy; + validateCiPolicy(policy); + + const checks = [ + ...compatibilityChecks(options.compatibilityReport, { strict: options.strict }), + ...refDiffChecks(options.refDiff, { strict: options.strict }), + ...executionChecks(options.executionResults, policy, { strict: options.strict }), + ].sort((left, right) => actionRank(left.action) - actionRank(right.action) || left.id.localeCompare(right.id)); + + return { + generatedAt: options.generatedAt ?? defaultCiPolicyReportOptions.generatedAt, + status: checks.some((check) => check.action === "fail") ? "fail" : "pass", + strict: Boolean(options.strict), + policy: { + allowedBlocked: policy.allowedBlocked.length, + expectedWarnings: policy.expectedWarnings.length, + fixtureSets: Object.keys(policy.fixtureSets).sort(), + thresholds: policy.thresholds, + }, + summary: { + checkCount: checks.length, + failCount: checks.filter((check) => check.action === "fail").length, + warnCount: checks.filter((check) => check.action === "warn").length, + passCount: checks.filter((check) => check.action === "pass").length, + }, + checks, + }; +} + +export function validateCiPolicy(policy) { + const errors = []; + if (policy?.version !== 1) { + errors.push("ci policy version must be 1"); + } + for (const key of ["allowedBlocked", "expectedWarnings"]) { + if (!Array.isArray(policy?.[key])) { + errors.push(`ci policy ${key} must be an array`); + } + } + if (!policy?.thresholds || typeof policy.thresholds !== "object") { + errors.push("ci policy thresholds are required"); + } + if (!policy?.fixtureSets || typeof policy.fixtureSets !== "object") { + errors.push("ci policy fixtureSets are required"); + } + if (errors.length > 0) { + throw new Error(errors.join("\n")); + } +} + +export function validateCiPolicyReport(report) { + return report.checks + .filter((check) => check.action === "fail") + .map((check) => `${check.id}: ${check.message}: ${check.evidence.join(", ")}`); +} + +export async function writeCiPolicyReport(report, options = {}) { + const rootDir = path.resolve(options.rootDir ?? process.cwd()); + const jsonPath = resolveFromRoot(rootDir, options.jsonPath ?? defaultCiPolicyReportOptions.jsonPath); + const markdownPath = resolveFromRoot(rootDir, options.markdownPath ?? defaultCiPolicyReportOptions.markdownPath); + return writeJsonMarkdownArtifacts({ + jsonPath, + markdownPath, + json: report, + markdown: renderCiPolicyMarkdown(report, options), + check: options.check, + }); +} + +export function renderCiPolicyMarkdown(report, options = {}) { + const title = options.title ?? options.reportTitle ?? defaultCiPolicyReportOptions.reportTitle; + return [ + `# ${title}`, + "", + `Generated: ${report.generatedAt}`, + `Status: ${report.status.toUpperCase()}`, + `Strict: ${report.strict}`, + "", + "## Summary", + "", + markdownTable( + [ + ["Checks", report.summary.checkCount], + ["Fail", report.summary.failCount], + ["Warn", report.summary.warnCount], + ["Pass", report.summary.passCount], + ["Allowed blocked rules", report.policy.allowedBlocked], + ["Expected warning rules", report.policy.expectedWarnings], + ["Fixture sets", report.policy.fixtureSets.join(", ")], + ], + ["Metric", "Value"], + ), + "", + "## Checks", + "", + markdownTable( + report.checks.map((check) => [ + check.action, + check.id, + check.message, + check.evidence.join(", ") || "-", + ]), + ["Action", "ID", "Message", "Evidence"], + ), + ].join("\n"); +} + +function compatibilityChecks(report, options) { + const checks = []; + if (!report) { + checks.push({ + id: "compatibility-report.missing", + action: "fail", + message: "compatibility report is missing", + evidence: [], + }); + return checks; + } + checks.push({ + id: "compatibility-report.breakages", + action: report.summary.breakageCount > 0 ? "fail" : "pass", + message: `${report.summary.breakageCount} hard breakages`, + evidence: (report.breakages ?? []).map((finding) => `${finding.fixture}:${finding.code}`), + }); + checks.push({ + id: "compatibility-report.p1-issues", + action: "pass", + message: `${report.summary.p1IssueCount} P1 issues tracked`, + evidence: (report.issues ?? []) + .filter((issue) => issue.severity === "P1") + .map((issue) => `${issue.fixture}:${issue.code}`), + }); + const issues = report.issues ?? []; + const liveP0Issues = issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0"); + const deprecationWarnings = issues.filter((issue) => issue.issueClass === "deprecation-warning"); + const inspectorGaps = issues.filter((issue) => issue.issueClass === "inspector-gap"); + checks.push({ + id: "compatibility-report.live-p0-issues", + action: liveP0Issues.length > 0 ? (options.strict ? "fail" : "warn") : "pass", + message: `${liveP0Issues.length} live P0 issues tracked`, + evidence: liveP0Issues.map((issue) => `${issue.fixture}:${issue.code}:${issue.compatStatus ?? "none"}`), + }); + checks.push({ + id: "compatibility-report.deprecation-warnings", + action: "pass", + message: `${deprecationWarnings.length} deprecated compat seams tracked`, + evidence: deprecationWarnings.map((issue) => `${issue.fixture}:${issue.code}`), + }); + checks.push({ + id: "compatibility-report.inspector-gaps", + action: "pass", + message: `${inspectorGaps.length} inspector proof gaps tracked`, + evidence: inspectorGaps.map((issue) => `${issue.fixture}:${issue.code}`), + }); + return checks; +} + +function refDiffChecks(refDiff, options) { + if (!refDiff) { + return [ + { + id: "ref-diff.not-run", + action: "pass", + message: "ref diff artifact was not present for this CI mode", + evidence: [], + }, + ]; + } + + return (refDiff.regressions ?? []).map((regression) => ({ + id: `ref-diff.${regression.code}`, + action: regression.action === "fail" || (options.strict && regression.action === "warn") ? "fail" : "warn", + message: regression.message, + evidence: regression.evidence ?? [], + })); +} + +function executionChecks(executionResults, policy, options) { + if (!executionResults) { + return [ + { + id: "execution-results.not-run", + action: "pass", + message: "isolated execution artifact was not present for this CI mode", + evidence: [], + }, + ]; + } + + const checks = [ + { + id: "execution-results.failures", + action: executionResults.summary.failCount > 0 ? "fail" : "pass", + message: `${executionResults.summary.failCount} failed synthetic probes`, + evidence: failedExecutionEvidence(executionResults), + }, + { + id: "execution-results.audit-findings", + action: executionResults.summary.auditFindingCount > 0 ? "warn" : "pass", + message: `${executionResults.summary.auditFindingCount ?? 0} package audit findings`, + evidence: executionResults.artifacts + .filter((artifact) => artifact.kind === "audit" && artifact.findingCount > 0) + .map((artifact) => `${artifact.fixture}:${artifact.findingCount}`), + }, + ]; + + const blocked = executionResults.artifacts.flatMap((artifact) => + (artifact.blocked ?? []).map((item) => ({ artifact, item })), + ); + for (const blockedItem of blocked) { + const expectedWarning = findPolicyMatch(policy.expectedWarnings, blockedItem.item); + const allowedBlocked = findPolicyMatch(policy.allowedBlocked, blockedItem.item); + const match = expectedWarning ?? allowedBlocked; + checks.push({ + id: `execution-results.blocked.${blockedItem.artifact.fixture}.${blockedItem.item.seam}.${blockedItem.item.captureIndex}`, + action: match ? (options.strict ? "fail" : "warn") : "fail", + message: match + ? `${match.decision}: ${blockedItem.item.reason}` + : `unknown blocked synthetic probe: ${blockedItem.item.reason}`, + evidence: [ + blockedItem.artifact.artifactPath, + blockedItem.item.seam, + blockedItem.item.reason, + match?.id ?? "unclassified", + ], + }); + } + return checks; +} + +function findPolicyMatch(rules, item) { + return rules.find((rule) => item.seam === rule.seam && item.reason?.includes(rule.reasonIncludes)); +} + +function failedExecutionEvidence(executionResults) { + return executionResults.artifacts.flatMap((artifact) => + (artifact.failures ?? []).map((failure) => `${artifact.fixture}:${failure.seam}:${failure.error}`), + ); +} + +function actionRank(value) { + return { fail: 0, warn: 1, pass: 2 }[value] ?? 3; +} + +function resolveFromRoot(rootDir, value) { + return path.isAbsolute(value) ? value : path.join(rootDir, value); +} + +function markdownTable(rows, headers) { + return renderMarkdownTable(rows, headers, { empty: "_none_", escape: false, padding: true }); +} diff --git a/src/index.js b/src/index.js index 3d2c762..90f7137 100644 --- a/src/index.js +++ b/src/index.js @@ -6,6 +6,14 @@ export { writeJsonMarkdownArtifacts, } from "./artifacts.js"; export { createCaptureApi } from "./capture-api.js"; +export { + buildCiPolicyReport, + defaultCiPolicyReportOptions, + renderCiPolicyMarkdown, + validateCiPolicy, + validateCiPolicyReport, + writeCiPolicyReport, +} from "./ci-policy.js"; export { buildCiSummary, defaultCiReportPaths, diff --git a/test/ci-policy.test.js b/test/ci-policy.test.js new file mode 100644 index 0000000..f5476f7 --- /dev/null +++ b/test/ci-policy.test.js @@ -0,0 +1,265 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + buildCiPolicyReport, + renderCiPolicyMarkdown, + validateCiPolicyReport, + writeCiPolicyReport, +} from "../src/index.js"; + +const policy = { + version: 1, + allowedBlocked: [ + { + id: "channel-runtime-harness", + seam: "registerChannel", + reasonIncludes: "includeChannelRuntime=true", + decision: "allowed-blocked", + until: "channel runtime harness lands", + }, + ], + expectedWarnings: [ + { + id: "tool-factory-descriptor", + seam: "registerTool", + reasonIncludes: "no object descriptor", + decision: "expected-warning", + until: "tool factory capture expansion lands", + }, + ], + thresholds: { + wallP95RegressionPercent: 50, + peakRssRegressionMb: 50, + bootRegressionMs: 500, + strictMinimumSamples: 3, + }, + fixtureSets: { + smoke: ["wecom"], + }, +}; + +test("ci policy allows known blocked probes but fails unknown blockers", () => { + const report = buildCiPolicyReport({ + policy, + compatibilityReport: compatibilityReport(), + executionResults: executionResults([ + { + seam: "registerChannel", + reason: "captured registration requires includeChannelRuntime=true", + }, + { + seam: "registerMystery", + reason: "new blocked reason", + }, + ]), + }); + + assert.equal(report.status, "fail"); + assert.ok(report.checks.some((check) => check.action === "warn" && check.id.includes("registerChannel"))); + assert.ok(validateCiPolicyReport(report).some((error) => error.includes("registerMystery"))); + assert.match(renderCiPolicyMarkdown(report), /Plugin Inspector CI Policy/); +}); + +test("ci policy fails ref diff hard regressions", () => { + const report = buildCiPolicyReport({ + policy, + compatibilityReport: compatibilityReport(), + refDiff: { + regressions: [ + { + code: "hookNames.removed-used", + action: "fail", + message: "Hook names removed values used by fixtures", + evidence: ["llm_output"], + }, + ], + }, + }); + + assert.equal(report.status, "fail"); + assert.ok(validateCiPolicyReport(report).some((error) => error.includes("hookNames.removed-used"))); +}); + +test("ci policy reports package audit findings as warnings", () => { + const report = buildCiPolicyReport({ + policy, + compatibilityReport: compatibilityReport(), + executionResults: { + summary: { + failCount: 0, + auditFindingCount: 2, + }, + artifacts: [ + { + fixture: "fixture", + kind: "audit", + findingCount: 2, + failures: [], + blocked: [], + }, + ], + }, + }); + + assert.equal(report.status, "pass"); + assert.ok(report.checks.some((check) => check.action === "warn" && check.id === "execution-results.audit-findings")); +}); + +test("ci policy surfaces P0 live issues without blocking default lanes", () => { + const report = buildCiPolicyReport({ + policy, + compatibilityReport: compatibilityReport({ + issues: [ + { + severity: "P0", + issueClass: "live-issue", + fixture: "codex-app-server", + code: "sdk-export-missing", + compatStatus: "untracked", + }, + { + severity: "P2", + issueClass: "deprecation-warning", + fixture: "connectclaw", + code: "legacy-before-agent-start", + }, + { + severity: "P1", + issueClass: "inspector-gap", + fixture: "wecom", + code: "registration-capture-gap", + }, + ], + }), + }); + + assert.equal(report.status, "pass"); + assert.ok(report.checks.some((check) => check.id === "compatibility-report.live-p0-issues" && check.action === "warn")); + assert.ok( + report.checks.some((check) => check.id === "compatibility-report.deprecation-warnings" && check.action === "pass"), + ); + assert.ok(report.checks.some((check) => check.id === "compatibility-report.inspector-gaps" && check.action === "pass")); +}); + +test("ci policy strict mode fails P0 live issues", () => { + const report = buildCiPolicyReport({ + policy, + strict: true, + compatibilityReport: compatibilityReport({ + issues: [ + { + severity: "P0", + issueClass: "live-issue", + fixture: "codex-app-server", + code: "sdk-export-missing", + compatStatus: "untracked", + }, + ], + }), + }); + + assert.equal(report.status, "fail"); + assert.match(validateCiPolicyReport(report).join("\n"), /compatibility-report\.live-p0-issues/); +}); + +test("ci policy strict mode escalates classified blocked probes", () => { + const report = buildCiPolicyReport({ + policy, + strict: true, + compatibilityReport: compatibilityReport(), + executionResults: executionResults([ + { + seam: "registerChannel", + reason: "captured registration requires includeChannelRuntime=true", + }, + { + seam: "registerTool", + reason: "factory had no object descriptor", + }, + ]), + }); + + assert.equal(report.status, "fail"); + assert.deepEqual( + report.checks.filter((check) => check.id.startsWith("execution-results.blocked.")).map((check) => check.action), + ["fail", "fail"], + ); + assert.match(validateCiPolicyReport(report).join("\n"), /channel-runtime-harness/); + assert.match(validateCiPolicyReport(report).join("\n"), /tool-factory-descriptor/); +}); + +test("ci policy validation rejects malformed policy files", () => { + assert.throws( + () => + buildCiPolicyReport({ + policy: { + version: 2, + allowedBlocked: {}, + expectedWarnings: null, + thresholds: null, + fixtureSets: null, + }, + compatibilityReport: compatibilityReport(), + }), + /ci policy version must be 1[\s\S]*allowedBlocked must be an array[\s\S]*expectedWarnings must be an array[\s\S]*thresholds are required[\s\S]*fixtureSets are required/, + ); +}); + +test("ci policy writer emits JSON and Markdown artifacts", async () => { + const outputDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ci-policy-")); + const jsonPath = path.join(outputDir, "policy.json"); + const markdownPath = path.join(outputDir, "policy.md"); + const report = buildCiPolicyReport({ + policy, + compatibilityReport: compatibilityReport(), + }); + + assert.deepEqual(await writeCiPolicyReport(report, { jsonPath, markdownPath }), { jsonPath, markdownPath }); + assert.equal(JSON.parse(await readFile(jsonPath, "utf8")).summary.failCount, 0); + assert.match(await readFile(markdownPath, "utf8"), /CI Policy/); +}); + +function compatibilityReport(overrides = {}) { + const issues = overrides.issues ?? [ + { + severity: "P1", + issueClass: "inspector-gap", + fixture: "fixture", + code: "registration-capture-gap", + }, + ]; + return { + summary: { + breakageCount: 0, + p1IssueCount: issues.filter((issue) => issue.severity === "P1").length, + }, + breakages: [], + issues, + }; +} + +function executionResults(blocked) { + return { + summary: { + failCount: 0, + auditFindingCount: 0, + }, + artifacts: [ + { + fixture: "fixture", + artifactPath: ".plugin-inspector/results/fixture/result.synthetic.json", + failures: [], + blocked: blocked.map((item, index) => ({ + captureIndex: index, + kind: "registration", + label: item.seam, + status: "blocked", + ...item, + })), + }, + ], + }; +} diff --git a/test/runtime-profile.test.js b/test/runtime-profile.test.js index 9488da3..186f04c 100644 --- a/test/runtime-profile.test.js +++ b/test/runtime-profile.test.js @@ -45,13 +45,13 @@ test("runtime profile records command timings and plugin surface summaries", asy id: "node-boot", label: "Node boot", category: "baseline", - args: ["-e", "setTimeout(() => undefined, 250)"], + args: ["-e", "setTimeout(() => undefined, 750)"], }, { id: "openclaw-aware", label: "OpenClaw aware command", category: "target-registry", - args: ["-e", "setTimeout(() => process.exit(process.argv.includes('--no-openclaw') ? 0 : 1), 250)", "--"], + args: ["-e", "setTimeout(() => process.exit(process.argv.includes('--no-openclaw') ? 0 : 1), 750)", "--"], openclaw: true, }, ],