diff --git a/README.md b/README.md index a678ad0..c106b71 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ import { buildImportLoopProfile, buildPlatformProbes, buildProfileDiff, + buildRefDiff, buildRuntimeProfile, buildWorkspacePlan, createCaptureApi, @@ -63,6 +64,7 @@ import { renderImportLoopProfileMarkdown, renderPlatformProbesMarkdown, renderProfileDiffMarkdown, + renderRefDiffMarkdown, renderRuntimeProfileMarkdown, renderWorkspacePlanMarkdown, renderMarkdownReport, @@ -74,6 +76,7 @@ import { writeImportLoopProfile, writePlatformProbes, writeProfileDiff, + writeRefDiff, writeRuntimeProfile, writeWorkspacePlan, writeReport, @@ -112,6 +115,9 @@ const runtimeProfile = await buildRuntimeProfile({ }); await writeRuntimeProfile(runtimeProfile); +const refDiff = await buildRefDiff({ baseReport, headReport }); +await writeRefDiff(refDiff); + const profileDiff = await buildProfileDiff({ current, baseline, policy }); await writeProfileDiff(profileDiff); ``` diff --git a/package.json b/package.json index 4a2710e..f2a3df0 100644 --- a/package.json +++ b/package.json @@ -31,6 +31,7 @@ "./openclaw-target": "./src/openclaw-target.js", "./platform-probes": "./src/platform-probes.js", "./profile-diff": "./src/profile-diff.js", + "./ref-diff": "./src/ref-diff.js", "./runtime-profile": "./src/runtime-profile.js", "./workspace-plan": "./src/workspace-plan.js" }, diff --git a/src/index.js b/src/index.js index ff0aa32..3d2c762 100644 --- a/src/index.js +++ b/src/index.js @@ -98,6 +98,14 @@ export { validateProfileDiff, writeProfileDiff, } from "./profile-diff.js"; +export { + buildRefDiff, + defaultRefDiffDimensions, + defaultRefDiffOptions, + renderRefDiffMarkdown, + validateRefDiff, + writeRefDiff, +} from "./ref-diff.js"; export { renderMarkdownReport, renderTextSummary, diff --git a/src/ref-diff.js b/src/ref-diff.js new file mode 100644 index 0000000..bc6b718 --- /dev/null +++ b/src/ref-diff.js @@ -0,0 +1,335 @@ +import { renderMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js"; + +export const defaultRefDiffOptions = { + generatedAt: "deterministic", + jsonPath: "reports/plugin-ref-diff.json", + markdownPath: "reports/plugin-ref-diff.md", + reportTitle: "Plugin Inspector Ref Diff", +}; + +export const defaultRefDiffDimensions = [ + { + id: "compatRecords", + label: "Compat records", + targetKey: "compatRecords", + used: () => new Set(), + hardWhenRemovedAndUsed: false, + }, + { + id: "hookNames", + label: "Hook names", + targetKey: "hookNames", + used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.hooks)), + hardWhenRemovedAndUsed: true, + }, + { + id: "apiRegistrars", + label: "API registrars", + targetKey: "apiRegistrars", + used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.registrations)), + hardWhenRemovedAndUsed: true, + }, + { + id: "capturedRegistrars", + label: "Captured registrars", + targetKey: "capturedRegistrars", + used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.registrations)), + hardWhenRemovedAndUsed: false, + }, + { + id: "sdkExports", + label: "SDK exports", + targetKey: "sdkExports", + used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.sdkImports)), + hardWhenRemovedAndUsed: true, + }, + { + id: "manifestFields", + label: "Manifest fields", + targetKey: "manifestFields", + used: (report) => + new Set( + report.fixtures.flatMap((fixture) => + fixture.pluginManifests.flatMap((pluginManifest) => Object.keys(pluginManifest)), + ), + ), + hardWhenRemovedAndUsed: true, + }, + { + id: "manifestContractFields", + label: "Manifest contract fields", + targetKey: "manifestContractFields", + used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.manifestContracts)), + hardWhenRemovedAndUsed: true, + }, +]; + +export async function buildRefDiff(options = {}) { + const baseReport = options.baseReport; + const headReport = options.headReport; + if (!baseReport || !headReport) { + throw new TypeError("buildRefDiff requires baseReport and headReport"); + } + + const dimensions = (options.dimensions ?? defaultRefDiffDimensions).map((dimension) => + compareDimension(dimension, baseReport, headReport), + ); + const issueDelta = compareIssues(baseReport, headReport); + const regressions = [ + ...dimensions.flatMap((dimension) => dimension.regressions), + ...issueDelta.regressions, + ...targetStatusRegressions(baseReport, headReport), + ].sort((left, right) => severityRank(left.severity) - severityRank(right.severity) || left.code.localeCompare(right.code)); + + const hardRegressionCount = regressions.filter((regression) => regression.action === "fail").length; + const warningRegressionCount = regressions.filter((regression) => regression.action === "warn").length; + + return { + generatedAt: options.generatedAt ?? defaultRefDiffOptions.generatedAt, + base: summarizeReport(options.baseLabel ?? "base", baseReport), + head: summarizeReport(options.headLabel ?? "head", headReport), + status: hardRegressionCount === 0 ? "pass" : "fail", + summary: { + dimensionCount: dimensions.length, + hardRegressionCount, + warningRegressionCount, + newIssueCount: issueDelta.added.length, + newP0IssueCount: issueDelta.added.filter((issue) => issue.severity === "P0").length, + newP1IssueCount: issueDelta.added.filter((issue) => issue.severity === "P1").length, + removedIssueCount: issueDelta.removed.length, + }, + dimensions, + issueDelta, + regressions, + }; +} + +export function validateRefDiff(diff, options = {}) { + return diff.regressions + .filter((regression) => regression.action === "fail" || (options.strict && regression.action === "warn")) + .map((regression) => `${regression.code}: ${regression.message}: ${regression.evidence.join(", ")}`); +} + +export async function writeRefDiff(diff, options = {}) { + return writeJsonMarkdownArtifacts({ + jsonPath: options.jsonPath, + markdownPath: options.markdownPath, + json: diff, + markdown: renderRefDiffMarkdown(diff, options), + check: options.check, + }); +} + +export function renderRefDiffMarkdown(diff, options = {}) { + const title = options.title ?? options.reportTitle ?? defaultRefDiffOptions.reportTitle; + return [ + `# ${title}`, + "", + `Generated: ${diff.generatedAt}`, + `Status: ${diff.status.toUpperCase()}`, + "", + "## Summary", + "", + markdownTable( + [ + ["Base", `${diff.base.label} (${diff.base.targetOpenClaw.status})`], + ["Head", `${diff.head.label} (${diff.head.targetOpenClaw.status})`], + ["Hard regressions", diff.summary.hardRegressionCount], + ["Warning regressions", diff.summary.warningRegressionCount], + ["New issues", diff.summary.newIssueCount], + ["New P0 issues", diff.summary.newP0IssueCount], + ["New P1 issues", diff.summary.newP1IssueCount], + ["Removed issues", diff.summary.removedIssueCount], + ], + ["Metric", "Value"], + ), + "", + "## Target Surface Delta", + "", + markdownTable( + diff.dimensions.map((dimension) => [ + dimension.label, + dimension.baseCount, + dimension.headCount, + signed(dimension.headCount - dimension.baseCount), + dimension.added.join(", ") || "-", + dimension.removed.join(", ") || "-", + dimension.removedUsed.join(", ") || "-", + ]), + ["Surface", "Base", "Head", "Delta", "Added", "Removed", "Removed used"], + ), + "", + "## Regressions", + "", + markdownTable( + diff.regressions.map((regression) => [ + regression.action, + regression.severity, + regression.dimension, + regression.code, + regression.message, + regression.evidence.join(", "), + ]), + ["Action", "Severity", "Surface", "Code", "Message", "Evidence"], + ), + "", + "## New Issues", + "", + markdownTable( + diff.issueDelta.added.map((issue) => [ + issue.severity, + issue.fixture, + issue.code, + issue.title, + issue.evidence.join(", "), + ]), + ["Severity", "Fixture", "Code", "Title", "Evidence"], + ), + "", + "## Removed Issues", + "", + markdownTable( + diff.issueDelta.removed.map((issue) => [ + issue.severity, + issue.fixture, + issue.code, + issue.title, + issue.evidence.join(", "), + ]), + ["Severity", "Fixture", "Code", "Title", "Evidence"], + ), + ].join("\n"); +} + +function summarizeReport(label, report) { + return { + label, + targetOpenClaw: { + status: report.targetOpenClaw.status, + configuredPath: report.targetOpenClaw.configuredPath, + compatRecordCount: report.targetOpenClaw.compatRecordCount ?? 0, + hookNameCount: report.targetOpenClaw.hookNameCount ?? 0, + apiRegistrarCount: report.targetOpenClaw.apiRegistrarCount ?? 0, + capturedRegistrarCount: report.targetOpenClaw.capturedRegistrarCount ?? 0, + sdkExportCount: report.targetOpenClaw.sdkExportCount ?? 0, + manifestFieldCount: report.targetOpenClaw.manifestFieldCount ?? 0, + manifestContractFieldCount: report.targetOpenClaw.manifestContractFieldCount ?? 0, + }, + report: { + status: report.status, + breakageCount: report.summary.breakageCount, + warningCount: report.summary.warningCount, + suggestionCount: report.summary.suggestionCount, + issueCount: report.summary.issueCount, + p0IssueCount: report.summary.p0IssueCount, + p1IssueCount: report.summary.p1IssueCount, + contractProbeCount: report.summary.contractProbeCount, + }, + }; +} + +function compareDimension(dimension, baseReport, headReport) { + const baseValues = sortedSet(baseReport.targetOpenClaw[dimension.targetKey] ?? []); + const headValues = sortedSet(headReport.targetOpenClaw[dimension.targetKey] ?? []); + const usedValues = sortedSet([...dimension.used(baseReport), ...dimension.used(headReport)]); + const added = headValues.filter((value) => !baseValues.includes(value)); + const removed = baseValues.filter((value) => !headValues.includes(value)); + const removedUsed = removed.filter((value) => usedValues.includes(value)); + const regressions = []; + + if (removedUsed.length > 0) { + regressions.push({ + code: `${dimension.id}.removed-used`, + severity: "P0", + action: dimension.hardWhenRemovedAndUsed ? "fail" : "warn", + dimension: dimension.id, + message: `${dimension.label} removed values used by fixtures`, + evidence: removedUsed, + }); + } + + const removedUnused = removed.filter((value) => !usedValues.includes(value)); + if (removedUnused.length > 0) { + regressions.push({ + code: `${dimension.id}.removed-unused`, + severity: "P3", + action: "warn", + dimension: dimension.id, + message: `${dimension.label} removed values not used by current fixtures`, + evidence: removedUnused, + }); + } + + return { + id: dimension.id, + label: dimension.label, + baseCount: baseValues.length, + headCount: headValues.length, + added, + removed, + removedUsed, + used: usedValues, + regressions, + }; +} + +function compareIssues(baseReport, headReport) { + const baseIssues = new Map(baseReport.issues.map((issue) => [issue.id, issue])); + const headIssues = new Map(headReport.issues.map((issue) => [issue.id, issue])); + const added = [...headIssues.values()].filter((issue) => !baseIssues.has(issue.id)).sort(issueSort); + const removed = [...baseIssues.values()].filter((issue) => !headIssues.has(issue.id)).sort(issueSort); + const regressions = added.map((issue) => ({ + code: `issue.${issue.id}`, + severity: issue.severity, + action: ["P0", "P1"].includes(issue.severity) ? "fail" : "warn", + dimension: "issues", + message: `new ${issue.severity} issue: ${issue.title}`, + evidence: [issue.fixture, issue.code, ...issue.evidence], + })); + + return { added, removed, regressions }; +} + +function targetStatusRegressions(baseReport, headReport) { + if (baseReport.targetOpenClaw.status === headReport.targetOpenClaw.status) { + return []; + } + return [ + { + code: "target.status.changed", + severity: headReport.targetOpenClaw.status === "ok" ? "P3" : "P0", + action: headReport.targetOpenClaw.status === "ok" ? "warn" : "fail", + dimension: "targetOpenClaw", + message: `target OpenClaw status changed from ${baseReport.targetOpenClaw.status} to ${headReport.targetOpenClaw.status}`, + evidence: [ + baseReport.targetOpenClaw.configuredPath ?? "base target path unknown", + headReport.targetOpenClaw.configuredPath ?? "head target path unknown", + ], + }, + ]; +} + +function sortedSet(values) { + return [...new Set(values.filter(Boolean))].sort(); +} + +function issueSort(left, right) { + return severityRank(left.severity) - severityRank(right.severity) || left.id.localeCompare(right.id); +} + +function severityRank(value) { + return { P0: 0, P1: 1, P2: 2, P3: 3 }[value] ?? 4; +} + +function signed(value) { + return value > 0 ? `+${value}` : String(value); +} + +function markdownTable(rows, headers) { + return renderMarkdownTable(rows, headers, { + empty: "_none_", + escape: false, + nullValue: "-", + padding: true, + }); +} diff --git a/test/ref-diff.test.js b/test/ref-diff.test.js new file mode 100644 index 0000000..40a29df --- /dev/null +++ b/test/ref-diff.test.js @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { buildRefDiff, renderRefDiffMarkdown, validateRefDiff, writeRefDiff } from "../src/ref-diff.js"; + +test("ref diff fails removed plugin-facing hooks and SDK exports", async () => { + const baseReport = reportFixture({ + hookNames: ["llm_input", "llm_output"], + sdkExports: ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech"], + }); + const headReport = reportFixture({ + hookNames: ["llm_input"], + sdkExports: ["openclaw/plugin-sdk"], + }); + + const diff = await buildRefDiff({ baseReport, headReport, baseLabel: "stable", headLabel: "candidate" }); + + assert.equal(diff.status, "fail"); + assert.ok(validateRefDiff(diff).some((error) => error.includes("hookNames.removed-used"))); + assert.ok(validateRefDiff(diff).some((error) => error.includes("sdkExports.removed-used"))); + assert.match(renderRefDiffMarkdown(diff), /Removed used/); +}); + +test("ref diff reports new P1 issues as hard regressions", async () => { + const baseReport = reportFixture({ issues: [] }); + const headReport = reportFixture({ + issues: [ + { + id: "PLUGIN-NEWP1", + fixture: "fixture", + severity: "P1", + code: "sdk-export-missing", + title: "plugin SDK import aliases are missing from target package exports", + evidence: ["openclaw/plugin-sdk/example"], + }, + ], + }); + + const diff = await buildRefDiff({ baseReport, headReport }); + + assert.equal(diff.summary.newP1IssueCount, 1); + assert.ok(validateRefDiff(diff).some((error) => error.includes("PLUGIN-NEWP1"))); +}); + +test("ref diff strict mode escalates warning regressions", async () => { + const baseReport = reportFixture({ + sdkExports: ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech", "openclaw/plugin-sdk/unused"], + }); + const headReport = reportFixture({ + sdkExports: ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech"], + }); + + const diff = await buildRefDiff({ baseReport, headReport }); + + assert.equal(diff.status, "pass"); + assert.equal(diff.summary.warningRegressionCount, 1); + assert.deepEqual(validateRefDiff(diff), []); + assert.match(validateRefDiff(diff, { strict: true }).join("\n"), /sdkExports\.removed-unused/); +}); + +test("ref diff fails when target OpenClaw status regresses", async () => { + const baseReport = reportFixture(); + const headReport = reportFixture({ targetStatus: "missing" }); + + const diff = await buildRefDiff({ baseReport, headReport }); + + assert.equal(diff.status, "fail"); + assert.match(validateRefDiff(diff).join("\n"), /target\.status\.changed/); +}); + +test("ref diff writer emits json and markdown artifacts", async () => { + const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ref-diff-")); + const jsonPath = path.join(dir, "ref-diff.json"); + const markdownPath = path.join(dir, "ref-diff.md"); + const diff = await buildRefDiff({ + baseReport: reportFixture({ issues: [] }), + headReport: reportFixture({ + issues: [ + { + id: "PLUGIN-NEWP2", + fixture: "fixture", + severity: "P2", + code: "manifest-unknown-fields", + title: "manifest uses unsupported top-level fields", + evidence: ["extra"], + }, + ], + }), + baseLabel: "base", + headLabel: "head", + }); + + assert.deepEqual(await writeRefDiff(diff, { jsonPath, markdownPath }), { jsonPath, markdownPath }); + assert.equal(JSON.parse(await readFile(jsonPath, "utf8")).summary.newIssueCount, 1); + assert.match(await readFile(markdownPath, "utf8"), /PLUGIN-NEWP2/); +}); + +function reportFixture(overrides = {}) { + const targetOpenClaw = { + status: overrides.targetStatus ?? "ok", + configuredPath: "../openclaw", + compatRecords: overrides.compatRecords ?? ["legacy-root-sdk-import"], + hookNames: overrides.hookNames ?? ["llm_input", "llm_output"], + apiRegistrars: overrides.apiRegistrars ?? ["registerService"], + capturedRegistrars: overrides.capturedRegistrars ?? ["registerService"], + sdkExports: overrides.sdkExports ?? ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech"], + manifestFields: overrides.manifestFields ?? ["id", "name", "configSchema"], + manifestContractFields: overrides.manifestContractFields ?? ["tools"], + }; + + return { + status: "pass", + targetOpenClaw: { + ...targetOpenClaw, + compatRecordCount: targetOpenClaw.compatRecords.length, + hookNameCount: targetOpenClaw.hookNames.length, + apiRegistrarCount: targetOpenClaw.apiRegistrars.length, + capturedRegistrarCount: targetOpenClaw.capturedRegistrars.length, + sdkExportCount: targetOpenClaw.sdkExports.length, + manifestFieldCount: targetOpenClaw.manifestFields.length, + manifestContractFieldCount: targetOpenClaw.manifestContractFields.length, + }, + summary: { + breakageCount: 0, + warningCount: 0, + suggestionCount: 0, + issueCount: overrides.issues?.length ?? 0, + p0IssueCount: overrides.issues?.filter((issue) => issue.severity === "P0").length ?? 0, + p1IssueCount: overrides.issues?.filter((issue) => issue.severity === "P1").length ?? 0, + contractProbeCount: 0, + }, + fixtures: [ + { + hooks: ["llm_output"], + registrations: ["registerService"], + sdkImports: ["openclaw/plugin-sdk/speech"], + manifestContracts: ["tools"], + pluginManifests: [{ id: "fixture", name: "Fixture", configSchema: {} }], + }, + ], + issues: overrides.issues ?? [], + }; +}