feat: add OpenClaw ref diff reports

This commit is contained in:
Vincent Koc 2026-04-26 21:59:19 -07:00
parent 4afedcf499
commit dee771af62
No known key found for this signature in database
5 changed files with 495 additions and 0 deletions

View File

@ -51,6 +51,7 @@ import {
buildImportLoopProfile,
buildPlatformProbes,
buildProfileDiff,
buildRefDiff,
buildRuntimeProfile,
buildWorkspacePlan,
createCaptureApi,
@ -63,6 +64,7 @@ import {
renderImportLoopProfileMarkdown,
renderPlatformProbesMarkdown,
renderProfileDiffMarkdown,
renderRefDiffMarkdown,
renderRuntimeProfileMarkdown,
renderWorkspacePlanMarkdown,
renderMarkdownReport,
@ -74,6 +76,7 @@ import {
writeImportLoopProfile,
writePlatformProbes,
writeProfileDiff,
writeRefDiff,
writeRuntimeProfile,
writeWorkspacePlan,
writeReport,
@ -112,6 +115,9 @@ const runtimeProfile = await buildRuntimeProfile({
});
await writeRuntimeProfile(runtimeProfile);
const refDiff = await buildRefDiff({ baseReport, headReport });
await writeRefDiff(refDiff);
const profileDiff = await buildProfileDiff({ current, baseline, policy });
await writeProfileDiff(profileDiff);
```

View File

@ -31,6 +31,7 @@
"./openclaw-target": "./src/openclaw-target.js",
"./platform-probes": "./src/platform-probes.js",
"./profile-diff": "./src/profile-diff.js",
"./ref-diff": "./src/ref-diff.js",
"./runtime-profile": "./src/runtime-profile.js",
"./workspace-plan": "./src/workspace-plan.js"
},

View File

@ -98,6 +98,14 @@ export {
validateProfileDiff,
writeProfileDiff,
} from "./profile-diff.js";
export {
buildRefDiff,
defaultRefDiffDimensions,
defaultRefDiffOptions,
renderRefDiffMarkdown,
validateRefDiff,
writeRefDiff,
} from "./ref-diff.js";
export {
renderMarkdownReport,
renderTextSummary,

335
src/ref-diff.js Normal file
View File

@ -0,0 +1,335 @@
import { renderMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
export const defaultRefDiffOptions = {
generatedAt: "deterministic",
jsonPath: "reports/plugin-ref-diff.json",
markdownPath: "reports/plugin-ref-diff.md",
reportTitle: "Plugin Inspector Ref Diff",
};
export const defaultRefDiffDimensions = [
{
id: "compatRecords",
label: "Compat records",
targetKey: "compatRecords",
used: () => new Set(),
hardWhenRemovedAndUsed: false,
},
{
id: "hookNames",
label: "Hook names",
targetKey: "hookNames",
used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.hooks)),
hardWhenRemovedAndUsed: true,
},
{
id: "apiRegistrars",
label: "API registrars",
targetKey: "apiRegistrars",
used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.registrations)),
hardWhenRemovedAndUsed: true,
},
{
id: "capturedRegistrars",
label: "Captured registrars",
targetKey: "capturedRegistrars",
used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.registrations)),
hardWhenRemovedAndUsed: false,
},
{
id: "sdkExports",
label: "SDK exports",
targetKey: "sdkExports",
used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.sdkImports)),
hardWhenRemovedAndUsed: true,
},
{
id: "manifestFields",
label: "Manifest fields",
targetKey: "manifestFields",
used: (report) =>
new Set(
report.fixtures.flatMap((fixture) =>
fixture.pluginManifests.flatMap((pluginManifest) => Object.keys(pluginManifest)),
),
),
hardWhenRemovedAndUsed: true,
},
{
id: "manifestContractFields",
label: "Manifest contract fields",
targetKey: "manifestContractFields",
used: (report) => new Set(report.fixtures.flatMap((fixture) => fixture.manifestContracts)),
hardWhenRemovedAndUsed: true,
},
];
export async function buildRefDiff(options = {}) {
const baseReport = options.baseReport;
const headReport = options.headReport;
if (!baseReport || !headReport) {
throw new TypeError("buildRefDiff requires baseReport and headReport");
}
const dimensions = (options.dimensions ?? defaultRefDiffDimensions).map((dimension) =>
compareDimension(dimension, baseReport, headReport),
);
const issueDelta = compareIssues(baseReport, headReport);
const regressions = [
...dimensions.flatMap((dimension) => dimension.regressions),
...issueDelta.regressions,
...targetStatusRegressions(baseReport, headReport),
].sort((left, right) => severityRank(left.severity) - severityRank(right.severity) || left.code.localeCompare(right.code));
const hardRegressionCount = regressions.filter((regression) => regression.action === "fail").length;
const warningRegressionCount = regressions.filter((regression) => regression.action === "warn").length;
return {
generatedAt: options.generatedAt ?? defaultRefDiffOptions.generatedAt,
base: summarizeReport(options.baseLabel ?? "base", baseReport),
head: summarizeReport(options.headLabel ?? "head", headReport),
status: hardRegressionCount === 0 ? "pass" : "fail",
summary: {
dimensionCount: dimensions.length,
hardRegressionCount,
warningRegressionCount,
newIssueCount: issueDelta.added.length,
newP0IssueCount: issueDelta.added.filter((issue) => issue.severity === "P0").length,
newP1IssueCount: issueDelta.added.filter((issue) => issue.severity === "P1").length,
removedIssueCount: issueDelta.removed.length,
},
dimensions,
issueDelta,
regressions,
};
}
export function validateRefDiff(diff, options = {}) {
return diff.regressions
.filter((regression) => regression.action === "fail" || (options.strict && regression.action === "warn"))
.map((regression) => `${regression.code}: ${regression.message}: ${regression.evidence.join(", ")}`);
}
export async function writeRefDiff(diff, options = {}) {
return writeJsonMarkdownArtifacts({
jsonPath: options.jsonPath,
markdownPath: options.markdownPath,
json: diff,
markdown: renderRefDiffMarkdown(diff, options),
check: options.check,
});
}
export function renderRefDiffMarkdown(diff, options = {}) {
const title = options.title ?? options.reportTitle ?? defaultRefDiffOptions.reportTitle;
return [
`# ${title}`,
"",
`Generated: ${diff.generatedAt}`,
`Status: ${diff.status.toUpperCase()}`,
"",
"## Summary",
"",
markdownTable(
[
["Base", `${diff.base.label} (${diff.base.targetOpenClaw.status})`],
["Head", `${diff.head.label} (${diff.head.targetOpenClaw.status})`],
["Hard regressions", diff.summary.hardRegressionCount],
["Warning regressions", diff.summary.warningRegressionCount],
["New issues", diff.summary.newIssueCount],
["New P0 issues", diff.summary.newP0IssueCount],
["New P1 issues", diff.summary.newP1IssueCount],
["Removed issues", diff.summary.removedIssueCount],
],
["Metric", "Value"],
),
"",
"## Target Surface Delta",
"",
markdownTable(
diff.dimensions.map((dimension) => [
dimension.label,
dimension.baseCount,
dimension.headCount,
signed(dimension.headCount - dimension.baseCount),
dimension.added.join(", ") || "-",
dimension.removed.join(", ") || "-",
dimension.removedUsed.join(", ") || "-",
]),
["Surface", "Base", "Head", "Delta", "Added", "Removed", "Removed used"],
),
"",
"## Regressions",
"",
markdownTable(
diff.regressions.map((regression) => [
regression.action,
regression.severity,
regression.dimension,
regression.code,
regression.message,
regression.evidence.join(", "),
]),
["Action", "Severity", "Surface", "Code", "Message", "Evidence"],
),
"",
"## New Issues",
"",
markdownTable(
diff.issueDelta.added.map((issue) => [
issue.severity,
issue.fixture,
issue.code,
issue.title,
issue.evidence.join(", "),
]),
["Severity", "Fixture", "Code", "Title", "Evidence"],
),
"",
"## Removed Issues",
"",
markdownTable(
diff.issueDelta.removed.map((issue) => [
issue.severity,
issue.fixture,
issue.code,
issue.title,
issue.evidence.join(", "),
]),
["Severity", "Fixture", "Code", "Title", "Evidence"],
),
].join("\n");
}
function summarizeReport(label, report) {
return {
label,
targetOpenClaw: {
status: report.targetOpenClaw.status,
configuredPath: report.targetOpenClaw.configuredPath,
compatRecordCount: report.targetOpenClaw.compatRecordCount ?? 0,
hookNameCount: report.targetOpenClaw.hookNameCount ?? 0,
apiRegistrarCount: report.targetOpenClaw.apiRegistrarCount ?? 0,
capturedRegistrarCount: report.targetOpenClaw.capturedRegistrarCount ?? 0,
sdkExportCount: report.targetOpenClaw.sdkExportCount ?? 0,
manifestFieldCount: report.targetOpenClaw.manifestFieldCount ?? 0,
manifestContractFieldCount: report.targetOpenClaw.manifestContractFieldCount ?? 0,
},
report: {
status: report.status,
breakageCount: report.summary.breakageCount,
warningCount: report.summary.warningCount,
suggestionCount: report.summary.suggestionCount,
issueCount: report.summary.issueCount,
p0IssueCount: report.summary.p0IssueCount,
p1IssueCount: report.summary.p1IssueCount,
contractProbeCount: report.summary.contractProbeCount,
},
};
}
function compareDimension(dimension, baseReport, headReport) {
const baseValues = sortedSet(baseReport.targetOpenClaw[dimension.targetKey] ?? []);
const headValues = sortedSet(headReport.targetOpenClaw[dimension.targetKey] ?? []);
const usedValues = sortedSet([...dimension.used(baseReport), ...dimension.used(headReport)]);
const added = headValues.filter((value) => !baseValues.includes(value));
const removed = baseValues.filter((value) => !headValues.includes(value));
const removedUsed = removed.filter((value) => usedValues.includes(value));
const regressions = [];
if (removedUsed.length > 0) {
regressions.push({
code: `${dimension.id}.removed-used`,
severity: "P0",
action: dimension.hardWhenRemovedAndUsed ? "fail" : "warn",
dimension: dimension.id,
message: `${dimension.label} removed values used by fixtures`,
evidence: removedUsed,
});
}
const removedUnused = removed.filter((value) => !usedValues.includes(value));
if (removedUnused.length > 0) {
regressions.push({
code: `${dimension.id}.removed-unused`,
severity: "P3",
action: "warn",
dimension: dimension.id,
message: `${dimension.label} removed values not used by current fixtures`,
evidence: removedUnused,
});
}
return {
id: dimension.id,
label: dimension.label,
baseCount: baseValues.length,
headCount: headValues.length,
added,
removed,
removedUsed,
used: usedValues,
regressions,
};
}
function compareIssues(baseReport, headReport) {
const baseIssues = new Map(baseReport.issues.map((issue) => [issue.id, issue]));
const headIssues = new Map(headReport.issues.map((issue) => [issue.id, issue]));
const added = [...headIssues.values()].filter((issue) => !baseIssues.has(issue.id)).sort(issueSort);
const removed = [...baseIssues.values()].filter((issue) => !headIssues.has(issue.id)).sort(issueSort);
const regressions = added.map((issue) => ({
code: `issue.${issue.id}`,
severity: issue.severity,
action: ["P0", "P1"].includes(issue.severity) ? "fail" : "warn",
dimension: "issues",
message: `new ${issue.severity} issue: ${issue.title}`,
evidence: [issue.fixture, issue.code, ...issue.evidence],
}));
return { added, removed, regressions };
}
function targetStatusRegressions(baseReport, headReport) {
if (baseReport.targetOpenClaw.status === headReport.targetOpenClaw.status) {
return [];
}
return [
{
code: "target.status.changed",
severity: headReport.targetOpenClaw.status === "ok" ? "P3" : "P0",
action: headReport.targetOpenClaw.status === "ok" ? "warn" : "fail",
dimension: "targetOpenClaw",
message: `target OpenClaw status changed from ${baseReport.targetOpenClaw.status} to ${headReport.targetOpenClaw.status}`,
evidence: [
baseReport.targetOpenClaw.configuredPath ?? "base target path unknown",
headReport.targetOpenClaw.configuredPath ?? "head target path unknown",
],
},
];
}
function sortedSet(values) {
return [...new Set(values.filter(Boolean))].sort();
}
function issueSort(left, right) {
return severityRank(left.severity) - severityRank(right.severity) || left.id.localeCompare(right.id);
}
function severityRank(value) {
return { P0: 0, P1: 1, P2: 2, P3: 3 }[value] ?? 4;
}
function signed(value) {
return value > 0 ? `+${value}` : String(value);
}
function markdownTable(rows, headers) {
return renderMarkdownTable(rows, headers, {
empty: "_none_",
escape: false,
nullValue: "-",
padding: true,
});
}

145
test/ref-diff.test.js Normal file
View File

@ -0,0 +1,145 @@
import assert from "node:assert/strict";
import { mkdtemp, readFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "node:test";
import { buildRefDiff, renderRefDiffMarkdown, validateRefDiff, writeRefDiff } from "../src/ref-diff.js";
test("ref diff fails removed plugin-facing hooks and SDK exports", async () => {
const baseReport = reportFixture({
hookNames: ["llm_input", "llm_output"],
sdkExports: ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech"],
});
const headReport = reportFixture({
hookNames: ["llm_input"],
sdkExports: ["openclaw/plugin-sdk"],
});
const diff = await buildRefDiff({ baseReport, headReport, baseLabel: "stable", headLabel: "candidate" });
assert.equal(diff.status, "fail");
assert.ok(validateRefDiff(diff).some((error) => error.includes("hookNames.removed-used")));
assert.ok(validateRefDiff(diff).some((error) => error.includes("sdkExports.removed-used")));
assert.match(renderRefDiffMarkdown(diff), /Removed used/);
});
test("ref diff reports new P1 issues as hard regressions", async () => {
const baseReport = reportFixture({ issues: [] });
const headReport = reportFixture({
issues: [
{
id: "PLUGIN-NEWP1",
fixture: "fixture",
severity: "P1",
code: "sdk-export-missing",
title: "plugin SDK import aliases are missing from target package exports",
evidence: ["openclaw/plugin-sdk/example"],
},
],
});
const diff = await buildRefDiff({ baseReport, headReport });
assert.equal(diff.summary.newP1IssueCount, 1);
assert.ok(validateRefDiff(diff).some((error) => error.includes("PLUGIN-NEWP1")));
});
test("ref diff strict mode escalates warning regressions", async () => {
const baseReport = reportFixture({
sdkExports: ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech", "openclaw/plugin-sdk/unused"],
});
const headReport = reportFixture({
sdkExports: ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech"],
});
const diff = await buildRefDiff({ baseReport, headReport });
assert.equal(diff.status, "pass");
assert.equal(diff.summary.warningRegressionCount, 1);
assert.deepEqual(validateRefDiff(diff), []);
assert.match(validateRefDiff(diff, { strict: true }).join("\n"), /sdkExports\.removed-unused/);
});
test("ref diff fails when target OpenClaw status regresses", async () => {
const baseReport = reportFixture();
const headReport = reportFixture({ targetStatus: "missing" });
const diff = await buildRefDiff({ baseReport, headReport });
assert.equal(diff.status, "fail");
assert.match(validateRefDiff(diff).join("\n"), /target\.status\.changed/);
});
test("ref diff writer emits json and markdown artifacts", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ref-diff-"));
const jsonPath = path.join(dir, "ref-diff.json");
const markdownPath = path.join(dir, "ref-diff.md");
const diff = await buildRefDiff({
baseReport: reportFixture({ issues: [] }),
headReport: reportFixture({
issues: [
{
id: "PLUGIN-NEWP2",
fixture: "fixture",
severity: "P2",
code: "manifest-unknown-fields",
title: "manifest uses unsupported top-level fields",
evidence: ["extra"],
},
],
}),
baseLabel: "base",
headLabel: "head",
});
assert.deepEqual(await writeRefDiff(diff, { jsonPath, markdownPath }), { jsonPath, markdownPath });
assert.equal(JSON.parse(await readFile(jsonPath, "utf8")).summary.newIssueCount, 1);
assert.match(await readFile(markdownPath, "utf8"), /PLUGIN-NEWP2/);
});
function reportFixture(overrides = {}) {
const targetOpenClaw = {
status: overrides.targetStatus ?? "ok",
configuredPath: "../openclaw",
compatRecords: overrides.compatRecords ?? ["legacy-root-sdk-import"],
hookNames: overrides.hookNames ?? ["llm_input", "llm_output"],
apiRegistrars: overrides.apiRegistrars ?? ["registerService"],
capturedRegistrars: overrides.capturedRegistrars ?? ["registerService"],
sdkExports: overrides.sdkExports ?? ["openclaw/plugin-sdk", "openclaw/plugin-sdk/speech"],
manifestFields: overrides.manifestFields ?? ["id", "name", "configSchema"],
manifestContractFields: overrides.manifestContractFields ?? ["tools"],
};
return {
status: "pass",
targetOpenClaw: {
...targetOpenClaw,
compatRecordCount: targetOpenClaw.compatRecords.length,
hookNameCount: targetOpenClaw.hookNames.length,
apiRegistrarCount: targetOpenClaw.apiRegistrars.length,
capturedRegistrarCount: targetOpenClaw.capturedRegistrars.length,
sdkExportCount: targetOpenClaw.sdkExports.length,
manifestFieldCount: targetOpenClaw.manifestFields.length,
manifestContractFieldCount: targetOpenClaw.manifestContractFields.length,
},
summary: {
breakageCount: 0,
warningCount: 0,
suggestionCount: 0,
issueCount: overrides.issues?.length ?? 0,
p0IssueCount: overrides.issues?.filter((issue) => issue.severity === "P0").length ?? 0,
p1IssueCount: overrides.issues?.filter((issue) => issue.severity === "P1").length ?? 0,
contractProbeCount: 0,
},
fixtures: [
{
hooks: ["llm_output"],
registrations: ["registerService"],
sdkImports: ["openclaw/plugin-sdk/speech"],
manifestContracts: ["tools"],
pluginManifests: [{ id: "fixture", name: "Fixture", configSchema: {} }],
},
],
issues: overrides.issues ?? [],
};
}