openclaw/plugin-inspector
openclaw/plugin-inspector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 11 Wiki Activity

feat(report): reconcile runtime capture evidence

Browse Source
This commit is contained in:
Vincent Koc 2026-05-02 11:22:20 -07:00
parent a6af8800e0
commit 4bc5fbcfa3
No known key found for this signature in database
9 changed files with 304 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/advanced.js
View File

@ -184,6 +184,10 @@ export {
validateRuntimeProfile,
writeRuntimeProfile,
} from "./runtime-profile.js";
export {
applyRuntimeExecutionCoverage,
buildRuntimeExecutionCoverage,
} from "./runtime-reconciliation.js";
export {
buildRuntimeCaptureReport,
renderRuntimeCaptureMarkdown,

4
src/api.js
View File

@ -45,6 +45,7 @@ export async function inspectPluginRoot(options = {}) {
return inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
});
}
@ -59,6 +60,7 @@ export async function inspectCompatibilityFixtureSetConfig(options = {}) {
return inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
});
}
@ -112,6 +114,7 @@ export async function buildFixtureSetColdImportReadiness(options = {}) {
(await inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
}));
@ -143,6 +146,7 @@ export async function buildFixtureSetWorkspacePlan(options = {}) {
(await inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
}));
const rootDir = options.rootDir ?? config?.rootDir ?? options.cwd;

42
src/compatibility-report.js
View File

@ -26,13 +26,20 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
["Warnings", report.summary.warningCount],
["Compatibility suggestions", report.summary.suggestionCount],
["Issue findings", report.summary.issueCount],
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
["P0 issues", report.summary.p0IssueCount],
["P1 issues", report.summary.p1IssueCount],
["Open P0 issues", report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
["Open P1 issues", report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
["Live issues", report.summary.liveIssueCount],
["Live P0 issues", report.summary.liveP0IssueCount],
["Compat gaps", report.summary.compatGapCount],
["Deprecation warnings", report.summary.deprecationWarningCount],
["Inspector gaps", report.summary.inspectorGapCount],
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
["Upstream metadata", report.summary.upstreamIssueCount],
["Contract probes", report.summary.contractProbeCount],
["Decision rows", report.summary.decisionCount],
@ -65,7 +72,11 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
"",
"## Inspector Proof Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
"",
"## Runtime-Covered Inspector Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
"",
"## Upstream Metadata Issues",
"",
@ -141,13 +152,20 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
markdownTable(
[
["Issue findings", report.summary.issueCount],
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
[severityLabel("P0", options), report.summary.p0IssueCount],
[severityLabel("P1", options), report.summary.p1IssueCount],
[`Open ${severityLabel("P0", options)}`, report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
[`Open ${severityLabel("P1", options)}`, report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
["Live issues", report.summary.liveIssueCount],
["Live P0 issues", report.summary.liveP0IssueCount],
["Compat gaps", report.summary.compatGapCount],
["Deprecation warnings", report.summary.deprecationWarningCount],
["Inspector gaps", report.summary.inspectorGapCount],
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
["Upstream metadata", report.summary.upstreamIssueCount],
["Contract probes", report.summary.contractProbeCount],
],
@ -179,7 +197,11 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
"",
"## Inspector Proof Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
"",
"## Runtime-Covered Inspector Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
"",
"## Upstream Metadata Issues",
"",
@ -228,6 +250,7 @@ function issueBlock(issue, options) {
` - state: ${issueState(issue)}`,
" - evidence:",
...evidenceList(issue.evidence, options).map((item) => ` - ${item}`),
...runtimeCoverageList(issue, options),
].join("\n");
}
@ -235,6 +258,7 @@ function issueState(issue) {
const flags = [
issue.status,
`compat:${issue.compatStatus ?? "none"}`,
issue.runtimeCoverage?.status ? `runtime:${issue.runtimeCoverage.status}` : null,
issue.live ? "live" : null,
issue.deprecated ? "deprecated" : null,
].filter(Boolean);
@ -266,7 +290,7 @@ function triageOverview(report) {
"inspector-gap",
report.summary.inspectorGapCount,
"-",
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments.",
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments. Runtime-covered rows are proof-backed and not open report work.",
],
[
"upstream-metadata",
@ -357,3 +381,15 @@ function evidenceList(evidence, options) {
const formatEvidence = options.formatEvidence ?? ((item) => item);
return items.map((item) => formatEvidence(item));
}
function runtimeCoverageList(issue, options) {
const runtimeCoverage = issue.runtimeCoverage;
if (!runtimeCoverage) {
return [];
}
return [
" - runtime coverage:",
...evidenceList(runtimeCoverage.captured, options).map((item) => ` - captured ${item}`),
...evidenceList(runtimeCoverage.artifacts, options).map((item) => ` - ${item}`),
];
}

7
src/index.js
View File

@ -13,6 +13,7 @@ import * as profileDiffApi from "./profile-diff.js";
import * as refDiffApi from "./ref-diff.js";
import * as reportApi from "./report.js";
import * as runtimeProfileApi from "./runtime-profile.js";
import * as runtimeReconciliationApi from "./runtime-reconciliation.js";
import * as syntheticProbeSuiteApi from "./synthetic-probe-suite.js";
import * as syntheticProbesApi from "./synthetic-probes.js";
@ -111,6 +112,8 @@ export const runtime = Object.freeze({
writeImportLoopProfile: importLoopProfileApi.writeImportLoopProfile,
renderImportLoopProfile: importLoopProfileApi.renderImportLoopProfileMarkdown,
validateImportLoopProfile: importLoopProfileApi.validateImportLoopProfile,
applyExecutionCoverage: runtimeReconciliationApi.applyRuntimeExecutionCoverage,
buildExecutionCoverage: runtimeReconciliationApi.buildRuntimeExecutionCoverage,
});
export const synthetic = Object.freeze({
@ -227,6 +230,10 @@ export {
validateRuntimeProfile,
writeRuntimeProfile,
} from "./runtime-profile.js";
export {
applyRuntimeExecutionCoverage,
buildRuntimeExecutionCoverage,
} from "./runtime-reconciliation.js";
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
export {
buildSyntheticProbePlan,

1
src/inspector.js
View File

@ -35,6 +35,7 @@ export async function inspectCompatibilityFixtureSet(config, options = {}) {
inspections,
failures,
generatedAt: options.generatedAt,
executionResults: options.executionResults,
targetOpenClaw,
buildFixtureReport: ({ fixture, inspection }) =>
buildCompatibilityFixtureReport({

3
src/issues.js
View File

@ -261,7 +261,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
owner: finding.owner,
code: finding.code,
decision: finding.decision,
status: finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open",
status: finding.status ?? (finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open"),
issueClass: finding.issueClass,
live: finding.live,
deprecated: finding.deprecated,
@ -269,6 +269,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
title: issueTitle(finding),
evidence: finding.evidence ?? [],
compatRecord: finding.compatRecord ?? null,
runtimeCoverage: finding.runtimeCoverage ?? null,
}));
}

20
src/report.js
View File

@ -5,6 +5,7 @@ import { buildContractProbes } from "./contract-probes.js";
import { classifyCompatibilityFixture } from "./fixture-summary.js";
import { buildIssues, summarizeIssueClasses } from "./issues.js";
import { sanitizeReportArtifact } from "./report-sanitizer.js";
import { applyRuntimeExecutionCoverage } from "./runtime-reconciliation.js";
export function buildReport({ config, inspections, failures = [], generatedAt = "deterministic" }) {
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
@ -140,6 +141,10 @@ export async function buildCompatibilityReport(options = {}) {
decisions,
});
const runtimeCoverage = applyRuntimeExecutionCoverage({
findings: [...warnings, ...suggestions],
executionResults: options.executionResults,
});
const issues = buildIssues({
breakages,
warnings,
@ -149,6 +154,8 @@ export async function buildCompatibilityReport(options = {}) {
});
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures: fixtureReports });
const issueSummary = summarizeIssueClasses(issues);
const openIssues = issues.filter((issue) => issue.status !== "runtime-covered");
const openIssueSummary = summarizeIssueClasses(openIssues);
return {
generatedAt: options.generatedAt ?? "deterministic",
@ -163,8 +170,11 @@ export async function buildCompatibilityReport(options = {}) {
decisionCount: decisions.length,
logCount: logs.length,
issueCount: issues.length,
openIssueCount: openIssues.length,
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
openP0IssueCount: openIssues.filter((issue) => issue.severity === "P0").length,
openP1IssueCount: openIssues.filter((issue) => issue.severity === "P1").length,
liveIssueCount: issueSummary["live-issue"],
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
compatGapCount: issueSummary["compat-gap"],
@ -172,6 +182,10 @@ export async function buildCompatibilityReport(options = {}) {
inspectorGapCount: issueSummary["inspector-gap"],
upstreamIssueCount: issueSummary["upstream-metadata"],
fixtureRegressionCount: issueSummary["fixture-regression"],
openInspectorGapCount: openIssueSummary["inspector-gap"],
runtimeCoveredIssueCount: runtimeCoverage.coveredFindingCount,
runtimePartiallyCoveredIssueCount: runtimeCoverage.partiallyCoveredFindingCount,
runtimeCoverageArtifactCount: runtimeCoverage.coverage.artifactCount,
contractProbeCount: contractProbes.length,
},
fixtures: fixtureReports,
@ -308,7 +322,11 @@ function topTextFindings(report, limit) {
return [
...(report.breakages ?? []).map((finding) => formatTextFinding(finding, "breakage")),
...(report.issues ?? [])
.filter((issue) => issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1")
.filter(
(issue) =>
issue.status !== "runtime-covered" &&
(issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1"),
)
.map((issue) => formatTextFinding(issue, issue.severity ?? "issue")),
...(report.warnings ?? []).map((finding) => formatTextFinding(finding, "warning")),
].slice(0, limit);

124
src/runtime-reconciliation.js Normal file
View File

@ -0,0 +1,124 @@
export function applyRuntimeExecutionCoverage({ findings = [], executionResults } = {}) {
const coverage = buildRuntimeExecutionCoverage(executionResults);
let coveredFindingCount = 0;
let partiallyCoveredFindingCount = 0;
for (const finding of findings) {
const findingCoverage = runtimeCoverageForFinding(finding, coverage);
if (!findingCoverage) {
continue;
}
finding.runtimeCoverage = findingCoverage;
if (findingCoverage.status === "covered") {
finding.status = "runtime-covered";
coveredFindingCount += 1;
} else {
partiallyCoveredFindingCount += 1;
}
}
return {
coverage,
coveredFindingCount,
partiallyCoveredFindingCount,
};
}
export function buildRuntimeExecutionCoverage(executionResults) {
const fixtures = new Map();
for (const artifact of executionResults?.artifacts ?? []) {
if (artifact.kind !== "capture") {
continue;
}
const fixture = String(artifact.fixture ?? "unknown");
const fixtureCoverage = ensureFixtureCoverage(fixtures, fixture);
if (artifact.artifactPath) {
fixtureCoverage.artifacts.add(artifact.artifactPath);
}
for (const captured of normalizeCaptured(artifact.captured)) {
fixtureCoverage.captured.add(captured);
}
}
return {
fixtures,
artifactCount: [...fixtures.values()].reduce((sum, fixture) => sum + fixture.artifacts.size, 0),
};
}
function runtimeCoverageForFinding(finding, coverage) {
const fixtureCoverage = coverage.fixtures.get(finding.fixture);
if (!fixtureCoverage || fixtureCoverage.captured.size === 0) {
return null;
}
const expected = expectedRuntimeCaptureKeys(finding);
if (expected.length === 0) {
return null;
}
const captured = expected.filter((item) => fixtureCoverage.captured.has(item));
if (captured.length === 0) {
return null;
}
return {
status: captured.length === expected.length ? "covered" : "partial",
captured,
expected,
artifacts: [...fixtureCoverage.artifacts].sort(),
};
}
function expectedRuntimeCaptureKeys(finding) {
const names = evidenceNames(finding.evidence);
if (finding.code === "registration-capture-gap") {
return names.map((name) => `registration:${name}`);
}
if (finding.code === "runtime-tool-capture") {
return ["registration:registerTool"];
}
if (finding.code === "conversation-access-hook") {
return names.map((name) => `hook:${name}`);
}
return [];
}
function normalizeCaptured(captured) {
return (captured ?? [])
.map((item) => {
if (typeof item === "string") {
return item;
}
if (item && typeof item === "object" && item.kind && item.name) {
return `${item.kind}:${item.name}`;
}
return "";
})
.filter(Boolean);
}
function evidenceNames(evidence) {
return [
...new Set(
(evidence ?? [])
.map((item) => String(item).split(" @ ")[0]?.trim())
.filter(Boolean),
),
];
}
function ensureFixtureCoverage(fixtures, fixture) {
let fixtureCoverage = fixtures.get(fixture);
if (!fixtureCoverage) {
fixtureCoverage = {
artifacts: new Set(),
captured: new Set(),
};
fixtures.set(fixture, fixtureCoverage);
}
return fixtureCoverage;
}

104
test/report.test.js
View File

@ -370,6 +370,110 @@ test("compatibility report assembly classifies fixtures, issues, probes, and com
assert.ok(report.decisions.some((decision) => decision.seam === "compat-registry"));
});
test("compatibility report marks inspector gaps covered by runtime execution artifacts", async () => {
const report = await buildCompatibilityReport({
generatedAt: "test",
fixtures: [
{
id: "fixture",
name: "Fixture",
path: "plugins/fixture",
priority: "high",
seams: ["native-tool"],
why: "covers runtime-only seams",
},
],
inspections: [
{
id: "fixture",
status: "ok",
hooks: ["llm_input"],
hookDetails: [{ name: "llm_input", ref: "plugins/fixture/src/index.ts:1" }],
registrations: ["registerTool", "registerService", "registerCommand"],
registrationDetails: [
{ name: "registerTool", ref: "plugins/fixture/src/index.ts:2" },
{ name: "registerService", ref: "plugins/fixture/src/index.ts:3" },
{ name: "registerCommand", ref: "plugins/fixture/src/index.ts:4" },
],
manifestContracts: [],
manifestFiles: [],
sdkImports: [],
sourceFiles: ["plugins/fixture/src/index.ts"],
},
],
targetOpenClaw: {
status: "ok",
compatRecords: [],
compatRecordStatuses: {},
hookNames: ["llm_input"],
apiRegistrars: ["registerTool", "registerService", "registerCommand"],
capturedRegistrars: [],
sdkExports: [],
manifestFields: ["id"],
manifestContractFields: [],
},
executionResults: {
artifacts: [
{
fixture: "fixture",
kind: "capture",
status: "pass",
artifactPath: ".crabpot/results/fixture/index.capture.json",
captured: [
"hook:llm_input",
"registration:registerTool",
"registration:registerService",
"registration:registerCommand",
],
},
],
},
buildFixtureReport: ({ fixture, inspection }) => ({
id: fixture.id,
name: fixture.name,
priority: fixture.priority,
seams: fixture.seams,
why: fixture.why,
status: inspection.status,
hooks: inspection.hooks,
hookDetails: inspection.hookDetails,
registrations: inspection.registrations,
registrationDetails: inspection.registrationDetails,
manifestContracts: inspection.manifestContracts,
manifestFiles: [],
sourceFiles: inspection.sourceFiles,
pluginManifests: [],
package: null,
packages: [],
sdkImports: [],
sdkImportDetails: [],
}),
});
const coveredCodes = report.issues
.filter((issue) => issue.status === "runtime-covered")
.map((issue) => issue.code)
.sort();
assert.deepEqual(coveredCodes, [
"conversation-access-hook",
"registration-capture-gap",
"runtime-tool-capture",
]);
assert.equal(report.summary.runtimeCoveredIssueCount, 3);
assert.equal(report.summary.openInspectorGapCount, 0);
assert.equal(report.summary.runtimeCoverageArtifactCount, 1);
const registrationIssue = report.issues.find((issue) => issue.code === "registration-capture-gap");
assert.deepEqual(registrationIssue.runtimeCoverage.captured, [
"registration:registerService",
"registration:registerCommand",
]);
const markdown = renderCompatibilityIssuesReport(report);
assert.match(markdown, /## Runtime-Covered Inspector Gaps/);
assert.match(markdown, /state: runtime-covered .* runtime:covered/);
});
test("compat record coverage logs unavailable targets", () => {
const logs = [];
classifyCompatRecordCoverage({