openclaw/plugin-inspector
openclaw/plugin-inspector
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 11 Wiki Activity

feat: add execution results inventory

Browse Source
This commit is contained in:
Vincent Koc 2026-04-26 21:21:16 -07:00
parent 50d92e106d
commit 686f588c24
No known key found for this signature in database
5 changed files with 520 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
README.md
View File

@ -47,6 +47,7 @@ import {
buildCiSummary,
buildColdImportReadiness,
buildContractCapture,
buildExecutionResultsReport,
buildPlatformProbes,
buildWorkspacePlan,
createCaptureApi,
@ -55,12 +56,14 @@ import {
readOpenClawTargetSurface,
renderColdImportReadinessMarkdown,
renderContractCaptureMarkdown,
renderExecutionResultsMarkdown,
renderPlatformProbesMarkdown,
renderWorkspacePlanMarkdown,
renderMarkdownReport,
writeCiSummary,
writeColdImportReadiness,
writeContractCapture,
writeExecutionResultsReport,
writePlatformProbes,
writeWorkspacePlan,
writeReport,
@ -86,6 +89,9 @@ await writeWorkspacePlan(workspacePlan);
const platformProbes = buildPlatformProbes({ plan: workspacePlan });
await writePlatformProbes(platformProbes);
const executionResults = await buildExecutionResultsReport({ resultsDir: ".plugin-inspector/results" });
await writeExecutionResultsReport(executionResults);
```
## Scope

1
package.json
View File

@ -25,6 +25,7 @@
"./capture-api": "./src/capture-api.js",
"./cold-import-readiness": "./src/cold-import-readiness.js",
"./contract-capture": "./src/contract-capture.js",
"./execution-results": "./src/execution-results.js",
"./openclaw-target": "./src/openclaw-target.js",
"./platform-probes": "./src/platform-probes.js",
"./workspace-plan": "./src/workspace-plan.js"

304
src/execution-results.js Normal file
View File

@ -0,0 +1,304 @@
import { existsSync } from "node:fs";
import { readdir, readFile } from "node:fs/promises";
import path from "node:path";
import { renderMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
export const defaultExecutionResultsOptions = {
generatedAt: "deterministic",
markdownPath: "reports/plugin-execution-results.md",
reportTitle: "Plugin Execution Results",
resultsDir: ".plugin-inspector/results",
jsonPath: "reports/plugin-execution-results.json",
};
export async function buildExecutionResultsReport(options = {}) {
const rootDir = path.resolve(options.rootDir ?? process.cwd());
const resultsDir = resolveFromRoot(rootDir, options.resultsDir ?? defaultExecutionResultsOptions.resultsDir);
const artifacts = existsSync(resultsDir) ? await readArtifacts(resultsDir, { rootDir }) : [];
const syntheticArtifacts = artifacts.filter((artifact) => artifact.kind === "synthetic");
const captureArtifacts = artifacts.filter((artifact) => artifact.kind === "capture");
const auditArtifacts = artifacts.filter((artifact) => artifact.kind === "audit");
const profileArtifacts = artifacts.filter((artifact) => artifact.kind === "profile");
return {
generatedAt: options.generatedAt ?? defaultExecutionResultsOptions.generatedAt,
resultsDir: repoRelative(resultsDir, { rootDir }),
summary: {
artifactCount: artifacts.length,
captureArtifactCount: captureArtifacts.length,
syntheticArtifactCount: syntheticArtifacts.length,
auditArtifactCount: auditArtifacts.length,
profileArtifactCount: profileArtifacts.length,
capturedRegistrationCount: captureArtifacts.reduce(
(sum, artifact) => sum + (artifact.capturedCount ?? 0),
0,
),
auditFindingCount: auditArtifacts.reduce((sum, artifact) => sum + artifact.findingCount, 0),
executionWallMs: profileArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.totalWallMs ?? 0), 0),
maxPeakRssMb: Math.max(0, ...profileArtifacts.map((artifact) => artifact.summary?.maxPeakRssMb ?? 0)),
maxCpuMsEstimate: Math.max(0, ...profileArtifacts.map((artifact) => artifact.summary?.maxCpuMsEstimate ?? 0)),
passCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.passCount ?? 0), 0),
failCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.failCount ?? 0), 0),
blockedCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.blockedCount ?? 0), 0),
},
artifacts,
};
}
export async function writeExecutionResultsReport(report, options = {}) {
const rootDir = path.resolve(options.rootDir ?? process.cwd());
const jsonPath = resolveFromRoot(rootDir, options.jsonPath ?? defaultExecutionResultsOptions.jsonPath);
const markdownPath = resolveFromRoot(rootDir, options.markdownPath ?? defaultExecutionResultsOptions.markdownPath);
return writeJsonMarkdownArtifacts({
jsonPath,
markdownPath,
json: report,
markdown: renderExecutionResultsMarkdown(report, options),
check: options.check,
});
}
export function renderExecutionResultsMarkdown(report, options = {}) {
const title = options.title ?? options.reportTitle ?? defaultExecutionResultsOptions.reportTitle;
return [
`# ${title}`,
"",
`Generated: ${report.generatedAt}`,
`Results dir: ${report.resultsDir}`,
"",
"## Summary",
"",
markdownTable(
[
["Artifacts", report.summary.artifactCount],
["Capture artifacts", report.summary.captureArtifactCount],
["Synthetic artifacts", report.summary.syntheticArtifactCount],
["Audit artifacts", report.summary.auditArtifactCount],
["Profile artifacts", report.summary.profileArtifactCount],
["Captured registrations/hooks", report.summary.capturedRegistrationCount],
["Audit findings", report.summary.auditFindingCount],
["Execution wall", `${report.summary.executionWallMs} ms`],
["Max peak RSS", `${report.summary.maxPeakRssMb} MB`],
["Max CPU estimate", `${report.summary.maxCpuMsEstimate} ms`],
["Pass", report.summary.passCount],
["Fail", report.summary.failCount],
["Blocked", report.summary.blockedCount],
],
["Metric", "Value"],
),
"",
"## Artifacts",
"",
markdownTable(
report.artifacts.map((artifact) => [
artifact.fixture,
artifact.kind,
artifact.status,
artifact.entrypoint,
summarizeArtifactResult(artifact),
artifact.artifactPath,
]),
["Fixture", "Kind", "Status", "Entrypoint", "Result", "Artifact"],
),
"",
"## Blocked Synthetic Probes",
"",
markdownTable(
report.artifacts.flatMap((artifact) =>
(artifact.blocked ?? []).map((item) => [
artifact.fixture,
item.kind,
item.seam,
item.label,
item.reason,
artifact.artifactPath,
]),
),
["Fixture", "Kind", "Seam", "Label", "Reason", "Artifact"],
),
"",
"## Failed Synthetic Probes",
"",
markdownTable(
report.artifacts.flatMap((artifact) =>
(artifact.failures ?? []).map((item) => [
artifact.fixture,
item.kind,
item.seam,
item.label,
item.error,
artifact.artifactPath,
]),
),
["Fixture", "Kind", "Seam", "Label", "Error", "Artifact"],
),
"",
"## Dependency Audit Artifacts",
"",
markdownTable(
report.artifacts
.filter((artifact) => artifact.kind === "audit")
.map((artifact) => [
artifact.fixture,
artifact.findingCount,
artifact.vulnerabilities ? JSON.stringify(artifact.vulnerabilities) : "-",
artifact.artifactPath,
]),
["Fixture", "Findings", "Vulnerabilities", "Artifact"],
),
"",
"## Execution Profiles",
"",
markdownTable(
report.artifacts.flatMap((artifact) =>
(artifact.slowestSteps ?? []).map((step) => [
artifact.fixture,
step.kind,
`${step.wallMs} ms`,
`${step.peakRssMb} MB`,
`${step.cpuMsEstimate} ms`,
step.command,
]),
),
["Fixture", "Step", "Wall", "Peak RSS", "CPU Estimate", "Command"],
),
].join("\n");
}
async function readArtifacts(resultsDir, options) {
const paths = await listJsonFiles(resultsDir);
const artifacts = [];
for (const artifactPath of paths) {
const parsed = JSON.parse(await readFile(artifactPath, "utf8"));
const relativePath = repoRelative(artifactPath, options);
artifacts.push(summarizeArtifact({ artifactPath: relativePath, parsed, rootDir: options.rootDir }));
}
return artifacts.sort((left, right) => left.artifactPath.localeCompare(right.artifactPath));
}
async function listJsonFiles(dir) {
const entries = await readdir(dir, { withFileTypes: true });
const files = [];
for (const entry of entries) {
const entryPath = path.join(dir, entry.name);
if (entry.isDirectory()) {
files.push(...(await listJsonFiles(entryPath)));
continue;
}
if (entry.isFile() && entry.name.endsWith(".json")) {
files.push(entryPath);
}
}
return files;
}
function summarizeArtifact({ artifactPath, parsed, rootDir }) {
const normalizedArtifactPath = toRepoPath(artifactPath);
const kind = normalizedArtifactPath.endsWith(".synthetic.json")
? "synthetic"
: normalizedArtifactPath.endsWith("package-audit.json")
? "audit"
: normalizedArtifactPath.endsWith("execution-profile.json")
? "profile"
: "capture";
const fixture = normalizedArtifactPath.split("/").at(-2) ?? "unknown";
if (kind === "synthetic") {
return {
artifactPath: normalizedArtifactPath,
fixture,
kind,
entrypoint: scrubPath(parsed.entrypoint, { rootDir }),
status: parsed.status,
summary: parsed.summary,
failures: (parsed.results ?? []).filter((result) => result.status === "fail"),
blocked: (parsed.results ?? []).filter((result) => result.status === "blocked"),
};
}
if (kind === "audit") {
return {
artifactPath: normalizedArtifactPath,
fixture,
kind,
entrypoint: "-",
status: "warning",
findingCount: auditFindingCount(parsed),
vulnerabilities: parsed.metadata?.vulnerabilities ?? null,
};
}
if (kind === "profile") {
return {
artifactPath: normalizedArtifactPath,
fixture,
kind,
entrypoint: "-",
status: parsed.summary?.failCount > 0 ? "fail" : "pass",
summary: parsed.summary,
slowestSteps: [...(parsed.steps ?? [])].sort((left, right) => right.wallMs - left.wallMs).slice(0, 5),
};
}
return {
artifactPath: normalizedArtifactPath,
fixture,
kind,
entrypoint: scrubPath(parsed.entrypoint, { rootDir }),
status: parsed.status,
capturedCount: parsed.captured?.length ?? 0,
captured: (parsed.captured ?? []).map((item) => `${item.kind}:${item.name}`),
};
}
function summarizeArtifactResult(artifact) {
if (artifact.kind === "audit") {
return `${artifact.findingCount} audit findings`;
}
if (artifact.kind === "profile") {
return `${artifact.summary?.stepCount ?? 0} steps / ${artifact.summary?.totalWallMs ?? 0} ms / ${artifact.summary?.maxPeakRssMb ?? 0} MB`;
}
if (artifact.summary) {
return `${artifact.summary.passCount} pass / ${artifact.summary.failCount} fail / ${artifact.summary.blockedCount} blocked`;
}
return `${artifact.capturedCount} captured`;
}
function auditFindingCount(parsed) {
const vulnerabilities = parsed.metadata?.vulnerabilities;
if (vulnerabilities && typeof vulnerabilities === "object") {
const severityTotal = Object.entries(vulnerabilities)
.filter(([key]) => key !== "total")
.reduce((sum, [, value]) => sum + (Number(value) || 0), 0);
return severityTotal || Number(vulnerabilities.total) || 0;
}
if (Array.isArray(parsed.vulnerabilities)) {
return parsed.vulnerabilities.length;
}
if (parsed.vulnerabilities && typeof parsed.vulnerabilities === "object") {
return Object.keys(parsed.vulnerabilities).length;
}
return 0;
}
function scrubPath(value, options) {
return typeof value === "string" ? repoRelative(value, options) : value;
}
function repoRelative(value, options = {}) {
const rootDir = path.resolve(options.rootDir ?? process.cwd());
const absolute = path.resolve(value);
const relative = path.relative(rootDir, absolute);
if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) {
return toRepoPath(relative || ".");
}
return toRepoPath(value);
}
function resolveFromRoot(rootDir, value) {
return path.isAbsolute(value) ? value : path.join(rootDir, value);
}
function toRepoPath(value) {
return String(value).replaceAll("\\", "/").replaceAll(path.sep, "/");
}
function markdownTable(rows, headers) {
return renderMarkdownTable(rows, headers, { empty: "_none_", escape: false, padding: true });
}

6
src/index.js
View File

@ -46,6 +46,12 @@ export {
knownIssueCodes,
summarizeIssueClasses,
} from "./issues.js";
export {
buildExecutionResultsReport,
defaultExecutionResultsOptions,
renderExecutionResultsMarkdown,
writeExecutionResultsReport,
} from "./execution-results.js";
export {
defaultOpenClawCheckoutPaths,
openClawTargetPathCandidates,

203
test/execution-results.test.js Normal file
View File

@ -0,0 +1,203 @@
import assert from "node:assert/strict";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "node:test";
import {
buildExecutionResultsReport,
renderExecutionResultsMarkdown,
writeExecutionResultsReport,
} from "../src/execution-results.js";
test("execution results summarize capture, synthetic, audit, and profile artifacts", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-"));
const resultsDir = path.join(rootDir, ".plugin-inspector", "results");
const fixtureDir = path.join(resultsDir, "wecom");
await mkdir(fixtureDir, { recursive: true });
await writeFile(
path.join(fixtureDir, "entry.capture.json"),
JSON.stringify({
entrypoint: path.join(rootDir, ".plugin-inspector/workspaces/wecom/index.js"),
status: "captured",
captured: [{ kind: "hook", name: "before_tool_call" }],
}),
"utf8",
);
await writeFile(
path.join(fixtureDir, "entry.synthetic.json"),
JSON.stringify({
entrypoint: path.join(rootDir, ".plugin-inspector/workspaces/wecom/index.js"),
status: "captured",
summary: { probeCount: 2, passCount: 1, failCount: 0, blockedCount: 1 },
results: [
{ kind: "hook", seam: "before_tool_call", label: "before_tool_call", status: "pass" },
{
kind: "registration",
seam: "registerChannel",
label: "registerChannel",
status: "blocked",
reason: "channel runtime opt-in",
},
],
}),
"utf8",
);
await writeFile(
path.join(fixtureDir, "package-audit.json"),
JSON.stringify({
metadata: {
vulnerabilities: {
info: 0,
low: 1,
moderate: 2,
high: 3,
critical: 0,
total: 6,
},
},
}),
"utf8",
);
await writeFile(
path.join(fixtureDir, "execution-profile.json"),
JSON.stringify({
summary: {
stepCount: 2,
failCount: 0,
totalWallMs: 123,
maxPeakRssMb: 42.5,
maxCpuMsEstimate: 80,
},
steps: [
{
kind: "install",
wallMs: 100,
peakRssMb: 42.5,
cpuMsEstimate: 80,
command: "npm install --ignore-scripts",
},
],
}),
"utf8",
);
const report = await buildExecutionResultsReport({ rootDir, resultsDir });
const markdown = renderExecutionResultsMarkdown(report, { title: "Fixture Execution Results" });
assert.equal(report.resultsDir, ".plugin-inspector/results");
assert.equal(report.summary.artifactCount, 4);
assert.equal(report.summary.auditArtifactCount, 1);
assert.equal(report.summary.profileArtifactCount, 1);
assert.equal(report.summary.auditFindingCount, 6);
assert.equal(report.summary.executionWallMs, 123);
assert.equal(report.summary.maxPeakRssMb, 42.5);
assert.equal(report.summary.maxCpuMsEstimate, 80);
assert.equal(report.summary.capturedRegistrationCount, 1);
assert.equal(report.summary.passCount, 1);
assert.equal(report.summary.blockedCount, 1);
assert.equal(report.artifacts.find((artifact) => artifact.kind === "synthetic").blocked[0].seam, "registerChannel");
assert.equal(report.artifacts.find((artifact) => artifact.kind === "audit").findingCount, 6);
assert.match(markdown, /# Fixture Execution Results/);
assert.match(markdown, /2 steps \/ 123 ms \/ 42.5 MB/);
assert.match(markdown, /Execution Profiles/);
});
test("execution results count total-only audit metadata", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-"));
const fixtureDir = path.join(rootDir, ".plugin-inspector", "results", "minimal");
await mkdir(fixtureDir, { recursive: true });
await writeFile(
path.join(fixtureDir, "package-audit.json"),
JSON.stringify({
metadata: {
vulnerabilities: {
total: 4,
},
},
}),
"utf8",
);
const report = await buildExecutionResultsReport({ rootDir });
assert.equal(report.summary.auditFindingCount, 4);
assert.equal(report.artifacts[0].findingCount, 4);
});
test("execution results handle empty result directories", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-"));
const resultsDir = path.join(rootDir, ".plugin-inspector", "missing-results");
const report = await buildExecutionResultsReport({ rootDir, resultsDir });
assert.equal(report.summary.artifactCount, 0);
assert.equal(report.summary.passCount, 0);
assert.deepEqual(report.artifacts, []);
assert.match(renderExecutionResultsMarkdown(report), /## Artifacts\n\n_none_/);
});
test("execution results recurse, sort artifacts, and count alternate audit shapes", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-"));
const resultsDir = path.join(rootDir, ".plugin-inspector", "results");
await mkdir(path.join(resultsDir, "z-fixture"), { recursive: true });
await mkdir(path.join(resultsDir, "a-fixture", "nested"), { recursive: true });
await writeFile(
path.join(resultsDir, "z-fixture", "package-audit.json"),
JSON.stringify({ vulnerabilities: [{ id: "one" }, { id: "two" }] }),
"utf8",
);
await writeFile(
path.join(resultsDir, "a-fixture", "nested", "package-audit.json"),
JSON.stringify({ vulnerabilities: { low: {}, high: {}, advisory: {} } }),
"utf8",
);
const report = await buildExecutionResultsReport({ rootDir });
assert.equal(report.summary.artifactCount, 2);
assert.equal(report.summary.auditFindingCount, 5);
assert.deepEqual(
report.artifacts.map((artifact) => artifact.fixture),
["nested", "z-fixture"],
);
assert.deepEqual(
report.artifacts.map((artifact) => artifact.findingCount),
[3, 2],
);
});
test("execution results writer preserves failures and repo-relative entrypoint paths", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-"));
const resultsDir = path.join(rootDir, ".plugin-inspector", "results");
const fixtureDir = path.join(resultsDir, "broken");
const outputDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-execution-report-"));
const jsonPath = path.join(outputDir, "execution.json");
const markdownPath = path.join(outputDir, "execution.md");
await mkdir(fixtureDir, { recursive: true });
await writeFile(
path.join(fixtureDir, "entry.synthetic.json"),
JSON.stringify({
entrypoint: path.join(rootDir, "plugins/broken/index.js"),
status: "captured",
summary: { probeCount: 1, passCount: 0, failCount: 1, blockedCount: 0 },
results: [
{
kind: "registration",
seam: "registerTool",
label: "registerTool.execute",
status: "fail",
error: "boom",
},
],
}),
"utf8",
);
const report = await buildExecutionResultsReport({ rootDir });
assert.equal(report.artifacts[0].entrypoint, "plugins/broken/index.js");
assert.match(renderExecutionResultsMarkdown(report), /registerTool\.execute/);
assert.deepEqual(await writeExecutionResultsReport(report, { jsonPath, markdownPath }), { jsonPath, markdownPath });
assert.equal(JSON.parse(await readFile(jsonPath, "utf8")).summary.failCount, 1);
assert.match(await readFile(markdownPath, "utf8"), /boom/);
});