From 686f588c24e3d21686e6c2da93b29e5bc7c66262 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sun, 26 Apr 2026 21:21:16 -0700 Subject: [PATCH] feat: add execution results inventory --- README.md | 6 + package.json | 1 + src/execution-results.js | 304 +++++++++++++++++++++++++++++++++ src/index.js | 6 + test/execution-results.test.js | 203 ++++++++++++++++++++++ 5 files changed, 520 insertions(+) create mode 100644 src/execution-results.js create mode 100644 test/execution-results.test.js diff --git a/README.md b/README.md index b18e3c9..a902c05 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ import { buildCiSummary, buildColdImportReadiness, buildContractCapture, + buildExecutionResultsReport, buildPlatformProbes, buildWorkspacePlan, createCaptureApi, @@ -55,12 +56,14 @@ import { readOpenClawTargetSurface, renderColdImportReadinessMarkdown, renderContractCaptureMarkdown, + renderExecutionResultsMarkdown, renderPlatformProbesMarkdown, renderWorkspacePlanMarkdown, renderMarkdownReport, writeCiSummary, writeColdImportReadiness, writeContractCapture, + writeExecutionResultsReport, writePlatformProbes, writeWorkspacePlan, writeReport, @@ -86,6 +89,9 @@ await writeWorkspacePlan(workspacePlan); const platformProbes = buildPlatformProbes({ plan: workspacePlan }); await writePlatformProbes(platformProbes); + +const executionResults = await buildExecutionResultsReport({ resultsDir: ".plugin-inspector/results" }); +await writeExecutionResultsReport(executionResults); ``` ## Scope diff --git a/package.json b/package.json index 9b0c2b7..45d5235 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ "./capture-api": "./src/capture-api.js", "./cold-import-readiness": "./src/cold-import-readiness.js", "./contract-capture": "./src/contract-capture.js", + "./execution-results": "./src/execution-results.js", "./openclaw-target": "./src/openclaw-target.js", "./platform-probes": "./src/platform-probes.js", "./workspace-plan": "./src/workspace-plan.js" diff --git a/src/execution-results.js b/src/execution-results.js new file mode 100644 index 0000000..1420436 --- /dev/null +++ b/src/execution-results.js @@ -0,0 +1,304 @@ +import { existsSync } from "node:fs"; +import { readdir, readFile } from "node:fs/promises"; +import path from "node:path"; +import { renderMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js"; + +export const defaultExecutionResultsOptions = { + generatedAt: "deterministic", + markdownPath: "reports/plugin-execution-results.md", + reportTitle: "Plugin Execution Results", + resultsDir: ".plugin-inspector/results", + jsonPath: "reports/plugin-execution-results.json", +}; + +export async function buildExecutionResultsReport(options = {}) { + const rootDir = path.resolve(options.rootDir ?? process.cwd()); + const resultsDir = resolveFromRoot(rootDir, options.resultsDir ?? defaultExecutionResultsOptions.resultsDir); + const artifacts = existsSync(resultsDir) ? await readArtifacts(resultsDir, { rootDir }) : []; + const syntheticArtifacts = artifacts.filter((artifact) => artifact.kind === "synthetic"); + const captureArtifacts = artifacts.filter((artifact) => artifact.kind === "capture"); + const auditArtifacts = artifacts.filter((artifact) => artifact.kind === "audit"); + const profileArtifacts = artifacts.filter((artifact) => artifact.kind === "profile"); + + return { + generatedAt: options.generatedAt ?? defaultExecutionResultsOptions.generatedAt, + resultsDir: repoRelative(resultsDir, { rootDir }), + summary: { + artifactCount: artifacts.length, + captureArtifactCount: captureArtifacts.length, + syntheticArtifactCount: syntheticArtifacts.length, + auditArtifactCount: auditArtifacts.length, + profileArtifactCount: profileArtifacts.length, + capturedRegistrationCount: captureArtifacts.reduce( + (sum, artifact) => sum + (artifact.capturedCount ?? 0), + 0, + ), + auditFindingCount: auditArtifacts.reduce((sum, artifact) => sum + artifact.findingCount, 0), + executionWallMs: profileArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.totalWallMs ?? 0), 0), + maxPeakRssMb: Math.max(0, ...profileArtifacts.map((artifact) => artifact.summary?.maxPeakRssMb ?? 0)), + maxCpuMsEstimate: Math.max(0, ...profileArtifacts.map((artifact) => artifact.summary?.maxCpuMsEstimate ?? 0)), + passCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.passCount ?? 0), 0), + failCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.failCount ?? 0), 0), + blockedCount: syntheticArtifacts.reduce((sum, artifact) => sum + (artifact.summary?.blockedCount ?? 0), 0), + }, + artifacts, + }; +} + +export async function writeExecutionResultsReport(report, options = {}) { + const rootDir = path.resolve(options.rootDir ?? process.cwd()); + const jsonPath = resolveFromRoot(rootDir, options.jsonPath ?? defaultExecutionResultsOptions.jsonPath); + const markdownPath = resolveFromRoot(rootDir, options.markdownPath ?? defaultExecutionResultsOptions.markdownPath); + return writeJsonMarkdownArtifacts({ + jsonPath, + markdownPath, + json: report, + markdown: renderExecutionResultsMarkdown(report, options), + check: options.check, + }); +} + +export function renderExecutionResultsMarkdown(report, options = {}) { + const title = options.title ?? options.reportTitle ?? defaultExecutionResultsOptions.reportTitle; + return [ + `# ${title}`, + "", + `Generated: ${report.generatedAt}`, + `Results dir: ${report.resultsDir}`, + "", + "## Summary", + "", + markdownTable( + [ + ["Artifacts", report.summary.artifactCount], + ["Capture artifacts", report.summary.captureArtifactCount], + ["Synthetic artifacts", report.summary.syntheticArtifactCount], + ["Audit artifacts", report.summary.auditArtifactCount], + ["Profile artifacts", report.summary.profileArtifactCount], + ["Captured registrations/hooks", report.summary.capturedRegistrationCount], + ["Audit findings", report.summary.auditFindingCount], + ["Execution wall", `${report.summary.executionWallMs} ms`], + ["Max peak RSS", `${report.summary.maxPeakRssMb} MB`], + ["Max CPU estimate", `${report.summary.maxCpuMsEstimate} ms`], + ["Pass", report.summary.passCount], + ["Fail", report.summary.failCount], + ["Blocked", report.summary.blockedCount], + ], + ["Metric", "Value"], + ), + "", + "## Artifacts", + "", + markdownTable( + report.artifacts.map((artifact) => [ + artifact.fixture, + artifact.kind, + artifact.status, + artifact.entrypoint, + summarizeArtifactResult(artifact), + artifact.artifactPath, + ]), + ["Fixture", "Kind", "Status", "Entrypoint", "Result", "Artifact"], + ), + "", + "## Blocked Synthetic Probes", + "", + markdownTable( + report.artifacts.flatMap((artifact) => + (artifact.blocked ?? []).map((item) => [ + artifact.fixture, + item.kind, + item.seam, + item.label, + item.reason, + artifact.artifactPath, + ]), + ), + ["Fixture", "Kind", "Seam", "Label", "Reason", "Artifact"], + ), + "", + "## Failed Synthetic Probes", + "", + markdownTable( + report.artifacts.flatMap((artifact) => + (artifact.failures ?? []).map((item) => [ + artifact.fixture, + item.kind, + item.seam, + item.label, + item.error, + artifact.artifactPath, + ]), + ), + ["Fixture", "Kind", "Seam", "Label", "Error", "Artifact"], + ), + "", + "## Dependency Audit Artifacts", + "", + markdownTable( + report.artifacts + .filter((artifact) => artifact.kind === "audit") + .map((artifact) => [ + artifact.fixture, + artifact.findingCount, + artifact.vulnerabilities ? JSON.stringify(artifact.vulnerabilities) : "-", + artifact.artifactPath, + ]), + ["Fixture", "Findings", "Vulnerabilities", "Artifact"], + ), + "", + "## Execution Profiles", + "", + markdownTable( + report.artifacts.flatMap((artifact) => + (artifact.slowestSteps ?? []).map((step) => [ + artifact.fixture, + step.kind, + `${step.wallMs} ms`, + `${step.peakRssMb} MB`, + `${step.cpuMsEstimate} ms`, + step.command, + ]), + ), + ["Fixture", "Step", "Wall", "Peak RSS", "CPU Estimate", "Command"], + ), + ].join("\n"); +} + +async function readArtifacts(resultsDir, options) { + const paths = await listJsonFiles(resultsDir); + const artifacts = []; + for (const artifactPath of paths) { + const parsed = JSON.parse(await readFile(artifactPath, "utf8")); + const relativePath = repoRelative(artifactPath, options); + artifacts.push(summarizeArtifact({ artifactPath: relativePath, parsed, rootDir: options.rootDir })); + } + return artifacts.sort((left, right) => left.artifactPath.localeCompare(right.artifactPath)); +} + +async function listJsonFiles(dir) { + const entries = await readdir(dir, { withFileTypes: true }); + const files = []; + for (const entry of entries) { + const entryPath = path.join(dir, entry.name); + if (entry.isDirectory()) { + files.push(...(await listJsonFiles(entryPath))); + continue; + } + if (entry.isFile() && entry.name.endsWith(".json")) { + files.push(entryPath); + } + } + return files; +} + +function summarizeArtifact({ artifactPath, parsed, rootDir }) { + const normalizedArtifactPath = toRepoPath(artifactPath); + const kind = normalizedArtifactPath.endsWith(".synthetic.json") + ? "synthetic" + : normalizedArtifactPath.endsWith("package-audit.json") + ? "audit" + : normalizedArtifactPath.endsWith("execution-profile.json") + ? "profile" + : "capture"; + const fixture = normalizedArtifactPath.split("/").at(-2) ?? "unknown"; + if (kind === "synthetic") { + return { + artifactPath: normalizedArtifactPath, + fixture, + kind, + entrypoint: scrubPath(parsed.entrypoint, { rootDir }), + status: parsed.status, + summary: parsed.summary, + failures: (parsed.results ?? []).filter((result) => result.status === "fail"), + blocked: (parsed.results ?? []).filter((result) => result.status === "blocked"), + }; + } + if (kind === "audit") { + return { + artifactPath: normalizedArtifactPath, + fixture, + kind, + entrypoint: "-", + status: "warning", + findingCount: auditFindingCount(parsed), + vulnerabilities: parsed.metadata?.vulnerabilities ?? null, + }; + } + if (kind === "profile") { + return { + artifactPath: normalizedArtifactPath, + fixture, + kind, + entrypoint: "-", + status: parsed.summary?.failCount > 0 ? "fail" : "pass", + summary: parsed.summary, + slowestSteps: [...(parsed.steps ?? [])].sort((left, right) => right.wallMs - left.wallMs).slice(0, 5), + }; + } + return { + artifactPath: normalizedArtifactPath, + fixture, + kind, + entrypoint: scrubPath(parsed.entrypoint, { rootDir }), + status: parsed.status, + capturedCount: parsed.captured?.length ?? 0, + captured: (parsed.captured ?? []).map((item) => `${item.kind}:${item.name}`), + }; +} + +function summarizeArtifactResult(artifact) { + if (artifact.kind === "audit") { + return `${artifact.findingCount} audit findings`; + } + if (artifact.kind === "profile") { + return `${artifact.summary?.stepCount ?? 0} steps / ${artifact.summary?.totalWallMs ?? 0} ms / ${artifact.summary?.maxPeakRssMb ?? 0} MB`; + } + if (artifact.summary) { + return `${artifact.summary.passCount} pass / ${artifact.summary.failCount} fail / ${artifact.summary.blockedCount} blocked`; + } + return `${artifact.capturedCount} captured`; +} + +function auditFindingCount(parsed) { + const vulnerabilities = parsed.metadata?.vulnerabilities; + if (vulnerabilities && typeof vulnerabilities === "object") { + const severityTotal = Object.entries(vulnerabilities) + .filter(([key]) => key !== "total") + .reduce((sum, [, value]) => sum + (Number(value) || 0), 0); + return severityTotal || Number(vulnerabilities.total) || 0; + } + if (Array.isArray(parsed.vulnerabilities)) { + return parsed.vulnerabilities.length; + } + if (parsed.vulnerabilities && typeof parsed.vulnerabilities === "object") { + return Object.keys(parsed.vulnerabilities).length; + } + return 0; +} + +function scrubPath(value, options) { + return typeof value === "string" ? repoRelative(value, options) : value; +} + +function repoRelative(value, options = {}) { + const rootDir = path.resolve(options.rootDir ?? process.cwd()); + const absolute = path.resolve(value); + const relative = path.relative(rootDir, absolute); + if (relative === "" || (!relative.startsWith("..") && !path.isAbsolute(relative))) { + return toRepoPath(relative || "."); + } + return toRepoPath(value); +} + +function resolveFromRoot(rootDir, value) { + return path.isAbsolute(value) ? value : path.join(rootDir, value); +} + +function toRepoPath(value) { + return String(value).replaceAll("\\", "/").replaceAll(path.sep, "/"); +} + +function markdownTable(rows, headers) { + return renderMarkdownTable(rows, headers, { empty: "_none_", escape: false, padding: true }); +} diff --git a/src/index.js b/src/index.js index 2397c5a..f6bb3da 100644 --- a/src/index.js +++ b/src/index.js @@ -46,6 +46,12 @@ export { knownIssueCodes, summarizeIssueClasses, } from "./issues.js"; +export { + buildExecutionResultsReport, + defaultExecutionResultsOptions, + renderExecutionResultsMarkdown, + writeExecutionResultsReport, +} from "./execution-results.js"; export { defaultOpenClawCheckoutPaths, openClawTargetPathCandidates, diff --git a/test/execution-results.test.js b/test/execution-results.test.js new file mode 100644 index 0000000..1f9ae8b --- /dev/null +++ b/test/execution-results.test.js @@ -0,0 +1,203 @@ +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { + buildExecutionResultsReport, + renderExecutionResultsMarkdown, + writeExecutionResultsReport, +} from "../src/execution-results.js"; + +test("execution results summarize capture, synthetic, audit, and profile artifacts", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-")); + const resultsDir = path.join(rootDir, ".plugin-inspector", "results"); + const fixtureDir = path.join(resultsDir, "wecom"); + await mkdir(fixtureDir, { recursive: true }); + await writeFile( + path.join(fixtureDir, "entry.capture.json"), + JSON.stringify({ + entrypoint: path.join(rootDir, ".plugin-inspector/workspaces/wecom/index.js"), + status: "captured", + captured: [{ kind: "hook", name: "before_tool_call" }], + }), + "utf8", + ); + await writeFile( + path.join(fixtureDir, "entry.synthetic.json"), + JSON.stringify({ + entrypoint: path.join(rootDir, ".plugin-inspector/workspaces/wecom/index.js"), + status: "captured", + summary: { probeCount: 2, passCount: 1, failCount: 0, blockedCount: 1 }, + results: [ + { kind: "hook", seam: "before_tool_call", label: "before_tool_call", status: "pass" }, + { + kind: "registration", + seam: "registerChannel", + label: "registerChannel", + status: "blocked", + reason: "channel runtime opt-in", + }, + ], + }), + "utf8", + ); + await writeFile( + path.join(fixtureDir, "package-audit.json"), + JSON.stringify({ + metadata: { + vulnerabilities: { + info: 0, + low: 1, + moderate: 2, + high: 3, + critical: 0, + total: 6, + }, + }, + }), + "utf8", + ); + await writeFile( + path.join(fixtureDir, "execution-profile.json"), + JSON.stringify({ + summary: { + stepCount: 2, + failCount: 0, + totalWallMs: 123, + maxPeakRssMb: 42.5, + maxCpuMsEstimate: 80, + }, + steps: [ + { + kind: "install", + wallMs: 100, + peakRssMb: 42.5, + cpuMsEstimate: 80, + command: "npm install --ignore-scripts", + }, + ], + }), + "utf8", + ); + + const report = await buildExecutionResultsReport({ rootDir, resultsDir }); + const markdown = renderExecutionResultsMarkdown(report, { title: "Fixture Execution Results" }); + + assert.equal(report.resultsDir, ".plugin-inspector/results"); + assert.equal(report.summary.artifactCount, 4); + assert.equal(report.summary.auditArtifactCount, 1); + assert.equal(report.summary.profileArtifactCount, 1); + assert.equal(report.summary.auditFindingCount, 6); + assert.equal(report.summary.executionWallMs, 123); + assert.equal(report.summary.maxPeakRssMb, 42.5); + assert.equal(report.summary.maxCpuMsEstimate, 80); + assert.equal(report.summary.capturedRegistrationCount, 1); + assert.equal(report.summary.passCount, 1); + assert.equal(report.summary.blockedCount, 1); + assert.equal(report.artifacts.find((artifact) => artifact.kind === "synthetic").blocked[0].seam, "registerChannel"); + assert.equal(report.artifacts.find((artifact) => artifact.kind === "audit").findingCount, 6); + assert.match(markdown, /# Fixture Execution Results/); + assert.match(markdown, /2 steps \/ 123 ms \/ 42.5 MB/); + assert.match(markdown, /Execution Profiles/); +}); + +test("execution results count total-only audit metadata", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-")); + const fixtureDir = path.join(rootDir, ".plugin-inspector", "results", "minimal"); + await mkdir(fixtureDir, { recursive: true }); + await writeFile( + path.join(fixtureDir, "package-audit.json"), + JSON.stringify({ + metadata: { + vulnerabilities: { + total: 4, + }, + }, + }), + "utf8", + ); + + const report = await buildExecutionResultsReport({ rootDir }); + + assert.equal(report.summary.auditFindingCount, 4); + assert.equal(report.artifacts[0].findingCount, 4); +}); + +test("execution results handle empty result directories", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-")); + const resultsDir = path.join(rootDir, ".plugin-inspector", "missing-results"); + + const report = await buildExecutionResultsReport({ rootDir, resultsDir }); + + assert.equal(report.summary.artifactCount, 0); + assert.equal(report.summary.passCount, 0); + assert.deepEqual(report.artifacts, []); + assert.match(renderExecutionResultsMarkdown(report), /## Artifacts\n\n_none_/); +}); + +test("execution results recurse, sort artifacts, and count alternate audit shapes", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-")); + const resultsDir = path.join(rootDir, ".plugin-inspector", "results"); + await mkdir(path.join(resultsDir, "z-fixture"), { recursive: true }); + await mkdir(path.join(resultsDir, "a-fixture", "nested"), { recursive: true }); + await writeFile( + path.join(resultsDir, "z-fixture", "package-audit.json"), + JSON.stringify({ vulnerabilities: [{ id: "one" }, { id: "two" }] }), + "utf8", + ); + await writeFile( + path.join(resultsDir, "a-fixture", "nested", "package-audit.json"), + JSON.stringify({ vulnerabilities: { low: {}, high: {}, advisory: {} } }), + "utf8", + ); + + const report = await buildExecutionResultsReport({ rootDir }); + + assert.equal(report.summary.artifactCount, 2); + assert.equal(report.summary.auditFindingCount, 5); + assert.deepEqual( + report.artifacts.map((artifact) => artifact.fixture), + ["nested", "z-fixture"], + ); + assert.deepEqual( + report.artifacts.map((artifact) => artifact.findingCount), + [3, 2], + ); +}); + +test("execution results writer preserves failures and repo-relative entrypoint paths", async () => { + const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-results-root-")); + const resultsDir = path.join(rootDir, ".plugin-inspector", "results"); + const fixtureDir = path.join(resultsDir, "broken"); + const outputDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-execution-report-")); + const jsonPath = path.join(outputDir, "execution.json"); + const markdownPath = path.join(outputDir, "execution.md"); + await mkdir(fixtureDir, { recursive: true }); + await writeFile( + path.join(fixtureDir, "entry.synthetic.json"), + JSON.stringify({ + entrypoint: path.join(rootDir, "plugins/broken/index.js"), + status: "captured", + summary: { probeCount: 1, passCount: 0, failCount: 1, blockedCount: 0 }, + results: [ + { + kind: "registration", + seam: "registerTool", + label: "registerTool.execute", + status: "fail", + error: "boom", + }, + ], + }), + "utf8", + ); + + const report = await buildExecutionResultsReport({ rootDir }); + + assert.equal(report.artifacts[0].entrypoint, "plugins/broken/index.js"); + assert.match(renderExecutionResultsMarkdown(report), /registerTool\.execute/); + assert.deepEqual(await writeExecutionResultsReport(report, { jsonPath, markdownPath }), { jsonPath, markdownPath }); + assert.equal(JSON.parse(await readFile(jsonPath, "utf8")).summary.failCount, 1); + assert.match(await readFile(markdownPath, "utf8"), /boom/); +});