2144 lines
74 KiB
JavaScript
2144 lines
74 KiB
JavaScript
import { createHash } from "node:crypto";
|
|
import { execFileSync } from "node:child_process";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
import { mkdir, readFile, readdir, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { inspectManifest } from "./inspect-fixtures.mjs";
|
|
import { fixtureCheckoutPath, fixtureSourceRoot, readManifest, repoRoot } from "./manifest-lib.mjs";
|
|
|
|
export const defaultReportDir = path.join(repoRoot, "reports");
|
|
export const defaultMarkdownReportPath = path.join(defaultReportDir, "crabpot-report.md");
|
|
export const defaultJsonReportPath = path.join(defaultReportDir, "crabpot-report.json");
|
|
export const defaultIssuesReportPath = path.join(defaultReportDir, "crabpot-issues.md");
|
|
|
|
const CONVERSATION_ACCESS_HOOKS = new Set(["agent_end", "llm_input", "llm_output"]);
|
|
const DEPRECATED_COMPAT_RECORDS = new Set([
|
|
"channel-env-vars",
|
|
"legacy-before-agent-start",
|
|
"legacy-root-sdk-import",
|
|
"provider-auth-env-vars",
|
|
]);
|
|
const CAPTURE_GAP_REGISTRATIONS = new Set([
|
|
"registerChannel",
|
|
"registerCommand",
|
|
"registerGatewayMethod",
|
|
"registerHttpRoute",
|
|
"registerInteractiveHandler",
|
|
"registerService",
|
|
]);
|
|
const CHANNEL_REGISTRATIONS = new Set([
|
|
"createChatChannelPlugin",
|
|
"defineChannelPluginEntry",
|
|
"registerChannel",
|
|
]);
|
|
const FALLBACK_OPENCLAW_CHECKOUT_PATHS = ["./openclaw", "../openclaw"];
|
|
let submoduleLinkTargets;
|
|
export const KNOWN_ISSUE_CODES = new Set([
|
|
"before-tool-call-probe",
|
|
"channel-contract-probe",
|
|
"channel-env-vars",
|
|
"conversation-access-hook",
|
|
"legacy-before-agent-start",
|
|
"legacy-root-sdk-import",
|
|
"manifest-unknown-contracts",
|
|
"manifest-unknown-fields",
|
|
"missing-expected-seam",
|
|
"missing-compat-record",
|
|
"unknown-hook-name",
|
|
"unknown-registration-name",
|
|
"package-build-artifact-entrypoint",
|
|
"package-dependency-install-required",
|
|
"package-entrypoint-missing",
|
|
"package-json-missing",
|
|
"package-manifest-version-drift",
|
|
"package-openclaw-entry-missing",
|
|
"package-openclaw-metadata-missing",
|
|
"package-plugin-api-compat-missing",
|
|
"package-typescript-source-entrypoint",
|
|
"provider-auth-env-vars",
|
|
"registration-capture-gap",
|
|
"runtime-tool-capture",
|
|
"sdk-export-missing",
|
|
]);
|
|
|
|
export async function buildReport(options = {}) {
|
|
const generatedAt = options.generatedAt ?? process.env.CRABPOT_REPORT_GENERATED_AT ?? "deterministic";
|
|
const manifest = await readManifest();
|
|
const targetOpenClaw = await readTargetOpenClaw(manifest, options.openclawPath);
|
|
const { inspections, failures } = await inspectManifest();
|
|
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
|
|
|
|
const fixtures = [];
|
|
const breakages = [];
|
|
const warnings = [];
|
|
const suggestions = [];
|
|
const logs = [];
|
|
const decisions = [];
|
|
|
|
for (const fixture of manifest.fixtures) {
|
|
const inspection = inspectionById.get(fixture.id) ?? emptyInspection(fixture);
|
|
const fixtureReport = await buildFixtureReport(fixture, inspection);
|
|
fixtures.push(fixtureReport);
|
|
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "seam-inventory",
|
|
level: "log",
|
|
message: `observed ${inspection.hooks.length} hooks, ${inspection.registrations.length} registrations, and ${inspection.manifestContracts.length} manifest contracts`,
|
|
evidence: [
|
|
...inspection.hooks.map((hook) => `hook:${hook}`),
|
|
...inspection.registrations.map((registration) => `registration:${registration}`),
|
|
...inspection.manifestContracts.map((contract) => `manifestContract:${contract}`),
|
|
],
|
|
});
|
|
|
|
classifyFixture({
|
|
fixture,
|
|
inspection,
|
|
fixtureReport,
|
|
targetOpenClaw,
|
|
breakages,
|
|
warnings,
|
|
suggestions,
|
|
logs,
|
|
decisions,
|
|
});
|
|
}
|
|
|
|
for (const failure of failures) {
|
|
const fixture = failure.split(":")[0] || "unknown";
|
|
breakages.push({
|
|
fixture,
|
|
code: "missing-expected-seam",
|
|
level: "breakage",
|
|
message: failure,
|
|
evidence: [failure],
|
|
});
|
|
decisions.push({
|
|
fixture,
|
|
decision: "inspector-follow-up",
|
|
seam: "expected-seam",
|
|
action: "Investigate whether OpenClaw removed a plugin-facing contract or the fixture pin changed upstream behavior.",
|
|
evidence: failure,
|
|
});
|
|
}
|
|
|
|
classifyCompatRecordCoverage({
|
|
targetOpenClaw,
|
|
findings: [...warnings, ...suggestions],
|
|
suggestions,
|
|
logs,
|
|
decisions,
|
|
});
|
|
|
|
const issues = buildIssues({ breakages, warnings, suggestions, targetOpenClaw });
|
|
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures });
|
|
const issueSummary = summarizeIssueClasses(issues);
|
|
|
|
const report = {
|
|
generatedAt,
|
|
targetOpenClaw,
|
|
status: breakages.length === 0 ? "pass" : "fail",
|
|
summary: {
|
|
fixtureCount: fixtures.length,
|
|
highPriorityFixtures: fixtures.filter((fixture) => fixture.priority === "high").length,
|
|
breakageCount: breakages.length,
|
|
warningCount: warnings.length,
|
|
suggestionCount: suggestions.length,
|
|
decisionCount: decisions.length,
|
|
issueCount: issues.length,
|
|
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
|
|
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
|
|
liveIssueCount: issueSummary["live-issue"],
|
|
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
|
|
compatGapCount: issueSummary["compat-gap"],
|
|
deprecationWarningCount: issueSummary["deprecation-warning"],
|
|
inspectorGapCount: issueSummary["inspector-gap"],
|
|
upstreamIssueCount: issueSummary["upstream-metadata"],
|
|
fixtureRegressionCount: issueSummary["fixture-regression"],
|
|
contractProbeCount: contractProbes.length,
|
|
},
|
|
fixtures,
|
|
breakages,
|
|
warnings,
|
|
suggestions,
|
|
issues,
|
|
contractProbes,
|
|
logs,
|
|
decisions,
|
|
};
|
|
|
|
return report;
|
|
}
|
|
|
|
export async function writeReport(report, options = {}) {
|
|
const markdownPath = options.markdownPath ?? defaultMarkdownReportPath;
|
|
const jsonPath = options.jsonPath ?? defaultJsonReportPath;
|
|
const issuesPath = options.issuesPath ?? defaultIssuesReportPath;
|
|
await mkdir(path.dirname(markdownPath), { recursive: true });
|
|
await mkdir(path.dirname(jsonPath), { recursive: true });
|
|
await mkdir(path.dirname(issuesPath), { recursive: true });
|
|
await writeFile(markdownPath, `${renderMarkdownReport(report)}\n`, "utf8");
|
|
await writeFile(jsonPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
await writeFile(issuesPath, `${renderIssuesReport(report)}\n`, "utf8");
|
|
return { markdownPath, jsonPath, issuesPath };
|
|
}
|
|
|
|
export function renderMarkdownReport(report) {
|
|
return [
|
|
"# Crabpot Compatibility Report",
|
|
"",
|
|
`Generated: ${report.generatedAt}`,
|
|
`Status: ${report.status.toUpperCase()}`,
|
|
"",
|
|
"## Summary",
|
|
"",
|
|
markdownTable(
|
|
[
|
|
["Fixtures", report.summary.fixtureCount],
|
|
["High-priority fixtures", report.summary.highPriorityFixtures],
|
|
["Hard breakages", report.summary.breakageCount],
|
|
["Warnings", report.summary.warningCount],
|
|
["Compatibility suggestions", report.summary.suggestionCount],
|
|
["Issue findings", report.summary.issueCount],
|
|
["P0 issues", report.summary.p0IssueCount],
|
|
["P1 issues", report.summary.p1IssueCount],
|
|
["Live issues", report.summary.liveIssueCount],
|
|
["Live P0 issues", report.summary.liveP0IssueCount],
|
|
["Compat gaps", report.summary.compatGapCount],
|
|
["Deprecation warnings", report.summary.deprecationWarningCount],
|
|
["Inspector gaps", report.summary.inspectorGapCount],
|
|
["Upstream metadata", report.summary.upstreamIssueCount],
|
|
["Contract probes", report.summary.contractProbeCount],
|
|
["Decision rows", report.summary.decisionCount],
|
|
],
|
|
["Metric", "Value"],
|
|
),
|
|
"",
|
|
"## Triage Overview",
|
|
"",
|
|
triageOverview(report),
|
|
"",
|
|
"## P0 Live Issues",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0")),
|
|
"",
|
|
"## Live Issues",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue")),
|
|
"",
|
|
"## Compat Gaps",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "compat-gap")),
|
|
"",
|
|
"## Deprecation Warnings",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "deprecation-warning")),
|
|
"",
|
|
"## Inspector Proof Gaps",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap")),
|
|
"",
|
|
"## Upstream Metadata Issues",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "upstream-metadata")),
|
|
"",
|
|
"## Hard Breakages",
|
|
"",
|
|
findingsTable(report.breakages),
|
|
"",
|
|
"## Target OpenClaw Compat Records",
|
|
"",
|
|
targetOpenClawTable(report.targetOpenClaw),
|
|
"",
|
|
"## Warnings",
|
|
"",
|
|
findingsTable(report.warnings),
|
|
"",
|
|
"## Suggestions To OpenClaw Compat Layer",
|
|
"",
|
|
findingsTable(report.suggestions),
|
|
"",
|
|
"## Issue Findings",
|
|
"",
|
|
issuesTable(report.issues),
|
|
"",
|
|
"## Contract Probe Backlog",
|
|
"",
|
|
contractProbesTable(report.contractProbes),
|
|
"",
|
|
"## Fixture Seam Inventory",
|
|
"",
|
|
markdownTable(
|
|
report.fixtures.map((fixture) => [
|
|
fixture.id,
|
|
fixture.priority,
|
|
fixture.seams.join(", "),
|
|
fixture.hooks.join(", ") || "-",
|
|
fixture.registrations.join(", ") || "-",
|
|
fixture.manifestContracts.join(", ") || "-",
|
|
]),
|
|
["Fixture", "Priority", "Seams", "Hooks", "Registrations", "Manifest contracts"],
|
|
),
|
|
"",
|
|
"## Decision Matrix",
|
|
"",
|
|
markdownTable(
|
|
report.decisions.map((decision) => [
|
|
decision.fixture,
|
|
decision.decision,
|
|
decision.seam,
|
|
decision.action,
|
|
decision.evidence,
|
|
]),
|
|
["Fixture", "Decision", "Seam", "Action", "Evidence"],
|
|
),
|
|
"",
|
|
"## Raw Logs",
|
|
"",
|
|
findingsTable(report.logs),
|
|
].join("\n");
|
|
}
|
|
|
|
export function renderIssuesReport(report) {
|
|
return [
|
|
"# Crabpot Issue Findings",
|
|
"",
|
|
`Generated: ${report.generatedAt}`,
|
|
`Status: ${report.status.toUpperCase()}`,
|
|
"",
|
|
"## Triage Summary",
|
|
"",
|
|
markdownTable(
|
|
[
|
|
["Issue findings", report.summary.issueCount],
|
|
[severityLabel("P0"), report.summary.p0IssueCount],
|
|
[severityLabel("P1"), report.summary.p1IssueCount],
|
|
["Live issues", report.summary.liveIssueCount],
|
|
["Live P0 issues", report.summary.liveP0IssueCount],
|
|
["Compat gaps", report.summary.compatGapCount],
|
|
["Deprecation warnings", report.summary.deprecationWarningCount],
|
|
["Inspector gaps", report.summary.inspectorGapCount],
|
|
["Upstream metadata", report.summary.upstreamIssueCount],
|
|
["Contract probes", report.summary.contractProbeCount],
|
|
],
|
|
["Metric", "Value"],
|
|
),
|
|
"",
|
|
"## Triage Overview",
|
|
"",
|
|
triageOverview(report),
|
|
"",
|
|
"## P0 Live Issues",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0")),
|
|
"",
|
|
"## Live Issues",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue")),
|
|
"",
|
|
"## Compat Gaps",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "compat-gap")),
|
|
"",
|
|
"## Deprecation Warnings",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "deprecation-warning")),
|
|
"",
|
|
"## Inspector Proof Gaps",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap")),
|
|
"",
|
|
"## Upstream Metadata Issues",
|
|
"",
|
|
issuesTable(report.issues.filter((issue) => issue.issueClass === "upstream-metadata")),
|
|
"",
|
|
"## Issues",
|
|
"",
|
|
issuesTable(report.issues),
|
|
"",
|
|
"## Contract Probe Backlog",
|
|
"",
|
|
contractProbesTable(report.contractProbes),
|
|
].join("\n");
|
|
}
|
|
|
|
function buildIssues({ breakages, warnings, suggestions, targetOpenClaw }) {
|
|
const findings = [
|
|
...breakages.map((finding) => issueMetadata({ ...finding, severity: "P0" }, targetOpenClaw)),
|
|
...warnings.map((finding) => issueMetadata(finding, targetOpenClaw)),
|
|
...suggestions.map((finding) => issueMetadata(finding, targetOpenClaw)),
|
|
];
|
|
|
|
return findings
|
|
.filter((finding) => finding.severity)
|
|
.sort(issueSort)
|
|
.map((finding) => ({
|
|
id: issueId(finding),
|
|
fixture: finding.fixture,
|
|
severity: finding.severity,
|
|
owner: finding.owner,
|
|
code: finding.code,
|
|
decision: finding.decision,
|
|
status: finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open",
|
|
issueClass: finding.issueClass,
|
|
live: finding.live,
|
|
deprecated: finding.deprecated,
|
|
compatStatus: finding.compatStatus,
|
|
title: issueTitle(finding),
|
|
evidence: finding.evidence ?? [],
|
|
compatRecord: finding.compatRecord ?? null,
|
|
}));
|
|
}
|
|
|
|
export function issueId(finding) {
|
|
const stableKey = [
|
|
finding.fixture,
|
|
finding.code,
|
|
finding.severity,
|
|
finding.compatRecord ?? "",
|
|
...(finding.evidence ?? []),
|
|
].join("\n");
|
|
return `CRABPOT-${createHash("sha256").update(stableKey).digest("hex").slice(0, 8).toUpperCase()}`;
|
|
}
|
|
|
|
function issueMetadata(finding, targetOpenClaw) {
|
|
const metadataByCode = {
|
|
"before-tool-call-probe": {
|
|
severity: "P1",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "before_tool_call needs terminal/block/approval probes",
|
|
},
|
|
"channel-contract-probe": {
|
|
severity: "P2",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "channel runtime needs envelope/config probes",
|
|
},
|
|
"channel-env-vars": {
|
|
severity: "P2",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "channelEnvVars legacy manifest metadata must stay covered",
|
|
},
|
|
"conversation-access-hook": {
|
|
severity: "P1",
|
|
owner: "core",
|
|
decision: "inspector-follow-up",
|
|
title: "conversation-access hooks need privacy-boundary probes",
|
|
},
|
|
"legacy-before-agent-start": {
|
|
severity: "P2",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "legacy before_agent_start hook compatibility is still used",
|
|
},
|
|
"legacy-root-sdk-import": {
|
|
severity: "P2",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "root plugin SDK barrel is still used by fixtures",
|
|
},
|
|
"sdk-export-missing": {
|
|
severity: "P1",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "plugin SDK import aliases are missing from target package exports",
|
|
},
|
|
"missing-compat-record": {
|
|
severity: "P1",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "compat-dependent behavior lacks registry coverage",
|
|
},
|
|
"missing-expected-seam": {
|
|
severity: "P0",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "fixture no longer exposes an expected seam",
|
|
},
|
|
"manifest-unknown-contracts": {
|
|
severity: "P1",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "manifest declares unsupported contract keys",
|
|
},
|
|
"manifest-unknown-fields": {
|
|
severity: "P2",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "manifest uses unsupported top-level fields",
|
|
},
|
|
"package-build-artifact-entrypoint": {
|
|
severity: "P2",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "cold import requires package build output",
|
|
},
|
|
"package-dependency-install-required": {
|
|
severity: "P2",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "cold import requires isolated dependency installation",
|
|
},
|
|
"package-entrypoint-missing": {
|
|
severity: "P1",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "OpenClaw package entrypoint is missing",
|
|
},
|
|
"package-json-missing": {
|
|
severity: "P2",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "package metadata is missing",
|
|
},
|
|
"package-manifest-version-drift": {
|
|
severity: "P2",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "package and manifest versions drift",
|
|
},
|
|
"package-openclaw-entry-missing": {
|
|
severity: "P2",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "OpenClaw package entrypoint metadata is missing",
|
|
},
|
|
"package-openclaw-metadata-missing": {
|
|
severity: "P2",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "OpenClaw package metadata is missing",
|
|
},
|
|
"package-plugin-api-compat-missing": {
|
|
severity: "P2",
|
|
owner: "plugin",
|
|
decision: "plugin-upstream-fix",
|
|
title: "plugin API compatibility range is missing",
|
|
},
|
|
"package-typescript-source-entrypoint": {
|
|
severity: "P2",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "cold import needs TypeScript source entrypoint support",
|
|
},
|
|
"provider-auth-env-vars": {
|
|
severity: "P2",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "providerAuthEnvVars legacy manifest metadata must stay covered",
|
|
},
|
|
"registration-capture-gap": {
|
|
severity: "P1",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "runtime registrations need capture before contract judgment",
|
|
},
|
|
"runtime-tool-capture": {
|
|
severity: "P2",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: "runtime tool schema needs registration capture",
|
|
},
|
|
"unknown-hook-name": {
|
|
severity: "P0",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "fixture uses a hook missing from target OpenClaw",
|
|
},
|
|
"unknown-registration-name": {
|
|
severity: "P0",
|
|
owner: "core",
|
|
decision: "core-compat-adapter",
|
|
title: "fixture calls a registrar missing from target OpenClaw",
|
|
},
|
|
};
|
|
return {
|
|
...finding,
|
|
...(metadataByCode[finding.code] ?? {
|
|
severity: "P3",
|
|
owner: "inspector",
|
|
decision: "inspector-follow-up",
|
|
title: finding.message,
|
|
}),
|
|
...classifyIssue(finding, metadataByCode[finding.code], targetOpenClaw),
|
|
};
|
|
}
|
|
|
|
export function classifyIssueFinding(finding, targetOpenClaw, metadata = {}) {
|
|
const compatStatus = compatStatusFor(finding, targetOpenClaw);
|
|
const deprecated = compatStatus === "deprecated";
|
|
const code = finding.code;
|
|
const issueClass = issueClassFor(code, { deprecated, compatRecord: finding.compatRecord });
|
|
const live = issueClass === "live-issue" || finding.level === "breakage";
|
|
const severity = severityForClass(code, metadata?.severity, {
|
|
issueClass,
|
|
compatRecord: finding.compatRecord,
|
|
compatStatus,
|
|
});
|
|
|
|
return {
|
|
compatStatus,
|
|
deprecated,
|
|
issueClass,
|
|
live,
|
|
severity,
|
|
};
|
|
}
|
|
|
|
function classifyIssue(finding, metadata, targetOpenClaw) {
|
|
return classifyIssueFinding(finding, targetOpenClaw, metadata);
|
|
}
|
|
|
|
function issueClassFor(code, options) {
|
|
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)) {
|
|
return "live-issue";
|
|
}
|
|
if (code === "missing-compat-record") {
|
|
return "compat-gap";
|
|
}
|
|
if (options.deprecated || ["channel-env-vars", "legacy-before-agent-start", "legacy-root-sdk-import", "provider-auth-env-vars"].includes(code)) {
|
|
return "deprecation-warning";
|
|
}
|
|
if (
|
|
[
|
|
"before-tool-call-probe",
|
|
"channel-contract-probe",
|
|
"conversation-access-hook",
|
|
"package-build-artifact-entrypoint",
|
|
"package-dependency-install-required",
|
|
"package-typescript-source-entrypoint",
|
|
"registration-capture-gap",
|
|
"runtime-tool-capture",
|
|
].includes(code)
|
|
) {
|
|
return "inspector-gap";
|
|
}
|
|
if (
|
|
[
|
|
"manifest-unknown-contracts",
|
|
"manifest-unknown-fields",
|
|
"package-json-missing",
|
|
"package-manifest-version-drift",
|
|
"package-openclaw-entry-missing",
|
|
"package-openclaw-metadata-missing",
|
|
"package-plugin-api-compat-missing",
|
|
].includes(code)
|
|
) {
|
|
return "upstream-metadata";
|
|
}
|
|
if (code === "missing-expected-seam") {
|
|
return "fixture-regression";
|
|
}
|
|
return options.compatRecord ? "compat-gap" : "inspector-gap";
|
|
}
|
|
|
|
function severityForClass(code, defaultSeverity, options) {
|
|
if (
|
|
options.issueClass === "live-issue" &&
|
|
["none", "untracked"].includes(options.compatStatus) &&
|
|
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)
|
|
) {
|
|
return "P0";
|
|
}
|
|
return defaultSeverity ?? "P3";
|
|
}
|
|
|
|
function compatStatusFor(finding, targetOpenClaw) {
|
|
if (finding.code === "missing-compat-record") {
|
|
return "missing";
|
|
}
|
|
if (!finding.compatRecord) {
|
|
return "none";
|
|
}
|
|
const targetStatus = targetOpenClaw?.compatRecordStatuses?.[finding.compatRecord];
|
|
if (targetStatus) {
|
|
return targetStatus;
|
|
}
|
|
if (DEPRECATED_COMPAT_RECORDS.has(finding.compatRecord)) {
|
|
return "deprecated";
|
|
}
|
|
return "untracked";
|
|
}
|
|
|
|
function summarizeIssueClasses(issues) {
|
|
const summary = {
|
|
"compat-gap": 0,
|
|
"deprecation-warning": 0,
|
|
"fixture-regression": 0,
|
|
"inspector-gap": 0,
|
|
"live-issue": 0,
|
|
"upstream-metadata": 0,
|
|
};
|
|
for (const issue of issues) {
|
|
summary[issue.issueClass] = (summary[issue.issueClass] ?? 0) + 1;
|
|
}
|
|
return summary;
|
|
}
|
|
|
|
function buildContractProbes({ warnings, suggestions, fixtures }) {
|
|
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
|
|
const probes = [];
|
|
const probeRules = {
|
|
"before-tool-call-probe": {
|
|
id: "hook.before_tool_call.terminal-block-approval",
|
|
contract: "Hook returns preserve terminal, block, and approval semantics.",
|
|
target: "hook-runner",
|
|
},
|
|
"channel-contract-probe": {
|
|
id: "channel.runtime.envelope-config-metadata",
|
|
contract: "Channel setup, message envelope, sender metadata, and config schema remain stable.",
|
|
target: "channel-runtime",
|
|
},
|
|
"conversation-access-hook": {
|
|
id: "hook.llm-observer.privacy-payload",
|
|
contract: "LLM observer hooks receive documented prompt/output fields with expected redaction behavior.",
|
|
target: "hook-runner",
|
|
},
|
|
"legacy-root-sdk-import": {
|
|
id: "sdk.import.root-barrel-cold-import",
|
|
contract: "Root plugin SDK barrel remains importable or has a machine-readable migration path.",
|
|
target: "sdk-alias",
|
|
},
|
|
"legacy-before-agent-start": {
|
|
id: "hook.compat.before-agent-start-migration",
|
|
contract: "Legacy before_agent_start remains wired until plugins migrate to before_model_resolve and before_prompt_build.",
|
|
target: "hook-runner",
|
|
},
|
|
"sdk-export-missing": {
|
|
id: "sdk.import.package-export-cold-import",
|
|
contract: "Every observed OpenClaw plugin SDK import remains exported by the target OpenClaw package.",
|
|
target: "sdk-alias",
|
|
},
|
|
"provider-auth-env-vars": {
|
|
id: "manifest.compat.provider-auth-env-vars",
|
|
contract: "Legacy provider auth env metadata continues to map into config/help surfaces.",
|
|
target: "manifest-loader",
|
|
},
|
|
"channel-env-vars": {
|
|
id: "manifest.compat.channel-env-vars",
|
|
contract: "Legacy channel env metadata continues to map into channel setup/help surfaces.",
|
|
target: "manifest-loader",
|
|
},
|
|
"manifest-unknown-contracts": {
|
|
id: "manifest.schema.contract-keys",
|
|
contract: "Manifest contract keys are represented in target OpenClaw PluginManifestContracts.",
|
|
target: "manifest-loader",
|
|
},
|
|
"manifest-unknown-fields": {
|
|
id: "manifest.schema.top-level-fields",
|
|
contract: "Manifest top-level fields are represented in target OpenClaw PluginManifest.",
|
|
target: "manifest-loader",
|
|
},
|
|
"registration-capture-gap": {
|
|
id: "api.capture.runtime-registrars",
|
|
contract: "External inspector capture records service, route, gateway, command, and interactive registrations.",
|
|
target: "inspector-capture-api",
|
|
},
|
|
"package-build-artifact-entrypoint": {
|
|
id: "package.entrypoint.build-before-cold-import",
|
|
contract: "Inspector can build or resolve source aliases before cold importing package entrypoints.",
|
|
target: "package-loader",
|
|
},
|
|
"package-dependency-install-required": {
|
|
id: "package.entrypoint.isolated-dependency-install",
|
|
contract: "Inspector installs package dependencies in an isolated workspace before cold import.",
|
|
target: "package-loader",
|
|
},
|
|
"package-entrypoint-missing": {
|
|
id: "package.entrypoint.exists",
|
|
contract: "OpenClaw package entrypoints resolve to files in the published or built plugin package.",
|
|
target: "package-loader",
|
|
},
|
|
"package-openclaw-entry-missing": {
|
|
id: "package.entrypoint.openclaw-metadata",
|
|
contract: "OpenClaw package metadata declares entrypoints for cold import and registration capture.",
|
|
target: "package-loader",
|
|
},
|
|
"package-openclaw-metadata-missing": {
|
|
id: "package.metadata.openclaw",
|
|
contract: "Plugins that register OpenClaw APIs declare OpenClaw install and entrypoint metadata.",
|
|
target: "package-loader",
|
|
},
|
|
"package-manifest-version-drift": {
|
|
id: "package.metadata.version-alignment",
|
|
contract: "Package and OpenClaw manifest versions stay aligned for release compatibility reporting.",
|
|
target: "package-loader",
|
|
},
|
|
"package-plugin-api-compat-missing": {
|
|
id: "package.compat.plugin-api-range",
|
|
contract: "Package metadata declares the OpenClaw plugin API range used by the plugin.",
|
|
target: "package-loader",
|
|
},
|
|
"package-typescript-source-entrypoint": {
|
|
id: "package.entrypoint.typescript-loader",
|
|
contract: "Inspector can compile or load TypeScript source entrypoints before registration capture.",
|
|
target: "package-loader",
|
|
},
|
|
"runtime-tool-capture": {
|
|
id: "tool.registration.schema-capture",
|
|
contract: "Registered runtime tools expose stable names, input schemas, and result metadata.",
|
|
target: "tool-runtime",
|
|
},
|
|
};
|
|
|
|
for (const finding of [...warnings, ...suggestions]) {
|
|
const rule = probeRules[finding.code];
|
|
if (!rule) {
|
|
continue;
|
|
}
|
|
probes.push({
|
|
id: `${rule.id}:${finding.fixture}`,
|
|
fixture: finding.fixture,
|
|
priority: probePriority(finding.code, fixtureById.get(finding.fixture)?.priority),
|
|
target: rule.target,
|
|
contract: rule.contract,
|
|
evidence: finding.evidence ?? [],
|
|
});
|
|
}
|
|
|
|
return dedupeBy(probes, (probe) => probe.id).sort(
|
|
(left, right) => priorityRank(left.priority) - priorityRank(right.priority) || left.id.localeCompare(right.id),
|
|
);
|
|
}
|
|
|
|
function classifyCompatRecordCoverage({ targetOpenClaw, findings, suggestions, logs, decisions }) {
|
|
if (targetOpenClaw.status !== "ok") {
|
|
logs.push({
|
|
fixture: "openclaw",
|
|
code: "target-openclaw-unavailable",
|
|
level: "log",
|
|
message: "target OpenClaw checkout was not available, so compat record coverage was not checked",
|
|
evidence: [targetOpenClaw.configuredPath ?? "not configured"],
|
|
});
|
|
return;
|
|
}
|
|
|
|
const knownRecords = new Set(targetOpenClaw.compatRecords);
|
|
for (const finding of findings.filter((item) => item.compatRecord)) {
|
|
if (knownRecords.has(finding.compatRecord)) {
|
|
logs.push({
|
|
fixture: finding.fixture,
|
|
code: "compat-record-present",
|
|
level: "log",
|
|
message: "target OpenClaw checkout has a matching compat registry record",
|
|
evidence: [finding.compatRecord, `status:${targetOpenClaw.compatRecordStatuses?.[finding.compatRecord] ?? "unknown"}`],
|
|
compatRecord: finding.compatRecord,
|
|
});
|
|
continue;
|
|
}
|
|
|
|
suggestions.push({
|
|
fixture: finding.fixture,
|
|
code: "missing-compat-record",
|
|
level: "suggestion",
|
|
message: "fixture depends on a compatibility behavior that is not represented in the target compat registry",
|
|
evidence: [finding.compatRecord],
|
|
compatRecord: finding.compatRecord,
|
|
});
|
|
decisions.push({
|
|
fixture: finding.fixture,
|
|
decision: "core-compat-adapter",
|
|
seam: "compat-registry",
|
|
action: "Add or restore a machine-readable OpenClaw compat record before changing this plugin-facing behavior.",
|
|
evidence: finding.compatRecord,
|
|
});
|
|
}
|
|
}
|
|
|
|
function classifyFixture({ fixture, inspection, fixtureReport, targetOpenClaw, breakages, warnings, suggestions, logs, decisions }) {
|
|
if (inspection.status !== "ok") {
|
|
return;
|
|
}
|
|
|
|
classifyHookNameCoverage({ fixture, inspection, targetOpenClaw, warnings, logs });
|
|
classifyRegistrationNameCoverage({ fixture, inspection, targetOpenClaw, warnings, logs });
|
|
classifySdkImportCoverage({ fixture, fixtureReport, targetOpenClaw, warnings, logs, decisions });
|
|
classifyManifestFieldCoverage({ fixture, fixtureReport, targetOpenClaw, warnings, logs, decisions });
|
|
classifyPackageContracts({ fixture, inspection, fixtureReport, warnings, suggestions, logs, decisions });
|
|
|
|
for (const pluginManifest of fixtureReport.pluginManifests) {
|
|
const providerAuthKeys = Object.keys(pluginManifest.providerAuthEnvVars ?? {});
|
|
if (providerAuthKeys.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "provider-auth-env-vars",
|
|
level: "warning",
|
|
message: "manifest uses providerAuthEnvVars legacy compatibility metadata",
|
|
evidence: providerAuthKeys,
|
|
compatRecord: "provider-auth-env-vars",
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "core-compat-adapter",
|
|
seam: "env-auth",
|
|
action: "Keep providerAuthEnvVars compatibility active while the inspector recommends manifest-schema migration upstream.",
|
|
evidence: providerAuthKeys.join(", "),
|
|
});
|
|
}
|
|
|
|
const channelEnvKeys = Object.keys(pluginManifest.channelEnvVars ?? {});
|
|
if (channelEnvKeys.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "channel-env-vars",
|
|
level: "warning",
|
|
message: "manifest uses channelEnvVars legacy compatibility metadata",
|
|
evidence: channelEnvKeys,
|
|
compatRecord: "channel-env-vars",
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "core-compat-adapter",
|
|
seam: "channel-env",
|
|
action: "Keep channelEnvVars compatibility active until channel setup metadata has a stable replacement path.",
|
|
evidence: channelEnvKeys.join(", "),
|
|
});
|
|
}
|
|
}
|
|
|
|
const conversationHooks = inspection.hooks.filter((hook) => CONVERSATION_ACCESS_HOOKS.has(hook));
|
|
const conversationHookDetails = inspection.hookDetails.filter((hook) => CONVERSATION_ACCESS_HOOKS.has(hook.name));
|
|
if (conversationHooks.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "conversation-access-hook",
|
|
level: "warning",
|
|
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
|
|
evidence: detailEvidence(conversationHookDetails),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "conversation-access",
|
|
action: "Add synthetic llm_input/llm_output/agent_end probes before tightening hook payloads or redaction behavior.",
|
|
evidence: conversationHooks.join(", "),
|
|
});
|
|
}
|
|
|
|
const rootSdkImports = fixtureReport.sdkImports.filter((specifier) => specifier === "openclaw/plugin-sdk");
|
|
const rootSdkImportDetails = fixtureReport.sdkImportDetails.filter(
|
|
(sdkImport) => sdkImport.specifier === "openclaw/plugin-sdk",
|
|
);
|
|
if (rootSdkImports.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "legacy-root-sdk-import",
|
|
level: "warning",
|
|
message: "fixture imports the root plugin SDK barrel",
|
|
evidence: detailEvidence(rootSdkImportDetails, "specifier"),
|
|
compatRecord: "legacy-root-sdk-import",
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "core-compat-adapter",
|
|
seam: "sdk-import",
|
|
action: "Keep the root SDK barrel stable or expose a machine-readable migration map before removing aliases.",
|
|
evidence: unique(rootSdkImports).join(", "),
|
|
});
|
|
}
|
|
|
|
const legacyBeforeAgentStartDetails = inspection.hookDetails.filter((hook) => hook.name === "before_agent_start");
|
|
if (legacyBeforeAgentStartDetails.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "legacy-before-agent-start",
|
|
level: "warning",
|
|
message: "fixture uses deprecated before_agent_start hook compatibility",
|
|
evidence: detailEvidence(legacyBeforeAgentStartDetails),
|
|
compatRecord: "legacy-before-agent-start",
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "core-compat-adapter",
|
|
seam: "hook-compat",
|
|
action: "Keep before_agent_start wired while plugin authors migrate to before_model_resolve and before_prompt_build.",
|
|
evidence: detailEvidence(legacyBeforeAgentStartDetails).join(", "),
|
|
});
|
|
}
|
|
|
|
const captureGapRegistrationDetails = registrationCaptureGapDetails(inspection, targetOpenClaw);
|
|
const captureGapRegistrations = unique(captureGapRegistrationDetails.map((registration) => registration.name));
|
|
if (captureGapRegistrations.length > 0) {
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "registration-capture-gap",
|
|
level: "suggestion",
|
|
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
|
|
evidence: detailEvidence(captureGapRegistrationDetails),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "registration-capture",
|
|
action: "Expose or mirror a full public API capture shim before treating these runtime-only seams as covered.",
|
|
evidence: captureGapRegistrations.join(", "),
|
|
});
|
|
}
|
|
|
|
if (inspection.hooks.includes("before_tool_call")) {
|
|
const hookDetails = inspection.hookDetails.filter((hook) => hook.name === "before_tool_call");
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "before-tool-call-probe",
|
|
level: "suggestion",
|
|
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
|
|
evidence: detailEvidence(hookDetails),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "tool-policy",
|
|
action: "Probe before_tool_call return shapes before changing tool-call approval or block behavior.",
|
|
evidence: "before_tool_call",
|
|
});
|
|
}
|
|
|
|
const channelRegistrations = inspection.registrations.filter((registration) =>
|
|
CHANNEL_REGISTRATIONS.has(registration),
|
|
);
|
|
const channelRegistrationDetails = inspection.registrationDetails.filter((registration) =>
|
|
CHANNEL_REGISTRATIONS.has(registration.name),
|
|
);
|
|
if (channelRegistrations.length > 0) {
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "channel-contract-probe",
|
|
level: "suggestion",
|
|
message: "add channel envelope, config-schema, and runtime metadata probes",
|
|
evidence: detailEvidence(channelRegistrationDetails),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "channel-runtime",
|
|
action: "Probe channel setup and message envelope contracts before changing channel runtime payloads.",
|
|
evidence: channelRegistrations.join(", "),
|
|
});
|
|
}
|
|
|
|
const runtimeToolOnly = inspection.registrations.includes("registerTool") && !inspection.manifestContracts.includes("tools");
|
|
if (runtimeToolOnly) {
|
|
const toolRegistrationDetails = inspection.registrationDetails.filter(
|
|
(registration) => registration.name === "registerTool",
|
|
);
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "runtime-tool-capture",
|
|
level: "suggestion",
|
|
message: "tool shape is only visible after runtime registration capture",
|
|
evidence: detailEvidence(toolRegistrationDetails),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "tool-schema",
|
|
action: "Capture registered tool schemas from plugin register() before judging tool compatibility.",
|
|
evidence: "registerTool without manifest contracts.tools",
|
|
});
|
|
}
|
|
|
|
if (inspection.manifestContracts.length > 0) {
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "declarative-contracts",
|
|
level: "log",
|
|
message: "fixture declares manifest contracts that can be checked without executing plugin code",
|
|
evidence: inspection.manifestContracts,
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "no-action",
|
|
seam: "manifest-contract",
|
|
action: "Keep checking this declarative contract in default offline CI.",
|
|
evidence: inspection.manifestContracts.join(", "),
|
|
});
|
|
}
|
|
}
|
|
|
|
function registrationCaptureGapDetails(inspection, targetOpenClaw) {
|
|
const apiRegistrationDetails = inspection.registrationDetails.filter((registration) =>
|
|
registration.name.startsWith("register"),
|
|
);
|
|
if (targetOpenClaw.status === "ok" && targetOpenClaw.capturedRegistrars.length > 0) {
|
|
const captured = new Set(targetOpenClaw.capturedRegistrars);
|
|
return apiRegistrationDetails.filter((registration) => !captured.has(registration.name));
|
|
}
|
|
return apiRegistrationDetails.filter((registration) => CAPTURE_GAP_REGISTRATIONS.has(registration.name));
|
|
}
|
|
|
|
function classifyHookNameCoverage({ fixture, inspection, targetOpenClaw, warnings, logs }) {
|
|
if (targetOpenClaw.status !== "ok" || targetOpenClaw.hookNames.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const knownHookNames = new Set(targetOpenClaw.hookNames);
|
|
const unknownHooks = inspection.hookDetails.filter((hook) => !knownHookNames.has(hook.name));
|
|
if (unknownHooks.length === 0) {
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "hook-names-present",
|
|
level: "log",
|
|
message: "all observed hooks exist in the target OpenClaw hook registry",
|
|
evidence: inspection.hooks,
|
|
});
|
|
return;
|
|
}
|
|
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "unknown-hook-name",
|
|
level: "warning",
|
|
message: "fixture registers hooks that are not present in the target OpenClaw hook registry",
|
|
evidence: detailEvidence(unknownHooks),
|
|
});
|
|
}
|
|
|
|
function classifyRegistrationNameCoverage({ fixture, inspection, targetOpenClaw, warnings, logs }) {
|
|
if (targetOpenClaw.status !== "ok" || targetOpenClaw.apiRegistrars.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const knownRegistrars = new Set(targetOpenClaw.apiRegistrars);
|
|
const apiRegistrations = inspection.registrationDetails.filter((registration) =>
|
|
registration.name.startsWith("register"),
|
|
);
|
|
const unknownRegistrations = apiRegistrations.filter((registration) => !knownRegistrars.has(registration.name));
|
|
if (unknownRegistrations.length === 0) {
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "api-registrars-present",
|
|
level: "log",
|
|
message: "all observed api.register* calls exist in the target OpenClaw plugin API builder",
|
|
evidence: unique(apiRegistrations.map((registration) => registration.name)).sort(),
|
|
});
|
|
return;
|
|
}
|
|
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "unknown-registration-name",
|
|
level: "warning",
|
|
message: "fixture calls api.register* methods that are not present in the target OpenClaw plugin API builder",
|
|
evidence: detailEvidence(unknownRegistrations),
|
|
});
|
|
}
|
|
|
|
function classifySdkImportCoverage({ fixture, fixtureReport, targetOpenClaw, warnings, logs, decisions }) {
|
|
if (targetOpenClaw.status !== "ok" || targetOpenClaw.sdkExports.length === 0 || fixtureReport.sdkImports.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const sdkExports = new Set(targetOpenClaw.sdkExports);
|
|
const unknownImports = fixtureReport.sdkImportDetails.filter((sdkImport) => !sdkExports.has(sdkImport.specifier));
|
|
if (unknownImports.length === 0) {
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "sdk-exports-present",
|
|
level: "log",
|
|
message: "all observed plugin SDK imports exist in target OpenClaw package exports",
|
|
evidence: fixtureReport.sdkImports,
|
|
});
|
|
return;
|
|
}
|
|
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "sdk-export-missing",
|
|
level: "warning",
|
|
message: "fixture imports plugin SDK aliases that are not exported by the target OpenClaw package",
|
|
evidence: detailEvidence(unknownImports, "specifier"),
|
|
compatRecord: "plugin-sdk-export-aliases",
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "core-compat-adapter",
|
|
seam: "sdk-alias",
|
|
action: "Restore the package export alias or publish a versioned migration map before cold-importing old plugins.",
|
|
evidence: unique(unknownImports.map((sdkImport) => sdkImport.specifier)).join(", "),
|
|
});
|
|
}
|
|
|
|
function classifyManifestFieldCoverage({ fixture, fixtureReport, targetOpenClaw, warnings, logs, decisions }) {
|
|
if (targetOpenClaw.status !== "ok" || targetOpenClaw.manifestFields.length === 0) {
|
|
return;
|
|
}
|
|
|
|
const manifestFields = new Set(targetOpenClaw.manifestFields);
|
|
const contractFields = new Set(targetOpenClaw.manifestContractFields);
|
|
for (const pluginManifest of fixtureReport.pluginManifests) {
|
|
const unknownFields = pluginManifest.keys.filter((key) => !manifestFields.has(key));
|
|
if (unknownFields.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "manifest-unknown-fields",
|
|
level: "warning",
|
|
message: "manifest uses top-level fields that are not present in the target OpenClaw PluginManifest type",
|
|
evidence: unknownFields.map((field) => `${field} @ ${pluginManifest.path}`),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "manifest-schema",
|
|
action: "Move unknown manifest metadata into supported package openclaw metadata or add a versioned OpenClaw manifest field.",
|
|
evidence: unknownFields.join(", "),
|
|
});
|
|
}
|
|
|
|
const unknownContractFields = pluginManifest.contracts.filter((field) => !contractFields.has(field));
|
|
if (unknownContractFields.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "manifest-unknown-contracts",
|
|
level: "warning",
|
|
message: "manifest declares contract keys that are not present in the target OpenClaw PluginManifestContracts type",
|
|
evidence: unknownContractFields.map((field) => `${field} @ ${pluginManifest.path}`),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "manifest-contract",
|
|
action: "Use a supported manifest contract key or add a versioned OpenClaw contract field.",
|
|
evidence: unknownContractFields.join(", "),
|
|
});
|
|
}
|
|
}
|
|
|
|
if (fixtureReport.pluginManifests.length > 0) {
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "manifest-fields-checked",
|
|
level: "log",
|
|
message: "plugin manifest fields were compared with target OpenClaw manifest types",
|
|
evidence: fixtureReport.pluginManifests.map((manifest) => manifest.path),
|
|
});
|
|
}
|
|
}
|
|
|
|
function classifyPackageContracts({ fixture, inspection, fixtureReport, warnings, suggestions, logs, decisions }) {
|
|
const packageSummary = fixtureReport.package;
|
|
if (!packageSummary) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "package-json-missing",
|
|
level: "warning",
|
|
message: "fixture has no package.json to describe install and plugin entrypoint metadata",
|
|
evidence: [fixture.path],
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "package-metadata",
|
|
action: "Ask the plugin to publish package metadata before treating install/cold-import checks as covered.",
|
|
evidence: fixture.path,
|
|
});
|
|
return;
|
|
}
|
|
|
|
logs.push({
|
|
fixture: fixture.id,
|
|
code: "package-metadata",
|
|
level: "log",
|
|
message: "selected package metadata for plugin contract checks",
|
|
evidence: [
|
|
packageSummary.path,
|
|
packageSummary.name ?? "unnamed",
|
|
packageSummary.version ? `version:${packageSummary.version}` : "version:missing",
|
|
],
|
|
});
|
|
|
|
const manifestVersions = fixtureReport.pluginManifests
|
|
.map((manifest) => manifest.version)
|
|
.filter((version) => typeof version === "string" && version.length > 0);
|
|
const mismatchedManifestVersions = manifestVersions.filter((version) => version !== packageSummary.version);
|
|
if (packageSummary.version && mismatchedManifestVersions.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "package-manifest-version-drift",
|
|
level: "warning",
|
|
message: "package.json and openclaw.plugin.json publish different versions",
|
|
evidence: [`package:${packageSummary.version}`, ...mismatchedManifestVersions.map((version) => `manifest:${version}`)],
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "package-metadata",
|
|
action: "Ask the plugin to keep package and manifest versions aligned before relying on release compatibility signals.",
|
|
evidence: `${packageSummary.version} != ${mismatchedManifestVersions.join(", ")}`,
|
|
});
|
|
}
|
|
|
|
if (packageSummary.openclaw && !packageSummary.openclaw.compatPluginApi) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "package-plugin-api-compat-missing",
|
|
level: "warning",
|
|
message: "package openclaw metadata does not declare compat.pluginApi",
|
|
evidence: [packageSummary.path],
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "package-metadata",
|
|
action: "Ask the plugin to declare the plugin API range it was built against.",
|
|
evidence: packageSummary.path,
|
|
});
|
|
}
|
|
|
|
if (packageSummary.openclaw && packageSummary.openclaw.entrypoints.length === 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "package-openclaw-entry-missing",
|
|
level: "warning",
|
|
message: "package openclaw metadata does not declare plugin entrypoints",
|
|
evidence: [packageSummary.path],
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "package-entrypoint",
|
|
action: "Ask the plugin to declare openclaw.extensions or runtimeExtensions so cold import can target the correct entrypoint.",
|
|
evidence: packageSummary.path,
|
|
});
|
|
}
|
|
|
|
const missingEntrypoints = packageSummary.openclaw?.entrypoints.filter((entrypoint) => !entrypoint.exists) ?? [];
|
|
const buildEntrypoints = missingEntrypoints.filter((entrypoint) => entrypoint.requiresBuild);
|
|
const plainMissingEntrypoints = missingEntrypoints.filter((entrypoint) => !entrypoint.requiresBuild);
|
|
|
|
if (buildEntrypoints.length > 0) {
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "package-build-artifact-entrypoint",
|
|
level: "suggestion",
|
|
message: "package OpenClaw entrypoint points at build output that is not present in the source fixture checkout",
|
|
evidence: buildEntrypoints.map((entrypoint) => `${entrypoint.kind}:${entrypoint.specifier} -> ${entrypoint.relativePath}`),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "cold-import",
|
|
action: "Run the plugin build or resolve source entrypoint aliases before cold-importing this fixture.",
|
|
evidence: buildEntrypoints.map((entrypoint) => entrypoint.specifier).join(", "),
|
|
});
|
|
}
|
|
|
|
if (plainMissingEntrypoints.length > 0) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "package-entrypoint-missing",
|
|
level: "warning",
|
|
message: "package OpenClaw entrypoint does not exist in the fixture checkout",
|
|
evidence: plainMissingEntrypoints.map((entrypoint) => `${entrypoint.kind}:${entrypoint.specifier} -> ${entrypoint.relativePath}`),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "package-entrypoint",
|
|
action: "Ask the plugin to publish a valid OpenClaw entrypoint or update package metadata.",
|
|
evidence: plainMissingEntrypoints.map((entrypoint) => entrypoint.specifier).join(", "),
|
|
});
|
|
}
|
|
|
|
const sourceEntrypoints = packageSummary.openclaw?.entrypoints.filter((entrypoint) => entrypoint.exists && entrypoint.relativePath.endsWith(".ts")) ?? [];
|
|
if (sourceEntrypoints.length > 0) {
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "package-typescript-source-entrypoint",
|
|
level: "suggestion",
|
|
message: "package OpenClaw entrypoint resolves to TypeScript source in this fixture checkout",
|
|
evidence: sourceEntrypoints.map((entrypoint) => `${entrypoint.kind}:${entrypoint.relativePath}`),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "cold-import",
|
|
action: "Compile TypeScript source or run a loader before cold-importing this fixture entrypoint.",
|
|
evidence: sourceEntrypoints.map((entrypoint) => entrypoint.relativePath).join(", "),
|
|
});
|
|
}
|
|
|
|
const runtimeDependencies = unique([
|
|
...packageSummary.dependencies,
|
|
...packageSummary.peerDependencies,
|
|
...packageSummary.optionalDependencies,
|
|
]);
|
|
if (packageSummary.openclaw?.entrypoints.length > 0 && runtimeDependencies.length > 0) {
|
|
suggestions.push({
|
|
fixture: fixture.id,
|
|
code: "package-dependency-install-required",
|
|
level: "suggestion",
|
|
message: "package declares runtime dependencies that must be installed before cold import",
|
|
evidence: runtimeDependencies.map((dependency) => `${dependency} @ ${packageSummary.path}`),
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "inspector-follow-up",
|
|
seam: "cold-import",
|
|
action: "Install runtime dependencies in an isolated workspace before executing this fixture entrypoint.",
|
|
evidence: runtimeDependencies.join(", "),
|
|
});
|
|
}
|
|
|
|
if (inspection.registrations.length > 0 && !packageSummary.openclaw) {
|
|
warnings.push({
|
|
fixture: fixture.id,
|
|
code: "package-openclaw-metadata-missing",
|
|
level: "warning",
|
|
message: "fixture registers plugin APIs but the selected package.json has no openclaw metadata",
|
|
evidence: [packageSummary.path, ...inspection.registrations],
|
|
});
|
|
decisions.push({
|
|
fixture: fixture.id,
|
|
decision: "plugin-upstream-fix",
|
|
seam: "package-metadata",
|
|
action: "Ask the plugin to declare OpenClaw install and entrypoint metadata in package.json.",
|
|
evidence: packageSummary.path,
|
|
});
|
|
}
|
|
}
|
|
|
|
async function buildFixtureReport(fixture, inspection) {
|
|
const checkoutPath = fixtureCheckoutPath(fixture);
|
|
const sourceRoot = fixtureSourceRoot(fixture);
|
|
|
|
const pluginManifests = await readPluginManifests(checkoutPath, sourceRoot);
|
|
const packageSummaries = await readPackageSummaries(checkoutPath, sourceRoot);
|
|
const packageJson = selectPrimaryPackage(packageSummaries);
|
|
const sdkImports = unique((inspection.sdkImports ?? []).map((sdkImport) => sdkImport.specifier));
|
|
|
|
return {
|
|
id: fixture.id,
|
|
name: fixture.name,
|
|
priority: fixture.priority,
|
|
seams: fixture.seams,
|
|
why: fixture.why,
|
|
status: inspection.status,
|
|
hooks: inspection.hooks,
|
|
hookDetails: inspection.hookDetails ?? [],
|
|
registrations: inspection.registrations,
|
|
registrationDetails: inspection.registrationDetails ?? [],
|
|
manifestContracts: inspection.manifestContracts,
|
|
manifestFiles: inspection.manifestFiles ?? [],
|
|
sourceFiles: inspection.sourceFiles ?? [],
|
|
pluginManifests,
|
|
package: packageJson,
|
|
packages: packageSummaries,
|
|
sdkImports,
|
|
sdkImportDetails: inspection.sdkImports ?? [],
|
|
};
|
|
}
|
|
|
|
async function readPluginManifests(checkoutPath, sourceRoot) {
|
|
const candidates = unique(
|
|
[path.join(sourceRoot, "openclaw.plugin.json"), path.join(checkoutPath, "openclaw.plugin.json")].filter(
|
|
existsSync,
|
|
),
|
|
);
|
|
const manifests = [];
|
|
|
|
for (const manifestPath of candidates) {
|
|
const raw = await readFile(manifestPath, "utf8");
|
|
const manifest = JSON.parse(raw);
|
|
manifests.push({
|
|
path: path.relative(repoRoot, manifestPath),
|
|
id: manifest.id ?? null,
|
|
name: manifest.name ?? null,
|
|
version: manifest.version ?? null,
|
|
keys: Object.keys(manifest).sort(),
|
|
contracts: Object.keys(manifest.contracts ?? {}).sort(),
|
|
providerAuthEnvVars: manifest.providerAuthEnvVars ?? {},
|
|
channelEnvVars: manifest.channelEnvVars ?? {},
|
|
activation: manifest.activation ?? null,
|
|
});
|
|
}
|
|
|
|
return manifests;
|
|
}
|
|
|
|
async function readTargetOpenClaw(manifest, configuredPath) {
|
|
if (configuredPath === false) {
|
|
return {
|
|
configuredPath: null,
|
|
status: "disabled",
|
|
compatRecords: [],
|
|
compatRecordStatuses: {},
|
|
hookNames: [],
|
|
apiRegistrars: [],
|
|
capturedRegistrars: [],
|
|
sdkExports: [],
|
|
manifestFields: [],
|
|
manifestContractFields: [],
|
|
};
|
|
}
|
|
|
|
const requestedPaths = targetOpenClawPathCandidates(manifest, configuredPath);
|
|
if (requestedPaths.length === 0) {
|
|
return {
|
|
configuredPath: null,
|
|
status: "not-configured",
|
|
compatRecords: [],
|
|
compatRecordStatuses: {},
|
|
hookNames: [],
|
|
apiRegistrars: [],
|
|
capturedRegistrars: [],
|
|
sdkExports: [],
|
|
manifestFields: [],
|
|
manifestContractFields: [],
|
|
};
|
|
}
|
|
|
|
let requestedPath = requestedPaths[0];
|
|
let resolvedPath = path.resolve(repoRoot, requestedPath);
|
|
let registryPath = path.join(resolvedPath, "src/plugins/compat/registry.ts");
|
|
for (const candidatePath of requestedPaths) {
|
|
const candidateResolvedPath = path.resolve(repoRoot, candidatePath);
|
|
const candidateRegistryPath = path.join(candidateResolvedPath, "src/plugins/compat/registry.ts");
|
|
if (existsSync(candidateRegistryPath)) {
|
|
requestedPath = candidatePath;
|
|
resolvedPath = candidateResolvedPath;
|
|
registryPath = candidateRegistryPath;
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (!existsSync(registryPath)) {
|
|
return {
|
|
configuredPath: requestedPath,
|
|
searchedPaths: requestedPaths,
|
|
status: "missing",
|
|
compatRecords: [],
|
|
compatRecordStatuses: {},
|
|
hookNames: [],
|
|
apiRegistrars: [],
|
|
capturedRegistrars: [],
|
|
sdkExports: [],
|
|
manifestFields: [],
|
|
manifestContractFields: [],
|
|
};
|
|
}
|
|
|
|
const hookTypesPath = path.join(resolvedPath, "src/plugins/hook-types.ts");
|
|
const registrySource = await readFile(registryPath, "utf8");
|
|
const compatRecordEntries = parseCompatRecordEntries(registrySource);
|
|
const compatRecords = compatRecordEntries.map((record) => record.code).sort();
|
|
const hookNames = existsSync(hookTypesPath)
|
|
? parseExportedStringArray(await readFile(hookTypesPath, "utf8"), "PLUGIN_HOOK_NAMES")
|
|
: [];
|
|
const apiBuilderPath = path.join(resolvedPath, "src/plugins/api-builder.ts");
|
|
const apiRegistrars = existsSync(apiBuilderPath)
|
|
? unique([...((await readFile(apiBuilderPath, "utf8")).matchAll(/\b(register[A-Za-z0-9]+)\b/g))].map((match) => match[1])).sort()
|
|
: [];
|
|
const capturedRegistrationPath = path.join(resolvedPath, "src/plugins/captured-registration.ts");
|
|
const capturedRegistrars = existsSync(capturedRegistrationPath)
|
|
? unique(
|
|
[...((await readFile(capturedRegistrationPath, "utf8")).matchAll(/^\s*(register[A-Za-z0-9]+)\s*\(/gm))].map(
|
|
(match) => match[1],
|
|
),
|
|
).sort()
|
|
: [];
|
|
const manifestTypesPath = path.join(resolvedPath, "src/plugins/manifest.ts");
|
|
const manifestTypesSource = existsSync(manifestTypesPath) ? await readFile(manifestTypesPath, "utf8") : "";
|
|
const manifestFields = manifestTypesSource ? parseTypeFields(manifestTypesSource, "PluginManifest") : [];
|
|
const manifestContractFields = manifestTypesSource ? parseTypeFields(manifestTypesSource, "PluginManifestContracts") : [];
|
|
const packagePath = path.join(resolvedPath, "package.json");
|
|
const sdkExports = existsSync(packagePath)
|
|
? parsePluginSdkExports(JSON.parse(await readFile(packagePath, "utf8")))
|
|
: [];
|
|
|
|
return {
|
|
configuredPath: requestedPath,
|
|
searchedPaths: requestedPaths,
|
|
status: "ok",
|
|
compatRegistryPath: path.relative(repoRoot, registryPath),
|
|
compatRecordCount: compatRecords.length,
|
|
compatRecords,
|
|
compatRecordStatuses: Object.fromEntries(compatRecordEntries.map((record) => [record.code, record.status])),
|
|
hookTypesPath: existsSync(hookTypesPath) ? path.relative(repoRoot, hookTypesPath) : null,
|
|
hookNameCount: hookNames.length,
|
|
hookNames,
|
|
apiBuilderPath: existsSync(apiBuilderPath) ? path.relative(repoRoot, apiBuilderPath) : null,
|
|
apiRegistrarCount: apiRegistrars.length,
|
|
apiRegistrars,
|
|
capturedRegistrationPath: existsSync(capturedRegistrationPath)
|
|
? path.relative(repoRoot, capturedRegistrationPath)
|
|
: null,
|
|
capturedRegistrarCount: capturedRegistrars.length,
|
|
capturedRegistrars,
|
|
packagePath: existsSync(packagePath) ? path.relative(repoRoot, packagePath) : null,
|
|
sdkExportCount: sdkExports.length,
|
|
sdkExports,
|
|
manifestTypesPath: existsSync(manifestTypesPath) ? path.relative(repoRoot, manifestTypesPath) : null,
|
|
manifestFieldCount: manifestFields.length,
|
|
manifestFields,
|
|
manifestContractFieldCount: manifestContractFields.length,
|
|
manifestContractFields,
|
|
};
|
|
}
|
|
|
|
function parseCompatRecordEntries(source) {
|
|
const entries = [];
|
|
for (const match of source.matchAll(/\{[\s\S]*?\bcode:\s*["'`]([^"'`]+)["'`][\s\S]*?\bstatus:\s*["'`]([^"'`]+)["'`][\s\S]*?\n\s*\}/g)) {
|
|
entries.push({ code: match[1], status: match[2] });
|
|
}
|
|
return dedupeBy(entries, (entry) => entry.code).sort((left, right) => left.code.localeCompare(right.code));
|
|
}
|
|
|
|
function parsePluginSdkExports(packageJson) {
|
|
return Object.keys(packageJson.exports ?? {})
|
|
.filter((specifier) => specifier === "./plugin-sdk" || specifier.startsWith("./plugin-sdk/"))
|
|
.map((specifier) => `openclaw/${specifier.slice(2)}`)
|
|
.sort();
|
|
}
|
|
|
|
export function targetOpenClawPathCandidates(manifest, configuredPath) {
|
|
if (typeof configuredPath === "string") {
|
|
return [configuredPath];
|
|
}
|
|
|
|
return unique([manifest.openclaw?.defaultCheckoutPath, ...FALLBACK_OPENCLAW_CHECKOUT_PATHS].filter(Boolean));
|
|
}
|
|
|
|
async function readPackageSummaries(checkoutPath, sourceRoot) {
|
|
const candidates = unique([
|
|
path.join(sourceRoot, "package.json"),
|
|
path.join(checkoutPath, "package.json"),
|
|
...(await findPackageFiles(checkoutPath, { maxDepth: 3 })),
|
|
].filter(existsSync));
|
|
const summaries = [];
|
|
|
|
for (const packagePath of candidates) {
|
|
const packageJson = JSON.parse(await readFile(packagePath, "utf8"));
|
|
summaries.push(summarizePackage(packagePath, packageJson));
|
|
}
|
|
|
|
return summaries.sort((left, right) => packageRank(left) - packageRank(right) || left.path.localeCompare(right.path));
|
|
}
|
|
|
|
function summarizePackage(packagePath, packageJson) {
|
|
const packageDir = path.dirname(packagePath);
|
|
const openclaw = packageJson.openclaw
|
|
? {
|
|
extensions: arrayValues(packageJson.openclaw.extensions),
|
|
runtimeExtensions: arrayValues(packageJson.openclaw.runtimeExtensions),
|
|
setupEntry: typeof packageJson.openclaw.setupEntry === "string" ? packageJson.openclaw.setupEntry : null,
|
|
compatPluginApi:
|
|
typeof packageJson.openclaw.compat?.pluginApi === "string" ? packageJson.openclaw.compat.pluginApi : null,
|
|
buildOpenClawVersion:
|
|
typeof packageJson.openclaw.build?.openclawVersion === "string"
|
|
? packageJson.openclaw.build.openclawVersion
|
|
: null,
|
|
buildPluginSdkVersion:
|
|
typeof packageJson.openclaw.build?.pluginSdkVersion === "string"
|
|
? packageJson.openclaw.build.pluginSdkVersion
|
|
: null,
|
|
}
|
|
: null;
|
|
|
|
if (openclaw) {
|
|
openclaw.entrypoints = collectOpenClawEntrypoints(packageDir, openclaw);
|
|
}
|
|
|
|
return {
|
|
path: path.relative(repoRoot, packagePath),
|
|
name: packageJson.name ?? null,
|
|
version: packageJson.version ?? null,
|
|
type: packageJson.type ?? null,
|
|
main: typeof packageJson.main === "string" ? packageJson.main : null,
|
|
dependencies: Object.keys(packageJson.dependencies ?? {}).sort(),
|
|
peerDependencies: Object.keys(packageJson.peerDependencies ?? {}).sort(),
|
|
optionalDependencies: Object.keys(packageJson.optionalDependencies ?? {}).sort(),
|
|
openclaw,
|
|
};
|
|
}
|
|
|
|
function collectOpenClawEntrypoints(packageDir, openclaw) {
|
|
const entrypoints = [
|
|
...openclaw.extensions.map((specifier) => ({ kind: "extension", specifier })),
|
|
...openclaw.runtimeExtensions.map((specifier) => ({ kind: "runtimeExtension", specifier })),
|
|
...(openclaw.setupEntry ? [{ kind: "setupEntry", specifier: openclaw.setupEntry }] : []),
|
|
];
|
|
|
|
return entrypoints.map((entrypoint) => {
|
|
const resolvedPath = path.resolve(packageDir, entrypoint.specifier);
|
|
const relativePath = path.relative(repoRoot, resolvedPath);
|
|
return {
|
|
...entrypoint,
|
|
relativePath,
|
|
exists: existsSync(resolvedPath),
|
|
requiresBuild: /(^|\/)dist\//.test(entrypoint.specifier) || /(^|\/)build\//.test(entrypoint.specifier),
|
|
};
|
|
});
|
|
}
|
|
|
|
async function findPackageFiles(root, options, depth = 0) {
|
|
if (!existsSync(root) || depth > options.maxDepth) {
|
|
return [];
|
|
}
|
|
|
|
const files = [];
|
|
for (const entry of await readdir(root, { withFileTypes: true })) {
|
|
const entryPath = path.join(root, entry.name);
|
|
if (entry.isFile() && entry.name === "package.json") {
|
|
files.push(entryPath);
|
|
continue;
|
|
}
|
|
if (!entry.isDirectory() || shouldSkipPackageDir(entry.name)) {
|
|
continue;
|
|
}
|
|
files.push(...(await findPackageFiles(entryPath, options, depth + 1)));
|
|
}
|
|
return files;
|
|
}
|
|
|
|
function shouldSkipPackageDir(name) {
|
|
return name === ".git" || name === "node_modules" || name === "dist" || name === "build" || name === "coverage";
|
|
}
|
|
|
|
function selectPrimaryPackage(packages) {
|
|
return packages[0] ?? null;
|
|
}
|
|
|
|
function packageRank(packageSummary) {
|
|
if (packageSummary.openclaw?.entrypoints.length > 0) {
|
|
return 0;
|
|
}
|
|
if (packageSummary.openclaw) {
|
|
return 1;
|
|
}
|
|
return 2;
|
|
}
|
|
|
|
function arrayValues(value) {
|
|
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
|
}
|
|
|
|
function emptyInspection(fixture) {
|
|
return {
|
|
id: fixture.id,
|
|
status: "missing",
|
|
hooks: [],
|
|
hookDetails: [],
|
|
registrations: [],
|
|
registrationDetails: [],
|
|
manifestContracts: [],
|
|
manifestFiles: [],
|
|
manifestErrors: [],
|
|
sdkImports: [],
|
|
sourceFiles: [],
|
|
};
|
|
}
|
|
|
|
function findingsTable(findings) {
|
|
if (findings.length === 0) {
|
|
return "_none_";
|
|
}
|
|
|
|
return markdownTable(
|
|
findings.map((finding) => [
|
|
finding.fixture,
|
|
finding.code,
|
|
finding.level,
|
|
finding.message,
|
|
(finding.evidence ?? []).join(", ") || "-",
|
|
finding.compatRecord ?? "-",
|
|
]),
|
|
["Fixture", "Code", "Level", "Message", "Evidence", "Compat record"],
|
|
);
|
|
}
|
|
|
|
function issuesTable(issues) {
|
|
if (issues.length === 0) {
|
|
return "_none_";
|
|
}
|
|
|
|
return issues.map((issue) => issueBlock(issue)).join("\n\n");
|
|
}
|
|
|
|
function issueBlock(issue) {
|
|
return [
|
|
`- ${severityLabel(issue.severity)} **${issue.fixture}** \`${issue.issueClass}\` \`${issue.decision}\``,
|
|
` - **${issue.code}**: ${issue.title}`,
|
|
` - state: ${issueState(issue)}`,
|
|
" - evidence:",
|
|
...evidenceList(issue.evidence).map((item) => ` - ${item}`),
|
|
].join("\n");
|
|
}
|
|
|
|
function issueState(issue) {
|
|
const flags = [
|
|
issue.status,
|
|
`compat:${issue.compatStatus ?? "none"}`,
|
|
issue.live ? "live" : null,
|
|
issue.deprecated ? "deprecated" : null,
|
|
].filter(Boolean);
|
|
return flags.join(" · ");
|
|
}
|
|
|
|
function triageOverview(report) {
|
|
return markdownTable(
|
|
[
|
|
[
|
|
"live-issue",
|
|
report.summary.liveIssueCount,
|
|
report.summary.liveP0IssueCount,
|
|
"Potential runtime breakage in the target OpenClaw/plugin pair. P0 only when it is not a deprecated compat seam.",
|
|
],
|
|
[
|
|
"compat-gap",
|
|
report.summary.compatGapCount,
|
|
"-",
|
|
"Compatibility behavior is needed but missing from the target OpenClaw compat registry.",
|
|
],
|
|
[
|
|
"deprecation-warning",
|
|
report.summary.deprecationWarningCount,
|
|
"-",
|
|
"Plugin uses a supported but deprecated compatibility seam; keep it wired while migration exists.",
|
|
],
|
|
[
|
|
"inspector-gap",
|
|
report.summary.inspectorGapCount,
|
|
"-",
|
|
"Crabpot needs stronger capture/probe evidence before making contract judgments.",
|
|
],
|
|
[
|
|
"upstream-metadata",
|
|
report.summary.upstreamIssueCount,
|
|
"-",
|
|
"Plugin package or manifest metadata should improve upstream; not a target OpenClaw live break by itself.",
|
|
],
|
|
[
|
|
"fixture-regression",
|
|
report.summary.fixtureRegressionCount,
|
|
"-",
|
|
"Fixture no longer exposes a seam Crabpot expected; investigate fixture pin or scanner drift.",
|
|
],
|
|
],
|
|
["Class", "Count", "P0", "Meaning"],
|
|
);
|
|
}
|
|
|
|
function contractProbesTable(probes) {
|
|
if (probes.length === 0) {
|
|
return "_none_";
|
|
}
|
|
|
|
return probes.map((probe) => contractProbeBlock(probe)).join("\n\n");
|
|
}
|
|
|
|
function contractProbeBlock(probe) {
|
|
return [
|
|
`- ${severityLabel(probe.priority)} **${probe.fixture}** \`${probe.target}\``,
|
|
` - contract: ${probe.contract}`,
|
|
` - id: \`${probe.id}\``,
|
|
" - evidence:",
|
|
...evidenceList(probe.evidence).map((item) => ` - ${item}`),
|
|
].join("\n");
|
|
}
|
|
|
|
function targetOpenClawTable(targetOpenClaw) {
|
|
const recordPreview = targetOpenClaw.compatRecords.length > 0 ? targetOpenClaw.compatRecords.join(", ") : "-";
|
|
const statusCounts = Object.values(targetOpenClaw.compatRecordStatuses ?? {}).reduce((counts, status) => {
|
|
counts[status] = (counts[status] ?? 0) + 1;
|
|
return counts;
|
|
}, {});
|
|
return markdownTable(
|
|
[
|
|
["Configured path", targetOpenClaw.configuredPath ?? "-"],
|
|
["Status", targetOpenClaw.status],
|
|
["Compat registry", targetOpenClaw.compatRegistryPath ?? "-"],
|
|
["Compat records", targetOpenClaw.compatRecordCount ?? 0],
|
|
["Compat status counts", Object.entries(statusCounts).map(([status, count]) => `${status}:${count}`).join(", ") || "-"],
|
|
["Record ids", recordPreview],
|
|
["Hook registry", targetOpenClaw.hookTypesPath ?? "-"],
|
|
["Hook names", targetOpenClaw.hookNameCount ?? 0],
|
|
["API builder", targetOpenClaw.apiBuilderPath ?? "-"],
|
|
["API registrars", targetOpenClaw.apiRegistrarCount ?? 0],
|
|
["Captured registration", targetOpenClaw.capturedRegistrationPath ?? "-"],
|
|
["Captured registrars", targetOpenClaw.capturedRegistrarCount ?? 0],
|
|
["Package metadata", targetOpenClaw.packagePath ?? "-"],
|
|
["Plugin SDK exports", targetOpenClaw.sdkExportCount ?? 0],
|
|
["Manifest types", targetOpenClaw.manifestTypesPath ?? "-"],
|
|
["Manifest fields", targetOpenClaw.manifestFieldCount ?? 0],
|
|
["Manifest contract fields", targetOpenClaw.manifestContractFieldCount ?? 0],
|
|
],
|
|
["Metric", "Value"],
|
|
);
|
|
}
|
|
|
|
function markdownTable(rows, headers) {
|
|
if (rows.length === 0) {
|
|
return "_none_";
|
|
}
|
|
|
|
const allRows = [headers, ...rows.map((row) => row.map(String))];
|
|
const widths = headers.map((_, columnIndex) =>
|
|
Math.max(...allRows.map((row) => escapeCell(row[columnIndex] ?? "").length)),
|
|
);
|
|
const formatRow = (row) =>
|
|
`| ${row.map((cell, index) => escapeCell(cell).padEnd(widths[index], " ")).join(" | ")} |`;
|
|
const separator = `| ${widths.map((width) => "-".repeat(width)).join(" | ")} |`;
|
|
|
|
return [formatRow(headers), separator, ...rows.map((row) => formatRow(row.map(String)))].join("\n");
|
|
}
|
|
|
|
function escapeCell(value) {
|
|
return String(value).replaceAll("|", "\\|").replaceAll("\n", " ");
|
|
}
|
|
|
|
function severityLabel(severity) {
|
|
const labels = {
|
|
P0: "🔴 P0",
|
|
P1: "🟠 P1",
|
|
P2: "🟡 P2",
|
|
P3: "🟢 P3",
|
|
};
|
|
return labels[severity] ?? severity ?? "-";
|
|
}
|
|
|
|
function evidenceList(evidence) {
|
|
const items = (evidence ?? []).filter(Boolean);
|
|
if (items.length === 0) {
|
|
return ["-"];
|
|
}
|
|
return items.map((item) => formatEvidenceLink(item));
|
|
}
|
|
|
|
function formatEvidenceLink(evidence) {
|
|
const parsed = parseEvidencePath(evidence);
|
|
if (!parsed) {
|
|
return evidence;
|
|
}
|
|
const target = submoduleLinkTarget(parsed.filePath);
|
|
if (!target) {
|
|
return evidence;
|
|
}
|
|
return `[${evidenceLinkLabel(evidence, parsed)}](${target.href}${parsed.line ? `#L${parsed.line}` : ""})`;
|
|
}
|
|
|
|
function parseEvidencePath(evidence) {
|
|
const match = String(evidence).match(/(?<filePath>plugins[\\/][^\s,)]+?)(?::(?<line>\d+))?(?=$|[\s,)])/);
|
|
if (!match?.groups?.filePath) {
|
|
return null;
|
|
}
|
|
return {
|
|
filePath: match.groups.filePath.replaceAll("\\", "/"),
|
|
index: match.index ?? 0,
|
|
line: match.groups.line,
|
|
};
|
|
}
|
|
|
|
function evidenceLinkLabel(evidence, parsed) {
|
|
const prefix = String(evidence)
|
|
.slice(0, parsed.index)
|
|
.trim()
|
|
.replace(/\s*(?:@|->|:)\s*$/, "");
|
|
const fileLabel = `${path.posix.basename(parsed.filePath)}${parsed.line ? `:${parsed.line}` : ""}`;
|
|
return prefix ? `${prefix} @ ${fileLabel}` : fileLabel;
|
|
}
|
|
|
|
function submoduleLinkTarget(filePath) {
|
|
const normalizedFilePath = filePath.replaceAll("\\", "/");
|
|
const target = submoduleLinkTargetsForRepo().find(
|
|
(candidate) => normalizedFilePath === candidate.path || normalizedFilePath.startsWith(`${candidate.path}/`),
|
|
);
|
|
if (!target) {
|
|
return null;
|
|
}
|
|
const subPath = normalizedFilePath === target.path ? "" : normalizedFilePath.slice(target.path.length + 1);
|
|
const encodedPath = subPath
|
|
.split("/")
|
|
.filter(Boolean)
|
|
.map((part) => encodeURIComponent(part))
|
|
.join("/");
|
|
return {
|
|
href: encodedPath ? `${target.webUrl}/blob/${target.sha}/${encodedPath}` : `${target.webUrl}/tree/${target.sha}`,
|
|
};
|
|
}
|
|
|
|
function submoduleLinkTargetsForRepo() {
|
|
if (submoduleLinkTargets) {
|
|
return submoduleLinkTargets;
|
|
}
|
|
const modules = readGitmodules();
|
|
const shas = readSubmoduleShas();
|
|
submoduleLinkTargets = [...modules.values()]
|
|
.map((entry) => ({
|
|
...entry,
|
|
sha: shas.get(entry.path),
|
|
webUrl: githubWebUrl(entry.url),
|
|
}))
|
|
.filter((entry) => entry.sha && entry.webUrl)
|
|
.sort((left, right) => right.path.length - left.path.length);
|
|
return submoduleLinkTargets;
|
|
}
|
|
|
|
function readGitmodules() {
|
|
const gitmodulesPath = path.join(repoRoot, ".gitmodules");
|
|
if (!existsSync(gitmodulesPath)) {
|
|
return new Map();
|
|
}
|
|
|
|
const modules = new Map();
|
|
let current = {};
|
|
for (const line of readFileSync(gitmodulesPath, "utf8").split(/\r?\n/)) {
|
|
if (/^\[submodule /.test(line)) {
|
|
if (current.path && current.url) {
|
|
modules.set(current.path, current);
|
|
}
|
|
current = {};
|
|
continue;
|
|
}
|
|
const pathMatch = line.match(/^\s*path\s*=\s*(.+)$/);
|
|
if (pathMatch) {
|
|
current.path = pathMatch[1].trim();
|
|
continue;
|
|
}
|
|
const urlMatch = line.match(/^\s*url\s*=\s*(.+)$/);
|
|
if (urlMatch) {
|
|
current.url = urlMatch[1].trim();
|
|
}
|
|
}
|
|
if (current.path && current.url) {
|
|
modules.set(current.path, current);
|
|
}
|
|
return modules;
|
|
}
|
|
|
|
function readSubmoduleShas() {
|
|
const gitlinkShas = readSubmoduleGitlinkShas();
|
|
if (gitlinkShas.size > 0) {
|
|
return gitlinkShas;
|
|
}
|
|
|
|
try {
|
|
const status = execFileSync("git", ["-c", "safe.directory=*", "submodule", "status", "--recursive"], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "ignore"],
|
|
});
|
|
return new Map(
|
|
status
|
|
.split(/\r?\n/)
|
|
.map((line) => line.match(/^[ +-]?([0-9a-f]{40})\s+(\S+)/i))
|
|
.filter(Boolean)
|
|
.map((match) => [match[2].replaceAll("\\", "/"), match[1]]),
|
|
);
|
|
} catch {
|
|
return new Map();
|
|
}
|
|
}
|
|
|
|
function readSubmoduleGitlinkShas() {
|
|
try {
|
|
const tree = execFileSync("git", ["-c", "safe.directory=*", "ls-tree", "-r", "HEAD", "plugins"], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
stdio: ["ignore", "pipe", "ignore"],
|
|
});
|
|
return new Map(
|
|
tree
|
|
.split(/\r?\n/)
|
|
.map((line) => line.match(/^160000 commit ([0-9a-f]{40})\s+(.+)$/i))
|
|
.filter(Boolean)
|
|
.map((match) => [match[2].replaceAll("\\", "/"), match[1]]),
|
|
);
|
|
} catch {
|
|
return new Map();
|
|
}
|
|
}
|
|
|
|
function githubWebUrl(url) {
|
|
const httpsMatch = url.match(/^https:\/\/github\.com\/(.+?)(?:\.git)?$/);
|
|
if (httpsMatch) {
|
|
return `https://github.com/${httpsMatch[1].replace(/\/$/, "")}`;
|
|
}
|
|
const sshMatch = url.match(/^git@github\.com:(.+?)(?:\.git)?$/);
|
|
if (sshMatch) {
|
|
return `https://github.com/${sshMatch[1].replace(/\/$/, "")}`;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function issueSort(left, right) {
|
|
return (
|
|
priorityRank(left.severity) - priorityRank(right.severity) ||
|
|
left.fixture.localeCompare(right.fixture) ||
|
|
left.code.localeCompare(right.code) ||
|
|
(left.evidence ?? []).join(",").localeCompare((right.evidence ?? []).join(","))
|
|
);
|
|
}
|
|
|
|
function issueTitle(finding) {
|
|
return `${finding.fixture}: ${finding.title ?? finding.message}`;
|
|
}
|
|
|
|
function probePriority(code, fixturePriority) {
|
|
if (
|
|
[
|
|
"before-tool-call-probe",
|
|
"conversation-access-hook",
|
|
"missing-compat-record",
|
|
"registration-capture-gap",
|
|
"sdk-export-missing",
|
|
].includes(code)
|
|
) {
|
|
return "P1";
|
|
}
|
|
if (fixturePriority === "high") {
|
|
return "P2";
|
|
}
|
|
return "P3";
|
|
}
|
|
|
|
function priorityRank(priority) {
|
|
return { P0: 0, P1: 1, P2: 2, P3: 3, high: 1, medium: 2, low: 3 }[priority] ?? 99;
|
|
}
|
|
|
|
function detailEvidence(details, key = "name") {
|
|
return unique(details.map((detail) => `${detail[key]} @ ${detail.ref}`));
|
|
}
|
|
|
|
function parseExportedStringArray(source, exportName) {
|
|
const match = source.match(new RegExp(`export\\s+const\\s+${exportName}\\s*=\\s*\\[([\\s\\S]*?)\\]\\s+as\\s+const`));
|
|
if (!match) {
|
|
return [];
|
|
}
|
|
|
|
return unique([...match[1].matchAll(/["'`]([^"'`]+)["'`]/g)].map((item) => item[1])).sort();
|
|
}
|
|
|
|
function parseTypeFields(source, typeName) {
|
|
const marker = `export type ${typeName} = {`;
|
|
const start = source.indexOf(marker);
|
|
if (start === -1) {
|
|
return [];
|
|
}
|
|
const bodyStart = start + marker.length;
|
|
const end = source.indexOf("\n};", bodyStart);
|
|
if (end === -1) {
|
|
return [];
|
|
}
|
|
const body = source.slice(bodyStart, end);
|
|
return unique(
|
|
[...body.matchAll(/^\s*([A-Za-z][A-Za-z0-9]*)\??:/gm)]
|
|
.map((match) => match[1])
|
|
.filter((field) => !field.startsWith("PluginManifest")),
|
|
).sort();
|
|
}
|
|
|
|
function dedupeBy(values, keyForValue) {
|
|
const byKey = new Map();
|
|
for (const value of values) {
|
|
byKey.set(keyForValue(value), value);
|
|
}
|
|
return [...byKey.values()];
|
|
}
|
|
|
|
function unique(values) {
|
|
return [...new Set(values)];
|
|
}
|