Compare commits
31 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8899fc796c | ||
|
|
feefb4ee23 | ||
|
|
f642fb5c9f | ||
|
|
68e10e0aaa | ||
|
|
12005b4658 | ||
|
|
4956ad1fbc | ||
|
|
a58e0785d5 | ||
|
|
9f45c8aeb6 | ||
|
|
677f6e5bc1 | ||
|
|
06cc55ce51 | ||
|
|
2eda65a8a9 | ||
|
|
4bc5fbcfa3 | ||
|
|
a6af8800e0 | ||
|
|
b919df78d3 | ||
|
|
ff78dccff7 | ||
|
|
b33c6f725d | ||
|
|
e38991a35f | ||
|
|
eb251cbae4 | ||
|
|
cc89d7cea7 | ||
|
|
1ee105e29d | ||
|
|
572956b8df | ||
|
|
e8fb9b4380 | ||
|
|
7b5f706398 | ||
|
|
862d8c9fb8 | ||
|
|
f6991de9b0 | ||
|
|
e04c3fc121 | ||
|
|
85c69fb24d | ||
|
|
9ab07bb316 | ||
|
|
d91d596d90 | ||
|
|
332706b014 | ||
|
|
e9e4b6704c |
@ -6,6 +6,9 @@
|
||||
- Do not publish npm packages without explicit owner approval.
|
||||
- Preserve stable report field names and finding codes; downstream CI and
|
||||
crabpot reports may consume them.
|
||||
- Treat a package dependency named `openclaw` as a host-linked workspace input,
|
||||
not an isolated dependency-install blocker. Keep third-party runtime
|
||||
dependencies classified as install/audit blockers.
|
||||
- When changing plugin-inspector behavior, CLI/package entrypoints, release
|
||||
metadata, or the npm package version, update crabpot's
|
||||
`@openclaw/plugin-inspector` pin/docs/smoke path as needed and run the
|
||||
|
||||
66
CHANGELOG.md
66
CHANGELOG.md
@ -2,6 +2,72 @@
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Fixed
|
||||
|
||||
- Stop classifying package source entrypoints as missing when the published package provides built runtime entrypoints, and collapse SDK alias findings into a single compat-gap row.
|
||||
- Treat compat-gap issues as reconciled contract coverage for their own compatibility record.
|
||||
|
||||
## 0.3.10 - 2026-05-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Accept valid mocked capture output when plugin code leaves `process.exitCode` dirty.
|
||||
|
||||
## 0.3.9 - 2026-05-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Follow bundled channel `loadBundledEntryExportSync` registration exports during mocked runtime capture.
|
||||
|
||||
## 0.3.8 - 2026-05-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Synthesize manifest config for isolated runtime capture so configured hooks can be observed without credentials.
|
||||
|
||||
## 0.3.7 - 2026-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Downgrade `registration-capture-gap` to advisory severity so missing capture evidence no longer reports as a P1 plugin contract risk.
|
||||
|
||||
## 0.3.6 - 2026-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Report import-loop RSS and CPU as baseline-adjusted plugin deltas alongside raw subprocess metrics so Crabpot dashboards do not treat harness import cost as plugin runtime cost.
|
||||
- Include optional OpenClaw loader lifecycle timings for import and full activation when a capture runner provides them.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Accept plugin install minimum-host floors as supported package metadata.
|
||||
- Flag unsupported legacy OpenClaw bundle metadata and advertised npm pack blockers.
|
||||
- Reconcile runtime capture evidence and harden mocked capture paths for downstream fixture reports.
|
||||
|
||||
## 0.3.5 - 2026-04-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Add immediate/faster subprocess RSS and CPU sampling plus explicit sample counts so short import-loop reports do not silently publish fake zero-memory metrics.
|
||||
- Classify `createChatChannelPlugin` as channel factory metadata in synthetic probe plans so channel-core plugins do not fail as unknown registrars.
|
||||
- Treat `createChatChannelPlugin` and `defineChannelPluginEntry` as channel registration equivalents when validating fixture expectations.
|
||||
- Label runtime profile wall-time summaries as command-median p95 and render missing sampled metrics as `n/a`.
|
||||
|
||||
## 0.3.4 - 2026-04-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Separate executor-covered platform portability findings from residual findings so downstream structured runners can keep reports blocking only on unhandled risks.
|
||||
- Sanitize absolute target OpenClaw paths from generated report artifacts and JSON CLI output.
|
||||
- Normalize the dependency-install inspector finding title to use isolated-workspace wording.
|
||||
- Treat `openclaw` package dependencies as host-linked workspace inputs instead of isolated dependency-install blockers.
|
||||
|
||||
## 0.3.3 - 2026-04-28
|
||||
|
||||
### Fixed
|
||||
|
||||
- Classify generated kitchen-sink public registrar coverage in synthetic probe plans so new API-surface fixtures do not fail as unknown execution profiles.
|
||||
|
||||
## 0.3.2 - 2026-04-28
|
||||
|
||||
### Fixed
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@openclaw/plugin-inspector",
|
||||
"version": "0.3.2",
|
||||
"version": "0.3.10",
|
||||
"private": false,
|
||||
"description": "Offline compatibility inspector for OpenClaw plugins.",
|
||||
"type": "module",
|
||||
|
||||
@ -43,6 +43,7 @@ export {
|
||||
} from "./ci-outputs.js";
|
||||
export {
|
||||
buildContractProbes,
|
||||
compatRecordForIssueCode,
|
||||
contractProbeRules,
|
||||
probePriority,
|
||||
} from "./contract-probes.js";
|
||||
@ -94,6 +95,7 @@ export {
|
||||
classifyTargetOpenClawCoverage,
|
||||
readPackageSummaries,
|
||||
readPluginManifests,
|
||||
readSecurityManifests,
|
||||
summarizePackage,
|
||||
} from "./fixture-summary.js";
|
||||
export {
|
||||
@ -170,6 +172,7 @@ export {
|
||||
classifyCompatRecordCoverage,
|
||||
renderMarkdownReport,
|
||||
renderTextSummary,
|
||||
sanitizeReportArtifact,
|
||||
writeCompatibilityReport,
|
||||
writeReport,
|
||||
} from "./report.js";
|
||||
@ -181,6 +184,10 @@ export {
|
||||
validateRuntimeProfile,
|
||||
writeRuntimeProfile,
|
||||
} from "./runtime-profile.js";
|
||||
export {
|
||||
applyRuntimeExecutionCoverage,
|
||||
buildRuntimeExecutionCoverage,
|
||||
} from "./runtime-reconciliation.js";
|
||||
export {
|
||||
buildRuntimeCaptureReport,
|
||||
renderRuntimeCaptureMarkdown,
|
||||
|
||||
@ -45,6 +45,7 @@ export async function inspectPluginRoot(options = {}) {
|
||||
return inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
});
|
||||
}
|
||||
@ -59,6 +60,7 @@ export async function inspectCompatibilityFixtureSetConfig(options = {}) {
|
||||
return inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
});
|
||||
}
|
||||
@ -112,6 +114,7 @@ export async function buildFixtureSetColdImportReadiness(options = {}) {
|
||||
(await inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
}));
|
||||
|
||||
@ -143,6 +146,7 @@ export async function buildFixtureSetWorkspacePlan(options = {}) {
|
||||
(await inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
}));
|
||||
const rootDir = options.rootDir ?? config?.rootDir ?? options.cwd;
|
||||
|
||||
@ -95,7 +95,7 @@ export function renderPaddedMarkdownTable(rows, headers, options = {}) {
|
||||
}
|
||||
|
||||
export function escapeMarkdownTableCell(value) {
|
||||
return value.replace(/\|/g, "\\|").replace(/\n/g, "<br>");
|
||||
return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, "<br>");
|
||||
}
|
||||
|
||||
async function assertFileMatches(filePath, expected) {
|
||||
|
||||
@ -148,6 +148,7 @@ export function createCaptureContext(options = {}) {
|
||||
config: options.config ?? {},
|
||||
logger: options.logger ?? console,
|
||||
pluginConfig: options.pluginConfig ?? {},
|
||||
resolvePath: options.resolvePath ?? ((value) => value),
|
||||
runtime: options.runtime ?? createRuntimeContext(options),
|
||||
secrets: options.secrets ?? createSecretContext(options),
|
||||
store: options.store ?? createStoreContext(options),
|
||||
|
||||
126
src/capture-config.js
Normal file
126
src/capture-config.js
Normal file
@ -0,0 +1,126 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export async function captureApiOptionsForPlugin(apiOptions = {}, options = {}) {
|
||||
if (apiOptions.pluginConfig !== undefined || !options.pluginRoot) {
|
||||
return apiOptions;
|
||||
}
|
||||
|
||||
const pluginConfig = await readSamplePluginConfig(options.pluginRoot);
|
||||
if (pluginConfig === undefined) {
|
||||
return apiOptions;
|
||||
}
|
||||
return {
|
||||
...apiOptions,
|
||||
pluginConfig,
|
||||
};
|
||||
}
|
||||
|
||||
async function readSamplePluginConfig(pluginRoot) {
|
||||
const manifestPath = path.join(pluginRoot, "openclaw.plugin.json");
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const sample = sampleJsonSchema(manifest.configSchema, { key: "config" });
|
||||
return isPlainObject(sample) && Object.keys(sample).length > 0 ? sample : undefined;
|
||||
}
|
||||
|
||||
function sampleJsonSchema(schema, context = {}) {
|
||||
if (!isPlainObject(schema)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (Array.isArray(schema.enum) && schema.enum.length > 0) {
|
||||
return schema.enum[0];
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(schema, "const")) {
|
||||
return schema.const;
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(schema, "default")) {
|
||||
return schema.default;
|
||||
}
|
||||
|
||||
const type = Array.isArray(schema.type) ? schema.type.find((item) => item !== "null") : schema.type;
|
||||
if (type === "object" || schema.properties) {
|
||||
return sampleObjectSchema(schema);
|
||||
}
|
||||
if (type === "array") {
|
||||
return [];
|
||||
}
|
||||
if (type === "boolean") {
|
||||
return false;
|
||||
}
|
||||
if (type === "number" || type === "integer") {
|
||||
return typeof schema.minimum === "number" ? schema.minimum : 1;
|
||||
}
|
||||
if (type === "string" || !type) {
|
||||
return sampleString(context.key);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function sampleObjectSchema(schema) {
|
||||
const properties = isPlainObject(schema.properties) ? schema.properties : {};
|
||||
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
||||
const output = {};
|
||||
|
||||
for (const key of Object.keys(properties)) {
|
||||
if (required.has(key)) {
|
||||
const value = sampleJsonSchema(properties[key], { key });
|
||||
if (value !== undefined) {
|
||||
output[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of Object.keys(properties)) {
|
||||
if (properties[key]?.type === "boolean") {
|
||||
output[key] = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(output).length === 0 && Number(schema.minProperties ?? 0) > 0) {
|
||||
const key = preferredSamplePropertyKey(properties);
|
||||
if (key) {
|
||||
const value = sampleJsonSchema(properties[key], { key });
|
||||
if (value !== undefined) {
|
||||
output[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function preferredSamplePropertyKey(properties) {
|
||||
for (const key of ["provider", "model", "apiKey", "id", "name", ...Object.keys(properties)]) {
|
||||
if (Object.prototype.hasOwnProperty.call(properties, key)) {
|
||||
return key;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function sampleString(key = "") {
|
||||
if (key === "provider") {
|
||||
return "openai";
|
||||
}
|
||||
if (key === "model") {
|
||||
return "text-embedding-3-small";
|
||||
}
|
||||
if (key === "apiKey") {
|
||||
return "fixture-api-key";
|
||||
}
|
||||
if (key === "dbPath") {
|
||||
return ".plugin-inspector/state/lancedb";
|
||||
}
|
||||
return "fixture";
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
@ -241,7 +241,9 @@ function executionChecks(executionResults, policy, options) {
|
||||
}
|
||||
|
||||
function findPolicyMatch(rules, item) {
|
||||
return rules.find((rule) => item.seam === rule.seam && item.reason?.includes(rule.reasonIncludes));
|
||||
return rules.find(
|
||||
(rule) => (rule.seam === "*" || item.seam === rule.seam) && item.reason?.includes(rule.reasonIncludes),
|
||||
);
|
||||
}
|
||||
|
||||
function failedExecutionEvidence(executionResults) {
|
||||
|
||||
@ -56,8 +56,14 @@ export async function buildCiSummary(options = {}) {
|
||||
loaderJitiCandidates: reports.platform?.summary?.jitiAlternativeCount ?? 0,
|
||||
importLoopP50Ms: reports.importLoop?.summary?.p50WallMs ?? 0,
|
||||
importLoopP95Ms: reports.importLoop?.summary?.p95WallMs ?? 0,
|
||||
importLoopMaxRssMb: reports.importLoop?.summary?.maxPeakRssMb ?? 0,
|
||||
importLoopMaxCpuMs: reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
|
||||
importLoopOpenClawLifecycleCount: reports.importLoop?.summary?.openClawLifecycleCount ?? 0,
|
||||
importLoopOpenClawImportP50Ms: reports.importLoop?.summary?.p50OpenClawImportMs ?? 0,
|
||||
importLoopOpenClawActivationP50Ms: reports.importLoop?.summary?.p50OpenClawActivationMs ?? 0,
|
||||
importLoopMetricBasis: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb === undefined ? "raw" : "baseline-adjusted",
|
||||
importLoopMaxRssMb: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb ?? reports.importLoop?.summary?.maxPeakRssMb ?? 0,
|
||||
importLoopMaxCpuMs: reports.importLoop?.summary?.maxPluginCpuDeltaMsEstimate ?? reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
|
||||
importLoopRssSampleCount: metricSampleCount(reports.importLoop, "rss", "maxPeakRssMb"),
|
||||
importLoopCpuSampleCount: metricSampleCount(reports.importLoop, "cpu", "maxCpuMsEstimate"),
|
||||
},
|
||||
topIssues: topIssues(reports.compatibility),
|
||||
refRegressions: (reports.refDiff?.regressions ?? []).slice(0, 20),
|
||||
@ -148,7 +154,7 @@ export function renderCiSummaryMarkdown(summary) {
|
||||
["Jiti loader candidates", summary.summary.loaderJitiCandidates],
|
||||
[
|
||||
"Import loop",
|
||||
`p50 ${summary.summary.importLoopP50Ms} ms / p95 ${summary.summary.importLoopP95Ms} ms / max RSS ${summary.summary.importLoopMaxRssMb} MB / CPU ${summary.summary.importLoopMaxCpuMs} ms`,
|
||||
importLoopSummaryLabel(summary.summary),
|
||||
],
|
||||
],
|
||||
["Metric", "Value"],
|
||||
@ -221,3 +227,44 @@ function topIssues(report) {
|
||||
function markdownTable(rows, headers) {
|
||||
return renderPaddedMarkdownTable(rows, headers, { nullValue: "-" });
|
||||
}
|
||||
|
||||
function metricSampleCount(report, kind, maxMetric) {
|
||||
const summaryKey = kind === "rss" ? "rssSampleCount" : "cpuSampleCount";
|
||||
const summaryCount = report?.summary?.[summaryKey];
|
||||
if (Number.isFinite(summaryCount)) {
|
||||
return summaryCount;
|
||||
}
|
||||
const sampleCount = inferSampleCount(report?.samples, kind);
|
||||
if (sampleCount > 0) {
|
||||
return sampleCount;
|
||||
}
|
||||
return (report?.summary?.[maxMetric] ?? 0) > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
function inferSampleCount(samples = [], kind) {
|
||||
if (!Array.isArray(samples)) {
|
||||
return 0;
|
||||
}
|
||||
return samples.reduce((sum, sample) => {
|
||||
if (kind === "rss") {
|
||||
return sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0));
|
||||
}
|
||||
return sum + (sample.cpuSampleCount ?? 0);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
function importLoopSummaryLabel(summary) {
|
||||
const metricLabel = summary.importLoopMetricBasis === "baseline-adjusted" ? "plugin delta" : "raw";
|
||||
const lifecycle =
|
||||
summary.importLoopOpenClawLifecycleCount > 0
|
||||
? ` / OpenClaw import ${summary.importLoopOpenClawImportP50Ms} ms / activate ${summary.importLoopOpenClawActivationP50Ms} ms`
|
||||
: "";
|
||||
return `p50 ${summary.importLoopP50Ms} ms / p95 ${summary.importLoopP95Ms} ms / ${metricLabel} RSS ${formatSampledMetric(summary.importLoopMaxRssMb, summary.importLoopRssSampleCount)} / ${metricLabel} CPU ${formatSampledMetric(summary.importLoopMaxCpuMs, summary.importLoopCpuSampleCount, "ms")}${lifecycle}`;
|
||||
}
|
||||
|
||||
function formatSampledMetric(value, count, unit = "MB") {
|
||||
if ((count ?? 0) <= 0) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
@ -3,6 +3,7 @@ import path from "node:path";
|
||||
import {
|
||||
loadPluginConfig,
|
||||
renderTextSummary,
|
||||
sanitizeReportArtifact,
|
||||
runPluginCheck,
|
||||
} from "./index.js";
|
||||
import {
|
||||
@ -90,7 +91,7 @@ async function runCheck(commandArgs) {
|
||||
});
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify(sanitizeReportArtifact(report), null, 2));
|
||||
} else {
|
||||
console.log(renderTextSummary(report, { artifacts: paths }));
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@ import path from "node:path";
|
||||
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
||||
import { slugForArtifact } from "./path-utils.js";
|
||||
|
||||
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
|
||||
|
||||
export function buildColdImportReadiness(options = {}) {
|
||||
const report = options.report;
|
||||
if (!report) {
|
||||
@ -182,7 +184,7 @@ function classifyEntrypointReadiness({ fixture, packageSummary, entrypoint, root
|
||||
...(packageSummary.dependencies ?? []),
|
||||
...(packageSummary.peerDependencies ?? []),
|
||||
...(packageSummary.optionalDependencies ?? []),
|
||||
]);
|
||||
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
|
||||
if (entrypoint.exists && runtimeDependencies.length > 0) {
|
||||
blockers.push({
|
||||
code: "dependency-install-required",
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { renderPaddedMarkdownTable } from "./artifacts.js";
|
||||
import { sanitizeReportArtifact } from "./report-sanitizer.js";
|
||||
|
||||
const defaultSeverityLabels = {
|
||||
P0: "P0",
|
||||
@ -8,6 +9,7 @@ const defaultSeverityLabels = {
|
||||
};
|
||||
|
||||
export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
report = sanitizeReportArtifact(report, options);
|
||||
return [
|
||||
`# ${options.title ?? "OpenClaw Plugin Compatibility Report"}`,
|
||||
"",
|
||||
@ -24,13 +26,20 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
["Warnings", report.summary.warningCount],
|
||||
["Compatibility suggestions", report.summary.suggestionCount],
|
||||
["Issue findings", report.summary.issueCount],
|
||||
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
|
||||
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
|
||||
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
|
||||
["P0 issues", report.summary.p0IssueCount],
|
||||
["P1 issues", report.summary.p1IssueCount],
|
||||
["Open P0 issues", report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
|
||||
["Open P1 issues", report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
|
||||
["Live issues", report.summary.liveIssueCount],
|
||||
["Live P0 issues", report.summary.liveP0IssueCount],
|
||||
["Compat gaps", report.summary.compatGapCount],
|
||||
["Deprecation warnings", report.summary.deprecationWarningCount],
|
||||
["Inspector gaps", report.summary.inspectorGapCount],
|
||||
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
|
||||
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
|
||||
["Upstream metadata", report.summary.upstreamIssueCount],
|
||||
["Contract probes", report.summary.contractProbeCount],
|
||||
["Decision rows", report.summary.decisionCount],
|
||||
@ -49,9 +58,9 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
options,
|
||||
),
|
||||
"",
|
||||
"## Live Issues",
|
||||
"## Other Live Issues",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
|
||||
"",
|
||||
"## Compat Gaps",
|
||||
"",
|
||||
@ -63,7 +72,11 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
"",
|
||||
"## Inspector Proof Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
|
||||
"",
|
||||
"## Runtime-Covered Inspector Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
|
||||
"",
|
||||
"## Upstream Metadata Issues",
|
||||
"",
|
||||
@ -127,6 +140,7 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
}
|
||||
|
||||
export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
report = sanitizeReportArtifact(report, options);
|
||||
return [
|
||||
`# ${options.title ?? "OpenClaw Plugin Issue Findings"}`,
|
||||
"",
|
||||
@ -138,13 +152,20 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
markdownTable(
|
||||
[
|
||||
["Issue findings", report.summary.issueCount],
|
||||
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
|
||||
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
|
||||
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
|
||||
[severityLabel("P0", options), report.summary.p0IssueCount],
|
||||
[severityLabel("P1", options), report.summary.p1IssueCount],
|
||||
[`Open ${severityLabel("P0", options)}`, report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
|
||||
[`Open ${severityLabel("P1", options)}`, report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
|
||||
["Live issues", report.summary.liveIssueCount],
|
||||
["Live P0 issues", report.summary.liveP0IssueCount],
|
||||
["Compat gaps", report.summary.compatGapCount],
|
||||
["Deprecation warnings", report.summary.deprecationWarningCount],
|
||||
["Inspector gaps", report.summary.inspectorGapCount],
|
||||
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
|
||||
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
|
||||
["Upstream metadata", report.summary.upstreamIssueCount],
|
||||
["Contract probes", report.summary.contractProbeCount],
|
||||
],
|
||||
@ -162,9 +183,9 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
options,
|
||||
),
|
||||
"",
|
||||
"## Live Issues",
|
||||
"## Other Live Issues",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
|
||||
"",
|
||||
"## Compat Gaps",
|
||||
"",
|
||||
@ -176,7 +197,11 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
"",
|
||||
"## Inspector Proof Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
|
||||
"",
|
||||
"## Runtime-Covered Inspector Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
|
||||
"",
|
||||
"## Upstream Metadata Issues",
|
||||
"",
|
||||
@ -225,6 +250,7 @@ function issueBlock(issue, options) {
|
||||
` - state: ${issueState(issue)}`,
|
||||
" - evidence:",
|
||||
...evidenceList(issue.evidence, options).map((item) => ` - ${item}`),
|
||||
...runtimeCoverageList(issue, options),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@ -232,6 +258,7 @@ function issueState(issue) {
|
||||
const flags = [
|
||||
issue.status,
|
||||
`compat:${issue.compatStatus ?? "none"}`,
|
||||
issue.runtimeCoverage?.status ? `runtime:${issue.runtimeCoverage.status}` : null,
|
||||
issue.live ? "live" : null,
|
||||
issue.deprecated ? "deprecated" : null,
|
||||
].filter(Boolean);
|
||||
@ -263,7 +290,7 @@ function triageOverview(report) {
|
||||
"inspector-gap",
|
||||
report.summary.inspectorGapCount,
|
||||
"-",
|
||||
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments.",
|
||||
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments. Runtime-covered rows are proof-backed and not open report work.",
|
||||
],
|
||||
[
|
||||
"upstream-metadata",
|
||||
@ -354,3 +381,15 @@ function evidenceList(evidence, options) {
|
||||
const formatEvidence = options.formatEvidence ?? ((item) => item);
|
||||
return items.map((item) => formatEvidence(item));
|
||||
}
|
||||
|
||||
function runtimeCoverageList(issue, options) {
|
||||
const runtimeCoverage = issue.runtimeCoverage;
|
||||
if (!runtimeCoverage) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
" - runtime coverage:",
|
||||
...evidenceList(runtimeCoverage.captured, options).map((item) => ` - captured ${item}`),
|
||||
...evidenceList(runtimeCoverage.artifacts, options).map((item) => ` - ${item}`),
|
||||
];
|
||||
}
|
||||
|
||||
@ -202,13 +202,43 @@ export function packageId(packageName) {
|
||||
if (!packageName) {
|
||||
return null;
|
||||
}
|
||||
return packageName
|
||||
const packageBase = packageName
|
||||
.split("/")
|
||||
.pop()
|
||||
.replace(/^openclaw-/, "")
|
||||
.replace(/[^a-zA-Z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.toLowerCase();
|
||||
.replace(/^openclaw-/, "");
|
||||
return trimHyphenEdges(collapsePackageIdSeparators(packageBase)).toLowerCase();
|
||||
}
|
||||
|
||||
function collapsePackageIdSeparators(value) {
|
||||
let result = "";
|
||||
let previousWasHyphen = false;
|
||||
for (const char of value) {
|
||||
if (isAsciiAlphaNumeric(char)) {
|
||||
result += char;
|
||||
previousWasHyphen = false;
|
||||
} else if (!previousWasHyphen) {
|
||||
result += "-";
|
||||
previousWasHyphen = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isAsciiAlphaNumeric(char) {
|
||||
const code = char.charCodeAt(0);
|
||||
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
|
||||
}
|
||||
|
||||
function trimHyphenEdges(value) {
|
||||
let start = 0;
|
||||
let end = value.length;
|
||||
while (start < end && value[start] === "-") {
|
||||
start += 1;
|
||||
}
|
||||
while (end > start && value[end - 1] === "-") {
|
||||
end -= 1;
|
||||
}
|
||||
return value.slice(start, end);
|
||||
}
|
||||
|
||||
export function inferPluginSeams(pluginManifest, packageJson) {
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
} from "./synthetic-probes.js";
|
||||
|
||||
export const defaultRegistrationAssertions = {
|
||||
createChatChannelPlugin: ["channel plugin id is stable", "channel factory metadata is captured"],
|
||||
defineChannelPluginEntry: ["channel id is stable", "setup/config schema can be read", "message envelope metadata is preserved"],
|
||||
definePluginEntry: ["entrypoint register function is callable", "entrypoint metadata is preserved"],
|
||||
registerChannel: ["channel id is stable", "inbound/outbound envelope shape is captured", "sender metadata is preserved"],
|
||||
|
||||
@ -157,10 +157,15 @@ function requireCompatRecordReconciliation(report, errors) {
|
||||
.filter((finding) => finding.code === "missing-compat-record")
|
||||
.map((finding) => `${finding.fixture}:${finding.compatRecord}`),
|
||||
);
|
||||
const compatGapRecords = new Set(
|
||||
report.issues
|
||||
.filter((issue) => issue.issueClass === "compat-gap" && issue.compatRecord)
|
||||
.map((issue) => `${issue.fixture}:${issue.compatRecord}`),
|
||||
);
|
||||
|
||||
for (const finding of [...report.warnings, ...report.suggestions].filter((item) => item.compatRecord)) {
|
||||
const key = `${finding.fixture}:${finding.compatRecord}`;
|
||||
if (!presentRecords.has(key) && !missingRecords.has(key)) {
|
||||
if (!presentRecords.has(key) && !missingRecords.has(key) && !compatGapRecords.has(key)) {
|
||||
errors.push(`${finding.fixture}: compat record ${finding.compatRecord} was not reconciled`);
|
||||
}
|
||||
}
|
||||
|
||||
@ -74,6 +74,31 @@ export const contractProbeRules = {
|
||||
contract: "OpenClaw package entrypoints resolve to files in the published or built plugin package.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-install-metadata-incomplete": {
|
||||
id: "package.metadata.install-release",
|
||||
contract: "Release publishing metadata declares canonical ClawHub and npm install specs.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-min-host-version-drift": {
|
||||
id: "package.metadata.min-host-version",
|
||||
contract: "Install minimum host version matches the OpenClaw package surface targeted by the plugin.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-npm-pack-entrypoint-missing": {
|
||||
id: "package.npm-pack.entrypoints",
|
||||
contract: "Advertised npm artifacts include every declared OpenClaw package entrypoint.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-npm-pack-metadata-missing": {
|
||||
id: "package.npm-pack.metadata",
|
||||
contract: "Advertised npm artifacts include OpenClaw manifest and package metadata.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-npm-pack-unavailable": {
|
||||
id: "package.npm-pack.available",
|
||||
contract: "Packages that advertise npm install support can produce an npm pack artifact.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-openclaw-entry-missing": {
|
||||
id: "package.entrypoint.openclaw-metadata",
|
||||
contract: "OpenClaw package metadata declares entrypoints for cold import and registration capture.",
|
||||
@ -106,6 +131,20 @@ export const contractProbeRules = {
|
||||
},
|
||||
};
|
||||
|
||||
const openClawOwnedProbeIssueCodes = new Set([
|
||||
"before-tool-call-probe",
|
||||
"channel-contract-probe",
|
||||
"conversation-access-hook",
|
||||
"registration-capture-gap",
|
||||
]);
|
||||
|
||||
export function compatRecordForIssueCode(code) {
|
||||
if (!openClawOwnedProbeIssueCodes.has(code)) {
|
||||
return undefined;
|
||||
}
|
||||
return contractProbeRules[code]?.id;
|
||||
}
|
||||
|
||||
export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
|
||||
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
|
||||
const probes = [];
|
||||
@ -136,7 +175,6 @@ export function probePriority(code, fixturePriority) {
|
||||
"before-tool-call-probe",
|
||||
"conversation-access-hook",
|
||||
"missing-compat-record",
|
||||
"registration-capture-gap",
|
||||
"sdk-export-missing",
|
||||
].includes(code)
|
||||
) {
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { compatRecordForIssueCode } from "./contract-probes.js";
|
||||
import { readJsonFile } from "./json-file.js";
|
||||
|
||||
const conversationAccessHooks = new Set(["agent_end", "llm_input", "llm_output"]);
|
||||
@ -17,9 +18,13 @@ const channelRegistrations = new Set([
|
||||
"defineChannelPluginEntry",
|
||||
"registerChannel",
|
||||
]);
|
||||
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
|
||||
const unsupportedSecurityManifestName = "openclaw.security.json";
|
||||
const unavailableSecurityManifestSchema = "https://openclaw.ai/schemas/plugin-security.json";
|
||||
|
||||
export async function buildCompatibilityFixtureReport({ fixture, inspection, checkoutPath, sourceRoot, rootDir = process.cwd() }) {
|
||||
const pluginManifests = await readPluginManifests({ checkoutPath, sourceRoot, rootDir });
|
||||
const securityManifests = await readSecurityManifests({ checkoutPath, sourceRoot, rootDir });
|
||||
const packageSummaries = await readPackageSummaries({ checkoutPath, sourceRoot, rootDir });
|
||||
const packageJson = selectPrimaryPackage(packageSummaries);
|
||||
const sdkImports = unique((inspection.sdkImports ?? []).map((sdkImport) => sdkImport.specifier));
|
||||
@ -39,6 +44,7 @@ export async function buildCompatibilityFixtureReport({ fixture, inspection, che
|
||||
manifestFiles: inspection.manifestFiles ?? [],
|
||||
sourceFiles: inspection.sourceFiles ?? [],
|
||||
pluginManifests,
|
||||
securityManifests,
|
||||
package: packageJson,
|
||||
packages: packageSummaries,
|
||||
sdkImports,
|
||||
@ -72,6 +78,46 @@ export async function readPluginManifests({ checkoutPath, sourceRoot, rootDir =
|
||||
return manifests;
|
||||
}
|
||||
|
||||
export async function readSecurityManifests({ checkoutPath, sourceRoot, rootDir = process.cwd() }) {
|
||||
const candidates = unique(
|
||||
[
|
||||
path.join(sourceRoot, unsupportedSecurityManifestName),
|
||||
path.join(checkoutPath, unsupportedSecurityManifestName),
|
||||
].filter(existsSync),
|
||||
);
|
||||
const manifests = [];
|
||||
|
||||
for (const manifestPath of candidates) {
|
||||
const relativePath = path.relative(rootDir, manifestPath);
|
||||
try {
|
||||
const manifest = await readJsonFile(manifestPath);
|
||||
manifests.push({
|
||||
path: relativePath,
|
||||
schema: typeof manifest.$schema === "string" ? manifest.$schema : null,
|
||||
version: typeof manifest.version === "string" ? manifest.version : null,
|
||||
plugin: typeof manifest.plugin === "string" ? manifest.plugin : null,
|
||||
expectedBehaviorCount: Array.isArray(manifest.expectedBehaviors)
|
||||
? manifest.expectedBehaviors.length
|
||||
: 0,
|
||||
securityNoteCount: Array.isArray(manifest.securityNotes) ? manifest.securityNotes.length : 0,
|
||||
validJson: true,
|
||||
});
|
||||
} catch {
|
||||
manifests.push({
|
||||
path: relativePath,
|
||||
schema: null,
|
||||
version: null,
|
||||
plugin: null,
|
||||
expectedBehaviorCount: 0,
|
||||
securityNoteCount: 0,
|
||||
validJson: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return manifests;
|
||||
}
|
||||
|
||||
export async function readPackageSummaries({ checkoutPath, sourceRoot, rootDir = process.cwd(), maxDepth = 3 }) {
|
||||
const candidates = unique([
|
||||
path.join(sourceRoot, "package.json"),
|
||||
@ -106,6 +152,9 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
|
||||
typeof packageJson.openclaw.build?.pluginSdkVersion === "string"
|
||||
? packageJson.openclaw.build.pluginSdkVersion
|
||||
: null,
|
||||
install: summarizeOpenClawInstall(packageJson.openclaw.install),
|
||||
release: summarizeOpenClawRelease(packageJson.openclaw.release),
|
||||
unsupportedMetadata: unsupportedOpenClawPackageMetadata(packageJson.openclaw),
|
||||
}
|
||||
: null;
|
||||
|
||||
@ -119,6 +168,7 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
|
||||
version: packageJson.version ?? null,
|
||||
type: packageJson.type ?? null,
|
||||
main: typeof packageJson.main === "string" ? packageJson.main : null,
|
||||
npmPack: summarizeNpmPack(packageJson, openclaw),
|
||||
dependencies: Object.keys(packageJson.dependencies ?? {}).sort(),
|
||||
peerDependencies: Object.keys(packageJson.peerDependencies ?? {}).sort(),
|
||||
optionalDependencies: Object.keys(packageJson.optionalDependencies ?? {}).sort(),
|
||||
@ -200,6 +250,79 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
|
||||
});
|
||||
}
|
||||
|
||||
if ((packageSummary.openclaw?.unsupportedMetadata ?? []).length > 0) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "package-openclaw-unsupported-metadata",
|
||||
level: "warning",
|
||||
message: "package declares unsupported OpenClaw metadata",
|
||||
evidence: packageSummary.openclaw.unsupportedMetadata,
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-metadata",
|
||||
action: "Remove unsupported OpenClaw metadata; native plugins use openclaw.plugin.json plus supported package openclaw fields.",
|
||||
evidence: packageSummary.openclaw.unsupportedMetadata.join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
const installMetadataIssues = packageInstallMetadataIssues(packageSummary);
|
||||
if (installMetadataIssues.length > 0) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "package-install-metadata-incomplete",
|
||||
level: "warning",
|
||||
message: "package OpenClaw install metadata does not match advertised release targets",
|
||||
evidence: installMetadataIssues,
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-metadata",
|
||||
action: "Ask the plugin to align openclaw.install metadata with openclaw.release publishing targets.",
|
||||
evidence: installMetadataIssues.join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
if (packageMinHostVersionDrift(packageSummary)) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "package-min-host-version-drift",
|
||||
level: "warning",
|
||||
message: "package openclaw.install.minHostVersion is not a semver floor for the target OpenClaw build version",
|
||||
evidence: [
|
||||
`minHostVersion:${packageSummary.openclaw.install.minHostVersion}`,
|
||||
`buildOpenClawVersion:${packageSummary.openclaw.buildOpenClawVersion}`,
|
||||
],
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-metadata",
|
||||
action: "Ask the plugin to publish install.minHostVersion as a semver floor for the OpenClaw package surface it targets.",
|
||||
evidence: packageSummary.path,
|
||||
});
|
||||
}
|
||||
|
||||
const npmPackIssues = packageNpmPackIssues(packageSummary, fixtureReport);
|
||||
for (const finding of npmPackIssues) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: finding.code,
|
||||
level: "warning",
|
||||
message: finding.message,
|
||||
evidence: finding.evidence,
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-artifact",
|
||||
action: "Ask the plugin to make its advertised npm install artifact match the published OpenClaw metadata.",
|
||||
evidence: finding.evidence.join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
if (packageSummary.openclaw && packageSummary.openclaw.entrypoints.length === 0) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
@ -217,9 +340,12 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
|
||||
});
|
||||
}
|
||||
|
||||
const missingEntrypoints = packageSummary.openclaw?.entrypoints.filter((entrypoint) => !entrypoint.exists) ?? [];
|
||||
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
|
||||
const missingEntrypoints = entrypoints.filter((entrypoint) => !entrypoint.exists);
|
||||
const buildEntrypoints = missingEntrypoints.filter((entrypoint) => entrypoint.requiresBuild);
|
||||
const plainMissingEntrypoints = missingEntrypoints.filter((entrypoint) => !entrypoint.requiresBuild);
|
||||
const plainMissingEntrypoints = missingEntrypoints.filter(
|
||||
(entrypoint) => !entrypoint.requiresBuild && !hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
|
||||
);
|
||||
|
||||
if (buildEntrypoints.length > 0) {
|
||||
suggestions.push({
|
||||
@ -278,7 +404,7 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
|
||||
...packageSummary.dependencies,
|
||||
...packageSummary.peerDependencies,
|
||||
...packageSummary.optionalDependencies,
|
||||
]);
|
||||
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
|
||||
if (packageSummary.openclaw?.entrypoints.length > 0 && runtimeDependencies.length > 0) {
|
||||
suggestions.push({
|
||||
fixture: fixture.id,
|
||||
@ -349,6 +475,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
suggestions.push(...packageContracts.suggestions);
|
||||
logs.push(...packageContracts.logs);
|
||||
decisions.push(...packageContracts.decisions);
|
||||
classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions });
|
||||
|
||||
for (const pluginManifest of fixtureReport.pluginManifests) {
|
||||
const providerAuthKeys = Object.keys(pluginManifest.providerAuthEnvVars ?? {});
|
||||
@ -399,6 +526,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "warning",
|
||||
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
|
||||
evidence: detailEvidence(conversationHookDetails),
|
||||
compatRecord: compatRecordForIssueCode("conversation-access-hook"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -459,6 +587,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
|
||||
evidence: detailEvidence(captureGapRegistrationDetails),
|
||||
compatRecord: compatRecordForIssueCode("registration-capture-gap"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -477,6 +606,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
|
||||
evidence: detailEvidence(hookDetails),
|
||||
compatRecord: compatRecordForIssueCode("before-tool-call-probe"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -500,6 +630,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "add channel envelope, config-schema, and runtime metadata probes",
|
||||
evidence: detailEvidence(channelRegistrationDetails),
|
||||
compatRecord: compatRecordForIssueCode("channel-contract-probe"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -551,6 +682,45 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
return { warnings, suggestions, logs, decisions };
|
||||
}
|
||||
|
||||
function classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions }) {
|
||||
for (const securityManifest of fixtureReport.securityManifests ?? []) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "unrecognized-security-manifest",
|
||||
level: "warning",
|
||||
message:
|
||||
"openclaw.security.json is not a supported OpenClaw or ClawHub security contract and is ignored by install safety checks",
|
||||
evidence: [securityManifest.path],
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "security-metadata",
|
||||
action:
|
||||
"Remove the advisory security manifest or replace it with a supported, versioned OpenClaw/ClawHub security contract once one exists.",
|
||||
evidence: securityManifest.path,
|
||||
});
|
||||
|
||||
if (securityManifest.schema === unavailableSecurityManifestSchema) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "security-manifest-schema-unavailable",
|
||||
level: "warning",
|
||||
message: "openclaw.security.json references an OpenClaw schema URL that is not currently published",
|
||||
evidence: [`${securityManifest.path}:$schema=${securityManifest.schema}`],
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "security-metadata",
|
||||
action:
|
||||
"Do not rely on the schema URL until OpenClaw publishes and documents a real plugin security metadata schema.",
|
||||
evidence: securityManifest.schema,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function registrationCaptureGapDetails(inspection, targetOpenClaw) {
|
||||
const apiRegistrationDetails = inspection.registrationDetails.filter((registration) =>
|
||||
registration.name.startsWith("register"),
|
||||
@ -753,6 +923,46 @@ function collectOpenClawEntrypoints(packageDir, openclaw, options) {
|
||||
});
|
||||
}
|
||||
|
||||
function hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
|
||||
if (!isSourceEntrypoint(entrypoint.specifier)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
|
||||
if (
|
||||
entrypoints.some(
|
||||
(candidate) =>
|
||||
candidate.exists &&
|
||||
candidate.requiresBuild &&
|
||||
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (entrypoint.kind === "extension" && entrypoints.some((candidate) => candidate.kind === "runtimeExtension" && candidate.exists)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const packageDir = path.dirname(packageSummary.path);
|
||||
return existsSync(path.resolve(packageDir, runtimeBuildSpecifier));
|
||||
}
|
||||
|
||||
function isSourceEntrypoint(specifier) {
|
||||
return /\.(?:ts|tsx)$/.test(specifier);
|
||||
}
|
||||
|
||||
function runtimeBuildSpecifierFor(specifier) {
|
||||
const normalized = normalizeEntrypointSpecifier(specifier);
|
||||
const basename = path.posix.basename(normalized).replace(/\.(?:ts|tsx)$/, ".js");
|
||||
return `./dist/${basename}`;
|
||||
}
|
||||
|
||||
function normalizeEntrypointSpecifier(specifier) {
|
||||
const normalized = specifier.replaceAll("\\", "/");
|
||||
return normalized.startsWith("./") ? normalized : `./${normalized}`;
|
||||
}
|
||||
|
||||
async function findPackageFiles(root, options, depth = 0) {
|
||||
if (!existsSync(root) || depth > options.maxDepth) {
|
||||
return [];
|
||||
@ -781,6 +991,269 @@ function selectPrimaryPackage(packages) {
|
||||
return packages[0] ?? null;
|
||||
}
|
||||
|
||||
function summarizeOpenClawInstall(install) {
|
||||
if (!install || typeof install !== "object") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
clawhubSpec: stringOrNull(install.clawhubSpec),
|
||||
npmSpec: stringOrNull(install.npmSpec),
|
||||
defaultChoice: stringOrNull(install.defaultChoice),
|
||||
minHostVersion: stringOrNull(install.minHostVersion),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeOpenClawRelease(release) {
|
||||
if (!release || typeof release !== "object") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
publishToClawHub: booleanOrNull(release.publishToClawHub),
|
||||
publishToNpm: booleanOrNull(release.publishToNpm),
|
||||
};
|
||||
}
|
||||
|
||||
function unsupportedOpenClawPackageMetadata(openclaw) {
|
||||
if (!openclaw || typeof openclaw !== "object") {
|
||||
return [];
|
||||
}
|
||||
return Object.keys(openclaw)
|
||||
.filter((key) => key === "bundle")
|
||||
.map((key) => `openclaw.${key}`);
|
||||
}
|
||||
|
||||
function summarizeNpmPack(packageJson, openclaw) {
|
||||
const files = arrayValues(packageJson.files).map(normalizePackagePath).filter((item) => item.length > 0);
|
||||
return {
|
||||
advertised: openclaw?.release?.publishToNpm === true || nonEmptyString(openclaw?.install?.npmSpec),
|
||||
private: packageJson.private === true,
|
||||
filesMode: files.length > 0 ? "allowlist" : "implicit",
|
||||
files,
|
||||
invalidFileSpecs: files.filter((item) => invalidPackageFileSpec(item)),
|
||||
};
|
||||
}
|
||||
|
||||
function packageInstallMetadataIssues(packageSummary) {
|
||||
const openclaw = packageSummary.openclaw;
|
||||
if (!openclaw) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const issues = [];
|
||||
const install = openclaw.install;
|
||||
const release = openclaw.release;
|
||||
const publishToClawHub = release?.publishToClawHub === true;
|
||||
const publishToNpm = release?.publishToNpm === true;
|
||||
|
||||
if (publishToClawHub && !nonEmptyString(install?.clawhubSpec)) {
|
||||
issues.push("openclaw.release.publishToClawHub requires openclaw.install.clawhubSpec");
|
||||
}
|
||||
if (publishToNpm && !nonEmptyString(install?.npmSpec)) {
|
||||
issues.push("openclaw.release.publishToNpm requires openclaw.install.npmSpec");
|
||||
}
|
||||
if (publishToNpm && nonEmptyString(install?.npmSpec) && nonEmptyString(packageSummary.name) && install.npmSpec !== packageSummary.name) {
|
||||
issues.push(`openclaw.install.npmSpec:${install.npmSpec} does not match package name:${packageSummary.name}`);
|
||||
}
|
||||
if (nonEmptyString(install?.defaultChoice) && !["clawhub", "npm"].includes(install.defaultChoice)) {
|
||||
issues.push(`openclaw.install.defaultChoice:${install.defaultChoice} must be clawhub or npm`);
|
||||
}
|
||||
if (install?.defaultChoice === "clawhub" && !nonEmptyString(install.clawhubSpec)) {
|
||||
issues.push("openclaw.install.defaultChoice clawhub requires openclaw.install.clawhubSpec");
|
||||
}
|
||||
if (install?.defaultChoice === "npm" && !nonEmptyString(install.npmSpec)) {
|
||||
issues.push("openclaw.install.defaultChoice npm requires openclaw.install.npmSpec");
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function packageNpmPackIssues(packageSummary, fixtureReport) {
|
||||
if (!packageSummary.npmPack?.advertised) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const findings = [];
|
||||
const unavailable = [];
|
||||
if (packageSummary.npmPack.private) {
|
||||
unavailable.push("package.json private:true");
|
||||
}
|
||||
if (!nonEmptyString(packageSummary.name)) {
|
||||
unavailable.push("package.json name missing");
|
||||
}
|
||||
if (!nonEmptyString(packageSummary.version)) {
|
||||
unavailable.push("package.json version missing");
|
||||
}
|
||||
unavailable.push(...packageSummary.npmPack.invalidFileSpecs.map((item) => `invalid files entry:${item}`));
|
||||
|
||||
if (unavailable.length > 0) {
|
||||
findings.push({
|
||||
code: "package-npm-pack-unavailable",
|
||||
message: "package advertises npm install or publish metadata but cannot produce a usable npm pack artifact",
|
||||
evidence: unavailable,
|
||||
});
|
||||
}
|
||||
|
||||
const missingMetadata = packageNpmPackMissingMetadata(packageSummary, fixtureReport);
|
||||
if (missingMetadata.length > 0) {
|
||||
findings.push({
|
||||
code: "package-npm-pack-metadata-missing",
|
||||
message: "advertised npm artifact would not include required OpenClaw package metadata",
|
||||
evidence: missingMetadata,
|
||||
});
|
||||
}
|
||||
|
||||
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
|
||||
const missingEntrypoints = entrypoints
|
||||
.filter(
|
||||
(entrypoint) =>
|
||||
!repoPathIncludedInNpmPack(packageSummary, entrypoint.relativePath) &&
|
||||
!hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
|
||||
)
|
||||
.map((entrypoint) => `${entrypoint.kind}:${entrypoint.specifier} -> ${entrypoint.relativePath}`) ?? [];
|
||||
if (missingEntrypoints.length > 0) {
|
||||
findings.push({
|
||||
code: "package-npm-pack-entrypoint-missing",
|
||||
message: "advertised npm artifact would not include declared OpenClaw entrypoints",
|
||||
evidence: missingEntrypoints,
|
||||
});
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
function packageNpmPackMissingMetadata(packageSummary, fixtureReport) {
|
||||
const missing = [];
|
||||
if (!repoPathIncludedInNpmPack(packageSummary, packageSummary.path)) {
|
||||
missing.push(packageSummary.path);
|
||||
}
|
||||
|
||||
for (const manifest of fixtureReport.pluginManifests ?? []) {
|
||||
if (repoPathWithinPackage(packageSummary, manifest.path) && !repoPathIncludedInNpmPack(packageSummary, manifest.path)) {
|
||||
missing.push(manifest.path);
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
function hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
|
||||
if (!isSourceEntrypoint(entrypoint.specifier)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
|
||||
const matchingRuntimeEntrypoint = entrypoints.find(
|
||||
(candidate) =>
|
||||
candidate.requiresBuild &&
|
||||
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
|
||||
);
|
||||
if (matchingRuntimeEntrypoint && repoPathIncludedInNpmPack(packageSummary, matchingRuntimeEntrypoint.relativePath)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
entrypoint.kind === "extension" &&
|
||||
entrypoints.some(
|
||||
(candidate) => candidate.kind === "runtimeExtension" && repoPathIncludedInNpmPack(packageSummary, candidate.relativePath),
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
|
||||
const runtimeBuildPath = path.posix.join(packageDir === "." ? "" : packageDir, normalizeEntrypointSpecifier(runtimeBuildSpecifier));
|
||||
return repoPathIncludedInNpmPack(packageSummary, runtimeBuildPath);
|
||||
}
|
||||
|
||||
function packageMinHostVersionDrift(packageSummary) {
|
||||
const openclaw = packageSummary.openclaw;
|
||||
if (!nonEmptyString(openclaw?.install?.minHostVersion) || !nonEmptyString(openclaw?.buildOpenClawVersion)) {
|
||||
return false;
|
||||
}
|
||||
return parseMinHostVersionFloor(openclaw.install.minHostVersion) !== openclaw.buildOpenClawVersion;
|
||||
}
|
||||
|
||||
function parseMinHostVersionFloor(value) {
|
||||
const match = /^>=([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/.exec(value);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function repoPathIncludedInNpmPack(packageSummary, repoPath) {
|
||||
const packageRelativePath = packageRelativeRepoPath(packageSummary, repoPath);
|
||||
if (!packageRelativePath) {
|
||||
return false;
|
||||
}
|
||||
if (npmAlwaysPacksPath(packageRelativePath)) {
|
||||
return true;
|
||||
}
|
||||
if (packageSummary.npmPack?.filesMode !== "allowlist") {
|
||||
return true;
|
||||
}
|
||||
return packageSummary.npmPack.files.some((spec) => packageFileSpecIncludesPath(spec, packageRelativePath));
|
||||
}
|
||||
|
||||
function repoPathWithinPackage(packageSummary, repoPath) {
|
||||
return packageRelativeRepoPath(packageSummary, repoPath) !== null;
|
||||
}
|
||||
|
||||
function packageRelativeRepoPath(packageSummary, repoPath) {
|
||||
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
|
||||
const normalized = normalizeRepoPath(repoPath);
|
||||
if (packageDir === ".") {
|
||||
return normalized;
|
||||
}
|
||||
if (normalized === packageDir) {
|
||||
return "";
|
||||
}
|
||||
return normalized.startsWith(`${packageDir}/`) ? normalized.slice(packageDir.length + 1) : null;
|
||||
}
|
||||
|
||||
function npmAlwaysPacksPath(packageRelativePath) {
|
||||
const base = path.posix.basename(packageRelativePath).toLowerCase();
|
||||
return packageRelativePath === "package.json" || /^readme(?:\..*)?$/u.test(base) || /^licen[cs]e(?:\..*)?$/u.test(base);
|
||||
}
|
||||
|
||||
function packageFileSpecIncludesPath(spec, packageRelativePath) {
|
||||
if (spec === "." || spec === packageRelativePath) {
|
||||
return true;
|
||||
}
|
||||
if (spec.includes("*")) {
|
||||
return globLikeSpecIncludesPath(spec, packageRelativePath);
|
||||
}
|
||||
return packageRelativePath.startsWith(`${spec.replace(/\/$/u, "")}/`);
|
||||
}
|
||||
|
||||
function globLikeSpecIncludesPath(spec, packageRelativePath) {
|
||||
return globSegmentsIncludePath(spec.split("/"), packageRelativePath.split("/"));
|
||||
}
|
||||
|
||||
function globSegmentsIncludePath(specSegments, packageSegments) {
|
||||
if (specSegments.length === 0) {
|
||||
return packageSegments.length === 0;
|
||||
}
|
||||
const [head, ...tail] = specSegments;
|
||||
if (head === "**") {
|
||||
return globSegmentsIncludePath(tail, packageSegments) || (packageSegments.length > 0 && globSegmentsIncludePath(specSegments, packageSegments.slice(1)));
|
||||
}
|
||||
if (packageSegments.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return globSegmentIncludesPath(head, packageSegments[0]) && globSegmentsIncludePath(tail, packageSegments.slice(1));
|
||||
}
|
||||
|
||||
function globSegmentIncludesPath(specSegment, packageSegment) {
|
||||
const escaped = specSegment.replace(/[.+?^${}()|[\]\\]/gu, "\\$&").replaceAll("*", ".*");
|
||||
return new RegExp(`^${escaped}$`, "u").test(packageSegment);
|
||||
}
|
||||
|
||||
function invalidPackageFileSpec(spec) {
|
||||
return spec.startsWith("/") || spec === ".." || spec.startsWith("../") || spec.includes("/../");
|
||||
}
|
||||
|
||||
function normalizePackagePath(value) {
|
||||
return normalizeRepoPath(value).replace(/^\.\/+/u, "").replace(/\/$/u, "");
|
||||
}
|
||||
|
||||
function packageRank(packageSummary) {
|
||||
if (packageSummary.openclaw?.entrypoints.length > 0) {
|
||||
return 0;
|
||||
@ -795,6 +1268,22 @@ function arrayValues(value) {
|
||||
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
function stringOrNull(value) {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function booleanOrNull(value) {
|
||||
return typeof value === "boolean" ? value : null;
|
||||
}
|
||||
|
||||
function nonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeRepoPath(value) {
|
||||
return String(value ?? "").replaceAll("\\", "/").replace(/^\.\/+/u, "");
|
||||
}
|
||||
|
||||
function detailEvidence(details, key = "name") {
|
||||
return unique(details.map((detail) => `${detail[key]} @ ${detail.ref}`));
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
||||
@ -24,22 +24,48 @@ export async function buildImportLoopProfile(options = {}) {
|
||||
const entrypoint = options.entrypoint ?? defaultImportLoopProfileOptions.entrypoint;
|
||||
assertRunCount(runs, 20);
|
||||
|
||||
const baseline = await buildBaselineProfile({ ...options, rootDir, runs });
|
||||
const samples = [];
|
||||
for (let index = 0; index < runs; index += 1) {
|
||||
samples.push(await runCaptureSample({ ...options, entrypoint, index, rootDir }));
|
||||
const sample = await runCaptureSample({ ...options, entrypoint, index, rootDir });
|
||||
samples.push(applyBaselineAdjustment(sample, baseline));
|
||||
}
|
||||
|
||||
const wallMs = samples.map((sample) => sample.wallMs).sort((left, right) => left - right);
|
||||
const pluginWallDeltaMs = samples.map((sample) => sample.pluginWallDeltaMs).sort((left, right) => left - right);
|
||||
const openClawImportMs = openClawLifecycleMetric(samples, "importMs");
|
||||
const openClawActivationMs = openClawLifecycleMetric(samples, "activationMs");
|
||||
const rssSampleCount = samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)), 0);
|
||||
const cpuSampleCount = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
|
||||
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
|
||||
return {
|
||||
generatedAt: options.generatedAt ?? defaultImportLoopProfileOptions.generatedAt,
|
||||
mode: options.mode ?? "subprocess-cold-import-loop",
|
||||
mode: options.mode ?? "baseline-adjusted-cold-capture-loop",
|
||||
entrypoint,
|
||||
baseline,
|
||||
summary: {
|
||||
runs,
|
||||
baselineRuns: baseline.runs,
|
||||
baselineFailCount: baseline.failCount,
|
||||
p50WallMs: percentile(wallMs, 0.5),
|
||||
p95WallMs: percentile(wallMs, 0.95),
|
||||
p50PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.5),
|
||||
p95PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.95),
|
||||
openClawLifecycleCount: openClawImportMs.length,
|
||||
p50OpenClawImportMs: percentile(openClawImportMs, 0.5),
|
||||
p95OpenClawImportMs: percentile(openClawImportMs, 0.95),
|
||||
p50OpenClawActivationMs: percentile(openClawActivationMs, 0.5),
|
||||
p95OpenClawActivationMs: percentile(openClawActivationMs, 0.95),
|
||||
maxPeakRssMb: Math.max(0, ...samples.map((sample) => sample.peakRssMb)),
|
||||
maxCpuMsEstimate: Math.max(0, ...samples.map((sample) => sample.cpuMsEstimate)),
|
||||
maxPluginPeakRssDeltaMb: Math.max(0, ...samples.map((sample) => sample.pluginPeakRssDeltaMb)),
|
||||
maxPluginCpuDeltaMsEstimate: Math.max(0, ...samples.map((sample) => sample.pluginCpuDeltaMsEstimate)),
|
||||
baselineReferenceWallMs: baseline.reference.wallMs,
|
||||
baselineReferencePeakRssMb: baseline.reference.peakRssMb,
|
||||
baselineReferenceCpuMsEstimate: baseline.reference.cpuMsEstimate,
|
||||
statSampleCount,
|
||||
rssSampleCount,
|
||||
cpuSampleCount,
|
||||
capturedCount: samples.reduce((sum, sample) => sum + sample.capturedCount, 0),
|
||||
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
|
||||
},
|
||||
@ -52,6 +78,9 @@ export function validateImportLoopProfile(report) {
|
||||
if (report.summary.failCount > 0) {
|
||||
errors.push(`import loop has ${report.summary.failCount} failed sample(s)`);
|
||||
}
|
||||
if ((report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0) > 0) {
|
||||
errors.push("import loop baseline capture failed");
|
||||
}
|
||||
if (report.summary.capturedCount < report.summary.runs) {
|
||||
errors.push("import loop did not capture at least one contract per run");
|
||||
}
|
||||
@ -85,7 +114,11 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
markdownTable(Object.entries(report.summary).map(([key, value]) => [key, value]), ["Metric", "Value"]),
|
||||
markdownTable(summaryRows(report), ["Metric", "Value"]),
|
||||
"",
|
||||
"## Harness Baseline",
|
||||
"",
|
||||
markdownTable(baselineRows(report), ["Metric", "Value"]),
|
||||
"",
|
||||
"## Samples",
|
||||
"",
|
||||
@ -94,22 +127,132 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
|
||||
sample.index,
|
||||
sample.status,
|
||||
sample.capturedCount,
|
||||
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.importMs),
|
||||
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.activationMs),
|
||||
formatOptionalMetric(sample.pluginWallDeltaMs, "ms"),
|
||||
formatSampledMetric(sample.pluginPeakRssDeltaMb, sample.rssSampleCount),
|
||||
formatSampledMetric(sample.pluginCpuDeltaMsEstimate, sample.cpuSampleCount, "ms"),
|
||||
`${sample.wallMs} ms`,
|
||||
`${sample.peakRssMb} MB`,
|
||||
`${sample.cpuMsEstimate} ms`,
|
||||
formatSampledMetric(sample.peakRssMb, sample.rssSampleCount),
|
||||
formatSampledMetric(sample.cpuMsEstimate, sample.cpuSampleCount, "ms"),
|
||||
`${sample.rssSampleCount ?? 0}/${sample.cpuSampleCount ?? 0}`,
|
||||
sample.exitCode,
|
||||
]),
|
||||
["Run", "Status", "Captured", "Wall", "Peak RSS", "CPU Estimate", "Exit"],
|
||||
[
|
||||
"Run",
|
||||
"Status",
|
||||
"Captured",
|
||||
"OpenClaw Import",
|
||||
"OpenClaw Activate",
|
||||
"Plugin Wall Delta",
|
||||
"Plugin RSS Delta",
|
||||
"Plugin CPU Delta",
|
||||
"Raw Wall",
|
||||
"Raw Peak RSS",
|
||||
"Raw CPU Estimate",
|
||||
"RSS/CPU samples",
|
||||
"Exit",
|
||||
],
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function buildBaselineProfile(options) {
|
||||
const baselineRuns = options.baseline === false ? 0 : options.baselineRuns ?? Math.min(options.runs, 3);
|
||||
if (baselineRuns <= 0) {
|
||||
return emptyBaseline();
|
||||
}
|
||||
|
||||
const entrypoint = await writeBaselineEntrypoint(options);
|
||||
const samples = [];
|
||||
for (let index = 0; index < baselineRuns; index += 1) {
|
||||
samples.push(
|
||||
await runCaptureSample({
|
||||
...options,
|
||||
entrypoint,
|
||||
index,
|
||||
sampleName: "baseline",
|
||||
rootDir: options.rootDir,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const wallMs = sortedMetric(samples, "wallMs");
|
||||
const peakRssMb = sortedMetric(samples, "peakRssMb");
|
||||
const cpuMsEstimate = sortedMetric(samples, "cpuMsEstimate");
|
||||
return {
|
||||
mode: "minimal-plugin-capture",
|
||||
runs: baselineRuns,
|
||||
entrypoint: path.relative(options.rootDir, entrypoint),
|
||||
reference: {
|
||||
wallMs: percentile(wallMs, 0.5),
|
||||
peakRssMb: percentile(peakRssMb, 0.5),
|
||||
cpuMsEstimate: percentile(cpuMsEstimate, 0.5),
|
||||
},
|
||||
max: {
|
||||
wallMs: wallMs.at(-1) ?? 0,
|
||||
peakRssMb: peakRssMb.at(-1) ?? 0,
|
||||
cpuMsEstimate: cpuMsEstimate.at(-1) ?? 0,
|
||||
},
|
||||
statSampleCount: samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0),
|
||||
rssSampleCount: samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? 0), 0),
|
||||
cpuSampleCount: samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0),
|
||||
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
|
||||
samples,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyBaseline() {
|
||||
return {
|
||||
mode: "disabled",
|
||||
runs: 0,
|
||||
entrypoint: null,
|
||||
reference: {
|
||||
wallMs: 0,
|
||||
peakRssMb: 0,
|
||||
cpuMsEstimate: 0,
|
||||
},
|
||||
max: {
|
||||
wallMs: 0,
|
||||
peakRssMb: 0,
|
||||
cpuMsEstimate: 0,
|
||||
},
|
||||
statSampleCount: 0,
|
||||
rssSampleCount: 0,
|
||||
cpuSampleCount: 0,
|
||||
failCount: 0,
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function writeBaselineEntrypoint(options) {
|
||||
const outputDir = resolveFromRoot(
|
||||
options.rootDir,
|
||||
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
|
||||
);
|
||||
const baselinePath = path.join(outputDir, "baseline-plugin.mjs");
|
||||
await mkdir(path.dirname(baselinePath), { recursive: true });
|
||||
await writeFile(
|
||||
baselinePath,
|
||||
[
|
||||
"export default {",
|
||||
" register(api) {",
|
||||
" api.registerTool({ name: 'baseline_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
" },",
|
||||
"};",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
return baselinePath;
|
||||
}
|
||||
|
||||
async function runCaptureSample(options) {
|
||||
const outputDir = resolveFromRoot(
|
||||
options.rootDir,
|
||||
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
|
||||
);
|
||||
const outputPath = path.join(outputDir, `capture-${options.index}.json`);
|
||||
const outputPath = path.join(outputDir, `${options.sampleName ?? "capture"}-${options.index}.json`);
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
|
||||
const command = buildCaptureCommand({ ...options, outputPath });
|
||||
@ -126,14 +269,125 @@ async function runCaptureSample(options) {
|
||||
exitCode: profile.exitCode,
|
||||
status: output?.status ?? "failed",
|
||||
capturedCount: output?.captured?.length ?? 0,
|
||||
openClawLifecycle: output?.openClawLifecycle ?? null,
|
||||
wallMs: profile.wallMs,
|
||||
peakRssMb: profile.peakRssMb,
|
||||
peakCpuPercent: profile.peakCpuPercent,
|
||||
cpuMsEstimate: profile.cpuMsEstimate,
|
||||
statSampleCount: profile.statSampleCount,
|
||||
rssSampleCount: profile.rssSampleCount,
|
||||
cpuSampleCount: profile.cpuSampleCount,
|
||||
stderrPreview: profile.stderrPreview,
|
||||
};
|
||||
}
|
||||
|
||||
function summaryRows(report) {
|
||||
return [
|
||||
["runs", report.summary.runs],
|
||||
["baselineRuns", report.summary.baselineRuns ?? report.baseline?.runs ?? 0],
|
||||
["baselineFailCount", report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0],
|
||||
["p50WallMs", report.summary.p50WallMs],
|
||||
["p95WallMs", report.summary.p95WallMs],
|
||||
...(Number.isFinite(report.summary.p50PluginWallDeltaMs)
|
||||
? [
|
||||
["p50PluginWallDeltaMs", report.summary.p50PluginWallDeltaMs],
|
||||
["p95PluginWallDeltaMs", report.summary.p95PluginWallDeltaMs],
|
||||
["maxPluginPeakRssDeltaMb", formatSampledMetric(report.summary.maxPluginPeakRssDeltaMb, report.summary.rssSampleCount)],
|
||||
[
|
||||
"maxPluginCpuDeltaMsEstimate",
|
||||
formatSampledMetric(report.summary.maxPluginCpuDeltaMsEstimate, report.summary.cpuSampleCount, "ms"),
|
||||
],
|
||||
]
|
||||
: []),
|
||||
...((report.summary.openClawLifecycleCount ?? 0) > 0
|
||||
? [
|
||||
["openClawLifecycleCount", report.summary.openClawLifecycleCount],
|
||||
["p50OpenClawImportMs", `${report.summary.p50OpenClawImportMs} ms`],
|
||||
["p95OpenClawImportMs", `${report.summary.p95OpenClawImportMs} ms`],
|
||||
["p50OpenClawActivationMs", `${report.summary.p50OpenClawActivationMs} ms`],
|
||||
["p95OpenClawActivationMs", `${report.summary.p95OpenClawActivationMs} ms`],
|
||||
]
|
||||
: []),
|
||||
["maxPeakRssMb", formatSampledMetric(report.summary.maxPeakRssMb, report.summary.rssSampleCount)],
|
||||
["maxCpuMsEstimate", formatSampledMetric(report.summary.maxCpuMsEstimate, report.summary.cpuSampleCount, "ms")],
|
||||
...(Number.isFinite(report.summary.baselineReferenceWallMs)
|
||||
? [
|
||||
["baselineReferenceWallMs", `${report.summary.baselineReferenceWallMs} ms`],
|
||||
["baselineReferencePeakRssMb", formatSampledMetric(report.summary.baselineReferencePeakRssMb, report.baseline?.rssSampleCount ?? 0)],
|
||||
[
|
||||
"baselineReferenceCpuMsEstimate",
|
||||
formatSampledMetric(report.summary.baselineReferenceCpuMsEstimate, report.baseline?.cpuSampleCount ?? 0, "ms"),
|
||||
],
|
||||
]
|
||||
: []),
|
||||
["statSampleCount", report.summary.statSampleCount ?? 0],
|
||||
["rssSampleCount", report.summary.rssSampleCount ?? 0],
|
||||
["cpuSampleCount", report.summary.cpuSampleCount ?? 0],
|
||||
["capturedCount", report.summary.capturedCount],
|
||||
["failCount", report.summary.failCount],
|
||||
];
|
||||
}
|
||||
|
||||
function baselineRows(report) {
|
||||
const baseline = report.baseline ?? emptyBaseline();
|
||||
return [
|
||||
["mode", baseline.mode],
|
||||
["runs", baseline.runs],
|
||||
["entrypoint", baseline.entrypoint ?? "-"],
|
||||
["referenceWallMs", `${baseline.reference?.wallMs ?? 0} ms`],
|
||||
["referencePeakRssMb", formatSampledMetric(baseline.reference?.peakRssMb ?? 0, baseline.rssSampleCount)],
|
||||
["referenceCpuMsEstimate", formatSampledMetric(baseline.reference?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
|
||||
["maxWallMs", `${baseline.max?.wallMs ?? 0} ms`],
|
||||
["maxPeakRssMb", formatSampledMetric(baseline.max?.peakRssMb ?? 0, baseline.rssSampleCount)],
|
||||
["maxCpuMsEstimate", formatSampledMetric(baseline.max?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
|
||||
["statSampleCount", baseline.statSampleCount ?? 0],
|
||||
["failCount", baseline.failCount ?? 0],
|
||||
];
|
||||
}
|
||||
|
||||
function formatSampledMetric(value, count, unit = "MB") {
|
||||
if ((count ?? 0) <= 0) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
function formatOptionalMetric(value, unit) {
|
||||
if (!Number.isFinite(value)) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
function formatOpenClawLifecycleMetric(value) {
|
||||
return Number.isFinite(value) ? `${value} ms` : "n/a";
|
||||
}
|
||||
|
||||
function openClawLifecycleMetric(samples, field) {
|
||||
return samples
|
||||
.map((sample) => sample.openClawLifecycle?.[field])
|
||||
.filter((value) => Number.isFinite(value))
|
||||
.sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
function applyBaselineAdjustment(sample, baseline) {
|
||||
return {
|
||||
...sample,
|
||||
pluginWallDeltaMs: roundNonNegative(sample.wallMs - baseline.reference.wallMs, 0),
|
||||
pluginPeakRssDeltaMb: roundNonNegative(sample.peakRssMb - baseline.reference.peakRssMb, 1),
|
||||
pluginCpuDeltaMsEstimate: roundNonNegative(sample.cpuMsEstimate - baseline.reference.cpuMsEstimate, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function sortedMetric(samples, field) {
|
||||
return samples.map((sample) => sample[field]).sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
function roundNonNegative(value, digits) {
|
||||
const scale = 10 ** digits;
|
||||
return Math.max(0, Math.round(value * scale) / scale);
|
||||
}
|
||||
|
||||
function buildCaptureCommand(options) {
|
||||
if (typeof options.captureCommand === "function") {
|
||||
return options.captureCommand({
|
||||
|
||||
13
src/index.js
13
src/index.js
@ -13,6 +13,7 @@ import * as profileDiffApi from "./profile-diff.js";
|
||||
import * as refDiffApi from "./ref-diff.js";
|
||||
import * as reportApi from "./report.js";
|
||||
import * as runtimeProfileApi from "./runtime-profile.js";
|
||||
import * as runtimeReconciliationApi from "./runtime-reconciliation.js";
|
||||
import * as syntheticProbeSuiteApi from "./synthetic-probe-suite.js";
|
||||
import * as syntheticProbesApi from "./synthetic-probes.js";
|
||||
|
||||
@ -53,11 +54,13 @@ export const staticInspection = Object.freeze({
|
||||
export const reports = Object.freeze({
|
||||
renderMarkdown: reportApi.renderMarkdownReport,
|
||||
renderTextSummary: pluginApi.renderTextSummary,
|
||||
sanitizeArtifact: reportApi.sanitizeReportArtifact,
|
||||
write: reportApi.writeReport,
|
||||
issueId: issuesApi.issueId,
|
||||
classifyIssueFinding: issuesApi.classifyIssueFinding,
|
||||
knownIssueCodes: issuesApi.knownIssueCodes,
|
||||
openClawTargetPathCandidates: openClawTargetApi.openClawTargetPathCandidates,
|
||||
readOpenClawTargetSurface: openClawTargetApi.readOpenClawTargetSurface,
|
||||
});
|
||||
|
||||
export const contracts = Object.freeze({
|
||||
@ -109,6 +112,8 @@ export const runtime = Object.freeze({
|
||||
writeImportLoopProfile: importLoopProfileApi.writeImportLoopProfile,
|
||||
renderImportLoopProfile: importLoopProfileApi.renderImportLoopProfileMarkdown,
|
||||
validateImportLoopProfile: importLoopProfileApi.validateImportLoopProfile,
|
||||
applyExecutionCoverage: runtimeReconciliationApi.applyRuntimeExecutionCoverage,
|
||||
buildExecutionCoverage: runtimeReconciliationApi.buildRuntimeExecutionCoverage,
|
||||
});
|
||||
|
||||
export const synthetic = Object.freeze({
|
||||
@ -200,7 +205,7 @@ export {
|
||||
} from "./import-loop-profile.js";
|
||||
export { classifyIssueFinding, issueId, knownIssueCodes } from "./issues.js";
|
||||
export { inspectFixtureSet, inspectPlugin, inspectSourceText } from "./inspector.js";
|
||||
export { openClawTargetPathCandidates } from "./openclaw-target.js";
|
||||
export { openClawTargetPathCandidates, readOpenClawTargetSurface } from "./openclaw-target.js";
|
||||
export {
|
||||
buildProfileDiff,
|
||||
defaultProfileDiffOptions,
|
||||
@ -216,7 +221,7 @@ export {
|
||||
validateRefDiff,
|
||||
writeRefDiff,
|
||||
} from "./ref-diff.js";
|
||||
export { renderMarkdownReport, writeReport } from "./report.js";
|
||||
export { renderMarkdownReport, sanitizeReportArtifact, writeReport } from "./report.js";
|
||||
export {
|
||||
buildRuntimeProfile,
|
||||
defaultRuntimeProfileCommands,
|
||||
@ -225,6 +230,10 @@ export {
|
||||
validateRuntimeProfile,
|
||||
writeRuntimeProfile,
|
||||
} from "./runtime-profile.js";
|
||||
export {
|
||||
applyRuntimeExecutionCoverage,
|
||||
buildRuntimeExecutionCoverage,
|
||||
} from "./runtime-reconciliation.js";
|
||||
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
|
||||
export {
|
||||
buildSyntheticProbePlan,
|
||||
|
||||
@ -5,12 +5,16 @@ import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { createCaptureApi } from "./capture-api.js";
|
||||
import { captureApiOptionsForPlugin } from "./capture-config.js";
|
||||
import { fixtureCheckoutPath, fixtureSourceRoot } from "./config.js";
|
||||
import { buildCompatibilityFixtureReport } from "./fixture-summary.js";
|
||||
import { readOpenClawTargetSurface } from "./openclaw-target.js";
|
||||
import { buildCompatibilityReport, buildReport } from "./report.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const registrationEquivalents = new Map([
|
||||
["registerChannel", new Set(["createChatChannelPlugin", "defineBundledChannelEntry", "defineChannelPluginEntry", "registerChannel"])],
|
||||
]);
|
||||
|
||||
export async function inspectFixtureSet(config, options = {}) {
|
||||
const { inspections, failures } = await inspectConfiguredFixtures(config, options);
|
||||
@ -32,6 +36,7 @@ export async function inspectCompatibilityFixtureSet(config, options = {}) {
|
||||
inspections,
|
||||
failures,
|
||||
generatedAt: options.generatedAt,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw,
|
||||
buildFixtureReport: ({ fixture, inspection }) =>
|
||||
buildCompatibilityFixtureReport({
|
||||
@ -58,7 +63,7 @@ async function inspectConfiguredFixtures(config, options = {}) {
|
||||
["manifestContracts", inspection.manifestContracts],
|
||||
]) {
|
||||
const expected = fixture.expect?.[key] ?? [];
|
||||
const missing = expected.filter((value) => !observed.includes(value));
|
||||
const missing = expected.filter((value) => !satisfiesExpectedSeam(key, value, observed));
|
||||
if (missing.length > 0) {
|
||||
failures.push(`${fixture.id}: missing ${key}: ${missing.join(", ")}`);
|
||||
}
|
||||
@ -68,6 +73,17 @@ async function inspectConfiguredFixtures(config, options = {}) {
|
||||
return { inspections, failures };
|
||||
}
|
||||
|
||||
function satisfiesExpectedSeam(key, expected, observed) {
|
||||
if (observed.includes(expected)) {
|
||||
return true;
|
||||
}
|
||||
if (key !== "registrations") {
|
||||
return false;
|
||||
}
|
||||
const equivalents = registrationEquivalents.get(expected);
|
||||
return Boolean(equivalents && observed.some((value) => equivalents.has(value)));
|
||||
}
|
||||
|
||||
export async function inspectPlugin(fixture, options = {}) {
|
||||
const config = options.config ?? { rootDir: options.rootDir ?? process.cwd() };
|
||||
const checkoutPath = fixtureCheckoutPath(config, fixture);
|
||||
@ -132,6 +148,7 @@ export function inspectSourceText(text, filePath = "source.js") {
|
||||
const hooks = collectDetailedMatches(searchableText, /\bapi\.on\(\s*["'`]([^"'`]+)["'`]/g, filePath, "name");
|
||||
const registrations = [
|
||||
...collectDetailedMatches(searchableText, /\bapi\.(register[A-Za-z0-9]+)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(defineBundledChannelEntry)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(defineChannelPluginEntry)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(createChatChannelPlugin)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(definePluginEntry)\s*\(/g, filePath, "name"),
|
||||
@ -172,7 +189,12 @@ export async function captureEntrypoint(entrypoint, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
const api = createCaptureApi(options.apiOptions);
|
||||
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
|
||||
pluginRoot: options.pluginRoot
|
||||
? path.resolve(options.cwd ?? process.cwd(), options.pluginRoot)
|
||||
: path.dirname(resolvedEntrypoint),
|
||||
});
|
||||
const api = createCaptureApi(apiOptions);
|
||||
try {
|
||||
await register(api);
|
||||
} catch (error) {
|
||||
@ -183,7 +205,7 @@ export async function captureEntrypoint(entrypoint, options = {}) {
|
||||
entrypoint: resolvedEntrypoint,
|
||||
captured: api.getCapturedContracts(),
|
||||
};
|
||||
if (options.apiOptions?.retainHandlers === true) {
|
||||
if (apiOptions?.retainHandlers === true) {
|
||||
result.retained = api.getRetainedContracts();
|
||||
}
|
||||
return result;
|
||||
@ -212,10 +234,34 @@ export async function captureEntrypointWithMockSdk(entrypoint, options = {}) {
|
||||
);
|
||||
return JSON.parse(stdout);
|
||||
} catch (error) {
|
||||
const captured = parseCaptureResultFromStdout(error?.stdout);
|
||||
if (captured) {
|
||||
return captured;
|
||||
}
|
||||
throw classifyMockSdkCaptureError(error);
|
||||
}
|
||||
}
|
||||
|
||||
function parseCaptureResultFromStdout(stdout) {
|
||||
if (!stdout) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(stdout);
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === "object" &&
|
||||
typeof parsed.status === "string" &&
|
||||
Array.isArray(parsed.captured)
|
||||
) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function classifyMockSdkCaptureError(error) {
|
||||
const rawMessage = [error?.stderr, error?.stdout, error?.message].filter(Boolean).join("\n");
|
||||
const missingExport = rawMessage.match(/does not provide an export named ['"]([^'"]+)['"]/)?.[1];
|
||||
@ -457,9 +503,38 @@ function lineForOffset(text, offset) {
|
||||
}
|
||||
|
||||
function stripComments(text) {
|
||||
return text
|
||||
.replace(/\/\*[\s\S]*?\*\//g, (comment) => comment.replace(/[^\n]/g, " "))
|
||||
.replace(/\/\/.*$/gm, (comment) => " ".repeat(comment.length));
|
||||
let result = "";
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
const next = text[index + 1];
|
||||
if (char === "/" && next === "*") {
|
||||
result += " ";
|
||||
index += 2;
|
||||
while (index < text.length && !(text[index] === "*" && text[index + 1] === "/")) {
|
||||
result += blankCommentChar(text[index]);
|
||||
index += 1;
|
||||
}
|
||||
if (index < text.length) {
|
||||
result += " ";
|
||||
index += 1;
|
||||
}
|
||||
} else if (char === "/" && next === "/") {
|
||||
result += " ";
|
||||
index += 2;
|
||||
while (index < text.length && text[index] !== "\n" && text[index] !== "\r") {
|
||||
result += " ";
|
||||
index += 1;
|
||||
}
|
||||
index -= 1;
|
||||
} else {
|
||||
result += char;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function blankCommentChar(char) {
|
||||
return char === "\n" || char === "\r" ? char : " ";
|
||||
}
|
||||
|
||||
function sortDetails(details) {
|
||||
|
||||
@ -23,17 +23,25 @@ export const knownIssueCodes = new Set([
|
||||
"package-build-artifact-entrypoint",
|
||||
"package-dependency-install-required",
|
||||
"package-entrypoint-missing",
|
||||
"package-install-metadata-incomplete",
|
||||
"package-json-missing",
|
||||
"package-manifest-version-drift",
|
||||
"package-min-host-version-drift",
|
||||
"package-npm-pack-entrypoint-missing",
|
||||
"package-npm-pack-metadata-missing",
|
||||
"package-npm-pack-unavailable",
|
||||
"package-openclaw-entry-missing",
|
||||
"package-openclaw-metadata-missing",
|
||||
"package-openclaw-unsupported-metadata",
|
||||
"package-plugin-api-compat-missing",
|
||||
"package-typescript-source-entrypoint",
|
||||
"provider-auth-env-vars",
|
||||
"registration-capture-gap",
|
||||
"runtime-tool-capture",
|
||||
"reserved-sdk-import",
|
||||
"security-manifest-schema-unavailable",
|
||||
"sdk-export-missing",
|
||||
"unrecognized-security-manifest",
|
||||
]);
|
||||
|
||||
export const issueMetadataByCode = {
|
||||
@ -85,6 +93,12 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "plugin imports reserved bundled-plugin SDK compatibility subpaths",
|
||||
},
|
||||
"security-manifest-schema-unavailable": {
|
||||
severity: "P3",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "plugin security manifest references an unavailable schema",
|
||||
},
|
||||
"missing-compat-record": {
|
||||
severity: "P1",
|
||||
owner: "core",
|
||||
@ -119,7 +133,7 @@ export const issueMetadataByCode = {
|
||||
severity: "P2",
|
||||
owner: "inspector",
|
||||
decision: "inspector-follow-up",
|
||||
title: "cold import requires isolated dependency installation",
|
||||
title: "cold import requires dependency installation in an isolated workspace",
|
||||
},
|
||||
"package-entrypoint-missing": {
|
||||
severity: "P1",
|
||||
@ -127,6 +141,12 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package entrypoint is missing",
|
||||
},
|
||||
"package-install-metadata-incomplete": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package install metadata is incomplete",
|
||||
},
|
||||
"package-json-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
@ -139,6 +159,30 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "package and manifest versions drift",
|
||||
},
|
||||
"package-min-host-version-drift": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package minimum host version drifts from build target",
|
||||
},
|
||||
"package-npm-pack-entrypoint-missing": {
|
||||
severity: "P1",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "advertised npm artifact is missing OpenClaw entrypoints",
|
||||
},
|
||||
"package-npm-pack-metadata-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "advertised npm artifact is missing OpenClaw metadata",
|
||||
},
|
||||
"package-npm-pack-unavailable": {
|
||||
severity: "P1",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "advertised npm artifact cannot be packed",
|
||||
},
|
||||
"package-openclaw-entry-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
@ -151,6 +195,12 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package metadata is missing",
|
||||
},
|
||||
"package-openclaw-unsupported-metadata": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "package declares unsupported OpenClaw metadata",
|
||||
},
|
||||
"package-plugin-api-compat-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
@ -170,10 +220,10 @@ export const issueMetadataByCode = {
|
||||
title: "providerAuthEnvVars legacy manifest metadata must stay covered",
|
||||
},
|
||||
"registration-capture-gap": {
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
owner: "inspector",
|
||||
decision: "inspector-follow-up",
|
||||
title: "runtime registrations need capture before contract judgment",
|
||||
title: "runtime registrations need capture evidence before final contract judgment",
|
||||
},
|
||||
"runtime-tool-capture": {
|
||||
severity: "P2",
|
||||
@ -193,6 +243,12 @@ export const issueMetadataByCode = {
|
||||
decision: "core-compat-adapter",
|
||||
title: "fixture calls a registrar missing from target OpenClaw",
|
||||
},
|
||||
"unrecognized-security-manifest": {
|
||||
severity: "P3",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "plugin ships an unsupported security manifest",
|
||||
},
|
||||
};
|
||||
|
||||
export function buildIssues({ breakages = [], warnings = [], suggestions = [], targetOpenClaw, idPrefix = "CRABPOT" }) {
|
||||
@ -212,7 +268,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
|
||||
owner: finding.owner,
|
||||
code: finding.code,
|
||||
decision: finding.decision,
|
||||
status: finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open",
|
||||
status: finding.status ?? (finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open"),
|
||||
issueClass: finding.issueClass,
|
||||
live: finding.live,
|
||||
deprecated: finding.deprecated,
|
||||
@ -220,6 +276,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
|
||||
title: issueTitle(finding),
|
||||
evidence: finding.evidence ?? [],
|
||||
compatRecord: finding.compatRecord ?? null,
|
||||
runtimeCoverage: finding.runtimeCoverage ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
@ -285,7 +342,10 @@ export function summarizeIssueClasses(issues) {
|
||||
}
|
||||
|
||||
function issueClassFor(code, options) {
|
||||
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)) {
|
||||
if (code === "sdk-export-missing" && options.compatRecord) {
|
||||
return "compat-gap";
|
||||
}
|
||||
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)) {
|
||||
return "live-issue";
|
||||
}
|
||||
if (code === "missing-compat-record") {
|
||||
@ -314,10 +374,18 @@ function issueClassFor(code, options) {
|
||||
"manifest-unknown-fields",
|
||||
"package-json-missing",
|
||||
"package-manifest-version-drift",
|
||||
"package-min-host-version-drift",
|
||||
"package-npm-pack-entrypoint-missing",
|
||||
"package-npm-pack-metadata-missing",
|
||||
"package-npm-pack-unavailable",
|
||||
"package-openclaw-entry-missing",
|
||||
"package-openclaw-metadata-missing",
|
||||
"package-openclaw-unsupported-metadata",
|
||||
"package-plugin-api-compat-missing",
|
||||
"package-install-metadata-incomplete",
|
||||
"reserved-sdk-import",
|
||||
"security-manifest-schema-unavailable",
|
||||
"unrecognized-security-manifest",
|
||||
].includes(code)
|
||||
) {
|
||||
return "upstream-metadata";
|
||||
@ -332,7 +400,7 @@ function severityForClass(code, defaultSeverity, options) {
|
||||
if (
|
||||
options.issueClass === "live-issue" &&
|
||||
["none", "untracked"].includes(options.compatStatus) &&
|
||||
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)
|
||||
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)
|
||||
) {
|
||||
return "P0";
|
||||
}
|
||||
|
||||
@ -6,6 +6,7 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { createCaptureApi } from "./capture-api.js";
|
||||
import { captureApiOptionsForPlugin } from "./capture-config.js";
|
||||
import { createMockSdkPackage } from "./sdk-mock.js";
|
||||
|
||||
const options = JSON.parse(process.argv[2] ?? "{}");
|
||||
@ -30,7 +31,7 @@ async function run(options) {
|
||||
cleanupTempDirOnExit(workspace);
|
||||
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
|
||||
register(pathToFileURL(loaderPath));
|
||||
return await captureLinkedEntrypoint(entrypoint, options);
|
||||
return await captureLinkedEntrypoint(entrypoint, { ...options, pluginRoot });
|
||||
}
|
||||
|
||||
function cleanupTempDirOnExit(dir) {
|
||||
@ -65,7 +66,10 @@ async function captureLinkedEntrypoint(entrypoint, options) {
|
||||
);
|
||||
}
|
||||
|
||||
const api = createCaptureApi(options.apiOptions);
|
||||
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
|
||||
pluginRoot: options.pluginRoot,
|
||||
});
|
||||
const api = createCaptureApi(apiOptions);
|
||||
try {
|
||||
await register(api);
|
||||
} catch (error) {
|
||||
@ -80,7 +84,7 @@ async function captureLinkedEntrypoint(entrypoint, options) {
|
||||
mockSdk: true,
|
||||
captured: api.getCapturedContracts(),
|
||||
};
|
||||
if (options.apiOptions?.retainHandlers === true) {
|
||||
if (apiOptions?.retainHandlers === true) {
|
||||
result.retained = api.getRetainedContracts();
|
||||
}
|
||||
return withProcessOutput(result, outputCapture);
|
||||
|
||||
@ -106,12 +106,89 @@ export function openClawTargetPathCandidates(manifest, configuredPath) {
|
||||
|
||||
export function parseCompatRecordEntries(source) {
|
||||
const entries = [];
|
||||
for (const match of source.matchAll(/\{[\s\S]*?\bcode:\s*["'`]([^"'`]+)["'`][\s\S]*?\bstatus:\s*["'`]([^"'`]+)["'`][\s\S]*?\}/g)) {
|
||||
entries.push({ code: match[1], status: match[2] });
|
||||
let cursor = 0;
|
||||
while (cursor < source.length) {
|
||||
const codeProperty = readStringProperty(source, "code", cursor);
|
||||
if (!codeProperty) {
|
||||
break;
|
||||
}
|
||||
|
||||
const statusProperty = readStringProperty(source, "status", codeProperty.end);
|
||||
if (statusProperty) {
|
||||
entries.push({ code: codeProperty.value, status: statusProperty.value });
|
||||
cursor = statusProperty.end;
|
||||
} else {
|
||||
cursor = codeProperty.end;
|
||||
}
|
||||
}
|
||||
return dedupeBy(entries, (entry) => entry.code).sort((left, right) => left.code.localeCompare(right.code));
|
||||
}
|
||||
|
||||
function readStringProperty(source, property, fromIndex) {
|
||||
const propertyIndex = findProperty(source, property, fromIndex);
|
||||
if (propertyIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
const colonIndex = source.indexOf(":", propertyIndex + property.length);
|
||||
if (colonIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
let quoteIndex = colonIndex + 1;
|
||||
while (quoteIndex < source.length && isWhitespace(source[quoteIndex])) {
|
||||
quoteIndex += 1;
|
||||
}
|
||||
if (!isQuote(source[quoteIndex])) {
|
||||
return null;
|
||||
}
|
||||
return readQuotedValue(source, quoteIndex);
|
||||
}
|
||||
|
||||
function findProperty(source, property, fromIndex) {
|
||||
let index = source.indexOf(property, fromIndex);
|
||||
while (index !== -1) {
|
||||
const previous = index === 0 ? "" : source[index - 1];
|
||||
const next = source[index + property.length] ?? "";
|
||||
if (!isIdentifierChar(previous) && !isIdentifierChar(next)) {
|
||||
return index;
|
||||
}
|
||||
index = source.indexOf(property, index + property.length);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function readQuotedValue(source, quoteIndex) {
|
||||
const quote = source[quoteIndex];
|
||||
let value = "";
|
||||
for (let index = quoteIndex + 1; index < source.length; index += 1) {
|
||||
const char = source[index];
|
||||
if (char === "\\") {
|
||||
value += source[index + 1] ?? "";
|
||||
index += 1;
|
||||
} else if (char === quote) {
|
||||
return { value, end: index + 1 };
|
||||
} else {
|
||||
value += char;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isQuote(char) {
|
||||
return char === '"' || char === "'" || char === "`";
|
||||
}
|
||||
|
||||
function isIdentifierChar(char) {
|
||||
if (char === "_" || char === "$") {
|
||||
return true;
|
||||
}
|
||||
const code = char.charCodeAt(0);
|
||||
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
|
||||
}
|
||||
|
||||
function isWhitespace(char) {
|
||||
return char === " " || char === "\n" || char === "\r" || char === "\t";
|
||||
}
|
||||
|
||||
export function parsePluginSdkExports(packageJson) {
|
||||
return Object.keys(packageJson.exports ?? {})
|
||||
.filter((specifier) => specifier === "./plugin-sdk" || specifier.startsWith("./plugin-sdk/"))
|
||||
|
||||
@ -12,13 +12,11 @@ export function buildPlatformProbes(options = {}) {
|
||||
const entrypoints = plan.fixtures.flatMap((fixture) =>
|
||||
fixture.entrypoints.map((entrypoint) => summarizeEntrypoint(fixture.id, entrypoint)),
|
||||
);
|
||||
const portabilityFindings = plan.fixtures.flatMap((fixture) =>
|
||||
fixture.entrypoints.flatMap((entrypoint) =>
|
||||
entrypoint.steps
|
||||
.map((step) => summarizeStep(fixture.id, entrypoint, step))
|
||||
.filter((finding) => finding.riskCodes.length > 0),
|
||||
),
|
||||
const stepFindings = plan.fixtures.flatMap((fixture) =>
|
||||
fixture.entrypoints.flatMap((entrypoint) => entrypoint.steps.map((step) => summarizeStep(fixture.id, entrypoint, step, options.stepCoverage))),
|
||||
);
|
||||
const portabilityFindings = stepFindings.flatMap((finding) => (finding.residual ? [finding.residual] : []));
|
||||
const coveredPortabilityFindings = stepFindings.flatMap((finding) => (finding.covered ? [finding.covered] : []));
|
||||
|
||||
return {
|
||||
generatedAt: plan.generatedAt,
|
||||
@ -31,6 +29,7 @@ export function buildPlatformProbes(options = {}) {
|
||||
jitiAlternativeCount: entrypoints.filter((entrypoint) => entrypoint.loaderAlternatives.includes("jiti")).length,
|
||||
lazyImportProbeCount: entrypoints.filter((entrypoint) => entrypoint.capturePlanned && entrypoint.syntheticProbePlanned).length,
|
||||
portabilityFindingCount: portabilityFindings.length,
|
||||
coveredPortabilityFindingCount: coveredPortabilityFindings.length,
|
||||
windowsRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("windows")).length,
|
||||
macosRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("macos")).length,
|
||||
linuxRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("linux")).length,
|
||||
@ -38,6 +37,7 @@ export function buildPlatformProbes(options = {}) {
|
||||
},
|
||||
entrypoints,
|
||||
portabilityFindings,
|
||||
coveredPortabilityFindings,
|
||||
recommendations: buildRecommendations(portabilityFindings, entrypoints),
|
||||
};
|
||||
}
|
||||
@ -55,8 +55,11 @@ export function validatePlatformProbes(report, options = {}) {
|
||||
errors.push("all TypeScript loader entrypoints must track a Jiti fallback candidate");
|
||||
}
|
||||
for (const entrypoint of report.entrypoints) {
|
||||
if (entrypoint.loaderPrimary === "tsx" && (!entrypoint.captureUsesTsx || !entrypoint.syntheticUsesTsx)) {
|
||||
errors.push(`${entrypoint.id}: tsx loader strategy is not reflected in capture and synthetic commands`);
|
||||
if (
|
||||
entrypoint.loaderPrimary === "tsx" &&
|
||||
(!entrypoint.captureUsesTypeScriptLoader || !entrypoint.syntheticUsesTypeScriptLoader)
|
||||
) {
|
||||
errors.push(`${entrypoint.id}: TypeScript loader strategy is not reflected in capture and synthetic commands`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
@ -94,9 +97,21 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
||||
entrypoint.loaderAlternatives.join(", ") || "-",
|
||||
entrypoint.captureUsesTsx ? "yes" : "no",
|
||||
entrypoint.syntheticUsesTsx ? "yes" : "no",
|
||||
entrypoint.captureUsesMockSdk ? "yes" : "no",
|
||||
entrypoint.syntheticUsesMockSdk ? "yes" : "no",
|
||||
entrypoint.entrypoint,
|
||||
]),
|
||||
["Fixture", "Status", "Primary", "Alternatives", "Capture TSX", "Synthetic TSX", "Entrypoint"],
|
||||
[
|
||||
"Fixture",
|
||||
"Status",
|
||||
"Primary",
|
||||
"Alternatives",
|
||||
"Capture TSX",
|
||||
"Synthetic TSX",
|
||||
"Capture Mock SDK",
|
||||
"Synthetic Mock SDK",
|
||||
"Entrypoint",
|
||||
],
|
||||
),
|
||||
"",
|
||||
"## Portability Findings",
|
||||
@ -112,6 +127,19 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
||||
["Fixture", "Step", "Platforms", "Risks", "Mitigation"],
|
||||
),
|
||||
"",
|
||||
"## Covered Portability Findings",
|
||||
"",
|
||||
markdownTable(
|
||||
(report.coveredPortabilityFindings ?? []).map((finding) => [
|
||||
finding.fixture,
|
||||
finding.kind,
|
||||
finding.platforms.join(", ") || "-",
|
||||
finding.riskCodes.join(", "),
|
||||
finding.coverage,
|
||||
]),
|
||||
["Fixture", "Step", "Platforms", "Covered Risks", "Coverage"],
|
||||
),
|
||||
"",
|
||||
"## Recommendations",
|
||||
"",
|
||||
markdownTable(
|
||||
@ -124,6 +152,10 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
||||
function summarizeEntrypoint(fixtureId, entrypoint) {
|
||||
const captureStep = entrypoint.steps.find((step) => step.kind === "capture");
|
||||
const syntheticStep = entrypoint.steps.find((step) => step.kind === "synthetic-probe");
|
||||
const captureUsesTsx = Boolean(captureStep?.command.includes("--import tsx"));
|
||||
const syntheticUsesTsx = Boolean(syntheticStep?.command.includes("--import tsx"));
|
||||
const captureUsesMockSdk = Boolean(captureStep?.command.includes("--mock-sdk"));
|
||||
const syntheticUsesMockSdk = Boolean(syntheticStep?.command.includes("--mock-sdk"));
|
||||
return {
|
||||
fixture: fixtureId,
|
||||
id: entrypoint.id,
|
||||
@ -135,21 +167,57 @@ function summarizeEntrypoint(fixtureId, entrypoint) {
|
||||
loaderAlternatives: entrypoint.loaderStrategy.alternatives,
|
||||
capturePlanned: Boolean(captureStep),
|
||||
syntheticProbePlanned: Boolean(syntheticStep),
|
||||
captureUsesTsx: Boolean(captureStep?.command.includes("--import tsx")),
|
||||
syntheticUsesTsx: Boolean(syntheticStep?.command.includes("--import tsx")),
|
||||
captureUsesTsx,
|
||||
syntheticUsesTsx,
|
||||
captureUsesMockSdk,
|
||||
syntheticUsesMockSdk,
|
||||
captureUsesTypeScriptLoader: captureUsesTsx || captureUsesMockSdk,
|
||||
syntheticUsesTypeScriptLoader: syntheticUsesTsx || syntheticUsesMockSdk,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeStep(fixtureId, entrypoint, step) {
|
||||
function summarizeStep(fixtureId, entrypoint, step, stepCoverage) {
|
||||
const riskCodes = stepRiskCodes(step);
|
||||
return {
|
||||
const coverage = normalizeStepCoverage(stepCoverage?.({ fixture: fixtureId, entrypoint, step, riskCodes }), riskCodes);
|
||||
const residualRiskCodes = riskCodes.filter((code) => !coverage.riskCodes.includes(code));
|
||||
const common = {
|
||||
fixture: fixtureId,
|
||||
entrypoint: entrypoint.id,
|
||||
kind: step.kind,
|
||||
platforms: platformsForRiskCodes(riskCodes),
|
||||
riskCodes,
|
||||
command: step.command,
|
||||
mitigation: mitigationForRiskCodes(riskCodes),
|
||||
};
|
||||
return {
|
||||
residual:
|
||||
residualRiskCodes.length > 0
|
||||
? {
|
||||
...common,
|
||||
coveredRiskCodes: coverage.riskCodes,
|
||||
platforms: platformsForRiskCodes(residualRiskCodes),
|
||||
riskCodes: residualRiskCodes,
|
||||
mitigation: mitigationForRiskCodes(residualRiskCodes),
|
||||
}
|
||||
: null,
|
||||
covered:
|
||||
coverage.riskCodes.length > 0
|
||||
? {
|
||||
...common,
|
||||
coverage: coverage.reason,
|
||||
platforms: platformsForRiskCodes(coverage.riskCodes),
|
||||
riskCodes: coverage.riskCodes,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeStepCoverage(coverage, riskCodes) {
|
||||
if (!coverage) {
|
||||
return { reason: "", riskCodes: [] };
|
||||
}
|
||||
const requested = coverage === true ? riskCodes : coverage.coveredRiskCodes ?? coverage.handledRiskCodes ?? coverage.riskCodes ?? coverage;
|
||||
const covered = Array.isArray(requested) ? requested : [];
|
||||
return {
|
||||
reason: typeof coverage.reason === "string" && coverage.reason ? coverage.reason : "covered by isolated workspace executor",
|
||||
riskCodes: covered.filter((code) => riskCodes.includes(code)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
@ -215,7 +283,7 @@ function buildRecommendations(portabilityFindings, entrypoints) {
|
||||
if (entrypoints.some((entrypoint) => entrypoint.loaderPrimary === "tsx")) {
|
||||
recommendations.push({
|
||||
area: "loader",
|
||||
action: "keep tsx as the source-entrypoint smoke path, add a Jiti execution lane before treating TS plugin source compatibility as covered",
|
||||
action: "keep mock-SDK TypeScript capture green, add a real host-loader/Jiti lane before treating TS plugin source compatibility as covered",
|
||||
});
|
||||
}
|
||||
if (portabilityFindings.some((finding) => finding.riskCodes.includes("rsync-required"))) {
|
||||
|
||||
@ -7,8 +7,12 @@ export async function runProfiledProcess(options) {
|
||||
let firstRssKb = 0;
|
||||
let peakRssKb = 0;
|
||||
let peakCpuPercent = 0;
|
||||
let statSampleCount = 0;
|
||||
let rssSampleCount = 0;
|
||||
let cpuSampleCount = 0;
|
||||
const cpuSamples = [];
|
||||
let pollInFlight = false;
|
||||
const pendingStats = new Set();
|
||||
|
||||
const child = spawn(options.command, options.args ?? [], {
|
||||
cwd: options.cwd,
|
||||
@ -21,27 +25,43 @@ export async function runProfiledProcess(options) {
|
||||
child.stderr?.on("data", (chunk) => stderr.push(chunk));
|
||||
|
||||
const recordStats = (stats) => {
|
||||
if (stats.rssKb > 0 && firstRssKb === 0) {
|
||||
if (stats.rssAvailable || stats.cpuAvailable) {
|
||||
statSampleCount += 1;
|
||||
}
|
||||
if (stats.rssAvailable) {
|
||||
rssSampleCount += 1;
|
||||
}
|
||||
if (stats.cpuAvailable) {
|
||||
cpuSampleCount += 1;
|
||||
}
|
||||
if (stats.rssAvailable && stats.rssKb > 0 && firstRssKb === 0) {
|
||||
firstRssKb = stats.rssKb;
|
||||
}
|
||||
peakRssKb = Math.max(peakRssKb, stats.rssKb);
|
||||
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
|
||||
if (stats.cpuPercent > 0) {
|
||||
if (stats.rssAvailable) {
|
||||
peakRssKb = Math.max(peakRssKb, stats.rssKb);
|
||||
}
|
||||
if (stats.cpuAvailable) {
|
||||
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
|
||||
cpuSamples.push(stats.cpuPercent);
|
||||
}
|
||||
};
|
||||
|
||||
const poll = setInterval(() => {
|
||||
const sampleStats = () => {
|
||||
if (pollInFlight) {
|
||||
return;
|
||||
}
|
||||
pollInFlight = true;
|
||||
readProcessStats(child.pid)
|
||||
const pending = readProcessStats(child.pid)
|
||||
.then(recordStats)
|
||||
.finally(() => {
|
||||
pollInFlight = false;
|
||||
pendingStats.delete(pending);
|
||||
});
|
||||
}, options.pollMs ?? 100);
|
||||
pendingStats.add(pending);
|
||||
};
|
||||
|
||||
sampleStats();
|
||||
const poll = setInterval(sampleStats, options.pollMs ?? 25);
|
||||
|
||||
const exitCode = await new Promise((resolve, reject) => {
|
||||
child.on("error", (error) => {
|
||||
@ -51,16 +71,10 @@ export async function runProfiledProcess(options) {
|
||||
child.on("exit", (code) => resolve(code ?? 1));
|
||||
});
|
||||
clearInterval(poll);
|
||||
await Promise.allSettled([...pendingStats]);
|
||||
|
||||
const finalStats = await readProcessStats(child.pid);
|
||||
if (finalStats.rssKb > 0 && firstRssKb === 0) {
|
||||
firstRssKb = finalStats.rssKb;
|
||||
}
|
||||
peakRssKb = Math.max(peakRssKb, finalStats.rssKb);
|
||||
peakCpuPercent = Math.max(peakCpuPercent, finalStats.cpuPercent);
|
||||
if (finalStats.cpuPercent > 0) {
|
||||
cpuSamples.push(finalStats.cpuPercent);
|
||||
}
|
||||
recordStats(finalStats);
|
||||
|
||||
const wallMs = Math.round(performance.now() - start);
|
||||
const averageCpuPercent =
|
||||
@ -79,6 +93,9 @@ export async function runProfiledProcess(options) {
|
||||
peakCpuPercent: Math.round(peakCpuPercent * 10) / 10,
|
||||
cpuMsEstimate: Math.round((wallMs * cpuPercentForEstimate) / 100),
|
||||
harnessHeapDeltaMb: Math.round((heapUsedMb() - heapStartMb) * 10) / 10,
|
||||
statSampleCount,
|
||||
rssSampleCount,
|
||||
cpuSampleCount,
|
||||
exitCode,
|
||||
stdoutPreview: previewLines(stdout),
|
||||
stderrPreview: previewLines(stderr),
|
||||
@ -87,7 +104,7 @@ export async function runProfiledProcess(options) {
|
||||
|
||||
async function readProcessStats(pid) {
|
||||
if (!pid || process.platform === "win32") {
|
||||
return { rssKb: 0, cpuPercent: 0 };
|
||||
return { rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 };
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const ps = spawn("ps", ["-o", "rss=", "-o", "%cpu=", "-p", String(pid)], {
|
||||
@ -95,14 +112,18 @@ async function readProcessStats(pid) {
|
||||
});
|
||||
const chunks = [];
|
||||
ps.stdout.on("data", (chunk) => chunks.push(chunk));
|
||||
ps.on("error", () => resolve({ rssKb: 0, cpuPercent: 0 }));
|
||||
ps.on("error", () => resolve({ rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 }));
|
||||
ps.on("exit", () => {
|
||||
const [rssRaw, cpuRaw] = Buffer.concat(chunks).toString("utf8").trim().split(/\s+/);
|
||||
const rssKb = Number.parseInt(rssRaw, 10);
|
||||
const cpuPercent = Number.parseFloat(cpuRaw);
|
||||
const rssAvailable = Number.isFinite(rssKb);
|
||||
const cpuAvailable = Number.isFinite(cpuPercent);
|
||||
resolve({
|
||||
rssKb: Number.isFinite(rssKb) ? rssKb : 0,
|
||||
cpuPercent: Number.isFinite(cpuPercent) ? cpuPercent : 0,
|
||||
rssAvailable,
|
||||
rssKb: rssAvailable ? rssKb : 0,
|
||||
cpuAvailable,
|
||||
cpuPercent: cpuAvailable ? cpuPercent : 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
23
src/prune-workspace-dev-deps-cli.js
Normal file
23
src/prune-workspace-dev-deps-cli.js
Normal file
@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const packageJsonPath = path.resolve(process.cwd(), "package.json");
|
||||
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
|
||||
let changed = false;
|
||||
|
||||
for (const [name, specifier] of Object.entries(packageJson.devDependencies ?? {})) {
|
||||
if (typeof specifier === "string" && specifier.startsWith("workspace:")) {
|
||||
delete packageJson.devDependencies[name];
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (packageJson.devDependencies && Object.keys(packageJson.devDependencies).length === 0) {
|
||||
delete packageJson.devDependencies;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
|
||||
}
|
||||
42
src/report-sanitizer.js
Normal file
42
src/report-sanitizer.js
Normal file
@ -0,0 +1,42 @@
|
||||
import path from "node:path";
|
||||
|
||||
export function sanitizeReportArtifact(report, options = {}) {
|
||||
const sensitivePaths = sensitiveOpenClawPaths(report);
|
||||
if (sensitivePaths.length === 0) {
|
||||
return report;
|
||||
}
|
||||
const placeholder = options.openclawPathPlaceholder ?? "<OPENCLAW_PATH>";
|
||||
return sanitizeValue(report, sensitivePaths, placeholder);
|
||||
}
|
||||
|
||||
function sensitiveOpenClawPaths(report) {
|
||||
const targetOpenClaw = report?.targetOpenClaw;
|
||||
return unique(
|
||||
[targetOpenClaw?.configuredPath, ...(targetOpenClaw?.searchedPaths ?? [])]
|
||||
.filter((value) => typeof value === "string" && isAbsolutePath(value))
|
||||
.sort((left, right) => right.length - left.length),
|
||||
);
|
||||
}
|
||||
|
||||
function sanitizeValue(value, sensitivePaths, placeholder) {
|
||||
if (typeof value === "string") {
|
||||
return sensitivePaths.reduce((result, sensitivePath) => result.replaceAll(sensitivePath, placeholder), value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => sanitizeValue(item, sensitivePaths, placeholder));
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, entryValue]) => [key, sanitizeValue(entryValue, sensitivePaths, placeholder)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function isAbsolutePath(value) {
|
||||
return path.isAbsolute(value) || /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith("\\\\");
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
@ -4,6 +4,8 @@ import { renderCompatibilityIssuesReport, renderCompatibilityMarkdownReport } fr
|
||||
import { buildContractProbes } from "./contract-probes.js";
|
||||
import { classifyCompatibilityFixture } from "./fixture-summary.js";
|
||||
import { buildIssues, summarizeIssueClasses } from "./issues.js";
|
||||
import { sanitizeReportArtifact } from "./report-sanitizer.js";
|
||||
import { applyRuntimeExecutionCoverage } from "./runtime-reconciliation.js";
|
||||
|
||||
export function buildReport({ config, inspections, failures = [], generatedAt = "deterministic" }) {
|
||||
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
|
||||
@ -139,6 +141,10 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
decisions,
|
||||
});
|
||||
|
||||
const runtimeCoverage = applyRuntimeExecutionCoverage({
|
||||
findings: [...warnings, ...suggestions],
|
||||
executionResults: options.executionResults,
|
||||
});
|
||||
const issues = buildIssues({
|
||||
breakages,
|
||||
warnings,
|
||||
@ -148,6 +154,8 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
});
|
||||
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures: fixtureReports });
|
||||
const issueSummary = summarizeIssueClasses(issues);
|
||||
const openIssues = issues.filter((issue) => issue.status !== "runtime-covered");
|
||||
const openIssueSummary = summarizeIssueClasses(openIssues);
|
||||
|
||||
return {
|
||||
generatedAt: options.generatedAt ?? "deterministic",
|
||||
@ -162,8 +170,11 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
decisionCount: decisions.length,
|
||||
logCount: logs.length,
|
||||
issueCount: issues.length,
|
||||
openIssueCount: openIssues.length,
|
||||
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
|
||||
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
|
||||
openP0IssueCount: openIssues.filter((issue) => issue.severity === "P0").length,
|
||||
openP1IssueCount: openIssues.filter((issue) => issue.severity === "P1").length,
|
||||
liveIssueCount: issueSummary["live-issue"],
|
||||
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
|
||||
compatGapCount: issueSummary["compat-gap"],
|
||||
@ -171,6 +182,10 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
inspectorGapCount: issueSummary["inspector-gap"],
|
||||
upstreamIssueCount: issueSummary["upstream-metadata"],
|
||||
fixtureRegressionCount: issueSummary["fixture-regression"],
|
||||
openInspectorGapCount: openIssueSummary["inspector-gap"],
|
||||
runtimeCoveredIssueCount: runtimeCoverage.coveredFindingCount,
|
||||
runtimePartiallyCoveredIssueCount: runtimeCoverage.partiallyCoveredFindingCount,
|
||||
runtimeCoverageArtifactCount: runtimeCoverage.coverage.artifactCount,
|
||||
contractProbeCount: contractProbes.length,
|
||||
},
|
||||
fixtures: fixtureReports,
|
||||
@ -210,6 +225,10 @@ export function classifyCompatRecordCoverage({ targetOpenClaw, findings, suggest
|
||||
continue;
|
||||
}
|
||||
|
||||
if (finding.code === "sdk-export-missing") {
|
||||
continue;
|
||||
}
|
||||
|
||||
suggestions.push({
|
||||
fixture: finding.fixture,
|
||||
code: "missing-compat-record",
|
||||
@ -233,12 +252,13 @@ export async function writeReport(report, options = {}) {
|
||||
const basename = options.basename ?? "plugin-inspector-report";
|
||||
const jsonPath = path.join(outDir, `${basename}.json`);
|
||||
const markdownPath = path.join(outDir, `${basename}.md`);
|
||||
const artifactReport = sanitizeReportArtifact(report, options);
|
||||
|
||||
return writeJsonMarkdownArtifacts({
|
||||
jsonPath,
|
||||
markdownPath,
|
||||
json: report,
|
||||
markdown: renderMarkdownReport(report),
|
||||
json: artifactReport,
|
||||
markdown: renderMarkdownReport(artifactReport),
|
||||
check: options.check,
|
||||
});
|
||||
}
|
||||
@ -249,6 +269,7 @@ export async function writeCompatibilityReport(report, options = {}) {
|
||||
const jsonPath = options.jsonPath ?? path.join(outDir, `${basename}.json`);
|
||||
const markdownPath = options.markdownPath ?? path.join(outDir, `${basename}.md`);
|
||||
const issuesPath = options.issuesPath ?? path.join(outDir, options.issuesBasename ?? "plugin-inspector-issues.md");
|
||||
const artifactReport = sanitizeReportArtifact(report, options);
|
||||
const markdownOptions = compatibilityRenderOptions(options, {
|
||||
title: options.markdownTitle ?? options.title,
|
||||
...options.markdownOptions,
|
||||
@ -260,14 +281,16 @@ export async function writeCompatibilityReport(report, options = {}) {
|
||||
|
||||
return writeArtifacts(
|
||||
[
|
||||
{ name: "jsonPath", path: jsonPath, json: report },
|
||||
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(report, markdownOptions) },
|
||||
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(report, issuesOptions) },
|
||||
{ name: "jsonPath", path: jsonPath, json: artifactReport },
|
||||
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(artifactReport, markdownOptions) },
|
||||
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(artifactReport, issuesOptions) },
|
||||
],
|
||||
{ check: options.check },
|
||||
);
|
||||
}
|
||||
|
||||
export { sanitizeReportArtifact };
|
||||
|
||||
function compatibilityRenderOptions(options, overrides) {
|
||||
const renderOptions = {
|
||||
formatEvidence: options.formatEvidence,
|
||||
@ -303,7 +326,11 @@ function topTextFindings(report, limit) {
|
||||
return [
|
||||
...(report.breakages ?? []).map((finding) => formatTextFinding(finding, "breakage")),
|
||||
...(report.issues ?? [])
|
||||
.filter((issue) => issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1")
|
||||
.filter(
|
||||
(issue) =>
|
||||
issue.status !== "runtime-covered" &&
|
||||
(issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1"),
|
||||
)
|
||||
.map((issue) => formatTextFinding(issue, issue.severity ?? "issue")),
|
||||
...(report.warnings ?? []).map((finding) => formatTextFinding(finding, "warning")),
|
||||
].slice(0, limit);
|
||||
|
||||
@ -46,8 +46,8 @@ export async function buildRuntimeProfile(options = {}) {
|
||||
os: process.platform,
|
||||
arch: process.arch,
|
||||
node: process.version,
|
||||
rssSampler: process.platform === "win32" ? "unavailable" : "ps",
|
||||
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent",
|
||||
rssSampler: process.platform === "win32" ? "unavailable" : "ps-immediate-25ms",
|
||||
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent-immediate-25ms",
|
||||
},
|
||||
summary: summarizeProfile(commands),
|
||||
groups: summarizeCommandGroups(commands),
|
||||
@ -65,8 +65,8 @@ export function validateRuntimeProfile(profile) {
|
||||
errors.push(`${command.id}: missing wall time`);
|
||||
}
|
||||
}
|
||||
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => command.peakRssMb.max <= 0)) {
|
||||
errors.push("all commands are missing peak RSS");
|
||||
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => !hasRssSample(command))) {
|
||||
errors.push("all commands are missing peak RSS samples");
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
@ -98,10 +98,17 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
||||
[
|
||||
["Commands", profile.summary.commandCount],
|
||||
["P50 wall time", `${profile.summary.p50WallMs} ms`],
|
||||
["P95 wall time", `${profile.summary.p95WallMs} ms`],
|
||||
["Max peak RSS", `${profile.summary.maxPeakRssMb} MB`],
|
||||
["Max RSS delta", `${profile.summary.maxRssDeltaMb} MB`],
|
||||
["Max CPU estimate", `${profile.summary.maxCpuMsEstimate} ms`],
|
||||
["Command P95 wall time", `${profile.summary.p95WallMs} ms`],
|
||||
["Wall time basis", profile.summary.wallTimeBasis ?? "command-median-p95"],
|
||||
["Profile samples", profile.summary.sampleCount ?? sampleCount(profile.commands)],
|
||||
["RSS samples", profile.summary.rssSampleCount ?? rssSampleCount(profile.commands)],
|
||||
["CPU samples", profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands)],
|
||||
["Max peak RSS", formatSampledMetric(profile.summary.maxPeakRssMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
|
||||
["Max RSS delta", formatSampledMetric(profile.summary.maxRssDeltaMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
|
||||
[
|
||||
"Max CPU estimate",
|
||||
formatSampledMetric(profile.summary.maxCpuMsEstimate, profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands), "ms"),
|
||||
],
|
||||
["Max harness heap delta", `${profile.summary.maxHarnessHeapDeltaMb} MB`],
|
||||
],
|
||||
["Metric", "Value"],
|
||||
@ -129,13 +136,14 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
||||
command.label,
|
||||
`${command.wallMs.median} ms`,
|
||||
`${command.wallMs.max} ms`,
|
||||
`${command.peakRssMb.max} MB`,
|
||||
`${command.rssDeltaMb.max} MB`,
|
||||
`${command.cpuMsEstimate.max} ms`,
|
||||
formatSampledMetric(command.peakRssMb.max, command.rssSampleCount),
|
||||
formatSampledMetric(command.rssDeltaMb.max, command.rssSampleCount),
|
||||
formatSampledMetric(command.cpuMsEstimate.max, command.cpuSampleCount, "ms"),
|
||||
`${command.harnessHeapDeltaMb.max} MB`,
|
||||
`${command.rssSampleCount ?? 0}/${command.cpuSampleCount ?? 0}`,
|
||||
command.exitCodes.join(", "),
|
||||
]),
|
||||
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "Exit codes"],
|
||||
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "RSS/CPU samples", "Exit codes"],
|
||||
),
|
||||
"",
|
||||
"## Category Rollups",
|
||||
@ -146,11 +154,12 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
||||
group.commandCount,
|
||||
`${group.p50WallMs} ms`,
|
||||
`${group.p95WallMs} ms`,
|
||||
`${group.maxPeakRssMb} MB`,
|
||||
`${group.maxCpuMsEstimate} ms`,
|
||||
formatSampledMetric(group.maxPeakRssMb, group.rssSampleCount),
|
||||
formatSampledMetric(group.maxCpuMsEstimate, group.cpuSampleCount, "ms"),
|
||||
`${group.rssSampleCount ?? 0}/${group.cpuSampleCount ?? 0}`,
|
||||
group.commands.join(", "),
|
||||
]),
|
||||
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "Command IDs"],
|
||||
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "RSS/CPU samples", "Command IDs"],
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
@ -189,8 +198,15 @@ function summarizeProfile(commands) {
|
||||
const maxRssDeltaMb = Math.max(0, ...commands.map((command) => command.rssDeltaMb.max));
|
||||
const maxCpuMsEstimate = Math.max(0, ...commands.map((command) => command.cpuMsEstimate.max));
|
||||
const maxHarnessHeapDeltaMb = Math.max(0, ...commands.map((command) => command.harnessHeapDeltaMb.max));
|
||||
const totalSampleCount = sampleCount(commands);
|
||||
const totalRssSampleCount = rssSampleCount(commands);
|
||||
const totalCpuSampleCount = cpuSampleCount(commands);
|
||||
return {
|
||||
commandCount: commands.length,
|
||||
sampleCount: totalSampleCount,
|
||||
rssSampleCount: totalRssSampleCount,
|
||||
cpuSampleCount: totalCpuSampleCount,
|
||||
wallTimeBasis: "command-median-p95",
|
||||
p50WallMs: percentile(wallTimes, 0.5),
|
||||
p95WallMs: percentile(wallTimes, 0.95),
|
||||
maxPeakRssMb,
|
||||
@ -206,6 +222,12 @@ function summarizeCommand(command, samples) {
|
||||
const rssDeltaMb = samples.map((sample) => sample.rssDeltaMb).sort((left, right) => left - right);
|
||||
const peakCpuPercent = samples.map((sample) => sample.peakCpuPercent).sort((left, right) => left - right);
|
||||
const cpuMsEstimate = samples.map((sample) => sample.cpuMsEstimate).sort((left, right) => left - right);
|
||||
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
|
||||
const rssSampleTotal = samples.reduce(
|
||||
(sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)),
|
||||
0,
|
||||
);
|
||||
const cpuSampleTotal = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
|
||||
const harnessHeapDeltaMb = samples
|
||||
.map((sample) => sample.harnessHeapDeltaMb)
|
||||
.sort((left, right) => left - right);
|
||||
@ -222,6 +244,9 @@ function summarizeCommand(command, samples) {
|
||||
peakCpuPercent: summarizeNumbers(peakCpuPercent),
|
||||
cpuMsEstimate: summarizeNumbers(cpuMsEstimate),
|
||||
harnessHeapDeltaMb: summarizeNumbers(harnessHeapDeltaMb),
|
||||
statSampleCount,
|
||||
rssSampleCount: rssSampleTotal,
|
||||
cpuSampleCount: cpuSampleTotal,
|
||||
exitCodes: [...new Set(samples.map((sample) => sample.exitCode))].sort(),
|
||||
};
|
||||
}
|
||||
@ -244,6 +269,8 @@ function summarizeCommandGroups(commands) {
|
||||
const cpuMs = categoryCommands
|
||||
.flatMap((command) => command.samples.map((sample) => sample.cpuMsEstimate))
|
||||
.sort((left, right) => left - right);
|
||||
const groupRssSampleCount = rssSampleCount(categoryCommands);
|
||||
const groupCpuSampleCount = cpuSampleCount(categoryCommands);
|
||||
return {
|
||||
category,
|
||||
commandCount: categoryCommands.length,
|
||||
@ -251,11 +278,36 @@ function summarizeCommandGroups(commands) {
|
||||
p95WallMs: percentile(wallTimes, 0.95),
|
||||
maxPeakRssMb: peakRss.at(-1) ?? 0,
|
||||
maxCpuMsEstimate: cpuMs.at(-1) ?? 0,
|
||||
rssSampleCount: groupRssSampleCount,
|
||||
cpuSampleCount: groupCpuSampleCount,
|
||||
commands: categoryCommands.map((command) => command.id),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function hasRssSample(command) {
|
||||
return (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)) > 0;
|
||||
}
|
||||
|
||||
function sampleCount(commands) {
|
||||
return commands.reduce((sum, command) => sum + (command.samples?.length ?? 0), 0);
|
||||
}
|
||||
|
||||
function rssSampleCount(commands) {
|
||||
return commands.reduce((sum, command) => sum + (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)), 0);
|
||||
}
|
||||
|
||||
function cpuSampleCount(commands) {
|
||||
return commands.reduce((sum, command) => sum + (command.cpuSampleCount ?? 0), 0);
|
||||
}
|
||||
|
||||
function formatSampledMetric(value, count, unit = "MB") {
|
||||
if ((count ?? 0) <= 0) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
function summarizeNumbers(values) {
|
||||
return {
|
||||
min: values[0],
|
||||
|
||||
124
src/runtime-reconciliation.js
Normal file
124
src/runtime-reconciliation.js
Normal file
@ -0,0 +1,124 @@
|
||||
export function applyRuntimeExecutionCoverage({ findings = [], executionResults } = {}) {
|
||||
const coverage = buildRuntimeExecutionCoverage(executionResults);
|
||||
let coveredFindingCount = 0;
|
||||
let partiallyCoveredFindingCount = 0;
|
||||
|
||||
for (const finding of findings) {
|
||||
const findingCoverage = runtimeCoverageForFinding(finding, coverage);
|
||||
if (!findingCoverage) {
|
||||
continue;
|
||||
}
|
||||
|
||||
finding.runtimeCoverage = findingCoverage;
|
||||
if (findingCoverage.status === "covered") {
|
||||
finding.status = "runtime-covered";
|
||||
coveredFindingCount += 1;
|
||||
} else {
|
||||
partiallyCoveredFindingCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
coverage,
|
||||
coveredFindingCount,
|
||||
partiallyCoveredFindingCount,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildRuntimeExecutionCoverage(executionResults) {
|
||||
const fixtures = new Map();
|
||||
for (const artifact of executionResults?.artifacts ?? []) {
|
||||
if (artifact.kind !== "capture") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fixture = String(artifact.fixture ?? "unknown");
|
||||
const fixtureCoverage = ensureFixtureCoverage(fixtures, fixture);
|
||||
if (artifact.artifactPath) {
|
||||
fixtureCoverage.artifacts.add(artifact.artifactPath);
|
||||
}
|
||||
|
||||
for (const captured of normalizeCaptured(artifact.captured)) {
|
||||
fixtureCoverage.captured.add(captured);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
fixtures,
|
||||
artifactCount: [...fixtures.values()].reduce((sum, fixture) => sum + fixture.artifacts.size, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeCoverageForFinding(finding, coverage) {
|
||||
const fixtureCoverage = coverage.fixtures.get(finding.fixture);
|
||||
if (!fixtureCoverage || fixtureCoverage.captured.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const expected = expectedRuntimeCaptureKeys(finding);
|
||||
if (expected.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const captured = expected.filter((item) => fixtureCoverage.captured.has(item));
|
||||
if (captured.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
status: captured.length === expected.length ? "covered" : "partial",
|
||||
captured,
|
||||
expected,
|
||||
artifacts: [...fixtureCoverage.artifacts].sort(),
|
||||
};
|
||||
}
|
||||
|
||||
function expectedRuntimeCaptureKeys(finding) {
|
||||
const names = evidenceNames(finding.evidence);
|
||||
if (finding.code === "registration-capture-gap") {
|
||||
return names.map((name) => `registration:${name}`);
|
||||
}
|
||||
if (finding.code === "runtime-tool-capture") {
|
||||
return ["registration:registerTool"];
|
||||
}
|
||||
if (finding.code === "conversation-access-hook") {
|
||||
return names.map((name) => `hook:${name}`);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function normalizeCaptured(captured) {
|
||||
return (captured ?? [])
|
||||
.map((item) => {
|
||||
if (typeof item === "string") {
|
||||
return item;
|
||||
}
|
||||
if (item && typeof item === "object" && item.kind && item.name) {
|
||||
return `${item.kind}:${item.name}`;
|
||||
}
|
||||
return "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function evidenceNames(evidence) {
|
||||
return [
|
||||
...new Set(
|
||||
(evidence ?? [])
|
||||
.map((item) => String(item).split(" @ ")[0]?.trim())
|
||||
.filter(Boolean),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function ensureFixtureCoverage(fixtures, fixture) {
|
||||
let fixtureCoverage = fixtures.get(fixture);
|
||||
if (!fixtureCoverage) {
|
||||
fixtureCoverage = {
|
||||
artifacts: new Set(),
|
||||
captured: new Set(),
|
||||
};
|
||||
fixtures.set(fixture, fixtureCoverage);
|
||||
}
|
||||
return fixtureCoverage;
|
||||
}
|
||||
152
src/sdk-mock.js
152
src/sdk-mock.js
@ -254,9 +254,20 @@ export async function createMockSdkPackage(rootDir, options = {}) {
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(), "utf8");
|
||||
const rootExportNames = new Set([
|
||||
...mockSdkExportNames,
|
||||
...(imports.bySpecifier.get("openclaw/plugin-sdk") ?? []),
|
||||
]);
|
||||
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(rootExportNames), "utf8");
|
||||
for (const [subpath, exportNames] of Object.entries(mockSdkSubpathExports)) {
|
||||
await writeFile(path.join(pluginSdkDir, `${subpath}.js`), mockSdkSubpathSource(exportNames), "utf8");
|
||||
const specifier = `openclaw/plugin-sdk/${subpath}`;
|
||||
await writeFile(
|
||||
path.join(pluginSdkDir, `${subpath}.js`),
|
||||
mockSdkSubpathSource(exportNames, imports.bySpecifier.get(specifier) ?? new Set(), {
|
||||
zod: subpath === "zod",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
for (const specifier of imports.openclawSdkSpecifiers) {
|
||||
if (specifier === "openclaw/plugin-sdk") {
|
||||
@ -428,6 +439,9 @@ export async function resolve(specifier, context, nextResolve) {
|
||||
const subpath = specifier.slice("openclaw/plugin-sdk/".length);
|
||||
return moduleUrl(path.join(pluginSdkDir, \`\${subpath}.js\`));
|
||||
}
|
||||
if (externalMap.has(specifier)) {
|
||||
return moduleUrl(externalMap.get(specifier));
|
||||
}
|
||||
try {
|
||||
return await nextResolve(specifier, context);
|
||||
} catch (error) {
|
||||
@ -531,6 +545,15 @@ function genericExportStatement(name) {
|
||||
if (["createChatChannelPlugin", "createPlugin", "defineChannelPluginEntry", "definePlugin", "definePluginEntry", "defineSetupPluginEntry"].includes(name)) {
|
||||
return name === "definePluginEntry" ? "export { definePluginEntry };" : `export const ${name} = definePluginEntry;`;
|
||||
}
|
||||
if (name === "defineBundledChannelEntry") {
|
||||
return "export { defineBundledChannelEntry };";
|
||||
}
|
||||
if (name === "defineBundledChannelSetupEntry") {
|
||||
return "export { defineBundledChannelSetupEntry };";
|
||||
}
|
||||
if (name === "loadBundledEntryExportSync") {
|
||||
return "export { loadBundledEntryExportSync };";
|
||||
}
|
||||
if (/^[A-Z].*Schema$/u.test(name)) {
|
||||
return `export const ${name} = createSchema();`;
|
||||
}
|
||||
@ -538,7 +561,13 @@ function genericExportStatement(name) {
|
||||
}
|
||||
|
||||
function genericMockRuntimeSource(options = {}) {
|
||||
return `${options.includeSdkRuntime ? `function definePluginEntry(entry) {
|
||||
return `${options.includeSdkRuntime ? `import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const pendingBundledEntryLoads = new Set();
|
||||
|
||||
function definePluginEntry(entry) {
|
||||
if (entry && typeof entry === "object" && typeof entry.register === "function") {
|
||||
return entry;
|
||||
}
|
||||
@ -547,9 +576,86 @@ function genericMockRuntimeSource(options = {}) {
|
||||
}
|
||||
return typeof entry === "function" ? { register: entry } : entry;
|
||||
}
|
||||
|
||||
function defineBundledChannelEntry(entry = {}) {
|
||||
return {
|
||||
...entry,
|
||||
kind: "bundled-channel-entry",
|
||||
async register(api) {
|
||||
if (api?.registrationMode === "cli-metadata") {
|
||||
return entry.registerCliMetadata?.(api);
|
||||
}
|
||||
if (api?.registrationMode !== "tool-discovery") {
|
||||
api?.registerChannel?.({
|
||||
id: entry.id,
|
||||
name: entry.name,
|
||||
description: entry.description,
|
||||
plugin: { id: entry.id, name: entry.name },
|
||||
});
|
||||
}
|
||||
entry.registerCliMetadata?.(api);
|
||||
const result = entry.registerFull?.(api);
|
||||
if (result && typeof result.then === "function") {
|
||||
await result;
|
||||
}
|
||||
await drainBundledEntryLoads();
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function defineBundledChannelSetupEntry(entry = {}) {
|
||||
return {
|
||||
...entry,
|
||||
kind: "bundled-channel-setup-entry",
|
||||
};
|
||||
}
|
||||
|
||||
function loadBundledEntryExportSync(importMetaUrl, options = {}) {
|
||||
return (...args) => {
|
||||
const promise = import(resolveBundledEntryUrl(importMetaUrl, options.specifier)).then((module) => {
|
||||
const loaded = module[options.exportName] ?? module.default;
|
||||
return typeof loaded === "function" ? loaded(...args) : loaded;
|
||||
});
|
||||
pendingBundledEntryLoads.add(promise);
|
||||
promise.finally(() => pendingBundledEntryLoads.delete(promise));
|
||||
return promise;
|
||||
};
|
||||
}
|
||||
|
||||
async function drainBundledEntryLoads() {
|
||||
while (pendingBundledEntryLoads.size > 0) {
|
||||
await Promise.all([...pendingBundledEntryLoads]);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveBundledEntryUrl(importMetaUrl, specifier) {
|
||||
const basePath = fileURLToPath(importMetaUrl);
|
||||
const target = specifier ? path.resolve(path.dirname(basePath), specifier) : basePath;
|
||||
const resolved = resolveExistingSourcePath(target);
|
||||
return pathToFileURL(resolved).href;
|
||||
}
|
||||
|
||||
function resolveExistingSourcePath(target) {
|
||||
if (existsSync(target)) {
|
||||
return target;
|
||||
}
|
||||
const parsed = path.parse(target);
|
||||
const withoutJsExtension = [".js", ".mjs", ".cjs"].includes(parsed.ext) ? path.join(parsed.dir, parsed.name) : null;
|
||||
const candidates = [
|
||||
...(withoutJsExtension ? [\`\${withoutJsExtension}.ts\`, \`\${withoutJsExtension}.mts\`, \`\${withoutJsExtension}.cts\`] : []),
|
||||
\`\${target}.js\`,
|
||||
\`\${target}.mjs\`,
|
||||
\`\${target}.ts\`,
|
||||
];
|
||||
return candidates.find((candidate) => existsSync(candidate)) ?? target;
|
||||
}
|
||||
` : ""}
|
||||
function createMockValue(name) {
|
||||
function fn(...args) {
|
||||
if (name === "resolvePreferredOpenClawTmpDir") {
|
||||
return process.env.TMPDIR || "/tmp";
|
||||
}
|
||||
if (name.startsWith("normalize")) {
|
||||
return typeof args[0] === "string" ? args[0] : "";
|
||||
}
|
||||
@ -728,7 +834,8 @@ function createTypeNamespace() {
|
||||
`;
|
||||
}
|
||||
|
||||
function mockSdkSource() {
|
||||
function mockSdkSource(exportNames = mockSdkExportNames) {
|
||||
const dynamicExportNames = [...exportNames].filter((name) => !mockSdkExportNames.includes(name));
|
||||
return `function normalizeEntry(entry) {
|
||||
return typeof entry === "function" ? { register: entry } : entry;
|
||||
}
|
||||
@ -1069,16 +1176,36 @@ export function normalizeSecretInputString(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function createSimpleSchema(defaultValue) {
|
||||
return {
|
||||
parse(value) {
|
||||
return value === undefined ? defaultValue : value;
|
||||
},
|
||||
default(value) {
|
||||
return createSimpleSchema(value);
|
||||
},
|
||||
optional() {
|
||||
return this;
|
||||
},
|
||||
nullable() {
|
||||
return this;
|
||||
},
|
||||
nullish() {
|
||||
return this;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSecretInputSchema() {
|
||||
return { type: "string" };
|
||||
return createSimpleSchema();
|
||||
}
|
||||
|
||||
export function buildOptionalSecretInputSchema() {
|
||||
return { anyOf: [buildSecretInputSchema(), { type: "undefined" }] };
|
||||
return createSimpleSchema();
|
||||
}
|
||||
|
||||
export function buildSecretInputArraySchema() {
|
||||
return { type: "array", items: buildSecretInputSchema() };
|
||||
return createSimpleSchema([]);
|
||||
}
|
||||
|
||||
export function registerPluginHttpRoute(options = {}) {
|
||||
@ -1538,15 +1665,20 @@ export const OPENAI_RESPONSES_STREAM_HOOKS = buildProviderStreamFamilyHooks("ope
|
||||
export const OPENROUTER_THINKING_STREAM_HOOKS = buildProviderStreamFamilyHooks("openrouter-thinking");
|
||||
export const TOOL_STREAM_DEFAULT_ON_HOOKS = buildProviderStreamFamilyHooks("tool-stream-default");
|
||||
export const pluginSdkMock = true;
|
||||
${dynamicExportNames.map(genericExportStatement).join("\n")}
|
||||
|
||||
export default {
|
||||
${mockSdkExportNames.map((name) => ` ${name},`).join("\n")}
|
||||
${[...exportNames].map((name) => ` ${name},`).join("\n")}
|
||||
};
|
||||
`;
|
||||
}
|
||||
|
||||
function mockSdkSubpathSource(exportNames) {
|
||||
return `${exportNames.map((name) => `export { ${name} } from "./index.js";`).join("\n")}
|
||||
function mockSdkSubpathSource(staticExportNames, importedExportNames, options = {}) {
|
||||
const staticNames = new Set(staticExportNames);
|
||||
const dynamicNames = [...importedExportNames].filter((name) => !staticNames.has(name));
|
||||
return `${[...staticNames].map((name) => `export { ${name} } from "./index.js";`).join("\n")}
|
||||
${dynamicNames.length > 0 ? genericMockRuntimeSource({ includeSdkRuntime: true, zod: options.zod }) : ""}
|
||||
${dynamicNames.map(genericExportStatement).join("\n")}
|
||||
export { default } from "./index.js";
|
||||
`;
|
||||
}
|
||||
|
||||
@ -1,11 +1,21 @@
|
||||
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
||||
|
||||
export const syntheticRegistrationExecutionProfiles = {
|
||||
createChatChannelPlugin: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "channel plugin factory metadata is captured before channel runtime execution",
|
||||
},
|
||||
defineChannelPluginEntry: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "entry wrapper metadata is captured before channel runtime execution",
|
||||
},
|
||||
defineBundledChannelEntry: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "bundled channel entry metadata is captured before channel runtime execution",
|
||||
},
|
||||
definePluginEntry: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
@ -16,19 +26,74 @@ export const syntheticRegistrationExecutionProfiles = {
|
||||
callableProperties: ["send", "sendMessage", "receive", "handleMessage"],
|
||||
option: "includeChannelRuntime",
|
||||
},
|
||||
registerAgentHarness: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "agent harness factories are captured as registration metadata; agent runtime execution remains isolated opt-in",
|
||||
},
|
||||
registerAgentEventSubscription: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "agent event subscriptions are captured as registration metadata before agent event dispatch",
|
||||
},
|
||||
registerAgentToolResultMiddleware: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "agent tool-result middleware is captured as registration metadata before tool-result pipeline execution",
|
||||
},
|
||||
registerAutoEnableProbe: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "auto-enable probes are captured as registration metadata before runtime activation checks",
|
||||
},
|
||||
registerCli: {
|
||||
mode: "direct",
|
||||
callableProperties: ["handler", "run", "execute"],
|
||||
},
|
||||
registerCliBackend: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "CLI backend descriptors are captured as registration metadata before backend process execution",
|
||||
},
|
||||
registerCommand: {
|
||||
mode: "direct",
|
||||
callableProperties: ["handler", "run", "execute"],
|
||||
},
|
||||
registerCodexAppServerExtensionFactory: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "Codex app server extension factories are captured as registration metadata before host UI execution",
|
||||
},
|
||||
registerCompactionProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "compaction providers are captured as registration metadata before compaction runtime execution",
|
||||
},
|
||||
registerConfigMigration: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "config migrations are captured as registration metadata before mutating stored plugin config",
|
||||
},
|
||||
registerContextEngine: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "context engine factories are captured as registration metadata; engine startup remains isolated opt-in",
|
||||
},
|
||||
registerControlUiDescriptor: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "control UI descriptors are captured as registration metadata before UI composition",
|
||||
},
|
||||
registerDetachedTaskRuntime: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "detached task runtimes are captured as registration metadata before async task execution",
|
||||
},
|
||||
registerGatewayDiscoveryService: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "gateway discovery services are captured as registration metadata before network discovery execution",
|
||||
},
|
||||
registerGatewayMethod: {
|
||||
mode: "direct",
|
||||
callableProperties: ["handler", "run", "execute", "invoke"],
|
||||
@ -46,21 +111,116 @@ export const syntheticRegistrationExecutionProfiles = {
|
||||
callableProperties: [],
|
||||
reason: "legacy hook registrar is captured as metadata; hook handlers are probed through hook events",
|
||||
},
|
||||
registerImageGenerationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "image generation providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerMemoryPromptSection: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory prompt section renderers are captured as metadata before prompt-runtime execution",
|
||||
},
|
||||
registerMediaUnderstandingProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "media understanding providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerMemoryCapability: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory capabilities are captured as registration metadata before memory runtime execution",
|
||||
},
|
||||
registerMemoryCorpusSupplement: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory corpus supplements are captured as registration metadata before memory runtime execution",
|
||||
},
|
||||
registerMemoryEmbeddingProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory embedding providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerMemoryFlushPlan: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory flush plans are captured as registration metadata before memory runtime execution",
|
||||
},
|
||||
registerMemoryRuntime: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory runtime factories are captured as metadata; external memory startup remains isolated opt-in",
|
||||
},
|
||||
registerMemoryPromptSupplement: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory prompt supplements are captured as registration metadata before prompt-runtime execution",
|
||||
},
|
||||
registerMigrationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "migration providers are captured as registration metadata before migration runtime execution",
|
||||
},
|
||||
registerMusicGenerationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "music generation providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerNodeHostCommand: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "node host commands are captured as registration metadata before host process execution",
|
||||
},
|
||||
registerNodeInvokePolicy: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "node invoke policies are captured as registration metadata before host authorization checks",
|
||||
},
|
||||
registerProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "provider descriptors are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerRealtimeTranscriptionProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "realtime transcription providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerRealtimeVoiceProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "realtime voice providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerReload: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "reload handlers are captured as registration metadata before runtime reload execution",
|
||||
},
|
||||
registerRuntimeLifecycle: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "runtime lifecycle handlers are captured as registration metadata before lifecycle dispatch",
|
||||
},
|
||||
registerSecurityAuditCollector: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "security audit collectors are captured as registration metadata before filesystem or policy scans",
|
||||
},
|
||||
registerService: {
|
||||
mode: "lifecycle-opt-in",
|
||||
callableProperties: ["start", "stop", "dispose"],
|
||||
option: "includeLifecycle",
|
||||
},
|
||||
registerSessionExtension: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "session extensions are captured as registration metadata before session runtime execution",
|
||||
},
|
||||
registerSessionSchedulerJob: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "session scheduler jobs are captured as registration metadata before scheduler execution",
|
||||
},
|
||||
registerSpeechProvider: {
|
||||
mode: "provider-opt-in",
|
||||
callableProperties: ["speak", "synthesize", "tts"],
|
||||
@ -70,6 +230,36 @@ export const syntheticRegistrationExecutionProfiles = {
|
||||
mode: "direct",
|
||||
callableProperties: ["run", "handler", "execute"],
|
||||
},
|
||||
registerToolMetadata: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "tool metadata descriptors are captured as registration metadata before tool runtime execution",
|
||||
},
|
||||
registerTextTransforms: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "text transforms are captured as registration metadata before content mutation execution",
|
||||
},
|
||||
registerVideoGenerationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "video generation providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerTrustedToolPolicy: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "trusted tool policies are captured as registration metadata before trust-policy enforcement",
|
||||
},
|
||||
registerWebFetchProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "web fetch providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerWebSearchProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "web search providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
};
|
||||
|
||||
export const defaultSyntheticHookEvents = {
|
||||
@ -186,6 +376,7 @@ export const defaultSyntheticHookContexts = {
|
||||
};
|
||||
|
||||
export const defaultSyntheticRegistrationArguments = {
|
||||
createChatChannelPlugin: [{ base: { id: "fixture-channel" }, outbound: { sendText: "function" } }],
|
||||
defineChannelPluginEntry: [{ id: "fixture-channel", setup: "function", receive: "function" }],
|
||||
definePluginEntry: [{ id: "fixture-plugin", register: "function" }],
|
||||
registerChannel: [{ id: "fixture-channel", send: "function", receive: "function" }],
|
||||
|
||||
@ -71,6 +71,7 @@ export async function buildWorkspacePlan(options = {}) {
|
||||
installStepCount: allSteps.filter((step) => step.kind === "install").length,
|
||||
auditStepCount: allSteps.filter((step) => step.kind === "audit").length,
|
||||
buildStepCount: allSteps.filter((step) => step.kind === "build").length,
|
||||
pruneDevWorkspaceDependencyStepCount: allSteps.filter((step) => step.kind === "prune-dev-workspace-deps").length,
|
||||
artifactStepCount: allSteps.filter((step) => step.kind === "prepare-artifacts").length,
|
||||
captureStepCount: allSteps.filter((step) => step.kind === "capture").length,
|
||||
syntheticProbeStepCount: allSteps.filter((step) => step.kind === "synthetic-probe").length,
|
||||
@ -179,6 +180,7 @@ export function renderWorkspacePlanMarkdown(plan, options = {}) {
|
||||
["Artifact dirs", plan.summary.artifactStepCount],
|
||||
["Install steps", plan.summary.installStepCount],
|
||||
["Audit steps", plan.summary.auditStepCount],
|
||||
["Prune dev workspace dependency steps", plan.summary.pruneDevWorkspaceDependencyStepCount],
|
||||
["Build steps", plan.summary.buildStepCount],
|
||||
["Capture steps", plan.summary.captureStepCount],
|
||||
["Synthetic probe steps", plan.summary.syntheticProbeStepCount],
|
||||
@ -218,7 +220,7 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
|
||||
const packageManager = detectPackageManager(settings.rootDir, packageDir, packageJson);
|
||||
const lockfile = findNearestLockfile(settings.rootDir, packageDir);
|
||||
const buildScript = packageJson.scripts?.build;
|
||||
const requiredCapabilities = requiredCapabilitiesFor(entrypoint);
|
||||
const requiredCapabilities = requiredCapabilitiesFor(entrypoint, packageSummary);
|
||||
const loaderStrategy = loaderStrategyFor(entrypoint);
|
||||
const blockers = [...entrypoint.blockers];
|
||||
const workspacePath = posixJoin(settings.workspaceRoot, fixtureId);
|
||||
@ -248,6 +250,14 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
|
||||
}
|
||||
|
||||
if (requiredCapabilities.includes("dependency-install")) {
|
||||
if (hasWorkspaceProtocolDevDependencies(packageJson)) {
|
||||
steps.push({
|
||||
kind: "prune-dev-workspace-deps",
|
||||
command: `node ${helperScript(settings, workspacePath, settings.pruneWorkspaceDevDepsScript, "prune-workspace-dev-deps-cli.js")}`,
|
||||
cwd: workspacePath,
|
||||
reason: "remove workspace: devDependencies from the isolated runtime install; the mock SDK supplies OpenClaw host imports",
|
||||
});
|
||||
}
|
||||
steps.push({
|
||||
kind: "install",
|
||||
command: installCommand(packageManager),
|
||||
@ -319,6 +329,7 @@ function workspaceSettings(options) {
|
||||
resultsRoot: repoRelative(options.resultsRoot ?? defaultWorkspacePlanOptions.resultsRoot),
|
||||
rootDir: path.resolve(options.rootDir ?? process.cwd()),
|
||||
syntheticProbeScript: options.syntheticProbeScript ?? defaultWorkspacePlanOptions.syntheticProbeScript,
|
||||
pruneWorkspaceDevDepsScript: options.pruneWorkspaceDevDepsScript,
|
||||
workspaceRoot: repoRelative(options.workspaceRoot ?? defaultWorkspacePlanOptions.workspaceRoot),
|
||||
};
|
||||
}
|
||||
@ -342,7 +353,7 @@ function loaderStrategyFor(entrypoint) {
|
||||
};
|
||||
}
|
||||
|
||||
function requiredCapabilitiesFor(entrypoint) {
|
||||
function requiredCapabilitiesFor(entrypoint, packageSummary = {}) {
|
||||
const capabilities = new Set();
|
||||
for (const blocker of entrypoint.blockers) {
|
||||
if (blocker.code === "dependency-install-required") {
|
||||
@ -361,7 +372,7 @@ function requiredCapabilitiesFor(entrypoint) {
|
||||
capabilities.add("side-effect-sandbox");
|
||||
}
|
||||
}
|
||||
if (entrypoint.blockers.some((blocker) => /\bopenclaw\b/.test(blocker.evidence ?? ""))) {
|
||||
if (hasHostLinkedOpenClawDependency(packageSummary)) {
|
||||
capabilities.add("target-openclaw-link");
|
||||
}
|
||||
capabilities.add("capture-shim");
|
||||
@ -369,6 +380,20 @@ function requiredCapabilitiesFor(entrypoint) {
|
||||
return [...capabilities].sort();
|
||||
}
|
||||
|
||||
function hasHostLinkedOpenClawDependency(packageSummary) {
|
||||
return [
|
||||
...(packageSummary.dependencies ?? []),
|
||||
...(packageSummary.peerDependencies ?? []),
|
||||
...(packageSummary.optionalDependencies ?? []),
|
||||
].includes("openclaw");
|
||||
}
|
||||
|
||||
function hasWorkspaceProtocolDevDependencies(packageJson) {
|
||||
return Object.values(packageJson.devDependencies ?? {}).some(
|
||||
(value) => typeof value === "string" && value.startsWith("workspace:"),
|
||||
);
|
||||
}
|
||||
|
||||
function detectPackageManager(rootDir, packageDir, packageJson) {
|
||||
const declared = typeof packageJson.packageManager === "string" ? packageJson.packageManager.split("@")[0] : null;
|
||||
if (declared) {
|
||||
@ -450,15 +475,13 @@ function runCommand(packageManager, script) {
|
||||
}
|
||||
|
||||
function captureCommand(settings, fixtureId, entrypoint, workspacePath) {
|
||||
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
|
||||
const script = helperScript(settings, workspacePath, settings.captureScript, "capture-cli.js");
|
||||
return `${settings.optInEnv} node${loader} ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
|
||||
return `${settings.optInEnv} node ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
|
||||
}
|
||||
|
||||
function syntheticProbeCommand(settings, fixtureId, entrypoint, workspacePath) {
|
||||
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
|
||||
const script = helperScript(settings, workspacePath, settings.syntheticProbeScript, "synthetic-probes-cli.js");
|
||||
return `${settings.optInEnv} node${loader} ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
|
||||
return `${settings.optInEnv} node ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
|
||||
}
|
||||
|
||||
function helperScript(settings, workspacePath, configuredScript, helperFileName) {
|
||||
|
||||
@ -3,6 +3,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { packageId } from "../src/config.js";
|
||||
import {
|
||||
buildCiPolicyReport,
|
||||
buildCiSummary,
|
||||
@ -85,6 +86,10 @@ import {
|
||||
writeSyntheticProbePlan,
|
||||
} from "../src/index.js";
|
||||
|
||||
test("package ids collapse separators and trim hyphen edges", () => {
|
||||
assert.equal(packageId("@openclaw/openclaw---Weather_Plugin!!!"), "weather-plugin");
|
||||
});
|
||||
|
||||
test("public API runs the plugin-root check and writes reports", async () => {
|
||||
const pluginRoot = await createPluginRoot();
|
||||
|
||||
@ -111,6 +116,8 @@ test("public API exposes grouped facades for common workflows", () => {
|
||||
assert.equal(fixtureSuites.runReport, runFixtureSetReport);
|
||||
assert.equal(staticInspection.inspectSourceText, inspectSourceText);
|
||||
assert.equal(reports.renderMarkdown, renderMarkdownReport);
|
||||
assert.equal(typeof reports.sanitizeArtifact, "function");
|
||||
assert.equal(typeof reports.readOpenClawTargetSurface, "function");
|
||||
assert.equal(contracts.buildCapture, buildContractCapture);
|
||||
assert.equal(contracts.validateCoverage, validateContractCoverage);
|
||||
assert.equal(ci.buildSummary, buildCiSummary);
|
||||
@ -251,6 +258,33 @@ test("public API builds fixture-set workspace and platform plans from config", a
|
||||
markdownPath: path.join(outDir, "workspace.md"),
|
||||
});
|
||||
const platform = await buildFixtureSetPlatformProbes({ plan });
|
||||
const coveredPlan = structuredClone(plan);
|
||||
coveredPlan.fixtures[0].entrypoints = [
|
||||
{
|
||||
id: "cold-import.extension:sample-plugin:index",
|
||||
status: "dependency-install-required",
|
||||
entrypoint: "plugins/sample-plugin/index.js",
|
||||
packageManager: "npm",
|
||||
loaderStrategy: {
|
||||
source: "javascript",
|
||||
primary: "node",
|
||||
alternatives: [],
|
||||
reason: "test",
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
kind: "prepare",
|
||||
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
const coveredPlatform = await buildFixtureSetPlatformProbes({
|
||||
plan: coveredPlan,
|
||||
stepCoverage({ riskCodes }) {
|
||||
return { riskCodes };
|
||||
},
|
||||
});
|
||||
const platformPaths = await writeFixtureSetPlatformProbes(platform, {
|
||||
jsonPath: path.join(outDir, "platform.json"),
|
||||
markdownPath: path.join(outDir, "platform.md"),
|
||||
@ -267,6 +301,8 @@ test("public API builds fixture-set workspace and platform plans from config", a
|
||||
assert.match(renderFixtureSetWorkspacePlanMarkdown(plan), /## Entrypoint Workspaces/);
|
||||
assert.equal(JSON.parse(await readFile(planPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
assert.equal(platform.summary.fixtureCount, 1);
|
||||
assert.equal(coveredPlatform.summary.portabilityFindingCount, 0);
|
||||
assert.ok(coveredPlatform.summary.coveredPortabilityFindingCount > 0);
|
||||
assert.deepEqual(validateFixtureSetPlatformProbes(platform), []);
|
||||
assert.match(renderFixtureSetPlatformProbesMarkdown(platform), /## Loader Probes/);
|
||||
assert.equal(JSON.parse(await readFile(platformPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
|
||||
@ -77,6 +77,7 @@ test("capture API accepts custom registrar return profiles", () => {
|
||||
|
||||
test("capture API exposes mock context helpers", async () => {
|
||||
const api = createCaptureApi({
|
||||
resolvePath: (value) => `/fixture/${value}`,
|
||||
secretValues: {
|
||||
token: "redacted",
|
||||
},
|
||||
@ -90,6 +91,7 @@ test("capture API exposes mock context helpers", async () => {
|
||||
assert.deepEqual(await api.store.list(), ["key"]);
|
||||
assert.equal(api.agent.id, "plugin-inspector-agent");
|
||||
assert.equal(api.paths.dataDir, ".plugin-inspector/data");
|
||||
assert.equal(api.resolvePath("state"), "/fixture/state");
|
||||
});
|
||||
|
||||
test("capture API can retain handlers for probes", () => {
|
||||
|
||||
@ -63,6 +63,41 @@ test("ci policy allows known blocked probes but fails unknown blockers", () => {
|
||||
assert.match(renderCiPolicyMarkdown(report), /Plugin Inspector CI Policy/);
|
||||
});
|
||||
|
||||
test("ci policy supports wildcard seam rules for generated surface blockers", () => {
|
||||
const report = buildCiPolicyReport({
|
||||
policy: {
|
||||
...policy,
|
||||
allowedBlocked: [
|
||||
...policy.allowedBlocked,
|
||||
{
|
||||
id: "generated-surface-runtime-gap",
|
||||
seam: "*",
|
||||
reasonIncludes: "generated surface has no callable runtime",
|
||||
decision: "allowed-blocked",
|
||||
until: "generated surface runtime harness lands",
|
||||
},
|
||||
],
|
||||
},
|
||||
compatibilityReport: compatibilityReport(),
|
||||
executionResults: executionResults([
|
||||
{
|
||||
seam: "before_tool_call",
|
||||
reason: "generated surface has no callable runtime",
|
||||
},
|
||||
{
|
||||
seam: "registerCommand",
|
||||
reason: "generated surface has no callable runtime",
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(
|
||||
report.checks.filter((check) => check.id.startsWith("execution-results.blocked.")).map((check) => check.action),
|
||||
["warn", "warn"],
|
||||
);
|
||||
});
|
||||
|
||||
test("ci policy fails ref diff hard regressions", () => {
|
||||
const report = buildCiPolicyReport({
|
||||
policy,
|
||||
@ -127,7 +162,7 @@ test("ci policy surfaces P0 live issues without blocking default lanes", () => {
|
||||
code: "legacy-before-agent-start",
|
||||
},
|
||||
{
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
issueClass: "inspector-gap",
|
||||
fixture: "wecom",
|
||||
code: "registration-capture-gap",
|
||||
@ -225,7 +260,7 @@ test("ci policy writer emits JSON and Markdown artifacts", async () => {
|
||||
function compatibilityReport(overrides = {}) {
|
||||
const issues = overrides.issues ?? [
|
||||
{
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
issueClass: "inspector-gap",
|
||||
fixture: "fixture",
|
||||
code: "registration-capture-gap",
|
||||
|
||||
@ -18,17 +18,17 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
|
||||
suggestionCount: 3,
|
||||
issueCount: 4,
|
||||
p0IssueCount: 1,
|
||||
p1IssueCount: 1,
|
||||
p1IssueCount: 0,
|
||||
liveIssueCount: 1,
|
||||
compatGapCount: 1,
|
||||
},
|
||||
issues: [
|
||||
{
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
issueClass: "inspector-gap",
|
||||
fixture: "fixture",
|
||||
code: "registration-capture-gap",
|
||||
title: "runtime registrations need capture",
|
||||
title: "runtime registrations need capture evidence",
|
||||
decision: "inspector-follow-up",
|
||||
},
|
||||
],
|
||||
@ -97,6 +97,13 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
|
||||
p95WallMs: 75,
|
||||
maxPeakRssMb: 40,
|
||||
maxCpuMsEstimate: 30,
|
||||
maxPluginPeakRssDeltaMb: 8,
|
||||
maxPluginCpuDeltaMsEstimate: 6,
|
||||
openClawLifecycleCount: 2,
|
||||
p50OpenClawImportMs: 12,
|
||||
p50OpenClawActivationMs: 3,
|
||||
rssSampleCount: 2,
|
||||
cpuSampleCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -108,9 +115,12 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
|
||||
assert.equal(summary.summary.platformWindowsRisks, 3);
|
||||
assert.equal(summary.summary.loaderJitiCandidates, 1);
|
||||
assert.equal(summary.summary.importLoopP50Ms, 50);
|
||||
assert.equal(summary.summary.importLoopMetricBasis, "baseline-adjusted");
|
||||
assert.equal(summary.summary.importLoopMaxRssMb, 8);
|
||||
assert.equal(summary.summary.importLoopOpenClawImportP50Ms, 12);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /Crabpot CI Summary/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /Windows portability risks/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ max RSS 40 MB \/ CPU 30 ms/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ plugin delta RSS 8 MB \/ plugin delta CPU 6 ms \/ OpenClaw import 12 ms \/ activate 3 ms/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /\| P0 issues\s+\| 1\s+\|/);
|
||||
});
|
||||
|
||||
|
||||
@ -87,6 +87,25 @@ test("check command can target a plugin root and use runtime aliases", async ()
|
||||
assert.equal(capture.summary.capturedCount, 1);
|
||||
});
|
||||
|
||||
test("check command sanitizes absolute OpenClaw paths in JSON output and artifacts", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-sanitize-");
|
||||
const openclawPath = await createTargetOpenClaw(rootDir);
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
const { stdout } = await execFileAsync(
|
||||
process.execPath,
|
||||
[cliPath, "check", "--out", "reports", "--openclaw", openclawPath, "--json"],
|
||||
{ cwd: rootDir },
|
||||
);
|
||||
const output = JSON.parse(stdout);
|
||||
const artifact = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector-report.json"), "utf8"));
|
||||
|
||||
assert.equal(output.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
|
||||
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
|
||||
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
|
||||
assert.doesNotMatch(stdout, new RegExp(escapeRegExp(openclawPath)));
|
||||
});
|
||||
|
||||
test("inspect command runs from a plugin root and can write CI outputs", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-inspect-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
@ -318,3 +337,18 @@ async function createCliPluginRoot(prefix) {
|
||||
);
|
||||
return rootDir;
|
||||
}
|
||||
|
||||
async function createTargetOpenClaw(rootDir) {
|
||||
const openclawPath = path.join(rootDir, "target-openclaw");
|
||||
await mkdir(path.join(openclawPath, "src/plugins/compat"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(openclawPath, "src/plugins/compat/registry.ts"),
|
||||
'export const records = [{ code: "legacy-root-sdk-import", status: "deprecated" }];\n',
|
||||
"utf8",
|
||||
);
|
||||
return openclawPath;
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
@ -71,6 +71,23 @@ test("cold import readiness preserves combined blocker evidence", () => {
|
||||
assert.deepEqual(validateColdImportReadiness(readiness), []);
|
||||
});
|
||||
|
||||
test("cold import readiness treats openclaw as a host-linked dependency", () => {
|
||||
const readiness = buildColdImportReadiness({
|
||||
report: readinessReport(
|
||||
[{ kind: "extension", specifier: "./index.js", relativePath: "index.js", exists: true }],
|
||||
{
|
||||
dependencies: ["openclaw"],
|
||||
},
|
||||
),
|
||||
});
|
||||
const entrypoint = readiness.fixtures[0].entrypoints[0];
|
||||
|
||||
assert.equal(entrypoint.status, "ready");
|
||||
assert.deepEqual(entrypoint.blockers, []);
|
||||
assert.equal(readiness.summary.dependencyInstallRequiredCount, 0);
|
||||
assert.deepEqual(validateColdImportReadiness(readiness), []);
|
||||
});
|
||||
|
||||
test("cold import readiness validation rejects incomplete entries", () => {
|
||||
const errors = validateColdImportReadiness({
|
||||
fixtures: [
|
||||
|
||||
@ -26,7 +26,7 @@ test("contract coverage fails missing evidence and P1 probe gaps", () => {
|
||||
fixture: "fixture",
|
||||
severity: "P1",
|
||||
issueClass: "inspector-gap",
|
||||
code: "registration-capture-gap",
|
||||
code: "conversation-access-hook",
|
||||
evidence: [],
|
||||
},
|
||||
],
|
||||
@ -165,4 +165,20 @@ test("contract coverage requires compat record reconciliation evidence", () => {
|
||||
compatRecord: "fixture.provider-auth-env-vars",
|
||||
});
|
||||
assert.deepEqual(validateContractCoverage(report), []);
|
||||
|
||||
report.logs = [];
|
||||
report.issues.push({
|
||||
id: "PLUGIN-COMPAT",
|
||||
fixture: "fixture",
|
||||
severity: "P1",
|
||||
issueClass: "compat-gap",
|
||||
code: "provider-auth-env-vars",
|
||||
evidence: ["fixture"],
|
||||
compatRecord: "fixture.provider-auth-env-vars",
|
||||
});
|
||||
report.contractProbes.push({
|
||||
fixture: "fixture",
|
||||
id: "compat.provider-auth-env-vars:fixture",
|
||||
});
|
||||
assert.deepEqual(validateContractCoverage(report), []);
|
||||
});
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { buildContractProbes, contractProbeRules, probePriority } from "../src/advanced.js";
|
||||
import { buildContractProbes, compatRecordForIssueCode, contractProbeRules, probePriority } from "../src/advanced.js";
|
||||
|
||||
test("contract probes map issue findings to executable backlog rows", () => {
|
||||
const probes = buildContractProbes({
|
||||
@ -41,11 +41,13 @@ test("contract probes map issue findings to executable backlog rows", () => {
|
||||
});
|
||||
|
||||
assert.ok(contractProbeRules["registration-capture-gap"]);
|
||||
assert.equal(compatRecordForIssueCode("registration-capture-gap"), "api.capture.runtime-registrars");
|
||||
assert.equal(compatRecordForIssueCode("package-dependency-install-required"), undefined);
|
||||
assert.deepEqual(
|
||||
probes.map((probe) => [probe.id, probe.priority, probe.target]),
|
||||
[
|
||||
["api.capture.runtime-registrars:wecom", "P1", "inspector-capture-api"],
|
||||
["sdk.import.package-export-cold-import:codex-app-server", "P1", "sdk-alias"],
|
||||
["api.capture.runtime-registrars:wecom", "P2", "inspector-capture-api"],
|
||||
["manifest.schema.top-level-fields:agentchat", "P3", "manifest-loader"],
|
||||
],
|
||||
);
|
||||
@ -53,6 +55,7 @@ test("contract probes map issue findings to executable backlog rows", () => {
|
||||
|
||||
test("contract probe priority escalates critical codes and high-priority fixtures", () => {
|
||||
assert.equal(probePriority("sdk-export-missing", "medium"), "P1");
|
||||
assert.equal(probePriority("registration-capture-gap", "high"), "P2");
|
||||
assert.equal(probePriority("manifest-unknown-fields", "high"), "P2");
|
||||
assert.equal(probePriority("manifest-unknown-fields", "medium"), "P3");
|
||||
});
|
||||
|
||||
@ -30,12 +30,19 @@ test("import loop profile measures repeated cold capture subprocesses", async ()
|
||||
|
||||
assert.deepEqual(validateImportLoopProfile(profile), []);
|
||||
assert.equal(profile.summary.runs, 2);
|
||||
assert.equal(profile.summary.baselineRuns, 2);
|
||||
assert.equal(profile.summary.baselineFailCount, 0);
|
||||
assert.equal(profile.summary.failCount, 0);
|
||||
assert.ok(profile.summary.capturedCount >= 2);
|
||||
assert.ok(profile.summary.p50WallMs > 0);
|
||||
assert.ok(profile.summary.p50PluginWallDeltaMs >= 0);
|
||||
assert.ok(profile.summary.maxPluginPeakRssDeltaMb >= 0);
|
||||
assert.ok(profile.baseline.reference.wallMs > 0);
|
||||
assert.ok(profile.samples.every((sample) => Number.isFinite(sample.pluginCpuDeltaMsEstimate)));
|
||||
assert.ok(profile.samples.every((sample) => sample.exitCode === 0));
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /Import Loop Profile/);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /CPU Estimate/);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /Harness Baseline/);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /Plugin CPU Delta/);
|
||||
});
|
||||
|
||||
test("import loop profile can use a custom capture script and opt-in env", async () => {
|
||||
@ -50,7 +57,7 @@ test("import loop profile can use a custom capture script and opt-in env", async
|
||||
"const [entrypoint,, outputPath] = process.argv.slice(2);",
|
||||
"if (process.env.CUSTOM_IMPORT_LOOP !== '1') throw new Error('missing opt-in');",
|
||||
"await mkdir(path.dirname(outputPath), { recursive: true });",
|
||||
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }] }));",
|
||||
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }], openClawLifecycle: { importMs: 12, activationMs: 3, importPhase: 'full', activationPhase: 'full:register' } }));",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
@ -64,7 +71,12 @@ test("import loop profile can use a custom capture script and opt-in env", async
|
||||
});
|
||||
|
||||
assert.equal(profile.summary.failCount, 0);
|
||||
assert.equal(profile.summary.baselineRuns, 1);
|
||||
assert.equal(profile.summary.capturedCount, 1);
|
||||
assert.equal(profile.summary.openClawLifecycleCount, 1);
|
||||
assert.equal(profile.summary.p50OpenClawImportMs, 12);
|
||||
assert.equal(profile.summary.p50OpenClawActivationMs, 3);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /OpenClaw Import/);
|
||||
});
|
||||
|
||||
test("import loop profile validation rejects failed or empty captures", () => {
|
||||
|
||||
@ -43,6 +43,18 @@ test("source inspection records hook, registrar, and SDK import evidence", () =>
|
||||
]);
|
||||
});
|
||||
|
||||
test("source inspection strips long comments before matching registrations", () => {
|
||||
const inspection = inspectSourceText(
|
||||
[`/* ${"a/*".repeat(512)} */`, "api.registerTool({ name: 'weather' });"].join("\n"),
|
||||
"plugins/example/index.ts",
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
inspection.registrations.map((registration) => `${registration.name}@${registration.ref}`),
|
||||
["registerTool@plugins/example/index.ts:2"],
|
||||
);
|
||||
});
|
||||
|
||||
test("fixture set inspection produces a passing report", async () => {
|
||||
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
|
||||
const report = await inspectFixtureSet(config);
|
||||
@ -65,6 +77,44 @@ test("fixture set inspection reports missing expected seams", async () => {
|
||||
assert.match(report.breakages[0].message, /llm_output/);
|
||||
});
|
||||
|
||||
test("fixture set inspection treats channel factories as channel registration coverage", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-channel-factory-"));
|
||||
await mkdir(path.join(dir, "fixture"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(dir, "fixture", "index.js"),
|
||||
[
|
||||
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
|
||||
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
|
||||
"",
|
||||
"export const channel = createChatChannelPlugin({ id: 'fixture-channel' });",
|
||||
"export default defineBundledChannelEntry({ id: 'bundled-channel' });",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const report = await inspectFixtureSet({
|
||||
version: 1,
|
||||
submoduleRoot: ".",
|
||||
rootDir: dir,
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
path: "fixture",
|
||||
repo: "https://github.com/openclaw/fixture.git",
|
||||
priority: "high",
|
||||
seams: ["channel"],
|
||||
expect: {
|
||||
registrations: ["registerChannel"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(report.breakages, []);
|
||||
assert.deepEqual(report.fixtures[0].registrations, ["createChatChannelPlugin", "defineBundledChannelEntry"]);
|
||||
});
|
||||
|
||||
test("capture entrypoint imports a local fixture and records registrations", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-capture-"));
|
||||
const entrypoint = path.join(dir, "fixture.mjs");
|
||||
@ -103,6 +153,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
|
||||
'import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";',
|
||||
'import { GPT5_BEHAVIOR_CONTRACT } from "openclaw/plugin-sdk/provider-model-shared";',
|
||||
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
|
||||
'import { registerPluginHttpRoute, resolveWebhookPath } from "openclaw/plugin-sdk/webhook-ingress";',
|
||||
"",
|
||||
@ -123,6 +174,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
|
||||
"",
|
||||
"export default definePluginEntry((api) => {",
|
||||
" if (!pluginSdkMock) throw new Error('expected mock SDK');",
|
||||
" if (!GPT5_BEHAVIOR_CONTRACT) throw new Error('expected dynamic subpath mock export');",
|
||||
" provider.register(api);",
|
||||
" api.registerHttpRoute({ path: resolveWebhookPath('hook'), handler() {} });",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
@ -144,3 +196,153 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
|
||||
["registration:registerProvider", "registration:registerHttpRoute", "registration:registerTool"],
|
||||
);
|
||||
});
|
||||
|
||||
test("mock capture accepts valid output when plugin code dirties process exit code", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-exit-code-"));
|
||||
const entrypoint = path.join(dir, "index.mjs");
|
||||
await writeFile(
|
||||
entrypoint,
|
||||
[
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
"process.exitCode = 1;",
|
||||
"export default definePluginEntry({",
|
||||
" register(api) {",
|
||||
" api.registerProvider({ id: 'fixture-provider' });",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.mjs", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
|
||||
"registration:registerProvider",
|
||||
]);
|
||||
});
|
||||
|
||||
test("mock capture prefers discovered bare mocks over installed dependency exports", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-bare-capture-"));
|
||||
await mkdir(path.join(dir, "node_modules/typebox"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(dir, "node_modules/typebox/package.json"),
|
||||
JSON.stringify({ name: "typebox", version: "0.0.0", type: "module", exports: "./index.js" }, null, 2),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(dir, "node_modules/typebox/index.js"), "export const Type = {};\n", "utf8");
|
||||
const entrypoint = path.join(dir, "index.mjs");
|
||||
await writeFile(
|
||||
entrypoint,
|
||||
[
|
||||
"import path from 'node:path';",
|
||||
'import { Static, Type } from "typebox";',
|
||||
'import { resolvePreferredOpenClawTmpDir } from "fixture-api";',
|
||||
"export function register(api) {",
|
||||
" if (!Static || !Type) throw new Error('expected mocked typebox exports');",
|
||||
" path.join(resolvePreferredOpenClawTmpDir(), 'fixture');",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: Type.Object({}), run() {} });",
|
||||
"}",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.mjs", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), ["registration:registerTool"]);
|
||||
});
|
||||
|
||||
test("mock capture expands bundled channel entry registration shells", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-capture-"));
|
||||
const entrypoint = path.join(dir, "index.mjs");
|
||||
await writeFile(
|
||||
entrypoint,
|
||||
[
|
||||
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
|
||||
"export default defineBundledChannelEntry({",
|
||||
" id: 'fixture-channel',",
|
||||
" name: 'Fixture Channel',",
|
||||
" description: 'Fixture channel',",
|
||||
" plugin: { specifier: './channel.js', exportName: 'fixtureChannel' },",
|
||||
" registerFull(api) {",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.mjs", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
|
||||
"registration:registerChannel",
|
||||
"registration:registerTool",
|
||||
]);
|
||||
});
|
||||
|
||||
test("mock capture follows bundled channel linked registerFull exports", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-linked-capture-"));
|
||||
await writeFile(
|
||||
path.join(dir, "index.ts"),
|
||||
[
|
||||
'import { defineBundledChannelEntry, loadBundledEntryExportSync } from "openclaw/plugin-sdk/channel-entry-contract";',
|
||||
"",
|
||||
"function registerFull(api) {",
|
||||
" const register = loadBundledEntryExportSync(import.meta.url, {",
|
||||
" specifier: './api.js',",
|
||||
" exportName: 'registerFixtureFull',",
|
||||
" });",
|
||||
" register(api);",
|
||||
"}",
|
||||
"",
|
||||
"export default defineBundledChannelEntry({",
|
||||
" id: 'fixture-channel',",
|
||||
" name: 'Fixture Channel',",
|
||||
" description: 'Fixture channel',",
|
||||
" plugin: { specifier: './channel-plugin-api.js', exportName: 'fixtureChannel' },",
|
||||
" registerFull,",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(dir, "api.ts"),
|
||||
[
|
||||
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
|
||||
"",
|
||||
"const schema = buildSecretInputSchema().optional();",
|
||||
"",
|
||||
"export function registerFixtureFull(api) {",
|
||||
" schema.parse(undefined);",
|
||||
" api.registerCommand({ name: 'fixture.command', run() {} });",
|
||||
"}",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.ts", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
|
||||
"registration:registerChannel",
|
||||
"registration:registerCommand",
|
||||
]);
|
||||
});
|
||||
|
||||
@ -19,18 +19,18 @@ test("issue ids are stable fingerprints", () => {
|
||||
test("issue classification separates live breaks from compat and deprecation buckets", () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "untracked SDK alias is a blocking live issue",
|
||||
name: "untracked SDK alias is a compat gap",
|
||||
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
|
||||
targetOpenClaw: { compatRecordStatuses: {} },
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "live-issue", compatStatus: "untracked", severity: "P0", live: true },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "untracked", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "active SDK alias compat avoids false P0 escalation",
|
||||
name: "active SDK alias compat stays a compat row",
|
||||
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
|
||||
targetOpenClaw: { compatRecordStatuses: { "plugin-sdk-export-aliases": "active" } },
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "live-issue", compatStatus: "active", severity: "P1", live: true },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "active", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "deprecated compat remains warning-class even when used",
|
||||
@ -46,6 +46,18 @@ test("issue classification separates live breaks from compat and deprecation buc
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "missing", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "active OpenClaw probe contract stays an inspector gap",
|
||||
finding: {
|
||||
code: "before-tool-call-probe",
|
||||
compatRecord: "hook.before_tool_call.terminal-block-approval",
|
||||
},
|
||||
targetOpenClaw: {
|
||||
compatRecordStatuses: { "hook.before_tool_call.terminal-block-approval": "active" },
|
||||
},
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "inspector-gap", compatStatus: "active", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "unknown untracked hook is P0 live break",
|
||||
finding: { code: "unknown-hook-name" },
|
||||
@ -100,17 +112,17 @@ test("issue builder applies metadata and class summaries", () => {
|
||||
assert.deepEqual(
|
||||
issues.map((issue) => [issue.fixture, issue.code, issue.severity, issue.issueClass, issue.status]),
|
||||
[
|
||||
["codex-app-server", "sdk-export-missing", "P0", "live-issue", "blocking"],
|
||||
["wecom", "registration-capture-gap", "P1", "inspector-gap", "open"],
|
||||
["codex-app-server", "sdk-export-missing", "P1", "compat-gap", "open"],
|
||||
["agentchat", "manifest-unknown-fields", "P2", "upstream-metadata", "open"],
|
||||
["wecom", "registration-capture-gap", "P2", "inspector-gap", "open"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(summarizeIssueClasses(issues), {
|
||||
"compat-gap": 0,
|
||||
"compat-gap": 1,
|
||||
"deprecation-warning": 0,
|
||||
"fixture-regression": 0,
|
||||
"inspector-gap": 1,
|
||||
"live-issue": 1,
|
||||
"live-issue": 0,
|
||||
"upstream-metadata": 1,
|
||||
});
|
||||
});
|
||||
|
||||
@ -136,6 +136,7 @@ test("OpenClaw target parsing helpers stay deterministic", () => {
|
||||
);
|
||||
assert.deepEqual(
|
||||
parseCompatRecordEntries(`
|
||||
${"{{".repeat(256)}
|
||||
{ code: "b", status: "supported" }
|
||||
{ code: "a", status: "deprecated" }
|
||||
{ code: "b", status: "supported" }
|
||||
|
||||
@ -40,11 +40,11 @@ test("platform probes classify loader and shell portability risks", () => {
|
||||
},
|
||||
{
|
||||
kind: "capture",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx capture.mjs ./src/index.ts",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./src/index.ts --mock-sdk",
|
||||
},
|
||||
{
|
||||
kind: "synthetic-probe",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx synthetic.mjs --entrypoint ./src/index.ts",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node synthetic.mjs --entrypoint ./src/index.ts --mock-sdk",
|
||||
},
|
||||
],
|
||||
},
|
||||
@ -63,7 +63,68 @@ test("platform probes classify loader and shell portability risks", () => {
|
||||
assert.match(renderPlatformProbesMarkdown(report), /rsync/);
|
||||
});
|
||||
|
||||
test("platform probe validation requires jiti fallback and reflected tsx commands", () => {
|
||||
test("platform probes separate executor-covered portability risks from residual risks", () => {
|
||||
const report = buildPlatformProbes({
|
||||
plan: {
|
||||
generatedAt: "test",
|
||||
mode: "plan-only",
|
||||
summary: {
|
||||
fixtureCount: 1,
|
||||
},
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
entrypoints: [
|
||||
{
|
||||
id: "cold-import.extension:fixture:index",
|
||||
status: "dependency-install-required",
|
||||
entrypoint: "plugins/fixture/index.js",
|
||||
packageManager: "pnpm",
|
||||
loaderStrategy: {
|
||||
source: "javascript",
|
||||
primary: "node",
|
||||
alternatives: [],
|
||||
reason: "test",
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
kind: "prepare",
|
||||
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
|
||||
},
|
||||
{
|
||||
kind: "audit",
|
||||
command: "pnpm audit --json > ../../results/fixture/package-audit.json || true",
|
||||
},
|
||||
{
|
||||
kind: "capture",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./index.js",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
stepCoverage({ riskCodes }) {
|
||||
return {
|
||||
reason: "covered by Crabpot structured executor",
|
||||
riskCodes: riskCodes.filter((code) =>
|
||||
["posix-mkdir", "posix-env-prefix", "posix-null-failure", "rsync-required", "shell-redirection"].includes(code),
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(report.summary.portabilityFindingCount, 1);
|
||||
assert.equal(report.summary.coveredPortabilityFindingCount, 3);
|
||||
assert.equal(report.summary.windowsRiskStepCount, 1);
|
||||
assert.deepEqual(report.portabilityFindings[0].riskCodes, ["package-manager-availability"]);
|
||||
assert.ok(report.coveredPortabilityFindings.every((finding) => finding.coverage === "covered by Crabpot structured executor"));
|
||||
assert.doesNotMatch(renderPlatformProbesMarkdown(report), /replace shell mkdir/);
|
||||
assert.match(renderPlatformProbesMarkdown(report), /Covered Portability Findings/);
|
||||
});
|
||||
|
||||
test("platform probe validation requires jiti fallback and reflected TypeScript loader commands", () => {
|
||||
const errors = validatePlatformProbes({
|
||||
mode: "plan-only",
|
||||
targets: ["linux", "macos", "windows", "container"],
|
||||
@ -76,11 +137,14 @@ test("platform probe validation requires jiti fallback and reflected tsx command
|
||||
id: "cold-import.extension:fixture:index",
|
||||
loaderPrimary: "tsx",
|
||||
captureUsesTsx: true,
|
||||
captureUsesTypeScriptLoader: true,
|
||||
syntheticUsesTsx: false,
|
||||
syntheticUsesMockSdk: false,
|
||||
syntheticUsesTypeScriptLoader: false,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.ok(errors.some((error) => error.includes("Jiti fallback")));
|
||||
assert.ok(errors.some((error) => error.includes("tsx loader strategy")));
|
||||
assert.ok(errors.some((error) => error.includes("TypeScript loader strategy")));
|
||||
});
|
||||
|
||||
@ -7,10 +7,12 @@ import {
|
||||
buildSarifReport,
|
||||
buildCompatibilityReport,
|
||||
buildCompatibilityFixtureReport,
|
||||
buildIssues,
|
||||
classifyCompatibilityFixture,
|
||||
classifyCompatRecordCoverage,
|
||||
classifyPackageContracts,
|
||||
classifyTargetOpenClawCoverage,
|
||||
escapeMarkdownTableCell,
|
||||
inspectFixtureSet,
|
||||
loadInspectorConfig,
|
||||
renderCompatibilityIssuesReport,
|
||||
@ -20,10 +22,16 @@ import {
|
||||
renderMarkdownTable,
|
||||
renderTextSummary,
|
||||
writeArtifacts,
|
||||
writeCompatibilityReport,
|
||||
writeCiOutputArtifacts,
|
||||
writeReport,
|
||||
} from "../src/advanced.js";
|
||||
|
||||
test("markdown table cell escaping preserves literal backslashes", () => {
|
||||
assert.equal(escapeMarkdownTableCell(String.raw`C:\tmp|next
|
||||
line`), String.raw`C:\\tmp\|next<br>line`);
|
||||
});
|
||||
|
||||
test("markdown report includes summary and inventory", async () => {
|
||||
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
|
||||
const report = await inspectFixtureSet(config);
|
||||
@ -185,6 +193,91 @@ test("compatibility report renderer supports issue metadata and evidence links",
|
||||
assert.match(issues, /\[linked\]\(plugins\/sample\/src\/index\.ts:1\)/);
|
||||
});
|
||||
|
||||
test("compatibility report artifacts sanitize absolute OpenClaw target paths", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-sanitized-report-"));
|
||||
const absoluteOpenClawPath = path.join(outDir, "openclaw");
|
||||
const report = {
|
||||
generatedAt: "test",
|
||||
status: "pass",
|
||||
targetOpenClaw: {
|
||||
status: "ok",
|
||||
configuredPath: absoluteOpenClawPath,
|
||||
searchedPaths: [absoluteOpenClawPath],
|
||||
compatRecords: [],
|
||||
compatRecordStatuses: {},
|
||||
},
|
||||
summary: {
|
||||
fixtureCount: 1,
|
||||
highPriorityFixtures: 1,
|
||||
breakageCount: 0,
|
||||
warningCount: 0,
|
||||
suggestionCount: 0,
|
||||
decisionCount: 0,
|
||||
issueCount: 1,
|
||||
p0IssueCount: 0,
|
||||
p1IssueCount: 0,
|
||||
liveIssueCount: 0,
|
||||
liveP0IssueCount: 0,
|
||||
compatGapCount: 0,
|
||||
deprecationWarningCount: 0,
|
||||
inspectorGapCount: 1,
|
||||
upstreamIssueCount: 0,
|
||||
fixtureRegressionCount: 0,
|
||||
contractProbeCount: 0,
|
||||
},
|
||||
fixtures: [
|
||||
{
|
||||
id: "sample-plugin",
|
||||
priority: "high",
|
||||
seams: ["native-tool"],
|
||||
hooks: [],
|
||||
registrations: [],
|
||||
manifestContracts: [],
|
||||
},
|
||||
],
|
||||
breakages: [],
|
||||
warnings: [],
|
||||
suggestions: [],
|
||||
issues: [
|
||||
{
|
||||
fixture: "sample-plugin",
|
||||
code: "package-dependency-install-required",
|
||||
issueClass: "inspector-gap",
|
||||
decision: "inspector-follow-up",
|
||||
severity: "P2",
|
||||
title: `sample-plugin: path ${absoluteOpenClawPath}`,
|
||||
status: "open",
|
||||
compatStatus: "none",
|
||||
live: false,
|
||||
evidence: [absoluteOpenClawPath],
|
||||
},
|
||||
],
|
||||
contractProbes: [],
|
||||
logs: [],
|
||||
decisions: [],
|
||||
};
|
||||
|
||||
const markdown = renderCompatibilityMarkdownReport(report);
|
||||
const issues = renderCompatibilityIssuesReport(report);
|
||||
const paths = await writeCompatibilityReport(report, {
|
||||
jsonPath: path.join(outDir, "report.json"),
|
||||
markdownPath: path.join(outDir, "report.md"),
|
||||
issuesPath: path.join(outDir, "issues.md"),
|
||||
});
|
||||
const artifact = JSON.parse(await readFile(paths.jsonPath, "utf8"));
|
||||
|
||||
assert.equal(report.targetOpenClaw.configuredPath, absoluteOpenClawPath);
|
||||
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
|
||||
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
|
||||
assert.equal(artifact.issues[0].evidence[0], "<OPENCLAW_PATH>");
|
||||
assert.equal(artifact.issues[0].title, "sample-plugin: path <OPENCLAW_PATH>");
|
||||
assert.doesNotMatch(markdown, new RegExp(escapeRegExp(absoluteOpenClawPath)));
|
||||
assert.doesNotMatch(issues, new RegExp(escapeRegExp(absoluteOpenClawPath)));
|
||||
assert.match(markdown, /<OPENCLAW_PATH>/);
|
||||
assert.match(await readFile(paths.markdownPath, "utf8"), /<OPENCLAW_PATH>/);
|
||||
assert.match(await readFile(paths.issuesPath, "utf8"), /<OPENCLAW_PATH>/);
|
||||
});
|
||||
|
||||
test("compatibility report assembly classifies fixtures, issues, probes, and compat records", async () => {
|
||||
const report = await buildCompatibilityReport({
|
||||
generatedAt: "test",
|
||||
@ -277,6 +370,110 @@ test("compatibility report assembly classifies fixtures, issues, probes, and com
|
||||
assert.ok(report.decisions.some((decision) => decision.seam === "compat-registry"));
|
||||
});
|
||||
|
||||
test("compatibility report marks inspector gaps covered by runtime execution artifacts", async () => {
|
||||
const report = await buildCompatibilityReport({
|
||||
generatedAt: "test",
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
name: "Fixture",
|
||||
path: "plugins/fixture",
|
||||
priority: "high",
|
||||
seams: ["native-tool"],
|
||||
why: "covers runtime-only seams",
|
||||
},
|
||||
],
|
||||
inspections: [
|
||||
{
|
||||
id: "fixture",
|
||||
status: "ok",
|
||||
hooks: ["llm_input"],
|
||||
hookDetails: [{ name: "llm_input", ref: "plugins/fixture/src/index.ts:1" }],
|
||||
registrations: ["registerTool", "registerService", "registerCommand"],
|
||||
registrationDetails: [
|
||||
{ name: "registerTool", ref: "plugins/fixture/src/index.ts:2" },
|
||||
{ name: "registerService", ref: "plugins/fixture/src/index.ts:3" },
|
||||
{ name: "registerCommand", ref: "plugins/fixture/src/index.ts:4" },
|
||||
],
|
||||
manifestContracts: [],
|
||||
manifestFiles: [],
|
||||
sdkImports: [],
|
||||
sourceFiles: ["plugins/fixture/src/index.ts"],
|
||||
},
|
||||
],
|
||||
targetOpenClaw: {
|
||||
status: "ok",
|
||||
compatRecords: [],
|
||||
compatRecordStatuses: {},
|
||||
hookNames: ["llm_input"],
|
||||
apiRegistrars: ["registerTool", "registerService", "registerCommand"],
|
||||
capturedRegistrars: [],
|
||||
sdkExports: [],
|
||||
manifestFields: ["id"],
|
||||
manifestContractFields: [],
|
||||
},
|
||||
executionResults: {
|
||||
artifacts: [
|
||||
{
|
||||
fixture: "fixture",
|
||||
kind: "capture",
|
||||
status: "pass",
|
||||
artifactPath: ".crabpot/results/fixture/index.capture.json",
|
||||
captured: [
|
||||
"hook:llm_input",
|
||||
"registration:registerTool",
|
||||
"registration:registerService",
|
||||
"registration:registerCommand",
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
buildFixtureReport: ({ fixture, inspection }) => ({
|
||||
id: fixture.id,
|
||||
name: fixture.name,
|
||||
priority: fixture.priority,
|
||||
seams: fixture.seams,
|
||||
why: fixture.why,
|
||||
status: inspection.status,
|
||||
hooks: inspection.hooks,
|
||||
hookDetails: inspection.hookDetails,
|
||||
registrations: inspection.registrations,
|
||||
registrationDetails: inspection.registrationDetails,
|
||||
manifestContracts: inspection.manifestContracts,
|
||||
manifestFiles: [],
|
||||
sourceFiles: inspection.sourceFiles,
|
||||
pluginManifests: [],
|
||||
package: null,
|
||||
packages: [],
|
||||
sdkImports: [],
|
||||
sdkImportDetails: [],
|
||||
}),
|
||||
});
|
||||
|
||||
const coveredCodes = report.issues
|
||||
.filter((issue) => issue.status === "runtime-covered")
|
||||
.map((issue) => issue.code)
|
||||
.sort();
|
||||
assert.deepEqual(coveredCodes, [
|
||||
"conversation-access-hook",
|
||||
"registration-capture-gap",
|
||||
"runtime-tool-capture",
|
||||
]);
|
||||
assert.equal(report.summary.runtimeCoveredIssueCount, 3);
|
||||
assert.equal(report.summary.openInspectorGapCount, 0);
|
||||
assert.equal(report.summary.runtimeCoverageArtifactCount, 1);
|
||||
|
||||
const registrationIssue = report.issues.find((issue) => issue.code === "registration-capture-gap");
|
||||
assert.deepEqual(registrationIssue.runtimeCoverage.captured, [
|
||||
"registration:registerService",
|
||||
"registration:registerCommand",
|
||||
]);
|
||||
|
||||
const markdown = renderCompatibilityIssuesReport(report);
|
||||
assert.match(markdown, /## Runtime-Covered Inspector Gaps/);
|
||||
assert.match(markdown, /state: runtime-covered .* runtime:covered/);
|
||||
});
|
||||
|
||||
test("compat record coverage logs unavailable targets", () => {
|
||||
const logs = [];
|
||||
classifyCompatRecordCoverage({
|
||||
@ -305,6 +502,21 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
|
||||
`${JSON.stringify({ id: "fixture", name: "Fixture", version: "1.0.0", contracts: { tools: {} } }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(fixtureDir, "openclaw.security.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
$schema: "https://openclaw.ai/schemas/plugin-security.json",
|
||||
version: "1.0.0",
|
||||
plugin: "fixture",
|
||||
expectedBehaviors: [{ id: "api-key", description: "requires an API key" }],
|
||||
securityNotes: [{ id: "storage", description: "stores local state" }],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(fixtureDir, "src", "index.js"), "export function register() {}\n", "utf8");
|
||||
await writeFile(
|
||||
path.join(fixtureDir, "package.json"),
|
||||
@ -313,10 +525,28 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
|
||||
name: "fixture-plugin",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
files: ["src", "openclaw.plugin.json"],
|
||||
dependencies: { zod: "^1.0.0" },
|
||||
openclaw: {
|
||||
extensions: ["src/index.js"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
build: {
|
||||
openclawVersion: "2026.5.2",
|
||||
pluginSdkVersion: "2026.5.2",
|
||||
},
|
||||
install: {
|
||||
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
defaultChoice: "clawhub",
|
||||
minHostVersion: ">=2026.5.2",
|
||||
},
|
||||
release: {
|
||||
publishToClawHub: true,
|
||||
publishToNpm: true,
|
||||
},
|
||||
bundle: {
|
||||
includeInCore: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
@ -349,8 +579,35 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
|
||||
});
|
||||
|
||||
assert.equal(report.pluginManifests[0].id, "fixture");
|
||||
assert.deepEqual(report.securityManifests[0], {
|
||||
path: "plugin/openclaw.security.json",
|
||||
schema: "https://openclaw.ai/schemas/plugin-security.json",
|
||||
version: "1.0.0",
|
||||
plugin: "fixture",
|
||||
expectedBehaviorCount: 1,
|
||||
securityNoteCount: 1,
|
||||
validJson: true,
|
||||
});
|
||||
assert.equal(report.package.name, "fixture-plugin");
|
||||
assert.deepEqual(report.package.npmPack, {
|
||||
advertised: true,
|
||||
private: false,
|
||||
filesMode: "allowlist",
|
||||
files: ["src", "openclaw.plugin.json"],
|
||||
invalidFileSpecs: [],
|
||||
});
|
||||
assert.equal(report.package.openclaw.compatPluginApi, "^1.0.0");
|
||||
assert.deepEqual(report.package.openclaw.install, {
|
||||
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
defaultChoice: "clawhub",
|
||||
minHostVersion: ">=2026.5.2",
|
||||
});
|
||||
assert.deepEqual(report.package.openclaw.release, {
|
||||
publishToClawHub: true,
|
||||
publishToNpm: true,
|
||||
});
|
||||
assert.deepEqual(report.package.openclaw.unsupportedMetadata, ["openclaw.bundle"]);
|
||||
assert.deepEqual(report.package.openclaw.entrypoints[0], {
|
||||
kind: "extension",
|
||||
specifier: "src/index.js",
|
||||
@ -401,6 +658,284 @@ test("package contract classifier reports install and entrypoint blockers", () =
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "package-build-artifact-entrypoint"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "package-dependency-install-required"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "cold-import"));
|
||||
|
||||
const issues = buildIssues({
|
||||
suggestions: result.suggestions,
|
||||
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
|
||||
});
|
||||
assert.ok(
|
||||
issues.some(
|
||||
(issue) =>
|
||||
issue.code === "package-dependency-install-required" &&
|
||||
issue.title === "fixture: cold import requires dependency installation in an isolated workspace",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("package contract classifier treats openclaw as a host-linked dependency", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "fixture-plugin",
|
||||
version: "1.0.0",
|
||||
dependencies: ["openclaw"],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "dist/index.js",
|
||||
relativePath: "plugins/fixture/dist/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
result.suggestions.some((finding) => finding.code === "package-dependency-install-required"),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("package contract classifier reports broken install and release metadata", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
dependencies: [],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
buildOpenClawVersion: "2026.5.2",
|
||||
install: {
|
||||
clawhubSpec: null,
|
||||
npmSpec: "fixture-plugin",
|
||||
defaultChoice: "clawhub",
|
||||
minHostVersion: ">=2026.5.1",
|
||||
},
|
||||
release: {
|
||||
publishToClawHub: true,
|
||||
publishToNpm: true,
|
||||
},
|
||||
unsupportedMetadata: ["openclaw.bundle"],
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "dist/index.js",
|
||||
relativePath: "plugins/fixture/dist/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-install-metadata-incomplete"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-min-host-version-drift"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-openclaw-unsupported-metadata"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "package-metadata"));
|
||||
});
|
||||
|
||||
test("package contract classifier reports advertised npm pack blockers", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
npmPack: {
|
||||
advertised: true,
|
||||
private: true,
|
||||
filesMode: "allowlist",
|
||||
files: ["README.md"],
|
||||
invalidFileSpecs: ["../secrets"],
|
||||
},
|
||||
dependencies: [],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
install: {
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
},
|
||||
release: {
|
||||
publishToNpm: true,
|
||||
},
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "src/index.js",
|
||||
relativePath: "plugins/fixture/src/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-unavailable"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-metadata-missing"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "package-artifact"));
|
||||
|
||||
const globResult = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
npmPack: {
|
||||
advertised: true,
|
||||
private: false,
|
||||
filesMode: "allowlist",
|
||||
files: ["src/**/*.js", "openclaw.plugin.json"],
|
||||
invalidFileSpecs: [],
|
||||
},
|
||||
dependencies: [],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
install: {
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
},
|
||||
release: {
|
||||
publishToNpm: true,
|
||||
},
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "src/index.js",
|
||||
relativePath: "plugins/fixture/src/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(globResult.warnings.some((finding) => finding.code.startsWith("package-npm-pack-")), false);
|
||||
});
|
||||
|
||||
test("package contract classifier accepts built runtime entries for source package metadata", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
npmPack: {
|
||||
advertised: true,
|
||||
private: false,
|
||||
filesMode: "allowlist",
|
||||
files: ["dist/**", "openclaw.plugin.json"],
|
||||
invalidFileSpecs: [],
|
||||
},
|
||||
dependencies: [],
|
||||
peerDependencies: ["openclaw"],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
install: {
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
},
|
||||
release: {
|
||||
publishToNpm: true,
|
||||
},
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "./index.ts",
|
||||
relativePath: "plugins/fixture/index.ts",
|
||||
exists: false,
|
||||
requiresBuild: false,
|
||||
},
|
||||
{
|
||||
kind: "runtimeExtension",
|
||||
specifier: "./dist/index.js",
|
||||
relativePath: "plugins/fixture/dist/index.js",
|
||||
exists: true,
|
||||
requiresBuild: true,
|
||||
},
|
||||
{
|
||||
kind: "setupEntry",
|
||||
specifier: "./setup-entry.ts",
|
||||
relativePath: "plugins/fixture/setup-entry.ts",
|
||||
exists: false,
|
||||
requiresBuild: false,
|
||||
},
|
||||
{
|
||||
kind: "runtimeExtension",
|
||||
specifier: "./dist/setup-entry.js",
|
||||
relativePath: "plugins/fixture/dist/setup-entry.js",
|
||||
exists: true,
|
||||
requiresBuild: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.warnings.some((finding) => finding.code === "package-entrypoint-missing"), false);
|
||||
assert.equal(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"), false);
|
||||
assert.equal(result.decisions.some((decision) => decision.seam === "package-entrypoint"), false);
|
||||
});
|
||||
|
||||
test("target OpenClaw coverage classifier reports missing public surface", () => {
|
||||
@ -478,6 +1013,17 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
|
||||
channelEnvVars: { CHANNEL_ID: "channel id" },
|
||||
},
|
||||
],
|
||||
securityManifests: [
|
||||
{
|
||||
path: "plugins/fixture/openclaw.security.json",
|
||||
schema: "https://openclaw.ai/schemas/plugin-security.json",
|
||||
version: "1.0.0",
|
||||
plugin: "fixture",
|
||||
expectedBehaviorCount: 1,
|
||||
securityNoteCount: 1,
|
||||
validJson: true,
|
||||
},
|
||||
],
|
||||
package: null,
|
||||
},
|
||||
targetOpenClaw: {
|
||||
@ -493,13 +1039,48 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "provider-auth-env-vars"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "channel-env-vars"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "conversation-access-hook"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "unrecognized-security-manifest"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "security-manifest-schema-unavailable"));
|
||||
assert.ok(
|
||||
result.warnings.some(
|
||||
(finding) =>
|
||||
finding.code === "conversation-access-hook" &&
|
||||
finding.compatRecord === "hook.llm-observer.privacy-payload",
|
||||
),
|
||||
);
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "legacy-root-sdk-import"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-json-missing"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "registration-capture-gap"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "before-tool-call-probe"));
|
||||
assert.ok(
|
||||
result.suggestions.some(
|
||||
(finding) =>
|
||||
finding.code === "registration-capture-gap" &&
|
||||
finding.compatRecord === "api.capture.runtime-registrars",
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
result.suggestions.some(
|
||||
(finding) =>
|
||||
finding.code === "before-tool-call-probe" &&
|
||||
finding.compatRecord === "hook.before_tool_call.terminal-block-approval",
|
||||
),
|
||||
);
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "runtime-tool-capture"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "conversation-access"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "security-metadata"));
|
||||
|
||||
const issues = buildIssues({
|
||||
warnings: result.warnings,
|
||||
suggestions: result.suggestions,
|
||||
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
|
||||
});
|
||||
assert.ok(
|
||||
issues.some(
|
||||
(issue) =>
|
||||
issue.code === "unrecognized-security-manifest" &&
|
||||
issue.issueClass === "upstream-metadata" &&
|
||||
issue.severity === "P3",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("writeReport writes JSON and Markdown artifacts", async () => {
|
||||
@ -585,3 +1166,7 @@ test("markdown table helper supports padded empty-table reports", () => {
|
||||
);
|
||||
assert.equal(renderMarkdownTable([], ["Name"], { empty: "_none_" }), "_none_");
|
||||
});
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
buildRuntimeCaptureReport,
|
||||
captureEntrypoint,
|
||||
inspectCompatibilityFixtureSet,
|
||||
loadPluginRootConfig,
|
||||
writeRuntimeCaptureReport,
|
||||
@ -106,14 +107,14 @@ test("runtime capture records conversation binding resolved callbacks", async ()
|
||||
);
|
||||
});
|
||||
|
||||
test("runtime capture report classifies missing mocked SDK exports", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-"));
|
||||
test("runtime capture report synthesizes newly imported mocked SDK exports", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-missing-sdk-export",
|
||||
name: "openclaw-dynamic-sdk-export",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
@ -131,9 +132,10 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
|
||||
[
|
||||
'import { definitelyMissing } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
"",
|
||||
"export default definitelyMissing({",
|
||||
" register() {},",
|
||||
"});",
|
||||
"export function register(api) {",
|
||||
" if (!definitelyMissing) throw new Error('expected dynamic mock export');",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
"}",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
@ -142,17 +144,18 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
|
||||
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
|
||||
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
|
||||
|
||||
assert.equal(captureReport.summary.failedCount, 1);
|
||||
assert.equal(captureReport.results[0].status, "error");
|
||||
assert.equal(captureReport.results[0].failureClass, "missing-sdk-export");
|
||||
assert.equal(captureReport.results[0].missingExport, "definitelyMissing");
|
||||
assert.equal(captureReport.summary.failedCount, 0);
|
||||
assert.equal(captureReport.results[0].status, "captured");
|
||||
assert.deepEqual(captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`), [
|
||||
"registration:registerTool",
|
||||
]);
|
||||
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-out-"));
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-out-"));
|
||||
await writeRuntimeCaptureReport(captureReport, {
|
||||
jsonPath: path.join(outDir, "capture.json"),
|
||||
markdownPath: path.join(outDir, "capture.md"),
|
||||
});
|
||||
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /missing-sdk-export/);
|
||||
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
|
||||
});
|
||||
|
||||
test("runtime capture report classifies registration execution failures", async () => {
|
||||
@ -272,6 +275,83 @@ test("runtime capture supports TypeScript entrypoints, SDK subpaths, external mo
|
||||
assert.match(captureReport.results[0].processOutput.stdout, /late plugin noise/);
|
||||
});
|
||||
|
||||
test("runtime capture synthesizes manifest config before plugin registration", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-config-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-configured-memory",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.ts"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "openclaw.plugin.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
id: "configured-memory",
|
||||
configSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
embedding: {
|
||||
type: "object",
|
||||
minProperties: 1,
|
||||
properties: {
|
||||
provider: { type: "string" },
|
||||
model: { type: "string" },
|
||||
},
|
||||
},
|
||||
autoCapture: { type: "boolean" },
|
||||
autoRecall: { type: "boolean" },
|
||||
},
|
||||
required: ["embedding"],
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "src", "index.ts"),
|
||||
[
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
"export default definePluginEntry({",
|
||||
" register(api) {",
|
||||
" if (!api.pluginConfig?.embedding) {",
|
||||
" api.registerService({ id: 'configured-memory-disabled', start() {} });",
|
||||
" return;",
|
||||
" }",
|
||||
" api.on('agent_end', () => undefined);",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("src/index.ts", {
|
||||
cwd: rootDir,
|
||||
pluginRoot: rootDir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(
|
||||
result.captured.map((item) => `${item.kind}:${item.name}`),
|
||||
["hook:agent_end"],
|
||||
);
|
||||
});
|
||||
|
||||
test("runtime capture supports namespace imports from mocked externals", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-zod-namespace-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
|
||||
@ -103,6 +103,85 @@ test("synthetic probe plan blocks unclassified registrars", () => {
|
||||
assert.match(validateSyntheticProbePlan(plan).join("\n"), /not been classified/);
|
||||
});
|
||||
|
||||
test("synthetic probe plan classifies generated kitchen-sink registrars", () => {
|
||||
const kitchenSinkRegistrars = [
|
||||
"createChatChannelPlugin",
|
||||
"registerAgentEventSubscription",
|
||||
"registerAgentHarness",
|
||||
"registerAgentToolResultMiddleware",
|
||||
"registerAutoEnableProbe",
|
||||
"registerChannel",
|
||||
"defineBundledChannelEntry",
|
||||
"registerCli",
|
||||
"registerCliBackend",
|
||||
"registerCodexAppServerExtensionFactory",
|
||||
"registerCommand",
|
||||
"registerCompactionProvider",
|
||||
"registerConfigMigration",
|
||||
"registerContextEngine",
|
||||
"registerControlUiDescriptor",
|
||||
"registerDetachedTaskRuntime",
|
||||
"registerGatewayDiscoveryService",
|
||||
"registerGatewayMethod",
|
||||
"registerHook",
|
||||
"registerHttpRoute",
|
||||
"registerImageGenerationProvider",
|
||||
"registerInteractiveHandler",
|
||||
"registerMediaUnderstandingProvider",
|
||||
"registerMemoryCapability",
|
||||
"registerMemoryCorpusSupplement",
|
||||
"registerMemoryEmbeddingProvider",
|
||||
"registerMemoryFlushPlan",
|
||||
"registerMemoryPromptSection",
|
||||
"registerMemoryPromptSupplement",
|
||||
"registerMemoryRuntime",
|
||||
"registerMigrationProvider",
|
||||
"registerMusicGenerationProvider",
|
||||
"registerNodeHostCommand",
|
||||
"registerNodeInvokePolicy",
|
||||
"registerProvider",
|
||||
"registerRealtimeTranscriptionProvider",
|
||||
"registerRealtimeVoiceProvider",
|
||||
"registerReload",
|
||||
"registerRuntimeLifecycle",
|
||||
"registerSecurityAuditCollector",
|
||||
"registerService",
|
||||
"registerSessionExtension",
|
||||
"registerSessionSchedulerJob",
|
||||
"registerSpeechProvider",
|
||||
"registerTextTransforms",
|
||||
"registerTool",
|
||||
"registerToolMetadata",
|
||||
"registerTrustedToolPolicy",
|
||||
"registerVideoGenerationProvider",
|
||||
"registerWebFetchProvider",
|
||||
"registerWebSearchProvider",
|
||||
];
|
||||
const plan = buildSyntheticProbePlan({
|
||||
capture: {
|
||||
generatedAt: "test",
|
||||
summary: { fixtureCount: 1 },
|
||||
fixtures: [
|
||||
{
|
||||
id: "kitchen-sink",
|
||||
hooks: [],
|
||||
registrations: kitchenSinkRegistrars.map((registrar) => ({
|
||||
id: `registration.${registrar}:kitchen-sink:index`,
|
||||
registrar,
|
||||
ref: "src/generated-registrars.js",
|
||||
assertions: [`${registrar} is classified`],
|
||||
syntheticArguments: [{}],
|
||||
})),
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(plan.summary.probeCount, kitchenSinkRegistrars.length);
|
||||
assert.equal(plan.summary.blockedCount, 0);
|
||||
assert.deepEqual(validateSyntheticProbePlan(plan), []);
|
||||
});
|
||||
|
||||
test("synthetic probes invoke retained hook and tool handlers", async () => {
|
||||
const capture = await captureLocalFixture([
|
||||
"export function register(api) {",
|
||||
|
||||
@ -22,7 +22,8 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
|
||||
name: "fixture",
|
||||
packageManager: "npm@10.0.0",
|
||||
scripts: { build: "tsup" },
|
||||
dependencies: { "left-pad": "^1.3.0" },
|
||||
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
|
||||
devDependencies: { "@openclaw/plugin-sdk": "workspace:*" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
@ -57,10 +58,11 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
|
||||
assert.equal(plan.summary.artifactStepCount, 2);
|
||||
assert.equal(plan.summary.installStepCount, 1);
|
||||
assert.equal(plan.summary.auditStepCount, 1);
|
||||
assert.equal(plan.summary.pruneDevWorkspaceDependencyStepCount, 1);
|
||||
assert.equal(plan.summary.buildStepCount, 1);
|
||||
assert.equal(plan.summary.captureStepCount, 2);
|
||||
assert.equal(plan.summary.syntheticProbeStepCount, 2);
|
||||
assert.equal(plan.summary.targetOpenClawLinkStepCount, 2);
|
||||
assert.equal(plan.summary.targetOpenClawLinkStepCount, 1);
|
||||
assert.equal(plan.summary.tsLoaderEntrypointCount, 1);
|
||||
assert.equal(plan.summary.jitiAlternativeCount, 1);
|
||||
|
||||
@ -73,7 +75,14 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
|
||||
assert.ok(entrypoint.requiredCapabilities.includes("sdk-alias-compat"));
|
||||
assert.ok(entrypoint.requiredCapabilities.includes("ts-loader"));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "install" && step.command === "npm install --ignore-scripts"));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node --import tsx capture.mjs")));
|
||||
assert.ok(
|
||||
entrypoint.steps.some(
|
||||
(step) => step.kind === "prune-dev-workspace-deps" && step.command.includes("prune-workspace-dev-deps-cli.js"),
|
||||
),
|
||||
);
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node capture.mjs")));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("--mock-sdk")));
|
||||
assert.ok(entrypoint.steps.every((step) => !step.command.includes("--import tsx")));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "synthetic-probe" && step.command.includes("synthetic.mjs")));
|
||||
const buildEntrypoint = plan.fixtures[0].entrypoints.find((item) => item.packageName === "build-fixture");
|
||||
assert.ok(buildEntrypoint);
|
||||
@ -94,7 +103,7 @@ test("workspace plan defaults point at packaged helper wrappers", async (t) => {
|
||||
name: "fixture",
|
||||
packageManager: "npm@10.0.0",
|
||||
scripts: { build: "tsup" },
|
||||
dependencies: { "left-pad": "^1.3.0" },
|
||||
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
@ -226,7 +235,7 @@ function readinessReport() {
|
||||
{
|
||||
path: "plugins\\fixture\\package.json",
|
||||
name: "fixture",
|
||||
dependencies: ["left-pad"],
|
||||
dependencies: ["left-pad", "openclaw"],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user