Compare commits

...

77 Commits
v0.1.3 ... main

Author SHA1 Message Date
Vincent Koc
8899fc796c
fix(reports): accept packaged runtime entrypoints
Some checks failed
Check / Node 22 (push) Has been cancelled
2026-05-06 02:46:10 -07:00
Vincent Koc
feefb4ee23
fix(contract): accept compat-gap coverage records
Some checks failed
Check / Node 22 (push) Has been cancelled
Treat compat-gap issues as reconciliation evidence for their own compatibility records.
2026-05-05 00:50:58 -07:00
Vincent Koc
f642fb5c9f
fix(reports): quiet resolved package entrypoint P0s
Fix compatibility report classification so built runtime package entrypoints satisfy source-form OpenClaw metadata, SDK export alias misses collapse into a single compat-gap row, and P0 live issues are not repeated under the general live section.
2026-05-05 00:34:18 -07:00
Vincent Koc
68e10e0aaa
chore(release): prepare 0.3.10
Some checks failed
Check / Node 22 (push) Has been cancelled
2026-05-03 01:13:33 -07:00
Vincent Koc
12005b4658
fix(capture): accept valid mocked capture output 2026-05-03 01:09:52 -07:00
Vincent Koc
4956ad1fbc
fix(capture): follow bundled channel exports 2026-05-03 00:59:42 -07:00
Vincent Koc
a58e0785d5
fix(capture): synthesize manifest plugin config 2026-05-03 00:11:24 -07:00
Vincent Koc
9f45c8aeb6
fix(report): downgrade registration capture gaps 2026-05-02 22:43:55 -07:00
Vincent Koc
677f6e5bc1
chore(release): prepare 0.3.6 2026-05-02 19:08:49 -07:00
Vincent Koc
06cc55ce51
fix(report): accept min host version floors (#17)
* fix(report): accept min host version floors

* fix(inspector): treat bundled channel entries as channel coverage

* fix(inspector): classify bundled channel probes

* fix(policy): allow wildcard seam rules
2026-05-02 18:20:13 -07:00
Vincent Koc
2eda65a8a9
fix(report): flag unsupported openclaw bundle metadata
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-05-02 11:31:30 -07:00
Vincent Koc
4bc5fbcfa3
feat(report): reconcile runtime capture evidence 2026-05-02 11:22:20 -07:00
Vincent Koc
a6af8800e0
fix(runtime): harden plugin capture mocks 2026-05-02 10:51:35 -07:00
Vincent Koc
b919df78d3
fix(reports): flag advertised npm pack blockers 2026-05-02 09:10:57 -07:00
Vincent Koc
ff78dccff7
fix(probes): classify kitchen sink registrars 2026-05-02 08:48:39 -07:00
Vincent Koc
b33c6f725d
fix(reports): surface plugin install metadata 2026-05-02 08:40:30 -07:00
Vincent Koc
e38991a35f
fix(security): harden inspector sanitizers
Some checks failed
Check / Node 22 (push) Has been cancelled
2026-04-30 02:54:02 -07:00
Vincent Koc
eb251cbae4
feat: report openclaw lifecycle timings 2026-04-29 19:38:38 -07:00
Vincent Koc
cc89d7cea7
feat: baseline import-loop profile metrics 2026-04-29 19:26:46 -07:00
Vincent Koc
1ee105e29d
feat: flag unsupported security manifests 2026-04-29 19:18:16 -07:00
Vincent Koc
572956b8df
chore(release): v0.3.5 2026-04-29 13:51:02 -07:00
Vincent Koc
e8fb9b4380
fix(profile): expose sampler availability
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-29 13:17:17 -07:00
Vincent Koc
7b5f706398
fix(inspector): accept channel factory seams 2026-04-29 06:32:59 -07:00
Vincent Koc
862d8c9fb8
fix(synthetic): classify chat channel plugin factory 2026-04-29 06:17:27 -07:00
Vincent Koc
f6991de9b0
docs(agents): clarify host-linked openclaw dependencies
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-28 22:07:49 -07:00
Vincent Koc
e04c3fc121
chore(release): prepare 0.3.4 2026-04-28 21:51:17 -07:00
Vincent Koc
85c69fb24d
fix(platform): separate covered portability risks 2026-04-28 21:39:27 -07:00
Vincent Koc
9ab07bb316
fix(reports): treat openclaw as host-linked dependency 2026-04-28 21:34:02 -07:00
Vincent Koc
d91d596d90
fix(reports): link probe gaps to compat records 2026-04-28 21:13:15 -07:00
Vincent Koc
332706b014
fix(reports): sanitize OpenClaw target paths 2026-04-28 18:32:50 -07:00
Vincent Koc
e9e4b6704c
fix(synthetic): classify kitchen-sink registrars
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-28 13:24:09 -07:00
Vincent Koc
98e5d775f8
chore(release): prepare 0.3.2 2026-04-28 02:26:34 -07:00
Patrick Erichsen
768c3fc4d6
Merge pull request #16 from openclaw/codex/capture-binding-resolved
[codex] fix runtime capture binding callbacks
2026-04-28 02:17:08 -07:00
Patrick Erichsen
41a47e9a18 fix runtime capture binding callbacks 2026-04-28 02:16:05 -07:00
Vincent Koc
5ea07a8fca
docs: rewrite plugin-inspector README 2026-04-28 00:45:47 -07:00
Vincent Koc
85f360548e
docs(readme): clarify plugin inspector onboarding 2026-04-27 22:31:34 -07:00
Vincent Koc
5cc4d04a97
chore(release): prepare 0.3.1 2026-04-27 22:12:56 -07:00
Vincent Koc
527b2eae82
chore(release): add patch release plan 2026-04-27 21:51:04 -07:00
Vincent Koc
bdef58eeb9
test(release): guard npm package contents 2026-04-27 21:45:20 -07:00
Vincent Koc
f8073df320
ci(release): share changelog release notes 2026-04-27 21:34:13 -07:00
Vincent Koc
ae08c3cc10
docs(readme): clarify package manager usage 2026-04-27 21:30:45 -07:00
Vincent Koc
7829f2652d
docs(release): add patch readiness notes 2026-04-27 21:28:09 -07:00
Vincent Koc
9718aca09c
feat(api): add grouped root facades 2026-04-27 21:22:08 -07:00
Vincent Koc
e296985559
test(release): guard crabpot public api migration 2026-04-27 21:10:40 -07:00
Vincent Koc
33657d547c
feat(api): expose synthetic probe helpers 2026-04-27 21:03:19 -07:00
Vincent Koc
36eb577373
feat(api): expose runtime profile helpers 2026-04-27 20:55:24 -07:00
Vincent Koc
441a44691b
feat(api): expose ci rollup helpers 2026-04-27 20:48:26 -07:00
Vincent Koc
2cdfd8523f
feat(api): expose contract capture helpers 2026-04-27 20:42:21 -07:00
Vincent Koc
4b00351801
feat(api): expose report metadata helpers 2026-04-27 20:34:31 -07:00
Vincent Koc
dc0b02513c
feat(api): add fixture workspace helpers 2026-04-27 17:30:26 -07:00
Vincent Koc
90ba519b4e
feat(api): add fixture cold import helpers 2026-04-27 17:24:37 -07:00
Vincent Koc
88f6ae551f
feat(api): expose static inspection primitives 2026-04-27 17:10:37 -07:00
Vincent Koc
ce32f311d2
chore(release): prepare 0.3.0 2026-04-27 15:44:21 -07:00
Vincent Koc
5c40771272
feat(init): add json summary output 2026-04-27 15:31:14 -07:00
Vincent Koc
e6c04b7038
feat(init): add dry run preview 2026-04-27 15:26:10 -07:00
Vincent Koc
c17a293c60
feat(cli): add explicit execution opt-in flag 2026-04-27 15:16:12 -07:00
Vincent Koc
1c94019fb7
fix(init): infer source root from export maps 2026-04-27 15:07:36 -07:00
Vincent Koc
2706a8e11a
docs(changelog): note init output cleanup 2026-04-27 15:02:49 -07:00
Vincent Koc
f88a593933
fix(init): make setup output repo-relative 2026-04-27 15:02:24 -07:00
Vincent Koc
c5c520b778
docs(changelog): note init and probe updates 2026-04-27 14:52:55 -07:00
Vincent Koc
11783e3973
feat(init): add package script setup 2026-04-27 14:46:21 -07:00
Vincent Koc
26543792f1
feat(init): detect plugin package managers 2026-04-27 14:36:01 -07:00
Vincent Koc
10d9c84865
chore(release): add crabpot follow-through check 2026-04-27 14:26:04 -07:00
Vincent Koc
66c3bb785a
feat(synthetic): build probe plans from reports 2026-04-27 14:12:45 -07:00
Vincent Koc
aa3aada85f
fix(mock-sdk): keep loader fixtures alive until exit 2026-04-27 13:54:24 -07:00
Vincent Koc
51e179f304
feat(api): add fixture set report helpers 2026-04-27 13:20:11 -07:00
Vincent Koc
ed80dc2d15
feat(cli): show artifacts and top findings
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-27 12:57:10 -07:00
Vincent Koc
c89fb0b1c8
docs(ci): add sarif and junit examples 2026-04-27 12:47:22 -07:00
Vincent Koc
0044797091
feat(mock-sdk): model channel and gateway runtime seams 2026-04-27 12:43:13 -07:00
Vincent Koc
71f7541edb
chore(release): prepare 0.2.0 2026-04-27 12:24:48 -07:00
Vincent Koc
b547026f5a
fix(mock-sdk): support config schemas and provider catalogs 2026-04-27 12:08:51 -07:00
Vincent Koc
04a1d44cf4
test(api): update generated ci workflow expectation 2026-04-27 11:58:20 -07:00
Vincent Koc
a2468f9628
feat(cli): make inspect and ci outputs first class 2026-04-27 11:54:32 -07:00
Vincent Koc
89d874a68d
feat(ci): add sarif and junit report outputs 2026-04-27 11:46:17 -07:00
Vincent Koc
0b875b26cc
feat(cli): print resolved plugin config 2026-04-27 11:21:08 -07:00
Vincent Koc
d7d1e253cd
fix(capture): probe string handler registrations 2026-04-27 11:17:39 -07:00
Vincent Koc
92da17561b
feat(config): support package json plugin inspector config 2026-04-27 11:14:11 -07:00
71 changed files with 7325 additions and 404 deletions

View File

@ -76,19 +76,7 @@ jobs:
run: |
set -euo pipefail
NOTES_FILE="$(mktemp)"
RELEASE_VERSION="${RELEASE_VERSION}" node <<'NODE' > "${NOTES_FILE}"
const fs = require("fs");
const version = process.env.RELEASE_VERSION;
const lines = fs.readFileSync("CHANGELOG.md", "utf8").split(/\r?\n/);
const start = lines.findIndex((line) => line.startsWith(`## ${version} - `));
if (start === -1) {
console.error(`CHANGELOG.md is missing a ${version} release section`);
process.exit(1);
}
const end = lines.findIndex((line, index) => index > start && line.startsWith("## "));
const body = lines.slice(start + 1, end === -1 ? lines.length : end).join("\n").trim();
process.stdout.write(`## plugin-inspector v${version}\n\n${body}\n`);
NODE
node scripts/release-notes.mjs "${RELEASE_VERSION}" > "${NOTES_FILE}"
release_flags=(--title "plugin-inspector ${RELEASE_TAG}" --notes-file "${NOTES_FILE}" --verify-tag)
if [[ "${RELEASE_TAG}" =~ (alpha|beta|rc) ]]; then

View File

@ -6,6 +6,9 @@
- Do not publish npm packages without explicit owner approval.
- Preserve stable report field names and finding codes; downstream CI and
crabpot reports may consume them.
- Treat a package dependency named `openclaw` as a host-linked workspace input,
not an isolated dependency-install blocker. Keep third-party runtime
dependencies classified as install/audit blockers.
- When changing plugin-inspector behavior, CLI/package entrypoints, release
metadata, or the npm package version, update crabpot's
`@openclaw/plugin-inspector` pin/docs/smoke path as needed and run the

View File

@ -1,5 +1,129 @@
# Changelog
## Unreleased
### Fixed
- Stop classifying package source entrypoints as missing when the published package provides built runtime entrypoints, and collapse SDK alias findings into a single compat-gap row.
- Treat compat-gap issues as reconciled contract coverage for their own compatibility record.
## 0.3.10 - 2026-05-03
### Fixed
- Accept valid mocked capture output when plugin code leaves `process.exitCode` dirty.
## 0.3.9 - 2026-05-03
### Fixed
- Follow bundled channel `loadBundledEntryExportSync` registration exports during mocked runtime capture.
## 0.3.8 - 2026-05-03
### Fixed
- Synthesize manifest config for isolated runtime capture so configured hooks can be observed without credentials.
## 0.3.7 - 2026-05-03
### Changed
- Downgrade `registration-capture-gap` to advisory severity so missing capture evidence no longer reports as a P1 plugin contract risk.
## 0.3.6 - 2026-05-03
### Changed
- Report import-loop RSS and CPU as baseline-adjusted plugin deltas alongside raw subprocess metrics so Crabpot dashboards do not treat harness import cost as plugin runtime cost.
- Include optional OpenClaw loader lifecycle timings for import and full activation when a capture runner provides them.
### Fixed
- Accept plugin install minimum-host floors as supported package metadata.
- Flag unsupported legacy OpenClaw bundle metadata and advertised npm pack blockers.
- Reconcile runtime capture evidence and harden mocked capture paths for downstream fixture reports.
## 0.3.5 - 2026-04-29
### Fixed
- Add immediate/faster subprocess RSS and CPU sampling plus explicit sample counts so short import-loop reports do not silently publish fake zero-memory metrics.
- Classify `createChatChannelPlugin` as channel factory metadata in synthetic probe plans so channel-core plugins do not fail as unknown registrars.
- Treat `createChatChannelPlugin` and `defineChannelPluginEntry` as channel registration equivalents when validating fixture expectations.
- Label runtime profile wall-time summaries as command-median p95 and render missing sampled metrics as `n/a`.
## 0.3.4 - 2026-04-29
### Fixed
- Separate executor-covered platform portability findings from residual findings so downstream structured runners can keep reports blocking only on unhandled risks.
- Sanitize absolute target OpenClaw paths from generated report artifacts and JSON CLI output.
- Normalize the dependency-install inspector finding title to use isolated-workspace wording.
- Treat `openclaw` package dependencies as host-linked workspace inputs instead of isolated dependency-install blockers.
## 0.3.3 - 2026-04-28
### Fixed
- Classify generated kitchen-sink public registrar coverage in synthetic probe plans so new API-surface fixtures do not fail as unknown execution profiles.
## 0.3.2 - 2026-04-28
### Fixed
- Preserve runtime capture bindings for callback-based registrations so captured hooks and registrations can resolve their bound callback metadata.
## 0.3.1 - 2026-04-28
### Added
- Add grouped root facades: `pluginRoot`, `fixtureSuites`, `staticInspection`, `reports`, `contracts`, `ci`, `runtime`, and `synthetic`.
- Expose contract capture, contract coverage, CI rollup, runtime profile, ref/profile diff, import-loop, and synthetic probe helpers from the root package API.
- Add a release follow-through guard that fails when Crabpot scripts regress to the legacy `advanced.js` bundle.
- Add a package-contents release guard for npm tarball entrypoints, examples, and README assets.
### Changed
- Move Crabpot integration scripts to the root public API while keeping Crabpot as the fixture corpus and report consumer.
- Keep generic artifact writing out of the root API; Crabpot-owned runner scripts write their own JSON outputs.
- Ship the README banner asset in the npm package so the published README does not reference a missing local image.
- Document the grouped root import path for embedding harnesses without turning the README into a full API dump.
## 0.3.0 - 2026-04-27
### Added
- Add `--allow-execute` as a cross-platform runtime capture opt-in flag.
- Add `plugin-inspector init --dry-run` for setup previews.
- Add `plugin-inspector init --json` for machine-readable setup summaries.
- Add `plugin-inspector init --scripts` for `plugin:check` and `plugin:ci` package scripts.
- Add public fixture-set report helpers and synthetic probe suite helpers for Crabpot and downstream compatibility suites.
- Add a Crabpot follow-through release checklist for source refs, package pins, and smoke commands.
### Changed
- Make generated runtime CI commands use `--allow-execute` instead of shell-specific inline environment syntax.
- Make `plugin-inspector init --ci` detect `packageManager` and common lockfiles before generating CI install/run commands.
- Make `plugin-inspector init` output repo-relative file paths and preflight generated files before writing.
- Make `plugin-inspector init` infer `sourceRoot: "src"` from package export maps like `"./src/index.js"`.
- Improve CLI failure summaries with report artifact paths and top blocking findings.
- Harden mock SDK capture by keeping generated loader fixtures available until subprocess exit.
## 0.2.0 - 2026-04-27
### Added
- Add package.json `pluginInspector` config discovery for plugin-root checks.
- Add `plugin-inspector config` for resolved plugin-root config summaries.
- Add author-facing `plugin-inspector inspect` plugin-root flow.
- Add CI-native SARIF and JUnit outputs; `plugin-inspector ci` writes them by default.
### Changed
- Make generated CI workflows use one `plugin-inspector ci --no-openclaw --runtime --mock-sdk` command.
- Harden runtime capture for string handler registrations, parse-capable config schema helpers, and provider auth/catalog SDK mocks.
## 0.1.3 - 2026-04-27
### Added

454
README.md
View File

@ -1,60 +1,131 @@
<img src="docs/plugin-inspector-banner.jpg" alt="openclaw plugin inspector banner"/>
<img src="docs/plugin-inspector-banner.jpg" alt="OpenClaw Plugin Inspector banner">
# OpenClaw Plugin Inspector
`plugin-inspector` is the offline compatibility check for OpenClaw plugins. Run
it from a plugin root to inspect package metadata, `openclaw.plugin.json`, SDK
imports, `api.on(...)`, `api.register*`, and optional runtime registration
capture.
`@openclaw/plugin-inspector` is the offline compatibility checker for OpenClaw
plugin packages and plugin fixture suites.
It answers the questions that matter before a plugin reaches users:
- can OpenClaw discover the package metadata and `openclaw.plugin.json`
manifest?
- which hooks, registration calls, manifest contracts, and SDK imports does the
plugin use?
- does the plugin still look compatible without local OpenClaw internals?
- if CI finds a breakage, which JSON, Markdown, SARIF, JUnit, and summary
artifacts should downstream automation read?
- when a fixture-suite harness such as Crabpot runs many plugins, which findings
are hard breakages, known warnings, live issues, deprecations, or inspector
proof gaps?
The default path is static, offline, and credential-free. Runtime capture exists,
but it is opt-in because it imports plugin code.
## Requirements
- Node.js 22 or newer.
- A plugin package root with `package.json`.
- `openclaw.plugin.json` when the plugin uses the OpenClaw manifest contract.
- No OpenClaw checkout, credentials, network service, or live provider access for
default inspection.
Pass `--no-openclaw` when CI should not compare against a local OpenClaw
checkout. If an OpenClaw checkout is supplied with `--openclaw <path>`, the
inspector only reads public compatibility surfaces such as compat records, SDK
exports, hook names, manifest fields, and registrar metadata.
## Quick Start
From a plugin package directory:
Run this from a plugin package root:
```bash
npx @openclaw/plugin-inspector
npx @openclaw/plugin-inspector inspect --no-openclaw
```
That runs `check`, writes report artifacts to `reports/`, and exits non-zero
when compatibility breakages are found.
Add a local config and GitHub Actions workflow:
Equivalent one-off runners:
```bash
npx @openclaw/plugin-inspector init --ci
pnpm dlx @openclaw/plugin-inspector inspect --no-openclaw
yarn dlx @openclaw/plugin-inspector inspect --no-openclaw
bunx @openclaw/plugin-inspector inspect --no-openclaw
```
Or install it as a dev dependency:
```bash
npm install --save-dev @openclaw/plugin-inspector
npx plugin-inspector check
```
## Commands
```bash
npx @openclaw/plugin-inspector check
npx @openclaw/plugin-inspector check --plugin-root ./plugins/weather
npx @openclaw/plugin-inspector init --ci --package-manager pnpm
```
`check` reads the current directory as one plugin unless `--plugin-root` is set.
It writes:
The command writes:
- `reports/plugin-inspector-report.json`
- `reports/plugin-inspector-report.md`
- `reports/plugin-inspector-issues.md`
Use `--no-openclaw` when CI should not compare against a local OpenClaw
checkout:
It exits non-zero when hard compatibility breakages are found. Warnings,
suggestions, issue classifications, and logs stay visible in the report without
necessarily failing the command.
## Install In A Plugin Repo
Install the package when you want repeatable local scripts and CI:
```bash
plugin-inspector check --no-openclaw
npm install --save-dev @openclaw/plugin-inspector
```
Use `plugin-inspector.config.json` when CI needs stable fixture metadata,
expected seams, or runtime capture defaults:
Add scripts:
```json
{
"scripts": {
"plugin:check": "plugin-inspector inspect --no-openclaw",
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
}
}
```
Then run:
```bash
npm run plugin:check
```
The initializer can write the starter config, package scripts, and GitHub
Actions workflow:
```bash
npx @openclaw/plugin-inspector init --ci --scripts --dry-run
npx @openclaw/plugin-inspector init --ci --scripts
```
`init` detects `packageManager` and common lockfiles. Override that with
`--package-manager npm`, `--package-manager pnpm`, `--package-manager yarn`, or
`--package-manager bun`. Existing files are protected unless you pass `--force`.
## Configuration
Small plugin repos can keep configuration in `package.json`:
```json
{
"scripts": {
"plugin:check": "plugin-inspector inspect --no-openclaw",
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
},
"pluginInspector": {
"version": 1,
"plugin": {
"id": "weather",
"priority": "high",
"seams": ["dynamic-tool"],
"sourceRoot": "src",
"expect": {
"registrations": ["registerTool"]
}
},
"capture": {
"mockSdk": true
}
}
}
```
Use `plugin-inspector.config.json` for a standalone config file:
```json
{
@ -77,57 +148,99 @@ expected seams, or runtime capture defaults:
}
```
Then run:
Inspect the resolved config before wiring CI:
```bash
plugin-inspector check --config plugin-inspector.config.json
plugin-inspector config --json
```
`init --ci` writes this shape for you, plus
`.github/workflows/plugin-inspector.yml`. Copy-ready examples also live in
`examples/plugin-inspector.config.json` and
`examples/github-actions-plugin-inspector.yml`.
Copy-ready examples live in:
- `examples/plugin-inspector.config.json`
- `examples/package-json-plugin-inspector.json`
## Commands
| Command | Purpose |
| --- | --- |
| `plugin-inspector` | Default alias for `check`. |
| `plugin-inspector check` | Script-friendly plugin-root check. |
| `plugin-inspector inspect` | Plugin-root check unless `--config` is supplied; with `--config`, runs a fixture report. |
| `plugin-inspector ci` | Compatibility report plus CI summary, SARIF, and JUnit outputs. |
| `plugin-inspector config` | Print resolved plugin-root config as text or JSON. |
| `plugin-inspector init` | Write starter config, scripts, and optional GitHub Actions workflow. |
| `plugin-inspector report` | Run a fixture-suite config with many plugins. |
| `plugin-inspector capture` | Runtime-capture one entrypoint directly. |
Common options:
| Option | Meaning |
| --- | --- |
| `--plugin-root <path>` / `--root <path>` | Check a plugin somewhere other than the current directory. |
| `--config <path>` | Read a standalone config file. Required for fixture-suite `report`. |
| `--out <dir>` | Write reports somewhere other than `reports/`. |
| `--openclaw <path>` | Compare against a local OpenClaw checkout. |
| `--no-openclaw` | Disable OpenClaw checkout comparison. |
| `--runtime` / `--capture` | Add opt-in runtime registration capture. |
| `--no-runtime` / `--no-capture` | Disable runtime capture even when config enables it. |
| `--mock-sdk` / `--sdk mock` | Use generated SDK and external-package mocks for runtime capture. |
| `--real-sdk` / `--sdk real` | Use installed real SDK dependencies instead of mocks. |
| `--allow-execute` | Permit commands that import plugin code. |
| `--json` | Print machine-readable JSON to stdout. |
| `--sarif [path]` | Write SARIF from `check` or `inspect`; `ci` enables this by default. |
| `--junit [path]` | Write JUnit XML from `check` or `inspect`; `ci` enables this by default. |
| `--no-sarif` / `--no-junit` | Disable default `ci` outputs. |
Run the built-in help for the exact CLI surface:
```bash
plugin-inspector --help
```
## Runtime Capture
Runtime capture imports plugin entrypoints in an isolated subprocess and records
the registrations made during `register(api)`. It is opt-in because it executes
plugin code:
what `register(api)` does. Use it when static inspection cannot prove the actual
registrations made at runtime.
```bash
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 npx @openclaw/plugin-inspector check --runtime --mock-sdk
plugin-inspector inspect --no-openclaw --runtime --mock-sdk --allow-execute
```
By default, runtime capture uses a generated mock for `openclaw/plugin-sdk` and
common external packages so plugin code can load in clean CI without OpenClaw
installed. Use `--real-sdk` only when the plugin workspace already has real SDK
dependencies installed and you intentionally want to test that path.
`--allow-execute` is the deliberate safety switch. Without it, modes that import
plugin code fail closed. The older environment guard still works for custom
harnesses:
```bash
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector inspect --no-openclaw --runtime --mock-sdk
```
By default, runtime capture uses generated mocks for `openclaw/plugin-sdk`
subpaths and unresolved external packages discovered in the plugin import graph.
That keeps compatibility CI offline and credential-free. It does not call live
services, launch OpenClaw, run provider SDKs, or emulate service lifecycle side
effects.
Use `--real-sdk` only when the plugin workspace already has real SDK
dependencies installed and you intentionally want that path.
Runtime capture writes:
- `reports/plugin-inspector-runtime-capture.json`
- `reports/plugin-inspector-runtime-capture.md`
You can also capture one entrypoint directly:
Capture one entrypoint directly:
```bash
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector capture ./dist/index.js --mock-sdk
plugin-inspector capture ./dist/index.js --mock-sdk --allow-execute
```
## CI
Minimal package scripts:
`plugin-inspector ci` writes the normal compatibility report plus CI-native
summary, SARIF, and JUnit artifacts.
```json
{
"scripts": {
"plugin:check": "plugin-inspector check --no-openclaw",
"plugin:check:runtime": "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector check --no-openclaw --runtime --mock-sdk"
}
}
```
GitHub Actions without a local dev dependency:
Minimal GitHub Actions workflow:
```yaml
name: plugin-inspector
@ -147,8 +260,7 @@ jobs:
node-version: 24
cache: npm
- run: npm ci
- run: npx @openclaw/plugin-inspector check --no-openclaw
- run: PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 npx @openclaw/plugin-inspector check --no-openclaw --runtime --mock-sdk
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
- uses: actions/upload-artifact@v5
if: always()
with:
@ -156,38 +268,214 @@ jobs:
path: reports/plugin-inspector-*
```
Generated `ci` artifacts:
- `reports/plugin-inspector-report.json`
- `reports/plugin-inspector-report.md`
- `reports/plugin-inspector-issues.md`
- `reports/plugin-inspector-ci-summary.json`
- `reports/plugin-inspector-ci-summary.md`
- `reports/plugin-inspector.sarif`
- `reports/plugin-inspector.junit.xml`
CI examples:
- `examples/github-actions-plugin-inspector.yml`
- `examples/github-actions-code-scanning.yml`
- `examples/gitlab-ci-plugin-inspector.yml`
- `examples/circleci-plugin-inspector.yml`
## Report Surfaces
The compatibility report is the primary contract. Preserve field names and
finding codes because downstream CI and Crabpot reports may consume them.
Important report sections:
| Field | Meaning |
| --- | --- |
| `status` | `pass` unless hard breakages exist. |
| `summary` | Counts for fixtures, breakages, warnings, suggestions, issues, issue classes, and contract probes. |
| `targetOpenClaw` | Status and public compatibility data read from the optional OpenClaw checkout. |
| `fixtures` | Per-plugin metadata, hooks, registrations, manifest contracts, package data, and SDK imports. |
| `breakages` | Blocking compatibility failures. |
| `warnings` / `suggestions` | Non-blocking compatibility findings. |
| `issues` | Normalized issue rows with severity and class. |
| `contractProbes` | Suggested synthetic probes derived from observed contracts. |
| `logs` | Informational inventory and coverage rows. |
| `decisions` | Maintainer-facing follow-up or compatibility-policy decisions. |
Issue classes currently flow through the reports as live issues, compat gaps,
deprecation warnings, inspector gaps, upstream metadata, and fixture regressions.
## CI Policy And Shared Reporting Primitives
`plugin-inspector` owns the shared CI policy and report rendering primitives.
Fixture-suite harnesses such as Crabpot should call these exports instead of
reimplementing scoring, summaries, Markdown, SARIF, or JUnit handling.
The root API exposes grouped helpers:
```js
import { ci } from "@openclaw/plugin-inspector";
const policyReport = ci.buildPolicyReport({
policy,
compatibilityReport,
executionResults,
strict: false,
});
await ci.writePolicyReport(policyReport);
```
CI policy reports default to:
- `reports/plugin-inspector-ci-policy.json`
- `reports/plugin-inspector-ci-policy.md`
A policy must use `version: 1` and define:
- `allowedBlocked`
- `expectedWarnings`
- `thresholds`
- `fixtureSets`
Policy scoring fails hard breakages, unknown blocked synthetic probes, hard ref
diff regressions, failed execution results, strict live P0 issues, and strict
classified blockers. Non-strict mode keeps classified blocked probes and live P0
issues visible as warnings.
CI summary helpers read the known report set from `reports/` and render one
machine-readable and one Markdown rollup:
- compatibility
- runtime capture
- synthetic probes
- cold import readiness
- workspace plan
- platform probes
- import-loop profile
- execution results
- runtime profile
- ref diff
- profile diff
- CI policy
## Fixture Suites
Fixture-set configs are still supported for crabpot-style compatibility suites:
Most plugin authors should use the plugin-root workflow. Use fixture suites when
one repository intentionally checks many plugins or packages, as Crabpot does.
```bash
plugin-inspector report --config crabpot.config.json --out reports
plugin-inspector report --config crabpot.config.json --out reports --check
plugin-inspector ci --config crabpot.config.json --out reports --no-openclaw
```
Use fixture suites when one repo wants to inspect many plugins. Use plugin-root
`check` for normal plugin CI.
Fixture-suite configs are loaded through the explicit fixture helpers. That keeps
normal plugin-root configuration simple while still supporting bulk compatibility
harnesses.
## Mocking Model
## Public API
Default inspection is static, offline, and credential-free. Runtime capture is
the only mode that imports plugin code.
Prefer the CLI for normal plugin repositories. Import the public API when a test
harness needs to compose workflows directly:
When `--mock-sdk` is enabled, the inspector generates temporary modules for
`openclaw/plugin-sdk` subpaths and unresolved external packages discovered in
the plugin import graph. The mock SDK captures registrations; it does not call
network services, launch OpenClaw, run provider SDKs, or emulate service
lifecycle side effects.
```js
import { pluginRoot } from "@openclaw/plugin-inspector";
Use the mock lane for plugin compatibility CI. Keep live provider/service tests
in the plugin repo behind their own credentials and explicit opt-in flags.
const { report, paths } = await pluginRoot.runCheck({
pluginRoot: process.cwd(),
openclawPath: false,
outDir: "reports",
});
## Scope
console.log(report.status, paths.jsonPath);
```
Default inspection is offline and credential-free. It reads manifests, package
metadata, and source files, then reports observed `api.on(...)`,
`api.register*`, `define*`, SDK imports, and manifest contracts.
OpenClaw target checkout parsing is limited to public compatibility registries,
SDK package exports, manifest types, hooks, and captured registrar metadata.
Stable grouped facades:
Cold import capture, synthetic contract probes, and runtime capture are explicit
opt-in modes. Live lanes stay credential-gated and must never run in default CI.
| Facade | Use |
| --- | --- |
| `pluginRoot` | Load config, inspect, run checks, capture entrypoints, or set up a plugin repo. |
| `fixtureSuites` | Load fixture-suite configs, run reports, and build fixture-suite readiness plans. |
| `staticInspection` | Inspect source text or fixture sets without the compatibility report layer. |
| `reports` | Render/write reports and classify issue findings. |
| `contracts` | Build, render, validate, and write contract captures and coverage. |
| `ci` | Build summaries, policy reports, execution results, SARIF, and JUnit outputs. |
| `runtime` | Build runtime profiles, profile diffs, ref diffs, and import-loop profiles. |
| `synthetic` | Build and run synthetic probe plans. |
Named exports remain available for existing automation. Prefer the grouped
facades for new code because they show ownership and keep downstream wrappers
thin.
## Development
Repository checks are intentionally small and offline:
```bash
npm test
npm run release:contents
npm run check
```
`npm run check` runs the Node test suite and the package-contents guard. The
contents guard shells through `npm pack --dry-run --json` and verifies the npm
tarball includes package entrypoints, examples, README assets, and no private
`test/`, `scripts/`, or `.github/` paths.
Useful release-prep commands:
```bash
npm run release:local
npm run release:readiness
npm run release:notes
npm run release:plan
npm run release:crabpot -- --crabpot ../crabpot
```
`release:readiness` proves the local package and verifies Crabpot follow-through.
It does not publish.
Keep this package dependency-light. Do not add runtime dependencies unless they
remove real complexity. Default checks must stay offline and credential-free.
## Release Notes
The package publishes from annotated `v*` tags through GitHub Actions. The
release workflow runs the test suite, verifies the npm tarball, publishes the
GitHub release, and publishes the public npm package through npm trusted
publishing.
Before tagging a release:
1. Move `CHANGELOG.md` `Unreleased` notes into a versioned section.
2. Update `package.json` to the same version.
3. Update Crabpot's `pluginInspectorRef` to the release commit.
4. Run `npm run release:readiness`.
5. Run the Crabpot plugin-inspector smoke commands printed by
`npm run release:crabpot -- --crabpot ../crabpot`.
After npm publish, update Crabpot's package pin and run:
```bash
npm run release:crabpot -- --crabpot ../crabpot --published
```
Do not publish npm packages without explicit owner approval.
## Contribution Notes
There is no `CONTRIBUTING.md` in this repository. Until one exists, use the repo
scripts above as the local contract and follow these project rules:
- preserve stable report field names and finding codes;
- prefer public OpenClaw plugin contracts over core internals;
- isolate any OpenClaw source parsing behind explicit helpers;
- keep runtime execution behind `--allow-execute` or
`PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1`;
- when behavior, entrypoints, release metadata, or the npm package version
change, update Crabpot's `@openclaw/plugin-inspector` pin/docs/smoke path and
run the Crabpot plugin-inspector smoke before calling the work done.

View File

@ -55,9 +55,67 @@ npm trust github @openclaw/plugin-inspector --repo openclaw/plugin-inspector --f
npm run release:local
```
This runs tests and `npm pack --dry-run`. Once a version has been published,
`npm publish --dry-run` rejects that same version, so the real publish check is
the tag workflow.
This runs tests and the package-contents guard, which shells through
`npm pack --dry-run --json` and fails if exported files, README assets, or
expected examples are missing from the npm tarball. Once a version has been
published, `npm publish --dry-run` rejects that same version, so the real
publish check is the tag workflow.
For normal patch prep before tagging, run the combined local readiness gate:
```bash
npm run release:readiness
```
That proves the package tarball locally and verifies Crabpot source-ref
follow-through. It does not publish anything.
Before creating a tag, move the `CHANGELOG.md` `Unreleased` notes into a
versioned section like `## 0.3.1 - 2026-04-28`, update `package.json` to the
same version, and update Crabpot's `pluginInspectorRef` to the exact release
commit.
Draft release notes from the current `Unreleased` section with:
```bash
npm run release:notes
```
Preview the exact patch-release edits without changing files:
```bash
npm run release:plan
```
That prints the next patch version, release tag, changelog heading, Crabpot
source ref, and post-publish Crabpot package pin.
## Crabpot follow-through
Before tagging a release, update Crabpot's `pluginInspectorRef` to the
plugin-inspector commit being released and run:
```bash
npm run release:crabpot -- --crabpot ../crabpot
```
The checklist verifies the source ref and prints the required Crabpot smoke
commands:
```bash
CRABPOT_PLUGIN_INSPECTOR_CLI=source npm run plugin-inspector:smoke
npm run plugin-inspector:smoke
```
After the npm package is published, update Crabpot's `pluginInspectorPackage` to
the released version and run the stricter post-publish check:
```bash
npm run release:crabpot -- --crabpot ../crabpot --published
```
Do not consider a release complete until the Crabpot source ref, package pin,
local smoke proof, and Crabpot CI proof are all current.
## Publish

View File

@ -0,0 +1,19 @@
version: 2.1
jobs:
plugin-inspector:
docker:
- image: cimg/node:24.0
steps:
- checkout
- run: npm ci
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
- store_test_results:
path: reports
- store_artifacts:
path: reports
workflows:
plugin-inspector:
jobs:
- plugin-inspector

View File

@ -0,0 +1,31 @@
name: plugin-inspector
on:
pull_request:
push:
branches: [main]
permissions:
contents: read
security-events: write
jobs:
check:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v5
- uses: actions/setup-node@v5
with:
node-version: 24
cache: npm
- run: npm ci
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
- uses: github/codeql-action/upload-sarif@v3
if: always()
with:
sarif_file: reports/plugin-inspector.sarif
- uses: actions/upload-artifact@v5
if: always()
with:
name: plugin-inspector-reports
path: reports/plugin-inspector-*

View File

@ -15,8 +15,7 @@ jobs:
node-version: 24
cache: npm
- run: npm ci
- run: npx @openclaw/plugin-inspector check --no-openclaw
- run: PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 npx @openclaw/plugin-inspector check --no-openclaw --runtime --mock-sdk
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
- uses: actions/upload-artifact@v5
if: always()
with:

View File

@ -0,0 +1,11 @@
plugin_inspector:
image: node:24
script:
- npm ci
- npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
artifacts:
when: always
paths:
- reports/plugin-inspector-*
reports:
junit: reports/plugin-inspector.junit.xml

View File

@ -0,0 +1,21 @@
{
"scripts": {
"plugin:check": "plugin-inspector inspect --no-openclaw",
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
},
"pluginInspector": {
"version": 1,
"plugin": {
"id": "weather",
"priority": "high",
"seams": ["dynamic-tool"],
"sourceRoot": "src",
"expect": {
"registrations": ["registerTool"]
}
},
"capture": {
"mockSdk": true
}
}
}

View File

@ -1,6 +1,6 @@
{
"name": "@openclaw/plugin-inspector",
"version": "0.1.3",
"version": "0.3.10",
"private": false,
"description": "Offline compatibility inspector for OpenClaw plugins.",
"type": "module",
@ -36,18 +36,26 @@
"./ref-diff": "./src/ref-diff.js",
"./runtime-capture-report": "./src/runtime-capture-report.js",
"./runtime-profile": "./src/runtime-profile.js",
"./synthetic-probe-suite": "./src/synthetic-probe-suite.js",
"./synthetic-probes": "./src/synthetic-probes.js",
"./workspace-plan": "./src/workspace-plan.js"
},
"files": [
"src",
"examples",
"docs/plugin-inspector-banner.jpg",
"README.md",
"CHANGELOG.md",
"LICENSE"
],
"scripts": {
"check": "npm test && npm pack --dry-run",
"check": "npm test && npm run release:contents",
"release:crabpot": "node scripts/check-crabpot-followthrough.mjs",
"release:contents": "node scripts/check-package-contents.mjs",
"release:plan": "node scripts/release-plan.mjs",
"release:readiness": "npm run release:local && npm run release:crabpot",
"release:local": "npm run check",
"release:notes": "node scripts/release-notes.mjs --unreleased",
"test": "node --test test/*.test.js"
},
"keywords": [

View File

@ -0,0 +1,194 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import { existsSync, readFileSync, readdirSync } from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const pluginInspectorRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const options = parseArgs(process.argv.slice(2));
const result = buildCrabpotFollowthroughChecklist({
pluginInspectorRoot,
crabpotRoot: options.crabpotRoot,
expectedRef: options.expectedRef,
expectedVersion: options.expectedVersion,
requirePublishedPin: options.requirePublishedPin,
});
if (options.json) {
console.log(JSON.stringify(result, null, 2));
} else {
printChecklist(result);
}
if (result.status === "fail") {
process.exitCode = 1;
}
}
export function buildCrabpotFollowthroughChecklist(options = {}) {
const root = options.pluginInspectorRoot ?? pluginInspectorRoot;
const crabpotRoot = path.resolve(root, options.crabpotRoot ?? "../crabpot");
const expectedRef = options.expectedRef ?? gitHead(root);
const expectedVersion = options.expectedVersion ?? packageVersion(root);
const sourcePath = path.join(crabpotRoot, "scripts", "plugin-inspector-source.mjs");
const pins = existsSync(sourcePath) ? readCrabpotPins(sourcePath) : {};
const advancedConsumers = findAdvancedConsumers(crabpotRoot);
const expectedPackage = `@openclaw/plugin-inspector@${expectedVersion}`;
const checks = [
{
id: "crabpot-source-ref",
status: pins.pluginInspectorRef === expectedRef ? "pass" : "fail",
message: `crabpot source ref points at plugin-inspector ${expectedRef}`,
expected: expectedRef,
actual: pins.pluginInspectorRef ?? "missing",
fix: `update ${path.relative(process.cwd(), sourcePath)} pluginInspectorRef to ${expectedRef}`,
},
{
id: "crabpot-public-api-migration",
status: advancedConsumers.length === 0 ? "pass" : "fail",
message: "crabpot scripts use the plugin-inspector root public API",
expected: "no advanced bundle consumers",
actual: advancedConsumers.length === 0 ? "none" : advancedConsumers.join(", "),
fix: "switch listed crabpot scripts from loadPluginInspector() to loadPluginInspectorPublicApi() or local helpers",
},
{
id: "crabpot-package-pin",
status: pins.pluginInspectorPackage === expectedPackage ? "pass" : options.requirePublishedPin ? "fail" : "manual",
message: `crabpot package smoke pin uses ${expectedPackage}`,
expected: expectedPackage,
actual: pins.pluginInspectorPackage ?? "missing",
fix: `after npm publish, update ${path.relative(process.cwd(), sourcePath)} pluginInspectorPackage to ${expectedPackage}`,
},
{
id: "crabpot-source-smoke",
status: "manual",
message: "run crabpot source-mode plugin-inspector smoke",
command: "CRABPOT_PLUGIN_INSPECTOR_CLI=source npm run plugin-inspector:smoke",
},
{
id: "crabpot-package-smoke",
status: "manual",
message: "run crabpot package-mode plugin-inspector smoke",
command: "npm run plugin-inspector:smoke",
},
];
return {
status: checks.some((check) => check.status === "fail") ? "fail" : "pass",
pluginInspectorRef: expectedRef,
pluginInspectorVersion: expectedVersion,
crabpotRoot,
checks,
};
}
function parseArgs(argv) {
const options = {
crabpotRoot: "../crabpot",
expectedRef: undefined,
expectedVersion: undefined,
json: false,
requirePublishedPin: false,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--crabpot") {
options.crabpotRoot = argv[index + 1];
index += 1;
continue;
}
if (arg === "--expected-ref") {
options.expectedRef = argv[index + 1];
index += 1;
continue;
}
if (arg === "--expected-version") {
options.expectedVersion = argv[index + 1];
index += 1;
continue;
}
if (arg === "--json") {
options.json = true;
continue;
}
if (arg === "--published") {
options.requirePublishedPin = true;
}
}
return options;
}
function readCrabpotPins(sourcePath) {
const text = readFileSync(sourcePath, "utf8");
return {
pluginInspectorRef: text.match(/pluginInspectorRef\s*=\s*"([^"]+)"/)?.[1],
pluginInspectorPackage: text.match(/pluginInspectorPackage\s*=\s*"([^"]+)"/)?.[1],
};
}
function findAdvancedConsumers(crabpotRoot) {
const scriptsDir = path.join(crabpotRoot, "scripts");
if (!existsSync(scriptsDir)) {
return [];
}
const consumers = [];
for (const filePath of walkFiles(scriptsDir)) {
if (!filePath.endsWith(".mjs") || path.basename(filePath) === "plugin-inspector-source.mjs") {
continue;
}
const text = readFileSync(filePath, "utf8");
if (/\bloadPluginInspector\s*\(/.test(text) || /src["']\s*,\s*["']advanced\.js/.test(text)) {
consumers.push(path.relative(crabpotRoot, filePath));
}
}
return consumers.sort();
}
function walkFiles(dir) {
const files = [];
for (const entry of readdirSync(dir, { withFileTypes: true })) {
const filePath = path.join(dir, entry.name);
if (entry.isDirectory()) {
files.push(...walkFiles(filePath));
} else if (entry.isFile()) {
files.push(filePath);
}
}
return files;
}
function gitHead(root) {
return execFileSync("git", ["rev-parse", "HEAD"], {
cwd: root,
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
}).trim();
}
function packageVersion(root) {
return JSON.parse(readFileSync(path.join(root, "package.json"), "utf8")).version;
}
function printChecklist(result) {
console.log(`crabpot follow-through: ${result.status}`);
console.log(`plugin-inspector ref: ${result.pluginInspectorRef}`);
console.log(`plugin-inspector version: ${result.pluginInspectorVersion}`);
for (const check of result.checks) {
console.log(`- ${check.status.toUpperCase()} ${check.id}: ${check.message}`);
if (check.actual && check.actual !== check.expected) {
console.log(` expected: ${check.expected}`);
console.log(` actual: ${check.actual}`);
}
if (check.command) {
console.log(` command: ${check.command}`);
}
if (check.status === "fail" && check.fix) {
console.log(` fix: ${check.fix}`);
}
}
}

View File

@ -0,0 +1,138 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import { readFileSync } from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const root = path.resolve(process.argv[2] ?? ".");
const packageJson = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
const readmeText = readFileSync(path.join(root, "README.md"), "utf8");
const filePaths = npmPackFilePaths(root);
const result = buildPackageContentsChecklist({ filePaths, packageJson, readmeText });
printChecklist(result);
if (result.status === "fail") {
process.exitCode = 1;
}
}
export function buildPackageContentsChecklist({ filePaths, packageJson, readmeText = "" }) {
const files = new Set(filePaths);
const checks = [
...requiredFileChecks(files),
...entrypointChecks(files, packageJson),
...readmeAssetChecks(files, readmeText),
...forbiddenPathChecks(filePaths),
];
return {
status: checks.some((check) => check.status === "fail") ? "fail" : "pass",
entryCount: filePaths.length,
checks,
};
}
function requiredFileChecks(files) {
return [
"package.json",
"README.md",
"CHANGELOG.md",
"LICENSE",
"src/cli.js",
"src/index.js",
"examples/github-actions-plugin-inspector.yml",
"examples/plugin-inspector.config.json",
].map((filePath) => presenceCheck(files, "package-required-file", filePath));
}
function entrypointChecks(files, packageJson) {
const checks = [];
for (const [name, filePath] of Object.entries(packageJson.bin ?? {})) {
checks.push(presenceCheck(files, "package-bin-entry", normalizePackagePath(filePath), name));
}
for (const [name, target] of Object.entries(packageJson.exports ?? {})) {
for (const filePath of exportTargets(target)) {
checks.push(presenceCheck(files, "package-export-entry", normalizePackagePath(filePath), name));
}
}
return checks;
}
function readmeAssetChecks(files, readmeText) {
const assetPaths = new Set();
for (const match of readmeText.matchAll(/<img\s+[^>]*src=["']([^"']+)["']/gi)) {
assetPaths.add(match[1]);
}
for (const match of readmeText.matchAll(/!\[[^\]]*]\(([^)]+)\)/g)) {
assetPaths.add(match[1]);
}
return [...assetPaths]
.filter((assetPath) => isPackageRelativeAsset(assetPath))
.map((assetPath) => presenceCheck(files, "package-readme-asset", normalizePackagePath(assetPath)));
}
function forbiddenPathChecks(filePaths) {
const forbiddenPrefixes = ["test/", "scripts/", ".github/"];
return filePaths
.filter((filePath) => forbiddenPrefixes.some((prefix) => filePath.startsWith(prefix)))
.map((filePath) => ({
id: "package-forbidden-path",
status: "fail",
message: `${filePath} should not be published in the npm package`,
}));
}
function presenceCheck(files, id, filePath, detail = filePath) {
return {
id,
status: files.has(filePath) ? "pass" : "fail",
message: `${detail} includes ${filePath}`,
expected: filePath,
actual: files.has(filePath) ? filePath : "missing",
};
}
function exportTargets(target) {
if (typeof target === "string") {
return [target];
}
if (target && typeof target === "object") {
return Object.values(target).flatMap((value) => exportTargets(value));
}
return [];
}
function isPackageRelativeAsset(assetPath) {
return !assetPath.startsWith("http://")
&& !assetPath.startsWith("https://")
&& !assetPath.startsWith("#")
&& !assetPath.startsWith("/");
}
function normalizePackagePath(filePath) {
return filePath.replace(/^\.\//, "");
}
function npmPackFilePaths(root) {
const output = execFileSync("npm", ["pack", "--dry-run", "--json"], {
cwd: root,
encoding: "utf8",
stdio: ["ignore", "pipe", "inherit"],
});
const pack = JSON.parse(output)[0];
return pack.files.map((file) => file.path).sort();
}
function printChecklist(result) {
console.log(`package contents: ${result.status}`);
console.log(`package entries: ${result.entryCount}`);
for (const check of result.checks) {
console.log(`- ${check.status.toUpperCase()} ${check.id}: ${check.message}`);
if (check.status === "fail" && check.actual !== check.expected) {
console.log(` expected: ${check.expected}`);
console.log(` actual: ${check.actual}`);
}
}
}

66
scripts/release-notes.mjs Normal file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env node
import { readFileSync } from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const options = parseArgs(process.argv.slice(2));
const changelogText = readFileSync(options.changelogPath, "utf8");
process.stdout.write(extractReleaseNotes({ changelogText, version: options.version }));
}
export function extractReleaseNotes({ changelogText, version }) {
const lines = changelogText.split(/\r?\n/);
const start = findReleaseStart(lines, version);
if (start === -1) {
throw new Error(`CHANGELOG.md is missing a ${version} release section`);
}
const end = lines.findIndex((line, index) => index > start && line.startsWith("## "));
const body = lines.slice(start + 1, end === -1 ? lines.length : end).join("\n").trim();
const title = version === "Unreleased" ? "plugin-inspector unreleased" : `plugin-inspector v${version}`;
return `## ${title}\n\n${body}\n`;
}
function findReleaseStart(lines, version) {
if (version === "Unreleased") {
return lines.findIndex((line) => line.trim() === "## Unreleased");
}
return lines.findIndex((line) => line.startsWith(`## ${version} - `));
}
function parseArgs(argv) {
const options = {
changelogPath: path.resolve("CHANGELOG.md"),
version: undefined,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--changelog") {
options.changelogPath = path.resolve(argv[index + 1]);
index += 1;
continue;
}
if (arg === "--unreleased") {
options.version = "Unreleased";
continue;
}
if (arg === "--version") {
options.version = argv[index + 1];
index += 1;
continue;
}
if (!options.version) {
options.version = arg;
continue;
}
throw new Error(`unknown argument: ${arg}`);
}
if (!options.version) {
throw new Error("usage: release-notes.mjs <version>|--unreleased [--changelog CHANGELOG.md]");
}
return options;
}

203
scripts/release-plan.mjs Normal file
View File

@ -0,0 +1,203 @@
#!/usr/bin/env node
import { execFileSync } from "node:child_process";
import { readFileSync } from "node:fs";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { extractReleaseNotes } from "./release-notes.mjs";
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
const options = parseArgs(process.argv.slice(2));
const root = path.resolve(options.root);
const packageJson = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
const changelogText = readFileSync(path.join(root, "CHANGELOG.md"), "utf8");
const plan = buildReleasePlan({
packageName: packageJson.name,
packageVersion: packageJson.version,
changelogText,
releaseRef: options.releaseRef ?? gitHead(root),
releaseDate: options.date,
nextVersion: options.version,
});
if (options.json) {
console.log(JSON.stringify(plan, null, 2));
} else {
printReleasePlan(plan);
}
if (plan.status === "fail") {
process.exitCode = 1;
}
}
export function buildReleasePlan({
packageName = "@openclaw/plugin-inspector",
packageVersion,
changelogText,
releaseRef,
releaseDate = today(),
nextVersion,
}) {
const version = nextVersion ?? bumpPatch(packageVersion);
const checks = [
{
id: "version-advance",
status: compareVersions(version, packageVersion) > 0 ? "pass" : "fail",
message: `${version} is newer than ${packageVersion}`,
expected: `>${packageVersion}`,
actual: version,
},
{
id: "changelog-unreleased",
status: hasUnreleasedNotes(changelogText) ? "pass" : "fail",
message: "CHANGELOG.md has non-empty Unreleased notes",
},
];
return {
status: checks.some((check) => check.status === "fail") ? "fail" : "pass",
packageName,
currentVersion: packageVersion,
nextVersion: version,
releaseDate,
releaseRef,
tagName: `v${version}`,
changelogHeading: `## ${version} - ${releaseDate}`,
crabpotSourceRef: releaseRef,
crabpotPackagePin: `${packageName}@${version}`,
releaseNotes: safeReleaseNotes(changelogText),
checks,
steps: [
`update package.json version to ${version}`,
`move CHANGELOG.md Unreleased notes to "${`## ${version} - ${releaseDate}`}"`,
`set crabpot pluginInspectorRef to ${releaseRef}`,
"run npm run release:readiness",
`create and push annotated tag ${`v${version}`}`,
`after npm publish, set crabpot pluginInspectorPackage to ${packageName}@${version}`,
"run npm run release:crabpot -- --published",
],
};
}
function hasUnreleasedNotes(changelogText) {
try {
return extractReleaseNotes({ changelogText, version: "Unreleased" })
.split(/\r?\n/)
.some((line) => line.trim().startsWith("- "));
} catch {
return false;
}
}
function safeReleaseNotes(changelogText) {
try {
return extractReleaseNotes({ changelogText, version: "Unreleased" });
} catch {
return "";
}
}
function bumpPatch(version) {
const match = version?.match(/^(\d+)\.(\d+)\.(\d+)$/);
if (!match) {
throw new Error(`cannot infer next patch version from ${version}`);
}
return `${match[1]}.${match[2]}.${Number(match[3]) + 1}`;
}
function compareVersions(left, right) {
const leftParts = parseVersion(left);
const rightParts = parseVersion(right);
for (let index = 0; index < 3; index += 1) {
if (leftParts[index] !== rightParts[index]) {
return leftParts[index] - rightParts[index];
}
}
return 0;
}
function parseVersion(version) {
const match = version?.match(/^(\d+)\.(\d+)\.(\d+)$/);
if (!match) {
throw new Error(`invalid semver version: ${version}`);
}
return match.slice(1).map(Number);
}
function today() {
return new Date().toISOString().slice(0, 10);
}
function gitHead(root) {
return execFileSync("git", ["rev-parse", "HEAD"], {
cwd: root,
encoding: "utf8",
stdio: ["ignore", "pipe", "ignore"],
}).trim();
}
function parseArgs(argv) {
const options = {
root: ".",
date: today(),
json: false,
releaseRef: undefined,
version: undefined,
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--root") {
options.root = argv[index + 1];
index += 1;
continue;
}
if (arg === "--date") {
options.date = argv[index + 1];
index += 1;
continue;
}
if (arg === "--json") {
options.json = true;
continue;
}
if (arg === "--ref") {
options.releaseRef = argv[index + 1];
index += 1;
continue;
}
if (arg === "--version") {
options.version = argv[index + 1];
index += 1;
continue;
}
if (!options.version) {
options.version = arg;
continue;
}
throw new Error(`unknown argument: ${arg}`);
}
return options;
}
function printReleasePlan(plan) {
console.log(`release plan: ${plan.status}`);
console.log(`package: ${plan.packageName}`);
console.log(`current version: ${plan.currentVersion}`);
console.log(`next version: ${plan.nextVersion}`);
console.log(`release date: ${plan.releaseDate}`);
console.log(`release ref: ${plan.releaseRef}`);
console.log(`tag: ${plan.tagName}`);
for (const check of plan.checks) {
console.log(`- ${check.status.toUpperCase()} ${check.id}: ${check.message}`);
if (check.status === "fail" && check.actual !== check.expected) {
console.log(` expected: ${check.expected}`);
console.log(` actual: ${check.actual}`);
}
}
console.log("steps:");
for (const step of plan.steps) {
console.log(`- ${step}`);
}
}

View File

@ -33,8 +33,17 @@ export {
renderCiSummaryMarkdown,
writeCiSummary,
} from "./ci-summary.js";
export {
buildSarifReport,
defaultJunitPath,
defaultSarifPath,
renderJunitXml,
reportFindings,
writeCiOutputArtifacts,
} from "./ci-outputs.js";
export {
buildContractProbes,
compatRecordForIssueCode,
contractProbeRules,
probePriority,
} from "./contract-probes.js";
@ -86,6 +95,7 @@ export {
classifyTargetOpenClawCoverage,
readPackageSummaries,
readPluginManifests,
readSecurityManifests,
summarizePackage,
} from "./fixture-summary.js";
export {
@ -122,13 +132,16 @@ export {
loadPluginRootConfig,
normalizeInspectorConfig,
normalizePluginRootConfig,
packageJsonConfigKeys,
packageId,
validateInspectorConfig,
} from "./config.js";
export {
buildPluginInspectorConfig,
defaultInitPackageScripts,
defaultInitConfigPath,
defaultInitWorkflowPath,
detectPackageManager,
renderGithubActionsWorkflow,
writePluginInspectorInit,
} from "./init.js";
@ -159,6 +172,7 @@ export {
classifyCompatRecordCoverage,
renderMarkdownReport,
renderTextSummary,
sanitizeReportArtifact,
writeCompatibilityReport,
writeReport,
} from "./report.js";
@ -170,6 +184,10 @@ export {
validateRuntimeProfile,
writeRuntimeProfile,
} from "./runtime-profile.js";
export {
applyRuntimeExecutionCoverage,
buildRuntimeExecutionCoverage,
} from "./runtime-reconciliation.js";
export {
buildRuntimeCaptureReport,
renderRuntimeCaptureMarkdown,
@ -187,6 +205,7 @@ export {
validateSyntheticProbePlan,
writeSyntheticProbePlan,
} from "./synthetic-probes.js";
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
export {
buildWorkspacePlan,
defaultWorkspacePlanOptions,
@ -194,3 +213,10 @@ export {
validateWorkspacePlan,
writeWorkspacePlan,
} from "./workspace-plan.js";
export {
inspectCompatibilityFixtureSetConfig,
renderFixtureSetIssuesReport,
renderFixtureSetMarkdownReport,
runFixtureSetReport,
writeFixtureSetReports,
} from "./api.js";

View File

@ -1,11 +1,31 @@
import path from "node:path";
import { createCaptureApi } from "./capture-api.js";
import { loadInspectorConfig, loadPluginRootConfig } from "./config.js";
import { renderCompatibilityIssuesReport, renderCompatibilityMarkdownReport } from "./compatibility-report.js";
import { writePluginInspectorInit } from "./init.js";
import { captureEntrypoint } from "./inspector.js";
import { renderTextSummary, writeCompatibilityReport } from "./report.js";
import { writeCiOutputArtifacts } from "./ci-outputs.js";
import { buildRuntimeCaptureReport, writeRuntimeCaptureReport } from "./runtime-capture-report.js";
import { inspectCompatibilityFixtureSet, inspectFixtureSet } from "./inspector.js";
import {
buildColdImportReadiness,
renderColdImportReadinessMarkdown,
validateColdImportReadiness,
writeColdImportReadiness,
} from "./cold-import-readiness.js";
import {
buildWorkspacePlan,
renderWorkspacePlanMarkdown,
validateWorkspacePlan,
writeWorkspacePlan,
} from "./workspace-plan.js";
import {
buildPlatformProbes,
renderPlatformProbesMarkdown,
validatePlatformProbes,
writePlatformProbes,
} from "./platform-probes.js";
export async function loadPluginConfig(options = {}) {
if (options.config) {
@ -25,6 +45,7 @@ export async function inspectPluginRoot(options = {}) {
return inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
});
}
@ -34,6 +55,16 @@ export async function inspectFixtureSetConfig(options = {}) {
return inspectFixtureSet(config, { generatedAt: options.generatedAt });
}
export async function inspectCompatibilityFixtureSetConfig(options = {}) {
const config = await loadFixtureSetConfig(options);
return inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
});
}
export async function writePluginReports(report, options = {}) {
return writeCompatibilityReport(report, {
basename: options.basename,
@ -44,6 +75,132 @@ export async function writePluginReports(report, options = {}) {
});
}
export async function writeFixtureSetReports(report, options = {}) {
return writeCompatibilityReport(report, {
basename: options.basename,
check: options.check,
cwd: options.cwd,
formatEvidence: options.formatEvidence,
issuesBasename: options.issuesBasename,
issuesPath: options.issuesPath,
issuesTitle: options.issuesTitle,
jsonPath: options.jsonPath,
markdownPath: options.markdownPath,
markdownTitle: options.markdownTitle,
outDir: options.outDir,
severityLabels: options.severityLabels,
title: options.title,
});
}
export function renderFixtureSetMarkdownReport(report, options = {}) {
return renderCompatibilityMarkdownReport(report, options);
}
export function renderFixtureSetIssuesReport(report, options = {}) {
return renderCompatibilityIssuesReport(report, options);
}
export async function runFixtureSetReport(options = {}) {
const report = await inspectCompatibilityFixtureSetConfig(options);
const paths = options.write === false ? null : await writeFixtureSetReports(report, options);
return { report, paths };
}
export async function buildFixtureSetColdImportReadiness(options = {}) {
const config = options.report ? null : await loadFixtureSetConfig(options);
const report =
options.report ??
(await inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
}));
return buildColdImportReadiness({
...options,
report,
rootDir: options.rootDir ?? config?.rootDir ?? options.cwd,
});
}
export function renderFixtureSetColdImportReadinessMarkdown(readiness, options = {}) {
return renderColdImportReadinessMarkdown(readiness, options);
}
export async function writeFixtureSetColdImportReadiness(readiness, options = {}) {
return writeColdImportReadiness(readiness, options);
}
export async function runFixtureSetColdImportReadiness(options = {}) {
const readiness = await buildFixtureSetColdImportReadiness(options);
const paths = options.write === false ? null : await writeFixtureSetColdImportReadiness(readiness, options);
return { readiness, paths };
}
export async function buildFixtureSetWorkspacePlan(options = {}) {
const config = options.report ? null : await loadFixtureSetConfig(options);
const report =
options.report ??
(await inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
}));
const rootDir = options.rootDir ?? config?.rootDir ?? options.cwd;
const readiness = options.readiness ?? buildColdImportReadiness({ ...options, report, rootDir });
return buildWorkspacePlan({
...options,
report,
readiness,
rootDir,
});
}
export function renderFixtureSetWorkspacePlanMarkdown(plan, options = {}) {
return renderWorkspacePlanMarkdown(plan, options);
}
export function validateFixtureSetWorkspacePlan(plan, options = {}) {
return validateWorkspacePlan(plan, options);
}
export async function writeFixtureSetWorkspacePlan(plan, options = {}) {
return writeWorkspacePlan(plan, options);
}
export async function runFixtureSetWorkspacePlan(options = {}) {
const plan = await buildFixtureSetWorkspacePlan(options);
const paths = options.write === false ? null : await writeFixtureSetWorkspacePlan(plan, options);
return { plan, paths };
}
export async function buildFixtureSetPlatformProbes(options = {}) {
const plan = options.plan ?? (await buildFixtureSetWorkspacePlan(options));
return buildPlatformProbes({ ...options, plan });
}
export function renderFixtureSetPlatformProbesMarkdown(report, options = {}) {
return renderPlatformProbesMarkdown(report, options);
}
export function validateFixtureSetPlatformProbes(report, options = {}) {
return validatePlatformProbes(report, options);
}
export async function writeFixtureSetPlatformProbes(report, options = {}) {
return writePlatformProbes(report, options);
}
export async function runFixtureSetPlatformProbes(options = {}) {
const report = await buildFixtureSetPlatformProbes(options);
const paths = options.write === false ? null : await writeFixtureSetPlatformProbes(report, options);
return { report, paths };
}
export async function runPluginCheck(options = {}) {
const outDir = options.outDir ?? "reports";
const config = await loadPluginConfig(options);
@ -54,8 +211,8 @@ export async function runPluginCheck(options = {}) {
const mockSdk = options.mockSdk ?? config.capture?.mockSdk ?? true;
if (capture === true) {
if (process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
throw new Error("runtime capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 in an isolated workspace");
if (!executionAllowed(options)) {
throw new Error("runtime capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 or --allow-execute in an isolated workspace");
}
const runtimeCapture = await buildRuntimeCaptureReport({
mockSdk,
@ -88,4 +245,18 @@ export async function setupPluginInspector(options = {}) {
return writePluginInspectorInit(options);
}
export { createCaptureApi, renderTextSummary };
export { createCaptureApi, renderTextSummary, validateColdImportReadiness, writeCiOutputArtifacts };
function executionAllowed(options) {
return options.allowExecution === true || process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED === "1";
}
async function loadFixtureSetConfig(options) {
if (options.config) {
return {
...options.config,
rootDir: options.config.rootDir ?? options.rootDir ?? options.cwd ?? process.cwd(),
};
}
return loadInspectorConfig(options.configPath, { cwd: options.cwd ?? options.rootDir });
}

View File

@ -95,7 +95,7 @@ export function renderPaddedMarkdownTable(rows, headers, options = {}) {
}
export function escapeMarkdownTableCell(value) {
return value.replace(/\|/g, "\\|").replace(/\n/g, "<br>");
return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, "<br>");
}
async function assertFileMatches(filePath, expected) {

View File

@ -1,6 +1,6 @@
export const defaultCaptureApiRegistrarProfiles = {
registerChannel: {
returnValue: ({ args }) => registrationObject(args, { id: "channel" }),
returnValue: ({ args }) => channelRegistrationObject(args),
},
registerCli: {
returnValue: ({ args }) => registrationObject(args, { name: "cli" }),
@ -12,7 +12,7 @@ export const defaultCaptureApiRegistrarProfiles = {
returnValue: ({ args }) => registrationObject(args, { id: "context-engine" }),
},
registerGatewayMethod: {
returnValue: ({ args }) => registrationObject(args, { name: "gateway.method" }),
returnValue: ({ args }) => gatewayMethodRegistrationObject(args),
},
registerHook: {
returnValue: ({ api }) => api,
@ -37,9 +37,10 @@ export const defaultCaptureApiRegistrarProfiles = {
},
registerService: {
returnValue: ({ args }) => ({
...registrationObject(args, { name: "service" }),
...registrationObject(args, { id: "service", name: "service" }),
start: async () => undefined,
stop: async () => undefined,
dispose: async () => undefined,
}),
},
registerSpeechProvider: {
@ -81,6 +82,24 @@ export function createCaptureApi(options = {}) {
}
return api;
},
onConversationBindingResolved(handler) {
const captureIndex =
captured.push({
kind: "hook",
name: "onConversationBindingResolved",
handlerType: typeof handler,
arguments: summarizeArguments([handler]),
}) - 1;
if (retainHandlers) {
retained.push({
kind: "hook",
name: "onConversationBindingResolved",
handler,
captureIndex,
});
}
return api;
},
},
{
get(target, property) {
@ -125,9 +144,11 @@ export function createCaptureApi(options = {}) {
export function createCaptureContext(options = {}) {
return {
registrationMode: options.registrationMode ?? "full",
config: options.config ?? {},
logger: options.logger ?? console,
pluginConfig: options.pluginConfig ?? {},
resolvePath: options.resolvePath ?? ((value) => value),
runtime: options.runtime ?? createRuntimeContext(options),
secrets: options.secrets ?? createSecretContext(options),
store: options.store ?? createStoreContext(options),
@ -142,6 +163,12 @@ export function createCaptureContext(options = {}) {
},
gateway: options.gateway ?? {
baseUrl: "http://127.0.0.1:0",
async call(method, params) {
return { ok: true, method, params };
},
respond(ok, result, error) {
return { ok, result, ...(error ? { error } : {}) };
},
registerRoute(route) {
return {
...route,
@ -223,6 +250,7 @@ function createStoreContext() {
function registrationObject(args, defaults) {
const first = args[0];
const callable = firstCallable(args);
if (first && typeof first === "object") {
return {
...defaults,
@ -232,13 +260,59 @@ function registrationObject(args, defaults) {
};
}
if (typeof first === "string") {
return {
return withCallableDefaults({
...defaults,
name: first,
id: defaults.id,
}, callable);
}
return withCallableDefaults({ ...defaults }, callable);
}
function channelRegistrationObject(args) {
const first = args[0];
const registration = registrationObject(args, { id: "channel" });
if (first?.plugin && typeof first.plugin === "object") {
return {
...registration,
id: objectId(first.plugin) ?? registration.id,
plugin: first.plugin,
};
}
return { ...defaults };
return registration;
}
function gatewayMethodRegistrationObject(args) {
const [method, handler, options] = args;
const registration = registrationObject(args, { name: "gateway.method" });
if (typeof method !== "string") {
return registration;
}
return {
...registration,
name: method,
method,
handler: typeof handler === "function" ? handler : registration.handler,
run: typeof handler === "function" ? handler : registration.run,
execute: typeof handler === "function" ? handler : registration.execute,
scope: options?.scope ?? registration.scope,
};
}
function firstCallable(args) {
return args.find((arg) => typeof arg === "function");
}
function withCallableDefaults(value, callable) {
if (!callable) {
return value;
}
return {
...value,
handler: typeof value.handler === "function" ? value.handler : callable,
run: typeof value.run === "function" ? value.run : callable,
execute: typeof value.execute === "function" ? value.execute : callable,
};
}
function summarizeArguments(args) {

126
src/capture-config.js Normal file
View File

@ -0,0 +1,126 @@
import { readFile } from "node:fs/promises";
import path from "node:path";
export async function captureApiOptionsForPlugin(apiOptions = {}, options = {}) {
if (apiOptions.pluginConfig !== undefined || !options.pluginRoot) {
return apiOptions;
}
const pluginConfig = await readSamplePluginConfig(options.pluginRoot);
if (pluginConfig === undefined) {
return apiOptions;
}
return {
...apiOptions,
pluginConfig,
};
}
async function readSamplePluginConfig(pluginRoot) {
const manifestPath = path.join(pluginRoot, "openclaw.plugin.json");
let manifest;
try {
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
} catch {
return undefined;
}
const sample = sampleJsonSchema(manifest.configSchema, { key: "config" });
return isPlainObject(sample) && Object.keys(sample).length > 0 ? sample : undefined;
}
function sampleJsonSchema(schema, context = {}) {
if (!isPlainObject(schema)) {
return undefined;
}
if (Array.isArray(schema.enum) && schema.enum.length > 0) {
return schema.enum[0];
}
if (Object.prototype.hasOwnProperty.call(schema, "const")) {
return schema.const;
}
if (Object.prototype.hasOwnProperty.call(schema, "default")) {
return schema.default;
}
const type = Array.isArray(schema.type) ? schema.type.find((item) => item !== "null") : schema.type;
if (type === "object" || schema.properties) {
return sampleObjectSchema(schema);
}
if (type === "array") {
return [];
}
if (type === "boolean") {
return false;
}
if (type === "number" || type === "integer") {
return typeof schema.minimum === "number" ? schema.minimum : 1;
}
if (type === "string" || !type) {
return sampleString(context.key);
}
return undefined;
}
function sampleObjectSchema(schema) {
const properties = isPlainObject(schema.properties) ? schema.properties : {};
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
const output = {};
for (const key of Object.keys(properties)) {
if (required.has(key)) {
const value = sampleJsonSchema(properties[key], { key });
if (value !== undefined) {
output[key] = value;
}
}
}
for (const key of Object.keys(properties)) {
if (properties[key]?.type === "boolean") {
output[key] = false;
}
}
if (Object.keys(output).length === 0 && Number(schema.minProperties ?? 0) > 0) {
const key = preferredSamplePropertyKey(properties);
if (key) {
const value = sampleJsonSchema(properties[key], { key });
if (value !== undefined) {
output[key] = value;
}
}
}
return output;
}
function preferredSamplePropertyKey(properties) {
for (const key of ["provider", "model", "apiKey", "id", "name", ...Object.keys(properties)]) {
if (Object.prototype.hasOwnProperty.call(properties, key)) {
return key;
}
}
return null;
}
function sampleString(key = "") {
if (key === "provider") {
return "openai";
}
if (key === "model") {
return "text-embedding-3-small";
}
if (key === "apiKey") {
return "fixture-api-key";
}
if (key === "dbPath") {
return ".plugin-inspector/state/lancedb";
}
return "fixture";
}
function isPlainObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}

235
src/ci-outputs.js Normal file
View File

@ -0,0 +1,235 @@
import path from "node:path";
import { writeArtifacts } from "./artifacts.js";
export const defaultSarifPath = "plugin-inspector.sarif";
export const defaultJunitPath = "plugin-inspector.junit.xml";
export async function writeCiOutputArtifacts(report, options = {}) {
const outDir = path.resolve(options.cwd ?? process.cwd(), options.outDir ?? "reports");
const artifacts = [];
if (options.sarifPath) {
artifacts.push({
name: "sarifPath",
path: path.resolve(outDir, options.sarifPath),
json: buildSarifReport(report),
});
}
if (options.junitPath) {
artifacts.push({
name: "junitPath",
path: path.resolve(outDir, options.junitPath),
content: renderJunitXml(report),
});
}
if (artifacts.length === 0) {
return {};
}
return writeArtifacts(artifacts, { check: options.check });
}
export function buildSarifReport(report) {
const findings = reportFindings(report);
const rules = [...new Map(findings.map((finding) => [finding.code, sarifRule(finding)])).values()];
const fixtureById = new Map((report.fixtures ?? []).map((fixture) => [fixture.id, fixture]));
return {
$schema: "https://json.schemastore.org/sarif-2.1.0.json",
version: "2.1.0",
runs: [
{
tool: {
driver: {
name: "plugin-inspector",
informationUri: "https://github.com/openclaw/plugin-inspector",
rules,
},
},
results: findings.map((finding) => sarifResult(finding, fixtureById)),
},
],
};
}
export function renderJunitXml(report) {
const findings = reportFindings(report);
const testcases = findings.length > 0 ? findings.map(junitFindingTestcase) : [junitPassingTestcase(report)];
const failures = findings.filter(isBlockingFinding).length;
const tests = testcases.length;
return [
'<?xml version="1.0" encoding="UTF-8"?>',
`<testsuite name="plugin-inspector" tests="${tests}" failures="${failures}" errors="0" skipped="0">`,
...testcases,
"</testsuite>",
"",
].join("\n");
}
export function reportFindings(report) {
const findings = new Map();
for (const finding of [...(report.breakages ?? []), ...(report.warnings ?? []), ...(report.suggestions ?? [])]) {
findings.set(findingKey(finding), finding);
}
for (const issue of report.issues ?? []) {
const finding = issueToFinding(issue);
findings.set(findingKey(finding), {
...findings.get(findingKey(finding)),
...finding,
});
}
return [...findings.values()];
}
function issueToFinding(issue) {
return {
fixture: issue.fixture,
code: issue.code,
level: issue.status === "blocking" ? "breakage" : "warning",
message: issue.title,
evidence: issue.evidence ?? [],
severity: issue.severity,
issueClass: issue.issueClass,
};
}
function findingKey(finding) {
return [
finding.fixture ?? "",
finding.code ?? "",
...normalizeEvidence(finding.evidence),
].join("\n");
}
function sarifRule(finding) {
return {
id: finding.code,
shortDescription: {
text: finding.code,
},
fullDescription: {
text: finding.message ?? finding.code,
},
defaultConfiguration: {
level: sarifLevel(finding),
},
};
}
function sarifResult(finding, fixtureById) {
return {
ruleId: finding.code,
level: sarifLevel(finding),
message: {
text: finding.message ?? finding.code,
},
locations: [sarifLocation(finding, fixtureById)],
properties: {
fixture: finding.fixture,
severity: finding.severity ?? finding.level,
issueClass: finding.issueClass,
evidence: normalizeEvidence(finding.evidence),
},
};
}
function sarifLocation(finding, fixtureById) {
const parsed = parseEvidenceLocation(normalizeEvidence(finding.evidence)[0]);
const fixture = fixtureById.get(finding.fixture);
const uri = parsed?.uri ?? fixture?.path ?? ".";
return {
physicalLocation: {
artifactLocation: {
uri: normalizeUri(uri),
},
region: {
startLine: parsed?.line ?? 1,
},
},
};
}
function parseEvidenceLocation(evidence) {
if (!evidence) {
return null;
}
const ref = evidence.includes(" @ ") ? evidence.split(" @ ").pop() : evidence;
const match = /^(?<uri>.+?):(?<line>\d+)(?::\d+)?$/.exec(ref);
if (!match?.groups?.uri) {
return null;
}
return {
uri: match.groups.uri,
line: Number(match.groups.line),
};
}
function junitFindingTestcase(finding) {
const classname = `plugin-inspector.${xmlName(finding.fixture ?? "unknown")}`;
const name = `${finding.level ?? "finding"}:${finding.code}`;
const output = normalizeEvidence(finding.evidence).join("\n");
if (!isBlockingFinding(finding)) {
return [
` <testcase classname="${escapeXml(classname)}" name="${escapeXml(name)}">`,
output ? ` <system-out>${escapeXml(output)}</system-out>` : "",
" </testcase>",
]
.filter(Boolean)
.join("\n");
}
return [
` <testcase classname="${escapeXml(classname)}" name="${escapeXml(name)}">`,
` <failure message="${escapeXml(finding.message ?? finding.code)}">${escapeXml(output || finding.message || finding.code)}</failure>`,
" </testcase>",
].join("\n");
}
function junitPassingTestcase(report) {
return ` <testcase classname="plugin-inspector" name="status:${escapeXml(report.status ?? "pass")}"/>`;
}
function isBlockingFinding(finding) {
return finding.level === "breakage" || finding.status === "blocking" || finding.severity === "P0";
}
function sarifLevel(finding) {
if (isBlockingFinding(finding) || finding.severity === "P1") {
return "error";
}
if (finding.level === "warning" || finding.severity === "P2") {
return "warning";
}
return "note";
}
function normalizeEvidence(evidence) {
if (Array.isArray(evidence)) {
return evidence.map(String);
}
if (evidence == null) {
return [];
}
return [String(evidence)];
}
function normalizeUri(uri) {
return String(uri).replaceAll(path.sep, "/");
}
function xmlName(value) {
return String(value).replace(/[^a-zA-Z0-9_.-]/g, "_");
}
function escapeXml(value) {
return String(value)
.replace(/&/g, "&amp;")
.replace(/</g, "&lt;")
.replace(/>/g, "&gt;")
.replace(/"/g, "&quot;")
.replace(/'/g, "&apos;");
}

View File

@ -241,7 +241,9 @@ function executionChecks(executionResults, policy, options) {
}
function findPolicyMatch(rules, item) {
return rules.find((rule) => item.seam === rule.seam && item.reason?.includes(rule.reasonIncludes));
return rules.find(
(rule) => (rule.seam === "*" || item.seam === rule.seam) && item.reason?.includes(rule.reasonIncludes),
);
}
function failedExecutionEvidence(executionResults) {

View File

@ -56,8 +56,14 @@ export async function buildCiSummary(options = {}) {
loaderJitiCandidates: reports.platform?.summary?.jitiAlternativeCount ?? 0,
importLoopP50Ms: reports.importLoop?.summary?.p50WallMs ?? 0,
importLoopP95Ms: reports.importLoop?.summary?.p95WallMs ?? 0,
importLoopMaxRssMb: reports.importLoop?.summary?.maxPeakRssMb ?? 0,
importLoopMaxCpuMs: reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
importLoopOpenClawLifecycleCount: reports.importLoop?.summary?.openClawLifecycleCount ?? 0,
importLoopOpenClawImportP50Ms: reports.importLoop?.summary?.p50OpenClawImportMs ?? 0,
importLoopOpenClawActivationP50Ms: reports.importLoop?.summary?.p50OpenClawActivationMs ?? 0,
importLoopMetricBasis: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb === undefined ? "raw" : "baseline-adjusted",
importLoopMaxRssMb: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb ?? reports.importLoop?.summary?.maxPeakRssMb ?? 0,
importLoopMaxCpuMs: reports.importLoop?.summary?.maxPluginCpuDeltaMsEstimate ?? reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
importLoopRssSampleCount: metricSampleCount(reports.importLoop, "rss", "maxPeakRssMb"),
importLoopCpuSampleCount: metricSampleCount(reports.importLoop, "cpu", "maxCpuMsEstimate"),
},
topIssues: topIssues(reports.compatibility),
refRegressions: (reports.refDiff?.regressions ?? []).slice(0, 20),
@ -148,7 +154,7 @@ export function renderCiSummaryMarkdown(summary) {
["Jiti loader candidates", summary.summary.loaderJitiCandidates],
[
"Import loop",
`p50 ${summary.summary.importLoopP50Ms} ms / p95 ${summary.summary.importLoopP95Ms} ms / max RSS ${summary.summary.importLoopMaxRssMb} MB / CPU ${summary.summary.importLoopMaxCpuMs} ms`,
importLoopSummaryLabel(summary.summary),
],
],
["Metric", "Value"],
@ -221,3 +227,44 @@ function topIssues(report) {
function markdownTable(rows, headers) {
return renderPaddedMarkdownTable(rows, headers, { nullValue: "-" });
}
function metricSampleCount(report, kind, maxMetric) {
const summaryKey = kind === "rss" ? "rssSampleCount" : "cpuSampleCount";
const summaryCount = report?.summary?.[summaryKey];
if (Number.isFinite(summaryCount)) {
return summaryCount;
}
const sampleCount = inferSampleCount(report?.samples, kind);
if (sampleCount > 0) {
return sampleCount;
}
return (report?.summary?.[maxMetric] ?? 0) > 0 ? 1 : 0;
}
function inferSampleCount(samples = [], kind) {
if (!Array.isArray(samples)) {
return 0;
}
return samples.reduce((sum, sample) => {
if (kind === "rss") {
return sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0));
}
return sum + (sample.cpuSampleCount ?? 0);
}, 0);
}
function importLoopSummaryLabel(summary) {
const metricLabel = summary.importLoopMetricBasis === "baseline-adjusted" ? "plugin delta" : "raw";
const lifecycle =
summary.importLoopOpenClawLifecycleCount > 0
? ` / OpenClaw import ${summary.importLoopOpenClawImportP50Ms} ms / activate ${summary.importLoopOpenClawActivationP50Ms} ms`
: "";
return `p50 ${summary.importLoopP50Ms} ms / p95 ${summary.importLoopP95Ms} ms / ${metricLabel} RSS ${formatSampledMetric(summary.importLoopMaxRssMb, summary.importLoopRssSampleCount)} / ${metricLabel} CPU ${formatSampledMetric(summary.importLoopMaxCpuMs, summary.importLoopCpuSampleCount, "ms")}${lifecycle}`;
}
function formatSampledMetric(value, count, unit = "MB") {
if ((count ?? 0) <= 0) {
return "n/a";
}
return `${value} ${unit}`;
}

View File

@ -1,15 +1,20 @@
#!/usr/bin/env node
import path from "node:path";
import {
loadPluginConfig,
renderTextSummary,
sanitizeReportArtifact,
runPluginCheck,
} from "./index.js";
import {
buildCiSummary,
captureEntrypoint,
defaultJunitPath,
defaultSarifPath,
inspectCompatibilityFixtureSet,
inspectFixtureSet,
loadInspectorConfig,
writeCiOutputArtifacts,
writeCiSummary,
writeCompatibilityReport,
writePluginInspectorInit,
@ -28,8 +33,14 @@ try {
await runCheck(commandArgs);
} else if (command === "init") {
await runInit(commandArgs);
} else if (command === "config") {
await runConfig(commandArgs);
} else if (command === "inspect" || command === "report") {
await runReport(command, commandArgs);
if (command === "inspect" && !commandArgs.includes("--config")) {
await runCheck(commandArgs);
} else {
await runReport(command, commandArgs);
}
} else if (command === "ci") {
await runCi(commandArgs);
} else if (command === "capture") {
@ -42,6 +53,18 @@ try {
process.exitCode = 1;
}
async function runConfig(commandArgs) {
const configPath = readFlag(commandArgs, "--config");
const pluginRoot = readFlag(commandArgs, "--plugin-root") ?? readFlag(commandArgs, "--root");
const config = await loadPluginConfig({ configPath, pluginRoot });
if (commandArgs.includes("--json")) {
console.log(JSON.stringify(config, null, 2));
} else {
console.log(renderConfigTextSummary(config));
}
}
async function runCheck(commandArgs) {
const configPath = readFlag(commandArgs, "--config");
const pluginRoot = readFlag(commandArgs, "--plugin-root") ?? readFlag(commandArgs, "--root");
@ -50,12 +73,27 @@ async function runCheck(commandArgs) {
const json = commandArgs.includes("--json");
const capture = readRuntimeFlag(commandArgs);
const mockSdk = readMockSdkFlag(commandArgs);
const { report } = await runPluginCheck({ configPath, pluginRoot, outDir, openclawPath, capture, mockSdk });
const allowExecution = readAllowExecutionFlag(commandArgs);
const ciOutputs = readCiOutputFlags(commandArgs);
const { report, paths } = await runPluginCheck({
allowExecution,
capture,
configPath,
mockSdk,
openclawPath,
outDir,
pluginRoot,
});
await writeCiOutputArtifacts(report, {
...ciOutputs,
cwd: path.dirname(paths.jsonPath),
outDir: ".",
});
if (json) {
console.log(JSON.stringify(report, null, 2));
console.log(JSON.stringify(sanitizeReportArtifact(report), null, 2));
} else {
console.log(renderTextSummary(report));
console.log(renderTextSummary(report, { artifacts: paths }));
}
if (report.status !== "pass") {
@ -67,19 +105,27 @@ async function runInit(commandArgs) {
const pluginRoot = readFlag(commandArgs, "--plugin-root") ?? readFlag(commandArgs, "--root");
const configPath = readFlag(commandArgs, "--config") ?? undefined;
const workflowPath = readFlag(commandArgs, "--workflow") ?? undefined;
const packageManager = readFlag(commandArgs, "--package-manager") ?? "npm";
const packageManager = readFlag(commandArgs, "--package-manager") ?? undefined;
const result = await writePluginInspectorInit({
pluginRoot,
configPath,
workflowPath,
packageManager,
ci: commandArgs.includes("--ci"),
dryRun: commandArgs.includes("--dry-run"),
scripts: commandArgs.includes("--scripts"),
force: commandArgs.includes("--force"),
});
for (const filePath of result.written) {
console.log(`wrote ${filePath}`);
if (commandArgs.includes("--json")) {
console.log(JSON.stringify(initCommandSummary(result), null, 2));
return;
}
for (const filePath of result.written) {
console.log(`${result.dryRun ? "would write" : "wrote"} ${path.relative(result.pluginRoot, filePath)}`);
}
console.log(`package manager: ${result.packageManager}`);
}
async function runReport(command, commandArgs) {
@ -87,14 +133,20 @@ async function runReport(command, commandArgs) {
const outDir = readFlag(commandArgs, "--out") ?? "reports";
const check = commandArgs.includes("--check") || command === "ci";
const json = commandArgs.includes("--json");
const ciOutputs = readCiOutputFlags(commandArgs);
const config = await loadInspectorConfig(configPath);
const report = await inspectFixtureSet(config);
await writeReport(report, { outDir });
const paths = await writeReport(report, { outDir });
await writeCiOutputArtifacts(report, {
...ciOutputs,
cwd: path.dirname(paths.jsonPath),
outDir: ".",
});
if (json) {
console.log(JSON.stringify(report, null, 2));
} else {
console.log(renderTextSummary(report));
console.log(renderTextSummary(report, { artifacts: paths }));
}
if (check && report.status !== "pass") {
@ -108,8 +160,15 @@ async function runCi(commandArgs) {
const outDir = readFlag(commandArgs, "--out") ?? "reports";
const openclawPath = commandArgs.includes("--no-openclaw") ? false : readFlag(commandArgs, "--openclaw");
const json = commandArgs.includes("--json");
const capture = readRuntimeFlag(commandArgs);
const mockSdk = readMockSdkFlag(commandArgs);
const allowExecution = readAllowExecutionFlag(commandArgs);
const ciOutputs = readCiOutputFlags(commandArgs, { defaultEnabled: true });
const { report, reportDir } = await runCiCompatibilityReport({
allowExecution,
capture,
configPath,
mockSdk,
openclawPath,
outDir,
pluginRoot,
@ -128,6 +187,11 @@ async function runCi(commandArgs) {
jsonPath: path.join(reportDir, "plugin-inspector-ci-summary.json"),
markdownPath: path.join(reportDir, "plugin-inspector-ci-summary.md"),
});
await writeCiOutputArtifacts(report, {
...ciOutputs,
cwd: reportDir,
outDir: ".",
});
if (json) {
console.log(JSON.stringify(summary, null, 2));
@ -140,7 +204,7 @@ async function runCi(commandArgs) {
}
}
async function runCiCompatibilityReport({ configPath, openclawPath, outDir, pluginRoot }) {
async function runCiCompatibilityReport({ allowExecution, capture, configPath, mockSdk, openclawPath, outDir, pluginRoot }) {
if (configPath) {
const config = await loadInspectorConfig(configPath, { cwd: pluginRoot });
const report = await inspectCompatibilityFixtureSet(config, { openclawPath });
@ -151,7 +215,7 @@ async function runCiCompatibilityReport({ configPath, openclawPath, outDir, plug
};
}
const { report } = await runPluginCheck({ pluginRoot, outDir, openclawPath });
const { report } = await runPluginCheck({ allowExecution, capture, mockSdk, openclawPath, outDir, pluginRoot });
return {
report,
reportDir: path.resolve(pluginRoot ?? process.cwd(), outDir),
@ -163,11 +227,12 @@ async function runCapture(commandArgs) {
const outputPath = readFlag(commandArgs, "--output");
const pluginRoot = readFlag(commandArgs, "--plugin-root");
const mockSdk = readMockSdkFlag(commandArgs) ?? commandArgs.includes("--mock-sdk");
const allowExecution = readAllowExecutionFlag(commandArgs);
if (!entrypoint) {
throw new Error("capture requires an entrypoint path");
}
if (process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
throw new Error("capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 in an isolated workspace");
if (!allowExecution && process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
throw new Error("capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 or --allow-execute in an isolated workspace");
}
const result = await captureEntrypoint(entrypoint, { mockSdk, pluginRoot });
@ -187,6 +252,26 @@ function readFlag(commandArgs, name) {
return commandArgs[index + 1] ?? null;
}
function readOptionalPathFlag(commandArgs, name, defaultPath) {
const index = commandArgs.indexOf(name);
if (index === -1) {
return null;
}
const value = commandArgs[index + 1];
return value && !value.startsWith("-") ? value : defaultPath;
}
function readCiOutputFlags(commandArgs, options = {}) {
return {
sarifPath: commandArgs.includes("--no-sarif")
? null
: (readOptionalPathFlag(commandArgs, "--sarif", defaultSarifPath) ?? (options.defaultEnabled ? defaultSarifPath : null)),
junitPath: commandArgs.includes("--no-junit")
? null
: (readOptionalPathFlag(commandArgs, "--junit", defaultJunitPath) ?? (options.defaultEnabled ? defaultJunitPath : null)),
};
}
function readRuntimeFlag(commandArgs) {
if (commandArgs.includes("--runtime") || commandArgs.includes("--capture")) {
return true;
@ -217,6 +302,10 @@ function readMockSdkFlag(commandArgs) {
return undefined;
}
function readAllowExecutionFlag(commandArgs) {
return commandArgs.includes("--allow-execute");
}
function renderCiTextSummary(summary) {
return [
`Status: ${summary.status.toUpperCase()}`,
@ -226,19 +315,43 @@ function renderCiTextSummary(summary) {
].join("\n");
}
function initCommandSummary(result) {
return {
dryRun: result.dryRun,
packageManager: result.packageManager,
pluginRoot: result.pluginRoot,
files: result.written.map((filePath) => path.relative(result.pluginRoot, filePath)),
};
}
function renderConfigTextSummary(config) {
const fixture = config.fixtures[0];
return [
`Plugin: ${fixture.id}`,
`Root: ${config.rootDir}`,
`Config: ${config.configPath ?? "auto"}`,
`Priority: ${fixture.priority}`,
`Seams: ${fixture.seams.join(", ")}`,
`Runtime capture: ${config.capture?.runtime === true ? "on" : "off"}`,
`Mock SDK: ${config.capture?.mockSdk === false ? "off" : "on"}`,
].join("\n");
}
function printHelp() {
console.log(`plugin-inspector
Usage:
plugin-inspector
plugin-inspector check [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--runtime] [--mock-sdk|--real-sdk] [--json]
plugin-inspector init [--plugin-root <path>] [--config <path>] [--ci] [--package-manager npm|pnpm|yarn|bun] [--force]
plugin-inspector check [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--runtime] [--mock-sdk|--real-sdk] [--allow-execute] [--json]
plugin-inspector config [--plugin-root <path>] [--config <path>] [--json]
plugin-inspector init [--plugin-root <path>] [--config <path>] [--ci] [--scripts] [--package-manager npm|pnpm|yarn|bun] [--dry-run] [--json] [--force]
plugin-inspector report --config <path> [--out <dir>] [--check] [--json]
plugin-inspector inspect --config <path> [--out <dir>] [--check] [--json]
plugin-inspector ci [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--json]
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector capture <entrypoint> [--mock-sdk|--real-sdk] [--plugin-root <path>] [--output <path>]
plugin-inspector inspect [--plugin-root <path>] [--config <path>] [--out <dir>] [--check] [--json] [--sarif [path]] [--junit [path]] [--allow-execute]
plugin-inspector ci [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--runtime] [--mock-sdk|--real-sdk] [--allow-execute] [--json] [--no-sarif] [--no-junit]
plugin-inspector capture <entrypoint> [--mock-sdk|--real-sdk] [--allow-execute] [--plugin-root <path>] [--output <path>]
Default check runs from the current plugin root and writes reports/ unless --out is set.
Runtime capture is opt-in because it imports plugin code; use --runtime with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1.
CI writes SARIF and JUnit artifacts by default; check/inspect can write them with --sarif and --junit.
Runtime capture is opt-in because it imports plugin code; use --runtime with --allow-execute or PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1.
`);
}

View File

@ -3,6 +3,8 @@ import path from "node:path";
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
import { slugForArtifact } from "./path-utils.js";
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
export function buildColdImportReadiness(options = {}) {
const report = options.report;
if (!report) {
@ -182,7 +184,7 @@ function classifyEntrypointReadiness({ fixture, packageSummary, entrypoint, root
...(packageSummary.dependencies ?? []),
...(packageSummary.peerDependencies ?? []),
...(packageSummary.optionalDependencies ?? []),
]);
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
if (entrypoint.exists && runtimeDependencies.length > 0) {
blockers.push({
code: "dependency-install-required",

View File

@ -1,4 +1,5 @@
import { renderPaddedMarkdownTable } from "./artifacts.js";
import { sanitizeReportArtifact } from "./report-sanitizer.js";
const defaultSeverityLabels = {
P0: "P0",
@ -8,6 +9,7 @@ const defaultSeverityLabels = {
};
export function renderCompatibilityMarkdownReport(report, options = {}) {
report = sanitizeReportArtifact(report, options);
return [
`# ${options.title ?? "OpenClaw Plugin Compatibility Report"}`,
"",
@ -24,13 +26,20 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
["Warnings", report.summary.warningCount],
["Compatibility suggestions", report.summary.suggestionCount],
["Issue findings", report.summary.issueCount],
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
["P0 issues", report.summary.p0IssueCount],
["P1 issues", report.summary.p1IssueCount],
["Open P0 issues", report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
["Open P1 issues", report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
["Live issues", report.summary.liveIssueCount],
["Live P0 issues", report.summary.liveP0IssueCount],
["Compat gaps", report.summary.compatGapCount],
["Deprecation warnings", report.summary.deprecationWarningCount],
["Inspector gaps", report.summary.inspectorGapCount],
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
["Upstream metadata", report.summary.upstreamIssueCount],
["Contract probes", report.summary.contractProbeCount],
["Decision rows", report.summary.decisionCount],
@ -49,9 +58,9 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
options,
),
"",
"## Live Issues",
"## Other Live Issues",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
"",
"## Compat Gaps",
"",
@ -63,7 +72,11 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
"",
"## Inspector Proof Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
"",
"## Runtime-Covered Inspector Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
"",
"## Upstream Metadata Issues",
"",
@ -127,6 +140,7 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
}
export function renderCompatibilityIssuesReport(report, options = {}) {
report = sanitizeReportArtifact(report, options);
return [
`# ${options.title ?? "OpenClaw Plugin Issue Findings"}`,
"",
@ -138,13 +152,20 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
markdownTable(
[
["Issue findings", report.summary.issueCount],
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
[severityLabel("P0", options), report.summary.p0IssueCount],
[severityLabel("P1", options), report.summary.p1IssueCount],
[`Open ${severityLabel("P0", options)}`, report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
[`Open ${severityLabel("P1", options)}`, report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
["Live issues", report.summary.liveIssueCount],
["Live P0 issues", report.summary.liveP0IssueCount],
["Compat gaps", report.summary.compatGapCount],
["Deprecation warnings", report.summary.deprecationWarningCount],
["Inspector gaps", report.summary.inspectorGapCount],
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
["Upstream metadata", report.summary.upstreamIssueCount],
["Contract probes", report.summary.contractProbeCount],
],
@ -162,9 +183,9 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
options,
),
"",
"## Live Issues",
"## Other Live Issues",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
"",
"## Compat Gaps",
"",
@ -176,7 +197,11 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
"",
"## Inspector Proof Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
"",
"## Runtime-Covered Inspector Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
"",
"## Upstream Metadata Issues",
"",
@ -225,6 +250,7 @@ function issueBlock(issue, options) {
` - state: ${issueState(issue)}`,
" - evidence:",
...evidenceList(issue.evidence, options).map((item) => ` - ${item}`),
...runtimeCoverageList(issue, options),
].join("\n");
}
@ -232,6 +258,7 @@ function issueState(issue) {
const flags = [
issue.status,
`compat:${issue.compatStatus ?? "none"}`,
issue.runtimeCoverage?.status ? `runtime:${issue.runtimeCoverage.status}` : null,
issue.live ? "live" : null,
issue.deprecated ? "deprecated" : null,
].filter(Boolean);
@ -263,7 +290,7 @@ function triageOverview(report) {
"inspector-gap",
report.summary.inspectorGapCount,
"-",
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments.",
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments. Runtime-covered rows are proof-backed and not open report work.",
],
[
"upstream-metadata",
@ -354,3 +381,15 @@ function evidenceList(evidence, options) {
const formatEvidence = options.formatEvidence ?? ((item) => item);
return items.map((item) => formatEvidence(item));
}
function runtimeCoverageList(issue, options) {
const runtimeCoverage = issue.runtimeCoverage;
if (!runtimeCoverage) {
return [];
}
return [
" - runtime coverage:",
...evidenceList(runtimeCoverage.captured, options).map((item) => ` - captured ${item}`),
...evidenceList(runtimeCoverage.artifacts, options).map((item) => ` - ${item}`),
];
}

View File

@ -4,6 +4,7 @@ import path from "node:path";
export const npmPackagePayloadDir = ".crabpot-package";
export const defaultPluginRootConfigFiles = ["plugin-inspector.config.json", ".plugin-inspector.json"];
export const packageJsonConfigKeys = ["pluginInspector", "plugin-inspector"];
export async function loadInspectorConfig(configPath, options = {}) {
if (!configPath) {
@ -24,16 +25,23 @@ export async function loadInspectorConfig(configPath, options = {}) {
export async function loadPluginRootConfig(configPath = null, options = {}) {
const rootDir = path.resolve(options.cwd ?? process.cwd());
const resolvedPath = configPath ? path.resolve(rootDir, configPath) : findPluginRootConfigPath(rootDir);
if (!resolvedPath && !existsSync(path.join(rootDir, "package.json")) && !existsSync(path.join(rootDir, "openclaw.plugin.json"))) {
const packageJsonPath = path.join(rootDir, "package.json");
const packageJson = await readJsonIfExists(packageJsonPath);
const packageConfig = packageJsonConfig(packageJson);
if (!resolvedPath && !packageJson && !existsSync(path.join(rootDir, "openclaw.plugin.json"))) {
throw new Error("run from a plugin root with package.json/openclaw.plugin.json, or pass --config");
}
const config = resolvedPath ? JSON.parse(await readFile(resolvedPath, "utf8")) : { version: 1 };
const config = resolvedPath
? JSON.parse(await readFile(resolvedPath, "utf8"))
: (packageConfig.config ?? { version: 1 });
const normalizedConfig = await normalizePluginRootConfig(config, { rootDir });
validateInspectorConfig(normalizedConfig);
return {
...normalizedConfig,
rootDir,
configPath: resolvedPath,
configPath: resolvedPath ?? packageConfig.configPath,
};
}
@ -164,6 +172,25 @@ function findPluginRootConfigPath(rootDir) {
return defaultPluginRootConfigFiles.map((file) => path.join(rootDir, file)).find(existsSync) ?? null;
}
function packageJsonConfig(packageJson) {
if (!packageJson) {
return { config: null, configPath: null };
}
for (const key of packageJsonConfigKeys) {
if (packageJson[key] === undefined) {
continue;
}
if (!packageJson[key] || typeof packageJson[key] !== "object" || Array.isArray(packageJson[key])) {
throw new Error(`package.json ${key} must be an object`);
}
return {
config: packageJson[key],
configPath: `package.json#${key}`,
};
}
return { config: null, configPath: null };
}
async function readJsonIfExists(filePath) {
if (!existsSync(filePath)) {
return null;
@ -175,13 +202,43 @@ export function packageId(packageName) {
if (!packageName) {
return null;
}
return packageName
const packageBase = packageName
.split("/")
.pop()
.replace(/^openclaw-/, "")
.replace(/[^a-zA-Z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.toLowerCase();
.replace(/^openclaw-/, "");
return trimHyphenEdges(collapsePackageIdSeparators(packageBase)).toLowerCase();
}
function collapsePackageIdSeparators(value) {
let result = "";
let previousWasHyphen = false;
for (const char of value) {
if (isAsciiAlphaNumeric(char)) {
result += char;
previousWasHyphen = false;
} else if (!previousWasHyphen) {
result += "-";
previousWasHyphen = true;
}
}
return result;
}
function isAsciiAlphaNumeric(char) {
const code = char.charCodeAt(0);
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
}
function trimHyphenEdges(value) {
let start = 0;
let end = value.length;
while (start < end && value[start] === "-") {
start += 1;
}
while (end > start && value[end - 1] === "-") {
end -= 1;
}
return value.slice(start, end);
}
export function inferPluginSeams(pluginManifest, packageJson) {

View File

@ -7,6 +7,7 @@ import {
} from "./synthetic-probes.js";
export const defaultRegistrationAssertions = {
createChatChannelPlugin: ["channel plugin id is stable", "channel factory metadata is captured"],
defineChannelPluginEntry: ["channel id is stable", "setup/config schema can be read", "message envelope metadata is preserved"],
definePluginEntry: ["entrypoint register function is callable", "entrypoint metadata is preserved"],
registerChannel: ["channel id is stable", "inbound/outbound envelope shape is captured", "sender metadata is preserved"],

View File

@ -157,10 +157,15 @@ function requireCompatRecordReconciliation(report, errors) {
.filter((finding) => finding.code === "missing-compat-record")
.map((finding) => `${finding.fixture}:${finding.compatRecord}`),
);
const compatGapRecords = new Set(
report.issues
.filter((issue) => issue.issueClass === "compat-gap" && issue.compatRecord)
.map((issue) => `${issue.fixture}:${issue.compatRecord}`),
);
for (const finding of [...report.warnings, ...report.suggestions].filter((item) => item.compatRecord)) {
const key = `${finding.fixture}:${finding.compatRecord}`;
if (!presentRecords.has(key) && !missingRecords.has(key)) {
if (!presentRecords.has(key) && !missingRecords.has(key) && !compatGapRecords.has(key)) {
errors.push(`${finding.fixture}: compat record ${finding.compatRecord} was not reconciled`);
}
}

View File

@ -74,6 +74,31 @@ export const contractProbeRules = {
contract: "OpenClaw package entrypoints resolve to files in the published or built plugin package.",
target: "package-loader",
},
"package-install-metadata-incomplete": {
id: "package.metadata.install-release",
contract: "Release publishing metadata declares canonical ClawHub and npm install specs.",
target: "package-loader",
},
"package-min-host-version-drift": {
id: "package.metadata.min-host-version",
contract: "Install minimum host version matches the OpenClaw package surface targeted by the plugin.",
target: "package-loader",
},
"package-npm-pack-entrypoint-missing": {
id: "package.npm-pack.entrypoints",
contract: "Advertised npm artifacts include every declared OpenClaw package entrypoint.",
target: "package-loader",
},
"package-npm-pack-metadata-missing": {
id: "package.npm-pack.metadata",
contract: "Advertised npm artifacts include OpenClaw manifest and package metadata.",
target: "package-loader",
},
"package-npm-pack-unavailable": {
id: "package.npm-pack.available",
contract: "Packages that advertise npm install support can produce an npm pack artifact.",
target: "package-loader",
},
"package-openclaw-entry-missing": {
id: "package.entrypoint.openclaw-metadata",
contract: "OpenClaw package metadata declares entrypoints for cold import and registration capture.",
@ -106,6 +131,20 @@ export const contractProbeRules = {
},
};
const openClawOwnedProbeIssueCodes = new Set([
"before-tool-call-probe",
"channel-contract-probe",
"conversation-access-hook",
"registration-capture-gap",
]);
export function compatRecordForIssueCode(code) {
if (!openClawOwnedProbeIssueCodes.has(code)) {
return undefined;
}
return contractProbeRules[code]?.id;
}
export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
const probes = [];
@ -136,7 +175,6 @@ export function probePriority(code, fixturePriority) {
"before-tool-call-probe",
"conversation-access-hook",
"missing-compat-record",
"registration-capture-gap",
"sdk-export-missing",
].includes(code)
) {

View File

@ -1,6 +1,7 @@
import { existsSync } from "node:fs";
import { readdir } from "node:fs/promises";
import path from "node:path";
import { compatRecordForIssueCode } from "./contract-probes.js";
import { readJsonFile } from "./json-file.js";
const conversationAccessHooks = new Set(["agent_end", "llm_input", "llm_output"]);
@ -17,9 +18,13 @@ const channelRegistrations = new Set([
"defineChannelPluginEntry",
"registerChannel",
]);
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
const unsupportedSecurityManifestName = "openclaw.security.json";
const unavailableSecurityManifestSchema = "https://openclaw.ai/schemas/plugin-security.json";
export async function buildCompatibilityFixtureReport({ fixture, inspection, checkoutPath, sourceRoot, rootDir = process.cwd() }) {
const pluginManifests = await readPluginManifests({ checkoutPath, sourceRoot, rootDir });
const securityManifests = await readSecurityManifests({ checkoutPath, sourceRoot, rootDir });
const packageSummaries = await readPackageSummaries({ checkoutPath, sourceRoot, rootDir });
const packageJson = selectPrimaryPackage(packageSummaries);
const sdkImports = unique((inspection.sdkImports ?? []).map((sdkImport) => sdkImport.specifier));
@ -39,6 +44,7 @@ export async function buildCompatibilityFixtureReport({ fixture, inspection, che
manifestFiles: inspection.manifestFiles ?? [],
sourceFiles: inspection.sourceFiles ?? [],
pluginManifests,
securityManifests,
package: packageJson,
packages: packageSummaries,
sdkImports,
@ -72,6 +78,46 @@ export async function readPluginManifests({ checkoutPath, sourceRoot, rootDir =
return manifests;
}
export async function readSecurityManifests({ checkoutPath, sourceRoot, rootDir = process.cwd() }) {
const candidates = unique(
[
path.join(sourceRoot, unsupportedSecurityManifestName),
path.join(checkoutPath, unsupportedSecurityManifestName),
].filter(existsSync),
);
const manifests = [];
for (const manifestPath of candidates) {
const relativePath = path.relative(rootDir, manifestPath);
try {
const manifest = await readJsonFile(manifestPath);
manifests.push({
path: relativePath,
schema: typeof manifest.$schema === "string" ? manifest.$schema : null,
version: typeof manifest.version === "string" ? manifest.version : null,
plugin: typeof manifest.plugin === "string" ? manifest.plugin : null,
expectedBehaviorCount: Array.isArray(manifest.expectedBehaviors)
? manifest.expectedBehaviors.length
: 0,
securityNoteCount: Array.isArray(manifest.securityNotes) ? manifest.securityNotes.length : 0,
validJson: true,
});
} catch {
manifests.push({
path: relativePath,
schema: null,
version: null,
plugin: null,
expectedBehaviorCount: 0,
securityNoteCount: 0,
validJson: false,
});
}
}
return manifests;
}
export async function readPackageSummaries({ checkoutPath, sourceRoot, rootDir = process.cwd(), maxDepth = 3 }) {
const candidates = unique([
path.join(sourceRoot, "package.json"),
@ -106,6 +152,9 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
typeof packageJson.openclaw.build?.pluginSdkVersion === "string"
? packageJson.openclaw.build.pluginSdkVersion
: null,
install: summarizeOpenClawInstall(packageJson.openclaw.install),
release: summarizeOpenClawRelease(packageJson.openclaw.release),
unsupportedMetadata: unsupportedOpenClawPackageMetadata(packageJson.openclaw),
}
: null;
@ -119,6 +168,7 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
version: packageJson.version ?? null,
type: packageJson.type ?? null,
main: typeof packageJson.main === "string" ? packageJson.main : null,
npmPack: summarizeNpmPack(packageJson, openclaw),
dependencies: Object.keys(packageJson.dependencies ?? {}).sort(),
peerDependencies: Object.keys(packageJson.peerDependencies ?? {}).sort(),
optionalDependencies: Object.keys(packageJson.optionalDependencies ?? {}).sort(),
@ -200,6 +250,79 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
});
}
if ((packageSummary.openclaw?.unsupportedMetadata ?? []).length > 0) {
warnings.push({
fixture: fixture.id,
code: "package-openclaw-unsupported-metadata",
level: "warning",
message: "package declares unsupported OpenClaw metadata",
evidence: packageSummary.openclaw.unsupportedMetadata,
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-metadata",
action: "Remove unsupported OpenClaw metadata; native plugins use openclaw.plugin.json plus supported package openclaw fields.",
evidence: packageSummary.openclaw.unsupportedMetadata.join(", "),
});
}
const installMetadataIssues = packageInstallMetadataIssues(packageSummary);
if (installMetadataIssues.length > 0) {
warnings.push({
fixture: fixture.id,
code: "package-install-metadata-incomplete",
level: "warning",
message: "package OpenClaw install metadata does not match advertised release targets",
evidence: installMetadataIssues,
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-metadata",
action: "Ask the plugin to align openclaw.install metadata with openclaw.release publishing targets.",
evidence: installMetadataIssues.join(", "),
});
}
if (packageMinHostVersionDrift(packageSummary)) {
warnings.push({
fixture: fixture.id,
code: "package-min-host-version-drift",
level: "warning",
message: "package openclaw.install.minHostVersion is not a semver floor for the target OpenClaw build version",
evidence: [
`minHostVersion:${packageSummary.openclaw.install.minHostVersion}`,
`buildOpenClawVersion:${packageSummary.openclaw.buildOpenClawVersion}`,
],
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-metadata",
action: "Ask the plugin to publish install.minHostVersion as a semver floor for the OpenClaw package surface it targets.",
evidence: packageSummary.path,
});
}
const npmPackIssues = packageNpmPackIssues(packageSummary, fixtureReport);
for (const finding of npmPackIssues) {
warnings.push({
fixture: fixture.id,
code: finding.code,
level: "warning",
message: finding.message,
evidence: finding.evidence,
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-artifact",
action: "Ask the plugin to make its advertised npm install artifact match the published OpenClaw metadata.",
evidence: finding.evidence.join(", "),
});
}
if (packageSummary.openclaw && packageSummary.openclaw.entrypoints.length === 0) {
warnings.push({
fixture: fixture.id,
@ -217,9 +340,12 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
});
}
const missingEntrypoints = packageSummary.openclaw?.entrypoints.filter((entrypoint) => !entrypoint.exists) ?? [];
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
const missingEntrypoints = entrypoints.filter((entrypoint) => !entrypoint.exists);
const buildEntrypoints = missingEntrypoints.filter((entrypoint) => entrypoint.requiresBuild);
const plainMissingEntrypoints = missingEntrypoints.filter((entrypoint) => !entrypoint.requiresBuild);
const plainMissingEntrypoints = missingEntrypoints.filter(
(entrypoint) => !entrypoint.requiresBuild && !hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
);
if (buildEntrypoints.length > 0) {
suggestions.push({
@ -278,7 +404,7 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
...packageSummary.dependencies,
...packageSummary.peerDependencies,
...packageSummary.optionalDependencies,
]);
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
if (packageSummary.openclaw?.entrypoints.length > 0 && runtimeDependencies.length > 0) {
suggestions.push({
fixture: fixture.id,
@ -349,6 +475,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
suggestions.push(...packageContracts.suggestions);
logs.push(...packageContracts.logs);
decisions.push(...packageContracts.decisions);
classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions });
for (const pluginManifest of fixtureReport.pluginManifests) {
const providerAuthKeys = Object.keys(pluginManifest.providerAuthEnvVars ?? {});
@ -399,6 +526,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "warning",
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
evidence: detailEvidence(conversationHookDetails),
compatRecord: compatRecordForIssueCode("conversation-access-hook"),
});
decisions.push({
fixture: fixture.id,
@ -459,6 +587,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
evidence: detailEvidence(captureGapRegistrationDetails),
compatRecord: compatRecordForIssueCode("registration-capture-gap"),
});
decisions.push({
fixture: fixture.id,
@ -477,6 +606,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
evidence: detailEvidence(hookDetails),
compatRecord: compatRecordForIssueCode("before-tool-call-probe"),
});
decisions.push({
fixture: fixture.id,
@ -500,6 +630,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "add channel envelope, config-schema, and runtime metadata probes",
evidence: detailEvidence(channelRegistrationDetails),
compatRecord: compatRecordForIssueCode("channel-contract-probe"),
});
decisions.push({
fixture: fixture.id,
@ -551,6 +682,45 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
return { warnings, suggestions, logs, decisions };
}
function classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions }) {
for (const securityManifest of fixtureReport.securityManifests ?? []) {
warnings.push({
fixture: fixture.id,
code: "unrecognized-security-manifest",
level: "warning",
message:
"openclaw.security.json is not a supported OpenClaw or ClawHub security contract and is ignored by install safety checks",
evidence: [securityManifest.path],
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "security-metadata",
action:
"Remove the advisory security manifest or replace it with a supported, versioned OpenClaw/ClawHub security contract once one exists.",
evidence: securityManifest.path,
});
if (securityManifest.schema === unavailableSecurityManifestSchema) {
warnings.push({
fixture: fixture.id,
code: "security-manifest-schema-unavailable",
level: "warning",
message: "openclaw.security.json references an OpenClaw schema URL that is not currently published",
evidence: [`${securityManifest.path}:$schema=${securityManifest.schema}`],
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "security-metadata",
action:
"Do not rely on the schema URL until OpenClaw publishes and documents a real plugin security metadata schema.",
evidence: securityManifest.schema,
});
}
}
}
function registrationCaptureGapDetails(inspection, targetOpenClaw) {
const apiRegistrationDetails = inspection.registrationDetails.filter((registration) =>
registration.name.startsWith("register"),
@ -753,6 +923,46 @@ function collectOpenClawEntrypoints(packageDir, openclaw, options) {
});
}
function hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
if (!isSourceEntrypoint(entrypoint.specifier)) {
return false;
}
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
if (
entrypoints.some(
(candidate) =>
candidate.exists &&
candidate.requiresBuild &&
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
)
) {
return true;
}
if (entrypoint.kind === "extension" && entrypoints.some((candidate) => candidate.kind === "runtimeExtension" && candidate.exists)) {
return true;
}
const packageDir = path.dirname(packageSummary.path);
return existsSync(path.resolve(packageDir, runtimeBuildSpecifier));
}
function isSourceEntrypoint(specifier) {
return /\.(?:ts|tsx)$/.test(specifier);
}
function runtimeBuildSpecifierFor(specifier) {
const normalized = normalizeEntrypointSpecifier(specifier);
const basename = path.posix.basename(normalized).replace(/\.(?:ts|tsx)$/, ".js");
return `./dist/${basename}`;
}
function normalizeEntrypointSpecifier(specifier) {
const normalized = specifier.replaceAll("\\", "/");
return normalized.startsWith("./") ? normalized : `./${normalized}`;
}
async function findPackageFiles(root, options, depth = 0) {
if (!existsSync(root) || depth > options.maxDepth) {
return [];
@ -781,6 +991,269 @@ function selectPrimaryPackage(packages) {
return packages[0] ?? null;
}
function summarizeOpenClawInstall(install) {
if (!install || typeof install !== "object") {
return null;
}
return {
clawhubSpec: stringOrNull(install.clawhubSpec),
npmSpec: stringOrNull(install.npmSpec),
defaultChoice: stringOrNull(install.defaultChoice),
minHostVersion: stringOrNull(install.minHostVersion),
};
}
function summarizeOpenClawRelease(release) {
if (!release || typeof release !== "object") {
return null;
}
return {
publishToClawHub: booleanOrNull(release.publishToClawHub),
publishToNpm: booleanOrNull(release.publishToNpm),
};
}
function unsupportedOpenClawPackageMetadata(openclaw) {
if (!openclaw || typeof openclaw !== "object") {
return [];
}
return Object.keys(openclaw)
.filter((key) => key === "bundle")
.map((key) => `openclaw.${key}`);
}
function summarizeNpmPack(packageJson, openclaw) {
const files = arrayValues(packageJson.files).map(normalizePackagePath).filter((item) => item.length > 0);
return {
advertised: openclaw?.release?.publishToNpm === true || nonEmptyString(openclaw?.install?.npmSpec),
private: packageJson.private === true,
filesMode: files.length > 0 ? "allowlist" : "implicit",
files,
invalidFileSpecs: files.filter((item) => invalidPackageFileSpec(item)),
};
}
function packageInstallMetadataIssues(packageSummary) {
const openclaw = packageSummary.openclaw;
if (!openclaw) {
return [];
}
const issues = [];
const install = openclaw.install;
const release = openclaw.release;
const publishToClawHub = release?.publishToClawHub === true;
const publishToNpm = release?.publishToNpm === true;
if (publishToClawHub && !nonEmptyString(install?.clawhubSpec)) {
issues.push("openclaw.release.publishToClawHub requires openclaw.install.clawhubSpec");
}
if (publishToNpm && !nonEmptyString(install?.npmSpec)) {
issues.push("openclaw.release.publishToNpm requires openclaw.install.npmSpec");
}
if (publishToNpm && nonEmptyString(install?.npmSpec) && nonEmptyString(packageSummary.name) && install.npmSpec !== packageSummary.name) {
issues.push(`openclaw.install.npmSpec:${install.npmSpec} does not match package name:${packageSummary.name}`);
}
if (nonEmptyString(install?.defaultChoice) && !["clawhub", "npm"].includes(install.defaultChoice)) {
issues.push(`openclaw.install.defaultChoice:${install.defaultChoice} must be clawhub or npm`);
}
if (install?.defaultChoice === "clawhub" && !nonEmptyString(install.clawhubSpec)) {
issues.push("openclaw.install.defaultChoice clawhub requires openclaw.install.clawhubSpec");
}
if (install?.defaultChoice === "npm" && !nonEmptyString(install.npmSpec)) {
issues.push("openclaw.install.defaultChoice npm requires openclaw.install.npmSpec");
}
return issues;
}
function packageNpmPackIssues(packageSummary, fixtureReport) {
if (!packageSummary.npmPack?.advertised) {
return [];
}
const findings = [];
const unavailable = [];
if (packageSummary.npmPack.private) {
unavailable.push("package.json private:true");
}
if (!nonEmptyString(packageSummary.name)) {
unavailable.push("package.json name missing");
}
if (!nonEmptyString(packageSummary.version)) {
unavailable.push("package.json version missing");
}
unavailable.push(...packageSummary.npmPack.invalidFileSpecs.map((item) => `invalid files entry:${item}`));
if (unavailable.length > 0) {
findings.push({
code: "package-npm-pack-unavailable",
message: "package advertises npm install or publish metadata but cannot produce a usable npm pack artifact",
evidence: unavailable,
});
}
const missingMetadata = packageNpmPackMissingMetadata(packageSummary, fixtureReport);
if (missingMetadata.length > 0) {
findings.push({
code: "package-npm-pack-metadata-missing",
message: "advertised npm artifact would not include required OpenClaw package metadata",
evidence: missingMetadata,
});
}
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
const missingEntrypoints = entrypoints
.filter(
(entrypoint) =>
!repoPathIncludedInNpmPack(packageSummary, entrypoint.relativePath) &&
!hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
)
.map((entrypoint) => `${entrypoint.kind}:${entrypoint.specifier} -> ${entrypoint.relativePath}`) ?? [];
if (missingEntrypoints.length > 0) {
findings.push({
code: "package-npm-pack-entrypoint-missing",
message: "advertised npm artifact would not include declared OpenClaw entrypoints",
evidence: missingEntrypoints,
});
}
return findings;
}
function packageNpmPackMissingMetadata(packageSummary, fixtureReport) {
const missing = [];
if (!repoPathIncludedInNpmPack(packageSummary, packageSummary.path)) {
missing.push(packageSummary.path);
}
for (const manifest of fixtureReport.pluginManifests ?? []) {
if (repoPathWithinPackage(packageSummary, manifest.path) && !repoPathIncludedInNpmPack(packageSummary, manifest.path)) {
missing.push(manifest.path);
}
}
return missing;
}
function hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
if (!isSourceEntrypoint(entrypoint.specifier)) {
return false;
}
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
const matchingRuntimeEntrypoint = entrypoints.find(
(candidate) =>
candidate.requiresBuild &&
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
);
if (matchingRuntimeEntrypoint && repoPathIncludedInNpmPack(packageSummary, matchingRuntimeEntrypoint.relativePath)) {
return true;
}
if (
entrypoint.kind === "extension" &&
entrypoints.some(
(candidate) => candidate.kind === "runtimeExtension" && repoPathIncludedInNpmPack(packageSummary, candidate.relativePath),
)
) {
return true;
}
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
const runtimeBuildPath = path.posix.join(packageDir === "." ? "" : packageDir, normalizeEntrypointSpecifier(runtimeBuildSpecifier));
return repoPathIncludedInNpmPack(packageSummary, runtimeBuildPath);
}
function packageMinHostVersionDrift(packageSummary) {
const openclaw = packageSummary.openclaw;
if (!nonEmptyString(openclaw?.install?.minHostVersion) || !nonEmptyString(openclaw?.buildOpenClawVersion)) {
return false;
}
return parseMinHostVersionFloor(openclaw.install.minHostVersion) !== openclaw.buildOpenClawVersion;
}
function parseMinHostVersionFloor(value) {
const match = /^>=([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/.exec(value);
return match?.[1] ?? null;
}
function repoPathIncludedInNpmPack(packageSummary, repoPath) {
const packageRelativePath = packageRelativeRepoPath(packageSummary, repoPath);
if (!packageRelativePath) {
return false;
}
if (npmAlwaysPacksPath(packageRelativePath)) {
return true;
}
if (packageSummary.npmPack?.filesMode !== "allowlist") {
return true;
}
return packageSummary.npmPack.files.some((spec) => packageFileSpecIncludesPath(spec, packageRelativePath));
}
function repoPathWithinPackage(packageSummary, repoPath) {
return packageRelativeRepoPath(packageSummary, repoPath) !== null;
}
function packageRelativeRepoPath(packageSummary, repoPath) {
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
const normalized = normalizeRepoPath(repoPath);
if (packageDir === ".") {
return normalized;
}
if (normalized === packageDir) {
return "";
}
return normalized.startsWith(`${packageDir}/`) ? normalized.slice(packageDir.length + 1) : null;
}
function npmAlwaysPacksPath(packageRelativePath) {
const base = path.posix.basename(packageRelativePath).toLowerCase();
return packageRelativePath === "package.json" || /^readme(?:\..*)?$/u.test(base) || /^licen[cs]e(?:\..*)?$/u.test(base);
}
function packageFileSpecIncludesPath(spec, packageRelativePath) {
if (spec === "." || spec === packageRelativePath) {
return true;
}
if (spec.includes("*")) {
return globLikeSpecIncludesPath(spec, packageRelativePath);
}
return packageRelativePath.startsWith(`${spec.replace(/\/$/u, "")}/`);
}
function globLikeSpecIncludesPath(spec, packageRelativePath) {
return globSegmentsIncludePath(spec.split("/"), packageRelativePath.split("/"));
}
function globSegmentsIncludePath(specSegments, packageSegments) {
if (specSegments.length === 0) {
return packageSegments.length === 0;
}
const [head, ...tail] = specSegments;
if (head === "**") {
return globSegmentsIncludePath(tail, packageSegments) || (packageSegments.length > 0 && globSegmentsIncludePath(specSegments, packageSegments.slice(1)));
}
if (packageSegments.length === 0) {
return false;
}
return globSegmentIncludesPath(head, packageSegments[0]) && globSegmentsIncludePath(tail, packageSegments.slice(1));
}
function globSegmentIncludesPath(specSegment, packageSegment) {
const escaped = specSegment.replace(/[.+?^${}()|[\]\\]/gu, "\\$&").replaceAll("*", ".*");
return new RegExp(`^${escaped}$`, "u").test(packageSegment);
}
function invalidPackageFileSpec(spec) {
return spec.startsWith("/") || spec === ".." || spec.startsWith("../") || spec.includes("/../");
}
function normalizePackagePath(value) {
return normalizeRepoPath(value).replace(/^\.\/+/u, "").replace(/\/$/u, "");
}
function packageRank(packageSummary) {
if (packageSummary.openclaw?.entrypoints.length > 0) {
return 0;
@ -795,6 +1268,22 @@ function arrayValues(value) {
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
}
function stringOrNull(value) {
return typeof value === "string" ? value : null;
}
function booleanOrNull(value) {
return typeof value === "boolean" ? value : null;
}
function nonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeRepoPath(value) {
return String(value ?? "").replaceAll("\\", "/").replace(/^\.\/+/u, "");
}
function detailEvidence(details, key = "name") {
return unique(details.map((detail) => `${detail[key]} @ ${detail.ref}`));
}

View File

@ -1,4 +1,4 @@
import { mkdir } from "node:fs/promises";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
@ -24,22 +24,48 @@ export async function buildImportLoopProfile(options = {}) {
const entrypoint = options.entrypoint ?? defaultImportLoopProfileOptions.entrypoint;
assertRunCount(runs, 20);
const baseline = await buildBaselineProfile({ ...options, rootDir, runs });
const samples = [];
for (let index = 0; index < runs; index += 1) {
samples.push(await runCaptureSample({ ...options, entrypoint, index, rootDir }));
const sample = await runCaptureSample({ ...options, entrypoint, index, rootDir });
samples.push(applyBaselineAdjustment(sample, baseline));
}
const wallMs = samples.map((sample) => sample.wallMs).sort((left, right) => left - right);
const pluginWallDeltaMs = samples.map((sample) => sample.pluginWallDeltaMs).sort((left, right) => left - right);
const openClawImportMs = openClawLifecycleMetric(samples, "importMs");
const openClawActivationMs = openClawLifecycleMetric(samples, "activationMs");
const rssSampleCount = samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)), 0);
const cpuSampleCount = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
return {
generatedAt: options.generatedAt ?? defaultImportLoopProfileOptions.generatedAt,
mode: options.mode ?? "subprocess-cold-import-loop",
mode: options.mode ?? "baseline-adjusted-cold-capture-loop",
entrypoint,
baseline,
summary: {
runs,
baselineRuns: baseline.runs,
baselineFailCount: baseline.failCount,
p50WallMs: percentile(wallMs, 0.5),
p95WallMs: percentile(wallMs, 0.95),
p50PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.5),
p95PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.95),
openClawLifecycleCount: openClawImportMs.length,
p50OpenClawImportMs: percentile(openClawImportMs, 0.5),
p95OpenClawImportMs: percentile(openClawImportMs, 0.95),
p50OpenClawActivationMs: percentile(openClawActivationMs, 0.5),
p95OpenClawActivationMs: percentile(openClawActivationMs, 0.95),
maxPeakRssMb: Math.max(0, ...samples.map((sample) => sample.peakRssMb)),
maxCpuMsEstimate: Math.max(0, ...samples.map((sample) => sample.cpuMsEstimate)),
maxPluginPeakRssDeltaMb: Math.max(0, ...samples.map((sample) => sample.pluginPeakRssDeltaMb)),
maxPluginCpuDeltaMsEstimate: Math.max(0, ...samples.map((sample) => sample.pluginCpuDeltaMsEstimate)),
baselineReferenceWallMs: baseline.reference.wallMs,
baselineReferencePeakRssMb: baseline.reference.peakRssMb,
baselineReferenceCpuMsEstimate: baseline.reference.cpuMsEstimate,
statSampleCount,
rssSampleCount,
cpuSampleCount,
capturedCount: samples.reduce((sum, sample) => sum + sample.capturedCount, 0),
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
},
@ -52,6 +78,9 @@ export function validateImportLoopProfile(report) {
if (report.summary.failCount > 0) {
errors.push(`import loop has ${report.summary.failCount} failed sample(s)`);
}
if ((report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0) > 0) {
errors.push("import loop baseline capture failed");
}
if (report.summary.capturedCount < report.summary.runs) {
errors.push("import loop did not capture at least one contract per run");
}
@ -85,7 +114,11 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
"",
"## Summary",
"",
markdownTable(Object.entries(report.summary).map(([key, value]) => [key, value]), ["Metric", "Value"]),
markdownTable(summaryRows(report), ["Metric", "Value"]),
"",
"## Harness Baseline",
"",
markdownTable(baselineRows(report), ["Metric", "Value"]),
"",
"## Samples",
"",
@ -94,22 +127,132 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
sample.index,
sample.status,
sample.capturedCount,
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.importMs),
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.activationMs),
formatOptionalMetric(sample.pluginWallDeltaMs, "ms"),
formatSampledMetric(sample.pluginPeakRssDeltaMb, sample.rssSampleCount),
formatSampledMetric(sample.pluginCpuDeltaMsEstimate, sample.cpuSampleCount, "ms"),
`${sample.wallMs} ms`,
`${sample.peakRssMb} MB`,
`${sample.cpuMsEstimate} ms`,
formatSampledMetric(sample.peakRssMb, sample.rssSampleCount),
formatSampledMetric(sample.cpuMsEstimate, sample.cpuSampleCount, "ms"),
`${sample.rssSampleCount ?? 0}/${sample.cpuSampleCount ?? 0}`,
sample.exitCode,
]),
["Run", "Status", "Captured", "Wall", "Peak RSS", "CPU Estimate", "Exit"],
[
"Run",
"Status",
"Captured",
"OpenClaw Import",
"OpenClaw Activate",
"Plugin Wall Delta",
"Plugin RSS Delta",
"Plugin CPU Delta",
"Raw Wall",
"Raw Peak RSS",
"Raw CPU Estimate",
"RSS/CPU samples",
"Exit",
],
),
].join("\n");
}
async function buildBaselineProfile(options) {
const baselineRuns = options.baseline === false ? 0 : options.baselineRuns ?? Math.min(options.runs, 3);
if (baselineRuns <= 0) {
return emptyBaseline();
}
const entrypoint = await writeBaselineEntrypoint(options);
const samples = [];
for (let index = 0; index < baselineRuns; index += 1) {
samples.push(
await runCaptureSample({
...options,
entrypoint,
index,
sampleName: "baseline",
rootDir: options.rootDir,
}),
);
}
const wallMs = sortedMetric(samples, "wallMs");
const peakRssMb = sortedMetric(samples, "peakRssMb");
const cpuMsEstimate = sortedMetric(samples, "cpuMsEstimate");
return {
mode: "minimal-plugin-capture",
runs: baselineRuns,
entrypoint: path.relative(options.rootDir, entrypoint),
reference: {
wallMs: percentile(wallMs, 0.5),
peakRssMb: percentile(peakRssMb, 0.5),
cpuMsEstimate: percentile(cpuMsEstimate, 0.5),
},
max: {
wallMs: wallMs.at(-1) ?? 0,
peakRssMb: peakRssMb.at(-1) ?? 0,
cpuMsEstimate: cpuMsEstimate.at(-1) ?? 0,
},
statSampleCount: samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0),
rssSampleCount: samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? 0), 0),
cpuSampleCount: samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0),
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
samples,
};
}
function emptyBaseline() {
return {
mode: "disabled",
runs: 0,
entrypoint: null,
reference: {
wallMs: 0,
peakRssMb: 0,
cpuMsEstimate: 0,
},
max: {
wallMs: 0,
peakRssMb: 0,
cpuMsEstimate: 0,
},
statSampleCount: 0,
rssSampleCount: 0,
cpuSampleCount: 0,
failCount: 0,
samples: [],
};
}
async function writeBaselineEntrypoint(options) {
const outputDir = resolveFromRoot(
options.rootDir,
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
);
const baselinePath = path.join(outputDir, "baseline-plugin.mjs");
await mkdir(path.dirname(baselinePath), { recursive: true });
await writeFile(
baselinePath,
[
"export default {",
" register(api) {",
" api.registerTool({ name: 'baseline_tool', inputSchema: { type: 'object' }, run() {} });",
" },",
"};",
"",
].join("\n"),
"utf8",
);
return baselinePath;
}
async function runCaptureSample(options) {
const outputDir = resolveFromRoot(
options.rootDir,
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
);
const outputPath = path.join(outputDir, `capture-${options.index}.json`);
const outputPath = path.join(outputDir, `${options.sampleName ?? "capture"}-${options.index}.json`);
await mkdir(path.dirname(outputPath), { recursive: true });
const command = buildCaptureCommand({ ...options, outputPath });
@ -126,14 +269,125 @@ async function runCaptureSample(options) {
exitCode: profile.exitCode,
status: output?.status ?? "failed",
capturedCount: output?.captured?.length ?? 0,
openClawLifecycle: output?.openClawLifecycle ?? null,
wallMs: profile.wallMs,
peakRssMb: profile.peakRssMb,
peakCpuPercent: profile.peakCpuPercent,
cpuMsEstimate: profile.cpuMsEstimate,
statSampleCount: profile.statSampleCount,
rssSampleCount: profile.rssSampleCount,
cpuSampleCount: profile.cpuSampleCount,
stderrPreview: profile.stderrPreview,
};
}
function summaryRows(report) {
return [
["runs", report.summary.runs],
["baselineRuns", report.summary.baselineRuns ?? report.baseline?.runs ?? 0],
["baselineFailCount", report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0],
["p50WallMs", report.summary.p50WallMs],
["p95WallMs", report.summary.p95WallMs],
...(Number.isFinite(report.summary.p50PluginWallDeltaMs)
? [
["p50PluginWallDeltaMs", report.summary.p50PluginWallDeltaMs],
["p95PluginWallDeltaMs", report.summary.p95PluginWallDeltaMs],
["maxPluginPeakRssDeltaMb", formatSampledMetric(report.summary.maxPluginPeakRssDeltaMb, report.summary.rssSampleCount)],
[
"maxPluginCpuDeltaMsEstimate",
formatSampledMetric(report.summary.maxPluginCpuDeltaMsEstimate, report.summary.cpuSampleCount, "ms"),
],
]
: []),
...((report.summary.openClawLifecycleCount ?? 0) > 0
? [
["openClawLifecycleCount", report.summary.openClawLifecycleCount],
["p50OpenClawImportMs", `${report.summary.p50OpenClawImportMs} ms`],
["p95OpenClawImportMs", `${report.summary.p95OpenClawImportMs} ms`],
["p50OpenClawActivationMs", `${report.summary.p50OpenClawActivationMs} ms`],
["p95OpenClawActivationMs", `${report.summary.p95OpenClawActivationMs} ms`],
]
: []),
["maxPeakRssMb", formatSampledMetric(report.summary.maxPeakRssMb, report.summary.rssSampleCount)],
["maxCpuMsEstimate", formatSampledMetric(report.summary.maxCpuMsEstimate, report.summary.cpuSampleCount, "ms")],
...(Number.isFinite(report.summary.baselineReferenceWallMs)
? [
["baselineReferenceWallMs", `${report.summary.baselineReferenceWallMs} ms`],
["baselineReferencePeakRssMb", formatSampledMetric(report.summary.baselineReferencePeakRssMb, report.baseline?.rssSampleCount ?? 0)],
[
"baselineReferenceCpuMsEstimate",
formatSampledMetric(report.summary.baselineReferenceCpuMsEstimate, report.baseline?.cpuSampleCount ?? 0, "ms"),
],
]
: []),
["statSampleCount", report.summary.statSampleCount ?? 0],
["rssSampleCount", report.summary.rssSampleCount ?? 0],
["cpuSampleCount", report.summary.cpuSampleCount ?? 0],
["capturedCount", report.summary.capturedCount],
["failCount", report.summary.failCount],
];
}
function baselineRows(report) {
const baseline = report.baseline ?? emptyBaseline();
return [
["mode", baseline.mode],
["runs", baseline.runs],
["entrypoint", baseline.entrypoint ?? "-"],
["referenceWallMs", `${baseline.reference?.wallMs ?? 0} ms`],
["referencePeakRssMb", formatSampledMetric(baseline.reference?.peakRssMb ?? 0, baseline.rssSampleCount)],
["referenceCpuMsEstimate", formatSampledMetric(baseline.reference?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
["maxWallMs", `${baseline.max?.wallMs ?? 0} ms`],
["maxPeakRssMb", formatSampledMetric(baseline.max?.peakRssMb ?? 0, baseline.rssSampleCount)],
["maxCpuMsEstimate", formatSampledMetric(baseline.max?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
["statSampleCount", baseline.statSampleCount ?? 0],
["failCount", baseline.failCount ?? 0],
];
}
function formatSampledMetric(value, count, unit = "MB") {
if ((count ?? 0) <= 0) {
return "n/a";
}
return `${value} ${unit}`;
}
function formatOptionalMetric(value, unit) {
if (!Number.isFinite(value)) {
return "n/a";
}
return `${value} ${unit}`;
}
function formatOpenClawLifecycleMetric(value) {
return Number.isFinite(value) ? `${value} ms` : "n/a";
}
function openClawLifecycleMetric(samples, field) {
return samples
.map((sample) => sample.openClawLifecycle?.[field])
.filter((value) => Number.isFinite(value))
.sort((left, right) => left - right);
}
function applyBaselineAdjustment(sample, baseline) {
return {
...sample,
pluginWallDeltaMs: roundNonNegative(sample.wallMs - baseline.reference.wallMs, 0),
pluginPeakRssDeltaMb: roundNonNegative(sample.peakRssMb - baseline.reference.peakRssMb, 1),
pluginCpuDeltaMsEstimate: roundNonNegative(sample.cpuMsEstimate - baseline.reference.cpuMsEstimate, 0),
};
}
function sortedMetric(samples, field) {
return samples.map((sample) => sample[field]).sort((left, right) => left - right);
}
function roundNonNegative(value, digits) {
const scale = 10 ** digits;
return Math.max(0, Math.round(value * scale) / scale);
}
function buildCaptureCommand(options) {
if (typeof options.captureCommand === "function") {
return options.captureCommand({

View File

@ -1,11 +1,248 @@
import * as pluginApi from "./api.js";
import * as ciPolicyApi from "./ci-policy.js";
import * as ciSummaryApi from "./ci-summary.js";
import * as configApi from "./config.js";
import * as contractCaptureApi from "./contract-capture.js";
import * as contractCoverageApi from "./contract-coverage.js";
import * as executionResultsApi from "./execution-results.js";
import * as importLoopProfileApi from "./import-loop-profile.js";
import * as inspectorApi from "./inspector.js";
import * as issuesApi from "./issues.js";
import * as openClawTargetApi from "./openclaw-target.js";
import * as profileDiffApi from "./profile-diff.js";
import * as refDiffApi from "./ref-diff.js";
import * as reportApi from "./report.js";
import * as runtimeProfileApi from "./runtime-profile.js";
import * as runtimeReconciliationApi from "./runtime-reconciliation.js";
import * as syntheticProbeSuiteApi from "./synthetic-probe-suite.js";
import * as syntheticProbesApi from "./synthetic-probes.js";
export const pluginRoot = Object.freeze({
loadConfig: pluginApi.loadPluginConfig,
inspect: pluginApi.inspectPluginRoot,
runCheck: pluginApi.runPluginCheck,
captureEntrypoint: pluginApi.capturePluginEntrypoint,
setup: pluginApi.setupPluginInspector,
});
export const fixtureSuites = Object.freeze({
loadConfig: configApi.loadInspectorConfig,
inspect: pluginApi.inspectCompatibilityFixtureSetConfig,
inspectStatic: pluginApi.inspectFixtureSetConfig,
runReport: pluginApi.runFixtureSetReport,
writeReports: pluginApi.writeFixtureSetReports,
renderReport: pluginApi.renderFixtureSetMarkdownReport,
renderIssues: pluginApi.renderFixtureSetIssuesReport,
buildColdImportReadiness: pluginApi.buildFixtureSetColdImportReadiness,
runColdImportReadiness: pluginApi.runFixtureSetColdImportReadiness,
writeColdImportReadiness: pluginApi.writeFixtureSetColdImportReadiness,
buildWorkspacePlan: pluginApi.buildFixtureSetWorkspacePlan,
runWorkspacePlan: pluginApi.runFixtureSetWorkspacePlan,
writeWorkspacePlan: pluginApi.writeFixtureSetWorkspacePlan,
buildPlatformProbes: pluginApi.buildFixtureSetPlatformProbes,
runPlatformProbes: pluginApi.runFixtureSetPlatformProbes,
writePlatformProbes: pluginApi.writeFixtureSetPlatformProbes,
});
export const staticInspection = Object.freeze({
loadConfig: configApi.loadInspectorConfig,
inspectSourceText: inspectorApi.inspectSourceText,
inspectPlugin: inspectorApi.inspectPlugin,
inspectFixtureSet: inspectorApi.inspectFixtureSet,
});
export const reports = Object.freeze({
renderMarkdown: reportApi.renderMarkdownReport,
renderTextSummary: pluginApi.renderTextSummary,
sanitizeArtifact: reportApi.sanitizeReportArtifact,
write: reportApi.writeReport,
issueId: issuesApi.issueId,
classifyIssueFinding: issuesApi.classifyIssueFinding,
knownIssueCodes: issuesApi.knownIssueCodes,
openClawTargetPathCandidates: openClawTargetApi.openClawTargetPathCandidates,
readOpenClawTargetSurface: openClawTargetApi.readOpenClawTargetSurface,
});
export const contracts = Object.freeze({
buildCapture: contractCaptureApi.buildContractCapture,
writeCapture: contractCaptureApi.writeContractCapture,
renderCapture: contractCaptureApi.renderContractCaptureMarkdown,
validateCapture: contractCaptureApi.validateContractCapture,
validateCoverage: contractCoverageApi.validateContractCoverage,
defaults: Object.freeze({
registrationAssertions: contractCaptureApi.defaultRegistrationAssertions,
registrationArguments: contractCaptureApi.defaultRegistrationArguments,
hookAssertions: contractCaptureApi.defaultHookAssertions,
hookEvents: contractCaptureApi.defaultHookEvents,
hookContexts: contractCaptureApi.defaultHookContexts,
}),
});
export const ci = Object.freeze({
buildSummary: ciSummaryApi.buildCiSummary,
writeSummary: ciSummaryApi.writeCiSummary,
renderSummary: ciSummaryApi.renderCiSummaryMarkdown,
readReports: ciSummaryApi.readCiReports,
deriveStatus: ciSummaryApi.deriveCiStatus,
buildPolicyReport: ciPolicyApi.buildCiPolicyReport,
writePolicyReport: ciPolicyApi.writeCiPolicyReport,
renderPolicyReport: ciPolicyApi.renderCiPolicyMarkdown,
validatePolicy: ciPolicyApi.validateCiPolicy,
validatePolicyReport: ciPolicyApi.validateCiPolicyReport,
buildExecutionResults: executionResultsApi.buildExecutionResultsReport,
writeExecutionResults: executionResultsApi.writeExecutionResultsReport,
renderExecutionResults: executionResultsApi.renderExecutionResultsMarkdown,
writeOutputs: pluginApi.writeCiOutputArtifacts,
});
export const runtime = Object.freeze({
buildProfile: runtimeProfileApi.buildRuntimeProfile,
writeProfile: runtimeProfileApi.writeRuntimeProfile,
renderProfile: runtimeProfileApi.renderRuntimeProfileMarkdown,
validateProfile: runtimeProfileApi.validateRuntimeProfile,
buildProfileDiff: profileDiffApi.buildProfileDiff,
writeProfileDiff: profileDiffApi.writeProfileDiff,
renderProfileDiff: profileDiffApi.renderProfileDiffMarkdown,
validateProfileDiff: profileDiffApi.validateProfileDiff,
buildRefDiff: refDiffApi.buildRefDiff,
writeRefDiff: refDiffApi.writeRefDiff,
renderRefDiff: refDiffApi.renderRefDiffMarkdown,
validateRefDiff: refDiffApi.validateRefDiff,
buildImportLoopProfile: importLoopProfileApi.buildImportLoopProfile,
writeImportLoopProfile: importLoopProfileApi.writeImportLoopProfile,
renderImportLoopProfile: importLoopProfileApi.renderImportLoopProfileMarkdown,
validateImportLoopProfile: importLoopProfileApi.validateImportLoopProfile,
applyExecutionCoverage: runtimeReconciliationApi.applyRuntimeExecutionCoverage,
buildExecutionCoverage: runtimeReconciliationApi.buildRuntimeExecutionCoverage,
});
export const synthetic = Object.freeze({
buildPlan: syntheticProbesApi.buildSyntheticProbePlan,
buildPlanFromReport: syntheticProbeSuiteApi.buildSyntheticProbePlanFromReport,
writePlan: syntheticProbesApi.writeSyntheticProbePlan,
renderPlan: syntheticProbesApi.renderSyntheticProbeMarkdown,
validatePlan: syntheticProbesApi.validateSyntheticProbePlan,
runCaptured: syntheticProbesApi.runCapturedSyntheticProbes,
registrationExecutionProfiles: syntheticProbesApi.syntheticRegistrationExecutionProfiles,
defaultHookEvents: syntheticProbesApi.defaultSyntheticHookEvents,
defaultHookContexts: syntheticProbesApi.defaultSyntheticHookContexts,
defaultRegistrationArguments: syntheticProbesApi.defaultSyntheticRegistrationArguments,
});
export {
capturePluginEntrypoint,
buildFixtureSetColdImportReadiness,
buildFixtureSetPlatformProbes,
buildFixtureSetWorkspacePlan,
createCaptureApi,
inspectCompatibilityFixtureSetConfig,
inspectFixtureSetConfig,
inspectPluginRoot,
loadPluginConfig,
renderFixtureSetColdImportReadinessMarkdown,
renderFixtureSetIssuesReport,
renderFixtureSetMarkdownReport,
renderFixtureSetPlatformProbesMarkdown,
renderFixtureSetWorkspacePlanMarkdown,
renderTextSummary,
runFixtureSetColdImportReadiness,
runFixtureSetPlatformProbes,
runFixtureSetReport,
runFixtureSetWorkspacePlan,
runPluginCheck,
setupPluginInspector,
validateColdImportReadiness,
validateFixtureSetPlatformProbes,
validateFixtureSetWorkspacePlan,
writeCiOutputArtifacts,
writeFixtureSetColdImportReadiness,
writeFixtureSetPlatformProbes,
writeFixtureSetReports,
writeFixtureSetWorkspacePlan,
writePluginReports,
} from "./api.js";
export {
buildContractCapture,
defaultHookAssertions,
defaultHookContexts,
defaultHookEvents,
defaultRegistrationArguments,
defaultRegistrationAssertions,
renderContractCaptureMarkdown,
validateContractCapture,
writeContractCapture,
} from "./contract-capture.js";
export { validateContractCoverage } from "./contract-coverage.js";
export {
buildCiPolicyReport,
defaultCiPolicyReportOptions,
renderCiPolicyMarkdown,
validateCiPolicy,
validateCiPolicyReport,
writeCiPolicyReport,
} from "./ci-policy.js";
export {
buildCiSummary,
defaultCiReportPaths,
deriveCiStatus,
readCiReports,
renderCiSummaryMarkdown,
writeCiSummary,
} from "./ci-summary.js";
export { loadInspectorConfig } from "./config.js";
export {
buildExecutionResultsReport,
defaultExecutionResultsOptions,
renderExecutionResultsMarkdown,
writeExecutionResultsReport,
} from "./execution-results.js";
export {
buildImportLoopProfile,
defaultImportLoopProfileOptions,
renderImportLoopProfileMarkdown,
validateImportLoopProfile,
writeImportLoopProfile,
} from "./import-loop-profile.js";
export { classifyIssueFinding, issueId, knownIssueCodes } from "./issues.js";
export { inspectFixtureSet, inspectPlugin, inspectSourceText } from "./inspector.js";
export { openClawTargetPathCandidates, readOpenClawTargetSurface } from "./openclaw-target.js";
export {
buildProfileDiff,
defaultProfileDiffOptions,
renderProfileDiffMarkdown,
validateProfileDiff,
writeProfileDiff,
} from "./profile-diff.js";
export {
buildRefDiff,
defaultRefDiffDimensions,
defaultRefDiffOptions,
renderRefDiffMarkdown,
validateRefDiff,
writeRefDiff,
} from "./ref-diff.js";
export { renderMarkdownReport, sanitizeReportArtifact, writeReport } from "./report.js";
export {
buildRuntimeProfile,
defaultRuntimeProfileCommands,
defaultRuntimeProfileOptions,
renderRuntimeProfileMarkdown,
validateRuntimeProfile,
writeRuntimeProfile,
} from "./runtime-profile.js";
export {
applyRuntimeExecutionCoverage,
buildRuntimeExecutionCoverage,
} from "./runtime-reconciliation.js";
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
export {
buildSyntheticProbePlan,
defaultSyntheticHookContexts,
defaultSyntheticHookEvents,
defaultSyntheticRegistrationArguments,
renderSyntheticProbeMarkdown,
runCapturedSyntheticProbes,
syntheticRegistrationExecutionProfiles,
validateSyntheticProbePlan,
writeSyntheticProbePlan,
} from "./synthetic-probes.js";

View File

@ -5,32 +5,66 @@ import { inferPluginSeams, packageId } from "./config.js";
export const defaultInitConfigPath = "plugin-inspector.config.json";
export const defaultInitWorkflowPath = ".github/workflows/plugin-inspector.yml";
export const defaultInitPackageScripts = {
"plugin:check": "plugin-inspector inspect --no-openclaw",
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute",
};
export async function writePluginInspectorInit(options = {}) {
const pluginRoot = path.resolve(options.pluginRoot ?? options.cwd ?? process.cwd());
const configPath = path.resolve(pluginRoot, options.configPath ?? defaultInitConfigPath);
const workflowPath = options.ci === true ? path.resolve(pluginRoot, options.workflowPath ?? defaultInitWorkflowPath) : null;
const packageManager = options.packageManager ?? (await detectPackageManager(pluginRoot));
const dryRun = options.dryRun === true;
const written = [];
if (existsSync(configPath) && options.force !== true) {
if (!dryRun && existsSync(configPath) && options.force !== true) {
throw new Error(`${path.relative(pluginRoot, configPath)} already exists; pass --force to overwrite it`);
}
if (!dryRun && workflowPath && existsSync(workflowPath) && options.force !== true) {
throw new Error(`${path.relative(pluginRoot, workflowPath)} already exists; pass --force to overwrite it`);
}
const packageJsonPath = path.join(pluginRoot, "package.json");
const packageJson = options.scripts === true ? await readJsonIfExists(packageJsonPath) : null;
if (options.scripts === true) {
if (!packageJson) {
throw new Error("package.json is required to write plugin-inspector package scripts");
}
for (const name of Object.keys(defaultInitPackageScripts)) {
if (!dryRun && packageJson.scripts?.[name] && options.force !== true) {
throw new Error(`package.json scripts.${name} already exists; pass --force to overwrite it`);
}
}
}
const config = await buildPluginInspectorConfig({ pluginRoot });
await mkdir(path.dirname(configPath), { recursive: true });
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
if (!dryRun) {
await mkdir(path.dirname(configPath), { recursive: true });
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
}
written.push(configPath);
if (options.ci === true) {
const workflowPath = path.resolve(pluginRoot, options.workflowPath ?? defaultInitWorkflowPath);
if (existsSync(workflowPath) && options.force !== true) {
throw new Error(`${path.relative(pluginRoot, workflowPath)} already exists; pass --force to overwrite it`);
if (workflowPath) {
if (!dryRun) {
await mkdir(path.dirname(workflowPath), { recursive: true });
await writeFile(workflowPath, renderGithubActionsWorkflow({ packageManager }), "utf8");
}
await mkdir(path.dirname(workflowPath), { recursive: true });
await writeFile(workflowPath, renderGithubActionsWorkflow({ packageManager: options.packageManager }), "utf8");
written.push(workflowPath);
}
return { pluginRoot, configPath, written };
if (options.scripts === true) {
const existingScripts = packageJson.scripts ?? {};
packageJson.scripts = {
...existingScripts,
...defaultInitPackageScripts,
};
if (!dryRun) {
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
}
written.push(packageJsonPath);
}
return { pluginRoot, configPath, dryRun, packageManager, written };
}
export async function buildPluginInspectorConfig(options = {}) {
@ -79,8 +113,7 @@ jobs:
node-version: 24
cache: ${setup.cache}
${setup.corepack ? " - run: corepack enable\n" : ""} - run: ${setup.install}
- run: ${setup.exec} @openclaw/plugin-inspector check --no-openclaw
- run: PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 ${setup.exec} @openclaw/plugin-inspector check --no-openclaw --runtime --mock-sdk
- run: ${setup.exec} @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
- uses: actions/upload-artifact@v5
if: always()
with:
@ -89,19 +122,62 @@ ${setup.corepack ? " - run: corepack enable\n" : ""} - run: ${setup.in
`;
}
export async function detectPackageManager(pluginRoot) {
const root = path.resolve(pluginRoot ?? process.cwd());
const packageJson = await readJsonIfExists(path.join(root, "package.json"));
const packageManager = packageJson?.packageManager;
if (typeof packageManager === "string") {
const [name] = packageManager.split("@");
if (["npm", "pnpm", "yarn", "bun"].includes(name)) {
return name;
}
}
if (existsSync(path.join(root, "pnpm-lock.yaml"))) {
return "pnpm";
}
if (existsSync(path.join(root, "yarn.lock"))) {
return "yarn";
}
if (existsSync(path.join(root, "bun.lockb")) || existsSync(path.join(root, "bun.lock"))) {
return "bun";
}
return "npm";
}
function inferSourceRoot(packageJson) {
const entrypoints = [
packageJson?.openclaw?.entrypoint,
...(packageJson?.openclaw?.extensions ?? []),
...(packageJson?.openclaw?.runtimeExtensions ?? []),
...entrypointStrings(packageJson?.exports?.["."]),
...entrypointStrings(packageJson?.exports),
packageJson?.module,
packageJson?.main,
].filter((value) => typeof value === "string");
const entrypoint = entrypoints[0] ?? packageJson?.exports?.["."] ?? packageJson?.main ?? "src/index.js";
if (typeof entrypoint === "string" && entrypoint.startsWith("src/")) {
const entrypoint = entrypoints[0] ?? "src/index.js";
if (stripRelativePrefix(entrypoint).startsWith("src/")) {
return "src";
}
return ".";
}
function entrypointStrings(value) {
if (typeof value === "string") {
return [value];
}
if (!value || typeof value !== "object" || Array.isArray(value)) {
return [];
}
return ["import", "default", "require", "node", "module"]
.map((key) => value[key])
.filter((item) => typeof item === "string");
}
function stripRelativePrefix(filePath) {
return filePath.replace(/^\.\//, "");
}
async function readJsonIfExists(filePath) {
if (!existsSync(filePath)) {
return null;

View File

@ -5,12 +5,16 @@ import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { createCaptureApi } from "./capture-api.js";
import { captureApiOptionsForPlugin } from "./capture-config.js";
import { fixtureCheckoutPath, fixtureSourceRoot } from "./config.js";
import { buildCompatibilityFixtureReport } from "./fixture-summary.js";
import { readOpenClawTargetSurface } from "./openclaw-target.js";
import { buildCompatibilityReport, buildReport } from "./report.js";
const execFileAsync = promisify(execFile);
const registrationEquivalents = new Map([
["registerChannel", new Set(["createChatChannelPlugin", "defineBundledChannelEntry", "defineChannelPluginEntry", "registerChannel"])],
]);
export async function inspectFixtureSet(config, options = {}) {
const { inspections, failures } = await inspectConfiguredFixtures(config, options);
@ -32,6 +36,7 @@ export async function inspectCompatibilityFixtureSet(config, options = {}) {
inspections,
failures,
generatedAt: options.generatedAt,
executionResults: options.executionResults,
targetOpenClaw,
buildFixtureReport: ({ fixture, inspection }) =>
buildCompatibilityFixtureReport({
@ -58,7 +63,7 @@ async function inspectConfiguredFixtures(config, options = {}) {
["manifestContracts", inspection.manifestContracts],
]) {
const expected = fixture.expect?.[key] ?? [];
const missing = expected.filter((value) => !observed.includes(value));
const missing = expected.filter((value) => !satisfiesExpectedSeam(key, value, observed));
if (missing.length > 0) {
failures.push(`${fixture.id}: missing ${key}: ${missing.join(", ")}`);
}
@ -68,6 +73,17 @@ async function inspectConfiguredFixtures(config, options = {}) {
return { inspections, failures };
}
function satisfiesExpectedSeam(key, expected, observed) {
if (observed.includes(expected)) {
return true;
}
if (key !== "registrations") {
return false;
}
const equivalents = registrationEquivalents.get(expected);
return Boolean(equivalents && observed.some((value) => equivalents.has(value)));
}
export async function inspectPlugin(fixture, options = {}) {
const config = options.config ?? { rootDir: options.rootDir ?? process.cwd() };
const checkoutPath = fixtureCheckoutPath(config, fixture);
@ -132,6 +148,7 @@ export function inspectSourceText(text, filePath = "source.js") {
const hooks = collectDetailedMatches(searchableText, /\bapi\.on\(\s*["'`]([^"'`]+)["'`]/g, filePath, "name");
const registrations = [
...collectDetailedMatches(searchableText, /\bapi\.(register[A-Za-z0-9]+)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(defineBundledChannelEntry)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(defineChannelPluginEntry)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(createChatChannelPlugin)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(definePluginEntry)\s*\(/g, filePath, "name"),
@ -172,7 +189,12 @@ export async function captureEntrypoint(entrypoint, options = {}) {
};
}
const api = createCaptureApi(options.apiOptions);
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
pluginRoot: options.pluginRoot
? path.resolve(options.cwd ?? process.cwd(), options.pluginRoot)
: path.dirname(resolvedEntrypoint),
});
const api = createCaptureApi(apiOptions);
try {
await register(api);
} catch (error) {
@ -183,7 +205,7 @@ export async function captureEntrypoint(entrypoint, options = {}) {
entrypoint: resolvedEntrypoint,
captured: api.getCapturedContracts(),
};
if (options.apiOptions?.retainHandlers === true) {
if (apiOptions?.retainHandlers === true) {
result.retained = api.getRetainedContracts();
}
return result;
@ -212,10 +234,34 @@ export async function captureEntrypointWithMockSdk(entrypoint, options = {}) {
);
return JSON.parse(stdout);
} catch (error) {
const captured = parseCaptureResultFromStdout(error?.stdout);
if (captured) {
return captured;
}
throw classifyMockSdkCaptureError(error);
}
}
function parseCaptureResultFromStdout(stdout) {
if (!stdout) {
return null;
}
try {
const parsed = JSON.parse(stdout);
if (
parsed &&
typeof parsed === "object" &&
typeof parsed.status === "string" &&
Array.isArray(parsed.captured)
) {
return parsed;
}
} catch {
return null;
}
return null;
}
export function classifyMockSdkCaptureError(error) {
const rawMessage = [error?.stderr, error?.stdout, error?.message].filter(Boolean).join("\n");
const missingExport = rawMessage.match(/does not provide an export named ['"]([^'"]+)['"]/)?.[1];
@ -457,9 +503,38 @@ function lineForOffset(text, offset) {
}
function stripComments(text) {
return text
.replace(/\/\*[\s\S]*?\*\//g, (comment) => comment.replace(/[^\n]/g, " "))
.replace(/\/\/.*$/gm, (comment) => " ".repeat(comment.length));
let result = "";
for (let index = 0; index < text.length; index += 1) {
const char = text[index];
const next = text[index + 1];
if (char === "/" && next === "*") {
result += " ";
index += 2;
while (index < text.length && !(text[index] === "*" && text[index + 1] === "/")) {
result += blankCommentChar(text[index]);
index += 1;
}
if (index < text.length) {
result += " ";
index += 1;
}
} else if (char === "/" && next === "/") {
result += " ";
index += 2;
while (index < text.length && text[index] !== "\n" && text[index] !== "\r") {
result += " ";
index += 1;
}
index -= 1;
} else {
result += char;
}
}
return result;
}
function blankCommentChar(char) {
return char === "\n" || char === "\r" ? char : " ";
}
function sortDetails(details) {

View File

@ -23,17 +23,25 @@ export const knownIssueCodes = new Set([
"package-build-artifact-entrypoint",
"package-dependency-install-required",
"package-entrypoint-missing",
"package-install-metadata-incomplete",
"package-json-missing",
"package-manifest-version-drift",
"package-min-host-version-drift",
"package-npm-pack-entrypoint-missing",
"package-npm-pack-metadata-missing",
"package-npm-pack-unavailable",
"package-openclaw-entry-missing",
"package-openclaw-metadata-missing",
"package-openclaw-unsupported-metadata",
"package-plugin-api-compat-missing",
"package-typescript-source-entrypoint",
"provider-auth-env-vars",
"registration-capture-gap",
"runtime-tool-capture",
"reserved-sdk-import",
"security-manifest-schema-unavailable",
"sdk-export-missing",
"unrecognized-security-manifest",
]);
export const issueMetadataByCode = {
@ -85,6 +93,12 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "plugin imports reserved bundled-plugin SDK compatibility subpaths",
},
"security-manifest-schema-unavailable": {
severity: "P3",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "plugin security manifest references an unavailable schema",
},
"missing-compat-record": {
severity: "P1",
owner: "core",
@ -119,7 +133,7 @@ export const issueMetadataByCode = {
severity: "P2",
owner: "inspector",
decision: "inspector-follow-up",
title: "cold import requires isolated dependency installation",
title: "cold import requires dependency installation in an isolated workspace",
},
"package-entrypoint-missing": {
severity: "P1",
@ -127,6 +141,12 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "OpenClaw package entrypoint is missing",
},
"package-install-metadata-incomplete": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "OpenClaw package install metadata is incomplete",
},
"package-json-missing": {
severity: "P2",
owner: "plugin",
@ -139,6 +159,30 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "package and manifest versions drift",
},
"package-min-host-version-drift": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "OpenClaw package minimum host version drifts from build target",
},
"package-npm-pack-entrypoint-missing": {
severity: "P1",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "advertised npm artifact is missing OpenClaw entrypoints",
},
"package-npm-pack-metadata-missing": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "advertised npm artifact is missing OpenClaw metadata",
},
"package-npm-pack-unavailable": {
severity: "P1",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "advertised npm artifact cannot be packed",
},
"package-openclaw-entry-missing": {
severity: "P2",
owner: "plugin",
@ -151,6 +195,12 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "OpenClaw package metadata is missing",
},
"package-openclaw-unsupported-metadata": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "package declares unsupported OpenClaw metadata",
},
"package-plugin-api-compat-missing": {
severity: "P2",
owner: "plugin",
@ -170,10 +220,10 @@ export const issueMetadataByCode = {
title: "providerAuthEnvVars legacy manifest metadata must stay covered",
},
"registration-capture-gap": {
severity: "P1",
severity: "P2",
owner: "inspector",
decision: "inspector-follow-up",
title: "runtime registrations need capture before contract judgment",
title: "runtime registrations need capture evidence before final contract judgment",
},
"runtime-tool-capture": {
severity: "P2",
@ -193,6 +243,12 @@ export const issueMetadataByCode = {
decision: "core-compat-adapter",
title: "fixture calls a registrar missing from target OpenClaw",
},
"unrecognized-security-manifest": {
severity: "P3",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "plugin ships an unsupported security manifest",
},
};
export function buildIssues({ breakages = [], warnings = [], suggestions = [], targetOpenClaw, idPrefix = "CRABPOT" }) {
@ -212,7 +268,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
owner: finding.owner,
code: finding.code,
decision: finding.decision,
status: finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open",
status: finding.status ?? (finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open"),
issueClass: finding.issueClass,
live: finding.live,
deprecated: finding.deprecated,
@ -220,6 +276,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
title: issueTitle(finding),
evidence: finding.evidence ?? [],
compatRecord: finding.compatRecord ?? null,
runtimeCoverage: finding.runtimeCoverage ?? null,
}));
}
@ -285,7 +342,10 @@ export function summarizeIssueClasses(issues) {
}
function issueClassFor(code, options) {
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)) {
if (code === "sdk-export-missing" && options.compatRecord) {
return "compat-gap";
}
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)) {
return "live-issue";
}
if (code === "missing-compat-record") {
@ -314,10 +374,18 @@ function issueClassFor(code, options) {
"manifest-unknown-fields",
"package-json-missing",
"package-manifest-version-drift",
"package-min-host-version-drift",
"package-npm-pack-entrypoint-missing",
"package-npm-pack-metadata-missing",
"package-npm-pack-unavailable",
"package-openclaw-entry-missing",
"package-openclaw-metadata-missing",
"package-openclaw-unsupported-metadata",
"package-plugin-api-compat-missing",
"package-install-metadata-incomplete",
"reserved-sdk-import",
"security-manifest-schema-unavailable",
"unrecognized-security-manifest",
].includes(code)
) {
return "upstream-metadata";
@ -332,7 +400,7 @@ function severityForClass(code, defaultSeverity, options) {
if (
options.issueClass === "live-issue" &&
["none", "untracked"].includes(options.compatStatus) &&
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)
) {
return "P0";
}

View File

@ -1,10 +1,12 @@
#!/usr/bin/env node
import { mkdtemp, rm } from "node:fs/promises";
import { rmSync } from "node:fs";
import { mkdtemp } from "node:fs/promises";
import { register } from "node:module";
import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { createCaptureApi } from "./capture-api.js";
import { captureApiOptionsForPlugin } from "./capture-config.js";
import { createMockSdkPackage } from "./sdk-mock.js";
const options = JSON.parse(process.argv[2] ?? "{}");
@ -26,13 +28,16 @@ async function run(options) {
const pluginRoot = path.resolve(options.cwd ?? process.cwd(), options.pluginRoot ?? path.dirname(entrypoint));
const workspace = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-sdk-"));
try {
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
register(pathToFileURL(loaderPath));
return await captureLinkedEntrypoint(entrypoint, options);
} finally {
await rm(workspace, { force: true, recursive: true });
}
cleanupTempDirOnExit(workspace);
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
register(pathToFileURL(loaderPath));
return await captureLinkedEntrypoint(entrypoint, { ...options, pluginRoot });
}
function cleanupTempDirOnExit(dir) {
process.once("exit", () => {
rmSync(dir, { force: true, recursive: true });
});
}
async function captureLinkedEntrypoint(entrypoint, options) {
@ -61,7 +66,10 @@ async function captureLinkedEntrypoint(entrypoint, options) {
);
}
const api = createCaptureApi(options.apiOptions);
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
pluginRoot: options.pluginRoot,
});
const api = createCaptureApi(apiOptions);
try {
await register(api);
} catch (error) {
@ -76,7 +84,7 @@ async function captureLinkedEntrypoint(entrypoint, options) {
mockSdk: true,
captured: api.getCapturedContracts(),
};
if (options.apiOptions?.retainHandlers === true) {
if (apiOptions?.retainHandlers === true) {
result.retained = api.getRetainedContracts();
}
return withProcessOutput(result, outputCapture);

View File

@ -106,12 +106,89 @@ export function openClawTargetPathCandidates(manifest, configuredPath) {
export function parseCompatRecordEntries(source) {
const entries = [];
for (const match of source.matchAll(/\{[\s\S]*?\bcode:\s*["'`]([^"'`]+)["'`][\s\S]*?\bstatus:\s*["'`]([^"'`]+)["'`][\s\S]*?\}/g)) {
entries.push({ code: match[1], status: match[2] });
let cursor = 0;
while (cursor < source.length) {
const codeProperty = readStringProperty(source, "code", cursor);
if (!codeProperty) {
break;
}
const statusProperty = readStringProperty(source, "status", codeProperty.end);
if (statusProperty) {
entries.push({ code: codeProperty.value, status: statusProperty.value });
cursor = statusProperty.end;
} else {
cursor = codeProperty.end;
}
}
return dedupeBy(entries, (entry) => entry.code).sort((left, right) => left.code.localeCompare(right.code));
}
function readStringProperty(source, property, fromIndex) {
const propertyIndex = findProperty(source, property, fromIndex);
if (propertyIndex === -1) {
return null;
}
const colonIndex = source.indexOf(":", propertyIndex + property.length);
if (colonIndex === -1) {
return null;
}
let quoteIndex = colonIndex + 1;
while (quoteIndex < source.length && isWhitespace(source[quoteIndex])) {
quoteIndex += 1;
}
if (!isQuote(source[quoteIndex])) {
return null;
}
return readQuotedValue(source, quoteIndex);
}
function findProperty(source, property, fromIndex) {
let index = source.indexOf(property, fromIndex);
while (index !== -1) {
const previous = index === 0 ? "" : source[index - 1];
const next = source[index + property.length] ?? "";
if (!isIdentifierChar(previous) && !isIdentifierChar(next)) {
return index;
}
index = source.indexOf(property, index + property.length);
}
return -1;
}
function readQuotedValue(source, quoteIndex) {
const quote = source[quoteIndex];
let value = "";
for (let index = quoteIndex + 1; index < source.length; index += 1) {
const char = source[index];
if (char === "\\") {
value += source[index + 1] ?? "";
index += 1;
} else if (char === quote) {
return { value, end: index + 1 };
} else {
value += char;
}
}
return null;
}
function isQuote(char) {
return char === '"' || char === "'" || char === "`";
}
function isIdentifierChar(char) {
if (char === "_" || char === "$") {
return true;
}
const code = char.charCodeAt(0);
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
}
function isWhitespace(char) {
return char === " " || char === "\n" || char === "\r" || char === "\t";
}
export function parsePluginSdkExports(packageJson) {
return Object.keys(packageJson.exports ?? {})
.filter((specifier) => specifier === "./plugin-sdk" || specifier.startsWith("./plugin-sdk/"))

View File

@ -12,13 +12,11 @@ export function buildPlatformProbes(options = {}) {
const entrypoints = plan.fixtures.flatMap((fixture) =>
fixture.entrypoints.map((entrypoint) => summarizeEntrypoint(fixture.id, entrypoint)),
);
const portabilityFindings = plan.fixtures.flatMap((fixture) =>
fixture.entrypoints.flatMap((entrypoint) =>
entrypoint.steps
.map((step) => summarizeStep(fixture.id, entrypoint, step))
.filter((finding) => finding.riskCodes.length > 0),
),
const stepFindings = plan.fixtures.flatMap((fixture) =>
fixture.entrypoints.flatMap((entrypoint) => entrypoint.steps.map((step) => summarizeStep(fixture.id, entrypoint, step, options.stepCoverage))),
);
const portabilityFindings = stepFindings.flatMap((finding) => (finding.residual ? [finding.residual] : []));
const coveredPortabilityFindings = stepFindings.flatMap((finding) => (finding.covered ? [finding.covered] : []));
return {
generatedAt: plan.generatedAt,
@ -31,6 +29,7 @@ export function buildPlatformProbes(options = {}) {
jitiAlternativeCount: entrypoints.filter((entrypoint) => entrypoint.loaderAlternatives.includes("jiti")).length,
lazyImportProbeCount: entrypoints.filter((entrypoint) => entrypoint.capturePlanned && entrypoint.syntheticProbePlanned).length,
portabilityFindingCount: portabilityFindings.length,
coveredPortabilityFindingCount: coveredPortabilityFindings.length,
windowsRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("windows")).length,
macosRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("macos")).length,
linuxRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("linux")).length,
@ -38,6 +37,7 @@ export function buildPlatformProbes(options = {}) {
},
entrypoints,
portabilityFindings,
coveredPortabilityFindings,
recommendations: buildRecommendations(portabilityFindings, entrypoints),
};
}
@ -55,8 +55,11 @@ export function validatePlatformProbes(report, options = {}) {
errors.push("all TypeScript loader entrypoints must track a Jiti fallback candidate");
}
for (const entrypoint of report.entrypoints) {
if (entrypoint.loaderPrimary === "tsx" && (!entrypoint.captureUsesTsx || !entrypoint.syntheticUsesTsx)) {
errors.push(`${entrypoint.id}: tsx loader strategy is not reflected in capture and synthetic commands`);
if (
entrypoint.loaderPrimary === "tsx" &&
(!entrypoint.captureUsesTypeScriptLoader || !entrypoint.syntheticUsesTypeScriptLoader)
) {
errors.push(`${entrypoint.id}: TypeScript loader strategy is not reflected in capture and synthetic commands`);
}
}
return errors;
@ -94,9 +97,21 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
entrypoint.loaderAlternatives.join(", ") || "-",
entrypoint.captureUsesTsx ? "yes" : "no",
entrypoint.syntheticUsesTsx ? "yes" : "no",
entrypoint.captureUsesMockSdk ? "yes" : "no",
entrypoint.syntheticUsesMockSdk ? "yes" : "no",
entrypoint.entrypoint,
]),
["Fixture", "Status", "Primary", "Alternatives", "Capture TSX", "Synthetic TSX", "Entrypoint"],
[
"Fixture",
"Status",
"Primary",
"Alternatives",
"Capture TSX",
"Synthetic TSX",
"Capture Mock SDK",
"Synthetic Mock SDK",
"Entrypoint",
],
),
"",
"## Portability Findings",
@ -112,6 +127,19 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
["Fixture", "Step", "Platforms", "Risks", "Mitigation"],
),
"",
"## Covered Portability Findings",
"",
markdownTable(
(report.coveredPortabilityFindings ?? []).map((finding) => [
finding.fixture,
finding.kind,
finding.platforms.join(", ") || "-",
finding.riskCodes.join(", "),
finding.coverage,
]),
["Fixture", "Step", "Platforms", "Covered Risks", "Coverage"],
),
"",
"## Recommendations",
"",
markdownTable(
@ -124,6 +152,10 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
function summarizeEntrypoint(fixtureId, entrypoint) {
const captureStep = entrypoint.steps.find((step) => step.kind === "capture");
const syntheticStep = entrypoint.steps.find((step) => step.kind === "synthetic-probe");
const captureUsesTsx = Boolean(captureStep?.command.includes("--import tsx"));
const syntheticUsesTsx = Boolean(syntheticStep?.command.includes("--import tsx"));
const captureUsesMockSdk = Boolean(captureStep?.command.includes("--mock-sdk"));
const syntheticUsesMockSdk = Boolean(syntheticStep?.command.includes("--mock-sdk"));
return {
fixture: fixtureId,
id: entrypoint.id,
@ -135,21 +167,57 @@ function summarizeEntrypoint(fixtureId, entrypoint) {
loaderAlternatives: entrypoint.loaderStrategy.alternatives,
capturePlanned: Boolean(captureStep),
syntheticProbePlanned: Boolean(syntheticStep),
captureUsesTsx: Boolean(captureStep?.command.includes("--import tsx")),
syntheticUsesTsx: Boolean(syntheticStep?.command.includes("--import tsx")),
captureUsesTsx,
syntheticUsesTsx,
captureUsesMockSdk,
syntheticUsesMockSdk,
captureUsesTypeScriptLoader: captureUsesTsx || captureUsesMockSdk,
syntheticUsesTypeScriptLoader: syntheticUsesTsx || syntheticUsesMockSdk,
};
}
function summarizeStep(fixtureId, entrypoint, step) {
function summarizeStep(fixtureId, entrypoint, step, stepCoverage) {
const riskCodes = stepRiskCodes(step);
return {
const coverage = normalizeStepCoverage(stepCoverage?.({ fixture: fixtureId, entrypoint, step, riskCodes }), riskCodes);
const residualRiskCodes = riskCodes.filter((code) => !coverage.riskCodes.includes(code));
const common = {
fixture: fixtureId,
entrypoint: entrypoint.id,
kind: step.kind,
platforms: platformsForRiskCodes(riskCodes),
riskCodes,
command: step.command,
mitigation: mitigationForRiskCodes(riskCodes),
};
return {
residual:
residualRiskCodes.length > 0
? {
...common,
coveredRiskCodes: coverage.riskCodes,
platforms: platformsForRiskCodes(residualRiskCodes),
riskCodes: residualRiskCodes,
mitigation: mitigationForRiskCodes(residualRiskCodes),
}
: null,
covered:
coverage.riskCodes.length > 0
? {
...common,
coverage: coverage.reason,
platforms: platformsForRiskCodes(coverage.riskCodes),
riskCodes: coverage.riskCodes,
}
: null,
};
}
function normalizeStepCoverage(coverage, riskCodes) {
if (!coverage) {
return { reason: "", riskCodes: [] };
}
const requested = coverage === true ? riskCodes : coverage.coveredRiskCodes ?? coverage.handledRiskCodes ?? coverage.riskCodes ?? coverage;
const covered = Array.isArray(requested) ? requested : [];
return {
reason: typeof coverage.reason === "string" && coverage.reason ? coverage.reason : "covered by isolated workspace executor",
riskCodes: covered.filter((code) => riskCodes.includes(code)).sort(),
};
}
@ -215,7 +283,7 @@ function buildRecommendations(portabilityFindings, entrypoints) {
if (entrypoints.some((entrypoint) => entrypoint.loaderPrimary === "tsx")) {
recommendations.push({
area: "loader",
action: "keep tsx as the source-entrypoint smoke path, add a Jiti execution lane before treating TS plugin source compatibility as covered",
action: "keep mock-SDK TypeScript capture green, add a real host-loader/Jiti lane before treating TS plugin source compatibility as covered",
});
}
if (portabilityFindings.some((finding) => finding.riskCodes.includes("rsync-required"))) {

View File

@ -7,8 +7,12 @@ export async function runProfiledProcess(options) {
let firstRssKb = 0;
let peakRssKb = 0;
let peakCpuPercent = 0;
let statSampleCount = 0;
let rssSampleCount = 0;
let cpuSampleCount = 0;
const cpuSamples = [];
let pollInFlight = false;
const pendingStats = new Set();
const child = spawn(options.command, options.args ?? [], {
cwd: options.cwd,
@ -21,27 +25,43 @@ export async function runProfiledProcess(options) {
child.stderr?.on("data", (chunk) => stderr.push(chunk));
const recordStats = (stats) => {
if (stats.rssKb > 0 && firstRssKb === 0) {
if (stats.rssAvailable || stats.cpuAvailable) {
statSampleCount += 1;
}
if (stats.rssAvailable) {
rssSampleCount += 1;
}
if (stats.cpuAvailable) {
cpuSampleCount += 1;
}
if (stats.rssAvailable && stats.rssKb > 0 && firstRssKb === 0) {
firstRssKb = stats.rssKb;
}
peakRssKb = Math.max(peakRssKb, stats.rssKb);
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
if (stats.cpuPercent > 0) {
if (stats.rssAvailable) {
peakRssKb = Math.max(peakRssKb, stats.rssKb);
}
if (stats.cpuAvailable) {
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
cpuSamples.push(stats.cpuPercent);
}
};
const poll = setInterval(() => {
const sampleStats = () => {
if (pollInFlight) {
return;
}
pollInFlight = true;
readProcessStats(child.pid)
const pending = readProcessStats(child.pid)
.then(recordStats)
.finally(() => {
pollInFlight = false;
pendingStats.delete(pending);
});
}, options.pollMs ?? 100);
pendingStats.add(pending);
};
sampleStats();
const poll = setInterval(sampleStats, options.pollMs ?? 25);
const exitCode = await new Promise((resolve, reject) => {
child.on("error", (error) => {
@ -51,16 +71,10 @@ export async function runProfiledProcess(options) {
child.on("exit", (code) => resolve(code ?? 1));
});
clearInterval(poll);
await Promise.allSettled([...pendingStats]);
const finalStats = await readProcessStats(child.pid);
if (finalStats.rssKb > 0 && firstRssKb === 0) {
firstRssKb = finalStats.rssKb;
}
peakRssKb = Math.max(peakRssKb, finalStats.rssKb);
peakCpuPercent = Math.max(peakCpuPercent, finalStats.cpuPercent);
if (finalStats.cpuPercent > 0) {
cpuSamples.push(finalStats.cpuPercent);
}
recordStats(finalStats);
const wallMs = Math.round(performance.now() - start);
const averageCpuPercent =
@ -79,6 +93,9 @@ export async function runProfiledProcess(options) {
peakCpuPercent: Math.round(peakCpuPercent * 10) / 10,
cpuMsEstimate: Math.round((wallMs * cpuPercentForEstimate) / 100),
harnessHeapDeltaMb: Math.round((heapUsedMb() - heapStartMb) * 10) / 10,
statSampleCount,
rssSampleCount,
cpuSampleCount,
exitCode,
stdoutPreview: previewLines(stdout),
stderrPreview: previewLines(stderr),
@ -87,7 +104,7 @@ export async function runProfiledProcess(options) {
async function readProcessStats(pid) {
if (!pid || process.platform === "win32") {
return { rssKb: 0, cpuPercent: 0 };
return { rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 };
}
return new Promise((resolve) => {
const ps = spawn("ps", ["-o", "rss=", "-o", "%cpu=", "-p", String(pid)], {
@ -95,14 +112,18 @@ async function readProcessStats(pid) {
});
const chunks = [];
ps.stdout.on("data", (chunk) => chunks.push(chunk));
ps.on("error", () => resolve({ rssKb: 0, cpuPercent: 0 }));
ps.on("error", () => resolve({ rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 }));
ps.on("exit", () => {
const [rssRaw, cpuRaw] = Buffer.concat(chunks).toString("utf8").trim().split(/\s+/);
const rssKb = Number.parseInt(rssRaw, 10);
const cpuPercent = Number.parseFloat(cpuRaw);
const rssAvailable = Number.isFinite(rssKb);
const cpuAvailable = Number.isFinite(cpuPercent);
resolve({
rssKb: Number.isFinite(rssKb) ? rssKb : 0,
cpuPercent: Number.isFinite(cpuPercent) ? cpuPercent : 0,
rssAvailable,
rssKb: rssAvailable ? rssKb : 0,
cpuAvailable,
cpuPercent: cpuAvailable ? cpuPercent : 0,
});
});
});

View File

@ -0,0 +1,23 @@
#!/usr/bin/env node
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
const packageJsonPath = path.resolve(process.cwd(), "package.json");
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
let changed = false;
for (const [name, specifier] of Object.entries(packageJson.devDependencies ?? {})) {
if (typeof specifier === "string" && specifier.startsWith("workspace:")) {
delete packageJson.devDependencies[name];
changed = true;
}
}
if (packageJson.devDependencies && Object.keys(packageJson.devDependencies).length === 0) {
delete packageJson.devDependencies;
changed = true;
}
if (changed) {
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
}

42
src/report-sanitizer.js Normal file
View File

@ -0,0 +1,42 @@
import path from "node:path";
export function sanitizeReportArtifact(report, options = {}) {
const sensitivePaths = sensitiveOpenClawPaths(report);
if (sensitivePaths.length === 0) {
return report;
}
const placeholder = options.openclawPathPlaceholder ?? "<OPENCLAW_PATH>";
return sanitizeValue(report, sensitivePaths, placeholder);
}
function sensitiveOpenClawPaths(report) {
const targetOpenClaw = report?.targetOpenClaw;
return unique(
[targetOpenClaw?.configuredPath, ...(targetOpenClaw?.searchedPaths ?? [])]
.filter((value) => typeof value === "string" && isAbsolutePath(value))
.sort((left, right) => right.length - left.length),
);
}
function sanitizeValue(value, sensitivePaths, placeholder) {
if (typeof value === "string") {
return sensitivePaths.reduce((result, sensitivePath) => result.replaceAll(sensitivePath, placeholder), value);
}
if (Array.isArray(value)) {
return value.map((item) => sanitizeValue(item, sensitivePaths, placeholder));
}
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value).map(([key, entryValue]) => [key, sanitizeValue(entryValue, sensitivePaths, placeholder)]),
);
}
return value;
}
function isAbsolutePath(value) {
return path.isAbsolute(value) || /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith("\\\\");
}
function unique(values) {
return [...new Set(values)];
}

View File

@ -4,6 +4,8 @@ import { renderCompatibilityIssuesReport, renderCompatibilityMarkdownReport } fr
import { buildContractProbes } from "./contract-probes.js";
import { classifyCompatibilityFixture } from "./fixture-summary.js";
import { buildIssues, summarizeIssueClasses } from "./issues.js";
import { sanitizeReportArtifact } from "./report-sanitizer.js";
import { applyRuntimeExecutionCoverage } from "./runtime-reconciliation.js";
export function buildReport({ config, inspections, failures = [], generatedAt = "deterministic" }) {
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
@ -139,6 +141,10 @@ export async function buildCompatibilityReport(options = {}) {
decisions,
});
const runtimeCoverage = applyRuntimeExecutionCoverage({
findings: [...warnings, ...suggestions],
executionResults: options.executionResults,
});
const issues = buildIssues({
breakages,
warnings,
@ -148,6 +154,8 @@ export async function buildCompatibilityReport(options = {}) {
});
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures: fixtureReports });
const issueSummary = summarizeIssueClasses(issues);
const openIssues = issues.filter((issue) => issue.status !== "runtime-covered");
const openIssueSummary = summarizeIssueClasses(openIssues);
return {
generatedAt: options.generatedAt ?? "deterministic",
@ -162,8 +170,11 @@ export async function buildCompatibilityReport(options = {}) {
decisionCount: decisions.length,
logCount: logs.length,
issueCount: issues.length,
openIssueCount: openIssues.length,
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
openP0IssueCount: openIssues.filter((issue) => issue.severity === "P0").length,
openP1IssueCount: openIssues.filter((issue) => issue.severity === "P1").length,
liveIssueCount: issueSummary["live-issue"],
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
compatGapCount: issueSummary["compat-gap"],
@ -171,6 +182,10 @@ export async function buildCompatibilityReport(options = {}) {
inspectorGapCount: issueSummary["inspector-gap"],
upstreamIssueCount: issueSummary["upstream-metadata"],
fixtureRegressionCount: issueSummary["fixture-regression"],
openInspectorGapCount: openIssueSummary["inspector-gap"],
runtimeCoveredIssueCount: runtimeCoverage.coveredFindingCount,
runtimePartiallyCoveredIssueCount: runtimeCoverage.partiallyCoveredFindingCount,
runtimeCoverageArtifactCount: runtimeCoverage.coverage.artifactCount,
contractProbeCount: contractProbes.length,
},
fixtures: fixtureReports,
@ -210,6 +225,10 @@ export function classifyCompatRecordCoverage({ targetOpenClaw, findings, suggest
continue;
}
if (finding.code === "sdk-export-missing") {
continue;
}
suggestions.push({
fixture: finding.fixture,
code: "missing-compat-record",
@ -233,12 +252,13 @@ export async function writeReport(report, options = {}) {
const basename = options.basename ?? "plugin-inspector-report";
const jsonPath = path.join(outDir, `${basename}.json`);
const markdownPath = path.join(outDir, `${basename}.md`);
const artifactReport = sanitizeReportArtifact(report, options);
return writeJsonMarkdownArtifacts({
jsonPath,
markdownPath,
json: report,
markdown: renderMarkdownReport(report),
json: artifactReport,
markdown: renderMarkdownReport(artifactReport),
check: options.check,
});
}
@ -246,27 +266,86 @@ export async function writeReport(report, options = {}) {
export async function writeCompatibilityReport(report, options = {}) {
const outDir = path.resolve(options.cwd ?? process.cwd(), options.outDir ?? "reports");
const basename = options.basename ?? "plugin-inspector-report";
const jsonPath = path.join(outDir, `${basename}.json`);
const markdownPath = path.join(outDir, `${basename}.md`);
const issuesPath = path.join(outDir, options.issuesBasename ?? "plugin-inspector-issues.md");
const jsonPath = options.jsonPath ?? path.join(outDir, `${basename}.json`);
const markdownPath = options.markdownPath ?? path.join(outDir, `${basename}.md`);
const issuesPath = options.issuesPath ?? path.join(outDir, options.issuesBasename ?? "plugin-inspector-issues.md");
const artifactReport = sanitizeReportArtifact(report, options);
const markdownOptions = compatibilityRenderOptions(options, {
title: options.markdownTitle ?? options.title,
...options.markdownOptions,
});
const issuesOptions = compatibilityRenderOptions(options, {
title: options.issuesTitle ?? options.title,
...options.issuesOptions,
});
return writeArtifacts(
[
{ name: "jsonPath", path: jsonPath, json: report },
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(report) },
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(report) },
{ name: "jsonPath", path: jsonPath, json: artifactReport },
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(artifactReport, markdownOptions) },
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(artifactReport, issuesOptions) },
],
{ check: options.check },
);
}
export function renderTextSummary(report) {
return [
export { sanitizeReportArtifact };
function compatibilityRenderOptions(options, overrides) {
const renderOptions = {
formatEvidence: options.formatEvidence,
severityLabels: options.severityLabels,
...overrides,
};
return Object.fromEntries(Object.entries(renderOptions).filter(([, value]) => value !== undefined));
}
export function renderTextSummary(report, options = {}) {
const lines = [
`Status: ${report.status.toUpperCase()}`,
`Fixtures: ${report.summary.fixtureCount}`,
`Breakages: ${report.summary.breakageCount}`,
...(typeof report.summary.issueCount === "number" ? [`Issues: ${report.summary.issueCount}`] : []),
`Logs: ${report.summary.logCount}`,
].join("\n");
];
const artifacts = Object.entries(options.artifacts ?? {}).filter(([, filePath]) => Boolean(filePath));
if (artifacts.length > 0) {
lines.push("", "Reports:", ...artifacts.map(([name, filePath]) => `- ${artifactLabel(name)}: ${filePath}`));
}
const findings = topTextFindings(report, options.topFindings ?? 3);
if (findings.length > 0) {
lines.push("", "Top findings:", ...findings.map((finding) => `- ${finding}`));
}
return lines.join("\n");
}
function topTextFindings(report, limit) {
if (report.status === "pass" || limit <= 0) {
return [];
}
return [
...(report.breakages ?? []).map((finding) => formatTextFinding(finding, "breakage")),
...(report.issues ?? [])
.filter(
(issue) =>
issue.status !== "runtime-covered" &&
(issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1"),
)
.map((issue) => formatTextFinding(issue, issue.severity ?? "issue")),
...(report.warnings ?? []).map((finding) => formatTextFinding(finding, "warning")),
].slice(0, limit);
}
function formatTextFinding(finding, fallbackLevel) {
const level = finding.level ?? finding.severity ?? fallbackLevel;
const code = finding.code ? ` ${finding.code}` : "";
const message = finding.message ?? finding.title ?? "see report";
const evidence = Array.isArray(finding.evidence) && finding.evidence.length > 0 ? ` (${finding.evidence[0]})` : "";
return `${String(level).toUpperCase()} ${finding.fixture ?? "unknown"}${code}: ${message}${evidence}`;
}
function artifactLabel(name) {
return String(name).replace(/Path$/u, "");
}
export function renderMarkdownReport(report) {

View File

@ -46,8 +46,8 @@ export async function buildRuntimeProfile(options = {}) {
os: process.platform,
arch: process.arch,
node: process.version,
rssSampler: process.platform === "win32" ? "unavailable" : "ps",
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent",
rssSampler: process.platform === "win32" ? "unavailable" : "ps-immediate-25ms",
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent-immediate-25ms",
},
summary: summarizeProfile(commands),
groups: summarizeCommandGroups(commands),
@ -65,8 +65,8 @@ export function validateRuntimeProfile(profile) {
errors.push(`${command.id}: missing wall time`);
}
}
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => command.peakRssMb.max <= 0)) {
errors.push("all commands are missing peak RSS");
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => !hasRssSample(command))) {
errors.push("all commands are missing peak RSS samples");
}
return errors;
}
@ -98,10 +98,17 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
[
["Commands", profile.summary.commandCount],
["P50 wall time", `${profile.summary.p50WallMs} ms`],
["P95 wall time", `${profile.summary.p95WallMs} ms`],
["Max peak RSS", `${profile.summary.maxPeakRssMb} MB`],
["Max RSS delta", `${profile.summary.maxRssDeltaMb} MB`],
["Max CPU estimate", `${profile.summary.maxCpuMsEstimate} ms`],
["Command P95 wall time", `${profile.summary.p95WallMs} ms`],
["Wall time basis", profile.summary.wallTimeBasis ?? "command-median-p95"],
["Profile samples", profile.summary.sampleCount ?? sampleCount(profile.commands)],
["RSS samples", profile.summary.rssSampleCount ?? rssSampleCount(profile.commands)],
["CPU samples", profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands)],
["Max peak RSS", formatSampledMetric(profile.summary.maxPeakRssMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
["Max RSS delta", formatSampledMetric(profile.summary.maxRssDeltaMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
[
"Max CPU estimate",
formatSampledMetric(profile.summary.maxCpuMsEstimate, profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands), "ms"),
],
["Max harness heap delta", `${profile.summary.maxHarnessHeapDeltaMb} MB`],
],
["Metric", "Value"],
@ -129,13 +136,14 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
command.label,
`${command.wallMs.median} ms`,
`${command.wallMs.max} ms`,
`${command.peakRssMb.max} MB`,
`${command.rssDeltaMb.max} MB`,
`${command.cpuMsEstimate.max} ms`,
formatSampledMetric(command.peakRssMb.max, command.rssSampleCount),
formatSampledMetric(command.rssDeltaMb.max, command.rssSampleCount),
formatSampledMetric(command.cpuMsEstimate.max, command.cpuSampleCount, "ms"),
`${command.harnessHeapDeltaMb.max} MB`,
`${command.rssSampleCount ?? 0}/${command.cpuSampleCount ?? 0}`,
command.exitCodes.join(", "),
]),
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "Exit codes"],
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "RSS/CPU samples", "Exit codes"],
),
"",
"## Category Rollups",
@ -146,11 +154,12 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
group.commandCount,
`${group.p50WallMs} ms`,
`${group.p95WallMs} ms`,
`${group.maxPeakRssMb} MB`,
`${group.maxCpuMsEstimate} ms`,
formatSampledMetric(group.maxPeakRssMb, group.rssSampleCount),
formatSampledMetric(group.maxCpuMsEstimate, group.cpuSampleCount, "ms"),
`${group.rssSampleCount ?? 0}/${group.cpuSampleCount ?? 0}`,
group.commands.join(", "),
]),
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "Command IDs"],
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "RSS/CPU samples", "Command IDs"],
),
].join("\n");
}
@ -189,8 +198,15 @@ function summarizeProfile(commands) {
const maxRssDeltaMb = Math.max(0, ...commands.map((command) => command.rssDeltaMb.max));
const maxCpuMsEstimate = Math.max(0, ...commands.map((command) => command.cpuMsEstimate.max));
const maxHarnessHeapDeltaMb = Math.max(0, ...commands.map((command) => command.harnessHeapDeltaMb.max));
const totalSampleCount = sampleCount(commands);
const totalRssSampleCount = rssSampleCount(commands);
const totalCpuSampleCount = cpuSampleCount(commands);
return {
commandCount: commands.length,
sampleCount: totalSampleCount,
rssSampleCount: totalRssSampleCount,
cpuSampleCount: totalCpuSampleCount,
wallTimeBasis: "command-median-p95",
p50WallMs: percentile(wallTimes, 0.5),
p95WallMs: percentile(wallTimes, 0.95),
maxPeakRssMb,
@ -206,6 +222,12 @@ function summarizeCommand(command, samples) {
const rssDeltaMb = samples.map((sample) => sample.rssDeltaMb).sort((left, right) => left - right);
const peakCpuPercent = samples.map((sample) => sample.peakCpuPercent).sort((left, right) => left - right);
const cpuMsEstimate = samples.map((sample) => sample.cpuMsEstimate).sort((left, right) => left - right);
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
const rssSampleTotal = samples.reduce(
(sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)),
0,
);
const cpuSampleTotal = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
const harnessHeapDeltaMb = samples
.map((sample) => sample.harnessHeapDeltaMb)
.sort((left, right) => left - right);
@ -222,6 +244,9 @@ function summarizeCommand(command, samples) {
peakCpuPercent: summarizeNumbers(peakCpuPercent),
cpuMsEstimate: summarizeNumbers(cpuMsEstimate),
harnessHeapDeltaMb: summarizeNumbers(harnessHeapDeltaMb),
statSampleCount,
rssSampleCount: rssSampleTotal,
cpuSampleCount: cpuSampleTotal,
exitCodes: [...new Set(samples.map((sample) => sample.exitCode))].sort(),
};
}
@ -244,6 +269,8 @@ function summarizeCommandGroups(commands) {
const cpuMs = categoryCommands
.flatMap((command) => command.samples.map((sample) => sample.cpuMsEstimate))
.sort((left, right) => left - right);
const groupRssSampleCount = rssSampleCount(categoryCommands);
const groupCpuSampleCount = cpuSampleCount(categoryCommands);
return {
category,
commandCount: categoryCommands.length,
@ -251,11 +278,36 @@ function summarizeCommandGroups(commands) {
p95WallMs: percentile(wallTimes, 0.95),
maxPeakRssMb: peakRss.at(-1) ?? 0,
maxCpuMsEstimate: cpuMs.at(-1) ?? 0,
rssSampleCount: groupRssSampleCount,
cpuSampleCount: groupCpuSampleCount,
commands: categoryCommands.map((command) => command.id),
};
});
}
function hasRssSample(command) {
return (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)) > 0;
}
function sampleCount(commands) {
return commands.reduce((sum, command) => sum + (command.samples?.length ?? 0), 0);
}
function rssSampleCount(commands) {
return commands.reduce((sum, command) => sum + (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)), 0);
}
function cpuSampleCount(commands) {
return commands.reduce((sum, command) => sum + (command.cpuSampleCount ?? 0), 0);
}
function formatSampledMetric(value, count, unit = "MB") {
if ((count ?? 0) <= 0) {
return "n/a";
}
return `${value} ${unit}`;
}
function summarizeNumbers(values) {
return {
min: values[0],

View File

@ -0,0 +1,124 @@
export function applyRuntimeExecutionCoverage({ findings = [], executionResults } = {}) {
const coverage = buildRuntimeExecutionCoverage(executionResults);
let coveredFindingCount = 0;
let partiallyCoveredFindingCount = 0;
for (const finding of findings) {
const findingCoverage = runtimeCoverageForFinding(finding, coverage);
if (!findingCoverage) {
continue;
}
finding.runtimeCoverage = findingCoverage;
if (findingCoverage.status === "covered") {
finding.status = "runtime-covered";
coveredFindingCount += 1;
} else {
partiallyCoveredFindingCount += 1;
}
}
return {
coverage,
coveredFindingCount,
partiallyCoveredFindingCount,
};
}
export function buildRuntimeExecutionCoverage(executionResults) {
const fixtures = new Map();
for (const artifact of executionResults?.artifacts ?? []) {
if (artifact.kind !== "capture") {
continue;
}
const fixture = String(artifact.fixture ?? "unknown");
const fixtureCoverage = ensureFixtureCoverage(fixtures, fixture);
if (artifact.artifactPath) {
fixtureCoverage.artifacts.add(artifact.artifactPath);
}
for (const captured of normalizeCaptured(artifact.captured)) {
fixtureCoverage.captured.add(captured);
}
}
return {
fixtures,
artifactCount: [...fixtures.values()].reduce((sum, fixture) => sum + fixture.artifacts.size, 0),
};
}
function runtimeCoverageForFinding(finding, coverage) {
const fixtureCoverage = coverage.fixtures.get(finding.fixture);
if (!fixtureCoverage || fixtureCoverage.captured.size === 0) {
return null;
}
const expected = expectedRuntimeCaptureKeys(finding);
if (expected.length === 0) {
return null;
}
const captured = expected.filter((item) => fixtureCoverage.captured.has(item));
if (captured.length === 0) {
return null;
}
return {
status: captured.length === expected.length ? "covered" : "partial",
captured,
expected,
artifacts: [...fixtureCoverage.artifacts].sort(),
};
}
function expectedRuntimeCaptureKeys(finding) {
const names = evidenceNames(finding.evidence);
if (finding.code === "registration-capture-gap") {
return names.map((name) => `registration:${name}`);
}
if (finding.code === "runtime-tool-capture") {
return ["registration:registerTool"];
}
if (finding.code === "conversation-access-hook") {
return names.map((name) => `hook:${name}`);
}
return [];
}
function normalizeCaptured(captured) {
return (captured ?? [])
.map((item) => {
if (typeof item === "string") {
return item;
}
if (item && typeof item === "object" && item.kind && item.name) {
return `${item.kind}:${item.name}`;
}
return "";
})
.filter(Boolean);
}
function evidenceNames(evidence) {
return [
...new Set(
(evidence ?? [])
.map((item) => String(item).split(" @ ")[0]?.trim())
.filter(Boolean),
),
];
}
function ensureFixtureCoverage(fixtures, fixture) {
let fixtureCoverage = fixtures.get(fixture);
if (!fixtureCoverage) {
fixtureCoverage = {
artifacts: new Set(),
captured: new Set(),
};
fixtures.set(fixture, fixtureCoverage);
}
return fixtureCoverage;
}

View File

@ -11,8 +11,11 @@ export const mockSdkSubpathExports = {
"emptyPluginConfigSchema",
],
core: [
"buildChannelOutboundSessionRoute",
"buildChannelConfigSchema",
"buildPluginConfigSchema",
"createActionGate",
"createChannelPluginBase",
"createChatChannelPlugin",
"createDedupeCache",
"defineChannelPluginEntry",
@ -22,12 +25,24 @@ export const mockSdkSubpathExports = {
"emptyPluginConfigSchema",
"jsonResult",
"readNumberParam",
"readReactionParams",
"readStringArrayParam",
"readStringParam",
],
"channel-actions": [
"createActionGate",
"jsonResult",
"readNumberParam",
"readReactionParams",
"readStringArrayParam",
"readStringParam",
],
"channel-core": [
"buildChannelConfigSchema",
"buildChannelOutboundSessionRoute",
"buildThreadAwareOutboundSessionRoute",
"clearAccountEntryFields",
"createChannelPluginBase",
"createChatChannelPlugin",
"defineChannelPluginEntry",
"defineSetupPluginEntry",
@ -239,9 +254,20 @@ export async function createMockSdkPackage(rootDir, options = {}) {
)}\n`,
"utf8",
);
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(), "utf8");
const rootExportNames = new Set([
...mockSdkExportNames,
...(imports.bySpecifier.get("openclaw/plugin-sdk") ?? []),
]);
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(rootExportNames), "utf8");
for (const [subpath, exportNames] of Object.entries(mockSdkSubpathExports)) {
await writeFile(path.join(pluginSdkDir, `${subpath}.js`), mockSdkSubpathSource(exportNames), "utf8");
const specifier = `openclaw/plugin-sdk/${subpath}`;
await writeFile(
path.join(pluginSdkDir, `${subpath}.js`),
mockSdkSubpathSource(exportNames, imports.bySpecifier.get(specifier) ?? new Set(), {
zod: subpath === "zod",
}),
"utf8",
);
}
for (const specifier of imports.openclawSdkSpecifiers) {
if (specifier === "openclaw/plugin-sdk") {
@ -413,6 +439,9 @@ export async function resolve(specifier, context, nextResolve) {
const subpath = specifier.slice("openclaw/plugin-sdk/".length);
return moduleUrl(path.join(pluginSdkDir, \`\${subpath}.js\`));
}
if (externalMap.has(specifier)) {
return moduleUrl(externalMap.get(specifier));
}
try {
return await nextResolve(specifier, context);
} catch (error) {
@ -516,6 +545,15 @@ function genericExportStatement(name) {
if (["createChatChannelPlugin", "createPlugin", "defineChannelPluginEntry", "definePlugin", "definePluginEntry", "defineSetupPluginEntry"].includes(name)) {
return name === "definePluginEntry" ? "export { definePluginEntry };" : `export const ${name} = definePluginEntry;`;
}
if (name === "defineBundledChannelEntry") {
return "export { defineBundledChannelEntry };";
}
if (name === "defineBundledChannelSetupEntry") {
return "export { defineBundledChannelSetupEntry };";
}
if (name === "loadBundledEntryExportSync") {
return "export { loadBundledEntryExportSync };";
}
if (/^[A-Z].*Schema$/u.test(name)) {
return `export const ${name} = createSchema();`;
}
@ -523,7 +561,13 @@ function genericExportStatement(name) {
}
function genericMockRuntimeSource(options = {}) {
return `${options.includeSdkRuntime ? `function definePluginEntry(entry) {
return `${options.includeSdkRuntime ? `import { existsSync } from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const pendingBundledEntryLoads = new Set();
function definePluginEntry(entry) {
if (entry && typeof entry === "object" && typeof entry.register === "function") {
return entry;
}
@ -532,9 +576,86 @@ function genericMockRuntimeSource(options = {}) {
}
return typeof entry === "function" ? { register: entry } : entry;
}
function defineBundledChannelEntry(entry = {}) {
return {
...entry,
kind: "bundled-channel-entry",
async register(api) {
if (api?.registrationMode === "cli-metadata") {
return entry.registerCliMetadata?.(api);
}
if (api?.registrationMode !== "tool-discovery") {
api?.registerChannel?.({
id: entry.id,
name: entry.name,
description: entry.description,
plugin: { id: entry.id, name: entry.name },
});
}
entry.registerCliMetadata?.(api);
const result = entry.registerFull?.(api);
if (result && typeof result.then === "function") {
await result;
}
await drainBundledEntryLoads();
return result;
},
};
}
function defineBundledChannelSetupEntry(entry = {}) {
return {
...entry,
kind: "bundled-channel-setup-entry",
};
}
function loadBundledEntryExportSync(importMetaUrl, options = {}) {
return (...args) => {
const promise = import(resolveBundledEntryUrl(importMetaUrl, options.specifier)).then((module) => {
const loaded = module[options.exportName] ?? module.default;
return typeof loaded === "function" ? loaded(...args) : loaded;
});
pendingBundledEntryLoads.add(promise);
promise.finally(() => pendingBundledEntryLoads.delete(promise));
return promise;
};
}
async function drainBundledEntryLoads() {
while (pendingBundledEntryLoads.size > 0) {
await Promise.all([...pendingBundledEntryLoads]);
}
}
function resolveBundledEntryUrl(importMetaUrl, specifier) {
const basePath = fileURLToPath(importMetaUrl);
const target = specifier ? path.resolve(path.dirname(basePath), specifier) : basePath;
const resolved = resolveExistingSourcePath(target);
return pathToFileURL(resolved).href;
}
function resolveExistingSourcePath(target) {
if (existsSync(target)) {
return target;
}
const parsed = path.parse(target);
const withoutJsExtension = [".js", ".mjs", ".cjs"].includes(parsed.ext) ? path.join(parsed.dir, parsed.name) : null;
const candidates = [
...(withoutJsExtension ? [\`\${withoutJsExtension}.ts\`, \`\${withoutJsExtension}.mts\`, \`\${withoutJsExtension}.cts\`] : []),
\`\${target}.js\`,
\`\${target}.mjs\`,
\`\${target}.ts\`,
];
return candidates.find((candidate) => existsSync(candidate)) ?? target;
}
` : ""}
function createMockValue(name) {
function fn(...args) {
if (name === "resolvePreferredOpenClawTmpDir") {
return process.env.TMPDIR || "/tmp";
}
if (name.startsWith("normalize")) {
return typeof args[0] === "string" ? args[0] : "";
}
@ -713,25 +834,185 @@ function createTypeNamespace() {
`;
}
function mockSdkSource() {
function mockSdkSource(exportNames = mockSdkExportNames) {
const dynamicExportNames = [...exportNames].filter((name) => !mockSdkExportNames.includes(name));
return `function normalizeEntry(entry) {
return typeof entry === "function" ? { register: entry } : entry;
}
function normalizeRegistrationMode(api) {
return api?.registrationMode ?? "full";
}
function isPlainObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}
function parseWithSchema(schema, value) {
return schema && typeof schema.parse === "function" ? schema.parse(value) : value;
}
function createConfigSchema(schema = {}) {
if (schema && typeof schema.parse === "function") {
return schema;
}
const shape = isPlainObject(schema?.shape) ? schema.shape : isPlainObject(schema?.properties) ? schema.properties : schema;
return {
...schema,
parse(value = {}) {
if (!isPlainObject(shape)) {
return isPlainObject(value) ? value : {};
}
const source = isPlainObject(value) ? value : {};
const output = { ...source };
for (const [key, fieldSchema] of Object.entries(shape)) {
if (Object.prototype.hasOwnProperty.call(source, key)) {
output[key] = parseWithSchema(fieldSchema, source[key]);
}
}
return output;
},
};
}
export function definePluginEntry(entry) {
return normalizeEntry(entry);
}
export function defineChannelPluginEntry(entry) {
return normalizeEntry(entry);
if (!isPlainObject(entry) || !entry.plugin) {
return normalizeEntry(entry);
}
const resolved = {
id: entry.id,
name: entry.name,
description: entry.description,
configSchema: createConfigSchema(entry.configSchema),
channelPlugin: entry.plugin,
register(api) {
const mode = normalizeRegistrationMode(api);
if (mode === "cli-metadata") {
entry.registerCliMetadata?.(api);
return;
}
api.registerChannel?.({ plugin: entry.plugin });
entry.setRuntime?.(api.runtime);
if (mode === "discovery") {
entry.registerCliMetadata?.(api);
return;
}
if (mode !== "full") {
return;
}
entry.registerCliMetadata?.(api);
entry.registerFull?.(api);
},
};
if (entry.setRuntime) {
resolved.setChannelRuntime = entry.setRuntime;
}
return resolved;
}
export function defineSetupPluginEntry(entry) {
return normalizeEntry(entry);
return isPlainObject(entry) && entry.plugin ? entry : { plugin: entry };
}
export function createChatChannelPlugin(entry) {
return normalizeEntry(entry);
if (!isPlainObject(entry) || !entry.base) {
return normalizeEntry(entry);
}
return {
...entry.base,
conversationBindings: {
supportsCurrentConversationBinding: true,
...(entry.base.conversationBindings ?? {}),
},
...(entry.security ? { security: resolveChannelSecurity(entry.security) } : {}),
...(entry.pairing ? { pairing: resolveChannelPairing(entry.pairing) } : {}),
...(entry.threading ? { threading: resolveChannelThreading(entry.threading) } : {}),
...(entry.outbound ? { outbound: resolveChannelOutbound(entry.outbound) } : {}),
};
}
export function createChannelPluginBase(params = {}) {
return {
id: params.id ?? "fixture-channel",
meta: { id: params.id ?? "fixture-channel", ...(params.meta ?? {}) },
...(params.setupWizard ? { setupWizard: params.setupWizard } : {}),
...(params.capabilities ? { capabilities: params.capabilities } : {}),
...(params.commands ? { commands: params.commands } : {}),
...(params.doctor ? { doctor: params.doctor } : {}),
...(params.agentPrompt ? { agentPrompt: params.agentPrompt } : {}),
...(params.streaming ? { streaming: params.streaming } : {}),
...(params.reload ? { reload: params.reload } : {}),
...(params.gatewayMethods ? { gatewayMethods: params.gatewayMethods } : {}),
...(params.configSchema ? { configSchema: createConfigSchema(params.configSchema) } : {}),
...(params.config ? { config: params.config } : {}),
...(params.security ? { security: params.security } : {}),
...(params.groups ? { groups: params.groups } : {}),
setup: params.setup ?? (() => ({})),
};
}
function resolveChannelSecurity(security) {
if (!isPlainObject(security) || !security.dm) {
return security;
}
return {
resolveDmPolicy: ({ account } = {}) => ({
policy: security.dm.resolvePolicy?.(account ?? {}) ?? security.dm.defaultPolicy ?? "allow",
allowFrom: security.dm.resolveAllowFrom?.(account ?? {}) ?? [],
}),
...(security.collectWarnings ? { collectWarnings: security.collectWarnings } : {}),
...(security.collectAuditFindings ? { collectAuditFindings: security.collectAuditFindings } : {}),
};
}
function resolveChannelPairing(pairing) {
if (!isPlainObject(pairing) || !pairing.text) {
return pairing;
}
return {
idLabel: pairing.text.idLabel,
normalizeAllowEntry: pairing.text.normalizeAllowEntry,
notifyApproval: (ctx) => pairing.text.notify?.({ ...ctx, message: pairing.text.message }),
};
}
function resolveChannelThreading(threading) {
if (!isPlainObject(threading)) {
return threading;
}
if (threading.resolveReplyToMode) {
return threading;
}
return {
...threading,
resolveReplyToMode: () =>
threading.topLevelReplyToMode ??
threading.scopedAccountReplyToMode?.fallback ??
"thread",
};
}
function resolveChannelOutbound(outbound) {
if (!isPlainObject(outbound) || !outbound.attachedResults) {
return outbound;
}
const { base = {}, attachedResults } = outbound;
return {
...base,
...(attachedResults.sendText
? { sendText: async (ctx) => ({ channel: attachedResults.channel, ...(await attachedResults.sendText(ctx)) }) }
: {}),
...(attachedResults.sendMedia
? { sendMedia: async (ctx) => ({ channel: attachedResults.channel, ...(await attachedResults.sendMedia(ctx)) }) }
: {}),
...(attachedResults.sendPoll
? { sendPoll: async (ctx) => ({ channel: attachedResults.channel, ...(await attachedResults.sendPoll(ctx)) }) }
: {}),
};
}
export function definePlugin(entry) {
@ -759,13 +1040,13 @@ export function defineSingleProviderPluginEntry(options) {
}
export function buildPluginConfigSchema(schema = {}) {
return schema;
return createConfigSchema(schema);
}
export const emptyPluginConfigSchema = { type: "object", properties: {}, additionalProperties: false };
export const emptyPluginConfigSchema = createConfigSchema({ type: "object", properties: {}, additionalProperties: false });
export function buildChannelConfigSchema(schema = {}) {
return schema;
return createConfigSchema(schema);
}
export const emptyChannelConfigSchema = emptyPluginConfigSchema;
@ -774,13 +1055,63 @@ export function jsonResult(value) {
return { content: [{ type: "text", text: JSON.stringify(value) }] };
}
export function readNumberParam(value, fallback = 0) {
const parsed = Number(value);
return Number.isFinite(parsed) ? parsed : fallback;
export function readNumberParam(value, keyOrFallback = 0, options = {}) {
const raw = isPlainObject(value) ? value[keyOrFallback] : value;
const parsed = Number(raw);
if (Number.isFinite(parsed)) {
return options.integer ? Math.trunc(parsed) : parsed;
}
return isPlainObject(value) ? undefined : keyOrFallback;
}
export function readStringParam(value, fallback = "") {
return typeof value === "string" ? value : fallback;
export function readStringParam(value, keyOrFallback = "") {
if (isPlainObject(value)) {
const raw = value[keyOrFallback];
return typeof raw === "string" ? raw : undefined;
}
return typeof value === "string" ? value : keyOrFallback;
}
export function readStringArrayParam(value, key) {
const raw = isPlainObject(value) ? value[key] : value;
if (Array.isArray(raw)) {
return raw.map((entry) => String(entry));
}
return typeof raw === "string" && raw ? [raw] : [];
}
export function readReactionParams(value = {}) {
return {
messageId: value.messageId ?? value.id ?? "",
reaction: value.reaction ?? value.emoji ?? "",
};
}
export function createActionGate(actions = {}) {
return (key, defaultValue = true) => {
const value = actions?.[key];
return value === undefined ? defaultValue : value !== false;
};
}
export function buildChannelOutboundSessionRoute(params = {}) {
const peer = params.peer ?? { kind: params.chatType ?? "direct", id: params.to ?? "fixture-peer" };
const baseSessionKey = [
params.agentId ?? "agent",
params.channel ?? "channel",
params.accountId ?? "default",
peer.kind,
peer.id,
].filter(Boolean).join(":");
return {
sessionKey: baseSessionKey,
baseSessionKey,
peer,
chatType: params.chatType ?? peer.kind ?? "direct",
from: params.from ?? "fixture-source",
to: params.to ?? peer.id,
...(params.threadId !== undefined ? { threadId: params.threadId } : {}),
};
}
export function createDedupeCache() {
@ -845,16 +1176,36 @@ export function normalizeSecretInputString(value) {
return String(value ?? "").trim();
}
function createSimpleSchema(defaultValue) {
return {
parse(value) {
return value === undefined ? defaultValue : value;
},
default(value) {
return createSimpleSchema(value);
},
optional() {
return this;
},
nullable() {
return this;
},
nullish() {
return this;
},
};
}
export function buildSecretInputSchema() {
return { type: "string" };
return createSimpleSchema();
}
export function buildOptionalSecretInputSchema() {
return { anyOf: [buildSecretInputSchema(), { type: "undefined" }] };
return createSimpleSchema();
}
export function buildSecretInputArraySchema() {
return { type: "array", items: buildSecretInputSchema() };
return createSimpleSchema([]);
}
export function registerPluginHttpRoute(options = {}) {
@ -951,14 +1302,28 @@ export function createAuthRateLimiter() {
}
export function createProviderApiKeyAuthMethod(options = {}) {
return { type: "apiKey", ...options };
return {
id: options.id ?? "apiKey",
type: "apiKey",
...options,
async resolve(ctx = {}) {
return ctx.apiKey ?? ctx.key ?? ctx.token ?? null;
},
};
}
export function buildSingleProviderApiKeyCatalog(options = {}) {
const auth = options.auth ?? createProviderApiKeyAuthMethod(options.authOptions);
return {
auth,
order: "simple",
async run(ctx) {
return { provider: await options.buildProvider?.(ctx) };
const provider = (await options.buildProvider?.(ctx)) ?? options.provider ?? { id: options.id ?? "provider", auth };
return {
provider,
providers: [provider],
models: (await options.buildModels?.(ctx)) ?? options.models ?? [],
};
},
};
}
@ -1084,7 +1449,7 @@ export function createSubsystemLogger() {
}
export function buildThreadAwareOutboundSessionRoute(route = {}) {
return route;
return route.route ?? route;
}
export function clearAccountEntryFields(entry = {}) {
@ -1300,15 +1665,20 @@ export const OPENAI_RESPONSES_STREAM_HOOKS = buildProviderStreamFamilyHooks("ope
export const OPENROUTER_THINKING_STREAM_HOOKS = buildProviderStreamFamilyHooks("openrouter-thinking");
export const TOOL_STREAM_DEFAULT_ON_HOOKS = buildProviderStreamFamilyHooks("tool-stream-default");
export const pluginSdkMock = true;
${dynamicExportNames.map(genericExportStatement).join("\n")}
export default {
${mockSdkExportNames.map((name) => ` ${name},`).join("\n")}
${[...exportNames].map((name) => ` ${name},`).join("\n")}
};
`;
}
function mockSdkSubpathSource(exportNames) {
return `${exportNames.map((name) => `export { ${name} } from "./index.js";`).join("\n")}
function mockSdkSubpathSource(staticExportNames, importedExportNames, options = {}) {
const staticNames = new Set(staticExportNames);
const dynamicNames = [...importedExportNames].filter((name) => !staticNames.has(name));
return `${[...staticNames].map((name) => `export { ${name} } from "./index.js";`).join("\n")}
${dynamicNames.length > 0 ? genericMockRuntimeSource({ includeSdkRuntime: true, zod: options.zod }) : ""}
${dynamicNames.map(genericExportStatement).join("\n")}
export { default } from "./index.js";
`;
}

View File

@ -0,0 +1,19 @@
import { buildContractCapture } from "./contract-capture.js";
import { buildSyntheticProbePlan } from "./synthetic-probes.js";
export function buildSyntheticProbePlanFromReport(report, options = {}) {
const capture = options.capture ?? buildContractCapture({
report,
hookAssertions: options.hookAssertions,
hookContexts: options.hookContexts,
hookEvents: options.hookEvents,
registrationArguments: options.registrationArguments,
registrationAssertions: options.registrationAssertions,
});
return buildSyntheticProbePlan({
capture,
hookContexts: options.hookContexts,
hookEvents: options.hookEvents,
registrationArguments: options.registrationArguments,
});
}

View File

@ -1,5 +1,6 @@
#!/usr/bin/env node
import { mkdtemp, rm } from "node:fs/promises";
import { rmSync } from "node:fs";
import { mkdtemp } from "node:fs/promises";
import { register } from "node:module";
import os from "node:os";
import path from "node:path";
@ -59,17 +60,20 @@ async function captureForSyntheticProbes(entrypoint, options) {
const resolvedEntrypoint = path.resolve(process.cwd(), entrypoint);
const pluginRoot = path.resolve(process.cwd(), options.pluginRoot ?? path.dirname(resolvedEntrypoint));
const workspace = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-synthetic-mock-sdk-"));
try {
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
register(pathToFileURL(loaderPath));
return captureEntrypoint(entrypoint, {
...options,
mockSdk: false,
pluginRoot,
});
} finally {
await rm(workspace, { force: true, recursive: true });
}
cleanupTempDirOnExit(workspace);
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
register(pathToFileURL(loaderPath));
return captureEntrypoint(entrypoint, {
...options,
mockSdk: false,
pluginRoot,
});
}
function cleanupTempDirOnExit(dir) {
process.once("exit", () => {
rmSync(dir, { force: true, recursive: true });
});
}
function readFlag(commandArgs, name) {

View File

@ -1,11 +1,21 @@
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
export const syntheticRegistrationExecutionProfiles = {
createChatChannelPlugin: {
mode: "metadata-only",
callableProperties: [],
reason: "channel plugin factory metadata is captured before channel runtime execution",
},
defineChannelPluginEntry: {
mode: "metadata-only",
callableProperties: [],
reason: "entry wrapper metadata is captured before channel runtime execution",
},
defineBundledChannelEntry: {
mode: "metadata-only",
callableProperties: [],
reason: "bundled channel entry metadata is captured before channel runtime execution",
},
definePluginEntry: {
mode: "metadata-only",
callableProperties: [],
@ -13,25 +23,80 @@ export const syntheticRegistrationExecutionProfiles = {
},
registerChannel: {
mode: "channel-opt-in",
callableProperties: ["send", "receive"],
callableProperties: ["send", "sendMessage", "receive", "handleMessage"],
option: "includeChannelRuntime",
},
registerAgentHarness: {
mode: "metadata-only",
callableProperties: [],
reason: "agent harness factories are captured as registration metadata; agent runtime execution remains isolated opt-in",
},
registerAgentEventSubscription: {
mode: "metadata-only",
callableProperties: [],
reason: "agent event subscriptions are captured as registration metadata before agent event dispatch",
},
registerAgentToolResultMiddleware: {
mode: "metadata-only",
callableProperties: [],
reason: "agent tool-result middleware is captured as registration metadata before tool-result pipeline execution",
},
registerAutoEnableProbe: {
mode: "metadata-only",
callableProperties: [],
reason: "auto-enable probes are captured as registration metadata before runtime activation checks",
},
registerCli: {
mode: "direct",
callableProperties: ["handler", "run", "execute"],
},
registerCliBackend: {
mode: "metadata-only",
callableProperties: [],
reason: "CLI backend descriptors are captured as registration metadata before backend process execution",
},
registerCommand: {
mode: "direct",
callableProperties: ["handler", "run", "execute"],
},
registerCodexAppServerExtensionFactory: {
mode: "metadata-only",
callableProperties: [],
reason: "Codex app server extension factories are captured as registration metadata before host UI execution",
},
registerCompactionProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "compaction providers are captured as registration metadata before compaction runtime execution",
},
registerConfigMigration: {
mode: "metadata-only",
callableProperties: [],
reason: "config migrations are captured as registration metadata before mutating stored plugin config",
},
registerContextEngine: {
mode: "metadata-only",
callableProperties: [],
reason: "context engine factories are captured as registration metadata; engine startup remains isolated opt-in",
},
registerControlUiDescriptor: {
mode: "metadata-only",
callableProperties: [],
reason: "control UI descriptors are captured as registration metadata before UI composition",
},
registerDetachedTaskRuntime: {
mode: "metadata-only",
callableProperties: [],
reason: "detached task runtimes are captured as registration metadata before async task execution",
},
registerGatewayDiscoveryService: {
mode: "metadata-only",
callableProperties: [],
reason: "gateway discovery services are captured as registration metadata before network discovery execution",
},
registerGatewayMethod: {
mode: "direct",
callableProperties: ["handler", "run", "execute"],
callableProperties: ["handler", "run", "execute", "invoke"],
},
registerHttpRoute: {
mode: "direct",
@ -46,30 +111,155 @@ export const syntheticRegistrationExecutionProfiles = {
callableProperties: [],
reason: "legacy hook registrar is captured as metadata; hook handlers are probed through hook events",
},
registerImageGenerationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "image generation providers are captured as registration metadata before provider runtime execution",
},
registerMemoryPromptSection: {
mode: "metadata-only",
callableProperties: [],
reason: "memory prompt section renderers are captured as metadata before prompt-runtime execution",
},
registerMediaUnderstandingProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "media understanding providers are captured as registration metadata before provider runtime execution",
},
registerMemoryCapability: {
mode: "metadata-only",
callableProperties: [],
reason: "memory capabilities are captured as registration metadata before memory runtime execution",
},
registerMemoryCorpusSupplement: {
mode: "metadata-only",
callableProperties: [],
reason: "memory corpus supplements are captured as registration metadata before memory runtime execution",
},
registerMemoryEmbeddingProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "memory embedding providers are captured as registration metadata before provider runtime execution",
},
registerMemoryFlushPlan: {
mode: "metadata-only",
callableProperties: [],
reason: "memory flush plans are captured as registration metadata before memory runtime execution",
},
registerMemoryRuntime: {
mode: "metadata-only",
callableProperties: [],
reason: "memory runtime factories are captured as metadata; external memory startup remains isolated opt-in",
},
registerMemoryPromptSupplement: {
mode: "metadata-only",
callableProperties: [],
reason: "memory prompt supplements are captured as registration metadata before prompt-runtime execution",
},
registerMigrationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "migration providers are captured as registration metadata before migration runtime execution",
},
registerMusicGenerationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "music generation providers are captured as registration metadata before provider runtime execution",
},
registerNodeHostCommand: {
mode: "metadata-only",
callableProperties: [],
reason: "node host commands are captured as registration metadata before host process execution",
},
registerNodeInvokePolicy: {
mode: "metadata-only",
callableProperties: [],
reason: "node invoke policies are captured as registration metadata before host authorization checks",
},
registerProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "provider descriptors are captured as registration metadata before provider runtime execution",
},
registerRealtimeTranscriptionProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "realtime transcription providers are captured as registration metadata before provider runtime execution",
},
registerRealtimeVoiceProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "realtime voice providers are captured as registration metadata before provider runtime execution",
},
registerReload: {
mode: "metadata-only",
callableProperties: [],
reason: "reload handlers are captured as registration metadata before runtime reload execution",
},
registerRuntimeLifecycle: {
mode: "metadata-only",
callableProperties: [],
reason: "runtime lifecycle handlers are captured as registration metadata before lifecycle dispatch",
},
registerSecurityAuditCollector: {
mode: "metadata-only",
callableProperties: [],
reason: "security audit collectors are captured as registration metadata before filesystem or policy scans",
},
registerService: {
mode: "lifecycle-opt-in",
callableProperties: ["start", "stop"],
callableProperties: ["start", "stop", "dispose"],
option: "includeLifecycle",
},
registerSessionExtension: {
mode: "metadata-only",
callableProperties: [],
reason: "session extensions are captured as registration metadata before session runtime execution",
},
registerSessionSchedulerJob: {
mode: "metadata-only",
callableProperties: [],
reason: "session scheduler jobs are captured as registration metadata before scheduler execution",
},
registerSpeechProvider: {
mode: "provider-opt-in",
callableProperties: ["speak", "synthesize"],
callableProperties: ["speak", "synthesize", "tts"],
option: "includeProviderCapabilities",
},
registerTool: {
mode: "direct",
callableProperties: ["run", "handler", "execute"],
},
registerToolMetadata: {
mode: "metadata-only",
callableProperties: [],
reason: "tool metadata descriptors are captured as registration metadata before tool runtime execution",
},
registerTextTransforms: {
mode: "metadata-only",
callableProperties: [],
reason: "text transforms are captured as registration metadata before content mutation execution",
},
registerVideoGenerationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "video generation providers are captured as registration metadata before provider runtime execution",
},
registerTrustedToolPolicy: {
mode: "metadata-only",
callableProperties: [],
reason: "trusted tool policies are captured as registration metadata before trust-policy enforcement",
},
registerWebFetchProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "web fetch providers are captured as registration metadata before provider runtime execution",
},
registerWebSearchProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "web search providers are captured as registration metadata before provider runtime execution",
},
};
export const defaultSyntheticHookEvents = {
@ -186,6 +376,7 @@ export const defaultSyntheticHookContexts = {
};
export const defaultSyntheticRegistrationArguments = {
createChatChannelPlugin: [{ base: { id: "fixture-channel" }, outbound: { sendText: "function" } }],
defineChannelPluginEntry: [{ id: "fixture-channel", setup: "function", receive: "function" }],
definePluginEntry: [{ id: "fixture-plugin", register: "function" }],
registerChannel: [{ id: "fixture-channel", send: "function", receive: "function" }],
@ -214,6 +405,12 @@ export const defaultSyntheticRegistrationProbeInputs = {
handler: commandProbeArgs,
run: commandProbeArgs,
},
registerChannel: {
handleMessage: channelReceiveProbeArgs,
receive: channelReceiveProbeArgs,
send: channelSendProbeArgs,
sendMessage: channelSendProbeArgs,
},
registerGatewayMethod: {
execute: gatewayProbeArgs,
handler: gatewayProbeArgs,
@ -452,7 +649,8 @@ async function runRegistrationProbes(entry, retainedEntry, captureIndex, options
return [metadataOnlyResult(entry, captureIndex, profile.reason)];
}
const descriptor = retainedEntry.arguments?.[0] ?? retainedEntry.returnValue;
const descriptor =
retainedEntry.arguments?.find((value) => value && typeof value === "object") ?? retainedEntry.returnValue;
if (!descriptor || typeof descriptor !== "object") {
return [blockedResult(entry, captureIndex, "captured registration has no object descriptor")];
}
@ -522,6 +720,9 @@ function syntheticRegistrationEvent(registrar, property, options) {
id: beforeToolCall.toolCallId,
name: beforeToolCall.toolName,
},
respond(ok, result, error) {
return { ok, result, ...(error ? { error } : {}) };
},
};
}
@ -575,9 +776,11 @@ function commandProbeArgs(event) {
function gatewayProbeArgs(event) {
return [
{
...event,
params: event.params,
body: event.body,
headers: event.headers,
respond: event.respond,
},
{
source: event.source,
@ -586,6 +789,47 @@ function gatewayProbeArgs(event) {
];
}
function channelSendProbeArgs(event) {
return [
{
source: event.source,
channelId: "fixture-channel",
accountId: "fixture-account",
to: "fixture-recipient",
text: "fixture message",
replyToId: "fixture-reply",
threadId: "fixture-thread",
logger: console,
signal: new AbortController().signal,
},
];
}
function channelReceiveProbeArgs(event) {
return [
{
source: event.source,
channelId: "fixture-channel",
accountId: "fixture-account",
message: {
id: "message-fixture",
text: "fixture inbound message",
sender: { id: "sender-fixture", displayName: "Fixture Sender" },
},
route: {
sessionKey: "fixture-session",
baseSessionKey: "fixture-base-session",
peer: { kind: "direct", id: "sender-fixture" },
chatType: "direct",
from: "sender-fixture",
to: "fixture-channel",
},
logger: console,
signal: new AbortController().signal,
},
];
}
function interactiveProbeArgs(event) {
return [
{
@ -603,7 +847,10 @@ function lifecycleProbeArgs(event) {
return [
{
source: event.source,
config: {},
logger: console,
runtime: { env: {}, logger: console },
secrets: { get: async () => null, has: async () => false },
signal: new AbortController().signal,
},
];

View File

@ -71,6 +71,7 @@ export async function buildWorkspacePlan(options = {}) {
installStepCount: allSteps.filter((step) => step.kind === "install").length,
auditStepCount: allSteps.filter((step) => step.kind === "audit").length,
buildStepCount: allSteps.filter((step) => step.kind === "build").length,
pruneDevWorkspaceDependencyStepCount: allSteps.filter((step) => step.kind === "prune-dev-workspace-deps").length,
artifactStepCount: allSteps.filter((step) => step.kind === "prepare-artifacts").length,
captureStepCount: allSteps.filter((step) => step.kind === "capture").length,
syntheticProbeStepCount: allSteps.filter((step) => step.kind === "synthetic-probe").length,
@ -179,6 +180,7 @@ export function renderWorkspacePlanMarkdown(plan, options = {}) {
["Artifact dirs", plan.summary.artifactStepCount],
["Install steps", plan.summary.installStepCount],
["Audit steps", plan.summary.auditStepCount],
["Prune dev workspace dependency steps", plan.summary.pruneDevWorkspaceDependencyStepCount],
["Build steps", plan.summary.buildStepCount],
["Capture steps", plan.summary.captureStepCount],
["Synthetic probe steps", plan.summary.syntheticProbeStepCount],
@ -218,7 +220,7 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
const packageManager = detectPackageManager(settings.rootDir, packageDir, packageJson);
const lockfile = findNearestLockfile(settings.rootDir, packageDir);
const buildScript = packageJson.scripts?.build;
const requiredCapabilities = requiredCapabilitiesFor(entrypoint);
const requiredCapabilities = requiredCapabilitiesFor(entrypoint, packageSummary);
const loaderStrategy = loaderStrategyFor(entrypoint);
const blockers = [...entrypoint.blockers];
const workspacePath = posixJoin(settings.workspaceRoot, fixtureId);
@ -248,6 +250,14 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
}
if (requiredCapabilities.includes("dependency-install")) {
if (hasWorkspaceProtocolDevDependencies(packageJson)) {
steps.push({
kind: "prune-dev-workspace-deps",
command: `node ${helperScript(settings, workspacePath, settings.pruneWorkspaceDevDepsScript, "prune-workspace-dev-deps-cli.js")}`,
cwd: workspacePath,
reason: "remove workspace: devDependencies from the isolated runtime install; the mock SDK supplies OpenClaw host imports",
});
}
steps.push({
kind: "install",
command: installCommand(packageManager),
@ -319,6 +329,7 @@ function workspaceSettings(options) {
resultsRoot: repoRelative(options.resultsRoot ?? defaultWorkspacePlanOptions.resultsRoot),
rootDir: path.resolve(options.rootDir ?? process.cwd()),
syntheticProbeScript: options.syntheticProbeScript ?? defaultWorkspacePlanOptions.syntheticProbeScript,
pruneWorkspaceDevDepsScript: options.pruneWorkspaceDevDepsScript,
workspaceRoot: repoRelative(options.workspaceRoot ?? defaultWorkspacePlanOptions.workspaceRoot),
};
}
@ -342,7 +353,7 @@ function loaderStrategyFor(entrypoint) {
};
}
function requiredCapabilitiesFor(entrypoint) {
function requiredCapabilitiesFor(entrypoint, packageSummary = {}) {
const capabilities = new Set();
for (const blocker of entrypoint.blockers) {
if (blocker.code === "dependency-install-required") {
@ -361,7 +372,7 @@ function requiredCapabilitiesFor(entrypoint) {
capabilities.add("side-effect-sandbox");
}
}
if (entrypoint.blockers.some((blocker) => /\bopenclaw\b/.test(blocker.evidence ?? ""))) {
if (hasHostLinkedOpenClawDependency(packageSummary)) {
capabilities.add("target-openclaw-link");
}
capabilities.add("capture-shim");
@ -369,6 +380,20 @@ function requiredCapabilitiesFor(entrypoint) {
return [...capabilities].sort();
}
function hasHostLinkedOpenClawDependency(packageSummary) {
return [
...(packageSummary.dependencies ?? []),
...(packageSummary.peerDependencies ?? []),
...(packageSummary.optionalDependencies ?? []),
].includes("openclaw");
}
function hasWorkspaceProtocolDevDependencies(packageJson) {
return Object.values(packageJson.devDependencies ?? {}).some(
(value) => typeof value === "string" && value.startsWith("workspace:"),
);
}
function detectPackageManager(rootDir, packageDir, packageJson) {
const declared = typeof packageJson.packageManager === "string" ? packageJson.packageManager.split("@")[0] : null;
if (declared) {
@ -450,15 +475,13 @@ function runCommand(packageManager, script) {
}
function captureCommand(settings, fixtureId, entrypoint, workspacePath) {
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
const script = helperScript(settings, workspacePath, settings.captureScript, "capture-cli.js");
return `${settings.optInEnv} node${loader} ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
return `${settings.optInEnv} node ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
}
function syntheticProbeCommand(settings, fixtureId, entrypoint, workspacePath) {
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
const script = helperScript(settings, workspacePath, settings.syntheticProbeScript, "synthetic-probes-cli.js");
return `${settings.optInEnv} node${loader} ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
return `${settings.optInEnv} node ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
}
function helperScript(settings, workspacePath, configuredScript, helperFileName) {

View File

@ -3,15 +3,93 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "node:test";
import { packageId } from "../src/config.js";
import {
buildCiPolicyReport,
buildCiSummary,
buildContractCapture,
buildExecutionResultsReport,
buildFixtureSetColdImportReadiness,
buildImportLoopProfile,
buildFixtureSetPlatformProbes,
buildFixtureSetWorkspacePlan,
buildProfileDiff,
buildRefDiff,
buildRuntimeProfile,
buildSyntheticProbePlanFromReport,
capturePluginEntrypoint,
ci,
classifyIssueFinding,
contracts,
fixtureSuites,
inspectFixtureSet,
inspectCompatibilityFixtureSetConfig,
inspectFixtureSetConfig,
inspectPluginRoot,
inspectSourceText,
issueId,
knownIssueCodes,
loadInspectorConfig,
loadPluginConfig,
openClawTargetPathCandidates,
pluginRoot,
renderCiPolicyMarkdown,
renderCiSummaryMarkdown,
renderContractCaptureMarkdown,
renderExecutionResultsMarkdown,
renderImportLoopProfileMarkdown,
renderMarkdownReport,
renderFixtureSetColdImportReadinessMarkdown,
renderFixtureSetIssuesReport,
renderFixtureSetPlatformProbesMarkdown,
renderFixtureSetWorkspacePlanMarkdown,
renderProfileDiffMarkdown,
renderRefDiffMarkdown,
renderRuntimeProfileMarkdown,
renderSyntheticProbeMarkdown,
reports,
runtime,
runCapturedSyntheticProbes,
runFixtureSetColdImportReadiness,
runFixtureSetPlatformProbes,
runFixtureSetReport,
runFixtureSetWorkspacePlan,
runPluginCheck,
setupPluginInspector,
staticInspection,
synthetic,
validateCiPolicy,
validateCiPolicyReport,
validateContractCapture,
validateContractCoverage,
validateColdImportReadiness,
validateFixtureSetPlatformProbes,
validateFixtureSetWorkspacePlan,
validateImportLoopProfile,
validateProfileDiff,
validateRefDiff,
validateRuntimeProfile,
validateSyntheticProbePlan,
writeFixtureSetColdImportReadiness,
writeFixtureSetPlatformProbes,
writeReport,
writeFixtureSetReports,
writeFixtureSetWorkspacePlan,
writeContractCapture,
writeCiPolicyReport,
writeCiSummary,
writeExecutionResultsReport,
writeImportLoopProfile,
writeProfileDiff,
writeRefDiff,
writeRuntimeProfile,
writeSyntheticProbePlan,
} from "../src/index.js";
test("package ids collapse separators and trim hyphen edges", () => {
assert.equal(packageId("@openclaw/openclaw---Weather_Plugin!!!"), "weather-plugin");
});
test("public API runs the plugin-root check and writes reports", async () => {
const pluginRoot = await createPluginRoot();
@ -30,6 +108,53 @@ test("public API runs the plugin-root check and writes reports", async () => {
assert.equal(paths.jsonPath, path.join(pluginRoot, "reports", "plugin-inspector-report.json"));
});
test("public API exposes grouped facades for common workflows", () => {
assert.equal(pluginRoot.loadConfig, loadPluginConfig);
assert.equal(pluginRoot.inspect, inspectPluginRoot);
assert.equal(pluginRoot.runCheck, runPluginCheck);
assert.equal(fixtureSuites.inspect, inspectCompatibilityFixtureSetConfig);
assert.equal(fixtureSuites.runReport, runFixtureSetReport);
assert.equal(staticInspection.inspectSourceText, inspectSourceText);
assert.equal(reports.renderMarkdown, renderMarkdownReport);
assert.equal(typeof reports.sanitizeArtifact, "function");
assert.equal(typeof reports.readOpenClawTargetSurface, "function");
assert.equal(contracts.buildCapture, buildContractCapture);
assert.equal(contracts.validateCoverage, validateContractCoverage);
assert.equal(ci.buildSummary, buildCiSummary);
assert.equal(ci.buildPolicyReport, buildCiPolicyReport);
assert.equal(runtime.buildProfile, buildRuntimeProfile);
assert.equal(runtime.buildRefDiff, buildRefDiff);
assert.equal(synthetic.buildPlanFromReport, buildSyntheticProbePlanFromReport);
assert.equal(synthetic.runCaptured, runCapturedSyntheticProbes);
});
test("public API reads plugin config from package.json", async () => {
const pluginRoot = await createPluginRoot({
packageConfig: {
plugin: {
id: "weather-pkg",
priority: "medium",
seams: ["channel"],
sourceRoot: "src",
expect: {
registrations: ["definePluginEntry"],
},
},
capture: {
mockSdk: true,
},
},
});
const config = await loadPluginConfig({ pluginRoot });
assert.equal(config.configPath, "package.json#pluginInspector");
assert.equal(config.fixtures[0].id, "weather-pkg");
assert.equal(config.fixtures[0].priority, "medium");
assert.deepEqual(config.fixtures[0].seams, ["channel"]);
assert.equal(config.capture.mockSdk, true);
});
test("public API keeps crabpot-style fixture configs behind an explicit helper", async () => {
const report = await inspectFixtureSetConfig({ configPath: "test/fixtures/inspector.config.json" });
@ -37,6 +162,154 @@ test("public API keeps crabpot-style fixture configs behind an explicit helper",
assert.equal(report.summary.fixtureCount, 1);
});
test("public API exposes static source and fixture-set inspection primitives", async () => {
const sourceInspection = inspectSourceText(
[
'import { definePluginEntry } from "openclaw/plugin-sdk";',
"export default definePluginEntry((api) => {",
' api.on("before_tool_call", () => undefined);',
' api.registerTool({ name: "weather" });',
"});",
].join("\n"),
"plugins/weather/src/index.js",
);
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
const report = await inspectFixtureSet(config);
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-root-report-"));
const paths = await writeReport(report, { outDir, basename: "static-inspection" });
assert.deepEqual(sourceInspection.hooks.map((hook) => hook.name), ["before_tool_call"]);
assert.deepEqual(sourceInspection.registrations.map((registration) => registration.name), [
"registerTool",
"definePluginEntry",
]);
assert.equal(report.status, "pass");
assert.match(renderMarkdownReport(report), /# OpenClaw Plugin Inspector Report/);
assert.equal(paths.jsonPath, path.join(outDir, "static-inspection.json"));
});
test("public API writes compatibility fixture-set reports with custom render options", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-fixture-api-"));
const report = await inspectCompatibilityFixtureSetConfig({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
});
const paths = await writeFixtureSetReports(report, {
jsonPath: path.join(outDir, "suite.json"),
markdownPath: path.join(outDir, "suite.md"),
issuesPath: path.join(outDir, "issues.md"),
markdownTitle: "Fixture Suite Compatibility",
issuesTitle: "Fixture Suite Issues",
formatEvidence: (evidence) => `linked:${evidence}`,
});
const result = await runFixtureSetReport({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
outDir,
basename: "run",
});
assert.equal(report.status, "pass");
assert.deepEqual(paths, {
jsonPath: path.join(outDir, "suite.json"),
markdownPath: path.join(outDir, "suite.md"),
issuesPath: path.join(outDir, "issues.md"),
});
assert.match(await readFile(paths.markdownPath, "utf8"), /# Fixture Suite Compatibility/);
assert.match(await readFile(paths.issuesPath, "utf8"), /# Fixture Suite Issues/);
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.fixtureCount, 1);
assert.equal(result.report.summary.fixtureCount, 1);
assert.equal(result.paths.jsonPath, path.join(outDir, "run.json"));
});
test("public API builds fixture-set cold import readiness from config", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-cold-import-api-"));
const readiness = await buildFixtureSetColdImportReadiness({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
});
const paths = await writeFixtureSetColdImportReadiness(readiness, {
jsonPath: path.join(outDir, "cold-import.json"),
markdownPath: path.join(outDir, "cold-import.md"),
title: "Fixture Cold Import",
});
const result = await runFixtureSetColdImportReadiness({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
write: false,
});
assert.equal(readiness.summary.fixtureCount, 1);
assert.deepEqual(validateColdImportReadiness(readiness), []);
assert.match(renderFixtureSetColdImportReadinessMarkdown(readiness), /## Entrypoints/);
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.fixtureCount, 1);
assert.equal(result.paths, null);
assert.equal(result.readiness.summary.fixtureCount, 1);
});
test("public API builds fixture-set workspace and platform plans from config", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-plan-api-"));
const plan = await buildFixtureSetWorkspacePlan({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
});
const planPaths = await writeFixtureSetWorkspacePlan(plan, {
jsonPath: path.join(outDir, "workspace.json"),
markdownPath: path.join(outDir, "workspace.md"),
});
const platform = await buildFixtureSetPlatformProbes({ plan });
const coveredPlan = structuredClone(plan);
coveredPlan.fixtures[0].entrypoints = [
{
id: "cold-import.extension:sample-plugin:index",
status: "dependency-install-required",
entrypoint: "plugins/sample-plugin/index.js",
packageManager: "npm",
loaderStrategy: {
source: "javascript",
primary: "node",
alternatives: [],
reason: "test",
},
steps: [
{
kind: "prepare",
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
},
],
},
];
const coveredPlatform = await buildFixtureSetPlatformProbes({
plan: coveredPlan,
stepCoverage({ riskCodes }) {
return { riskCodes };
},
});
const platformPaths = await writeFixtureSetPlatformProbes(platform, {
jsonPath: path.join(outDir, "platform.json"),
markdownPath: path.join(outDir, "platform.md"),
});
const planResult = await runFixtureSetWorkspacePlan({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
write: false,
});
const platformResult = await runFixtureSetPlatformProbes({ plan, write: false });
assert.equal(plan.summary.fixtureCount, 1);
assert.deepEqual(validateFixtureSetWorkspacePlan(plan), []);
assert.match(renderFixtureSetWorkspacePlanMarkdown(plan), /## Entrypoint Workspaces/);
assert.equal(JSON.parse(await readFile(planPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
assert.equal(platform.summary.fixtureCount, 1);
assert.equal(coveredPlatform.summary.portabilityFindingCount, 0);
assert.ok(coveredPlatform.summary.coveredPortabilityFindingCount > 0);
assert.deepEqual(validateFixtureSetPlatformProbes(platform), []);
assert.match(renderFixtureSetPlatformProbesMarkdown(platform), /## Loader Probes/);
assert.equal(JSON.parse(await readFile(platformPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
assert.equal(planResult.paths, null);
assert.equal(platformResult.paths, null);
});
test("public API exposes capture through an explicit entrypoint helper", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-api-capture-"));
const entrypoint = path.join(dir, "fixture.mjs");
@ -65,14 +338,265 @@ test("public API exposes capture through an explicit entrypoint helper", async (
test("public API can initialize plugin inspector files", async () => {
const pluginRoot = await createPluginRoot();
const result = await setupPluginInspector({ pluginRoot, ci: true, packageManager: "npm" });
const result = await setupPluginInspector({ pluginRoot, ci: true, scripts: true, packageManager: "npm" });
const config = JSON.parse(await readFile(path.join(pluginRoot, "plugin-inspector.config.json"), "utf8"));
const packageJson = JSON.parse(await readFile(path.join(pluginRoot, "package.json"), "utf8"));
const workflow = await readFile(path.join(pluginRoot, ".github", "workflows", "plugin-inspector.yml"), "utf8");
assert.equal(result.written.length, 2);
assert.equal(result.written.length, 3);
assert.equal(result.packageManager, "npm");
assert.equal(config.plugin.id, "weather");
assert.equal(config.capture.mockSdk, true);
assert.match(workflow, /npx @openclaw\/plugin-inspector check --no-openclaw/);
assert.equal(packageJson.scripts["plugin:check"], "plugin-inspector inspect --no-openclaw");
assert.equal(packageJson.scripts["plugin:ci"], "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute");
assert.match(workflow, /npx @openclaw\/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute/);
});
test("public API initializes source root from package export maps", async () => {
const pluginRoot = await createPluginRoot({
packageJson: {
openclaw: null,
exports: {
".": {
import: "./src/plugin-entry.js",
},
},
},
});
await setupPluginInspector({ pluginRoot, force: true });
const config = JSON.parse(await readFile(path.join(pluginRoot, "plugin-inspector.config.json"), "utf8"));
assert.equal(config.plugin.sourceRoot, "src");
});
test("fixture-set issue renderer is available without advanced internals", () => {
const markdown = renderFixtureSetIssuesReport(
{
generatedAt: "test",
status: "pass",
summary: {
issueCount: 0,
p0IssueCount: 0,
p1IssueCount: 0,
liveIssueCount: 0,
liveP0IssueCount: 0,
compatGapCount: 0,
deprecationWarningCount: 0,
inspectorGapCount: 0,
upstreamIssueCount: 0,
contractProbeCount: 0,
},
issues: [],
contractProbes: [],
},
{ title: "Fixture Issues" },
);
assert.match(markdown, /# Fixture Issues/);
assert.match(markdown, /## Triage Summary/);
});
test("public API exposes report issue metadata helpers", () => {
assert.ok(knownIssueCodes.has("registration-capture-gap"));
assert.match(issueId({ fixture: "weather", code: "registration-capture-gap" }), /^CRABPOT-[A-F0-9]{8}$/);
assert.equal(classifyIssueFinding({ code: "registration-capture-gap" }).issueClass, "inspector-gap");
assert.ok(openClawTargetPathCandidates().some((candidate) => candidate.includes("openclaw")));
});
test("public API exposes contract capture and coverage helpers", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-contract-api-"));
const report = await inspectCompatibilityFixtureSetConfig({
configPath: "test/fixtures/inspector.config.json",
openclawPath: false,
});
const capture = buildContractCapture({ report });
const paths = await writeContractCapture(capture, {
jsonPath: path.join(outDir, "capture.json"),
markdownPath: path.join(outDir, "capture.md"),
});
assert.equal(capture.summary.fixtureCount, 1);
assert.deepEqual(validateContractCapture(capture), []);
assert.deepEqual(validateContractCoverage(report), []);
assert.match(renderContractCaptureMarkdown(capture), /## Registration Capture/);
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.fixtureCount, 1);
});
test("public API exposes execution and CI rollup helpers", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ci-api-"));
const resultDir = path.join(rootDir, ".plugin-inspector", "results", "weather");
const outDir = path.join(rootDir, "reports");
await mkdir(resultDir, { recursive: true });
await writeFile(
path.join(resultDir, "entry.synthetic.json"),
JSON.stringify({
summary: { probeCount: 1, passCount: 1, failCount: 0, blockedCount: 0 },
results: [{ kind: "hook", seam: "before_tool_call", label: "before_tool_call", status: "pass" }],
}),
"utf8",
);
const policy = {
version: 1,
allowedBlocked: [],
expectedWarnings: [],
thresholds: {
wallP95RegressionPercent: 50,
peakRssRegressionMb: 50,
bootRegressionMs: 500,
strictMinimumSamples: 3,
},
fixtureSets: { smoke: ["weather"] },
};
const compatibilityReport = {
summary: { breakageCount: 0, p1IssueCount: 0 },
breakages: [],
issues: [],
};
const execution = await buildExecutionResultsReport({ rootDir });
const policyReport = buildCiPolicyReport({ policy, compatibilityReport, executionResults: execution });
const summary = await buildCiSummary({
reports: { compatibility: compatibilityReport, execution, ciPolicy: policyReport },
});
const executionPaths = await writeExecutionResultsReport(execution, {
jsonPath: path.join(outDir, "execution.json"),
markdownPath: path.join(outDir, "execution.md"),
});
const policyPaths = await writeCiPolicyReport(policyReport, {
jsonPath: path.join(outDir, "policy.json"),
markdownPath: path.join(outDir, "policy.md"),
});
const summaryPaths = await writeCiSummary(summary, {
jsonPath: path.join(outDir, "summary.json"),
markdownPath: path.join(outDir, "summary.md"),
});
assert.equal(execution.summary.passCount, 1);
assert.doesNotThrow(() => validateCiPolicy(policy));
assert.deepEqual(validateCiPolicyReport(policyReport), []);
assert.equal(summary.status, "pass");
assert.match(renderExecutionResultsMarkdown(execution), /Execution Results/);
assert.match(renderCiPolicyMarkdown(policyReport), /CI Policy/);
assert.match(renderCiSummaryMarkdown(summary), /CI Summary/);
assert.equal(JSON.parse(await readFile(executionPaths.jsonPath, "utf8")).summary.passCount, 1);
assert.equal(JSON.parse(await readFile(policyPaths.jsonPath, "utf8")).status, "pass");
assert.equal(JSON.parse(await readFile(summaryPaths.jsonPath, "utf8")).status, "pass");
});
test("public API exposes runtime profile and diff helpers", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-profile-api-"));
const runtimeProfile = await buildRuntimeProfile({ runs: 1 });
const profileDiff = await buildProfileDiff({
baseline: profileFixture({ p95WallMs: 100, maxPeakRssMb: 80, nodeBootMs: 25 }),
current: profileFixture({ p95WallMs: 120, maxPeakRssMb: 90, nodeBootMs: 30 }),
policy: {
thresholds: {
wallP95RegressionPercent: 50,
peakRssRegressionMb: 50,
bootRegressionMs: 500,
strictMinimumSamples: 3,
},
},
});
const refDiff = await buildRefDiff({
baseReport: diffReport({ hookNames: ["before_tool_call"], issues: [] }),
headReport: diffReport({ hookNames: ["before_tool_call"], issues: [] }),
});
const importLoopProfile = {
generatedAt: "deterministic",
mode: "subprocess-cold-import-loop",
entrypoint: "fixtures/plugin.mjs",
summary: {
runs: 1,
p50WallMs: 5,
p95WallMs: 5,
maxPeakRssMb: 10,
maxCpuMsEstimate: 2,
capturedCount: 1,
failCount: 0,
},
samples: [
{
index: 0,
status: "captured",
capturedCount: 1,
wallMs: 5,
peakRssMb: 10,
cpuMsEstimate: 2,
exitCode: 0,
},
],
};
assert.equal(typeof buildImportLoopProfile, "function");
const portableRuntimeProfile = {
...runtimeProfile,
platform: { ...runtimeProfile.platform, rssSampler: "unavailable" },
};
assert.deepEqual(validateRuntimeProfile(portableRuntimeProfile), []);
assert.deepEqual(validateProfileDiff(profileDiff), []);
assert.deepEqual(validateRefDiff(refDiff), []);
assert.deepEqual(validateImportLoopProfile(importLoopProfile), []);
assert.match(renderRuntimeProfileMarkdown(portableRuntimeProfile), /Runtime Profile/);
assert.match(renderProfileDiffMarkdown(profileDiff), /Runtime Profile Diff/);
assert.match(renderRefDiffMarkdown(refDiff), /Ref Diff/);
assert.match(renderImportLoopProfileMarkdown(importLoopProfile), /Import Loop Profile/);
const runtimePaths = await writeRuntimeProfile(portableRuntimeProfile, {
jsonPath: path.join(outDir, "runtime.json"),
markdownPath: path.join(outDir, "runtime.md"),
});
const profileDiffPaths = await writeProfileDiff(profileDiff, {
jsonPath: path.join(outDir, "profile-diff.json"),
markdownPath: path.join(outDir, "profile-diff.md"),
});
const refDiffPaths = await writeRefDiff(refDiff, {
jsonPath: path.join(outDir, "ref-diff.json"),
markdownPath: path.join(outDir, "ref-diff.md"),
});
const importLoopPaths = await writeImportLoopProfile(importLoopProfile, {
jsonPath: path.join(outDir, "import-loop.json"),
markdownPath: path.join(outDir, "import-loop.md"),
});
assert.equal(JSON.parse(await readFile(runtimePaths.jsonPath, "utf8")).summary.commandCount, 1);
assert.equal(JSON.parse(await readFile(profileDiffPaths.jsonPath, "utf8")).status, "pass");
assert.equal(JSON.parse(await readFile(refDiffPaths.jsonPath, "utf8")).status, "pass");
assert.equal(JSON.parse(await readFile(importLoopPaths.jsonPath, "utf8")).summary.runs, 1);
});
test("public API exposes synthetic probe helpers", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-synthetic-api-"));
const plan = buildSyntheticProbePlanFromReport({
generatedAt: "test",
targetOpenClaw: {
capturedRegistrars: ["registerTool"],
sdkExports: [],
},
summary: {},
fixtures: [
{
id: "weather",
priority: "high",
hookDetails: [{ name: "before_tool_call", ref: "src/index.js:1" }],
registrationDetails: [{ name: "registerTool", ref: "src/index.js:2" }],
sdkImportDetails: [],
packages: [],
},
],
contractProbes: [],
});
const paths = await writeSyntheticProbePlan(plan, {
jsonPath: path.join(outDir, "synthetic.json"),
markdownPath: path.join(outDir, "synthetic.md"),
});
assert.equal(typeof runCapturedSyntheticProbes, "function");
assert.deepEqual(validateSyntheticProbePlan(plan), []);
assert.match(renderSyntheticProbeMarkdown(plan), /registerTool/);
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.probeCount, 2);
});
test("public API honors config-driven runtime capture", async () => {
@ -83,38 +607,86 @@ test("public API honors config-driven runtime capture", async () => {
"utf8",
);
const previous = process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED;
process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED = "1";
try {
const result = await runPluginCheck({ pluginRoot, outDir: "reports", openclawPath: false });
assert.equal(result.runtimeCapture.summary.registrationCount, 1);
} finally {
if (previous === undefined) {
delete process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED;
} else {
process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED = previous;
}
}
const result = await runPluginCheck({ pluginRoot, outDir: "reports", openclawPath: false, allowExecution: true });
assert.equal(result.runtimeCapture.summary.registrationCount, 1);
});
async function createPluginRoot() {
function profileFixture({ p95WallMs, maxPeakRssMb, nodeBootMs }) {
return {
runs: 3,
summary: { p95WallMs, maxPeakRssMb },
targetOpenClaw: {
compatRecords: 1,
hookNames: 1,
apiRegistrars: 1,
capturedRegistrars: 1,
sdkExports: 1,
manifestFields: 1,
manifestContractFields: 1,
},
fixtureInventory: {},
commands: [{ id: "node-boot", wallMs: { median: nodeBootMs } }],
};
}
function diffReport({ hookNames, issues }) {
return {
summary: {
fixtureCount: 1,
breakageCount: 0,
issueCount: issues.length,
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
},
targetOpenClaw: {
status: "available",
compatRecords: [],
hookNames,
apiRegistrars: ["registerTool"],
capturedRegistrars: ["registerTool"],
sdkExports: ["definePluginEntry"],
manifestFields: ["name"],
manifestContractFields: ["permissions"],
},
fixtures: [
{
id: "weather",
hooks: hookNames,
registrations: ["registerTool"],
sdkImports: ["definePluginEntry"],
pluginManifests: [{ name: "weather" }],
manifestContracts: ["permissions"],
},
],
issues,
};
}
async function createPluginRoot(options = {}) {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-api-root-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
const packageJson = {
name: "@example/openclaw-weather",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.js"],
compat: { pluginApi: "^1.0.0" },
},
...(options.packageJson ?? {}),
};
if (packageJson.openclaw === null) {
delete packageJson.openclaw;
}
if (options.packageConfig) {
packageJson.pluginInspector = {
version: 1,
...options.packageConfig,
};
}
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "@example/openclaw-weather",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.js"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
`${JSON.stringify(packageJson, null, 2)}\n`,
"utf8",
);
await writeFile(

View File

@ -31,6 +31,35 @@ test("capture API defaults to known OpenClaw registrar profiles", () => {
assert.ok(defaultCaptureApiRegistrarProfiles.registerTool);
});
test("capture API returns useful channel, gateway, and lifecycle descriptors", () => {
const api = createCaptureApi();
const channel = api.registerChannel({ plugin: { id: "fixture-channel", outbound: { sendText() {} } } });
const gatewayMethod = api.registerGatewayMethod(
"fixture.ping",
({ respond }) => respond(true, { ok: true }),
{ scope: "operator.read" },
);
const service = api.registerService({ id: "fixture-service" });
assert.equal(api.registrationMode, "full");
assert.equal(channel.id, "fixture-channel");
assert.equal(channel.plugin.id, "fixture-channel");
assert.equal(gatewayMethod.method, "fixture.ping");
assert.equal(gatewayMethod.scope, "operator.read");
assert.deepEqual(gatewayMethod.handler({ respond: api.gateway.respond }), { ok: true, result: { ok: true } });
assert.equal(typeof service.dispose, "function");
});
test("capture API records conversation binding resolved callbacks", () => {
const api = createCaptureApi();
assert.equal(api.onConversationBindingResolved(() => undefined), api);
assert.deepEqual(
api.getCapturedContracts().map((entry) => `${entry.kind}:${entry.name}`),
["hook:onConversationBindingResolved"],
);
});
test("capture API accepts custom registrar return profiles", () => {
const api = createCaptureApi({
registrarProfiles: {
@ -48,6 +77,7 @@ test("capture API accepts custom registrar return profiles", () => {
test("capture API exposes mock context helpers", async () => {
const api = createCaptureApi({
resolvePath: (value) => `/fixture/${value}`,
secretValues: {
token: "redacted",
},
@ -61,6 +91,7 @@ test("capture API exposes mock context helpers", async () => {
assert.deepEqual(await api.store.list(), ["key"]);
assert.equal(api.agent.id, "plugin-inspector-agent");
assert.equal(api.paths.dataDir, ".plugin-inspector/data");
assert.equal(api.resolvePath("state"), "/fixture/state");
});
test("capture API can retain handlers for probes", () => {

View File

@ -63,6 +63,41 @@ test("ci policy allows known blocked probes but fails unknown blockers", () => {
assert.match(renderCiPolicyMarkdown(report), /Plugin Inspector CI Policy/);
});
test("ci policy supports wildcard seam rules for generated surface blockers", () => {
const report = buildCiPolicyReport({
policy: {
...policy,
allowedBlocked: [
...policy.allowedBlocked,
{
id: "generated-surface-runtime-gap",
seam: "*",
reasonIncludes: "generated surface has no callable runtime",
decision: "allowed-blocked",
until: "generated surface runtime harness lands",
},
],
},
compatibilityReport: compatibilityReport(),
executionResults: executionResults([
{
seam: "before_tool_call",
reason: "generated surface has no callable runtime",
},
{
seam: "registerCommand",
reason: "generated surface has no callable runtime",
},
]),
});
assert.equal(report.status, "pass");
assert.deepEqual(
report.checks.filter((check) => check.id.startsWith("execution-results.blocked.")).map((check) => check.action),
["warn", "warn"],
);
});
test("ci policy fails ref diff hard regressions", () => {
const report = buildCiPolicyReport({
policy,
@ -127,7 +162,7 @@ test("ci policy surfaces P0 live issues without blocking default lanes", () => {
code: "legacy-before-agent-start",
},
{
severity: "P1",
severity: "P2",
issueClass: "inspector-gap",
fixture: "wecom",
code: "registration-capture-gap",
@ -225,7 +260,7 @@ test("ci policy writer emits JSON and Markdown artifacts", async () => {
function compatibilityReport(overrides = {}) {
const issues = overrides.issues ?? [
{
severity: "P1",
severity: "P2",
issueClass: "inspector-gap",
fixture: "fixture",
code: "registration-capture-gap",

View File

@ -18,17 +18,17 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
suggestionCount: 3,
issueCount: 4,
p0IssueCount: 1,
p1IssueCount: 1,
p1IssueCount: 0,
liveIssueCount: 1,
compatGapCount: 1,
},
issues: [
{
severity: "P1",
severity: "P2",
issueClass: "inspector-gap",
fixture: "fixture",
code: "registration-capture-gap",
title: "runtime registrations need capture",
title: "runtime registrations need capture evidence",
decision: "inspector-follow-up",
},
],
@ -97,6 +97,13 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
p95WallMs: 75,
maxPeakRssMb: 40,
maxCpuMsEstimate: 30,
maxPluginPeakRssDeltaMb: 8,
maxPluginCpuDeltaMsEstimate: 6,
openClawLifecycleCount: 2,
p50OpenClawImportMs: 12,
p50OpenClawActivationMs: 3,
rssSampleCount: 2,
cpuSampleCount: 2,
},
},
},
@ -108,9 +115,12 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
assert.equal(summary.summary.platformWindowsRisks, 3);
assert.equal(summary.summary.loaderJitiCandidates, 1);
assert.equal(summary.summary.importLoopP50Ms, 50);
assert.equal(summary.summary.importLoopMetricBasis, "baseline-adjusted");
assert.equal(summary.summary.importLoopMaxRssMb, 8);
assert.equal(summary.summary.importLoopOpenClawImportP50Ms, 12);
assert.match(renderCiSummaryMarkdown(summary), /Crabpot CI Summary/);
assert.match(renderCiSummaryMarkdown(summary), /Windows portability risks/);
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ max RSS 40 MB \/ CPU 30 ms/);
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ plugin delta RSS 8 MB \/ plugin delta CPU 6 ms \/ OpenClaw import 12 ms \/ activate 3 ms/);
assert.match(renderCiSummaryMarkdown(summary), /\| P0 issues\s+\| 1\s+\|/);
});

View File

@ -55,13 +55,11 @@ test("check command runs from a plugin root without fixture config", async () =>
assert.ok(report.fixtures[0].package.openclaw.entrypoints.some((entrypoint) => entrypoint.exists));
assert.match(issues, /# OpenClaw Plugin Issue Findings/);
await execFileAsync(process.execPath, [cliPath, "check", "--out", "capture-reports", "--no-openclaw", "--capture"], {
cwd: rootDir,
env: {
...process.env,
PLUGIN_INSPECTOR_EXECUTE_ISOLATED: "1",
},
});
await execFileAsync(
process.execPath,
[cliPath, "check", "--out", "capture-reports", "--no-openclaw", "--capture", "--allow-execute"],
{ cwd: rootDir },
);
const capture = JSON.parse(
await readFile(path.join(rootDir, "capture-reports", "plugin-inspector-runtime-capture.json"), "utf8"),
);
@ -75,13 +73,9 @@ test("check command can target a plugin root and use runtime aliases", async ()
await execFileAsync(
process.execPath,
[cliPath, "--plugin-root", rootDir, "--out", "reports", "--no-openclaw", "--runtime", "--mock-sdk"],
[cliPath, "--plugin-root", rootDir, "--out", "reports", "--no-openclaw", "--runtime", "--mock-sdk", "--allow-execute"],
{
cwd: os.tmpdir(),
env: {
...process.env,
PLUGIN_INSPECTOR_EXECUTE_ISOLATED: "1",
},
},
);
@ -93,6 +87,49 @@ test("check command can target a plugin root and use runtime aliases", async ()
assert.equal(capture.summary.capturedCount, 1);
});
test("check command sanitizes absolute OpenClaw paths in JSON output and artifacts", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-sanitize-");
const openclawPath = await createTargetOpenClaw(rootDir);
const cliPath = path.resolve("src/cli.js");
const { stdout } = await execFileAsync(
process.execPath,
[cliPath, "check", "--out", "reports", "--openclaw", openclawPath, "--json"],
{ cwd: rootDir },
);
const output = JSON.parse(stdout);
const artifact = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector-report.json"), "utf8"));
assert.equal(output.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
assert.doesNotMatch(stdout, new RegExp(escapeRegExp(openclawPath)));
});
test("inspect command runs from a plugin root and can write CI outputs", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-inspect-");
const cliPath = path.resolve("src/cli.js");
const { stdout } = await execFileAsync(process.execPath, [
cliPath,
"inspect",
"--out",
"reports",
"--no-openclaw",
"--sarif",
"--junit",
], {
cwd: rootDir,
});
const sarif = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector.sarif"), "utf8"));
const junit = await readFile(path.join(rootDir, "reports", "plugin-inspector.junit.xml"), "utf8");
assert.match(stdout, /Status: PASS/);
assert.equal(sarif.version, "2.1.0");
assert.match(junit, /<testsuite name="plugin-inspector"/);
});
test("check command can enable runtime capture from plugin config", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-config-runtime-");
await writeFile(
@ -102,12 +139,8 @@ test("check command can enable runtime capture from plugin config", async () =>
);
const cliPath = path.resolve("src/cli.js");
await execFileAsync(process.execPath, [cliPath, "check", "--out", "reports", "--no-openclaw"], {
await execFileAsync(process.execPath, [cliPath, "check", "--out", "reports", "--no-openclaw", "--allow-execute"], {
cwd: rootDir,
env: {
...process.env,
PLUGIN_INSPECTOR_EXECUTE_ISOLATED: "1",
},
});
const capture = JSON.parse(
@ -116,6 +149,26 @@ test("check command can enable runtime capture from plugin config", async () =>
assert.equal(capture.summary.registrationCount, 1);
});
test("config command prints resolved plugin root config", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-config-print-");
const cliPath = path.resolve("src/cli.js");
const { stdout } = await execFileAsync(process.execPath, [cliPath, "config", "--plugin-root", rootDir]);
const { stdout: jsonStdout } = await execFileAsync(process.execPath, [
cliPath,
"config",
"--plugin-root",
rootDir,
"--json",
]);
const config = JSON.parse(jsonStdout);
assert.match(stdout, /Plugin: weather/);
assert.match(stdout, /Runtime capture: off/);
assert.equal(config.fixtures[0].id, "weather");
assert.equal(config.fixtures[0].subdir, "src");
});
test("ci command writes CI summary artifacts", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-ci-");
const cliPath = path.resolve("src/cli.js");
@ -133,6 +186,8 @@ test("ci command writes CI summary artifacts", async () => {
await readFile(path.join(rootDir, "reports", "plugin-inspector-ci-summary.json"), "utf8"),
);
const markdown = await readFile(path.join(rootDir, "reports", "plugin-inspector-ci-summary.md"), "utf8");
const sarif = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector.sarif"), "utf8"));
const junit = await readFile(path.join(rootDir, "reports", "plugin-inspector.junit.xml"), "utf8");
assert.match(stdout, /Status: PASS/);
assert.match(stdout, /Artifacts: 1/);
@ -143,6 +198,8 @@ test("ci command writes CI summary artifacts", async () => {
assert.equal(summary.summary.issues, report.summary.issueCount);
assert.equal(summary.artifacts.compatibility, "plugin-inspector-report.json");
assert.match(markdown, /# Plugin Inspector CI Summary/);
assert.equal(sarif.runs[0].tool.driver.name, "plugin-inspector");
assert.match(junit, /failures="0"/);
});
test("init command writes plugin config and CI workflow", async () => {
@ -156,12 +213,91 @@ test("init command writes plugin config and CI workflow", async () => {
const config = JSON.parse(await readFile(path.join(rootDir, "plugin-inspector.config.json"), "utf8"));
const workflow = await readFile(path.join(rootDir, ".github", "workflows", "plugin-inspector.yml"), "utf8");
assert.match(stdout, /plugin-inspector\.config\.json/);
assert.match(stdout, /^wrote plugin-inspector\.config\.json$/m);
assert.match(stdout, /^wrote \.github\/workflows\/plugin-inspector\.yml$/m);
assert.match(stdout, /^package manager: pnpm$/m);
assert.equal(stdout.includes(rootDir), false);
assert.equal(config.plugin.id, "weather");
assert.equal(config.plugin.sourceRoot, "src");
assert.equal(config.capture.mockSdk, true);
assert.match(workflow, /pnpm dlx @openclaw\/plugin-inspector check --no-openclaw/);
assert.match(workflow, /--runtime --mock-sdk/);
assert.match(workflow, /pnpm dlx @openclaw\/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute/);
});
test("init command detects plugin package managers", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-pm-");
const cliPath = path.resolve("src/cli.js");
await writeFile(path.join(rootDir, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n", "utf8");
await execFileAsync(process.execPath, [cliPath, "init", "--plugin-root", rootDir, "--ci", "--force"]);
const workflow = await readFile(path.join(rootDir, ".github", "workflows", "plugin-inspector.yml"), "utf8");
assert.match(workflow, /cache: pnpm/);
assert.match(workflow, /corepack enable/);
assert.match(workflow, /pnpm install --frozen-lockfile/);
assert.match(workflow, /pnpm dlx @openclaw\/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute/);
});
test("init command can add package scripts", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-scripts-");
const cliPath = path.resolve("src/cli.js");
await execFileAsync(process.execPath, [cliPath, "init", "--plugin-root", rootDir, "--scripts", "--force"]);
const packageJson = JSON.parse(await readFile(path.join(rootDir, "package.json"), "utf8"));
assert.equal(packageJson.scripts["plugin:check"], "plugin-inspector inspect --no-openclaw");
assert.equal(packageJson.scripts["plugin:ci"], "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute");
});
test("init command can preview generated files", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-dry-run-");
const cliPath = path.resolve("src/cli.js");
const beforeConfig = await readFile(path.join(rootDir, "plugin-inspector.config.json"), "utf8");
const beforePackageJson = await readFile(path.join(rootDir, "package.json"), "utf8");
const { stdout } = await execFileAsync(process.execPath, [
cliPath,
"init",
"--plugin-root",
rootDir,
"--ci",
"--scripts",
"--dry-run",
]);
assert.match(stdout, /^would write plugin-inspector\.config\.json$/m);
assert.match(stdout, /^would write \.github\/workflows\/plugin-inspector\.yml$/m);
assert.match(stdout, /^would write package\.json$/m);
assert.equal(await readFile(path.join(rootDir, "plugin-inspector.config.json"), "utf8"), beforeConfig);
assert.equal(await readFile(path.join(rootDir, "package.json"), "utf8"), beforePackageJson);
await assert.rejects(readFile(path.join(rootDir, ".github", "workflows", "plugin-inspector.yml"), "utf8"), {
code: "ENOENT",
});
});
test("init command can print a JSON summary", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-json-");
const cliPath = path.resolve("src/cli.js");
const { stdout } = await execFileAsync(process.execPath, [
cliPath,
"init",
"--plugin-root",
rootDir,
"--ci",
"--scripts",
"--dry-run",
"--json",
]);
const summary = JSON.parse(stdout);
assert.equal(summary.dryRun, true);
assert.equal(summary.packageManager, "npm");
assert.equal(summary.pluginRoot, rootDir);
assert.deepEqual(summary.files, [
"plugin-inspector.config.json",
".github/workflows/plugin-inspector.yml",
"package.json",
]);
});
async function createCliPluginRoot(prefix) {
@ -201,3 +337,18 @@ async function createCliPluginRoot(prefix) {
);
return rootDir;
}
async function createTargetOpenClaw(rootDir) {
const openclawPath = path.join(rootDir, "target-openclaw");
await mkdir(path.join(openclawPath, "src/plugins/compat"), { recursive: true });
await writeFile(
path.join(openclawPath, "src/plugins/compat/registry.ts"),
'export const records = [{ code: "legacy-root-sdk-import", status: "deprecated" }];\n',
"utf8",
);
return openclawPath;
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

View File

@ -71,6 +71,23 @@ test("cold import readiness preserves combined blocker evidence", () => {
assert.deepEqual(validateColdImportReadiness(readiness), []);
});
test("cold import readiness treats openclaw as a host-linked dependency", () => {
const readiness = buildColdImportReadiness({
report: readinessReport(
[{ kind: "extension", specifier: "./index.js", relativePath: "index.js", exists: true }],
{
dependencies: ["openclaw"],
},
),
});
const entrypoint = readiness.fixtures[0].entrypoints[0];
assert.equal(entrypoint.status, "ready");
assert.deepEqual(entrypoint.blockers, []);
assert.equal(readiness.summary.dependencyInstallRequiredCount, 0);
assert.deepEqual(validateColdImportReadiness(readiness), []);
});
test("cold import readiness validation rejects incomplete entries", () => {
const errors = validateColdImportReadiness({
fixtures: [

View File

@ -26,7 +26,7 @@ test("contract coverage fails missing evidence and P1 probe gaps", () => {
fixture: "fixture",
severity: "P1",
issueClass: "inspector-gap",
code: "registration-capture-gap",
code: "conversation-access-hook",
evidence: [],
},
],
@ -165,4 +165,20 @@ test("contract coverage requires compat record reconciliation evidence", () => {
compatRecord: "fixture.provider-auth-env-vars",
});
assert.deepEqual(validateContractCoverage(report), []);
report.logs = [];
report.issues.push({
id: "PLUGIN-COMPAT",
fixture: "fixture",
severity: "P1",
issueClass: "compat-gap",
code: "provider-auth-env-vars",
evidence: ["fixture"],
compatRecord: "fixture.provider-auth-env-vars",
});
report.contractProbes.push({
fixture: "fixture",
id: "compat.provider-auth-env-vars:fixture",
});
assert.deepEqual(validateContractCoverage(report), []);
});

View File

@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { buildContractProbes, contractProbeRules, probePriority } from "../src/advanced.js";
import { buildContractProbes, compatRecordForIssueCode, contractProbeRules, probePriority } from "../src/advanced.js";
test("contract probes map issue findings to executable backlog rows", () => {
const probes = buildContractProbes({
@ -41,11 +41,13 @@ test("contract probes map issue findings to executable backlog rows", () => {
});
assert.ok(contractProbeRules["registration-capture-gap"]);
assert.equal(compatRecordForIssueCode("registration-capture-gap"), "api.capture.runtime-registrars");
assert.equal(compatRecordForIssueCode("package-dependency-install-required"), undefined);
assert.deepEqual(
probes.map((probe) => [probe.id, probe.priority, probe.target]),
[
["api.capture.runtime-registrars:wecom", "P1", "inspector-capture-api"],
["sdk.import.package-export-cold-import:codex-app-server", "P1", "sdk-alias"],
["api.capture.runtime-registrars:wecom", "P2", "inspector-capture-api"],
["manifest.schema.top-level-fields:agentchat", "P3", "manifest-loader"],
],
);
@ -53,6 +55,7 @@ test("contract probes map issue findings to executable backlog rows", () => {
test("contract probe priority escalates critical codes and high-priority fixtures", () => {
assert.equal(probePriority("sdk-export-missing", "medium"), "P1");
assert.equal(probePriority("registration-capture-gap", "high"), "P2");
assert.equal(probePriority("manifest-unknown-fields", "high"), "P2");
assert.equal(probePriority("manifest-unknown-fields", "medium"), "P3");
});

View File

@ -30,12 +30,19 @@ test("import loop profile measures repeated cold capture subprocesses", async ()
assert.deepEqual(validateImportLoopProfile(profile), []);
assert.equal(profile.summary.runs, 2);
assert.equal(profile.summary.baselineRuns, 2);
assert.equal(profile.summary.baselineFailCount, 0);
assert.equal(profile.summary.failCount, 0);
assert.ok(profile.summary.capturedCount >= 2);
assert.ok(profile.summary.p50WallMs > 0);
assert.ok(profile.summary.p50PluginWallDeltaMs >= 0);
assert.ok(profile.summary.maxPluginPeakRssDeltaMb >= 0);
assert.ok(profile.baseline.reference.wallMs > 0);
assert.ok(profile.samples.every((sample) => Number.isFinite(sample.pluginCpuDeltaMsEstimate)));
assert.ok(profile.samples.every((sample) => sample.exitCode === 0));
assert.match(renderImportLoopProfileMarkdown(profile), /Import Loop Profile/);
assert.match(renderImportLoopProfileMarkdown(profile), /CPU Estimate/);
assert.match(renderImportLoopProfileMarkdown(profile), /Harness Baseline/);
assert.match(renderImportLoopProfileMarkdown(profile), /Plugin CPU Delta/);
});
test("import loop profile can use a custom capture script and opt-in env", async () => {
@ -50,7 +57,7 @@ test("import loop profile can use a custom capture script and opt-in env", async
"const [entrypoint,, outputPath] = process.argv.slice(2);",
"if (process.env.CUSTOM_IMPORT_LOOP !== '1') throw new Error('missing opt-in');",
"await mkdir(path.dirname(outputPath), { recursive: true });",
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }] }));",
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }], openClawLifecycle: { importMs: 12, activationMs: 3, importPhase: 'full', activationPhase: 'full:register' } }));",
].join("\n"),
"utf8",
);
@ -64,7 +71,12 @@ test("import loop profile can use a custom capture script and opt-in env", async
});
assert.equal(profile.summary.failCount, 0);
assert.equal(profile.summary.baselineRuns, 1);
assert.equal(profile.summary.capturedCount, 1);
assert.equal(profile.summary.openClawLifecycleCount, 1);
assert.equal(profile.summary.p50OpenClawImportMs, 12);
assert.equal(profile.summary.p50OpenClawActivationMs, 3);
assert.match(renderImportLoopProfileMarkdown(profile), /OpenClaw Import/);
});
test("import loop profile validation rejects failed or empty captures", () => {

View File

@ -43,6 +43,18 @@ test("source inspection records hook, registrar, and SDK import evidence", () =>
]);
});
test("source inspection strips long comments before matching registrations", () => {
const inspection = inspectSourceText(
[`/* ${"a/*".repeat(512)} */`, "api.registerTool({ name: 'weather' });"].join("\n"),
"plugins/example/index.ts",
);
assert.deepEqual(
inspection.registrations.map((registration) => `${registration.name}@${registration.ref}`),
["registerTool@plugins/example/index.ts:2"],
);
});
test("fixture set inspection produces a passing report", async () => {
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
const report = await inspectFixtureSet(config);
@ -65,6 +77,44 @@ test("fixture set inspection reports missing expected seams", async () => {
assert.match(report.breakages[0].message, /llm_output/);
});
test("fixture set inspection treats channel factories as channel registration coverage", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-channel-factory-"));
await mkdir(path.join(dir, "fixture"), { recursive: true });
await writeFile(
path.join(dir, "fixture", "index.js"),
[
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
"",
"export const channel = createChatChannelPlugin({ id: 'fixture-channel' });",
"export default defineBundledChannelEntry({ id: 'bundled-channel' });",
].join("\n"),
"utf8",
);
const report = await inspectFixtureSet({
version: 1,
submoduleRoot: ".",
rootDir: dir,
fixtures: [
{
id: "fixture",
path: "fixture",
repo: "https://github.com/openclaw/fixture.git",
priority: "high",
seams: ["channel"],
expect: {
registrations: ["registerChannel"],
},
},
],
});
assert.equal(report.status, "pass");
assert.deepEqual(report.breakages, []);
assert.deepEqual(report.fixtures[0].registrations, ["createChatChannelPlugin", "defineBundledChannelEntry"]);
});
test("capture entrypoint imports a local fixture and records registrations", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-capture-"));
const entrypoint = path.join(dir, "fixture.mjs");
@ -103,6 +153,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
'import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";',
'import { GPT5_BEHAVIOR_CONTRACT } from "openclaw/plugin-sdk/provider-model-shared";',
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
'import { registerPluginHttpRoute, resolveWebhookPath } from "openclaw/plugin-sdk/webhook-ingress";',
"",
@ -123,6 +174,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
"",
"export default definePluginEntry((api) => {",
" if (!pluginSdkMock) throw new Error('expected mock SDK');",
" if (!GPT5_BEHAVIOR_CONTRACT) throw new Error('expected dynamic subpath mock export');",
" provider.register(api);",
" api.registerHttpRoute({ path: resolveWebhookPath('hook'), handler() {} });",
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
@ -144,3 +196,153 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
["registration:registerProvider", "registration:registerHttpRoute", "registration:registerTool"],
);
});
test("mock capture accepts valid output when plugin code dirties process exit code", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-exit-code-"));
const entrypoint = path.join(dir, "index.mjs");
await writeFile(
entrypoint,
[
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
"process.exitCode = 1;",
"export default definePluginEntry({",
" register(api) {",
" api.registerProvider({ id: 'fixture-provider' });",
" },",
"});",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.mjs", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
"registration:registerProvider",
]);
});
test("mock capture prefers discovered bare mocks over installed dependency exports", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-bare-capture-"));
await mkdir(path.join(dir, "node_modules/typebox"), { recursive: true });
await writeFile(
path.join(dir, "node_modules/typebox/package.json"),
JSON.stringify({ name: "typebox", version: "0.0.0", type: "module", exports: "./index.js" }, null, 2),
"utf8",
);
await writeFile(path.join(dir, "node_modules/typebox/index.js"), "export const Type = {};\n", "utf8");
const entrypoint = path.join(dir, "index.mjs");
await writeFile(
entrypoint,
[
"import path from 'node:path';",
'import { Static, Type } from "typebox";',
'import { resolvePreferredOpenClawTmpDir } from "fixture-api";',
"export function register(api) {",
" if (!Static || !Type) throw new Error('expected mocked typebox exports');",
" path.join(resolvePreferredOpenClawTmpDir(), 'fixture');",
" api.registerTool({ name: 'fixture_tool', inputSchema: Type.Object({}), run() {} });",
"}",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.mjs", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), ["registration:registerTool"]);
});
test("mock capture expands bundled channel entry registration shells", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-capture-"));
const entrypoint = path.join(dir, "index.mjs");
await writeFile(
entrypoint,
[
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
"export default defineBundledChannelEntry({",
" id: 'fixture-channel',",
" name: 'Fixture Channel',",
" description: 'Fixture channel',",
" plugin: { specifier: './channel.js', exportName: 'fixtureChannel' },",
" registerFull(api) {",
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
" },",
"});",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.mjs", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
"registration:registerChannel",
"registration:registerTool",
]);
});
test("mock capture follows bundled channel linked registerFull exports", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-linked-capture-"));
await writeFile(
path.join(dir, "index.ts"),
[
'import { defineBundledChannelEntry, loadBundledEntryExportSync } from "openclaw/plugin-sdk/channel-entry-contract";',
"",
"function registerFull(api) {",
" const register = loadBundledEntryExportSync(import.meta.url, {",
" specifier: './api.js',",
" exportName: 'registerFixtureFull',",
" });",
" register(api);",
"}",
"",
"export default defineBundledChannelEntry({",
" id: 'fixture-channel',",
" name: 'Fixture Channel',",
" description: 'Fixture channel',",
" plugin: { specifier: './channel-plugin-api.js', exportName: 'fixtureChannel' },",
" registerFull,",
"});",
].join("\n"),
"utf8",
);
await writeFile(
path.join(dir, "api.ts"),
[
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
"",
"const schema = buildSecretInputSchema().optional();",
"",
"export function registerFixtureFull(api) {",
" schema.parse(undefined);",
" api.registerCommand({ name: 'fixture.command', run() {} });",
"}",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.ts", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
"registration:registerChannel",
"registration:registerCommand",
]);
});

View File

@ -19,18 +19,18 @@ test("issue ids are stable fingerprints", () => {
test("issue classification separates live breaks from compat and deprecation buckets", () => {
const cases = [
{
name: "untracked SDK alias is a blocking live issue",
name: "untracked SDK alias is a compat gap",
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
targetOpenClaw: { compatRecordStatuses: {} },
metadata: { severity: "P1" },
expected: { issueClass: "live-issue", compatStatus: "untracked", severity: "P0", live: true },
expected: { issueClass: "compat-gap", compatStatus: "untracked", severity: "P1", live: false },
},
{
name: "active SDK alias compat avoids false P0 escalation",
name: "active SDK alias compat stays a compat row",
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
targetOpenClaw: { compatRecordStatuses: { "plugin-sdk-export-aliases": "active" } },
metadata: { severity: "P1" },
expected: { issueClass: "live-issue", compatStatus: "active", severity: "P1", live: true },
expected: { issueClass: "compat-gap", compatStatus: "active", severity: "P1", live: false },
},
{
name: "deprecated compat remains warning-class even when used",
@ -46,6 +46,18 @@ test("issue classification separates live breaks from compat and deprecation buc
metadata: { severity: "P1" },
expected: { issueClass: "compat-gap", compatStatus: "missing", severity: "P1", live: false },
},
{
name: "active OpenClaw probe contract stays an inspector gap",
finding: {
code: "before-tool-call-probe",
compatRecord: "hook.before_tool_call.terminal-block-approval",
},
targetOpenClaw: {
compatRecordStatuses: { "hook.before_tool_call.terminal-block-approval": "active" },
},
metadata: { severity: "P1" },
expected: { issueClass: "inspector-gap", compatStatus: "active", severity: "P1", live: false },
},
{
name: "unknown untracked hook is P0 live break",
finding: { code: "unknown-hook-name" },
@ -100,17 +112,17 @@ test("issue builder applies metadata and class summaries", () => {
assert.deepEqual(
issues.map((issue) => [issue.fixture, issue.code, issue.severity, issue.issueClass, issue.status]),
[
["codex-app-server", "sdk-export-missing", "P0", "live-issue", "blocking"],
["wecom", "registration-capture-gap", "P1", "inspector-gap", "open"],
["codex-app-server", "sdk-export-missing", "P1", "compat-gap", "open"],
["agentchat", "manifest-unknown-fields", "P2", "upstream-metadata", "open"],
["wecom", "registration-capture-gap", "P2", "inspector-gap", "open"],
],
);
assert.deepEqual(summarizeIssueClasses(issues), {
"compat-gap": 0,
"compat-gap": 1,
"deprecation-warning": 0,
"fixture-regression": 0,
"inspector-gap": 1,
"live-issue": 1,
"live-issue": 0,
"upstream-metadata": 1,
});
});

View File

@ -136,6 +136,7 @@ test("OpenClaw target parsing helpers stay deterministic", () => {
);
assert.deepEqual(
parseCompatRecordEntries(`
${"{{".repeat(256)}
{ code: "b", status: "supported" }
{ code: "a", status: "deprecated" }
{ code: "b", status: "supported" }

View File

@ -0,0 +1,60 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { buildPackageContentsChecklist } from "../scripts/check-package-contents.mjs";
const packageJson = {
bin: {
"plugin-inspector": "src/cli.js",
},
exports: {
".": "./src/index.js",
"./capture-api": "./src/capture-api.js",
},
};
const requiredFiles = [
"package.json",
"README.md",
"CHANGELOG.md",
"LICENSE",
"src/cli.js",
"src/index.js",
"src/capture-api.js",
"examples/github-actions-plugin-inspector.yml",
"examples/plugin-inspector.config.json",
"docs/plugin-inspector-banner.jpg",
];
test("package contents pass when entrypoints and README assets are packed", () => {
const result = buildPackageContentsChecklist({
filePaths: requiredFiles,
packageJson,
readmeText: '<img src="docs/plugin-inspector-banner.jpg" alt="banner"/>',
});
assert.equal(result.status, "pass");
});
test("package contents fail when README assets are missing", () => {
const result = buildPackageContentsChecklist({
filePaths: requiredFiles.filter((filePath) => filePath !== "docs/plugin-inspector-banner.jpg"),
packageJson,
readmeText: '<img src="docs/plugin-inspector-banner.jpg" alt="banner"/>',
});
assert.equal(result.status, "fail");
assert.equal(result.checks.find((check) => check.id === "package-readme-asset").actual, "missing");
});
test("package contents fail when private release scripts are packed", () => {
const result = buildPackageContentsChecklist({
filePaths: [...requiredFiles, "scripts/release-notes.mjs"],
packageJson,
});
assert.equal(result.status, "fail");
assert.equal(
result.checks.find((check) => check.id === "package-forbidden-path").message,
"scripts/release-notes.mjs should not be published in the npm package",
);
});

View File

@ -40,11 +40,11 @@ test("platform probes classify loader and shell portability risks", () => {
},
{
kind: "capture",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx capture.mjs ./src/index.ts",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./src/index.ts --mock-sdk",
},
{
kind: "synthetic-probe",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx synthetic.mjs --entrypoint ./src/index.ts",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node synthetic.mjs --entrypoint ./src/index.ts --mock-sdk",
},
],
},
@ -63,7 +63,68 @@ test("platform probes classify loader and shell portability risks", () => {
assert.match(renderPlatformProbesMarkdown(report), /rsync/);
});
test("platform probe validation requires jiti fallback and reflected tsx commands", () => {
test("platform probes separate executor-covered portability risks from residual risks", () => {
const report = buildPlatformProbes({
plan: {
generatedAt: "test",
mode: "plan-only",
summary: {
fixtureCount: 1,
},
fixtures: [
{
id: "fixture",
entrypoints: [
{
id: "cold-import.extension:fixture:index",
status: "dependency-install-required",
entrypoint: "plugins/fixture/index.js",
packageManager: "pnpm",
loaderStrategy: {
source: "javascript",
primary: "node",
alternatives: [],
reason: "test",
},
steps: [
{
kind: "prepare",
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
},
{
kind: "audit",
command: "pnpm audit --json > ../../results/fixture/package-audit.json || true",
},
{
kind: "capture",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./index.js",
},
],
},
],
},
],
},
stepCoverage({ riskCodes }) {
return {
reason: "covered by Crabpot structured executor",
riskCodes: riskCodes.filter((code) =>
["posix-mkdir", "posix-env-prefix", "posix-null-failure", "rsync-required", "shell-redirection"].includes(code),
),
};
},
});
assert.equal(report.summary.portabilityFindingCount, 1);
assert.equal(report.summary.coveredPortabilityFindingCount, 3);
assert.equal(report.summary.windowsRiskStepCount, 1);
assert.deepEqual(report.portabilityFindings[0].riskCodes, ["package-manager-availability"]);
assert.ok(report.coveredPortabilityFindings.every((finding) => finding.coverage === "covered by Crabpot structured executor"));
assert.doesNotMatch(renderPlatformProbesMarkdown(report), /replace shell mkdir/);
assert.match(renderPlatformProbesMarkdown(report), /Covered Portability Findings/);
});
test("platform probe validation requires jiti fallback and reflected TypeScript loader commands", () => {
const errors = validatePlatformProbes({
mode: "plan-only",
targets: ["linux", "macos", "windows", "container"],
@ -76,11 +137,14 @@ test("platform probe validation requires jiti fallback and reflected tsx command
id: "cold-import.extension:fixture:index",
loaderPrimary: "tsx",
captureUsesTsx: true,
captureUsesTypeScriptLoader: true,
syntheticUsesTsx: false,
syntheticUsesMockSdk: false,
syntheticUsesTypeScriptLoader: false,
},
],
});
assert.ok(errors.some((error) => error.includes("Jiti fallback")));
assert.ok(errors.some((error) => error.includes("tsx loader strategy")));
assert.ok(errors.some((error) => error.includes("TypeScript loader strategy")));
});

View File

@ -0,0 +1,103 @@
import assert from "node:assert/strict";
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "node:test";
import { buildCrabpotFollowthroughChecklist } from "../scripts/check-crabpot-followthrough.mjs";
test("crabpot follow-through checklist passes matching source refs", async () => {
const roots = await createFixtureRoots({
ref: "abc123",
packagePin: "@openclaw/plugin-inspector@1.2.3",
});
const result = buildCrabpotFollowthroughChecklist({
pluginInspectorRoot: roots.pluginInspectorRoot,
crabpotRoot: roots.crabpotRoot,
expectedRef: "abc123",
expectedVersion: "1.2.3",
});
assert.equal(result.status, "pass");
assert.equal(result.checks.find((check) => check.id === "crabpot-source-ref").status, "pass");
assert.equal(result.checks.find((check) => check.id === "crabpot-public-api-migration").status, "pass");
assert.equal(result.checks.find((check) => check.id === "crabpot-package-pin").status, "pass");
});
test("crabpot follow-through checklist fails stale refs and optionally stale package pins", async () => {
const roots = await createFixtureRoots({
ref: "old-ref",
packagePin: "@openclaw/plugin-inspector@1.0.0",
});
const preRelease = buildCrabpotFollowthroughChecklist({
pluginInspectorRoot: roots.pluginInspectorRoot,
crabpotRoot: roots.crabpotRoot,
expectedRef: "new-ref",
expectedVersion: "1.2.3",
});
const postRelease = buildCrabpotFollowthroughChecklist({
pluginInspectorRoot: roots.pluginInspectorRoot,
crabpotRoot: roots.crabpotRoot,
expectedRef: "old-ref",
expectedVersion: "1.2.3",
requirePublishedPin: true,
});
assert.equal(preRelease.status, "fail");
assert.equal(preRelease.checks.find((check) => check.id === "crabpot-package-pin").status, "manual");
assert.equal(postRelease.status, "fail");
assert.equal(postRelease.checks.find((check) => check.id === "crabpot-package-pin").status, "fail");
});
test("crabpot follow-through checklist fails advanced bundle script consumers", async () => {
const roots = await createFixtureRoots({
ref: "abc123",
packagePin: "@openclaw/plugin-inspector@1.2.3",
scripts: {
"synthetic-probes.mjs": [
'import { loadPluginInspector } from "./plugin-inspector-source.mjs";',
"const pluginInspector = await loadPluginInspector();",
"export const validateSyntheticProbePlan = pluginInspector.validateSyntheticProbePlan;",
"",
].join("\n"),
},
});
const result = buildCrabpotFollowthroughChecklist({
pluginInspectorRoot: roots.pluginInspectorRoot,
crabpotRoot: roots.crabpotRoot,
expectedRef: "abc123",
expectedVersion: "1.2.3",
});
assert.equal(result.status, "fail");
assert.equal(result.checks.find((check) => check.id === "crabpot-public-api-migration").status, "fail");
assert.match(result.checks.find((check) => check.id === "crabpot-public-api-migration").actual, /synthetic-probes/);
});
async function createFixtureRoots({ ref, packagePin, scripts = {} }) {
const root = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-release-followthrough-"));
const pluginInspectorRoot = path.join(root, "plugin-inspector");
const crabpotRoot = path.join(root, "crabpot");
await mkdir(path.join(crabpotRoot, "scripts"), { recursive: true });
await mkdir(pluginInspectorRoot, { recursive: true });
await writeFile(
path.join(pluginInspectorRoot, "package.json"),
`${JSON.stringify({ version: "1.2.3" }, null, 2)}\n`,
"utf8",
);
await writeFile(
path.join(crabpotRoot, "scripts", "plugin-inspector-source.mjs"),
[
`export const pluginInspectorRef = "${ref}";`,
`export const pluginInspectorPackage = "${packagePin}";`,
"",
].join("\n"),
"utf8",
);
for (const [scriptPath, content] of Object.entries(scripts)) {
await writeFile(path.join(crabpotRoot, "scripts", scriptPath), content, "utf8");
}
return { pluginInspectorRoot, crabpotRoot };
}

View File

@ -0,0 +1,38 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { extractReleaseNotes } from "../scripts/release-notes.mjs";
const changelog = [
"# Changelog",
"",
"## Unreleased",
"",
"### Added",
"",
"- Add grouped API helpers.",
"",
"## 0.3.0 - 2026-04-27",
"",
"### Changed",
"",
"- Improve setup.",
"",
].join("\n");
test("release notes can render unreleased draft notes", () => {
assert.equal(
extractReleaseNotes({ changelogText: changelog, version: "Unreleased" }),
["## plugin-inspector unreleased", "", "### Added", "", "- Add grouped API helpers.", ""].join("\n"),
);
});
test("release notes can render versioned changelog sections", () => {
assert.equal(
extractReleaseNotes({ changelogText: changelog, version: "0.3.0" }),
["## plugin-inspector v0.3.0", "", "### Changed", "", "- Improve setup.", ""].join("\n"),
);
});
test("release notes fail when a version section is missing", () => {
assert.throws(() => extractReleaseNotes({ changelogText: changelog, version: "9.9.9" }), /missing a 9\.9\.9/);
});

57
test/release-plan.test.js Normal file
View File

@ -0,0 +1,57 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { buildReleasePlan } from "../scripts/release-plan.mjs";
const changelog = [
"# Changelog",
"",
"## Unreleased",
"",
"### Changed",
"",
"- Tighten package contents.",
"",
"## 0.3.0 - 2026-04-27",
"",
].join("\n");
test("release plan infers the next patch version", () => {
const plan = buildReleasePlan({
packageVersion: "0.3.0",
changelogText: changelog,
releaseDate: "2026-04-28",
releaseRef: "abc123",
});
assert.equal(plan.status, "pass");
assert.equal(plan.nextVersion, "0.3.1");
assert.equal(plan.tagName, "v0.3.1");
assert.equal(plan.changelogHeading, "## 0.3.1 - 2026-04-28");
assert.equal(plan.crabpotSourceRef, "abc123");
assert.equal(plan.crabpotPackagePin, "@openclaw/plugin-inspector@0.3.1");
});
test("release plan rejects non-advancing versions", () => {
const plan = buildReleasePlan({
packageVersion: "0.3.0",
nextVersion: "0.3.0",
changelogText: changelog,
releaseDate: "2026-04-28",
releaseRef: "abc123",
});
assert.equal(plan.status, "fail");
assert.equal(plan.checks.find((check) => check.id === "version-advance").status, "fail");
});
test("release plan requires unreleased changelog bullets", () => {
const plan = buildReleasePlan({
packageVersion: "0.3.0",
changelogText: "# Changelog\n\n## Unreleased\n",
releaseDate: "2026-04-28",
releaseRef: "abc123",
});
assert.equal(plan.status, "fail");
assert.equal(plan.checks.find((check) => check.id === "changelog-unreleased").status, "fail");
});

View File

@ -4,22 +4,34 @@ import os from "node:os";
import path from "node:path";
import { test } from "node:test";
import {
buildSarifReport,
buildCompatibilityReport,
buildCompatibilityFixtureReport,
buildIssues,
classifyCompatibilityFixture,
classifyCompatRecordCoverage,
classifyPackageContracts,
classifyTargetOpenClawCoverage,
escapeMarkdownTableCell,
inspectFixtureSet,
loadInspectorConfig,
renderCompatibilityIssuesReport,
renderCompatibilityMarkdownReport,
renderJunitXml,
renderMarkdownReport,
renderMarkdownTable,
renderTextSummary,
writeArtifacts,
writeCompatibilityReport,
writeCiOutputArtifacts,
writeReport,
} from "../src/advanced.js";
test("markdown table cell escaping preserves literal backslashes", () => {
assert.equal(escapeMarkdownTableCell(String.raw`C:\tmp|next
line`), String.raw`C:\\tmp\|next<br>line`);
});
test("markdown report includes summary and inventory", async () => {
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
const report = await inspectFixtureSet(config);
@ -29,6 +41,53 @@ test("markdown report includes summary and inventory", async () => {
assert.match(markdown, /\| sample-plugin \| high \| native-tool \| before_tool_call \| definePluginEntry, registerTool \| tools \|/);
});
test("text summary includes artifact paths and top blocking findings", () => {
const summary = renderTextSummary(
{
status: "fail",
summary: {
fixtureCount: 1,
breakageCount: 1,
issueCount: 1,
logCount: 0,
},
breakages: [
{
fixture: "weather",
code: "missing-expected-seam",
level: "breakage",
message: "weather: missing expected registration registerTool",
evidence: ["src/index.js:12"],
},
],
warnings: [],
issues: [
{
fixture: "weather",
code: "sdk-export-missing",
severity: "P0",
status: "blocking",
title: "SDK export is missing",
evidence: ["src/index.js:1"],
},
],
},
{
artifacts: {
jsonPath: "reports/plugin-inspector-report.json",
markdownPath: "reports/plugin-inspector-report.md",
},
},
);
assert.match(summary, /Status: FAIL/);
assert.match(summary, /Issues: 1/);
assert.match(summary, /Reports:/);
assert.match(summary, /json: reports\/plugin-inspector-report\.json/);
assert.match(summary, /Top findings:/);
assert.match(summary, /BREAKAGE weather missing-expected-seam/);
});
test("compatibility report renderer supports issue metadata and evidence links", () => {
const report = {
generatedAt: "test",
@ -134,6 +193,91 @@ test("compatibility report renderer supports issue metadata and evidence links",
assert.match(issues, /\[linked\]\(plugins\/sample\/src\/index\.ts:1\)/);
});
test("compatibility report artifacts sanitize absolute OpenClaw target paths", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-sanitized-report-"));
const absoluteOpenClawPath = path.join(outDir, "openclaw");
const report = {
generatedAt: "test",
status: "pass",
targetOpenClaw: {
status: "ok",
configuredPath: absoluteOpenClawPath,
searchedPaths: [absoluteOpenClawPath],
compatRecords: [],
compatRecordStatuses: {},
},
summary: {
fixtureCount: 1,
highPriorityFixtures: 1,
breakageCount: 0,
warningCount: 0,
suggestionCount: 0,
decisionCount: 0,
issueCount: 1,
p0IssueCount: 0,
p1IssueCount: 0,
liveIssueCount: 0,
liveP0IssueCount: 0,
compatGapCount: 0,
deprecationWarningCount: 0,
inspectorGapCount: 1,
upstreamIssueCount: 0,
fixtureRegressionCount: 0,
contractProbeCount: 0,
},
fixtures: [
{
id: "sample-plugin",
priority: "high",
seams: ["native-tool"],
hooks: [],
registrations: [],
manifestContracts: [],
},
],
breakages: [],
warnings: [],
suggestions: [],
issues: [
{
fixture: "sample-plugin",
code: "package-dependency-install-required",
issueClass: "inspector-gap",
decision: "inspector-follow-up",
severity: "P2",
title: `sample-plugin: path ${absoluteOpenClawPath}`,
status: "open",
compatStatus: "none",
live: false,
evidence: [absoluteOpenClawPath],
},
],
contractProbes: [],
logs: [],
decisions: [],
};
const markdown = renderCompatibilityMarkdownReport(report);
const issues = renderCompatibilityIssuesReport(report);
const paths = await writeCompatibilityReport(report, {
jsonPath: path.join(outDir, "report.json"),
markdownPath: path.join(outDir, "report.md"),
issuesPath: path.join(outDir, "issues.md"),
});
const artifact = JSON.parse(await readFile(paths.jsonPath, "utf8"));
assert.equal(report.targetOpenClaw.configuredPath, absoluteOpenClawPath);
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
assert.equal(artifact.issues[0].evidence[0], "<OPENCLAW_PATH>");
assert.equal(artifact.issues[0].title, "sample-plugin: path <OPENCLAW_PATH>");
assert.doesNotMatch(markdown, new RegExp(escapeRegExp(absoluteOpenClawPath)));
assert.doesNotMatch(issues, new RegExp(escapeRegExp(absoluteOpenClawPath)));
assert.match(markdown, /<OPENCLAW_PATH>/);
assert.match(await readFile(paths.markdownPath, "utf8"), /<OPENCLAW_PATH>/);
assert.match(await readFile(paths.issuesPath, "utf8"), /<OPENCLAW_PATH>/);
});
test("compatibility report assembly classifies fixtures, issues, probes, and compat records", async () => {
const report = await buildCompatibilityReport({
generatedAt: "test",
@ -226,6 +370,110 @@ test("compatibility report assembly classifies fixtures, issues, probes, and com
assert.ok(report.decisions.some((decision) => decision.seam === "compat-registry"));
});
test("compatibility report marks inspector gaps covered by runtime execution artifacts", async () => {
const report = await buildCompatibilityReport({
generatedAt: "test",
fixtures: [
{
id: "fixture",
name: "Fixture",
path: "plugins/fixture",
priority: "high",
seams: ["native-tool"],
why: "covers runtime-only seams",
},
],
inspections: [
{
id: "fixture",
status: "ok",
hooks: ["llm_input"],
hookDetails: [{ name: "llm_input", ref: "plugins/fixture/src/index.ts:1" }],
registrations: ["registerTool", "registerService", "registerCommand"],
registrationDetails: [
{ name: "registerTool", ref: "plugins/fixture/src/index.ts:2" },
{ name: "registerService", ref: "plugins/fixture/src/index.ts:3" },
{ name: "registerCommand", ref: "plugins/fixture/src/index.ts:4" },
],
manifestContracts: [],
manifestFiles: [],
sdkImports: [],
sourceFiles: ["plugins/fixture/src/index.ts"],
},
],
targetOpenClaw: {
status: "ok",
compatRecords: [],
compatRecordStatuses: {},
hookNames: ["llm_input"],
apiRegistrars: ["registerTool", "registerService", "registerCommand"],
capturedRegistrars: [],
sdkExports: [],
manifestFields: ["id"],
manifestContractFields: [],
},
executionResults: {
artifacts: [
{
fixture: "fixture",
kind: "capture",
status: "pass",
artifactPath: ".crabpot/results/fixture/index.capture.json",
captured: [
"hook:llm_input",
"registration:registerTool",
"registration:registerService",
"registration:registerCommand",
],
},
],
},
buildFixtureReport: ({ fixture, inspection }) => ({
id: fixture.id,
name: fixture.name,
priority: fixture.priority,
seams: fixture.seams,
why: fixture.why,
status: inspection.status,
hooks: inspection.hooks,
hookDetails: inspection.hookDetails,
registrations: inspection.registrations,
registrationDetails: inspection.registrationDetails,
manifestContracts: inspection.manifestContracts,
manifestFiles: [],
sourceFiles: inspection.sourceFiles,
pluginManifests: [],
package: null,
packages: [],
sdkImports: [],
sdkImportDetails: [],
}),
});
const coveredCodes = report.issues
.filter((issue) => issue.status === "runtime-covered")
.map((issue) => issue.code)
.sort();
assert.deepEqual(coveredCodes, [
"conversation-access-hook",
"registration-capture-gap",
"runtime-tool-capture",
]);
assert.equal(report.summary.runtimeCoveredIssueCount, 3);
assert.equal(report.summary.openInspectorGapCount, 0);
assert.equal(report.summary.runtimeCoverageArtifactCount, 1);
const registrationIssue = report.issues.find((issue) => issue.code === "registration-capture-gap");
assert.deepEqual(registrationIssue.runtimeCoverage.captured, [
"registration:registerService",
"registration:registerCommand",
]);
const markdown = renderCompatibilityIssuesReport(report);
assert.match(markdown, /## Runtime-Covered Inspector Gaps/);
assert.match(markdown, /state: runtime-covered .* runtime:covered/);
});
test("compat record coverage logs unavailable targets", () => {
const logs = [];
classifyCompatRecordCoverage({
@ -254,6 +502,21 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
`${JSON.stringify({ id: "fixture", name: "Fixture", version: "1.0.0", contracts: { tools: {} } }, null, 2)}\n`,
"utf8",
);
await writeFile(
path.join(fixtureDir, "openclaw.security.json"),
`${JSON.stringify(
{
$schema: "https://openclaw.ai/schemas/plugin-security.json",
version: "1.0.0",
plugin: "fixture",
expectedBehaviors: [{ id: "api-key", description: "requires an API key" }],
securityNotes: [{ id: "storage", description: "stores local state" }],
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(path.join(fixtureDir, "src", "index.js"), "export function register() {}\n", "utf8");
await writeFile(
path.join(fixtureDir, "package.json"),
@ -262,10 +525,28 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
name: "fixture-plugin",
version: "1.0.0",
type: "module",
files: ["src", "openclaw.plugin.json"],
dependencies: { zod: "^1.0.0" },
openclaw: {
extensions: ["src/index.js"],
compat: { pluginApi: "^1.0.0" },
build: {
openclawVersion: "2026.5.2",
pluginSdkVersion: "2026.5.2",
},
install: {
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
npmSpec: "@openclaw/fixture-plugin",
defaultChoice: "clawhub",
minHostVersion: ">=2026.5.2",
},
release: {
publishToClawHub: true,
publishToNpm: true,
},
bundle: {
includeInCore: false,
},
},
},
null,
@ -298,8 +579,35 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
});
assert.equal(report.pluginManifests[0].id, "fixture");
assert.deepEqual(report.securityManifests[0], {
path: "plugin/openclaw.security.json",
schema: "https://openclaw.ai/schemas/plugin-security.json",
version: "1.0.0",
plugin: "fixture",
expectedBehaviorCount: 1,
securityNoteCount: 1,
validJson: true,
});
assert.equal(report.package.name, "fixture-plugin");
assert.deepEqual(report.package.npmPack, {
advertised: true,
private: false,
filesMode: "allowlist",
files: ["src", "openclaw.plugin.json"],
invalidFileSpecs: [],
});
assert.equal(report.package.openclaw.compatPluginApi, "^1.0.0");
assert.deepEqual(report.package.openclaw.install, {
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
npmSpec: "@openclaw/fixture-plugin",
defaultChoice: "clawhub",
minHostVersion: ">=2026.5.2",
});
assert.deepEqual(report.package.openclaw.release, {
publishToClawHub: true,
publishToNpm: true,
});
assert.deepEqual(report.package.openclaw.unsupportedMetadata, ["openclaw.bundle"]);
assert.deepEqual(report.package.openclaw.entrypoints[0], {
kind: "extension",
specifier: "src/index.js",
@ -350,6 +658,284 @@ test("package contract classifier reports install and entrypoint blockers", () =
assert.ok(result.suggestions.some((finding) => finding.code === "package-build-artifact-entrypoint"));
assert.ok(result.suggestions.some((finding) => finding.code === "package-dependency-install-required"));
assert.ok(result.decisions.some((decision) => decision.seam === "cold-import"));
const issues = buildIssues({
suggestions: result.suggestions,
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
});
assert.ok(
issues.some(
(issue) =>
issue.code === "package-dependency-install-required" &&
issue.title === "fixture: cold import requires dependency installation in an isolated workspace",
),
);
});
test("package contract classifier treats openclaw as a host-linked dependency", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "fixture-plugin",
version: "1.0.0",
dependencies: ["openclaw"],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
entrypoints: [
{
kind: "extension",
specifier: "dist/index.js",
relativePath: "plugins/fixture/dist/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.equal(
result.suggestions.some((finding) => finding.code === "package-dependency-install-required"),
false,
);
});
test("package contract classifier reports broken install and release metadata", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
dependencies: [],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
buildOpenClawVersion: "2026.5.2",
install: {
clawhubSpec: null,
npmSpec: "fixture-plugin",
defaultChoice: "clawhub",
minHostVersion: ">=2026.5.1",
},
release: {
publishToClawHub: true,
publishToNpm: true,
},
unsupportedMetadata: ["openclaw.bundle"],
entrypoints: [
{
kind: "extension",
specifier: "dist/index.js",
relativePath: "plugins/fixture/dist/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.ok(result.warnings.some((finding) => finding.code === "package-install-metadata-incomplete"));
assert.ok(result.warnings.some((finding) => finding.code === "package-min-host-version-drift"));
assert.ok(result.warnings.some((finding) => finding.code === "package-openclaw-unsupported-metadata"));
assert.ok(result.decisions.some((decision) => decision.seam === "package-metadata"));
});
test("package contract classifier reports advertised npm pack blockers", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
npmPack: {
advertised: true,
private: true,
filesMode: "allowlist",
files: ["README.md"],
invalidFileSpecs: ["../secrets"],
},
dependencies: [],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
install: {
npmSpec: "@openclaw/fixture-plugin",
},
release: {
publishToNpm: true,
},
entrypoints: [
{
kind: "extension",
specifier: "src/index.js",
relativePath: "plugins/fixture/src/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-unavailable"));
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-metadata-missing"));
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"));
assert.ok(result.decisions.some((decision) => decision.seam === "package-artifact"));
const globResult = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
npmPack: {
advertised: true,
private: false,
filesMode: "allowlist",
files: ["src/**/*.js", "openclaw.plugin.json"],
invalidFileSpecs: [],
},
dependencies: [],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
install: {
npmSpec: "@openclaw/fixture-plugin",
},
release: {
publishToNpm: true,
},
entrypoints: [
{
kind: "extension",
specifier: "src/index.js",
relativePath: "plugins/fixture/src/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.equal(globResult.warnings.some((finding) => finding.code.startsWith("package-npm-pack-")), false);
});
test("package contract classifier accepts built runtime entries for source package metadata", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
npmPack: {
advertised: true,
private: false,
filesMode: "allowlist",
files: ["dist/**", "openclaw.plugin.json"],
invalidFileSpecs: [],
},
dependencies: [],
peerDependencies: ["openclaw"],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
install: {
npmSpec: "@openclaw/fixture-plugin",
},
release: {
publishToNpm: true,
},
entrypoints: [
{
kind: "extension",
specifier: "./index.ts",
relativePath: "plugins/fixture/index.ts",
exists: false,
requiresBuild: false,
},
{
kind: "runtimeExtension",
specifier: "./dist/index.js",
relativePath: "plugins/fixture/dist/index.js",
exists: true,
requiresBuild: true,
},
{
kind: "setupEntry",
specifier: "./setup-entry.ts",
relativePath: "plugins/fixture/setup-entry.ts",
exists: false,
requiresBuild: false,
},
{
kind: "runtimeExtension",
specifier: "./dist/setup-entry.js",
relativePath: "plugins/fixture/dist/setup-entry.js",
exists: true,
requiresBuild: true,
},
],
},
},
},
});
assert.equal(result.warnings.some((finding) => finding.code === "package-entrypoint-missing"), false);
assert.equal(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"), false);
assert.equal(result.decisions.some((decision) => decision.seam === "package-entrypoint"), false);
});
test("target OpenClaw coverage classifier reports missing public surface", () => {
@ -427,6 +1013,17 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
channelEnvVars: { CHANNEL_ID: "channel id" },
},
],
securityManifests: [
{
path: "plugins/fixture/openclaw.security.json",
schema: "https://openclaw.ai/schemas/plugin-security.json",
version: "1.0.0",
plugin: "fixture",
expectedBehaviorCount: 1,
securityNoteCount: 1,
validJson: true,
},
],
package: null,
},
targetOpenClaw: {
@ -442,13 +1039,48 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
assert.ok(result.warnings.some((finding) => finding.code === "provider-auth-env-vars"));
assert.ok(result.warnings.some((finding) => finding.code === "channel-env-vars"));
assert.ok(result.warnings.some((finding) => finding.code === "conversation-access-hook"));
assert.ok(result.warnings.some((finding) => finding.code === "unrecognized-security-manifest"));
assert.ok(result.warnings.some((finding) => finding.code === "security-manifest-schema-unavailable"));
assert.ok(
result.warnings.some(
(finding) =>
finding.code === "conversation-access-hook" &&
finding.compatRecord === "hook.llm-observer.privacy-payload",
),
);
assert.ok(result.warnings.some((finding) => finding.code === "legacy-root-sdk-import"));
assert.ok(result.warnings.some((finding) => finding.code === "package-json-missing"));
assert.ok(result.suggestions.some((finding) => finding.code === "registration-capture-gap"));
assert.ok(result.suggestions.some((finding) => finding.code === "before-tool-call-probe"));
assert.ok(
result.suggestions.some(
(finding) =>
finding.code === "registration-capture-gap" &&
finding.compatRecord === "api.capture.runtime-registrars",
),
);
assert.ok(
result.suggestions.some(
(finding) =>
finding.code === "before-tool-call-probe" &&
finding.compatRecord === "hook.before_tool_call.terminal-block-approval",
),
);
assert.ok(result.suggestions.some((finding) => finding.code === "runtime-tool-capture"));
assert.ok(result.decisions.some((decision) => decision.seam === "conversation-access"));
assert.ok(result.decisions.some((decision) => decision.seam === "security-metadata"));
const issues = buildIssues({
warnings: result.warnings,
suggestions: result.suggestions,
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
});
assert.ok(
issues.some(
(issue) =>
issue.code === "unrecognized-security-manifest" &&
issue.issueClass === "upstream-metadata" &&
issue.severity === "P3",
),
);
});
test("writeReport writes JSON and Markdown artifacts", async () => {
@ -464,6 +1096,44 @@ test("writeReport writes JSON and Markdown artifacts", async () => {
assert.match(markdown, /Status: PASS/);
});
test("CI output helpers write SARIF and JUnit artifacts", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ci-outputs-"));
const report = {
status: "fail",
summary: { fixtureCount: 1, breakageCount: 1 },
fixtures: [{ id: "weather", path: "." }],
breakages: [
{
fixture: "weather",
code: "missing-expected-seam",
level: "breakage",
message: "weather: missing expected registration registerTool",
evidence: ["registerTool @ src/index.js:12:4"],
},
],
warnings: [],
suggestions: [],
issues: [],
};
const sarif = buildSarifReport(report);
const junit = renderJunitXml(report);
const paths = await writeCiOutputArtifacts(report, {
outDir,
sarifPath: "plugin-inspector.sarif",
junitPath: "plugin-inspector.junit.xml",
});
assert.equal(sarif.version, "2.1.0");
assert.equal(sarif.runs[0].results[0].ruleId, "missing-expected-seam");
assert.equal(sarif.runs[0].results[0].level, "error");
assert.equal(sarif.runs[0].results[0].locations[0].physicalLocation.artifactLocation.uri, "src/index.js");
assert.match(junit, /tests="1" failures="1"/);
assert.match(junit, /missing expected registration registerTool/);
assert.equal(JSON.parse(await readFile(paths.sarifPath, "utf8")).runs[0].results.length, 1);
assert.match(await readFile(paths.junitPath, "utf8"), /<testsuite name="plugin-inspector"/);
});
test("artifact helpers write stable CI files", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-artifacts-"));
const jsonPath = path.join(outDir, "summary.json");
@ -496,3 +1166,7 @@ test("markdown table helper supports padded empty-table reports", () => {
);
assert.equal(renderMarkdownTable([], ["Name"], { empty: "_none_" }), "_none_");
});
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

View File

@ -5,6 +5,7 @@ import path from "node:path";
import { test } from "node:test";
import {
buildRuntimeCaptureReport,
captureEntrypoint,
inspectCompatibilityFixtureSet,
loadPluginRootConfig,
writeRuntimeCaptureReport,
@ -61,14 +62,59 @@ test("runtime capture report imports plugin entrypoints with mocked SDK", async
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
});
test("runtime capture report classifies missing mocked SDK exports", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-"));
test("runtime capture records conversation binding resolved callbacks", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-binding-resolved-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-missing-sdk-export",
name: "openclaw-binding-resolved",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.mjs"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "src", "index.mjs"),
[
'import { definePluginEntry } from "openclaw/plugin-sdk";',
"",
"export default definePluginEntry((api) => {",
" api.onConversationBindingResolved(() => undefined);",
"});",
].join("\n"),
"utf8",
);
const config = await loadPluginRootConfig(null, { cwd: rootDir });
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
assert.equal(captureReport.summary.failedCount, 0);
assert.equal(captureReport.summary.capturedCount, 1);
assert.equal(captureReport.summary.hookCount, 1);
assert.deepEqual(
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
["hook:onConversationBindingResolved"],
);
});
test("runtime capture report synthesizes newly imported mocked SDK exports", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-dynamic-sdk-export",
version: "1.0.0",
type: "module",
openclaw: {
@ -86,9 +132,10 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
[
'import { definitelyMissing } from "openclaw/plugin-sdk/plugin-entry";',
"",
"export default definitelyMissing({",
" register() {},",
"});",
"export function register(api) {",
" if (!definitelyMissing) throw new Error('expected dynamic mock export');",
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
"}",
].join("\n"),
"utf8",
);
@ -97,17 +144,18 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
assert.equal(captureReport.summary.failedCount, 1);
assert.equal(captureReport.results[0].status, "error");
assert.equal(captureReport.results[0].failureClass, "missing-sdk-export");
assert.equal(captureReport.results[0].missingExport, "definitelyMissing");
assert.equal(captureReport.summary.failedCount, 0);
assert.equal(captureReport.results[0].status, "captured");
assert.deepEqual(captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`), [
"registration:registerTool",
]);
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-out-"));
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-out-"));
await writeRuntimeCaptureReport(captureReport, {
jsonPath: path.join(outDir, "capture.json"),
markdownPath: path.join(outDir, "capture.md"),
});
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /missing-sdk-export/);
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
});
test("runtime capture report classifies registration execution failures", async () => {
@ -227,6 +275,83 @@ test("runtime capture supports TypeScript entrypoints, SDK subpaths, external mo
assert.match(captureReport.results[0].processOutput.stdout, /late plugin noise/);
});
test("runtime capture synthesizes manifest config before plugin registration", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-config-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-configured-memory",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.ts"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "openclaw.plugin.json"),
`${JSON.stringify(
{
id: "configured-memory",
configSchema: {
type: "object",
properties: {
embedding: {
type: "object",
minProperties: 1,
properties: {
provider: { type: "string" },
model: { type: "string" },
},
},
autoCapture: { type: "boolean" },
autoRecall: { type: "boolean" },
},
required: ["embedding"],
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "src", "index.ts"),
[
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
"export default definePluginEntry({",
" register(api) {",
" if (!api.pluginConfig?.embedding) {",
" api.registerService({ id: 'configured-memory-disabled', start() {} });",
" return;",
" }",
" api.on('agent_end', () => undefined);",
" },",
"});",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("src/index.ts", {
cwd: rootDir,
pluginRoot: rootDir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(
result.captured.map((item) => `${item.kind}:${item.name}`),
["hook:agent_end"],
);
});
test("runtime capture supports namespace imports from mocked externals", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-zod-namespace-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
@ -268,6 +393,134 @@ test("runtime capture supports namespace imports from mocked externals", async (
assert.equal(captureReport.summary.registrationCount, 1);
});
test("runtime capture mock SDK supports config schemas and provider catalogs", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-provider-catalog-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-provider-catalog",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.mjs"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "src", "index.mjs"),
[
'import { buildPluginConfigSchema } from "openclaw/plugin-sdk/plugin-entry";',
'import { buildSingleProviderApiKeyCatalog, createProviderApiKeyAuthMethod, defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";',
"",
"const config = buildPluginConfigSchema({ providerId: { parse: (value) => value ?? 'fixture-provider' } }).parse({});",
"const auth = createProviderApiKeyAuthMethod({ id: 'fixture-key' });",
"const catalog = buildSingleProviderApiKeyCatalog({",
" id: config.providerId,",
" auth,",
" buildModels: async () => [{ id: 'fixture-model' }],",
"});",
"const listed = await catalog.run({ apiKey: 'redacted' });",
"",
"export default defineSingleProviderPluginEntry({",
" id: config.providerId,",
" provider: listed.provider,",
" register(api) {",
" api.registerProvider({ id: listed.provider.id, auth, catalog, modelCount: listed.models.length });",
" },",
"});",
].join("\n"),
"utf8",
);
const config = await loadPluginRootConfig(null, { cwd: rootDir });
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
assert.equal(captureReport.summary.failedCount, 0);
assert.equal(captureReport.summary.registrationCount, 2);
assert.deepEqual(
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
["registration:registerProvider", "registration:registerProvider"],
);
});
test("runtime capture mock SDK supports channel entries, gateway responders, and action gates", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-channel-entry-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-channel-entry",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.mjs"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "src", "index.mjs"),
[
'import { createActionGate } from "openclaw/plugin-sdk/channel-actions";',
'import { buildChannelOutboundSessionRoute, createChannelPluginBase, createChatChannelPlugin, defineChannelPluginEntry } from "openclaw/plugin-sdk/channel-core";',
"",
"const gate = createActionGate({ gateway: true });",
"const route = buildChannelOutboundSessionRoute({ agentId: 'agent', channel: 'fixture-channel', to: 'sender', chatType: 'direct' });",
"const plugin = createChatChannelPlugin({",
" base: createChannelPluginBase({",
" id: 'fixture-channel',",
" setup: () => ({ accountId: 'default' }),",
" }),",
" outbound: {",
" base: { resolveOutboundSessionRoute: () => route },",
" attachedResults: {",
" channel: 'fixture-channel',",
" sendText: async ({ text }) => ({ messageId: text }),",
" },",
" },",
"});",
"",
"export default defineChannelPluginEntry({",
" id: 'fixture-channel-entry',",
" name: 'Fixture channel',",
" description: 'Fixture channel',",
" plugin,",
" registerFull(api) {",
" if (gate('gateway')) {",
" api.registerGatewayMethod('fixture.ping', ({ respond }) => respond(true, { ok: true }), { scope: 'operator.read' });",
" }",
" api.registerService({ id: 'fixture-service', start() {}, stop() {} });",
" },",
"});",
].join("\n"),
"utf8",
);
const config = await loadPluginRootConfig(null, { cwd: rootDir });
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
assert.equal(captureReport.summary.failedCount, 0);
assert.deepEqual(
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
["registration:registerChannel", "registration:registerGatewayMethod", "registration:registerService"],
);
assert.equal(captureReport.results[0].captured[0].arguments[0].keys.includes("plugin"), true);
});
test("runtime capture keeps dist chunk imports rooted at their original package", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-dist-"));
await mkdir(path.join(rootDir, "dist", "extensions", "weather"), { recursive: true });

View File

@ -10,6 +10,7 @@ import {
runCapturedSyntheticProbes,
validateSyntheticProbePlan,
} from "../src/advanced.js";
import { buildSyntheticProbePlanFromReport } from "../src/synthetic-probe-suite.js";
test("synthetic probe plan maps capture inventory to executable probes", () => {
const plan = buildSyntheticProbePlan({
@ -48,6 +49,33 @@ test("synthetic probe plan maps capture inventory to executable probes", () => {
assert.match(renderSyntheticProbeMarkdown(plan), /registerTool/);
});
test("synthetic probe plan can be built from a compatibility report", () => {
const plan = buildSyntheticProbePlanFromReport({
generatedAt: "test",
targetOpenClaw: {
capturedRegistrars: ["registerTool"],
sdkExports: [],
},
summary: {},
fixtures: [
{
id: "fixture",
priority: "high",
hookDetails: [{ name: "before_tool_call", ref: "src/index.js:1" }],
registrationDetails: [{ name: "registerTool", ref: "src/index.js:2" }],
sdkImportDetails: [],
packages: [],
},
],
contractProbes: [],
});
assert.equal(plan.generatedAt, "test");
assert.equal(plan.summary.probeCount, 2);
assert.equal(plan.summary.readyCount, 2);
assert.deepEqual(validateSyntheticProbePlan(plan), []);
});
test("synthetic probe plan blocks unclassified registrars", () => {
const plan = buildSyntheticProbePlan({
capture: {
@ -75,6 +103,85 @@ test("synthetic probe plan blocks unclassified registrars", () => {
assert.match(validateSyntheticProbePlan(plan).join("\n"), /not been classified/);
});
test("synthetic probe plan classifies generated kitchen-sink registrars", () => {
const kitchenSinkRegistrars = [
"createChatChannelPlugin",
"registerAgentEventSubscription",
"registerAgentHarness",
"registerAgentToolResultMiddleware",
"registerAutoEnableProbe",
"registerChannel",
"defineBundledChannelEntry",
"registerCli",
"registerCliBackend",
"registerCodexAppServerExtensionFactory",
"registerCommand",
"registerCompactionProvider",
"registerConfigMigration",
"registerContextEngine",
"registerControlUiDescriptor",
"registerDetachedTaskRuntime",
"registerGatewayDiscoveryService",
"registerGatewayMethod",
"registerHook",
"registerHttpRoute",
"registerImageGenerationProvider",
"registerInteractiveHandler",
"registerMediaUnderstandingProvider",
"registerMemoryCapability",
"registerMemoryCorpusSupplement",
"registerMemoryEmbeddingProvider",
"registerMemoryFlushPlan",
"registerMemoryPromptSection",
"registerMemoryPromptSupplement",
"registerMemoryRuntime",
"registerMigrationProvider",
"registerMusicGenerationProvider",
"registerNodeHostCommand",
"registerNodeInvokePolicy",
"registerProvider",
"registerRealtimeTranscriptionProvider",
"registerRealtimeVoiceProvider",
"registerReload",
"registerRuntimeLifecycle",
"registerSecurityAuditCollector",
"registerService",
"registerSessionExtension",
"registerSessionSchedulerJob",
"registerSpeechProvider",
"registerTextTransforms",
"registerTool",
"registerToolMetadata",
"registerTrustedToolPolicy",
"registerVideoGenerationProvider",
"registerWebFetchProvider",
"registerWebSearchProvider",
];
const plan = buildSyntheticProbePlan({
capture: {
generatedAt: "test",
summary: { fixtureCount: 1 },
fixtures: [
{
id: "kitchen-sink",
hooks: [],
registrations: kitchenSinkRegistrars.map((registrar) => ({
id: `registration.${registrar}:kitchen-sink:index`,
registrar,
ref: "src/generated-registrars.js",
assertions: [`${registrar} is classified`],
syntheticArguments: [{}],
})),
},
],
},
});
assert.equal(plan.summary.probeCount, kitchenSinkRegistrars.length);
assert.equal(plan.summary.blockedCount, 0);
assert.deepEqual(validateSyntheticProbePlan(plan), []);
});
test("synthetic probes invoke retained hook and tool handlers", async () => {
const capture = await captureLocalFixture([
"export function register(api) {",
@ -120,6 +227,57 @@ test("synthetic probes pass registrar-specific handler inputs", async () => {
);
});
test("synthetic probes pass channel envelopes and gateway responders", async () => {
const capture = await captureLocalFixture([
"export function register(api) {",
" api.registerChannel({",
" id: 'fixture_channel',",
" async send(ctx) { return { messageId: ctx.replyToId, to: ctx.to }; },",
" async receive(ctx) { return { messageId: ctx.message.id, peer: ctx.route.peer.id }; },",
" });",
" api.registerGatewayMethod('fixture.ping', ({ respond, params }) => respond(true, { sawParams: typeof params === 'object' }));",
"}",
]);
const blocked = await runCapturedSyntheticProbes(capture);
assert.equal(blocked.summary.blockedCount, 1);
const result = await runCapturedSyntheticProbes(capture, { includeChannelRuntime: true });
assert.equal(result.summary.failCount, 0);
assert.deepEqual(
result.results.map((item) => `${item.status}:${item.label}`),
[
"pass:registerChannel.send",
"pass:registerChannel.receive",
"pass:registerGatewayMethod.handler",
"pass:registerGatewayMethod.run",
"pass:registerGatewayMethod.execute",
],
);
});
test("synthetic probes can execute string plus handler registrations", async () => {
const capture = await captureLocalFixture([
"export function register(api) {",
" api.registerGatewayMethod('fixture.ping', (event) => ({ method: event.registrar, property: event.property }));",
"}",
]);
const result = await runCapturedSyntheticProbes(capture);
assert.equal(result.summary.failCount, 0);
assert.equal(result.summary.blockedCount, 0);
assert.deepEqual(
result.results.map((item) => `${item.status}:${item.label}`),
[
"pass:registerGatewayMethod.handler",
"pass:registerGatewayMethod.run",
"pass:registerGatewayMethod.execute",
],
);
});
test("synthetic probes keep opt-in registrations guarded", async () => {
const capture = await captureLocalFixture([
"export function register(api) {",

View File

@ -22,7 +22,8 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
name: "fixture",
packageManager: "npm@10.0.0",
scripts: { build: "tsup" },
dependencies: { "left-pad": "^1.3.0" },
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
devDependencies: { "@openclaw/plugin-sdk": "workspace:*" },
},
null,
2,
@ -57,10 +58,11 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
assert.equal(plan.summary.artifactStepCount, 2);
assert.equal(plan.summary.installStepCount, 1);
assert.equal(plan.summary.auditStepCount, 1);
assert.equal(plan.summary.pruneDevWorkspaceDependencyStepCount, 1);
assert.equal(plan.summary.buildStepCount, 1);
assert.equal(plan.summary.captureStepCount, 2);
assert.equal(plan.summary.syntheticProbeStepCount, 2);
assert.equal(plan.summary.targetOpenClawLinkStepCount, 2);
assert.equal(plan.summary.targetOpenClawLinkStepCount, 1);
assert.equal(plan.summary.tsLoaderEntrypointCount, 1);
assert.equal(plan.summary.jitiAlternativeCount, 1);
@ -73,7 +75,14 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
assert.ok(entrypoint.requiredCapabilities.includes("sdk-alias-compat"));
assert.ok(entrypoint.requiredCapabilities.includes("ts-loader"));
assert.ok(entrypoint.steps.some((step) => step.kind === "install" && step.command === "npm install --ignore-scripts"));
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node --import tsx capture.mjs")));
assert.ok(
entrypoint.steps.some(
(step) => step.kind === "prune-dev-workspace-deps" && step.command.includes("prune-workspace-dev-deps-cli.js"),
),
);
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node capture.mjs")));
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("--mock-sdk")));
assert.ok(entrypoint.steps.every((step) => !step.command.includes("--import tsx")));
assert.ok(entrypoint.steps.some((step) => step.kind === "synthetic-probe" && step.command.includes("synthetic.mjs")));
const buildEntrypoint = plan.fixtures[0].entrypoints.find((item) => item.packageName === "build-fixture");
assert.ok(buildEntrypoint);
@ -94,7 +103,7 @@ test("workspace plan defaults point at packaged helper wrappers", async (t) => {
name: "fixture",
packageManager: "npm@10.0.0",
scripts: { build: "tsup" },
dependencies: { "left-pad": "^1.3.0" },
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
},
null,
2,
@ -226,7 +235,7 @@ function readinessReport() {
{
path: "plugins\\fixture\\package.json",
name: "fixture",
dependencies: ["left-pad"],
dependencies: ["left-pad", "openclaw"],
peerDependencies: [],
optionalDependencies: [],
openclaw: {