Compare commits

...

36 Commits
v0.3.1 ... main

Author SHA1 Message Date
Vincent Koc
8899fc796c
fix(reports): accept packaged runtime entrypoints
Some checks failed
Check / Node 22 (push) Has been cancelled
2026-05-06 02:46:10 -07:00
Vincent Koc
feefb4ee23
fix(contract): accept compat-gap coverage records
Some checks failed
Check / Node 22 (push) Has been cancelled
Treat compat-gap issues as reconciliation evidence for their own compatibility records.
2026-05-05 00:50:58 -07:00
Vincent Koc
f642fb5c9f
fix(reports): quiet resolved package entrypoint P0s
Fix compatibility report classification so built runtime package entrypoints satisfy source-form OpenClaw metadata, SDK export alias misses collapse into a single compat-gap row, and P0 live issues are not repeated under the general live section.
2026-05-05 00:34:18 -07:00
Vincent Koc
68e10e0aaa
chore(release): prepare 0.3.10
Some checks failed
Check / Node 22 (push) Has been cancelled
2026-05-03 01:13:33 -07:00
Vincent Koc
12005b4658
fix(capture): accept valid mocked capture output 2026-05-03 01:09:52 -07:00
Vincent Koc
4956ad1fbc
fix(capture): follow bundled channel exports 2026-05-03 00:59:42 -07:00
Vincent Koc
a58e0785d5
fix(capture): synthesize manifest plugin config 2026-05-03 00:11:24 -07:00
Vincent Koc
9f45c8aeb6
fix(report): downgrade registration capture gaps 2026-05-02 22:43:55 -07:00
Vincent Koc
677f6e5bc1
chore(release): prepare 0.3.6 2026-05-02 19:08:49 -07:00
Vincent Koc
06cc55ce51
fix(report): accept min host version floors (#17)
* fix(report): accept min host version floors

* fix(inspector): treat bundled channel entries as channel coverage

* fix(inspector): classify bundled channel probes

* fix(policy): allow wildcard seam rules
2026-05-02 18:20:13 -07:00
Vincent Koc
2eda65a8a9
fix(report): flag unsupported openclaw bundle metadata
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-05-02 11:31:30 -07:00
Vincent Koc
4bc5fbcfa3
feat(report): reconcile runtime capture evidence 2026-05-02 11:22:20 -07:00
Vincent Koc
a6af8800e0
fix(runtime): harden plugin capture mocks 2026-05-02 10:51:35 -07:00
Vincent Koc
b919df78d3
fix(reports): flag advertised npm pack blockers 2026-05-02 09:10:57 -07:00
Vincent Koc
ff78dccff7
fix(probes): classify kitchen sink registrars 2026-05-02 08:48:39 -07:00
Vincent Koc
b33c6f725d
fix(reports): surface plugin install metadata 2026-05-02 08:40:30 -07:00
Vincent Koc
e38991a35f
fix(security): harden inspector sanitizers
Some checks failed
Check / Node 22 (push) Has been cancelled
2026-04-30 02:54:02 -07:00
Vincent Koc
eb251cbae4
feat: report openclaw lifecycle timings 2026-04-29 19:38:38 -07:00
Vincent Koc
cc89d7cea7
feat: baseline import-loop profile metrics 2026-04-29 19:26:46 -07:00
Vincent Koc
1ee105e29d
feat: flag unsupported security manifests 2026-04-29 19:18:16 -07:00
Vincent Koc
572956b8df
chore(release): v0.3.5 2026-04-29 13:51:02 -07:00
Vincent Koc
e8fb9b4380
fix(profile): expose sampler availability
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-29 13:17:17 -07:00
Vincent Koc
7b5f706398
fix(inspector): accept channel factory seams 2026-04-29 06:32:59 -07:00
Vincent Koc
862d8c9fb8
fix(synthetic): classify chat channel plugin factory 2026-04-29 06:17:27 -07:00
Vincent Koc
f6991de9b0
docs(agents): clarify host-linked openclaw dependencies
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-28 22:07:49 -07:00
Vincent Koc
e04c3fc121
chore(release): prepare 0.3.4 2026-04-28 21:51:17 -07:00
Vincent Koc
85c69fb24d
fix(platform): separate covered portability risks 2026-04-28 21:39:27 -07:00
Vincent Koc
9ab07bb316
fix(reports): treat openclaw as host-linked dependency 2026-04-28 21:34:02 -07:00
Vincent Koc
d91d596d90
fix(reports): link probe gaps to compat records 2026-04-28 21:13:15 -07:00
Vincent Koc
332706b014
fix(reports): sanitize OpenClaw target paths 2026-04-28 18:32:50 -07:00
Vincent Koc
e9e4b6704c
fix(synthetic): classify kitchen-sink registrars
Some checks are pending
Check / Node 22 (push) Waiting to run
2026-04-28 13:24:09 -07:00
Vincent Koc
98e5d775f8
chore(release): prepare 0.3.2 2026-04-28 02:26:34 -07:00
Patrick Erichsen
768c3fc4d6
Merge pull request #16 from openclaw/codex/capture-binding-resolved
[codex] fix runtime capture binding callbacks
2026-04-28 02:17:08 -07:00
Patrick Erichsen
41a47e9a18 fix runtime capture binding callbacks 2026-04-28 02:16:05 -07:00
Vincent Koc
5ea07a8fca
docs: rewrite plugin-inspector README 2026-04-28 00:45:47 -07:00
Vincent Koc
85f360548e
docs(readme): clarify plugin inspector onboarding 2026-04-27 22:31:34 -07:00
52 changed files with 3830 additions and 324 deletions

View File

@ -6,6 +6,9 @@
- Do not publish npm packages without explicit owner approval.
- Preserve stable report field names and finding codes; downstream CI and
crabpot reports may consume them.
- Treat a package dependency named `openclaw` as a host-linked workspace input,
not an isolated dependency-install blocker. Keep third-party runtime
dependencies classified as install/audit blockers.
- When changing plugin-inspector behavior, CLI/package entrypoints, release
metadata, or the npm package version, update crabpot's
`@openclaw/plugin-inspector` pin/docs/smoke path as needed and run the

View File

@ -2,6 +2,78 @@
## Unreleased
### Fixed
- Stop classifying package source entrypoints as missing when the published package provides built runtime entrypoints, and collapse SDK alias findings into a single compat-gap row.
- Treat compat-gap issues as reconciled contract coverage for their own compatibility record.
## 0.3.10 - 2026-05-03
### Fixed
- Accept valid mocked capture output when plugin code leaves `process.exitCode` dirty.
## 0.3.9 - 2026-05-03
### Fixed
- Follow bundled channel `loadBundledEntryExportSync` registration exports during mocked runtime capture.
## 0.3.8 - 2026-05-03
### Fixed
- Synthesize manifest config for isolated runtime capture so configured hooks can be observed without credentials.
## 0.3.7 - 2026-05-03
### Changed
- Downgrade `registration-capture-gap` to advisory severity so missing capture evidence no longer reports as a P1 plugin contract risk.
## 0.3.6 - 2026-05-03
### Changed
- Report import-loop RSS and CPU as baseline-adjusted plugin deltas alongside raw subprocess metrics so Crabpot dashboards do not treat harness import cost as plugin runtime cost.
- Include optional OpenClaw loader lifecycle timings for import and full activation when a capture runner provides them.
### Fixed
- Accept plugin install minimum-host floors as supported package metadata.
- Flag unsupported legacy OpenClaw bundle metadata and advertised npm pack blockers.
- Reconcile runtime capture evidence and harden mocked capture paths for downstream fixture reports.
## 0.3.5 - 2026-04-29
### Fixed
- Add immediate/faster subprocess RSS and CPU sampling plus explicit sample counts so short import-loop reports do not silently publish fake zero-memory metrics.
- Classify `createChatChannelPlugin` as channel factory metadata in synthetic probe plans so channel-core plugins do not fail as unknown registrars.
- Treat `createChatChannelPlugin` and `defineChannelPluginEntry` as channel registration equivalents when validating fixture expectations.
- Label runtime profile wall-time summaries as command-median p95 and render missing sampled metrics as `n/a`.
## 0.3.4 - 2026-04-29
### Fixed
- Separate executor-covered platform portability findings from residual findings so downstream structured runners can keep reports blocking only on unhandled risks.
- Sanitize absolute target OpenClaw paths from generated report artifacts and JSON CLI output.
- Normalize the dependency-install inspector finding title to use isolated-workspace wording.
- Treat `openclaw` package dependencies as host-linked workspace inputs instead of isolated dependency-install blockers.
## 0.3.3 - 2026-04-28
### Fixed
- Classify generated kitchen-sink public registrar coverage in synthetic probe plans so new API-surface fixtures do not fail as unknown execution profiles.
## 0.3.2 - 2026-04-28
### Fixed
- Preserve runtime capture bindings for callback-based registrations so captured hooks and registrations can resolve their bound callback metadata.
## 0.3.1 - 2026-04-28
### Added

489
README.md
View File

@ -1,51 +1,74 @@
<img src="docs/plugin-inspector-banner.jpg" alt="openclaw plugin inspector banner"/>
<img src="docs/plugin-inspector-banner.jpg" alt="OpenClaw Plugin Inspector banner">
# OpenClaw Plugin Inspector
`plugin-inspector` is the offline compatibility check for OpenClaw plugins. Run
it from a plugin root to inspect package metadata, `openclaw.plugin.json`, SDK
imports, `api.on(...)`, `api.register*`, and optional runtime registration
capture.
`@openclaw/plugin-inspector` is the offline compatibility checker for OpenClaw
plugin packages and plugin fixture suites.
It answers the questions that matter before a plugin reaches users:
- can OpenClaw discover the package metadata and `openclaw.plugin.json`
manifest?
- which hooks, registration calls, manifest contracts, and SDK imports does the
plugin use?
- does the plugin still look compatible without local OpenClaw internals?
- if CI finds a breakage, which JSON, Markdown, SARIF, JUnit, and summary
artifacts should downstream automation read?
- when a fixture-suite harness such as Crabpot runs many plugins, which findings
are hard breakages, known warnings, live issues, deprecations, or inspector
proof gaps?
The default path is static, offline, and credential-free. Runtime capture exists,
but it is opt-in because it imports plugin code.
## Requirements
- Node.js 22 or newer.
- A plugin package root with `package.json`.
- `openclaw.plugin.json` when the plugin uses the OpenClaw manifest contract.
- No OpenClaw checkout, credentials, network service, or live provider access for
default inspection.
Pass `--no-openclaw` when CI should not compare against a local OpenClaw
checkout. If an OpenClaw checkout is supplied with `--openclaw <path>`, the
inspector only reads public compatibility surfaces such as compat records, SDK
exports, hook names, manifest fields, and registrar metadata.
## Quick Start
From a plugin package directory:
Run this from a plugin package root:
```bash
npx @openclaw/plugin-inspector
npx @openclaw/plugin-inspector inspect --no-openclaw
```
That runs `check`, writes report artifacts to `reports/`, and exits non-zero
when compatibility breakages are found.
Package-manager equivalents:
Equivalent one-off runners:
```bash
pnpm dlx @openclaw/plugin-inspector
yarn dlx @openclaw/plugin-inspector
bunx @openclaw/plugin-inspector
pnpm dlx @openclaw/plugin-inspector inspect --no-openclaw
yarn dlx @openclaw/plugin-inspector inspect --no-openclaw
bunx @openclaw/plugin-inspector inspect --no-openclaw
```
Add a local config and GitHub Actions workflow:
The command writes:
```bash
npx @openclaw/plugin-inspector init --ci
```
- `reports/plugin-inspector-report.json`
- `reports/plugin-inspector-report.md`
- `reports/plugin-inspector-issues.md`
`init --ci` detects `packageManager` and common lockfiles. Pass
`--package-manager pnpm`, `npm`, `yarn`, or `bun` when you want to override it.
Add `--scripts` to write `plugin:check` and `plugin:ci` package scripts.
Use `--dry-run` to preview the files first.
It exits non-zero when hard compatibility breakages are found. Warnings,
suggestions, issue classifications, and logs stay visible in the report without
necessarily failing the command.
Or install it as a dev dependency:
## Install In A Plugin Repo
Install the package when you want repeatable local scripts and CI:
```bash
npm install --save-dev @openclaw/plugin-inspector
npx plugin-inspector check
```
With a local dev dependency, prefer package scripts so CI and local checks use
the same command:
Add scripts:
```json
{
@ -56,55 +79,53 @@ the same command:
}
```
## Commands
Then run:
```bash
npm run plugin:check
```
The initializer can write the starter config, package scripts, and GitHub
Actions workflow:
```bash
npx @openclaw/plugin-inspector check
npx @openclaw/plugin-inspector inspect
npx @openclaw/plugin-inspector ci --no-openclaw
npx @openclaw/plugin-inspector config
npx @openclaw/plugin-inspector check --plugin-root ./plugins/weather
npx @openclaw/plugin-inspector init --ci --package-manager pnpm
npx @openclaw/plugin-inspector init --ci --scripts --dry-run
npx @openclaw/plugin-inspector init --ci --scripts --dry-run --json
npx @openclaw/plugin-inspector init --ci --scripts
```
`check` and `inspect` read the current directory as one plugin unless
`--plugin-root` is set. `inspect` is the friendly author-facing alias; `check`
is kept for scripts. Both write:
`init` detects `packageManager` and common lockfiles. Override that with
`--package-manager npm`, `--package-manager pnpm`, `--package-manager yarn`, or
`--package-manager bun`. Existing files are protected unless you pass `--force`.
- `reports/plugin-inspector-report.json`
- `reports/plugin-inspector-report.md`
- `reports/plugin-inspector-issues.md`
## Configuration
Use CI-native outputs when you want annotations or test-summary ingestion:
Small plugin repos can keep configuration in `package.json`:
```bash
plugin-inspector inspect --no-openclaw --sarif --junit
```json
{
"scripts": {
"plugin:check": "plugin-inspector inspect --no-openclaw",
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
},
"pluginInspector": {
"version": 1,
"plugin": {
"id": "weather",
"priority": "high",
"seams": ["dynamic-tool"],
"sourceRoot": "src",
"expect": {
"registrations": ["registerTool"]
}
},
"capture": {
"mockSdk": true
}
}
}
```
That also writes:
- `reports/plugin-inspector.sarif`
- `reports/plugin-inspector.junit.xml`
`config` prints the resolved plugin root, fixture id, seams, and capture
settings before CI runs:
```bash
plugin-inspector config --json
```
Use `--no-openclaw` when CI should not compare against a local OpenClaw
checkout:
```bash
plugin-inspector check --no-openclaw
```
Use either `plugin-inspector.config.json` or a `package.json`
`pluginInspector` block when CI needs stable fixture metadata, expected seams,
or runtime capture defaults:
Use `plugin-inspector.config.json` for a standalone config file:
```json
{
@ -127,66 +148,88 @@ or runtime capture defaults:
}
```
Then run:
Inspect the resolved config before wiring CI:
```bash
plugin-inspector check --config plugin-inspector.config.json
plugin-inspector config --json
```
For a single plugin package, the same config can live in `package.json`:
Copy-ready examples live in:
```json
{
"scripts": {
"plugin:check": "plugin-inspector inspect --no-openclaw"
},
"pluginInspector": {
"version": 1,
"plugin": {
"id": "weather",
"priority": "high",
"seams": ["dynamic-tool"],
"sourceRoot": "src"
},
"capture": {
"mockSdk": true
}
}
}
- `examples/plugin-inspector.config.json`
- `examples/package-json-plugin-inspector.json`
## Commands
| Command | Purpose |
| --- | --- |
| `plugin-inspector` | Default alias for `check`. |
| `plugin-inspector check` | Script-friendly plugin-root check. |
| `plugin-inspector inspect` | Plugin-root check unless `--config` is supplied; with `--config`, runs a fixture report. |
| `plugin-inspector ci` | Compatibility report plus CI summary, SARIF, and JUnit outputs. |
| `plugin-inspector config` | Print resolved plugin-root config as text or JSON. |
| `plugin-inspector init` | Write starter config, scripts, and optional GitHub Actions workflow. |
| `plugin-inspector report` | Run a fixture-suite config with many plugins. |
| `plugin-inspector capture` | Runtime-capture one entrypoint directly. |
Common options:
| Option | Meaning |
| --- | --- |
| `--plugin-root <path>` / `--root <path>` | Check a plugin somewhere other than the current directory. |
| `--config <path>` | Read a standalone config file. Required for fixture-suite `report`. |
| `--out <dir>` | Write reports somewhere other than `reports/`. |
| `--openclaw <path>` | Compare against a local OpenClaw checkout. |
| `--no-openclaw` | Disable OpenClaw checkout comparison. |
| `--runtime` / `--capture` | Add opt-in runtime registration capture. |
| `--no-runtime` / `--no-capture` | Disable runtime capture even when config enables it. |
| `--mock-sdk` / `--sdk mock` | Use generated SDK and external-package mocks for runtime capture. |
| `--real-sdk` / `--sdk real` | Use installed real SDK dependencies instead of mocks. |
| `--allow-execute` | Permit commands that import plugin code. |
| `--json` | Print machine-readable JSON to stdout. |
| `--sarif [path]` | Write SARIF from `check` or `inspect`; `ci` enables this by default. |
| `--junit [path]` | Write JUnit XML from `check` or `inspect`; `ci` enables this by default. |
| `--no-sarif` / `--no-junit` | Disable default `ci` outputs. |
Run the built-in help for the exact CLI surface:
```bash
plugin-inspector --help
```
`init --ci` writes this shape for you, plus
`.github/workflows/plugin-inspector.yml`. Copy-ready examples also live in
`examples/plugin-inspector.config.json` and
`examples/package-json-plugin-inspector.json` and
`examples/github-actions-plugin-inspector.yml`. SARIF/JUnit CI consumption
examples live alongside them.
## Runtime Capture
Runtime capture imports plugin entrypoints in an isolated subprocess and records
the registrations made during `register(api)`. It is opt-in because it executes
plugin code:
what `register(api)` does. Use it when static inspection cannot prove the actual
registrations made at runtime.
```bash
npx @openclaw/plugin-inspector check --runtime --mock-sdk --allow-execute
plugin-inspector inspect --no-openclaw --runtime --mock-sdk --allow-execute
```
`--allow-execute` is the explicit guard for modes that import plugin code. The
older `PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1` environment guard still works for
custom harnesses.
`--allow-execute` is the deliberate safety switch. Without it, modes that import
plugin code fail closed. The older environment guard still works for custom
harnesses:
By default, runtime capture uses a generated mock for `openclaw/plugin-sdk` and
common external packages so plugin code can load in clean CI without OpenClaw
installed. Use `--real-sdk` only when the plugin workspace already has real SDK
dependencies installed and you intentionally want to test that path.
```bash
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector inspect --no-openclaw --runtime --mock-sdk
```
By default, runtime capture uses generated mocks for `openclaw/plugin-sdk`
subpaths and unresolved external packages discovered in the plugin import graph.
That keeps compatibility CI offline and credential-free. It does not call live
services, launch OpenClaw, run provider SDKs, or emulate service lifecycle side
effects.
Use `--real-sdk` only when the plugin workspace already has real SDK
dependencies installed and you intentionally want that path.
Runtime capture writes:
- `reports/plugin-inspector-runtime-capture.json`
- `reports/plugin-inspector-runtime-capture.md`
You can also capture one entrypoint directly:
Capture one entrypoint directly:
```bash
plugin-inspector capture ./dist/index.js --mock-sdk --allow-execute
@ -194,18 +237,10 @@ plugin-inspector capture ./dist/index.js --mock-sdk --allow-execute
## CI
Minimal package scripts:
`plugin-inspector ci` writes the normal compatibility report plus CI-native
summary, SARIF, and JUnit artifacts.
```json
{
"scripts": {
"plugin:check": "plugin-inspector inspect --no-openclaw",
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
}
}
```
GitHub Actions without a local dev dependency:
Minimal GitHub Actions workflow:
```yaml
name: plugin-inspector
@ -233,70 +268,214 @@ jobs:
path: reports/plugin-inspector-*
```
`ci` writes the normal report, CI summary, SARIF, and JUnit files by default.
Pass `--no-sarif` or `--no-junit` only if your CI surface cannot consume them.
Generated `ci` artifacts:
For GitHub code scanning, use
`examples/github-actions-code-scanning.yml`; it uploads
`reports/plugin-inspector.sarif` through CodeQL's SARIF upload action.
- `reports/plugin-inspector-report.json`
- `reports/plugin-inspector-report.md`
- `reports/plugin-inspector-issues.md`
- `reports/plugin-inspector-ci-summary.json`
- `reports/plugin-inspector-ci-summary.md`
- `reports/plugin-inspector.sarif`
- `reports/plugin-inspector.junit.xml`
For CI test-summary UIs, point JUnit ingestion at
`reports/plugin-inspector.junit.xml`. Copy-ready GitLab and CircleCI examples
live in `examples/gitlab-ci-plugin-inspector.yml` and
`examples/circleci-plugin-inspector.yml`.
CI examples:
- `examples/github-actions-plugin-inspector.yml`
- `examples/github-actions-code-scanning.yml`
- `examples/gitlab-ci-plugin-inspector.yml`
- `examples/circleci-plugin-inspector.yml`
## Report Surfaces
The compatibility report is the primary contract. Preserve field names and
finding codes because downstream CI and Crabpot reports may consume them.
Important report sections:
| Field | Meaning |
| --- | --- |
| `status` | `pass` unless hard breakages exist. |
| `summary` | Counts for fixtures, breakages, warnings, suggestions, issues, issue classes, and contract probes. |
| `targetOpenClaw` | Status and public compatibility data read from the optional OpenClaw checkout. |
| `fixtures` | Per-plugin metadata, hooks, registrations, manifest contracts, package data, and SDK imports. |
| `breakages` | Blocking compatibility failures. |
| `warnings` / `suggestions` | Non-blocking compatibility findings. |
| `issues` | Normalized issue rows with severity and class. |
| `contractProbes` | Suggested synthetic probes derived from observed contracts. |
| `logs` | Informational inventory and coverage rows. |
| `decisions` | Maintainer-facing follow-up or compatibility-policy decisions. |
Issue classes currently flow through the reports as live issues, compat gaps,
deprecation warnings, inspector gaps, upstream metadata, and fixture regressions.
## CI Policy And Shared Reporting Primitives
`plugin-inspector` owns the shared CI policy and report rendering primitives.
Fixture-suite harnesses such as Crabpot should call these exports instead of
reimplementing scoring, summaries, Markdown, SARIF, or JUnit handling.
The root API exposes grouped helpers:
```js
import { ci } from "@openclaw/plugin-inspector";
const policyReport = ci.buildPolicyReport({
policy,
compatibilityReport,
executionResults,
strict: false,
});
await ci.writePolicyReport(policyReport);
```
CI policy reports default to:
- `reports/plugin-inspector-ci-policy.json`
- `reports/plugin-inspector-ci-policy.md`
A policy must use `version: 1` and define:
- `allowedBlocked`
- `expectedWarnings`
- `thresholds`
- `fixtureSets`
Policy scoring fails hard breakages, unknown blocked synthetic probes, hard ref
diff regressions, failed execution results, strict live P0 issues, and strict
classified blockers. Non-strict mode keeps classified blocked probes and live P0
issues visible as warnings.
CI summary helpers read the known report set from `reports/` and render one
machine-readable and one Markdown rollup:
- compatibility
- runtime capture
- synthetic probes
- cold import readiness
- workspace plan
- platform probes
- import-loop profile
- execution results
- runtime profile
- ref diff
- profile diff
- CI policy
## Fixture Suites
Fixture-set configs are still supported for crabpot-style compatibility suites:
Most plugin authors should use the plugin-root workflow. Use fixture suites when
one repository intentionally checks many plugins or packages, as Crabpot does.
```bash
plugin-inspector report --config crabpot.config.json --out reports
plugin-inspector report --config crabpot.config.json --out reports --check
plugin-inspector ci --config crabpot.config.json --out reports --no-openclaw
```
Use fixture suites when one repo wants to inspect many plugins. Use plugin-root
`check` for normal plugin CI.
Fixture-suite configs are loaded through the explicit fixture helpers. That keeps
normal plugin-root configuration simple while still supporting bulk compatibility
harnesses.
## Library Use
## Public API
Most plugin repos should use the CLI. Harnesses can import grouped root helpers
when they need to embed the inspector:
Prefer the CLI for normal plugin repositories. Import the public API when a test
harness needs to compose workflows directly:
```js
import { pluginRoot, fixtureSuites, contracts, ci } from "@openclaw/plugin-inspector";
import { pluginRoot } from "@openclaw/plugin-inspector";
const { report } = await pluginRoot.runCheck({
const { report, paths } = await pluginRoot.runCheck({
pluginRoot: process.cwd(),
openclawPath: false,
outDir: "reports",
});
const capture = contracts.buildCapture({ report });
const summary = await ci.buildSummary({ reports: { compatibility: report } });
console.log(report.status, paths.jsonPath);
```
The root package groups stable workflows as `pluginRoot`, `fixtureSuites`,
`staticInspection`, `reports`, `contracts`, `ci`, `runtime`, and `synthetic`.
Named exports remain available for existing automation.
Stable grouped facades:
## Mocking Model
| Facade | Use |
| --- | --- |
| `pluginRoot` | Load config, inspect, run checks, capture entrypoints, or set up a plugin repo. |
| `fixtureSuites` | Load fixture-suite configs, run reports, and build fixture-suite readiness plans. |
| `staticInspection` | Inspect source text or fixture sets without the compatibility report layer. |
| `reports` | Render/write reports and classify issue findings. |
| `contracts` | Build, render, validate, and write contract captures and coverage. |
| `ci` | Build summaries, policy reports, execution results, SARIF, and JUnit outputs. |
| `runtime` | Build runtime profiles, profile diffs, ref diffs, and import-loop profiles. |
| `synthetic` | Build and run synthetic probe plans. |
Default inspection is static, offline, and credential-free. Runtime capture is
the only mode that imports plugin code.
Named exports remain available for existing automation. Prefer the grouped
facades for new code because they show ownership and keep downstream wrappers
thin.
When `--mock-sdk` is enabled, the inspector generates temporary modules for
`openclaw/plugin-sdk` subpaths and unresolved external packages discovered in
the plugin import graph. The mock SDK captures registrations; it does not call
network services, launch OpenClaw, run provider SDKs, or emulate service
lifecycle side effects.
## Development
Use the mock lane for plugin compatibility CI. Keep live provider/service tests
in the plugin repo behind their own credentials and explicit opt-in flags.
Repository checks are intentionally small and offline:
## Scope
```bash
npm test
npm run release:contents
npm run check
```
Default inspection is offline and credential-free. It reads manifests, package
metadata, and source files, then reports observed `api.on(...)`,
`api.register*`, `define*`, SDK imports, and manifest contracts.
OpenClaw target checkout parsing is limited to public compatibility registries,
SDK package exports, manifest types, hooks, and captured registrar metadata.
`npm run check` runs the Node test suite and the package-contents guard. The
contents guard shells through `npm pack --dry-run --json` and verifies the npm
tarball includes package entrypoints, examples, README assets, and no private
`test/`, `scripts/`, or `.github/` paths.
Cold import capture, synthetic contract probes, and runtime capture are explicit
opt-in modes. Live lanes stay credential-gated and must never run in default CI.
Useful release-prep commands:
```bash
npm run release:local
npm run release:readiness
npm run release:notes
npm run release:plan
npm run release:crabpot -- --crabpot ../crabpot
```
`release:readiness` proves the local package and verifies Crabpot follow-through.
It does not publish.
Keep this package dependency-light. Do not add runtime dependencies unless they
remove real complexity. Default checks must stay offline and credential-free.
## Release Notes
The package publishes from annotated `v*` tags through GitHub Actions. The
release workflow runs the test suite, verifies the npm tarball, publishes the
GitHub release, and publishes the public npm package through npm trusted
publishing.
Before tagging a release:
1. Move `CHANGELOG.md` `Unreleased` notes into a versioned section.
2. Update `package.json` to the same version.
3. Update Crabpot's `pluginInspectorRef` to the release commit.
4. Run `npm run release:readiness`.
5. Run the Crabpot plugin-inspector smoke commands printed by
`npm run release:crabpot -- --crabpot ../crabpot`.
After npm publish, update Crabpot's package pin and run:
```bash
npm run release:crabpot -- --crabpot ../crabpot --published
```
Do not publish npm packages without explicit owner approval.
## Contribution Notes
There is no `CONTRIBUTING.md` in this repository. Until one exists, use the repo
scripts above as the local contract and follow these project rules:
- preserve stable report field names and finding codes;
- prefer public OpenClaw plugin contracts over core internals;
- isolate any OpenClaw source parsing behind explicit helpers;
- keep runtime execution behind `--allow-execute` or
`PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1`;
- when behavior, entrypoints, release metadata, or the npm package version
change, update Crabpot's `@openclaw/plugin-inspector` pin/docs/smoke path and
run the Crabpot plugin-inspector smoke before calling the work done.

View File

@ -1,6 +1,6 @@
{
"name": "@openclaw/plugin-inspector",
"version": "0.3.1",
"version": "0.3.10",
"private": false,
"description": "Offline compatibility inspector for OpenClaw plugins.",
"type": "module",

View File

@ -43,6 +43,7 @@ export {
} from "./ci-outputs.js";
export {
buildContractProbes,
compatRecordForIssueCode,
contractProbeRules,
probePriority,
} from "./contract-probes.js";
@ -94,6 +95,7 @@ export {
classifyTargetOpenClawCoverage,
readPackageSummaries,
readPluginManifests,
readSecurityManifests,
summarizePackage,
} from "./fixture-summary.js";
export {
@ -170,6 +172,7 @@ export {
classifyCompatRecordCoverage,
renderMarkdownReport,
renderTextSummary,
sanitizeReportArtifact,
writeCompatibilityReport,
writeReport,
} from "./report.js";
@ -181,6 +184,10 @@ export {
validateRuntimeProfile,
writeRuntimeProfile,
} from "./runtime-profile.js";
export {
applyRuntimeExecutionCoverage,
buildRuntimeExecutionCoverage,
} from "./runtime-reconciliation.js";
export {
buildRuntimeCaptureReport,
renderRuntimeCaptureMarkdown,

View File

@ -45,6 +45,7 @@ export async function inspectPluginRoot(options = {}) {
return inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
});
}
@ -59,6 +60,7 @@ export async function inspectCompatibilityFixtureSetConfig(options = {}) {
return inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
});
}
@ -112,6 +114,7 @@ export async function buildFixtureSetColdImportReadiness(options = {}) {
(await inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
}));
@ -143,6 +146,7 @@ export async function buildFixtureSetWorkspacePlan(options = {}) {
(await inspectCompatibilityFixtureSet(config, {
generatedAt: options.generatedAt,
openclawPath: options.openclawPath,
executionResults: options.executionResults,
targetOpenClaw: options.targetOpenClaw,
}));
const rootDir = options.rootDir ?? config?.rootDir ?? options.cwd;

View File

@ -95,7 +95,7 @@ export function renderPaddedMarkdownTable(rows, headers, options = {}) {
}
export function escapeMarkdownTableCell(value) {
return value.replace(/\|/g, "\\|").replace(/\n/g, "<br>");
return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, "<br>");
}
async function assertFileMatches(filePath, expected) {

View File

@ -82,6 +82,24 @@ export function createCaptureApi(options = {}) {
}
return api;
},
onConversationBindingResolved(handler) {
const captureIndex =
captured.push({
kind: "hook",
name: "onConversationBindingResolved",
handlerType: typeof handler,
arguments: summarizeArguments([handler]),
}) - 1;
if (retainHandlers) {
retained.push({
kind: "hook",
name: "onConversationBindingResolved",
handler,
captureIndex,
});
}
return api;
},
},
{
get(target, property) {
@ -130,6 +148,7 @@ export function createCaptureContext(options = {}) {
config: options.config ?? {},
logger: options.logger ?? console,
pluginConfig: options.pluginConfig ?? {},
resolvePath: options.resolvePath ?? ((value) => value),
runtime: options.runtime ?? createRuntimeContext(options),
secrets: options.secrets ?? createSecretContext(options),
store: options.store ?? createStoreContext(options),

126
src/capture-config.js Normal file
View File

@ -0,0 +1,126 @@
import { readFile } from "node:fs/promises";
import path from "node:path";
export async function captureApiOptionsForPlugin(apiOptions = {}, options = {}) {
if (apiOptions.pluginConfig !== undefined || !options.pluginRoot) {
return apiOptions;
}
const pluginConfig = await readSamplePluginConfig(options.pluginRoot);
if (pluginConfig === undefined) {
return apiOptions;
}
return {
...apiOptions,
pluginConfig,
};
}
async function readSamplePluginConfig(pluginRoot) {
const manifestPath = path.join(pluginRoot, "openclaw.plugin.json");
let manifest;
try {
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
} catch {
return undefined;
}
const sample = sampleJsonSchema(manifest.configSchema, { key: "config" });
return isPlainObject(sample) && Object.keys(sample).length > 0 ? sample : undefined;
}
function sampleJsonSchema(schema, context = {}) {
if (!isPlainObject(schema)) {
return undefined;
}
if (Array.isArray(schema.enum) && schema.enum.length > 0) {
return schema.enum[0];
}
if (Object.prototype.hasOwnProperty.call(schema, "const")) {
return schema.const;
}
if (Object.prototype.hasOwnProperty.call(schema, "default")) {
return schema.default;
}
const type = Array.isArray(schema.type) ? schema.type.find((item) => item !== "null") : schema.type;
if (type === "object" || schema.properties) {
return sampleObjectSchema(schema);
}
if (type === "array") {
return [];
}
if (type === "boolean") {
return false;
}
if (type === "number" || type === "integer") {
return typeof schema.minimum === "number" ? schema.minimum : 1;
}
if (type === "string" || !type) {
return sampleString(context.key);
}
return undefined;
}
function sampleObjectSchema(schema) {
const properties = isPlainObject(schema.properties) ? schema.properties : {};
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
const output = {};
for (const key of Object.keys(properties)) {
if (required.has(key)) {
const value = sampleJsonSchema(properties[key], { key });
if (value !== undefined) {
output[key] = value;
}
}
}
for (const key of Object.keys(properties)) {
if (properties[key]?.type === "boolean") {
output[key] = false;
}
}
if (Object.keys(output).length === 0 && Number(schema.minProperties ?? 0) > 0) {
const key = preferredSamplePropertyKey(properties);
if (key) {
const value = sampleJsonSchema(properties[key], { key });
if (value !== undefined) {
output[key] = value;
}
}
}
return output;
}
function preferredSamplePropertyKey(properties) {
for (const key of ["provider", "model", "apiKey", "id", "name", ...Object.keys(properties)]) {
if (Object.prototype.hasOwnProperty.call(properties, key)) {
return key;
}
}
return null;
}
function sampleString(key = "") {
if (key === "provider") {
return "openai";
}
if (key === "model") {
return "text-embedding-3-small";
}
if (key === "apiKey") {
return "fixture-api-key";
}
if (key === "dbPath") {
return ".plugin-inspector/state/lancedb";
}
return "fixture";
}
function isPlainObject(value) {
return value !== null && typeof value === "object" && !Array.isArray(value);
}

View File

@ -241,7 +241,9 @@ function executionChecks(executionResults, policy, options) {
}
function findPolicyMatch(rules, item) {
return rules.find((rule) => item.seam === rule.seam && item.reason?.includes(rule.reasonIncludes));
return rules.find(
(rule) => (rule.seam === "*" || item.seam === rule.seam) && item.reason?.includes(rule.reasonIncludes),
);
}
function failedExecutionEvidence(executionResults) {

View File

@ -56,8 +56,14 @@ export async function buildCiSummary(options = {}) {
loaderJitiCandidates: reports.platform?.summary?.jitiAlternativeCount ?? 0,
importLoopP50Ms: reports.importLoop?.summary?.p50WallMs ?? 0,
importLoopP95Ms: reports.importLoop?.summary?.p95WallMs ?? 0,
importLoopMaxRssMb: reports.importLoop?.summary?.maxPeakRssMb ?? 0,
importLoopMaxCpuMs: reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
importLoopOpenClawLifecycleCount: reports.importLoop?.summary?.openClawLifecycleCount ?? 0,
importLoopOpenClawImportP50Ms: reports.importLoop?.summary?.p50OpenClawImportMs ?? 0,
importLoopOpenClawActivationP50Ms: reports.importLoop?.summary?.p50OpenClawActivationMs ?? 0,
importLoopMetricBasis: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb === undefined ? "raw" : "baseline-adjusted",
importLoopMaxRssMb: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb ?? reports.importLoop?.summary?.maxPeakRssMb ?? 0,
importLoopMaxCpuMs: reports.importLoop?.summary?.maxPluginCpuDeltaMsEstimate ?? reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
importLoopRssSampleCount: metricSampleCount(reports.importLoop, "rss", "maxPeakRssMb"),
importLoopCpuSampleCount: metricSampleCount(reports.importLoop, "cpu", "maxCpuMsEstimate"),
},
topIssues: topIssues(reports.compatibility),
refRegressions: (reports.refDiff?.regressions ?? []).slice(0, 20),
@ -148,7 +154,7 @@ export function renderCiSummaryMarkdown(summary) {
["Jiti loader candidates", summary.summary.loaderJitiCandidates],
[
"Import loop",
`p50 ${summary.summary.importLoopP50Ms} ms / p95 ${summary.summary.importLoopP95Ms} ms / max RSS ${summary.summary.importLoopMaxRssMb} MB / CPU ${summary.summary.importLoopMaxCpuMs} ms`,
importLoopSummaryLabel(summary.summary),
],
],
["Metric", "Value"],
@ -221,3 +227,44 @@ function topIssues(report) {
function markdownTable(rows, headers) {
return renderPaddedMarkdownTable(rows, headers, { nullValue: "-" });
}
function metricSampleCount(report, kind, maxMetric) {
const summaryKey = kind === "rss" ? "rssSampleCount" : "cpuSampleCount";
const summaryCount = report?.summary?.[summaryKey];
if (Number.isFinite(summaryCount)) {
return summaryCount;
}
const sampleCount = inferSampleCount(report?.samples, kind);
if (sampleCount > 0) {
return sampleCount;
}
return (report?.summary?.[maxMetric] ?? 0) > 0 ? 1 : 0;
}
function inferSampleCount(samples = [], kind) {
if (!Array.isArray(samples)) {
return 0;
}
return samples.reduce((sum, sample) => {
if (kind === "rss") {
return sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0));
}
return sum + (sample.cpuSampleCount ?? 0);
}, 0);
}
function importLoopSummaryLabel(summary) {
const metricLabel = summary.importLoopMetricBasis === "baseline-adjusted" ? "plugin delta" : "raw";
const lifecycle =
summary.importLoopOpenClawLifecycleCount > 0
? ` / OpenClaw import ${summary.importLoopOpenClawImportP50Ms} ms / activate ${summary.importLoopOpenClawActivationP50Ms} ms`
: "";
return `p50 ${summary.importLoopP50Ms} ms / p95 ${summary.importLoopP95Ms} ms / ${metricLabel} RSS ${formatSampledMetric(summary.importLoopMaxRssMb, summary.importLoopRssSampleCount)} / ${metricLabel} CPU ${formatSampledMetric(summary.importLoopMaxCpuMs, summary.importLoopCpuSampleCount, "ms")}${lifecycle}`;
}
function formatSampledMetric(value, count, unit = "MB") {
if ((count ?? 0) <= 0) {
return "n/a";
}
return `${value} ${unit}`;
}

View File

@ -3,6 +3,7 @@ import path from "node:path";
import {
loadPluginConfig,
renderTextSummary,
sanitizeReportArtifact,
runPluginCheck,
} from "./index.js";
import {
@ -90,7 +91,7 @@ async function runCheck(commandArgs) {
});
if (json) {
console.log(JSON.stringify(report, null, 2));
console.log(JSON.stringify(sanitizeReportArtifact(report), null, 2));
} else {
console.log(renderTextSummary(report, { artifacts: paths }));
}

View File

@ -3,6 +3,8 @@ import path from "node:path";
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
import { slugForArtifact } from "./path-utils.js";
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
export function buildColdImportReadiness(options = {}) {
const report = options.report;
if (!report) {
@ -182,7 +184,7 @@ function classifyEntrypointReadiness({ fixture, packageSummary, entrypoint, root
...(packageSummary.dependencies ?? []),
...(packageSummary.peerDependencies ?? []),
...(packageSummary.optionalDependencies ?? []),
]);
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
if (entrypoint.exists && runtimeDependencies.length > 0) {
blockers.push({
code: "dependency-install-required",

View File

@ -1,4 +1,5 @@
import { renderPaddedMarkdownTable } from "./artifacts.js";
import { sanitizeReportArtifact } from "./report-sanitizer.js";
const defaultSeverityLabels = {
P0: "P0",
@ -8,6 +9,7 @@ const defaultSeverityLabels = {
};
export function renderCompatibilityMarkdownReport(report, options = {}) {
report = sanitizeReportArtifact(report, options);
return [
`# ${options.title ?? "OpenClaw Plugin Compatibility Report"}`,
"",
@ -24,13 +26,20 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
["Warnings", report.summary.warningCount],
["Compatibility suggestions", report.summary.suggestionCount],
["Issue findings", report.summary.issueCount],
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
["P0 issues", report.summary.p0IssueCount],
["P1 issues", report.summary.p1IssueCount],
["Open P0 issues", report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
["Open P1 issues", report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
["Live issues", report.summary.liveIssueCount],
["Live P0 issues", report.summary.liveP0IssueCount],
["Compat gaps", report.summary.compatGapCount],
["Deprecation warnings", report.summary.deprecationWarningCount],
["Inspector gaps", report.summary.inspectorGapCount],
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
["Upstream metadata", report.summary.upstreamIssueCount],
["Contract probes", report.summary.contractProbeCount],
["Decision rows", report.summary.decisionCount],
@ -49,9 +58,9 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
options,
),
"",
"## Live Issues",
"## Other Live Issues",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
"",
"## Compat Gaps",
"",
@ -63,7 +72,11 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
"",
"## Inspector Proof Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
"",
"## Runtime-Covered Inspector Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
"",
"## Upstream Metadata Issues",
"",
@ -127,6 +140,7 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
}
export function renderCompatibilityIssuesReport(report, options = {}) {
report = sanitizeReportArtifact(report, options);
return [
`# ${options.title ?? "OpenClaw Plugin Issue Findings"}`,
"",
@ -138,13 +152,20 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
markdownTable(
[
["Issue findings", report.summary.issueCount],
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
[severityLabel("P0", options), report.summary.p0IssueCount],
[severityLabel("P1", options), report.summary.p1IssueCount],
[`Open ${severityLabel("P0", options)}`, report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
[`Open ${severityLabel("P1", options)}`, report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
["Live issues", report.summary.liveIssueCount],
["Live P0 issues", report.summary.liveP0IssueCount],
["Compat gaps", report.summary.compatGapCount],
["Deprecation warnings", report.summary.deprecationWarningCount],
["Inspector gaps", report.summary.inspectorGapCount],
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
["Upstream metadata", report.summary.upstreamIssueCount],
["Contract probes", report.summary.contractProbeCount],
],
@ -162,9 +183,9 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
options,
),
"",
"## Live Issues",
"## Other Live Issues",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
"",
"## Compat Gaps",
"",
@ -176,7 +197,11 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
"",
"## Inspector Proof Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
"",
"## Runtime-Covered Inspector Gaps",
"",
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
"",
"## Upstream Metadata Issues",
"",
@ -225,6 +250,7 @@ function issueBlock(issue, options) {
` - state: ${issueState(issue)}`,
" - evidence:",
...evidenceList(issue.evidence, options).map((item) => ` - ${item}`),
...runtimeCoverageList(issue, options),
].join("\n");
}
@ -232,6 +258,7 @@ function issueState(issue) {
const flags = [
issue.status,
`compat:${issue.compatStatus ?? "none"}`,
issue.runtimeCoverage?.status ? `runtime:${issue.runtimeCoverage.status}` : null,
issue.live ? "live" : null,
issue.deprecated ? "deprecated" : null,
].filter(Boolean);
@ -263,7 +290,7 @@ function triageOverview(report) {
"inspector-gap",
report.summary.inspectorGapCount,
"-",
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments.",
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments. Runtime-covered rows are proof-backed and not open report work.",
],
[
"upstream-metadata",
@ -354,3 +381,15 @@ function evidenceList(evidence, options) {
const formatEvidence = options.formatEvidence ?? ((item) => item);
return items.map((item) => formatEvidence(item));
}
function runtimeCoverageList(issue, options) {
const runtimeCoverage = issue.runtimeCoverage;
if (!runtimeCoverage) {
return [];
}
return [
" - runtime coverage:",
...evidenceList(runtimeCoverage.captured, options).map((item) => ` - captured ${item}`),
...evidenceList(runtimeCoverage.artifacts, options).map((item) => ` - ${item}`),
];
}

View File

@ -202,13 +202,43 @@ export function packageId(packageName) {
if (!packageName) {
return null;
}
return packageName
const packageBase = packageName
.split("/")
.pop()
.replace(/^openclaw-/, "")
.replace(/[^a-zA-Z0-9]+/g, "-")
.replace(/^-+|-+$/g, "")
.toLowerCase();
.replace(/^openclaw-/, "");
return trimHyphenEdges(collapsePackageIdSeparators(packageBase)).toLowerCase();
}
function collapsePackageIdSeparators(value) {
let result = "";
let previousWasHyphen = false;
for (const char of value) {
if (isAsciiAlphaNumeric(char)) {
result += char;
previousWasHyphen = false;
} else if (!previousWasHyphen) {
result += "-";
previousWasHyphen = true;
}
}
return result;
}
function isAsciiAlphaNumeric(char) {
const code = char.charCodeAt(0);
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
}
function trimHyphenEdges(value) {
let start = 0;
let end = value.length;
while (start < end && value[start] === "-") {
start += 1;
}
while (end > start && value[end - 1] === "-") {
end -= 1;
}
return value.slice(start, end);
}
export function inferPluginSeams(pluginManifest, packageJson) {

View File

@ -7,6 +7,7 @@ import {
} from "./synthetic-probes.js";
export const defaultRegistrationAssertions = {
createChatChannelPlugin: ["channel plugin id is stable", "channel factory metadata is captured"],
defineChannelPluginEntry: ["channel id is stable", "setup/config schema can be read", "message envelope metadata is preserved"],
definePluginEntry: ["entrypoint register function is callable", "entrypoint metadata is preserved"],
registerChannel: ["channel id is stable", "inbound/outbound envelope shape is captured", "sender metadata is preserved"],

View File

@ -157,10 +157,15 @@ function requireCompatRecordReconciliation(report, errors) {
.filter((finding) => finding.code === "missing-compat-record")
.map((finding) => `${finding.fixture}:${finding.compatRecord}`),
);
const compatGapRecords = new Set(
report.issues
.filter((issue) => issue.issueClass === "compat-gap" && issue.compatRecord)
.map((issue) => `${issue.fixture}:${issue.compatRecord}`),
);
for (const finding of [...report.warnings, ...report.suggestions].filter((item) => item.compatRecord)) {
const key = `${finding.fixture}:${finding.compatRecord}`;
if (!presentRecords.has(key) && !missingRecords.has(key)) {
if (!presentRecords.has(key) && !missingRecords.has(key) && !compatGapRecords.has(key)) {
errors.push(`${finding.fixture}: compat record ${finding.compatRecord} was not reconciled`);
}
}

View File

@ -74,6 +74,31 @@ export const contractProbeRules = {
contract: "OpenClaw package entrypoints resolve to files in the published or built plugin package.",
target: "package-loader",
},
"package-install-metadata-incomplete": {
id: "package.metadata.install-release",
contract: "Release publishing metadata declares canonical ClawHub and npm install specs.",
target: "package-loader",
},
"package-min-host-version-drift": {
id: "package.metadata.min-host-version",
contract: "Install minimum host version matches the OpenClaw package surface targeted by the plugin.",
target: "package-loader",
},
"package-npm-pack-entrypoint-missing": {
id: "package.npm-pack.entrypoints",
contract: "Advertised npm artifacts include every declared OpenClaw package entrypoint.",
target: "package-loader",
},
"package-npm-pack-metadata-missing": {
id: "package.npm-pack.metadata",
contract: "Advertised npm artifacts include OpenClaw manifest and package metadata.",
target: "package-loader",
},
"package-npm-pack-unavailable": {
id: "package.npm-pack.available",
contract: "Packages that advertise npm install support can produce an npm pack artifact.",
target: "package-loader",
},
"package-openclaw-entry-missing": {
id: "package.entrypoint.openclaw-metadata",
contract: "OpenClaw package metadata declares entrypoints for cold import and registration capture.",
@ -106,6 +131,20 @@ export const contractProbeRules = {
},
};
const openClawOwnedProbeIssueCodes = new Set([
"before-tool-call-probe",
"channel-contract-probe",
"conversation-access-hook",
"registration-capture-gap",
]);
export function compatRecordForIssueCode(code) {
if (!openClawOwnedProbeIssueCodes.has(code)) {
return undefined;
}
return contractProbeRules[code]?.id;
}
export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
const probes = [];
@ -136,7 +175,6 @@ export function probePriority(code, fixturePriority) {
"before-tool-call-probe",
"conversation-access-hook",
"missing-compat-record",
"registration-capture-gap",
"sdk-export-missing",
].includes(code)
) {

View File

@ -1,6 +1,7 @@
import { existsSync } from "node:fs";
import { readdir } from "node:fs/promises";
import path from "node:path";
import { compatRecordForIssueCode } from "./contract-probes.js";
import { readJsonFile } from "./json-file.js";
const conversationAccessHooks = new Set(["agent_end", "llm_input", "llm_output"]);
@ -17,9 +18,13 @@ const channelRegistrations = new Set([
"defineChannelPluginEntry",
"registerChannel",
]);
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
const unsupportedSecurityManifestName = "openclaw.security.json";
const unavailableSecurityManifestSchema = "https://openclaw.ai/schemas/plugin-security.json";
export async function buildCompatibilityFixtureReport({ fixture, inspection, checkoutPath, sourceRoot, rootDir = process.cwd() }) {
const pluginManifests = await readPluginManifests({ checkoutPath, sourceRoot, rootDir });
const securityManifests = await readSecurityManifests({ checkoutPath, sourceRoot, rootDir });
const packageSummaries = await readPackageSummaries({ checkoutPath, sourceRoot, rootDir });
const packageJson = selectPrimaryPackage(packageSummaries);
const sdkImports = unique((inspection.sdkImports ?? []).map((sdkImport) => sdkImport.specifier));
@ -39,6 +44,7 @@ export async function buildCompatibilityFixtureReport({ fixture, inspection, che
manifestFiles: inspection.manifestFiles ?? [],
sourceFiles: inspection.sourceFiles ?? [],
pluginManifests,
securityManifests,
package: packageJson,
packages: packageSummaries,
sdkImports,
@ -72,6 +78,46 @@ export async function readPluginManifests({ checkoutPath, sourceRoot, rootDir =
return manifests;
}
export async function readSecurityManifests({ checkoutPath, sourceRoot, rootDir = process.cwd() }) {
const candidates = unique(
[
path.join(sourceRoot, unsupportedSecurityManifestName),
path.join(checkoutPath, unsupportedSecurityManifestName),
].filter(existsSync),
);
const manifests = [];
for (const manifestPath of candidates) {
const relativePath = path.relative(rootDir, manifestPath);
try {
const manifest = await readJsonFile(manifestPath);
manifests.push({
path: relativePath,
schema: typeof manifest.$schema === "string" ? manifest.$schema : null,
version: typeof manifest.version === "string" ? manifest.version : null,
plugin: typeof manifest.plugin === "string" ? manifest.plugin : null,
expectedBehaviorCount: Array.isArray(manifest.expectedBehaviors)
? manifest.expectedBehaviors.length
: 0,
securityNoteCount: Array.isArray(manifest.securityNotes) ? manifest.securityNotes.length : 0,
validJson: true,
});
} catch {
manifests.push({
path: relativePath,
schema: null,
version: null,
plugin: null,
expectedBehaviorCount: 0,
securityNoteCount: 0,
validJson: false,
});
}
}
return manifests;
}
export async function readPackageSummaries({ checkoutPath, sourceRoot, rootDir = process.cwd(), maxDepth = 3 }) {
const candidates = unique([
path.join(sourceRoot, "package.json"),
@ -106,6 +152,9 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
typeof packageJson.openclaw.build?.pluginSdkVersion === "string"
? packageJson.openclaw.build.pluginSdkVersion
: null,
install: summarizeOpenClawInstall(packageJson.openclaw.install),
release: summarizeOpenClawRelease(packageJson.openclaw.release),
unsupportedMetadata: unsupportedOpenClawPackageMetadata(packageJson.openclaw),
}
: null;
@ -119,6 +168,7 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
version: packageJson.version ?? null,
type: packageJson.type ?? null,
main: typeof packageJson.main === "string" ? packageJson.main : null,
npmPack: summarizeNpmPack(packageJson, openclaw),
dependencies: Object.keys(packageJson.dependencies ?? {}).sort(),
peerDependencies: Object.keys(packageJson.peerDependencies ?? {}).sort(),
optionalDependencies: Object.keys(packageJson.optionalDependencies ?? {}).sort(),
@ -200,6 +250,79 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
});
}
if ((packageSummary.openclaw?.unsupportedMetadata ?? []).length > 0) {
warnings.push({
fixture: fixture.id,
code: "package-openclaw-unsupported-metadata",
level: "warning",
message: "package declares unsupported OpenClaw metadata",
evidence: packageSummary.openclaw.unsupportedMetadata,
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-metadata",
action: "Remove unsupported OpenClaw metadata; native plugins use openclaw.plugin.json plus supported package openclaw fields.",
evidence: packageSummary.openclaw.unsupportedMetadata.join(", "),
});
}
const installMetadataIssues = packageInstallMetadataIssues(packageSummary);
if (installMetadataIssues.length > 0) {
warnings.push({
fixture: fixture.id,
code: "package-install-metadata-incomplete",
level: "warning",
message: "package OpenClaw install metadata does not match advertised release targets",
evidence: installMetadataIssues,
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-metadata",
action: "Ask the plugin to align openclaw.install metadata with openclaw.release publishing targets.",
evidence: installMetadataIssues.join(", "),
});
}
if (packageMinHostVersionDrift(packageSummary)) {
warnings.push({
fixture: fixture.id,
code: "package-min-host-version-drift",
level: "warning",
message: "package openclaw.install.minHostVersion is not a semver floor for the target OpenClaw build version",
evidence: [
`minHostVersion:${packageSummary.openclaw.install.minHostVersion}`,
`buildOpenClawVersion:${packageSummary.openclaw.buildOpenClawVersion}`,
],
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-metadata",
action: "Ask the plugin to publish install.minHostVersion as a semver floor for the OpenClaw package surface it targets.",
evidence: packageSummary.path,
});
}
const npmPackIssues = packageNpmPackIssues(packageSummary, fixtureReport);
for (const finding of npmPackIssues) {
warnings.push({
fixture: fixture.id,
code: finding.code,
level: "warning",
message: finding.message,
evidence: finding.evidence,
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "package-artifact",
action: "Ask the plugin to make its advertised npm install artifact match the published OpenClaw metadata.",
evidence: finding.evidence.join(", "),
});
}
if (packageSummary.openclaw && packageSummary.openclaw.entrypoints.length === 0) {
warnings.push({
fixture: fixture.id,
@ -217,9 +340,12 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
});
}
const missingEntrypoints = packageSummary.openclaw?.entrypoints.filter((entrypoint) => !entrypoint.exists) ?? [];
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
const missingEntrypoints = entrypoints.filter((entrypoint) => !entrypoint.exists);
const buildEntrypoints = missingEntrypoints.filter((entrypoint) => entrypoint.requiresBuild);
const plainMissingEntrypoints = missingEntrypoints.filter((entrypoint) => !entrypoint.requiresBuild);
const plainMissingEntrypoints = missingEntrypoints.filter(
(entrypoint) => !entrypoint.requiresBuild && !hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
);
if (buildEntrypoints.length > 0) {
suggestions.push({
@ -278,7 +404,7 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
...packageSummary.dependencies,
...packageSummary.peerDependencies,
...packageSummary.optionalDependencies,
]);
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
if (packageSummary.openclaw?.entrypoints.length > 0 && runtimeDependencies.length > 0) {
suggestions.push({
fixture: fixture.id,
@ -349,6 +475,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
suggestions.push(...packageContracts.suggestions);
logs.push(...packageContracts.logs);
decisions.push(...packageContracts.decisions);
classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions });
for (const pluginManifest of fixtureReport.pluginManifests) {
const providerAuthKeys = Object.keys(pluginManifest.providerAuthEnvVars ?? {});
@ -399,6 +526,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "warning",
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
evidence: detailEvidence(conversationHookDetails),
compatRecord: compatRecordForIssueCode("conversation-access-hook"),
});
decisions.push({
fixture: fixture.id,
@ -459,6 +587,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
evidence: detailEvidence(captureGapRegistrationDetails),
compatRecord: compatRecordForIssueCode("registration-capture-gap"),
});
decisions.push({
fixture: fixture.id,
@ -477,6 +606,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
evidence: detailEvidence(hookDetails),
compatRecord: compatRecordForIssueCode("before-tool-call-probe"),
});
decisions.push({
fixture: fixture.id,
@ -500,6 +630,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "add channel envelope, config-schema, and runtime metadata probes",
evidence: detailEvidence(channelRegistrationDetails),
compatRecord: compatRecordForIssueCode("channel-contract-probe"),
});
decisions.push({
fixture: fixture.id,
@ -551,6 +682,45 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
return { warnings, suggestions, logs, decisions };
}
function classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions }) {
for (const securityManifest of fixtureReport.securityManifests ?? []) {
warnings.push({
fixture: fixture.id,
code: "unrecognized-security-manifest",
level: "warning",
message:
"openclaw.security.json is not a supported OpenClaw or ClawHub security contract and is ignored by install safety checks",
evidence: [securityManifest.path],
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "security-metadata",
action:
"Remove the advisory security manifest or replace it with a supported, versioned OpenClaw/ClawHub security contract once one exists.",
evidence: securityManifest.path,
});
if (securityManifest.schema === unavailableSecurityManifestSchema) {
warnings.push({
fixture: fixture.id,
code: "security-manifest-schema-unavailable",
level: "warning",
message: "openclaw.security.json references an OpenClaw schema URL that is not currently published",
evidence: [`${securityManifest.path}:$schema=${securityManifest.schema}`],
});
decisions.push({
fixture: fixture.id,
decision: "plugin-upstream-fix",
seam: "security-metadata",
action:
"Do not rely on the schema URL until OpenClaw publishes and documents a real plugin security metadata schema.",
evidence: securityManifest.schema,
});
}
}
}
function registrationCaptureGapDetails(inspection, targetOpenClaw) {
const apiRegistrationDetails = inspection.registrationDetails.filter((registration) =>
registration.name.startsWith("register"),
@ -753,6 +923,46 @@ function collectOpenClawEntrypoints(packageDir, openclaw, options) {
});
}
function hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
if (!isSourceEntrypoint(entrypoint.specifier)) {
return false;
}
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
if (
entrypoints.some(
(candidate) =>
candidate.exists &&
candidate.requiresBuild &&
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
)
) {
return true;
}
if (entrypoint.kind === "extension" && entrypoints.some((candidate) => candidate.kind === "runtimeExtension" && candidate.exists)) {
return true;
}
const packageDir = path.dirname(packageSummary.path);
return existsSync(path.resolve(packageDir, runtimeBuildSpecifier));
}
function isSourceEntrypoint(specifier) {
return /\.(?:ts|tsx)$/.test(specifier);
}
function runtimeBuildSpecifierFor(specifier) {
const normalized = normalizeEntrypointSpecifier(specifier);
const basename = path.posix.basename(normalized).replace(/\.(?:ts|tsx)$/, ".js");
return `./dist/${basename}`;
}
function normalizeEntrypointSpecifier(specifier) {
const normalized = specifier.replaceAll("\\", "/");
return normalized.startsWith("./") ? normalized : `./${normalized}`;
}
async function findPackageFiles(root, options, depth = 0) {
if (!existsSync(root) || depth > options.maxDepth) {
return [];
@ -781,6 +991,269 @@ function selectPrimaryPackage(packages) {
return packages[0] ?? null;
}
function summarizeOpenClawInstall(install) {
if (!install || typeof install !== "object") {
return null;
}
return {
clawhubSpec: stringOrNull(install.clawhubSpec),
npmSpec: stringOrNull(install.npmSpec),
defaultChoice: stringOrNull(install.defaultChoice),
minHostVersion: stringOrNull(install.minHostVersion),
};
}
function summarizeOpenClawRelease(release) {
if (!release || typeof release !== "object") {
return null;
}
return {
publishToClawHub: booleanOrNull(release.publishToClawHub),
publishToNpm: booleanOrNull(release.publishToNpm),
};
}
function unsupportedOpenClawPackageMetadata(openclaw) {
if (!openclaw || typeof openclaw !== "object") {
return [];
}
return Object.keys(openclaw)
.filter((key) => key === "bundle")
.map((key) => `openclaw.${key}`);
}
function summarizeNpmPack(packageJson, openclaw) {
const files = arrayValues(packageJson.files).map(normalizePackagePath).filter((item) => item.length > 0);
return {
advertised: openclaw?.release?.publishToNpm === true || nonEmptyString(openclaw?.install?.npmSpec),
private: packageJson.private === true,
filesMode: files.length > 0 ? "allowlist" : "implicit",
files,
invalidFileSpecs: files.filter((item) => invalidPackageFileSpec(item)),
};
}
function packageInstallMetadataIssues(packageSummary) {
const openclaw = packageSummary.openclaw;
if (!openclaw) {
return [];
}
const issues = [];
const install = openclaw.install;
const release = openclaw.release;
const publishToClawHub = release?.publishToClawHub === true;
const publishToNpm = release?.publishToNpm === true;
if (publishToClawHub && !nonEmptyString(install?.clawhubSpec)) {
issues.push("openclaw.release.publishToClawHub requires openclaw.install.clawhubSpec");
}
if (publishToNpm && !nonEmptyString(install?.npmSpec)) {
issues.push("openclaw.release.publishToNpm requires openclaw.install.npmSpec");
}
if (publishToNpm && nonEmptyString(install?.npmSpec) && nonEmptyString(packageSummary.name) && install.npmSpec !== packageSummary.name) {
issues.push(`openclaw.install.npmSpec:${install.npmSpec} does not match package name:${packageSummary.name}`);
}
if (nonEmptyString(install?.defaultChoice) && !["clawhub", "npm"].includes(install.defaultChoice)) {
issues.push(`openclaw.install.defaultChoice:${install.defaultChoice} must be clawhub or npm`);
}
if (install?.defaultChoice === "clawhub" && !nonEmptyString(install.clawhubSpec)) {
issues.push("openclaw.install.defaultChoice clawhub requires openclaw.install.clawhubSpec");
}
if (install?.defaultChoice === "npm" && !nonEmptyString(install.npmSpec)) {
issues.push("openclaw.install.defaultChoice npm requires openclaw.install.npmSpec");
}
return issues;
}
function packageNpmPackIssues(packageSummary, fixtureReport) {
if (!packageSummary.npmPack?.advertised) {
return [];
}
const findings = [];
const unavailable = [];
if (packageSummary.npmPack.private) {
unavailable.push("package.json private:true");
}
if (!nonEmptyString(packageSummary.name)) {
unavailable.push("package.json name missing");
}
if (!nonEmptyString(packageSummary.version)) {
unavailable.push("package.json version missing");
}
unavailable.push(...packageSummary.npmPack.invalidFileSpecs.map((item) => `invalid files entry:${item}`));
if (unavailable.length > 0) {
findings.push({
code: "package-npm-pack-unavailable",
message: "package advertises npm install or publish metadata but cannot produce a usable npm pack artifact",
evidence: unavailable,
});
}
const missingMetadata = packageNpmPackMissingMetadata(packageSummary, fixtureReport);
if (missingMetadata.length > 0) {
findings.push({
code: "package-npm-pack-metadata-missing",
message: "advertised npm artifact would not include required OpenClaw package metadata",
evidence: missingMetadata,
});
}
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
const missingEntrypoints = entrypoints
.filter(
(entrypoint) =>
!repoPathIncludedInNpmPack(packageSummary, entrypoint.relativePath) &&
!hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
)
.map((entrypoint) => `${entrypoint.kind}:${entrypoint.specifier} -> ${entrypoint.relativePath}`) ?? [];
if (missingEntrypoints.length > 0) {
findings.push({
code: "package-npm-pack-entrypoint-missing",
message: "advertised npm artifact would not include declared OpenClaw entrypoints",
evidence: missingEntrypoints,
});
}
return findings;
}
function packageNpmPackMissingMetadata(packageSummary, fixtureReport) {
const missing = [];
if (!repoPathIncludedInNpmPack(packageSummary, packageSummary.path)) {
missing.push(packageSummary.path);
}
for (const manifest of fixtureReport.pluginManifests ?? []) {
if (repoPathWithinPackage(packageSummary, manifest.path) && !repoPathIncludedInNpmPack(packageSummary, manifest.path)) {
missing.push(manifest.path);
}
}
return missing;
}
function hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
if (!isSourceEntrypoint(entrypoint.specifier)) {
return false;
}
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
const matchingRuntimeEntrypoint = entrypoints.find(
(candidate) =>
candidate.requiresBuild &&
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
);
if (matchingRuntimeEntrypoint && repoPathIncludedInNpmPack(packageSummary, matchingRuntimeEntrypoint.relativePath)) {
return true;
}
if (
entrypoint.kind === "extension" &&
entrypoints.some(
(candidate) => candidate.kind === "runtimeExtension" && repoPathIncludedInNpmPack(packageSummary, candidate.relativePath),
)
) {
return true;
}
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
const runtimeBuildPath = path.posix.join(packageDir === "." ? "" : packageDir, normalizeEntrypointSpecifier(runtimeBuildSpecifier));
return repoPathIncludedInNpmPack(packageSummary, runtimeBuildPath);
}
function packageMinHostVersionDrift(packageSummary) {
const openclaw = packageSummary.openclaw;
if (!nonEmptyString(openclaw?.install?.minHostVersion) || !nonEmptyString(openclaw?.buildOpenClawVersion)) {
return false;
}
return parseMinHostVersionFloor(openclaw.install.minHostVersion) !== openclaw.buildOpenClawVersion;
}
function parseMinHostVersionFloor(value) {
const match = /^>=([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/.exec(value);
return match?.[1] ?? null;
}
function repoPathIncludedInNpmPack(packageSummary, repoPath) {
const packageRelativePath = packageRelativeRepoPath(packageSummary, repoPath);
if (!packageRelativePath) {
return false;
}
if (npmAlwaysPacksPath(packageRelativePath)) {
return true;
}
if (packageSummary.npmPack?.filesMode !== "allowlist") {
return true;
}
return packageSummary.npmPack.files.some((spec) => packageFileSpecIncludesPath(spec, packageRelativePath));
}
function repoPathWithinPackage(packageSummary, repoPath) {
return packageRelativeRepoPath(packageSummary, repoPath) !== null;
}
function packageRelativeRepoPath(packageSummary, repoPath) {
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
const normalized = normalizeRepoPath(repoPath);
if (packageDir === ".") {
return normalized;
}
if (normalized === packageDir) {
return "";
}
return normalized.startsWith(`${packageDir}/`) ? normalized.slice(packageDir.length + 1) : null;
}
function npmAlwaysPacksPath(packageRelativePath) {
const base = path.posix.basename(packageRelativePath).toLowerCase();
return packageRelativePath === "package.json" || /^readme(?:\..*)?$/u.test(base) || /^licen[cs]e(?:\..*)?$/u.test(base);
}
function packageFileSpecIncludesPath(spec, packageRelativePath) {
if (spec === "." || spec === packageRelativePath) {
return true;
}
if (spec.includes("*")) {
return globLikeSpecIncludesPath(spec, packageRelativePath);
}
return packageRelativePath.startsWith(`${spec.replace(/\/$/u, "")}/`);
}
function globLikeSpecIncludesPath(spec, packageRelativePath) {
return globSegmentsIncludePath(spec.split("/"), packageRelativePath.split("/"));
}
function globSegmentsIncludePath(specSegments, packageSegments) {
if (specSegments.length === 0) {
return packageSegments.length === 0;
}
const [head, ...tail] = specSegments;
if (head === "**") {
return globSegmentsIncludePath(tail, packageSegments) || (packageSegments.length > 0 && globSegmentsIncludePath(specSegments, packageSegments.slice(1)));
}
if (packageSegments.length === 0) {
return false;
}
return globSegmentIncludesPath(head, packageSegments[0]) && globSegmentsIncludePath(tail, packageSegments.slice(1));
}
function globSegmentIncludesPath(specSegment, packageSegment) {
const escaped = specSegment.replace(/[.+?^${}()|[\]\\]/gu, "\\$&").replaceAll("*", ".*");
return new RegExp(`^${escaped}$`, "u").test(packageSegment);
}
function invalidPackageFileSpec(spec) {
return spec.startsWith("/") || spec === ".." || spec.startsWith("../") || spec.includes("/../");
}
function normalizePackagePath(value) {
return normalizeRepoPath(value).replace(/^\.\/+/u, "").replace(/\/$/u, "");
}
function packageRank(packageSummary) {
if (packageSummary.openclaw?.entrypoints.length > 0) {
return 0;
@ -795,6 +1268,22 @@ function arrayValues(value) {
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
}
function stringOrNull(value) {
return typeof value === "string" ? value : null;
}
function booleanOrNull(value) {
return typeof value === "boolean" ? value : null;
}
function nonEmptyString(value) {
return typeof value === "string" && value.trim().length > 0;
}
function normalizeRepoPath(value) {
return String(value ?? "").replaceAll("\\", "/").replace(/^\.\/+/u, "");
}
function detailEvidence(details, key = "name") {
return unique(details.map((detail) => `${detail[key]} @ ${detail.ref}`));
}

View File

@ -1,4 +1,4 @@
import { mkdir } from "node:fs/promises";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
@ -24,22 +24,48 @@ export async function buildImportLoopProfile(options = {}) {
const entrypoint = options.entrypoint ?? defaultImportLoopProfileOptions.entrypoint;
assertRunCount(runs, 20);
const baseline = await buildBaselineProfile({ ...options, rootDir, runs });
const samples = [];
for (let index = 0; index < runs; index += 1) {
samples.push(await runCaptureSample({ ...options, entrypoint, index, rootDir }));
const sample = await runCaptureSample({ ...options, entrypoint, index, rootDir });
samples.push(applyBaselineAdjustment(sample, baseline));
}
const wallMs = samples.map((sample) => sample.wallMs).sort((left, right) => left - right);
const pluginWallDeltaMs = samples.map((sample) => sample.pluginWallDeltaMs).sort((left, right) => left - right);
const openClawImportMs = openClawLifecycleMetric(samples, "importMs");
const openClawActivationMs = openClawLifecycleMetric(samples, "activationMs");
const rssSampleCount = samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)), 0);
const cpuSampleCount = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
return {
generatedAt: options.generatedAt ?? defaultImportLoopProfileOptions.generatedAt,
mode: options.mode ?? "subprocess-cold-import-loop",
mode: options.mode ?? "baseline-adjusted-cold-capture-loop",
entrypoint,
baseline,
summary: {
runs,
baselineRuns: baseline.runs,
baselineFailCount: baseline.failCount,
p50WallMs: percentile(wallMs, 0.5),
p95WallMs: percentile(wallMs, 0.95),
p50PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.5),
p95PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.95),
openClawLifecycleCount: openClawImportMs.length,
p50OpenClawImportMs: percentile(openClawImportMs, 0.5),
p95OpenClawImportMs: percentile(openClawImportMs, 0.95),
p50OpenClawActivationMs: percentile(openClawActivationMs, 0.5),
p95OpenClawActivationMs: percentile(openClawActivationMs, 0.95),
maxPeakRssMb: Math.max(0, ...samples.map((sample) => sample.peakRssMb)),
maxCpuMsEstimate: Math.max(0, ...samples.map((sample) => sample.cpuMsEstimate)),
maxPluginPeakRssDeltaMb: Math.max(0, ...samples.map((sample) => sample.pluginPeakRssDeltaMb)),
maxPluginCpuDeltaMsEstimate: Math.max(0, ...samples.map((sample) => sample.pluginCpuDeltaMsEstimate)),
baselineReferenceWallMs: baseline.reference.wallMs,
baselineReferencePeakRssMb: baseline.reference.peakRssMb,
baselineReferenceCpuMsEstimate: baseline.reference.cpuMsEstimate,
statSampleCount,
rssSampleCount,
cpuSampleCount,
capturedCount: samples.reduce((sum, sample) => sum + sample.capturedCount, 0),
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
},
@ -52,6 +78,9 @@ export function validateImportLoopProfile(report) {
if (report.summary.failCount > 0) {
errors.push(`import loop has ${report.summary.failCount} failed sample(s)`);
}
if ((report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0) > 0) {
errors.push("import loop baseline capture failed");
}
if (report.summary.capturedCount < report.summary.runs) {
errors.push("import loop did not capture at least one contract per run");
}
@ -85,7 +114,11 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
"",
"## Summary",
"",
markdownTable(Object.entries(report.summary).map(([key, value]) => [key, value]), ["Metric", "Value"]),
markdownTable(summaryRows(report), ["Metric", "Value"]),
"",
"## Harness Baseline",
"",
markdownTable(baselineRows(report), ["Metric", "Value"]),
"",
"## Samples",
"",
@ -94,22 +127,132 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
sample.index,
sample.status,
sample.capturedCount,
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.importMs),
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.activationMs),
formatOptionalMetric(sample.pluginWallDeltaMs, "ms"),
formatSampledMetric(sample.pluginPeakRssDeltaMb, sample.rssSampleCount),
formatSampledMetric(sample.pluginCpuDeltaMsEstimate, sample.cpuSampleCount, "ms"),
`${sample.wallMs} ms`,
`${sample.peakRssMb} MB`,
`${sample.cpuMsEstimate} ms`,
formatSampledMetric(sample.peakRssMb, sample.rssSampleCount),
formatSampledMetric(sample.cpuMsEstimate, sample.cpuSampleCount, "ms"),
`${sample.rssSampleCount ?? 0}/${sample.cpuSampleCount ?? 0}`,
sample.exitCode,
]),
["Run", "Status", "Captured", "Wall", "Peak RSS", "CPU Estimate", "Exit"],
[
"Run",
"Status",
"Captured",
"OpenClaw Import",
"OpenClaw Activate",
"Plugin Wall Delta",
"Plugin RSS Delta",
"Plugin CPU Delta",
"Raw Wall",
"Raw Peak RSS",
"Raw CPU Estimate",
"RSS/CPU samples",
"Exit",
],
),
].join("\n");
}
async function buildBaselineProfile(options) {
const baselineRuns = options.baseline === false ? 0 : options.baselineRuns ?? Math.min(options.runs, 3);
if (baselineRuns <= 0) {
return emptyBaseline();
}
const entrypoint = await writeBaselineEntrypoint(options);
const samples = [];
for (let index = 0; index < baselineRuns; index += 1) {
samples.push(
await runCaptureSample({
...options,
entrypoint,
index,
sampleName: "baseline",
rootDir: options.rootDir,
}),
);
}
const wallMs = sortedMetric(samples, "wallMs");
const peakRssMb = sortedMetric(samples, "peakRssMb");
const cpuMsEstimate = sortedMetric(samples, "cpuMsEstimate");
return {
mode: "minimal-plugin-capture",
runs: baselineRuns,
entrypoint: path.relative(options.rootDir, entrypoint),
reference: {
wallMs: percentile(wallMs, 0.5),
peakRssMb: percentile(peakRssMb, 0.5),
cpuMsEstimate: percentile(cpuMsEstimate, 0.5),
},
max: {
wallMs: wallMs.at(-1) ?? 0,
peakRssMb: peakRssMb.at(-1) ?? 0,
cpuMsEstimate: cpuMsEstimate.at(-1) ?? 0,
},
statSampleCount: samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0),
rssSampleCount: samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? 0), 0),
cpuSampleCount: samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0),
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
samples,
};
}
function emptyBaseline() {
return {
mode: "disabled",
runs: 0,
entrypoint: null,
reference: {
wallMs: 0,
peakRssMb: 0,
cpuMsEstimate: 0,
},
max: {
wallMs: 0,
peakRssMb: 0,
cpuMsEstimate: 0,
},
statSampleCount: 0,
rssSampleCount: 0,
cpuSampleCount: 0,
failCount: 0,
samples: [],
};
}
async function writeBaselineEntrypoint(options) {
const outputDir = resolveFromRoot(
options.rootDir,
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
);
const baselinePath = path.join(outputDir, "baseline-plugin.mjs");
await mkdir(path.dirname(baselinePath), { recursive: true });
await writeFile(
baselinePath,
[
"export default {",
" register(api) {",
" api.registerTool({ name: 'baseline_tool', inputSchema: { type: 'object' }, run() {} });",
" },",
"};",
"",
].join("\n"),
"utf8",
);
return baselinePath;
}
async function runCaptureSample(options) {
const outputDir = resolveFromRoot(
options.rootDir,
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
);
const outputPath = path.join(outputDir, `capture-${options.index}.json`);
const outputPath = path.join(outputDir, `${options.sampleName ?? "capture"}-${options.index}.json`);
await mkdir(path.dirname(outputPath), { recursive: true });
const command = buildCaptureCommand({ ...options, outputPath });
@ -126,14 +269,125 @@ async function runCaptureSample(options) {
exitCode: profile.exitCode,
status: output?.status ?? "failed",
capturedCount: output?.captured?.length ?? 0,
openClawLifecycle: output?.openClawLifecycle ?? null,
wallMs: profile.wallMs,
peakRssMb: profile.peakRssMb,
peakCpuPercent: profile.peakCpuPercent,
cpuMsEstimate: profile.cpuMsEstimate,
statSampleCount: profile.statSampleCount,
rssSampleCount: profile.rssSampleCount,
cpuSampleCount: profile.cpuSampleCount,
stderrPreview: profile.stderrPreview,
};
}
function summaryRows(report) {
return [
["runs", report.summary.runs],
["baselineRuns", report.summary.baselineRuns ?? report.baseline?.runs ?? 0],
["baselineFailCount", report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0],
["p50WallMs", report.summary.p50WallMs],
["p95WallMs", report.summary.p95WallMs],
...(Number.isFinite(report.summary.p50PluginWallDeltaMs)
? [
["p50PluginWallDeltaMs", report.summary.p50PluginWallDeltaMs],
["p95PluginWallDeltaMs", report.summary.p95PluginWallDeltaMs],
["maxPluginPeakRssDeltaMb", formatSampledMetric(report.summary.maxPluginPeakRssDeltaMb, report.summary.rssSampleCount)],
[
"maxPluginCpuDeltaMsEstimate",
formatSampledMetric(report.summary.maxPluginCpuDeltaMsEstimate, report.summary.cpuSampleCount, "ms"),
],
]
: []),
...((report.summary.openClawLifecycleCount ?? 0) > 0
? [
["openClawLifecycleCount", report.summary.openClawLifecycleCount],
["p50OpenClawImportMs", `${report.summary.p50OpenClawImportMs} ms`],
["p95OpenClawImportMs", `${report.summary.p95OpenClawImportMs} ms`],
["p50OpenClawActivationMs", `${report.summary.p50OpenClawActivationMs} ms`],
["p95OpenClawActivationMs", `${report.summary.p95OpenClawActivationMs} ms`],
]
: []),
["maxPeakRssMb", formatSampledMetric(report.summary.maxPeakRssMb, report.summary.rssSampleCount)],
["maxCpuMsEstimate", formatSampledMetric(report.summary.maxCpuMsEstimate, report.summary.cpuSampleCount, "ms")],
...(Number.isFinite(report.summary.baselineReferenceWallMs)
? [
["baselineReferenceWallMs", `${report.summary.baselineReferenceWallMs} ms`],
["baselineReferencePeakRssMb", formatSampledMetric(report.summary.baselineReferencePeakRssMb, report.baseline?.rssSampleCount ?? 0)],
[
"baselineReferenceCpuMsEstimate",
formatSampledMetric(report.summary.baselineReferenceCpuMsEstimate, report.baseline?.cpuSampleCount ?? 0, "ms"),
],
]
: []),
["statSampleCount", report.summary.statSampleCount ?? 0],
["rssSampleCount", report.summary.rssSampleCount ?? 0],
["cpuSampleCount", report.summary.cpuSampleCount ?? 0],
["capturedCount", report.summary.capturedCount],
["failCount", report.summary.failCount],
];
}
function baselineRows(report) {
const baseline = report.baseline ?? emptyBaseline();
return [
["mode", baseline.mode],
["runs", baseline.runs],
["entrypoint", baseline.entrypoint ?? "-"],
["referenceWallMs", `${baseline.reference?.wallMs ?? 0} ms`],
["referencePeakRssMb", formatSampledMetric(baseline.reference?.peakRssMb ?? 0, baseline.rssSampleCount)],
["referenceCpuMsEstimate", formatSampledMetric(baseline.reference?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
["maxWallMs", `${baseline.max?.wallMs ?? 0} ms`],
["maxPeakRssMb", formatSampledMetric(baseline.max?.peakRssMb ?? 0, baseline.rssSampleCount)],
["maxCpuMsEstimate", formatSampledMetric(baseline.max?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
["statSampleCount", baseline.statSampleCount ?? 0],
["failCount", baseline.failCount ?? 0],
];
}
function formatSampledMetric(value, count, unit = "MB") {
if ((count ?? 0) <= 0) {
return "n/a";
}
return `${value} ${unit}`;
}
function formatOptionalMetric(value, unit) {
if (!Number.isFinite(value)) {
return "n/a";
}
return `${value} ${unit}`;
}
function formatOpenClawLifecycleMetric(value) {
return Number.isFinite(value) ? `${value} ms` : "n/a";
}
function openClawLifecycleMetric(samples, field) {
return samples
.map((sample) => sample.openClawLifecycle?.[field])
.filter((value) => Number.isFinite(value))
.sort((left, right) => left - right);
}
function applyBaselineAdjustment(sample, baseline) {
return {
...sample,
pluginWallDeltaMs: roundNonNegative(sample.wallMs - baseline.reference.wallMs, 0),
pluginPeakRssDeltaMb: roundNonNegative(sample.peakRssMb - baseline.reference.peakRssMb, 1),
pluginCpuDeltaMsEstimate: roundNonNegative(sample.cpuMsEstimate - baseline.reference.cpuMsEstimate, 0),
};
}
function sortedMetric(samples, field) {
return samples.map((sample) => sample[field]).sort((left, right) => left - right);
}
function roundNonNegative(value, digits) {
const scale = 10 ** digits;
return Math.max(0, Math.round(value * scale) / scale);
}
function buildCaptureCommand(options) {
if (typeof options.captureCommand === "function") {
return options.captureCommand({

View File

@ -13,6 +13,7 @@ import * as profileDiffApi from "./profile-diff.js";
import * as refDiffApi from "./ref-diff.js";
import * as reportApi from "./report.js";
import * as runtimeProfileApi from "./runtime-profile.js";
import * as runtimeReconciliationApi from "./runtime-reconciliation.js";
import * as syntheticProbeSuiteApi from "./synthetic-probe-suite.js";
import * as syntheticProbesApi from "./synthetic-probes.js";
@ -53,11 +54,13 @@ export const staticInspection = Object.freeze({
export const reports = Object.freeze({
renderMarkdown: reportApi.renderMarkdownReport,
renderTextSummary: pluginApi.renderTextSummary,
sanitizeArtifact: reportApi.sanitizeReportArtifact,
write: reportApi.writeReport,
issueId: issuesApi.issueId,
classifyIssueFinding: issuesApi.classifyIssueFinding,
knownIssueCodes: issuesApi.knownIssueCodes,
openClawTargetPathCandidates: openClawTargetApi.openClawTargetPathCandidates,
readOpenClawTargetSurface: openClawTargetApi.readOpenClawTargetSurface,
});
export const contracts = Object.freeze({
@ -109,6 +112,8 @@ export const runtime = Object.freeze({
writeImportLoopProfile: importLoopProfileApi.writeImportLoopProfile,
renderImportLoopProfile: importLoopProfileApi.renderImportLoopProfileMarkdown,
validateImportLoopProfile: importLoopProfileApi.validateImportLoopProfile,
applyExecutionCoverage: runtimeReconciliationApi.applyRuntimeExecutionCoverage,
buildExecutionCoverage: runtimeReconciliationApi.buildRuntimeExecutionCoverage,
});
export const synthetic = Object.freeze({
@ -200,7 +205,7 @@ export {
} from "./import-loop-profile.js";
export { classifyIssueFinding, issueId, knownIssueCodes } from "./issues.js";
export { inspectFixtureSet, inspectPlugin, inspectSourceText } from "./inspector.js";
export { openClawTargetPathCandidates } from "./openclaw-target.js";
export { openClawTargetPathCandidates, readOpenClawTargetSurface } from "./openclaw-target.js";
export {
buildProfileDiff,
defaultProfileDiffOptions,
@ -216,7 +221,7 @@ export {
validateRefDiff,
writeRefDiff,
} from "./ref-diff.js";
export { renderMarkdownReport, writeReport } from "./report.js";
export { renderMarkdownReport, sanitizeReportArtifact, writeReport } from "./report.js";
export {
buildRuntimeProfile,
defaultRuntimeProfileCommands,
@ -225,6 +230,10 @@ export {
validateRuntimeProfile,
writeRuntimeProfile,
} from "./runtime-profile.js";
export {
applyRuntimeExecutionCoverage,
buildRuntimeExecutionCoverage,
} from "./runtime-reconciliation.js";
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
export {
buildSyntheticProbePlan,

View File

@ -5,12 +5,16 @@ import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
import { promisify } from "node:util";
import { createCaptureApi } from "./capture-api.js";
import { captureApiOptionsForPlugin } from "./capture-config.js";
import { fixtureCheckoutPath, fixtureSourceRoot } from "./config.js";
import { buildCompatibilityFixtureReport } from "./fixture-summary.js";
import { readOpenClawTargetSurface } from "./openclaw-target.js";
import { buildCompatibilityReport, buildReport } from "./report.js";
const execFileAsync = promisify(execFile);
const registrationEquivalents = new Map([
["registerChannel", new Set(["createChatChannelPlugin", "defineBundledChannelEntry", "defineChannelPluginEntry", "registerChannel"])],
]);
export async function inspectFixtureSet(config, options = {}) {
const { inspections, failures } = await inspectConfiguredFixtures(config, options);
@ -32,6 +36,7 @@ export async function inspectCompatibilityFixtureSet(config, options = {}) {
inspections,
failures,
generatedAt: options.generatedAt,
executionResults: options.executionResults,
targetOpenClaw,
buildFixtureReport: ({ fixture, inspection }) =>
buildCompatibilityFixtureReport({
@ -58,7 +63,7 @@ async function inspectConfiguredFixtures(config, options = {}) {
["manifestContracts", inspection.manifestContracts],
]) {
const expected = fixture.expect?.[key] ?? [];
const missing = expected.filter((value) => !observed.includes(value));
const missing = expected.filter((value) => !satisfiesExpectedSeam(key, value, observed));
if (missing.length > 0) {
failures.push(`${fixture.id}: missing ${key}: ${missing.join(", ")}`);
}
@ -68,6 +73,17 @@ async function inspectConfiguredFixtures(config, options = {}) {
return { inspections, failures };
}
function satisfiesExpectedSeam(key, expected, observed) {
if (observed.includes(expected)) {
return true;
}
if (key !== "registrations") {
return false;
}
const equivalents = registrationEquivalents.get(expected);
return Boolean(equivalents && observed.some((value) => equivalents.has(value)));
}
export async function inspectPlugin(fixture, options = {}) {
const config = options.config ?? { rootDir: options.rootDir ?? process.cwd() };
const checkoutPath = fixtureCheckoutPath(config, fixture);
@ -132,6 +148,7 @@ export function inspectSourceText(text, filePath = "source.js") {
const hooks = collectDetailedMatches(searchableText, /\bapi\.on\(\s*["'`]([^"'`]+)["'`]/g, filePath, "name");
const registrations = [
...collectDetailedMatches(searchableText, /\bapi\.(register[A-Za-z0-9]+)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(defineBundledChannelEntry)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(defineChannelPluginEntry)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(createChatChannelPlugin)\s*\(/g, filePath, "name"),
...collectDetailedMatches(searchableText, /\b(definePluginEntry)\s*\(/g, filePath, "name"),
@ -172,7 +189,12 @@ export async function captureEntrypoint(entrypoint, options = {}) {
};
}
const api = createCaptureApi(options.apiOptions);
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
pluginRoot: options.pluginRoot
? path.resolve(options.cwd ?? process.cwd(), options.pluginRoot)
: path.dirname(resolvedEntrypoint),
});
const api = createCaptureApi(apiOptions);
try {
await register(api);
} catch (error) {
@ -183,7 +205,7 @@ export async function captureEntrypoint(entrypoint, options = {}) {
entrypoint: resolvedEntrypoint,
captured: api.getCapturedContracts(),
};
if (options.apiOptions?.retainHandlers === true) {
if (apiOptions?.retainHandlers === true) {
result.retained = api.getRetainedContracts();
}
return result;
@ -212,10 +234,34 @@ export async function captureEntrypointWithMockSdk(entrypoint, options = {}) {
);
return JSON.parse(stdout);
} catch (error) {
const captured = parseCaptureResultFromStdout(error?.stdout);
if (captured) {
return captured;
}
throw classifyMockSdkCaptureError(error);
}
}
function parseCaptureResultFromStdout(stdout) {
if (!stdout) {
return null;
}
try {
const parsed = JSON.parse(stdout);
if (
parsed &&
typeof parsed === "object" &&
typeof parsed.status === "string" &&
Array.isArray(parsed.captured)
) {
return parsed;
}
} catch {
return null;
}
return null;
}
export function classifyMockSdkCaptureError(error) {
const rawMessage = [error?.stderr, error?.stdout, error?.message].filter(Boolean).join("\n");
const missingExport = rawMessage.match(/does not provide an export named ['"]([^'"]+)['"]/)?.[1];
@ -457,9 +503,38 @@ function lineForOffset(text, offset) {
}
function stripComments(text) {
return text
.replace(/\/\*[\s\S]*?\*\//g, (comment) => comment.replace(/[^\n]/g, " "))
.replace(/\/\/.*$/gm, (comment) => " ".repeat(comment.length));
let result = "";
for (let index = 0; index < text.length; index += 1) {
const char = text[index];
const next = text[index + 1];
if (char === "/" && next === "*") {
result += " ";
index += 2;
while (index < text.length && !(text[index] === "*" && text[index + 1] === "/")) {
result += blankCommentChar(text[index]);
index += 1;
}
if (index < text.length) {
result += " ";
index += 1;
}
} else if (char === "/" && next === "/") {
result += " ";
index += 2;
while (index < text.length && text[index] !== "\n" && text[index] !== "\r") {
result += " ";
index += 1;
}
index -= 1;
} else {
result += char;
}
}
return result;
}
function blankCommentChar(char) {
return char === "\n" || char === "\r" ? char : " ";
}
function sortDetails(details) {

View File

@ -23,17 +23,25 @@ export const knownIssueCodes = new Set([
"package-build-artifact-entrypoint",
"package-dependency-install-required",
"package-entrypoint-missing",
"package-install-metadata-incomplete",
"package-json-missing",
"package-manifest-version-drift",
"package-min-host-version-drift",
"package-npm-pack-entrypoint-missing",
"package-npm-pack-metadata-missing",
"package-npm-pack-unavailable",
"package-openclaw-entry-missing",
"package-openclaw-metadata-missing",
"package-openclaw-unsupported-metadata",
"package-plugin-api-compat-missing",
"package-typescript-source-entrypoint",
"provider-auth-env-vars",
"registration-capture-gap",
"runtime-tool-capture",
"reserved-sdk-import",
"security-manifest-schema-unavailable",
"sdk-export-missing",
"unrecognized-security-manifest",
]);
export const issueMetadataByCode = {
@ -85,6 +93,12 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "plugin imports reserved bundled-plugin SDK compatibility subpaths",
},
"security-manifest-schema-unavailable": {
severity: "P3",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "plugin security manifest references an unavailable schema",
},
"missing-compat-record": {
severity: "P1",
owner: "core",
@ -119,7 +133,7 @@ export const issueMetadataByCode = {
severity: "P2",
owner: "inspector",
decision: "inspector-follow-up",
title: "cold import requires isolated dependency installation",
title: "cold import requires dependency installation in an isolated workspace",
},
"package-entrypoint-missing": {
severity: "P1",
@ -127,6 +141,12 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "OpenClaw package entrypoint is missing",
},
"package-install-metadata-incomplete": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "OpenClaw package install metadata is incomplete",
},
"package-json-missing": {
severity: "P2",
owner: "plugin",
@ -139,6 +159,30 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "package and manifest versions drift",
},
"package-min-host-version-drift": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "OpenClaw package minimum host version drifts from build target",
},
"package-npm-pack-entrypoint-missing": {
severity: "P1",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "advertised npm artifact is missing OpenClaw entrypoints",
},
"package-npm-pack-metadata-missing": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "advertised npm artifact is missing OpenClaw metadata",
},
"package-npm-pack-unavailable": {
severity: "P1",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "advertised npm artifact cannot be packed",
},
"package-openclaw-entry-missing": {
severity: "P2",
owner: "plugin",
@ -151,6 +195,12 @@ export const issueMetadataByCode = {
decision: "plugin-upstream-fix",
title: "OpenClaw package metadata is missing",
},
"package-openclaw-unsupported-metadata": {
severity: "P2",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "package declares unsupported OpenClaw metadata",
},
"package-plugin-api-compat-missing": {
severity: "P2",
owner: "plugin",
@ -170,10 +220,10 @@ export const issueMetadataByCode = {
title: "providerAuthEnvVars legacy manifest metadata must stay covered",
},
"registration-capture-gap": {
severity: "P1",
severity: "P2",
owner: "inspector",
decision: "inspector-follow-up",
title: "runtime registrations need capture before contract judgment",
title: "runtime registrations need capture evidence before final contract judgment",
},
"runtime-tool-capture": {
severity: "P2",
@ -193,6 +243,12 @@ export const issueMetadataByCode = {
decision: "core-compat-adapter",
title: "fixture calls a registrar missing from target OpenClaw",
},
"unrecognized-security-manifest": {
severity: "P3",
owner: "plugin",
decision: "plugin-upstream-fix",
title: "plugin ships an unsupported security manifest",
},
};
export function buildIssues({ breakages = [], warnings = [], suggestions = [], targetOpenClaw, idPrefix = "CRABPOT" }) {
@ -212,7 +268,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
owner: finding.owner,
code: finding.code,
decision: finding.decision,
status: finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open",
status: finding.status ?? (finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open"),
issueClass: finding.issueClass,
live: finding.live,
deprecated: finding.deprecated,
@ -220,6 +276,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
title: issueTitle(finding),
evidence: finding.evidence ?? [],
compatRecord: finding.compatRecord ?? null,
runtimeCoverage: finding.runtimeCoverage ?? null,
}));
}
@ -285,7 +342,10 @@ export function summarizeIssueClasses(issues) {
}
function issueClassFor(code, options) {
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)) {
if (code === "sdk-export-missing" && options.compatRecord) {
return "compat-gap";
}
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)) {
return "live-issue";
}
if (code === "missing-compat-record") {
@ -314,10 +374,18 @@ function issueClassFor(code, options) {
"manifest-unknown-fields",
"package-json-missing",
"package-manifest-version-drift",
"package-min-host-version-drift",
"package-npm-pack-entrypoint-missing",
"package-npm-pack-metadata-missing",
"package-npm-pack-unavailable",
"package-openclaw-entry-missing",
"package-openclaw-metadata-missing",
"package-openclaw-unsupported-metadata",
"package-plugin-api-compat-missing",
"package-install-metadata-incomplete",
"reserved-sdk-import",
"security-manifest-schema-unavailable",
"unrecognized-security-manifest",
].includes(code)
) {
return "upstream-metadata";
@ -332,7 +400,7 @@ function severityForClass(code, defaultSeverity, options) {
if (
options.issueClass === "live-issue" &&
["none", "untracked"].includes(options.compatStatus) &&
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)
) {
return "P0";
}

View File

@ -6,6 +6,7 @@ import os from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { createCaptureApi } from "./capture-api.js";
import { captureApiOptionsForPlugin } from "./capture-config.js";
import { createMockSdkPackage } from "./sdk-mock.js";
const options = JSON.parse(process.argv[2] ?? "{}");
@ -30,7 +31,7 @@ async function run(options) {
cleanupTempDirOnExit(workspace);
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
register(pathToFileURL(loaderPath));
return await captureLinkedEntrypoint(entrypoint, options);
return await captureLinkedEntrypoint(entrypoint, { ...options, pluginRoot });
}
function cleanupTempDirOnExit(dir) {
@ -65,7 +66,10 @@ async function captureLinkedEntrypoint(entrypoint, options) {
);
}
const api = createCaptureApi(options.apiOptions);
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
pluginRoot: options.pluginRoot,
});
const api = createCaptureApi(apiOptions);
try {
await register(api);
} catch (error) {
@ -80,7 +84,7 @@ async function captureLinkedEntrypoint(entrypoint, options) {
mockSdk: true,
captured: api.getCapturedContracts(),
};
if (options.apiOptions?.retainHandlers === true) {
if (apiOptions?.retainHandlers === true) {
result.retained = api.getRetainedContracts();
}
return withProcessOutput(result, outputCapture);

View File

@ -106,12 +106,89 @@ export function openClawTargetPathCandidates(manifest, configuredPath) {
export function parseCompatRecordEntries(source) {
const entries = [];
for (const match of source.matchAll(/\{[\s\S]*?\bcode:\s*["'`]([^"'`]+)["'`][\s\S]*?\bstatus:\s*["'`]([^"'`]+)["'`][\s\S]*?\}/g)) {
entries.push({ code: match[1], status: match[2] });
let cursor = 0;
while (cursor < source.length) {
const codeProperty = readStringProperty(source, "code", cursor);
if (!codeProperty) {
break;
}
const statusProperty = readStringProperty(source, "status", codeProperty.end);
if (statusProperty) {
entries.push({ code: codeProperty.value, status: statusProperty.value });
cursor = statusProperty.end;
} else {
cursor = codeProperty.end;
}
}
return dedupeBy(entries, (entry) => entry.code).sort((left, right) => left.code.localeCompare(right.code));
}
function readStringProperty(source, property, fromIndex) {
const propertyIndex = findProperty(source, property, fromIndex);
if (propertyIndex === -1) {
return null;
}
const colonIndex = source.indexOf(":", propertyIndex + property.length);
if (colonIndex === -1) {
return null;
}
let quoteIndex = colonIndex + 1;
while (quoteIndex < source.length && isWhitespace(source[quoteIndex])) {
quoteIndex += 1;
}
if (!isQuote(source[quoteIndex])) {
return null;
}
return readQuotedValue(source, quoteIndex);
}
function findProperty(source, property, fromIndex) {
let index = source.indexOf(property, fromIndex);
while (index !== -1) {
const previous = index === 0 ? "" : source[index - 1];
const next = source[index + property.length] ?? "";
if (!isIdentifierChar(previous) && !isIdentifierChar(next)) {
return index;
}
index = source.indexOf(property, index + property.length);
}
return -1;
}
function readQuotedValue(source, quoteIndex) {
const quote = source[quoteIndex];
let value = "";
for (let index = quoteIndex + 1; index < source.length; index += 1) {
const char = source[index];
if (char === "\\") {
value += source[index + 1] ?? "";
index += 1;
} else if (char === quote) {
return { value, end: index + 1 };
} else {
value += char;
}
}
return null;
}
function isQuote(char) {
return char === '"' || char === "'" || char === "`";
}
function isIdentifierChar(char) {
if (char === "_" || char === "$") {
return true;
}
const code = char.charCodeAt(0);
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
}
function isWhitespace(char) {
return char === " " || char === "\n" || char === "\r" || char === "\t";
}
export function parsePluginSdkExports(packageJson) {
return Object.keys(packageJson.exports ?? {})
.filter((specifier) => specifier === "./plugin-sdk" || specifier.startsWith("./plugin-sdk/"))

View File

@ -12,13 +12,11 @@ export function buildPlatformProbes(options = {}) {
const entrypoints = plan.fixtures.flatMap((fixture) =>
fixture.entrypoints.map((entrypoint) => summarizeEntrypoint(fixture.id, entrypoint)),
);
const portabilityFindings = plan.fixtures.flatMap((fixture) =>
fixture.entrypoints.flatMap((entrypoint) =>
entrypoint.steps
.map((step) => summarizeStep(fixture.id, entrypoint, step))
.filter((finding) => finding.riskCodes.length > 0),
),
const stepFindings = plan.fixtures.flatMap((fixture) =>
fixture.entrypoints.flatMap((entrypoint) => entrypoint.steps.map((step) => summarizeStep(fixture.id, entrypoint, step, options.stepCoverage))),
);
const portabilityFindings = stepFindings.flatMap((finding) => (finding.residual ? [finding.residual] : []));
const coveredPortabilityFindings = stepFindings.flatMap((finding) => (finding.covered ? [finding.covered] : []));
return {
generatedAt: plan.generatedAt,
@ -31,6 +29,7 @@ export function buildPlatformProbes(options = {}) {
jitiAlternativeCount: entrypoints.filter((entrypoint) => entrypoint.loaderAlternatives.includes("jiti")).length,
lazyImportProbeCount: entrypoints.filter((entrypoint) => entrypoint.capturePlanned && entrypoint.syntheticProbePlanned).length,
portabilityFindingCount: portabilityFindings.length,
coveredPortabilityFindingCount: coveredPortabilityFindings.length,
windowsRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("windows")).length,
macosRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("macos")).length,
linuxRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("linux")).length,
@ -38,6 +37,7 @@ export function buildPlatformProbes(options = {}) {
},
entrypoints,
portabilityFindings,
coveredPortabilityFindings,
recommendations: buildRecommendations(portabilityFindings, entrypoints),
};
}
@ -55,8 +55,11 @@ export function validatePlatformProbes(report, options = {}) {
errors.push("all TypeScript loader entrypoints must track a Jiti fallback candidate");
}
for (const entrypoint of report.entrypoints) {
if (entrypoint.loaderPrimary === "tsx" && (!entrypoint.captureUsesTsx || !entrypoint.syntheticUsesTsx)) {
errors.push(`${entrypoint.id}: tsx loader strategy is not reflected in capture and synthetic commands`);
if (
entrypoint.loaderPrimary === "tsx" &&
(!entrypoint.captureUsesTypeScriptLoader || !entrypoint.syntheticUsesTypeScriptLoader)
) {
errors.push(`${entrypoint.id}: TypeScript loader strategy is not reflected in capture and synthetic commands`);
}
}
return errors;
@ -94,9 +97,21 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
entrypoint.loaderAlternatives.join(", ") || "-",
entrypoint.captureUsesTsx ? "yes" : "no",
entrypoint.syntheticUsesTsx ? "yes" : "no",
entrypoint.captureUsesMockSdk ? "yes" : "no",
entrypoint.syntheticUsesMockSdk ? "yes" : "no",
entrypoint.entrypoint,
]),
["Fixture", "Status", "Primary", "Alternatives", "Capture TSX", "Synthetic TSX", "Entrypoint"],
[
"Fixture",
"Status",
"Primary",
"Alternatives",
"Capture TSX",
"Synthetic TSX",
"Capture Mock SDK",
"Synthetic Mock SDK",
"Entrypoint",
],
),
"",
"## Portability Findings",
@ -112,6 +127,19 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
["Fixture", "Step", "Platforms", "Risks", "Mitigation"],
),
"",
"## Covered Portability Findings",
"",
markdownTable(
(report.coveredPortabilityFindings ?? []).map((finding) => [
finding.fixture,
finding.kind,
finding.platforms.join(", ") || "-",
finding.riskCodes.join(", "),
finding.coverage,
]),
["Fixture", "Step", "Platforms", "Covered Risks", "Coverage"],
),
"",
"## Recommendations",
"",
markdownTable(
@ -124,6 +152,10 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
function summarizeEntrypoint(fixtureId, entrypoint) {
const captureStep = entrypoint.steps.find((step) => step.kind === "capture");
const syntheticStep = entrypoint.steps.find((step) => step.kind === "synthetic-probe");
const captureUsesTsx = Boolean(captureStep?.command.includes("--import tsx"));
const syntheticUsesTsx = Boolean(syntheticStep?.command.includes("--import tsx"));
const captureUsesMockSdk = Boolean(captureStep?.command.includes("--mock-sdk"));
const syntheticUsesMockSdk = Boolean(syntheticStep?.command.includes("--mock-sdk"));
return {
fixture: fixtureId,
id: entrypoint.id,
@ -135,21 +167,57 @@ function summarizeEntrypoint(fixtureId, entrypoint) {
loaderAlternatives: entrypoint.loaderStrategy.alternatives,
capturePlanned: Boolean(captureStep),
syntheticProbePlanned: Boolean(syntheticStep),
captureUsesTsx: Boolean(captureStep?.command.includes("--import tsx")),
syntheticUsesTsx: Boolean(syntheticStep?.command.includes("--import tsx")),
captureUsesTsx,
syntheticUsesTsx,
captureUsesMockSdk,
syntheticUsesMockSdk,
captureUsesTypeScriptLoader: captureUsesTsx || captureUsesMockSdk,
syntheticUsesTypeScriptLoader: syntheticUsesTsx || syntheticUsesMockSdk,
};
}
function summarizeStep(fixtureId, entrypoint, step) {
function summarizeStep(fixtureId, entrypoint, step, stepCoverage) {
const riskCodes = stepRiskCodes(step);
return {
const coverage = normalizeStepCoverage(stepCoverage?.({ fixture: fixtureId, entrypoint, step, riskCodes }), riskCodes);
const residualRiskCodes = riskCodes.filter((code) => !coverage.riskCodes.includes(code));
const common = {
fixture: fixtureId,
entrypoint: entrypoint.id,
kind: step.kind,
platforms: platformsForRiskCodes(riskCodes),
riskCodes,
command: step.command,
mitigation: mitigationForRiskCodes(riskCodes),
};
return {
residual:
residualRiskCodes.length > 0
? {
...common,
coveredRiskCodes: coverage.riskCodes,
platforms: platformsForRiskCodes(residualRiskCodes),
riskCodes: residualRiskCodes,
mitigation: mitigationForRiskCodes(residualRiskCodes),
}
: null,
covered:
coverage.riskCodes.length > 0
? {
...common,
coverage: coverage.reason,
platforms: platformsForRiskCodes(coverage.riskCodes),
riskCodes: coverage.riskCodes,
}
: null,
};
}
function normalizeStepCoverage(coverage, riskCodes) {
if (!coverage) {
return { reason: "", riskCodes: [] };
}
const requested = coverage === true ? riskCodes : coverage.coveredRiskCodes ?? coverage.handledRiskCodes ?? coverage.riskCodes ?? coverage;
const covered = Array.isArray(requested) ? requested : [];
return {
reason: typeof coverage.reason === "string" && coverage.reason ? coverage.reason : "covered by isolated workspace executor",
riskCodes: covered.filter((code) => riskCodes.includes(code)).sort(),
};
}
@ -215,7 +283,7 @@ function buildRecommendations(portabilityFindings, entrypoints) {
if (entrypoints.some((entrypoint) => entrypoint.loaderPrimary === "tsx")) {
recommendations.push({
area: "loader",
action: "keep tsx as the source-entrypoint smoke path, add a Jiti execution lane before treating TS plugin source compatibility as covered",
action: "keep mock-SDK TypeScript capture green, add a real host-loader/Jiti lane before treating TS plugin source compatibility as covered",
});
}
if (portabilityFindings.some((finding) => finding.riskCodes.includes("rsync-required"))) {

View File

@ -7,8 +7,12 @@ export async function runProfiledProcess(options) {
let firstRssKb = 0;
let peakRssKb = 0;
let peakCpuPercent = 0;
let statSampleCount = 0;
let rssSampleCount = 0;
let cpuSampleCount = 0;
const cpuSamples = [];
let pollInFlight = false;
const pendingStats = new Set();
const child = spawn(options.command, options.args ?? [], {
cwd: options.cwd,
@ -21,27 +25,43 @@ export async function runProfiledProcess(options) {
child.stderr?.on("data", (chunk) => stderr.push(chunk));
const recordStats = (stats) => {
if (stats.rssKb > 0 && firstRssKb === 0) {
if (stats.rssAvailable || stats.cpuAvailable) {
statSampleCount += 1;
}
if (stats.rssAvailable) {
rssSampleCount += 1;
}
if (stats.cpuAvailable) {
cpuSampleCount += 1;
}
if (stats.rssAvailable && stats.rssKb > 0 && firstRssKb === 0) {
firstRssKb = stats.rssKb;
}
peakRssKb = Math.max(peakRssKb, stats.rssKb);
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
if (stats.cpuPercent > 0) {
if (stats.rssAvailable) {
peakRssKb = Math.max(peakRssKb, stats.rssKb);
}
if (stats.cpuAvailable) {
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
cpuSamples.push(stats.cpuPercent);
}
};
const poll = setInterval(() => {
const sampleStats = () => {
if (pollInFlight) {
return;
}
pollInFlight = true;
readProcessStats(child.pid)
const pending = readProcessStats(child.pid)
.then(recordStats)
.finally(() => {
pollInFlight = false;
pendingStats.delete(pending);
});
}, options.pollMs ?? 100);
pendingStats.add(pending);
};
sampleStats();
const poll = setInterval(sampleStats, options.pollMs ?? 25);
const exitCode = await new Promise((resolve, reject) => {
child.on("error", (error) => {
@ -51,16 +71,10 @@ export async function runProfiledProcess(options) {
child.on("exit", (code) => resolve(code ?? 1));
});
clearInterval(poll);
await Promise.allSettled([...pendingStats]);
const finalStats = await readProcessStats(child.pid);
if (finalStats.rssKb > 0 && firstRssKb === 0) {
firstRssKb = finalStats.rssKb;
}
peakRssKb = Math.max(peakRssKb, finalStats.rssKb);
peakCpuPercent = Math.max(peakCpuPercent, finalStats.cpuPercent);
if (finalStats.cpuPercent > 0) {
cpuSamples.push(finalStats.cpuPercent);
}
recordStats(finalStats);
const wallMs = Math.round(performance.now() - start);
const averageCpuPercent =
@ -79,6 +93,9 @@ export async function runProfiledProcess(options) {
peakCpuPercent: Math.round(peakCpuPercent * 10) / 10,
cpuMsEstimate: Math.round((wallMs * cpuPercentForEstimate) / 100),
harnessHeapDeltaMb: Math.round((heapUsedMb() - heapStartMb) * 10) / 10,
statSampleCount,
rssSampleCount,
cpuSampleCount,
exitCode,
stdoutPreview: previewLines(stdout),
stderrPreview: previewLines(stderr),
@ -87,7 +104,7 @@ export async function runProfiledProcess(options) {
async function readProcessStats(pid) {
if (!pid || process.platform === "win32") {
return { rssKb: 0, cpuPercent: 0 };
return { rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 };
}
return new Promise((resolve) => {
const ps = spawn("ps", ["-o", "rss=", "-o", "%cpu=", "-p", String(pid)], {
@ -95,14 +112,18 @@ async function readProcessStats(pid) {
});
const chunks = [];
ps.stdout.on("data", (chunk) => chunks.push(chunk));
ps.on("error", () => resolve({ rssKb: 0, cpuPercent: 0 }));
ps.on("error", () => resolve({ rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 }));
ps.on("exit", () => {
const [rssRaw, cpuRaw] = Buffer.concat(chunks).toString("utf8").trim().split(/\s+/);
const rssKb = Number.parseInt(rssRaw, 10);
const cpuPercent = Number.parseFloat(cpuRaw);
const rssAvailable = Number.isFinite(rssKb);
const cpuAvailable = Number.isFinite(cpuPercent);
resolve({
rssKb: Number.isFinite(rssKb) ? rssKb : 0,
cpuPercent: Number.isFinite(cpuPercent) ? cpuPercent : 0,
rssAvailable,
rssKb: rssAvailable ? rssKb : 0,
cpuAvailable,
cpuPercent: cpuAvailable ? cpuPercent : 0,
});
});
});

View File

@ -0,0 +1,23 @@
#!/usr/bin/env node
import { readFile, writeFile } from "node:fs/promises";
import path from "node:path";
const packageJsonPath = path.resolve(process.cwd(), "package.json");
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
let changed = false;
for (const [name, specifier] of Object.entries(packageJson.devDependencies ?? {})) {
if (typeof specifier === "string" && specifier.startsWith("workspace:")) {
delete packageJson.devDependencies[name];
changed = true;
}
}
if (packageJson.devDependencies && Object.keys(packageJson.devDependencies).length === 0) {
delete packageJson.devDependencies;
changed = true;
}
if (changed) {
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
}

42
src/report-sanitizer.js Normal file
View File

@ -0,0 +1,42 @@
import path from "node:path";
export function sanitizeReportArtifact(report, options = {}) {
const sensitivePaths = sensitiveOpenClawPaths(report);
if (sensitivePaths.length === 0) {
return report;
}
const placeholder = options.openclawPathPlaceholder ?? "<OPENCLAW_PATH>";
return sanitizeValue(report, sensitivePaths, placeholder);
}
function sensitiveOpenClawPaths(report) {
const targetOpenClaw = report?.targetOpenClaw;
return unique(
[targetOpenClaw?.configuredPath, ...(targetOpenClaw?.searchedPaths ?? [])]
.filter((value) => typeof value === "string" && isAbsolutePath(value))
.sort((left, right) => right.length - left.length),
);
}
function sanitizeValue(value, sensitivePaths, placeholder) {
if (typeof value === "string") {
return sensitivePaths.reduce((result, sensitivePath) => result.replaceAll(sensitivePath, placeholder), value);
}
if (Array.isArray(value)) {
return value.map((item) => sanitizeValue(item, sensitivePaths, placeholder));
}
if (value && typeof value === "object") {
return Object.fromEntries(
Object.entries(value).map(([key, entryValue]) => [key, sanitizeValue(entryValue, sensitivePaths, placeholder)]),
);
}
return value;
}
function isAbsolutePath(value) {
return path.isAbsolute(value) || /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith("\\\\");
}
function unique(values) {
return [...new Set(values)];
}

View File

@ -4,6 +4,8 @@ import { renderCompatibilityIssuesReport, renderCompatibilityMarkdownReport } fr
import { buildContractProbes } from "./contract-probes.js";
import { classifyCompatibilityFixture } from "./fixture-summary.js";
import { buildIssues, summarizeIssueClasses } from "./issues.js";
import { sanitizeReportArtifact } from "./report-sanitizer.js";
import { applyRuntimeExecutionCoverage } from "./runtime-reconciliation.js";
export function buildReport({ config, inspections, failures = [], generatedAt = "deterministic" }) {
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
@ -139,6 +141,10 @@ export async function buildCompatibilityReport(options = {}) {
decisions,
});
const runtimeCoverage = applyRuntimeExecutionCoverage({
findings: [...warnings, ...suggestions],
executionResults: options.executionResults,
});
const issues = buildIssues({
breakages,
warnings,
@ -148,6 +154,8 @@ export async function buildCompatibilityReport(options = {}) {
});
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures: fixtureReports });
const issueSummary = summarizeIssueClasses(issues);
const openIssues = issues.filter((issue) => issue.status !== "runtime-covered");
const openIssueSummary = summarizeIssueClasses(openIssues);
return {
generatedAt: options.generatedAt ?? "deterministic",
@ -162,8 +170,11 @@ export async function buildCompatibilityReport(options = {}) {
decisionCount: decisions.length,
logCount: logs.length,
issueCount: issues.length,
openIssueCount: openIssues.length,
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
openP0IssueCount: openIssues.filter((issue) => issue.severity === "P0").length,
openP1IssueCount: openIssues.filter((issue) => issue.severity === "P1").length,
liveIssueCount: issueSummary["live-issue"],
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
compatGapCount: issueSummary["compat-gap"],
@ -171,6 +182,10 @@ export async function buildCompatibilityReport(options = {}) {
inspectorGapCount: issueSummary["inspector-gap"],
upstreamIssueCount: issueSummary["upstream-metadata"],
fixtureRegressionCount: issueSummary["fixture-regression"],
openInspectorGapCount: openIssueSummary["inspector-gap"],
runtimeCoveredIssueCount: runtimeCoverage.coveredFindingCount,
runtimePartiallyCoveredIssueCount: runtimeCoverage.partiallyCoveredFindingCount,
runtimeCoverageArtifactCount: runtimeCoverage.coverage.artifactCount,
contractProbeCount: contractProbes.length,
},
fixtures: fixtureReports,
@ -210,6 +225,10 @@ export function classifyCompatRecordCoverage({ targetOpenClaw, findings, suggest
continue;
}
if (finding.code === "sdk-export-missing") {
continue;
}
suggestions.push({
fixture: finding.fixture,
code: "missing-compat-record",
@ -233,12 +252,13 @@ export async function writeReport(report, options = {}) {
const basename = options.basename ?? "plugin-inspector-report";
const jsonPath = path.join(outDir, `${basename}.json`);
const markdownPath = path.join(outDir, `${basename}.md`);
const artifactReport = sanitizeReportArtifact(report, options);
return writeJsonMarkdownArtifacts({
jsonPath,
markdownPath,
json: report,
markdown: renderMarkdownReport(report),
json: artifactReport,
markdown: renderMarkdownReport(artifactReport),
check: options.check,
});
}
@ -249,6 +269,7 @@ export async function writeCompatibilityReport(report, options = {}) {
const jsonPath = options.jsonPath ?? path.join(outDir, `${basename}.json`);
const markdownPath = options.markdownPath ?? path.join(outDir, `${basename}.md`);
const issuesPath = options.issuesPath ?? path.join(outDir, options.issuesBasename ?? "plugin-inspector-issues.md");
const artifactReport = sanitizeReportArtifact(report, options);
const markdownOptions = compatibilityRenderOptions(options, {
title: options.markdownTitle ?? options.title,
...options.markdownOptions,
@ -260,14 +281,16 @@ export async function writeCompatibilityReport(report, options = {}) {
return writeArtifacts(
[
{ name: "jsonPath", path: jsonPath, json: report },
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(report, markdownOptions) },
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(report, issuesOptions) },
{ name: "jsonPath", path: jsonPath, json: artifactReport },
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(artifactReport, markdownOptions) },
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(artifactReport, issuesOptions) },
],
{ check: options.check },
);
}
export { sanitizeReportArtifact };
function compatibilityRenderOptions(options, overrides) {
const renderOptions = {
formatEvidence: options.formatEvidence,
@ -303,7 +326,11 @@ function topTextFindings(report, limit) {
return [
...(report.breakages ?? []).map((finding) => formatTextFinding(finding, "breakage")),
...(report.issues ?? [])
.filter((issue) => issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1")
.filter(
(issue) =>
issue.status !== "runtime-covered" &&
(issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1"),
)
.map((issue) => formatTextFinding(issue, issue.severity ?? "issue")),
...(report.warnings ?? []).map((finding) => formatTextFinding(finding, "warning")),
].slice(0, limit);

View File

@ -46,8 +46,8 @@ export async function buildRuntimeProfile(options = {}) {
os: process.platform,
arch: process.arch,
node: process.version,
rssSampler: process.platform === "win32" ? "unavailable" : "ps",
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent",
rssSampler: process.platform === "win32" ? "unavailable" : "ps-immediate-25ms",
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent-immediate-25ms",
},
summary: summarizeProfile(commands),
groups: summarizeCommandGroups(commands),
@ -65,8 +65,8 @@ export function validateRuntimeProfile(profile) {
errors.push(`${command.id}: missing wall time`);
}
}
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => command.peakRssMb.max <= 0)) {
errors.push("all commands are missing peak RSS");
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => !hasRssSample(command))) {
errors.push("all commands are missing peak RSS samples");
}
return errors;
}
@ -98,10 +98,17 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
[
["Commands", profile.summary.commandCount],
["P50 wall time", `${profile.summary.p50WallMs} ms`],
["P95 wall time", `${profile.summary.p95WallMs} ms`],
["Max peak RSS", `${profile.summary.maxPeakRssMb} MB`],
["Max RSS delta", `${profile.summary.maxRssDeltaMb} MB`],
["Max CPU estimate", `${profile.summary.maxCpuMsEstimate} ms`],
["Command P95 wall time", `${profile.summary.p95WallMs} ms`],
["Wall time basis", profile.summary.wallTimeBasis ?? "command-median-p95"],
["Profile samples", profile.summary.sampleCount ?? sampleCount(profile.commands)],
["RSS samples", profile.summary.rssSampleCount ?? rssSampleCount(profile.commands)],
["CPU samples", profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands)],
["Max peak RSS", formatSampledMetric(profile.summary.maxPeakRssMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
["Max RSS delta", formatSampledMetric(profile.summary.maxRssDeltaMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
[
"Max CPU estimate",
formatSampledMetric(profile.summary.maxCpuMsEstimate, profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands), "ms"),
],
["Max harness heap delta", `${profile.summary.maxHarnessHeapDeltaMb} MB`],
],
["Metric", "Value"],
@ -129,13 +136,14 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
command.label,
`${command.wallMs.median} ms`,
`${command.wallMs.max} ms`,
`${command.peakRssMb.max} MB`,
`${command.rssDeltaMb.max} MB`,
`${command.cpuMsEstimate.max} ms`,
formatSampledMetric(command.peakRssMb.max, command.rssSampleCount),
formatSampledMetric(command.rssDeltaMb.max, command.rssSampleCount),
formatSampledMetric(command.cpuMsEstimate.max, command.cpuSampleCount, "ms"),
`${command.harnessHeapDeltaMb.max} MB`,
`${command.rssSampleCount ?? 0}/${command.cpuSampleCount ?? 0}`,
command.exitCodes.join(", "),
]),
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "Exit codes"],
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "RSS/CPU samples", "Exit codes"],
),
"",
"## Category Rollups",
@ -146,11 +154,12 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
group.commandCount,
`${group.p50WallMs} ms`,
`${group.p95WallMs} ms`,
`${group.maxPeakRssMb} MB`,
`${group.maxCpuMsEstimate} ms`,
formatSampledMetric(group.maxPeakRssMb, group.rssSampleCount),
formatSampledMetric(group.maxCpuMsEstimate, group.cpuSampleCount, "ms"),
`${group.rssSampleCount ?? 0}/${group.cpuSampleCount ?? 0}`,
group.commands.join(", "),
]),
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "Command IDs"],
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "RSS/CPU samples", "Command IDs"],
),
].join("\n");
}
@ -189,8 +198,15 @@ function summarizeProfile(commands) {
const maxRssDeltaMb = Math.max(0, ...commands.map((command) => command.rssDeltaMb.max));
const maxCpuMsEstimate = Math.max(0, ...commands.map((command) => command.cpuMsEstimate.max));
const maxHarnessHeapDeltaMb = Math.max(0, ...commands.map((command) => command.harnessHeapDeltaMb.max));
const totalSampleCount = sampleCount(commands);
const totalRssSampleCount = rssSampleCount(commands);
const totalCpuSampleCount = cpuSampleCount(commands);
return {
commandCount: commands.length,
sampleCount: totalSampleCount,
rssSampleCount: totalRssSampleCount,
cpuSampleCount: totalCpuSampleCount,
wallTimeBasis: "command-median-p95",
p50WallMs: percentile(wallTimes, 0.5),
p95WallMs: percentile(wallTimes, 0.95),
maxPeakRssMb,
@ -206,6 +222,12 @@ function summarizeCommand(command, samples) {
const rssDeltaMb = samples.map((sample) => sample.rssDeltaMb).sort((left, right) => left - right);
const peakCpuPercent = samples.map((sample) => sample.peakCpuPercent).sort((left, right) => left - right);
const cpuMsEstimate = samples.map((sample) => sample.cpuMsEstimate).sort((left, right) => left - right);
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
const rssSampleTotal = samples.reduce(
(sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)),
0,
);
const cpuSampleTotal = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
const harnessHeapDeltaMb = samples
.map((sample) => sample.harnessHeapDeltaMb)
.sort((left, right) => left - right);
@ -222,6 +244,9 @@ function summarizeCommand(command, samples) {
peakCpuPercent: summarizeNumbers(peakCpuPercent),
cpuMsEstimate: summarizeNumbers(cpuMsEstimate),
harnessHeapDeltaMb: summarizeNumbers(harnessHeapDeltaMb),
statSampleCount,
rssSampleCount: rssSampleTotal,
cpuSampleCount: cpuSampleTotal,
exitCodes: [...new Set(samples.map((sample) => sample.exitCode))].sort(),
};
}
@ -244,6 +269,8 @@ function summarizeCommandGroups(commands) {
const cpuMs = categoryCommands
.flatMap((command) => command.samples.map((sample) => sample.cpuMsEstimate))
.sort((left, right) => left - right);
const groupRssSampleCount = rssSampleCount(categoryCommands);
const groupCpuSampleCount = cpuSampleCount(categoryCommands);
return {
category,
commandCount: categoryCommands.length,
@ -251,11 +278,36 @@ function summarizeCommandGroups(commands) {
p95WallMs: percentile(wallTimes, 0.95),
maxPeakRssMb: peakRss.at(-1) ?? 0,
maxCpuMsEstimate: cpuMs.at(-1) ?? 0,
rssSampleCount: groupRssSampleCount,
cpuSampleCount: groupCpuSampleCount,
commands: categoryCommands.map((command) => command.id),
};
});
}
function hasRssSample(command) {
return (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)) > 0;
}
function sampleCount(commands) {
return commands.reduce((sum, command) => sum + (command.samples?.length ?? 0), 0);
}
function rssSampleCount(commands) {
return commands.reduce((sum, command) => sum + (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)), 0);
}
function cpuSampleCount(commands) {
return commands.reduce((sum, command) => sum + (command.cpuSampleCount ?? 0), 0);
}
function formatSampledMetric(value, count, unit = "MB") {
if ((count ?? 0) <= 0) {
return "n/a";
}
return `${value} ${unit}`;
}
function summarizeNumbers(values) {
return {
min: values[0],

View File

@ -0,0 +1,124 @@
export function applyRuntimeExecutionCoverage({ findings = [], executionResults } = {}) {
const coverage = buildRuntimeExecutionCoverage(executionResults);
let coveredFindingCount = 0;
let partiallyCoveredFindingCount = 0;
for (const finding of findings) {
const findingCoverage = runtimeCoverageForFinding(finding, coverage);
if (!findingCoverage) {
continue;
}
finding.runtimeCoverage = findingCoverage;
if (findingCoverage.status === "covered") {
finding.status = "runtime-covered";
coveredFindingCount += 1;
} else {
partiallyCoveredFindingCount += 1;
}
}
return {
coverage,
coveredFindingCount,
partiallyCoveredFindingCount,
};
}
export function buildRuntimeExecutionCoverage(executionResults) {
const fixtures = new Map();
for (const artifact of executionResults?.artifacts ?? []) {
if (artifact.kind !== "capture") {
continue;
}
const fixture = String(artifact.fixture ?? "unknown");
const fixtureCoverage = ensureFixtureCoverage(fixtures, fixture);
if (artifact.artifactPath) {
fixtureCoverage.artifacts.add(artifact.artifactPath);
}
for (const captured of normalizeCaptured(artifact.captured)) {
fixtureCoverage.captured.add(captured);
}
}
return {
fixtures,
artifactCount: [...fixtures.values()].reduce((sum, fixture) => sum + fixture.artifacts.size, 0),
};
}
function runtimeCoverageForFinding(finding, coverage) {
const fixtureCoverage = coverage.fixtures.get(finding.fixture);
if (!fixtureCoverage || fixtureCoverage.captured.size === 0) {
return null;
}
const expected = expectedRuntimeCaptureKeys(finding);
if (expected.length === 0) {
return null;
}
const captured = expected.filter((item) => fixtureCoverage.captured.has(item));
if (captured.length === 0) {
return null;
}
return {
status: captured.length === expected.length ? "covered" : "partial",
captured,
expected,
artifacts: [...fixtureCoverage.artifacts].sort(),
};
}
function expectedRuntimeCaptureKeys(finding) {
const names = evidenceNames(finding.evidence);
if (finding.code === "registration-capture-gap") {
return names.map((name) => `registration:${name}`);
}
if (finding.code === "runtime-tool-capture") {
return ["registration:registerTool"];
}
if (finding.code === "conversation-access-hook") {
return names.map((name) => `hook:${name}`);
}
return [];
}
function normalizeCaptured(captured) {
return (captured ?? [])
.map((item) => {
if (typeof item === "string") {
return item;
}
if (item && typeof item === "object" && item.kind && item.name) {
return `${item.kind}:${item.name}`;
}
return "";
})
.filter(Boolean);
}
function evidenceNames(evidence) {
return [
...new Set(
(evidence ?? [])
.map((item) => String(item).split(" @ ")[0]?.trim())
.filter(Boolean),
),
];
}
function ensureFixtureCoverage(fixtures, fixture) {
let fixtureCoverage = fixtures.get(fixture);
if (!fixtureCoverage) {
fixtureCoverage = {
artifacts: new Set(),
captured: new Set(),
};
fixtures.set(fixture, fixtureCoverage);
}
return fixtureCoverage;
}

View File

@ -254,9 +254,20 @@ export async function createMockSdkPackage(rootDir, options = {}) {
)}\n`,
"utf8",
);
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(), "utf8");
const rootExportNames = new Set([
...mockSdkExportNames,
...(imports.bySpecifier.get("openclaw/plugin-sdk") ?? []),
]);
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(rootExportNames), "utf8");
for (const [subpath, exportNames] of Object.entries(mockSdkSubpathExports)) {
await writeFile(path.join(pluginSdkDir, `${subpath}.js`), mockSdkSubpathSource(exportNames), "utf8");
const specifier = `openclaw/plugin-sdk/${subpath}`;
await writeFile(
path.join(pluginSdkDir, `${subpath}.js`),
mockSdkSubpathSource(exportNames, imports.bySpecifier.get(specifier) ?? new Set(), {
zod: subpath === "zod",
}),
"utf8",
);
}
for (const specifier of imports.openclawSdkSpecifiers) {
if (specifier === "openclaw/plugin-sdk") {
@ -428,6 +439,9 @@ export async function resolve(specifier, context, nextResolve) {
const subpath = specifier.slice("openclaw/plugin-sdk/".length);
return moduleUrl(path.join(pluginSdkDir, \`\${subpath}.js\`));
}
if (externalMap.has(specifier)) {
return moduleUrl(externalMap.get(specifier));
}
try {
return await nextResolve(specifier, context);
} catch (error) {
@ -531,6 +545,15 @@ function genericExportStatement(name) {
if (["createChatChannelPlugin", "createPlugin", "defineChannelPluginEntry", "definePlugin", "definePluginEntry", "defineSetupPluginEntry"].includes(name)) {
return name === "definePluginEntry" ? "export { definePluginEntry };" : `export const ${name} = definePluginEntry;`;
}
if (name === "defineBundledChannelEntry") {
return "export { defineBundledChannelEntry };";
}
if (name === "defineBundledChannelSetupEntry") {
return "export { defineBundledChannelSetupEntry };";
}
if (name === "loadBundledEntryExportSync") {
return "export { loadBundledEntryExportSync };";
}
if (/^[A-Z].*Schema$/u.test(name)) {
return `export const ${name} = createSchema();`;
}
@ -538,7 +561,13 @@ function genericExportStatement(name) {
}
function genericMockRuntimeSource(options = {}) {
return `${options.includeSdkRuntime ? `function definePluginEntry(entry) {
return `${options.includeSdkRuntime ? `import { existsSync } from "node:fs";
import path from "node:path";
import { fileURLToPath, pathToFileURL } from "node:url";
const pendingBundledEntryLoads = new Set();
function definePluginEntry(entry) {
if (entry && typeof entry === "object" && typeof entry.register === "function") {
return entry;
}
@ -547,9 +576,86 @@ function genericMockRuntimeSource(options = {}) {
}
return typeof entry === "function" ? { register: entry } : entry;
}
function defineBundledChannelEntry(entry = {}) {
return {
...entry,
kind: "bundled-channel-entry",
async register(api) {
if (api?.registrationMode === "cli-metadata") {
return entry.registerCliMetadata?.(api);
}
if (api?.registrationMode !== "tool-discovery") {
api?.registerChannel?.({
id: entry.id,
name: entry.name,
description: entry.description,
plugin: { id: entry.id, name: entry.name },
});
}
entry.registerCliMetadata?.(api);
const result = entry.registerFull?.(api);
if (result && typeof result.then === "function") {
await result;
}
await drainBundledEntryLoads();
return result;
},
};
}
function defineBundledChannelSetupEntry(entry = {}) {
return {
...entry,
kind: "bundled-channel-setup-entry",
};
}
function loadBundledEntryExportSync(importMetaUrl, options = {}) {
return (...args) => {
const promise = import(resolveBundledEntryUrl(importMetaUrl, options.specifier)).then((module) => {
const loaded = module[options.exportName] ?? module.default;
return typeof loaded === "function" ? loaded(...args) : loaded;
});
pendingBundledEntryLoads.add(promise);
promise.finally(() => pendingBundledEntryLoads.delete(promise));
return promise;
};
}
async function drainBundledEntryLoads() {
while (pendingBundledEntryLoads.size > 0) {
await Promise.all([...pendingBundledEntryLoads]);
}
}
function resolveBundledEntryUrl(importMetaUrl, specifier) {
const basePath = fileURLToPath(importMetaUrl);
const target = specifier ? path.resolve(path.dirname(basePath), specifier) : basePath;
const resolved = resolveExistingSourcePath(target);
return pathToFileURL(resolved).href;
}
function resolveExistingSourcePath(target) {
if (existsSync(target)) {
return target;
}
const parsed = path.parse(target);
const withoutJsExtension = [".js", ".mjs", ".cjs"].includes(parsed.ext) ? path.join(parsed.dir, parsed.name) : null;
const candidates = [
...(withoutJsExtension ? [\`\${withoutJsExtension}.ts\`, \`\${withoutJsExtension}.mts\`, \`\${withoutJsExtension}.cts\`] : []),
\`\${target}.js\`,
\`\${target}.mjs\`,
\`\${target}.ts\`,
];
return candidates.find((candidate) => existsSync(candidate)) ?? target;
}
` : ""}
function createMockValue(name) {
function fn(...args) {
if (name === "resolvePreferredOpenClawTmpDir") {
return process.env.TMPDIR || "/tmp";
}
if (name.startsWith("normalize")) {
return typeof args[0] === "string" ? args[0] : "";
}
@ -728,7 +834,8 @@ function createTypeNamespace() {
`;
}
function mockSdkSource() {
function mockSdkSource(exportNames = mockSdkExportNames) {
const dynamicExportNames = [...exportNames].filter((name) => !mockSdkExportNames.includes(name));
return `function normalizeEntry(entry) {
return typeof entry === "function" ? { register: entry } : entry;
}
@ -1069,16 +1176,36 @@ export function normalizeSecretInputString(value) {
return String(value ?? "").trim();
}
function createSimpleSchema(defaultValue) {
return {
parse(value) {
return value === undefined ? defaultValue : value;
},
default(value) {
return createSimpleSchema(value);
},
optional() {
return this;
},
nullable() {
return this;
},
nullish() {
return this;
},
};
}
export function buildSecretInputSchema() {
return { type: "string" };
return createSimpleSchema();
}
export function buildOptionalSecretInputSchema() {
return { anyOf: [buildSecretInputSchema(), { type: "undefined" }] };
return createSimpleSchema();
}
export function buildSecretInputArraySchema() {
return { type: "array", items: buildSecretInputSchema() };
return createSimpleSchema([]);
}
export function registerPluginHttpRoute(options = {}) {
@ -1538,15 +1665,20 @@ export const OPENAI_RESPONSES_STREAM_HOOKS = buildProviderStreamFamilyHooks("ope
export const OPENROUTER_THINKING_STREAM_HOOKS = buildProviderStreamFamilyHooks("openrouter-thinking");
export const TOOL_STREAM_DEFAULT_ON_HOOKS = buildProviderStreamFamilyHooks("tool-stream-default");
export const pluginSdkMock = true;
${dynamicExportNames.map(genericExportStatement).join("\n")}
export default {
${mockSdkExportNames.map((name) => ` ${name},`).join("\n")}
${[...exportNames].map((name) => ` ${name},`).join("\n")}
};
`;
}
function mockSdkSubpathSource(exportNames) {
return `${exportNames.map((name) => `export { ${name} } from "./index.js";`).join("\n")}
function mockSdkSubpathSource(staticExportNames, importedExportNames, options = {}) {
const staticNames = new Set(staticExportNames);
const dynamicNames = [...importedExportNames].filter((name) => !staticNames.has(name));
return `${[...staticNames].map((name) => `export { ${name} } from "./index.js";`).join("\n")}
${dynamicNames.length > 0 ? genericMockRuntimeSource({ includeSdkRuntime: true, zod: options.zod }) : ""}
${dynamicNames.map(genericExportStatement).join("\n")}
export { default } from "./index.js";
`;
}

View File

@ -1,11 +1,21 @@
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
export const syntheticRegistrationExecutionProfiles = {
createChatChannelPlugin: {
mode: "metadata-only",
callableProperties: [],
reason: "channel plugin factory metadata is captured before channel runtime execution",
},
defineChannelPluginEntry: {
mode: "metadata-only",
callableProperties: [],
reason: "entry wrapper metadata is captured before channel runtime execution",
},
defineBundledChannelEntry: {
mode: "metadata-only",
callableProperties: [],
reason: "bundled channel entry metadata is captured before channel runtime execution",
},
definePluginEntry: {
mode: "metadata-only",
callableProperties: [],
@ -16,19 +26,74 @@ export const syntheticRegistrationExecutionProfiles = {
callableProperties: ["send", "sendMessage", "receive", "handleMessage"],
option: "includeChannelRuntime",
},
registerAgentHarness: {
mode: "metadata-only",
callableProperties: [],
reason: "agent harness factories are captured as registration metadata; agent runtime execution remains isolated opt-in",
},
registerAgentEventSubscription: {
mode: "metadata-only",
callableProperties: [],
reason: "agent event subscriptions are captured as registration metadata before agent event dispatch",
},
registerAgentToolResultMiddleware: {
mode: "metadata-only",
callableProperties: [],
reason: "agent tool-result middleware is captured as registration metadata before tool-result pipeline execution",
},
registerAutoEnableProbe: {
mode: "metadata-only",
callableProperties: [],
reason: "auto-enable probes are captured as registration metadata before runtime activation checks",
},
registerCli: {
mode: "direct",
callableProperties: ["handler", "run", "execute"],
},
registerCliBackend: {
mode: "metadata-only",
callableProperties: [],
reason: "CLI backend descriptors are captured as registration metadata before backend process execution",
},
registerCommand: {
mode: "direct",
callableProperties: ["handler", "run", "execute"],
},
registerCodexAppServerExtensionFactory: {
mode: "metadata-only",
callableProperties: [],
reason: "Codex app server extension factories are captured as registration metadata before host UI execution",
},
registerCompactionProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "compaction providers are captured as registration metadata before compaction runtime execution",
},
registerConfigMigration: {
mode: "metadata-only",
callableProperties: [],
reason: "config migrations are captured as registration metadata before mutating stored plugin config",
},
registerContextEngine: {
mode: "metadata-only",
callableProperties: [],
reason: "context engine factories are captured as registration metadata; engine startup remains isolated opt-in",
},
registerControlUiDescriptor: {
mode: "metadata-only",
callableProperties: [],
reason: "control UI descriptors are captured as registration metadata before UI composition",
},
registerDetachedTaskRuntime: {
mode: "metadata-only",
callableProperties: [],
reason: "detached task runtimes are captured as registration metadata before async task execution",
},
registerGatewayDiscoveryService: {
mode: "metadata-only",
callableProperties: [],
reason: "gateway discovery services are captured as registration metadata before network discovery execution",
},
registerGatewayMethod: {
mode: "direct",
callableProperties: ["handler", "run", "execute", "invoke"],
@ -46,21 +111,116 @@ export const syntheticRegistrationExecutionProfiles = {
callableProperties: [],
reason: "legacy hook registrar is captured as metadata; hook handlers are probed through hook events",
},
registerImageGenerationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "image generation providers are captured as registration metadata before provider runtime execution",
},
registerMemoryPromptSection: {
mode: "metadata-only",
callableProperties: [],
reason: "memory prompt section renderers are captured as metadata before prompt-runtime execution",
},
registerMediaUnderstandingProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "media understanding providers are captured as registration metadata before provider runtime execution",
},
registerMemoryCapability: {
mode: "metadata-only",
callableProperties: [],
reason: "memory capabilities are captured as registration metadata before memory runtime execution",
},
registerMemoryCorpusSupplement: {
mode: "metadata-only",
callableProperties: [],
reason: "memory corpus supplements are captured as registration metadata before memory runtime execution",
},
registerMemoryEmbeddingProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "memory embedding providers are captured as registration metadata before provider runtime execution",
},
registerMemoryFlushPlan: {
mode: "metadata-only",
callableProperties: [],
reason: "memory flush plans are captured as registration metadata before memory runtime execution",
},
registerMemoryRuntime: {
mode: "metadata-only",
callableProperties: [],
reason: "memory runtime factories are captured as metadata; external memory startup remains isolated opt-in",
},
registerMemoryPromptSupplement: {
mode: "metadata-only",
callableProperties: [],
reason: "memory prompt supplements are captured as registration metadata before prompt-runtime execution",
},
registerMigrationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "migration providers are captured as registration metadata before migration runtime execution",
},
registerMusicGenerationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "music generation providers are captured as registration metadata before provider runtime execution",
},
registerNodeHostCommand: {
mode: "metadata-only",
callableProperties: [],
reason: "node host commands are captured as registration metadata before host process execution",
},
registerNodeInvokePolicy: {
mode: "metadata-only",
callableProperties: [],
reason: "node invoke policies are captured as registration metadata before host authorization checks",
},
registerProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "provider descriptors are captured as registration metadata before provider runtime execution",
},
registerRealtimeTranscriptionProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "realtime transcription providers are captured as registration metadata before provider runtime execution",
},
registerRealtimeVoiceProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "realtime voice providers are captured as registration metadata before provider runtime execution",
},
registerReload: {
mode: "metadata-only",
callableProperties: [],
reason: "reload handlers are captured as registration metadata before runtime reload execution",
},
registerRuntimeLifecycle: {
mode: "metadata-only",
callableProperties: [],
reason: "runtime lifecycle handlers are captured as registration metadata before lifecycle dispatch",
},
registerSecurityAuditCollector: {
mode: "metadata-only",
callableProperties: [],
reason: "security audit collectors are captured as registration metadata before filesystem or policy scans",
},
registerService: {
mode: "lifecycle-opt-in",
callableProperties: ["start", "stop", "dispose"],
option: "includeLifecycle",
},
registerSessionExtension: {
mode: "metadata-only",
callableProperties: [],
reason: "session extensions are captured as registration metadata before session runtime execution",
},
registerSessionSchedulerJob: {
mode: "metadata-only",
callableProperties: [],
reason: "session scheduler jobs are captured as registration metadata before scheduler execution",
},
registerSpeechProvider: {
mode: "provider-opt-in",
callableProperties: ["speak", "synthesize", "tts"],
@ -70,6 +230,36 @@ export const syntheticRegistrationExecutionProfiles = {
mode: "direct",
callableProperties: ["run", "handler", "execute"],
},
registerToolMetadata: {
mode: "metadata-only",
callableProperties: [],
reason: "tool metadata descriptors are captured as registration metadata before tool runtime execution",
},
registerTextTransforms: {
mode: "metadata-only",
callableProperties: [],
reason: "text transforms are captured as registration metadata before content mutation execution",
},
registerVideoGenerationProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "video generation providers are captured as registration metadata before provider runtime execution",
},
registerTrustedToolPolicy: {
mode: "metadata-only",
callableProperties: [],
reason: "trusted tool policies are captured as registration metadata before trust-policy enforcement",
},
registerWebFetchProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "web fetch providers are captured as registration metadata before provider runtime execution",
},
registerWebSearchProvider: {
mode: "metadata-only",
callableProperties: [],
reason: "web search providers are captured as registration metadata before provider runtime execution",
},
};
export const defaultSyntheticHookEvents = {
@ -186,6 +376,7 @@ export const defaultSyntheticHookContexts = {
};
export const defaultSyntheticRegistrationArguments = {
createChatChannelPlugin: [{ base: { id: "fixture-channel" }, outbound: { sendText: "function" } }],
defineChannelPluginEntry: [{ id: "fixture-channel", setup: "function", receive: "function" }],
definePluginEntry: [{ id: "fixture-plugin", register: "function" }],
registerChannel: [{ id: "fixture-channel", send: "function", receive: "function" }],

View File

@ -71,6 +71,7 @@ export async function buildWorkspacePlan(options = {}) {
installStepCount: allSteps.filter((step) => step.kind === "install").length,
auditStepCount: allSteps.filter((step) => step.kind === "audit").length,
buildStepCount: allSteps.filter((step) => step.kind === "build").length,
pruneDevWorkspaceDependencyStepCount: allSteps.filter((step) => step.kind === "prune-dev-workspace-deps").length,
artifactStepCount: allSteps.filter((step) => step.kind === "prepare-artifacts").length,
captureStepCount: allSteps.filter((step) => step.kind === "capture").length,
syntheticProbeStepCount: allSteps.filter((step) => step.kind === "synthetic-probe").length,
@ -179,6 +180,7 @@ export function renderWorkspacePlanMarkdown(plan, options = {}) {
["Artifact dirs", plan.summary.artifactStepCount],
["Install steps", plan.summary.installStepCount],
["Audit steps", plan.summary.auditStepCount],
["Prune dev workspace dependency steps", plan.summary.pruneDevWorkspaceDependencyStepCount],
["Build steps", plan.summary.buildStepCount],
["Capture steps", plan.summary.captureStepCount],
["Synthetic probe steps", plan.summary.syntheticProbeStepCount],
@ -218,7 +220,7 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
const packageManager = detectPackageManager(settings.rootDir, packageDir, packageJson);
const lockfile = findNearestLockfile(settings.rootDir, packageDir);
const buildScript = packageJson.scripts?.build;
const requiredCapabilities = requiredCapabilitiesFor(entrypoint);
const requiredCapabilities = requiredCapabilitiesFor(entrypoint, packageSummary);
const loaderStrategy = loaderStrategyFor(entrypoint);
const blockers = [...entrypoint.blockers];
const workspacePath = posixJoin(settings.workspaceRoot, fixtureId);
@ -248,6 +250,14 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
}
if (requiredCapabilities.includes("dependency-install")) {
if (hasWorkspaceProtocolDevDependencies(packageJson)) {
steps.push({
kind: "prune-dev-workspace-deps",
command: `node ${helperScript(settings, workspacePath, settings.pruneWorkspaceDevDepsScript, "prune-workspace-dev-deps-cli.js")}`,
cwd: workspacePath,
reason: "remove workspace: devDependencies from the isolated runtime install; the mock SDK supplies OpenClaw host imports",
});
}
steps.push({
kind: "install",
command: installCommand(packageManager),
@ -319,6 +329,7 @@ function workspaceSettings(options) {
resultsRoot: repoRelative(options.resultsRoot ?? defaultWorkspacePlanOptions.resultsRoot),
rootDir: path.resolve(options.rootDir ?? process.cwd()),
syntheticProbeScript: options.syntheticProbeScript ?? defaultWorkspacePlanOptions.syntheticProbeScript,
pruneWorkspaceDevDepsScript: options.pruneWorkspaceDevDepsScript,
workspaceRoot: repoRelative(options.workspaceRoot ?? defaultWorkspacePlanOptions.workspaceRoot),
};
}
@ -342,7 +353,7 @@ function loaderStrategyFor(entrypoint) {
};
}
function requiredCapabilitiesFor(entrypoint) {
function requiredCapabilitiesFor(entrypoint, packageSummary = {}) {
const capabilities = new Set();
for (const blocker of entrypoint.blockers) {
if (blocker.code === "dependency-install-required") {
@ -361,7 +372,7 @@ function requiredCapabilitiesFor(entrypoint) {
capabilities.add("side-effect-sandbox");
}
}
if (entrypoint.blockers.some((blocker) => /\bopenclaw\b/.test(blocker.evidence ?? ""))) {
if (hasHostLinkedOpenClawDependency(packageSummary)) {
capabilities.add("target-openclaw-link");
}
capabilities.add("capture-shim");
@ -369,6 +380,20 @@ function requiredCapabilitiesFor(entrypoint) {
return [...capabilities].sort();
}
function hasHostLinkedOpenClawDependency(packageSummary) {
return [
...(packageSummary.dependencies ?? []),
...(packageSummary.peerDependencies ?? []),
...(packageSummary.optionalDependencies ?? []),
].includes("openclaw");
}
function hasWorkspaceProtocolDevDependencies(packageJson) {
return Object.values(packageJson.devDependencies ?? {}).some(
(value) => typeof value === "string" && value.startsWith("workspace:"),
);
}
function detectPackageManager(rootDir, packageDir, packageJson) {
const declared = typeof packageJson.packageManager === "string" ? packageJson.packageManager.split("@")[0] : null;
if (declared) {
@ -450,15 +475,13 @@ function runCommand(packageManager, script) {
}
function captureCommand(settings, fixtureId, entrypoint, workspacePath) {
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
const script = helperScript(settings, workspacePath, settings.captureScript, "capture-cli.js");
return `${settings.optInEnv} node${loader} ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
return `${settings.optInEnv} node ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
}
function syntheticProbeCommand(settings, fixtureId, entrypoint, workspacePath) {
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
const script = helperScript(settings, workspacePath, settings.syntheticProbeScript, "synthetic-probes-cli.js");
return `${settings.optInEnv} node${loader} ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
return `${settings.optInEnv} node ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
}
function helperScript(settings, workspacePath, configuredScript, helperFileName) {

View File

@ -3,6 +3,7 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "node:test";
import { packageId } from "../src/config.js";
import {
buildCiPolicyReport,
buildCiSummary,
@ -85,6 +86,10 @@ import {
writeSyntheticProbePlan,
} from "../src/index.js";
test("package ids collapse separators and trim hyphen edges", () => {
assert.equal(packageId("@openclaw/openclaw---Weather_Plugin!!!"), "weather-plugin");
});
test("public API runs the plugin-root check and writes reports", async () => {
const pluginRoot = await createPluginRoot();
@ -111,6 +116,8 @@ test("public API exposes grouped facades for common workflows", () => {
assert.equal(fixtureSuites.runReport, runFixtureSetReport);
assert.equal(staticInspection.inspectSourceText, inspectSourceText);
assert.equal(reports.renderMarkdown, renderMarkdownReport);
assert.equal(typeof reports.sanitizeArtifact, "function");
assert.equal(typeof reports.readOpenClawTargetSurface, "function");
assert.equal(contracts.buildCapture, buildContractCapture);
assert.equal(contracts.validateCoverage, validateContractCoverage);
assert.equal(ci.buildSummary, buildCiSummary);
@ -251,6 +258,33 @@ test("public API builds fixture-set workspace and platform plans from config", a
markdownPath: path.join(outDir, "workspace.md"),
});
const platform = await buildFixtureSetPlatformProbes({ plan });
const coveredPlan = structuredClone(plan);
coveredPlan.fixtures[0].entrypoints = [
{
id: "cold-import.extension:sample-plugin:index",
status: "dependency-install-required",
entrypoint: "plugins/sample-plugin/index.js",
packageManager: "npm",
loaderStrategy: {
source: "javascript",
primary: "node",
alternatives: [],
reason: "test",
},
steps: [
{
kind: "prepare",
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
},
],
},
];
const coveredPlatform = await buildFixtureSetPlatformProbes({
plan: coveredPlan,
stepCoverage({ riskCodes }) {
return { riskCodes };
},
});
const platformPaths = await writeFixtureSetPlatformProbes(platform, {
jsonPath: path.join(outDir, "platform.json"),
markdownPath: path.join(outDir, "platform.md"),
@ -267,6 +301,8 @@ test("public API builds fixture-set workspace and platform plans from config", a
assert.match(renderFixtureSetWorkspacePlanMarkdown(plan), /## Entrypoint Workspaces/);
assert.equal(JSON.parse(await readFile(planPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
assert.equal(platform.summary.fixtureCount, 1);
assert.equal(coveredPlatform.summary.portabilityFindingCount, 0);
assert.ok(coveredPlatform.summary.coveredPortabilityFindingCount > 0);
assert.deepEqual(validateFixtureSetPlatformProbes(platform), []);
assert.match(renderFixtureSetPlatformProbesMarkdown(platform), /## Loader Probes/);
assert.equal(JSON.parse(await readFile(platformPaths.jsonPath, "utf8")).summary.fixtureCount, 1);

View File

@ -50,6 +50,16 @@ test("capture API returns useful channel, gateway, and lifecycle descriptors", (
assert.equal(typeof service.dispose, "function");
});
test("capture API records conversation binding resolved callbacks", () => {
const api = createCaptureApi();
assert.equal(api.onConversationBindingResolved(() => undefined), api);
assert.deepEqual(
api.getCapturedContracts().map((entry) => `${entry.kind}:${entry.name}`),
["hook:onConversationBindingResolved"],
);
});
test("capture API accepts custom registrar return profiles", () => {
const api = createCaptureApi({
registrarProfiles: {
@ -67,6 +77,7 @@ test("capture API accepts custom registrar return profiles", () => {
test("capture API exposes mock context helpers", async () => {
const api = createCaptureApi({
resolvePath: (value) => `/fixture/${value}`,
secretValues: {
token: "redacted",
},
@ -80,6 +91,7 @@ test("capture API exposes mock context helpers", async () => {
assert.deepEqual(await api.store.list(), ["key"]);
assert.equal(api.agent.id, "plugin-inspector-agent");
assert.equal(api.paths.dataDir, ".plugin-inspector/data");
assert.equal(api.resolvePath("state"), "/fixture/state");
});
test("capture API can retain handlers for probes", () => {

View File

@ -63,6 +63,41 @@ test("ci policy allows known blocked probes but fails unknown blockers", () => {
assert.match(renderCiPolicyMarkdown(report), /Plugin Inspector CI Policy/);
});
test("ci policy supports wildcard seam rules for generated surface blockers", () => {
const report = buildCiPolicyReport({
policy: {
...policy,
allowedBlocked: [
...policy.allowedBlocked,
{
id: "generated-surface-runtime-gap",
seam: "*",
reasonIncludes: "generated surface has no callable runtime",
decision: "allowed-blocked",
until: "generated surface runtime harness lands",
},
],
},
compatibilityReport: compatibilityReport(),
executionResults: executionResults([
{
seam: "before_tool_call",
reason: "generated surface has no callable runtime",
},
{
seam: "registerCommand",
reason: "generated surface has no callable runtime",
},
]),
});
assert.equal(report.status, "pass");
assert.deepEqual(
report.checks.filter((check) => check.id.startsWith("execution-results.blocked.")).map((check) => check.action),
["warn", "warn"],
);
});
test("ci policy fails ref diff hard regressions", () => {
const report = buildCiPolicyReport({
policy,
@ -127,7 +162,7 @@ test("ci policy surfaces P0 live issues without blocking default lanes", () => {
code: "legacy-before-agent-start",
},
{
severity: "P1",
severity: "P2",
issueClass: "inspector-gap",
fixture: "wecom",
code: "registration-capture-gap",
@ -225,7 +260,7 @@ test("ci policy writer emits JSON and Markdown artifacts", async () => {
function compatibilityReport(overrides = {}) {
const issues = overrides.issues ?? [
{
severity: "P1",
severity: "P2",
issueClass: "inspector-gap",
fixture: "fixture",
code: "registration-capture-gap",

View File

@ -18,17 +18,17 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
suggestionCount: 3,
issueCount: 4,
p0IssueCount: 1,
p1IssueCount: 1,
p1IssueCount: 0,
liveIssueCount: 1,
compatGapCount: 1,
},
issues: [
{
severity: "P1",
severity: "P2",
issueClass: "inspector-gap",
fixture: "fixture",
code: "registration-capture-gap",
title: "runtime registrations need capture",
title: "runtime registrations need capture evidence",
decision: "inspector-follow-up",
},
],
@ -97,6 +97,13 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
p95WallMs: 75,
maxPeakRssMb: 40,
maxCpuMsEstimate: 30,
maxPluginPeakRssDeltaMb: 8,
maxPluginCpuDeltaMsEstimate: 6,
openClawLifecycleCount: 2,
p50OpenClawImportMs: 12,
p50OpenClawActivationMs: 3,
rssSampleCount: 2,
cpuSampleCount: 2,
},
},
},
@ -108,9 +115,12 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
assert.equal(summary.summary.platformWindowsRisks, 3);
assert.equal(summary.summary.loaderJitiCandidates, 1);
assert.equal(summary.summary.importLoopP50Ms, 50);
assert.equal(summary.summary.importLoopMetricBasis, "baseline-adjusted");
assert.equal(summary.summary.importLoopMaxRssMb, 8);
assert.equal(summary.summary.importLoopOpenClawImportP50Ms, 12);
assert.match(renderCiSummaryMarkdown(summary), /Crabpot CI Summary/);
assert.match(renderCiSummaryMarkdown(summary), /Windows portability risks/);
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ max RSS 40 MB \/ CPU 30 ms/);
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ plugin delta RSS 8 MB \/ plugin delta CPU 6 ms \/ OpenClaw import 12 ms \/ activate 3 ms/);
assert.match(renderCiSummaryMarkdown(summary), /\| P0 issues\s+\| 1\s+\|/);
});

View File

@ -87,6 +87,25 @@ test("check command can target a plugin root and use runtime aliases", async ()
assert.equal(capture.summary.capturedCount, 1);
});
test("check command sanitizes absolute OpenClaw paths in JSON output and artifacts", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-sanitize-");
const openclawPath = await createTargetOpenClaw(rootDir);
const cliPath = path.resolve("src/cli.js");
const { stdout } = await execFileAsync(
process.execPath,
[cliPath, "check", "--out", "reports", "--openclaw", openclawPath, "--json"],
{ cwd: rootDir },
);
const output = JSON.parse(stdout);
const artifact = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector-report.json"), "utf8"));
assert.equal(output.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
assert.doesNotMatch(stdout, new RegExp(escapeRegExp(openclawPath)));
});
test("inspect command runs from a plugin root and can write CI outputs", async () => {
const rootDir = await createCliPluginRoot("plugin-inspector-cli-inspect-");
const cliPath = path.resolve("src/cli.js");
@ -318,3 +337,18 @@ async function createCliPluginRoot(prefix) {
);
return rootDir;
}
async function createTargetOpenClaw(rootDir) {
const openclawPath = path.join(rootDir, "target-openclaw");
await mkdir(path.join(openclawPath, "src/plugins/compat"), { recursive: true });
await writeFile(
path.join(openclawPath, "src/plugins/compat/registry.ts"),
'export const records = [{ code: "legacy-root-sdk-import", status: "deprecated" }];\n',
"utf8",
);
return openclawPath;
}
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

View File

@ -71,6 +71,23 @@ test("cold import readiness preserves combined blocker evidence", () => {
assert.deepEqual(validateColdImportReadiness(readiness), []);
});
test("cold import readiness treats openclaw as a host-linked dependency", () => {
const readiness = buildColdImportReadiness({
report: readinessReport(
[{ kind: "extension", specifier: "./index.js", relativePath: "index.js", exists: true }],
{
dependencies: ["openclaw"],
},
),
});
const entrypoint = readiness.fixtures[0].entrypoints[0];
assert.equal(entrypoint.status, "ready");
assert.deepEqual(entrypoint.blockers, []);
assert.equal(readiness.summary.dependencyInstallRequiredCount, 0);
assert.deepEqual(validateColdImportReadiness(readiness), []);
});
test("cold import readiness validation rejects incomplete entries", () => {
const errors = validateColdImportReadiness({
fixtures: [

View File

@ -26,7 +26,7 @@ test("contract coverage fails missing evidence and P1 probe gaps", () => {
fixture: "fixture",
severity: "P1",
issueClass: "inspector-gap",
code: "registration-capture-gap",
code: "conversation-access-hook",
evidence: [],
},
],
@ -165,4 +165,20 @@ test("contract coverage requires compat record reconciliation evidence", () => {
compatRecord: "fixture.provider-auth-env-vars",
});
assert.deepEqual(validateContractCoverage(report), []);
report.logs = [];
report.issues.push({
id: "PLUGIN-COMPAT",
fixture: "fixture",
severity: "P1",
issueClass: "compat-gap",
code: "provider-auth-env-vars",
evidence: ["fixture"],
compatRecord: "fixture.provider-auth-env-vars",
});
report.contractProbes.push({
fixture: "fixture",
id: "compat.provider-auth-env-vars:fixture",
});
assert.deepEqual(validateContractCoverage(report), []);
});

View File

@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { buildContractProbes, contractProbeRules, probePriority } from "../src/advanced.js";
import { buildContractProbes, compatRecordForIssueCode, contractProbeRules, probePriority } from "../src/advanced.js";
test("contract probes map issue findings to executable backlog rows", () => {
const probes = buildContractProbes({
@ -41,11 +41,13 @@ test("contract probes map issue findings to executable backlog rows", () => {
});
assert.ok(contractProbeRules["registration-capture-gap"]);
assert.equal(compatRecordForIssueCode("registration-capture-gap"), "api.capture.runtime-registrars");
assert.equal(compatRecordForIssueCode("package-dependency-install-required"), undefined);
assert.deepEqual(
probes.map((probe) => [probe.id, probe.priority, probe.target]),
[
["api.capture.runtime-registrars:wecom", "P1", "inspector-capture-api"],
["sdk.import.package-export-cold-import:codex-app-server", "P1", "sdk-alias"],
["api.capture.runtime-registrars:wecom", "P2", "inspector-capture-api"],
["manifest.schema.top-level-fields:agentchat", "P3", "manifest-loader"],
],
);
@ -53,6 +55,7 @@ test("contract probes map issue findings to executable backlog rows", () => {
test("contract probe priority escalates critical codes and high-priority fixtures", () => {
assert.equal(probePriority("sdk-export-missing", "medium"), "P1");
assert.equal(probePriority("registration-capture-gap", "high"), "P2");
assert.equal(probePriority("manifest-unknown-fields", "high"), "P2");
assert.equal(probePriority("manifest-unknown-fields", "medium"), "P3");
});

View File

@ -30,12 +30,19 @@ test("import loop profile measures repeated cold capture subprocesses", async ()
assert.deepEqual(validateImportLoopProfile(profile), []);
assert.equal(profile.summary.runs, 2);
assert.equal(profile.summary.baselineRuns, 2);
assert.equal(profile.summary.baselineFailCount, 0);
assert.equal(profile.summary.failCount, 0);
assert.ok(profile.summary.capturedCount >= 2);
assert.ok(profile.summary.p50WallMs > 0);
assert.ok(profile.summary.p50PluginWallDeltaMs >= 0);
assert.ok(profile.summary.maxPluginPeakRssDeltaMb >= 0);
assert.ok(profile.baseline.reference.wallMs > 0);
assert.ok(profile.samples.every((sample) => Number.isFinite(sample.pluginCpuDeltaMsEstimate)));
assert.ok(profile.samples.every((sample) => sample.exitCode === 0));
assert.match(renderImportLoopProfileMarkdown(profile), /Import Loop Profile/);
assert.match(renderImportLoopProfileMarkdown(profile), /CPU Estimate/);
assert.match(renderImportLoopProfileMarkdown(profile), /Harness Baseline/);
assert.match(renderImportLoopProfileMarkdown(profile), /Plugin CPU Delta/);
});
test("import loop profile can use a custom capture script and opt-in env", async () => {
@ -50,7 +57,7 @@ test("import loop profile can use a custom capture script and opt-in env", async
"const [entrypoint,, outputPath] = process.argv.slice(2);",
"if (process.env.CUSTOM_IMPORT_LOOP !== '1') throw new Error('missing opt-in');",
"await mkdir(path.dirname(outputPath), { recursive: true });",
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }] }));",
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }], openClawLifecycle: { importMs: 12, activationMs: 3, importPhase: 'full', activationPhase: 'full:register' } }));",
].join("\n"),
"utf8",
);
@ -64,7 +71,12 @@ test("import loop profile can use a custom capture script and opt-in env", async
});
assert.equal(profile.summary.failCount, 0);
assert.equal(profile.summary.baselineRuns, 1);
assert.equal(profile.summary.capturedCount, 1);
assert.equal(profile.summary.openClawLifecycleCount, 1);
assert.equal(profile.summary.p50OpenClawImportMs, 12);
assert.equal(profile.summary.p50OpenClawActivationMs, 3);
assert.match(renderImportLoopProfileMarkdown(profile), /OpenClaw Import/);
});
test("import loop profile validation rejects failed or empty captures", () => {

View File

@ -43,6 +43,18 @@ test("source inspection records hook, registrar, and SDK import evidence", () =>
]);
});
test("source inspection strips long comments before matching registrations", () => {
const inspection = inspectSourceText(
[`/* ${"a/*".repeat(512)} */`, "api.registerTool({ name: 'weather' });"].join("\n"),
"plugins/example/index.ts",
);
assert.deepEqual(
inspection.registrations.map((registration) => `${registration.name}@${registration.ref}`),
["registerTool@plugins/example/index.ts:2"],
);
});
test("fixture set inspection produces a passing report", async () => {
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
const report = await inspectFixtureSet(config);
@ -65,6 +77,44 @@ test("fixture set inspection reports missing expected seams", async () => {
assert.match(report.breakages[0].message, /llm_output/);
});
test("fixture set inspection treats channel factories as channel registration coverage", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-channel-factory-"));
await mkdir(path.join(dir, "fixture"), { recursive: true });
await writeFile(
path.join(dir, "fixture", "index.js"),
[
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
"",
"export const channel = createChatChannelPlugin({ id: 'fixture-channel' });",
"export default defineBundledChannelEntry({ id: 'bundled-channel' });",
].join("\n"),
"utf8",
);
const report = await inspectFixtureSet({
version: 1,
submoduleRoot: ".",
rootDir: dir,
fixtures: [
{
id: "fixture",
path: "fixture",
repo: "https://github.com/openclaw/fixture.git",
priority: "high",
seams: ["channel"],
expect: {
registrations: ["registerChannel"],
},
},
],
});
assert.equal(report.status, "pass");
assert.deepEqual(report.breakages, []);
assert.deepEqual(report.fixtures[0].registrations, ["createChatChannelPlugin", "defineBundledChannelEntry"]);
});
test("capture entrypoint imports a local fixture and records registrations", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-capture-"));
const entrypoint = path.join(dir, "fixture.mjs");
@ -103,6 +153,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
'import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";',
'import { GPT5_BEHAVIOR_CONTRACT } from "openclaw/plugin-sdk/provider-model-shared";',
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
'import { registerPluginHttpRoute, resolveWebhookPath } from "openclaw/plugin-sdk/webhook-ingress";',
"",
@ -123,6 +174,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
"",
"export default definePluginEntry((api) => {",
" if (!pluginSdkMock) throw new Error('expected mock SDK');",
" if (!GPT5_BEHAVIOR_CONTRACT) throw new Error('expected dynamic subpath mock export');",
" provider.register(api);",
" api.registerHttpRoute({ path: resolveWebhookPath('hook'), handler() {} });",
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
@ -144,3 +196,153 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
["registration:registerProvider", "registration:registerHttpRoute", "registration:registerTool"],
);
});
test("mock capture accepts valid output when plugin code dirties process exit code", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-exit-code-"));
const entrypoint = path.join(dir, "index.mjs");
await writeFile(
entrypoint,
[
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
"process.exitCode = 1;",
"export default definePluginEntry({",
" register(api) {",
" api.registerProvider({ id: 'fixture-provider' });",
" },",
"});",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.mjs", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
"registration:registerProvider",
]);
});
test("mock capture prefers discovered bare mocks over installed dependency exports", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-bare-capture-"));
await mkdir(path.join(dir, "node_modules/typebox"), { recursive: true });
await writeFile(
path.join(dir, "node_modules/typebox/package.json"),
JSON.stringify({ name: "typebox", version: "0.0.0", type: "module", exports: "./index.js" }, null, 2),
"utf8",
);
await writeFile(path.join(dir, "node_modules/typebox/index.js"), "export const Type = {};\n", "utf8");
const entrypoint = path.join(dir, "index.mjs");
await writeFile(
entrypoint,
[
"import path from 'node:path';",
'import { Static, Type } from "typebox";',
'import { resolvePreferredOpenClawTmpDir } from "fixture-api";',
"export function register(api) {",
" if (!Static || !Type) throw new Error('expected mocked typebox exports');",
" path.join(resolvePreferredOpenClawTmpDir(), 'fixture');",
" api.registerTool({ name: 'fixture_tool', inputSchema: Type.Object({}), run() {} });",
"}",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.mjs", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), ["registration:registerTool"]);
});
test("mock capture expands bundled channel entry registration shells", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-capture-"));
const entrypoint = path.join(dir, "index.mjs");
await writeFile(
entrypoint,
[
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
"export default defineBundledChannelEntry({",
" id: 'fixture-channel',",
" name: 'Fixture Channel',",
" description: 'Fixture channel',",
" plugin: { specifier: './channel.js', exportName: 'fixtureChannel' },",
" registerFull(api) {",
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
" },",
"});",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.mjs", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
"registration:registerChannel",
"registration:registerTool",
]);
});
test("mock capture follows bundled channel linked registerFull exports", async () => {
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-linked-capture-"));
await writeFile(
path.join(dir, "index.ts"),
[
'import { defineBundledChannelEntry, loadBundledEntryExportSync } from "openclaw/plugin-sdk/channel-entry-contract";',
"",
"function registerFull(api) {",
" const register = loadBundledEntryExportSync(import.meta.url, {",
" specifier: './api.js',",
" exportName: 'registerFixtureFull',",
" });",
" register(api);",
"}",
"",
"export default defineBundledChannelEntry({",
" id: 'fixture-channel',",
" name: 'Fixture Channel',",
" description: 'Fixture channel',",
" plugin: { specifier: './channel-plugin-api.js', exportName: 'fixtureChannel' },",
" registerFull,",
"});",
].join("\n"),
"utf8",
);
await writeFile(
path.join(dir, "api.ts"),
[
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
"",
"const schema = buildSecretInputSchema().optional();",
"",
"export function registerFixtureFull(api) {",
" schema.parse(undefined);",
" api.registerCommand({ name: 'fixture.command', run() {} });",
"}",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("index.ts", {
cwd: dir,
pluginRoot: dir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
"registration:registerChannel",
"registration:registerCommand",
]);
});

View File

@ -19,18 +19,18 @@ test("issue ids are stable fingerprints", () => {
test("issue classification separates live breaks from compat and deprecation buckets", () => {
const cases = [
{
name: "untracked SDK alias is a blocking live issue",
name: "untracked SDK alias is a compat gap",
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
targetOpenClaw: { compatRecordStatuses: {} },
metadata: { severity: "P1" },
expected: { issueClass: "live-issue", compatStatus: "untracked", severity: "P0", live: true },
expected: { issueClass: "compat-gap", compatStatus: "untracked", severity: "P1", live: false },
},
{
name: "active SDK alias compat avoids false P0 escalation",
name: "active SDK alias compat stays a compat row",
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
targetOpenClaw: { compatRecordStatuses: { "plugin-sdk-export-aliases": "active" } },
metadata: { severity: "P1" },
expected: { issueClass: "live-issue", compatStatus: "active", severity: "P1", live: true },
expected: { issueClass: "compat-gap", compatStatus: "active", severity: "P1", live: false },
},
{
name: "deprecated compat remains warning-class even when used",
@ -46,6 +46,18 @@ test("issue classification separates live breaks from compat and deprecation buc
metadata: { severity: "P1" },
expected: { issueClass: "compat-gap", compatStatus: "missing", severity: "P1", live: false },
},
{
name: "active OpenClaw probe contract stays an inspector gap",
finding: {
code: "before-tool-call-probe",
compatRecord: "hook.before_tool_call.terminal-block-approval",
},
targetOpenClaw: {
compatRecordStatuses: { "hook.before_tool_call.terminal-block-approval": "active" },
},
metadata: { severity: "P1" },
expected: { issueClass: "inspector-gap", compatStatus: "active", severity: "P1", live: false },
},
{
name: "unknown untracked hook is P0 live break",
finding: { code: "unknown-hook-name" },
@ -100,17 +112,17 @@ test("issue builder applies metadata and class summaries", () => {
assert.deepEqual(
issues.map((issue) => [issue.fixture, issue.code, issue.severity, issue.issueClass, issue.status]),
[
["codex-app-server", "sdk-export-missing", "P0", "live-issue", "blocking"],
["wecom", "registration-capture-gap", "P1", "inspector-gap", "open"],
["codex-app-server", "sdk-export-missing", "P1", "compat-gap", "open"],
["agentchat", "manifest-unknown-fields", "P2", "upstream-metadata", "open"],
["wecom", "registration-capture-gap", "P2", "inspector-gap", "open"],
],
);
assert.deepEqual(summarizeIssueClasses(issues), {
"compat-gap": 0,
"compat-gap": 1,
"deprecation-warning": 0,
"fixture-regression": 0,
"inspector-gap": 1,
"live-issue": 1,
"live-issue": 0,
"upstream-metadata": 1,
});
});

View File

@ -136,6 +136,7 @@ test("OpenClaw target parsing helpers stay deterministic", () => {
);
assert.deepEqual(
parseCompatRecordEntries(`
${"{{".repeat(256)}
{ code: "b", status: "supported" }
{ code: "a", status: "deprecated" }
{ code: "b", status: "supported" }

View File

@ -40,11 +40,11 @@ test("platform probes classify loader and shell portability risks", () => {
},
{
kind: "capture",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx capture.mjs ./src/index.ts",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./src/index.ts --mock-sdk",
},
{
kind: "synthetic-probe",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx synthetic.mjs --entrypoint ./src/index.ts",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node synthetic.mjs --entrypoint ./src/index.ts --mock-sdk",
},
],
},
@ -63,7 +63,68 @@ test("platform probes classify loader and shell portability risks", () => {
assert.match(renderPlatformProbesMarkdown(report), /rsync/);
});
test("platform probe validation requires jiti fallback and reflected tsx commands", () => {
test("platform probes separate executor-covered portability risks from residual risks", () => {
const report = buildPlatformProbes({
plan: {
generatedAt: "test",
mode: "plan-only",
summary: {
fixtureCount: 1,
},
fixtures: [
{
id: "fixture",
entrypoints: [
{
id: "cold-import.extension:fixture:index",
status: "dependency-install-required",
entrypoint: "plugins/fixture/index.js",
packageManager: "pnpm",
loaderStrategy: {
source: "javascript",
primary: "node",
alternatives: [],
reason: "test",
},
steps: [
{
kind: "prepare",
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
},
{
kind: "audit",
command: "pnpm audit --json > ../../results/fixture/package-audit.json || true",
},
{
kind: "capture",
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./index.js",
},
],
},
],
},
],
},
stepCoverage({ riskCodes }) {
return {
reason: "covered by Crabpot structured executor",
riskCodes: riskCodes.filter((code) =>
["posix-mkdir", "posix-env-prefix", "posix-null-failure", "rsync-required", "shell-redirection"].includes(code),
),
};
},
});
assert.equal(report.summary.portabilityFindingCount, 1);
assert.equal(report.summary.coveredPortabilityFindingCount, 3);
assert.equal(report.summary.windowsRiskStepCount, 1);
assert.deepEqual(report.portabilityFindings[0].riskCodes, ["package-manager-availability"]);
assert.ok(report.coveredPortabilityFindings.every((finding) => finding.coverage === "covered by Crabpot structured executor"));
assert.doesNotMatch(renderPlatformProbesMarkdown(report), /replace shell mkdir/);
assert.match(renderPlatformProbesMarkdown(report), /Covered Portability Findings/);
});
test("platform probe validation requires jiti fallback and reflected TypeScript loader commands", () => {
const errors = validatePlatformProbes({
mode: "plan-only",
targets: ["linux", "macos", "windows", "container"],
@ -76,11 +137,14 @@ test("platform probe validation requires jiti fallback and reflected tsx command
id: "cold-import.extension:fixture:index",
loaderPrimary: "tsx",
captureUsesTsx: true,
captureUsesTypeScriptLoader: true,
syntheticUsesTsx: false,
syntheticUsesMockSdk: false,
syntheticUsesTypeScriptLoader: false,
},
],
});
assert.ok(errors.some((error) => error.includes("Jiti fallback")));
assert.ok(errors.some((error) => error.includes("tsx loader strategy")));
assert.ok(errors.some((error) => error.includes("TypeScript loader strategy")));
});

View File

@ -7,10 +7,12 @@ import {
buildSarifReport,
buildCompatibilityReport,
buildCompatibilityFixtureReport,
buildIssues,
classifyCompatibilityFixture,
classifyCompatRecordCoverage,
classifyPackageContracts,
classifyTargetOpenClawCoverage,
escapeMarkdownTableCell,
inspectFixtureSet,
loadInspectorConfig,
renderCompatibilityIssuesReport,
@ -20,10 +22,16 @@ import {
renderMarkdownTable,
renderTextSummary,
writeArtifacts,
writeCompatibilityReport,
writeCiOutputArtifacts,
writeReport,
} from "../src/advanced.js";
test("markdown table cell escaping preserves literal backslashes", () => {
assert.equal(escapeMarkdownTableCell(String.raw`C:\tmp|next
line`), String.raw`C:\\tmp\|next<br>line`);
});
test("markdown report includes summary and inventory", async () => {
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
const report = await inspectFixtureSet(config);
@ -185,6 +193,91 @@ test("compatibility report renderer supports issue metadata and evidence links",
assert.match(issues, /\[linked\]\(plugins\/sample\/src\/index\.ts:1\)/);
});
test("compatibility report artifacts sanitize absolute OpenClaw target paths", async () => {
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-sanitized-report-"));
const absoluteOpenClawPath = path.join(outDir, "openclaw");
const report = {
generatedAt: "test",
status: "pass",
targetOpenClaw: {
status: "ok",
configuredPath: absoluteOpenClawPath,
searchedPaths: [absoluteOpenClawPath],
compatRecords: [],
compatRecordStatuses: {},
},
summary: {
fixtureCount: 1,
highPriorityFixtures: 1,
breakageCount: 0,
warningCount: 0,
suggestionCount: 0,
decisionCount: 0,
issueCount: 1,
p0IssueCount: 0,
p1IssueCount: 0,
liveIssueCount: 0,
liveP0IssueCount: 0,
compatGapCount: 0,
deprecationWarningCount: 0,
inspectorGapCount: 1,
upstreamIssueCount: 0,
fixtureRegressionCount: 0,
contractProbeCount: 0,
},
fixtures: [
{
id: "sample-plugin",
priority: "high",
seams: ["native-tool"],
hooks: [],
registrations: [],
manifestContracts: [],
},
],
breakages: [],
warnings: [],
suggestions: [],
issues: [
{
fixture: "sample-plugin",
code: "package-dependency-install-required",
issueClass: "inspector-gap",
decision: "inspector-follow-up",
severity: "P2",
title: `sample-plugin: path ${absoluteOpenClawPath}`,
status: "open",
compatStatus: "none",
live: false,
evidence: [absoluteOpenClawPath],
},
],
contractProbes: [],
logs: [],
decisions: [],
};
const markdown = renderCompatibilityMarkdownReport(report);
const issues = renderCompatibilityIssuesReport(report);
const paths = await writeCompatibilityReport(report, {
jsonPath: path.join(outDir, "report.json"),
markdownPath: path.join(outDir, "report.md"),
issuesPath: path.join(outDir, "issues.md"),
});
const artifact = JSON.parse(await readFile(paths.jsonPath, "utf8"));
assert.equal(report.targetOpenClaw.configuredPath, absoluteOpenClawPath);
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
assert.equal(artifact.issues[0].evidence[0], "<OPENCLAW_PATH>");
assert.equal(artifact.issues[0].title, "sample-plugin: path <OPENCLAW_PATH>");
assert.doesNotMatch(markdown, new RegExp(escapeRegExp(absoluteOpenClawPath)));
assert.doesNotMatch(issues, new RegExp(escapeRegExp(absoluteOpenClawPath)));
assert.match(markdown, /<OPENCLAW_PATH>/);
assert.match(await readFile(paths.markdownPath, "utf8"), /<OPENCLAW_PATH>/);
assert.match(await readFile(paths.issuesPath, "utf8"), /<OPENCLAW_PATH>/);
});
test("compatibility report assembly classifies fixtures, issues, probes, and compat records", async () => {
const report = await buildCompatibilityReport({
generatedAt: "test",
@ -277,6 +370,110 @@ test("compatibility report assembly classifies fixtures, issues, probes, and com
assert.ok(report.decisions.some((decision) => decision.seam === "compat-registry"));
});
test("compatibility report marks inspector gaps covered by runtime execution artifacts", async () => {
const report = await buildCompatibilityReport({
generatedAt: "test",
fixtures: [
{
id: "fixture",
name: "Fixture",
path: "plugins/fixture",
priority: "high",
seams: ["native-tool"],
why: "covers runtime-only seams",
},
],
inspections: [
{
id: "fixture",
status: "ok",
hooks: ["llm_input"],
hookDetails: [{ name: "llm_input", ref: "plugins/fixture/src/index.ts:1" }],
registrations: ["registerTool", "registerService", "registerCommand"],
registrationDetails: [
{ name: "registerTool", ref: "plugins/fixture/src/index.ts:2" },
{ name: "registerService", ref: "plugins/fixture/src/index.ts:3" },
{ name: "registerCommand", ref: "plugins/fixture/src/index.ts:4" },
],
manifestContracts: [],
manifestFiles: [],
sdkImports: [],
sourceFiles: ["plugins/fixture/src/index.ts"],
},
],
targetOpenClaw: {
status: "ok",
compatRecords: [],
compatRecordStatuses: {},
hookNames: ["llm_input"],
apiRegistrars: ["registerTool", "registerService", "registerCommand"],
capturedRegistrars: [],
sdkExports: [],
manifestFields: ["id"],
manifestContractFields: [],
},
executionResults: {
artifacts: [
{
fixture: "fixture",
kind: "capture",
status: "pass",
artifactPath: ".crabpot/results/fixture/index.capture.json",
captured: [
"hook:llm_input",
"registration:registerTool",
"registration:registerService",
"registration:registerCommand",
],
},
],
},
buildFixtureReport: ({ fixture, inspection }) => ({
id: fixture.id,
name: fixture.name,
priority: fixture.priority,
seams: fixture.seams,
why: fixture.why,
status: inspection.status,
hooks: inspection.hooks,
hookDetails: inspection.hookDetails,
registrations: inspection.registrations,
registrationDetails: inspection.registrationDetails,
manifestContracts: inspection.manifestContracts,
manifestFiles: [],
sourceFiles: inspection.sourceFiles,
pluginManifests: [],
package: null,
packages: [],
sdkImports: [],
sdkImportDetails: [],
}),
});
const coveredCodes = report.issues
.filter((issue) => issue.status === "runtime-covered")
.map((issue) => issue.code)
.sort();
assert.deepEqual(coveredCodes, [
"conversation-access-hook",
"registration-capture-gap",
"runtime-tool-capture",
]);
assert.equal(report.summary.runtimeCoveredIssueCount, 3);
assert.equal(report.summary.openInspectorGapCount, 0);
assert.equal(report.summary.runtimeCoverageArtifactCount, 1);
const registrationIssue = report.issues.find((issue) => issue.code === "registration-capture-gap");
assert.deepEqual(registrationIssue.runtimeCoverage.captured, [
"registration:registerService",
"registration:registerCommand",
]);
const markdown = renderCompatibilityIssuesReport(report);
assert.match(markdown, /## Runtime-Covered Inspector Gaps/);
assert.match(markdown, /state: runtime-covered .* runtime:covered/);
});
test("compat record coverage logs unavailable targets", () => {
const logs = [];
classifyCompatRecordCoverage({
@ -305,6 +502,21 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
`${JSON.stringify({ id: "fixture", name: "Fixture", version: "1.0.0", contracts: { tools: {} } }, null, 2)}\n`,
"utf8",
);
await writeFile(
path.join(fixtureDir, "openclaw.security.json"),
`${JSON.stringify(
{
$schema: "https://openclaw.ai/schemas/plugin-security.json",
version: "1.0.0",
plugin: "fixture",
expectedBehaviors: [{ id: "api-key", description: "requires an API key" }],
securityNotes: [{ id: "storage", description: "stores local state" }],
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(path.join(fixtureDir, "src", "index.js"), "export function register() {}\n", "utf8");
await writeFile(
path.join(fixtureDir, "package.json"),
@ -313,10 +525,28 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
name: "fixture-plugin",
version: "1.0.0",
type: "module",
files: ["src", "openclaw.plugin.json"],
dependencies: { zod: "^1.0.0" },
openclaw: {
extensions: ["src/index.js"],
compat: { pluginApi: "^1.0.0" },
build: {
openclawVersion: "2026.5.2",
pluginSdkVersion: "2026.5.2",
},
install: {
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
npmSpec: "@openclaw/fixture-plugin",
defaultChoice: "clawhub",
minHostVersion: ">=2026.5.2",
},
release: {
publishToClawHub: true,
publishToNpm: true,
},
bundle: {
includeInCore: false,
},
},
},
null,
@ -349,8 +579,35 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
});
assert.equal(report.pluginManifests[0].id, "fixture");
assert.deepEqual(report.securityManifests[0], {
path: "plugin/openclaw.security.json",
schema: "https://openclaw.ai/schemas/plugin-security.json",
version: "1.0.0",
plugin: "fixture",
expectedBehaviorCount: 1,
securityNoteCount: 1,
validJson: true,
});
assert.equal(report.package.name, "fixture-plugin");
assert.deepEqual(report.package.npmPack, {
advertised: true,
private: false,
filesMode: "allowlist",
files: ["src", "openclaw.plugin.json"],
invalidFileSpecs: [],
});
assert.equal(report.package.openclaw.compatPluginApi, "^1.0.0");
assert.deepEqual(report.package.openclaw.install, {
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
npmSpec: "@openclaw/fixture-plugin",
defaultChoice: "clawhub",
minHostVersion: ">=2026.5.2",
});
assert.deepEqual(report.package.openclaw.release, {
publishToClawHub: true,
publishToNpm: true,
});
assert.deepEqual(report.package.openclaw.unsupportedMetadata, ["openclaw.bundle"]);
assert.deepEqual(report.package.openclaw.entrypoints[0], {
kind: "extension",
specifier: "src/index.js",
@ -401,6 +658,284 @@ test("package contract classifier reports install and entrypoint blockers", () =
assert.ok(result.suggestions.some((finding) => finding.code === "package-build-artifact-entrypoint"));
assert.ok(result.suggestions.some((finding) => finding.code === "package-dependency-install-required"));
assert.ok(result.decisions.some((decision) => decision.seam === "cold-import"));
const issues = buildIssues({
suggestions: result.suggestions,
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
});
assert.ok(
issues.some(
(issue) =>
issue.code === "package-dependency-install-required" &&
issue.title === "fixture: cold import requires dependency installation in an isolated workspace",
),
);
});
test("package contract classifier treats openclaw as a host-linked dependency", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "fixture-plugin",
version: "1.0.0",
dependencies: ["openclaw"],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
entrypoints: [
{
kind: "extension",
specifier: "dist/index.js",
relativePath: "plugins/fixture/dist/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.equal(
result.suggestions.some((finding) => finding.code === "package-dependency-install-required"),
false,
);
});
test("package contract classifier reports broken install and release metadata", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
dependencies: [],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
buildOpenClawVersion: "2026.5.2",
install: {
clawhubSpec: null,
npmSpec: "fixture-plugin",
defaultChoice: "clawhub",
minHostVersion: ">=2026.5.1",
},
release: {
publishToClawHub: true,
publishToNpm: true,
},
unsupportedMetadata: ["openclaw.bundle"],
entrypoints: [
{
kind: "extension",
specifier: "dist/index.js",
relativePath: "plugins/fixture/dist/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.ok(result.warnings.some((finding) => finding.code === "package-install-metadata-incomplete"));
assert.ok(result.warnings.some((finding) => finding.code === "package-min-host-version-drift"));
assert.ok(result.warnings.some((finding) => finding.code === "package-openclaw-unsupported-metadata"));
assert.ok(result.decisions.some((decision) => decision.seam === "package-metadata"));
});
test("package contract classifier reports advertised npm pack blockers", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
npmPack: {
advertised: true,
private: true,
filesMode: "allowlist",
files: ["README.md"],
invalidFileSpecs: ["../secrets"],
},
dependencies: [],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
install: {
npmSpec: "@openclaw/fixture-plugin",
},
release: {
publishToNpm: true,
},
entrypoints: [
{
kind: "extension",
specifier: "src/index.js",
relativePath: "plugins/fixture/src/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-unavailable"));
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-metadata-missing"));
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"));
assert.ok(result.decisions.some((decision) => decision.seam === "package-artifact"));
const globResult = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
npmPack: {
advertised: true,
private: false,
filesMode: "allowlist",
files: ["src/**/*.js", "openclaw.plugin.json"],
invalidFileSpecs: [],
},
dependencies: [],
peerDependencies: [],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
install: {
npmSpec: "@openclaw/fixture-plugin",
},
release: {
publishToNpm: true,
},
entrypoints: [
{
kind: "extension",
specifier: "src/index.js",
relativePath: "plugins/fixture/src/index.js",
exists: true,
requiresBuild: false,
},
],
},
},
},
});
assert.equal(globResult.warnings.some((finding) => finding.code.startsWith("package-npm-pack-")), false);
});
test("package contract classifier accepts built runtime entries for source package metadata", () => {
const result = classifyPackageContracts({
fixture: {
id: "fixture",
path: "plugins/fixture",
},
inspection: {
registrations: ["registerTool"],
},
fixtureReport: {
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
package: {
path: "plugins/fixture/package.json",
name: "@openclaw/fixture-plugin",
version: "1.0.0",
npmPack: {
advertised: true,
private: false,
filesMode: "allowlist",
files: ["dist/**", "openclaw.plugin.json"],
invalidFileSpecs: [],
},
dependencies: [],
peerDependencies: ["openclaw"],
optionalDependencies: [],
openclaw: {
compatPluginApi: "^1.0.0",
install: {
npmSpec: "@openclaw/fixture-plugin",
},
release: {
publishToNpm: true,
},
entrypoints: [
{
kind: "extension",
specifier: "./index.ts",
relativePath: "plugins/fixture/index.ts",
exists: false,
requiresBuild: false,
},
{
kind: "runtimeExtension",
specifier: "./dist/index.js",
relativePath: "plugins/fixture/dist/index.js",
exists: true,
requiresBuild: true,
},
{
kind: "setupEntry",
specifier: "./setup-entry.ts",
relativePath: "plugins/fixture/setup-entry.ts",
exists: false,
requiresBuild: false,
},
{
kind: "runtimeExtension",
specifier: "./dist/setup-entry.js",
relativePath: "plugins/fixture/dist/setup-entry.js",
exists: true,
requiresBuild: true,
},
],
},
},
},
});
assert.equal(result.warnings.some((finding) => finding.code === "package-entrypoint-missing"), false);
assert.equal(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"), false);
assert.equal(result.decisions.some((decision) => decision.seam === "package-entrypoint"), false);
});
test("target OpenClaw coverage classifier reports missing public surface", () => {
@ -478,6 +1013,17 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
channelEnvVars: { CHANNEL_ID: "channel id" },
},
],
securityManifests: [
{
path: "plugins/fixture/openclaw.security.json",
schema: "https://openclaw.ai/schemas/plugin-security.json",
version: "1.0.0",
plugin: "fixture",
expectedBehaviorCount: 1,
securityNoteCount: 1,
validJson: true,
},
],
package: null,
},
targetOpenClaw: {
@ -493,13 +1039,48 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
assert.ok(result.warnings.some((finding) => finding.code === "provider-auth-env-vars"));
assert.ok(result.warnings.some((finding) => finding.code === "channel-env-vars"));
assert.ok(result.warnings.some((finding) => finding.code === "conversation-access-hook"));
assert.ok(result.warnings.some((finding) => finding.code === "unrecognized-security-manifest"));
assert.ok(result.warnings.some((finding) => finding.code === "security-manifest-schema-unavailable"));
assert.ok(
result.warnings.some(
(finding) =>
finding.code === "conversation-access-hook" &&
finding.compatRecord === "hook.llm-observer.privacy-payload",
),
);
assert.ok(result.warnings.some((finding) => finding.code === "legacy-root-sdk-import"));
assert.ok(result.warnings.some((finding) => finding.code === "package-json-missing"));
assert.ok(result.suggestions.some((finding) => finding.code === "registration-capture-gap"));
assert.ok(result.suggestions.some((finding) => finding.code === "before-tool-call-probe"));
assert.ok(
result.suggestions.some(
(finding) =>
finding.code === "registration-capture-gap" &&
finding.compatRecord === "api.capture.runtime-registrars",
),
);
assert.ok(
result.suggestions.some(
(finding) =>
finding.code === "before-tool-call-probe" &&
finding.compatRecord === "hook.before_tool_call.terminal-block-approval",
),
);
assert.ok(result.suggestions.some((finding) => finding.code === "runtime-tool-capture"));
assert.ok(result.decisions.some((decision) => decision.seam === "conversation-access"));
assert.ok(result.decisions.some((decision) => decision.seam === "security-metadata"));
const issues = buildIssues({
warnings: result.warnings,
suggestions: result.suggestions,
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
});
assert.ok(
issues.some(
(issue) =>
issue.code === "unrecognized-security-manifest" &&
issue.issueClass === "upstream-metadata" &&
issue.severity === "P3",
),
);
});
test("writeReport writes JSON and Markdown artifacts", async () => {
@ -585,3 +1166,7 @@ test("markdown table helper supports padded empty-table reports", () => {
);
assert.equal(renderMarkdownTable([], ["Name"], { empty: "_none_" }), "_none_");
});
function escapeRegExp(value) {
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
}

View File

@ -5,6 +5,7 @@ import path from "node:path";
import { test } from "node:test";
import {
buildRuntimeCaptureReport,
captureEntrypoint,
inspectCompatibilityFixtureSet,
loadPluginRootConfig,
writeRuntimeCaptureReport,
@ -61,14 +62,59 @@ test("runtime capture report imports plugin entrypoints with mocked SDK", async
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
});
test("runtime capture report classifies missing mocked SDK exports", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-"));
test("runtime capture records conversation binding resolved callbacks", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-binding-resolved-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-missing-sdk-export",
name: "openclaw-binding-resolved",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.mjs"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "src", "index.mjs"),
[
'import { definePluginEntry } from "openclaw/plugin-sdk";',
"",
"export default definePluginEntry((api) => {",
" api.onConversationBindingResolved(() => undefined);",
"});",
].join("\n"),
"utf8",
);
const config = await loadPluginRootConfig(null, { cwd: rootDir });
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
assert.equal(captureReport.summary.failedCount, 0);
assert.equal(captureReport.summary.capturedCount, 1);
assert.equal(captureReport.summary.hookCount, 1);
assert.deepEqual(
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
["hook:onConversationBindingResolved"],
);
});
test("runtime capture report synthesizes newly imported mocked SDK exports", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-dynamic-sdk-export",
version: "1.0.0",
type: "module",
openclaw: {
@ -86,9 +132,10 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
[
'import { definitelyMissing } from "openclaw/plugin-sdk/plugin-entry";',
"",
"export default definitelyMissing({",
" register() {},",
"});",
"export function register(api) {",
" if (!definitelyMissing) throw new Error('expected dynamic mock export');",
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
"}",
].join("\n"),
"utf8",
);
@ -97,17 +144,18 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
assert.equal(captureReport.summary.failedCount, 1);
assert.equal(captureReport.results[0].status, "error");
assert.equal(captureReport.results[0].failureClass, "missing-sdk-export");
assert.equal(captureReport.results[0].missingExport, "definitelyMissing");
assert.equal(captureReport.summary.failedCount, 0);
assert.equal(captureReport.results[0].status, "captured");
assert.deepEqual(captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`), [
"registration:registerTool",
]);
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-out-"));
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-out-"));
await writeRuntimeCaptureReport(captureReport, {
jsonPath: path.join(outDir, "capture.json"),
markdownPath: path.join(outDir, "capture.md"),
});
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /missing-sdk-export/);
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
});
test("runtime capture report classifies registration execution failures", async () => {
@ -227,6 +275,83 @@ test("runtime capture supports TypeScript entrypoints, SDK subpaths, external mo
assert.match(captureReport.results[0].processOutput.stdout, /late plugin noise/);
});
test("runtime capture synthesizes manifest config before plugin registration", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-config-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });
await writeFile(
path.join(rootDir, "package.json"),
`${JSON.stringify(
{
name: "openclaw-configured-memory",
version: "1.0.0",
type: "module",
openclaw: {
extensions: ["src/index.ts"],
compat: { pluginApi: "^1.0.0" },
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "openclaw.plugin.json"),
`${JSON.stringify(
{
id: "configured-memory",
configSchema: {
type: "object",
properties: {
embedding: {
type: "object",
minProperties: 1,
properties: {
provider: { type: "string" },
model: { type: "string" },
},
},
autoCapture: { type: "boolean" },
autoRecall: { type: "boolean" },
},
required: ["embedding"],
},
},
null,
2,
)}\n`,
"utf8",
);
await writeFile(
path.join(rootDir, "src", "index.ts"),
[
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
"export default definePluginEntry({",
" register(api) {",
" if (!api.pluginConfig?.embedding) {",
" api.registerService({ id: 'configured-memory-disabled', start() {} });",
" return;",
" }",
" api.on('agent_end', () => undefined);",
" },",
"});",
].join("\n"),
"utf8",
);
const result = await captureEntrypoint("src/index.ts", {
cwd: rootDir,
pluginRoot: rootDir,
mockSdk: true,
});
assert.equal(result.status, "captured");
assert.deepEqual(
result.captured.map((item) => `${item.kind}:${item.name}`),
["hook:agent_end"],
);
});
test("runtime capture supports namespace imports from mocked externals", async () => {
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-zod-namespace-"));
await mkdir(path.join(rootDir, "src"), { recursive: true });

View File

@ -103,6 +103,85 @@ test("synthetic probe plan blocks unclassified registrars", () => {
assert.match(validateSyntheticProbePlan(plan).join("\n"), /not been classified/);
});
test("synthetic probe plan classifies generated kitchen-sink registrars", () => {
const kitchenSinkRegistrars = [
"createChatChannelPlugin",
"registerAgentEventSubscription",
"registerAgentHarness",
"registerAgentToolResultMiddleware",
"registerAutoEnableProbe",
"registerChannel",
"defineBundledChannelEntry",
"registerCli",
"registerCliBackend",
"registerCodexAppServerExtensionFactory",
"registerCommand",
"registerCompactionProvider",
"registerConfigMigration",
"registerContextEngine",
"registerControlUiDescriptor",
"registerDetachedTaskRuntime",
"registerGatewayDiscoveryService",
"registerGatewayMethod",
"registerHook",
"registerHttpRoute",
"registerImageGenerationProvider",
"registerInteractiveHandler",
"registerMediaUnderstandingProvider",
"registerMemoryCapability",
"registerMemoryCorpusSupplement",
"registerMemoryEmbeddingProvider",
"registerMemoryFlushPlan",
"registerMemoryPromptSection",
"registerMemoryPromptSupplement",
"registerMemoryRuntime",
"registerMigrationProvider",
"registerMusicGenerationProvider",
"registerNodeHostCommand",
"registerNodeInvokePolicy",
"registerProvider",
"registerRealtimeTranscriptionProvider",
"registerRealtimeVoiceProvider",
"registerReload",
"registerRuntimeLifecycle",
"registerSecurityAuditCollector",
"registerService",
"registerSessionExtension",
"registerSessionSchedulerJob",
"registerSpeechProvider",
"registerTextTransforms",
"registerTool",
"registerToolMetadata",
"registerTrustedToolPolicy",
"registerVideoGenerationProvider",
"registerWebFetchProvider",
"registerWebSearchProvider",
];
const plan = buildSyntheticProbePlan({
capture: {
generatedAt: "test",
summary: { fixtureCount: 1 },
fixtures: [
{
id: "kitchen-sink",
hooks: [],
registrations: kitchenSinkRegistrars.map((registrar) => ({
id: `registration.${registrar}:kitchen-sink:index`,
registrar,
ref: "src/generated-registrars.js",
assertions: [`${registrar} is classified`],
syntheticArguments: [{}],
})),
},
],
},
});
assert.equal(plan.summary.probeCount, kitchenSinkRegistrars.length);
assert.equal(plan.summary.blockedCount, 0);
assert.deepEqual(validateSyntheticProbePlan(plan), []);
});
test("synthetic probes invoke retained hook and tool handlers", async () => {
const capture = await captureLocalFixture([
"export function register(api) {",

View File

@ -22,7 +22,8 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
name: "fixture",
packageManager: "npm@10.0.0",
scripts: { build: "tsup" },
dependencies: { "left-pad": "^1.3.0" },
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
devDependencies: { "@openclaw/plugin-sdk": "workspace:*" },
},
null,
2,
@ -57,10 +58,11 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
assert.equal(plan.summary.artifactStepCount, 2);
assert.equal(plan.summary.installStepCount, 1);
assert.equal(plan.summary.auditStepCount, 1);
assert.equal(plan.summary.pruneDevWorkspaceDependencyStepCount, 1);
assert.equal(plan.summary.buildStepCount, 1);
assert.equal(plan.summary.captureStepCount, 2);
assert.equal(plan.summary.syntheticProbeStepCount, 2);
assert.equal(plan.summary.targetOpenClawLinkStepCount, 2);
assert.equal(plan.summary.targetOpenClawLinkStepCount, 1);
assert.equal(plan.summary.tsLoaderEntrypointCount, 1);
assert.equal(plan.summary.jitiAlternativeCount, 1);
@ -73,7 +75,14 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
assert.ok(entrypoint.requiredCapabilities.includes("sdk-alias-compat"));
assert.ok(entrypoint.requiredCapabilities.includes("ts-loader"));
assert.ok(entrypoint.steps.some((step) => step.kind === "install" && step.command === "npm install --ignore-scripts"));
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node --import tsx capture.mjs")));
assert.ok(
entrypoint.steps.some(
(step) => step.kind === "prune-dev-workspace-deps" && step.command.includes("prune-workspace-dev-deps-cli.js"),
),
);
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node capture.mjs")));
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("--mock-sdk")));
assert.ok(entrypoint.steps.every((step) => !step.command.includes("--import tsx")));
assert.ok(entrypoint.steps.some((step) => step.kind === "synthetic-probe" && step.command.includes("synthetic.mjs")));
const buildEntrypoint = plan.fixtures[0].entrypoints.find((item) => item.packageName === "build-fixture");
assert.ok(buildEntrypoint);
@ -94,7 +103,7 @@ test("workspace plan defaults point at packaged helper wrappers", async (t) => {
name: "fixture",
packageManager: "npm@10.0.0",
scripts: { build: "tsup" },
dependencies: { "left-pad": "^1.3.0" },
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
},
null,
2,
@ -226,7 +235,7 @@ function readinessReport() {
{
path: "plugins\\fixture\\package.json",
name: "fixture",
dependencies: ["left-pad"],
dependencies: ["left-pad", "openclaw"],
peerDependencies: [],
optionalDependencies: [],
openclaw: {