Compare commits
77 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8899fc796c | ||
|
|
feefb4ee23 | ||
|
|
f642fb5c9f | ||
|
|
68e10e0aaa | ||
|
|
12005b4658 | ||
|
|
4956ad1fbc | ||
|
|
a58e0785d5 | ||
|
|
9f45c8aeb6 | ||
|
|
677f6e5bc1 | ||
|
|
06cc55ce51 | ||
|
|
2eda65a8a9 | ||
|
|
4bc5fbcfa3 | ||
|
|
a6af8800e0 | ||
|
|
b919df78d3 | ||
|
|
ff78dccff7 | ||
|
|
b33c6f725d | ||
|
|
e38991a35f | ||
|
|
eb251cbae4 | ||
|
|
cc89d7cea7 | ||
|
|
1ee105e29d | ||
|
|
572956b8df | ||
|
|
e8fb9b4380 | ||
|
|
7b5f706398 | ||
|
|
862d8c9fb8 | ||
|
|
f6991de9b0 | ||
|
|
e04c3fc121 | ||
|
|
85c69fb24d | ||
|
|
9ab07bb316 | ||
|
|
d91d596d90 | ||
|
|
332706b014 | ||
|
|
e9e4b6704c | ||
|
|
98e5d775f8 | ||
|
|
768c3fc4d6 | ||
|
|
41a47e9a18 | ||
|
|
5ea07a8fca | ||
|
|
85f360548e | ||
|
|
5cc4d04a97 | ||
|
|
527b2eae82 | ||
|
|
bdef58eeb9 | ||
|
|
f8073df320 | ||
|
|
ae08c3cc10 | ||
|
|
7829f2652d | ||
|
|
9718aca09c | ||
|
|
e296985559 | ||
|
|
33657d547c | ||
|
|
36eb577373 | ||
|
|
441a44691b | ||
|
|
2cdfd8523f | ||
|
|
4b00351801 | ||
|
|
dc0b02513c | ||
|
|
90ba519b4e | ||
|
|
88f6ae551f | ||
|
|
ce32f311d2 | ||
|
|
5c40771272 | ||
|
|
e6c04b7038 | ||
|
|
c17a293c60 | ||
|
|
1c94019fb7 | ||
|
|
2706a8e11a | ||
|
|
f88a593933 | ||
|
|
c5c520b778 | ||
|
|
11783e3973 | ||
|
|
26543792f1 | ||
|
|
10d9c84865 | ||
|
|
66c3bb785a | ||
|
|
aa3aada85f | ||
|
|
51e179f304 | ||
|
|
ed80dc2d15 | ||
|
|
c89fb0b1c8 | ||
|
|
0044797091 | ||
|
|
71f7541edb | ||
|
|
b547026f5a | ||
|
|
04a1d44cf4 | ||
|
|
a2468f9628 | ||
|
|
89d874a68d | ||
|
|
0b875b26cc | ||
|
|
d7d1e253cd | ||
|
|
92da17561b |
14
.github/workflows/release.yml
vendored
14
.github/workflows/release.yml
vendored
@ -76,19 +76,7 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
NOTES_FILE="$(mktemp)"
|
||||
RELEASE_VERSION="${RELEASE_VERSION}" node <<'NODE' > "${NOTES_FILE}"
|
||||
const fs = require("fs");
|
||||
const version = process.env.RELEASE_VERSION;
|
||||
const lines = fs.readFileSync("CHANGELOG.md", "utf8").split(/\r?\n/);
|
||||
const start = lines.findIndex((line) => line.startsWith(`## ${version} - `));
|
||||
if (start === -1) {
|
||||
console.error(`CHANGELOG.md is missing a ${version} release section`);
|
||||
process.exit(1);
|
||||
}
|
||||
const end = lines.findIndex((line, index) => index > start && line.startsWith("## "));
|
||||
const body = lines.slice(start + 1, end === -1 ? lines.length : end).join("\n").trim();
|
||||
process.stdout.write(`## plugin-inspector v${version}\n\n${body}\n`);
|
||||
NODE
|
||||
node scripts/release-notes.mjs "${RELEASE_VERSION}" > "${NOTES_FILE}"
|
||||
|
||||
release_flags=(--title "plugin-inspector ${RELEASE_TAG}" --notes-file "${NOTES_FILE}" --verify-tag)
|
||||
if [[ "${RELEASE_TAG}" =~ (alpha|beta|rc) ]]; then
|
||||
|
||||
@ -6,6 +6,9 @@
|
||||
- Do not publish npm packages without explicit owner approval.
|
||||
- Preserve stable report field names and finding codes; downstream CI and
|
||||
crabpot reports may consume them.
|
||||
- Treat a package dependency named `openclaw` as a host-linked workspace input,
|
||||
not an isolated dependency-install blocker. Keep third-party runtime
|
||||
dependencies classified as install/audit blockers.
|
||||
- When changing plugin-inspector behavior, CLI/package entrypoints, release
|
||||
metadata, or the npm package version, update crabpot's
|
||||
`@openclaw/plugin-inspector` pin/docs/smoke path as needed and run the
|
||||
|
||||
124
CHANGELOG.md
124
CHANGELOG.md
@ -1,5 +1,129 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Fixed
|
||||
|
||||
- Stop classifying package source entrypoints as missing when the published package provides built runtime entrypoints, and collapse SDK alias findings into a single compat-gap row.
|
||||
- Treat compat-gap issues as reconciled contract coverage for their own compatibility record.
|
||||
|
||||
## 0.3.10 - 2026-05-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Accept valid mocked capture output when plugin code leaves `process.exitCode` dirty.
|
||||
|
||||
## 0.3.9 - 2026-05-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Follow bundled channel `loadBundledEntryExportSync` registration exports during mocked runtime capture.
|
||||
|
||||
## 0.3.8 - 2026-05-03
|
||||
|
||||
### Fixed
|
||||
|
||||
- Synthesize manifest config for isolated runtime capture so configured hooks can be observed without credentials.
|
||||
|
||||
## 0.3.7 - 2026-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Downgrade `registration-capture-gap` to advisory severity so missing capture evidence no longer reports as a P1 plugin contract risk.
|
||||
|
||||
## 0.3.6 - 2026-05-03
|
||||
|
||||
### Changed
|
||||
|
||||
- Report import-loop RSS and CPU as baseline-adjusted plugin deltas alongside raw subprocess metrics so Crabpot dashboards do not treat harness import cost as plugin runtime cost.
|
||||
- Include optional OpenClaw loader lifecycle timings for import and full activation when a capture runner provides them.
|
||||
|
||||
### Fixed
|
||||
|
||||
- Accept plugin install minimum-host floors as supported package metadata.
|
||||
- Flag unsupported legacy OpenClaw bundle metadata and advertised npm pack blockers.
|
||||
- Reconcile runtime capture evidence and harden mocked capture paths for downstream fixture reports.
|
||||
|
||||
## 0.3.5 - 2026-04-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Add immediate/faster subprocess RSS and CPU sampling plus explicit sample counts so short import-loop reports do not silently publish fake zero-memory metrics.
|
||||
- Classify `createChatChannelPlugin` as channel factory metadata in synthetic probe plans so channel-core plugins do not fail as unknown registrars.
|
||||
- Treat `createChatChannelPlugin` and `defineChannelPluginEntry` as channel registration equivalents when validating fixture expectations.
|
||||
- Label runtime profile wall-time summaries as command-median p95 and render missing sampled metrics as `n/a`.
|
||||
|
||||
## 0.3.4 - 2026-04-29
|
||||
|
||||
### Fixed
|
||||
|
||||
- Separate executor-covered platform portability findings from residual findings so downstream structured runners can keep reports blocking only on unhandled risks.
|
||||
- Sanitize absolute target OpenClaw paths from generated report artifacts and JSON CLI output.
|
||||
- Normalize the dependency-install inspector finding title to use isolated-workspace wording.
|
||||
- Treat `openclaw` package dependencies as host-linked workspace inputs instead of isolated dependency-install blockers.
|
||||
|
||||
## 0.3.3 - 2026-04-28
|
||||
|
||||
### Fixed
|
||||
|
||||
- Classify generated kitchen-sink public registrar coverage in synthetic probe plans so new API-surface fixtures do not fail as unknown execution profiles.
|
||||
|
||||
## 0.3.2 - 2026-04-28
|
||||
|
||||
### Fixed
|
||||
|
||||
- Preserve runtime capture bindings for callback-based registrations so captured hooks and registrations can resolve their bound callback metadata.
|
||||
|
||||
## 0.3.1 - 2026-04-28
|
||||
|
||||
### Added
|
||||
|
||||
- Add grouped root facades: `pluginRoot`, `fixtureSuites`, `staticInspection`, `reports`, `contracts`, `ci`, `runtime`, and `synthetic`.
|
||||
- Expose contract capture, contract coverage, CI rollup, runtime profile, ref/profile diff, import-loop, and synthetic probe helpers from the root package API.
|
||||
- Add a release follow-through guard that fails when Crabpot scripts regress to the legacy `advanced.js` bundle.
|
||||
- Add a package-contents release guard for npm tarball entrypoints, examples, and README assets.
|
||||
|
||||
### Changed
|
||||
|
||||
- Move Crabpot integration scripts to the root public API while keeping Crabpot as the fixture corpus and report consumer.
|
||||
- Keep generic artifact writing out of the root API; Crabpot-owned runner scripts write their own JSON outputs.
|
||||
- Ship the README banner asset in the npm package so the published README does not reference a missing local image.
|
||||
- Document the grouped root import path for embedding harnesses without turning the README into a full API dump.
|
||||
|
||||
## 0.3.0 - 2026-04-27
|
||||
|
||||
### Added
|
||||
|
||||
- Add `--allow-execute` as a cross-platform runtime capture opt-in flag.
|
||||
- Add `plugin-inspector init --dry-run` for setup previews.
|
||||
- Add `plugin-inspector init --json` for machine-readable setup summaries.
|
||||
- Add `plugin-inspector init --scripts` for `plugin:check` and `plugin:ci` package scripts.
|
||||
- Add public fixture-set report helpers and synthetic probe suite helpers for Crabpot and downstream compatibility suites.
|
||||
- Add a Crabpot follow-through release checklist for source refs, package pins, and smoke commands.
|
||||
|
||||
### Changed
|
||||
|
||||
- Make generated runtime CI commands use `--allow-execute` instead of shell-specific inline environment syntax.
|
||||
- Make `plugin-inspector init --ci` detect `packageManager` and common lockfiles before generating CI install/run commands.
|
||||
- Make `plugin-inspector init` output repo-relative file paths and preflight generated files before writing.
|
||||
- Make `plugin-inspector init` infer `sourceRoot: "src"` from package export maps like `"./src/index.js"`.
|
||||
- Improve CLI failure summaries with report artifact paths and top blocking findings.
|
||||
- Harden mock SDK capture by keeping generated loader fixtures available until subprocess exit.
|
||||
|
||||
## 0.2.0 - 2026-04-27
|
||||
|
||||
### Added
|
||||
|
||||
- Add package.json `pluginInspector` config discovery for plugin-root checks.
|
||||
- Add `plugin-inspector config` for resolved plugin-root config summaries.
|
||||
- Add author-facing `plugin-inspector inspect` plugin-root flow.
|
||||
- Add CI-native SARIF and JUnit outputs; `plugin-inspector ci` writes them by default.
|
||||
|
||||
### Changed
|
||||
|
||||
- Make generated CI workflows use one `plugin-inspector ci --no-openclaw --runtime --mock-sdk` command.
|
||||
- Harden runtime capture for string handler registrations, parse-capable config schema helpers, and provider auth/catalog SDK mocks.
|
||||
|
||||
## 0.1.3 - 2026-04-27
|
||||
|
||||
### Added
|
||||
|
||||
454
README.md
454
README.md
@ -1,60 +1,131 @@
|
||||
<img src="docs/plugin-inspector-banner.jpg" alt="openclaw plugin inspector banner"/>
|
||||
<img src="docs/plugin-inspector-banner.jpg" alt="OpenClaw Plugin Inspector banner">
|
||||
|
||||
# OpenClaw Plugin Inspector
|
||||
|
||||
`plugin-inspector` is the offline compatibility check for OpenClaw plugins. Run
|
||||
it from a plugin root to inspect package metadata, `openclaw.plugin.json`, SDK
|
||||
imports, `api.on(...)`, `api.register*`, and optional runtime registration
|
||||
capture.
|
||||
`@openclaw/plugin-inspector` is the offline compatibility checker for OpenClaw
|
||||
plugin packages and plugin fixture suites.
|
||||
|
||||
It answers the questions that matter before a plugin reaches users:
|
||||
|
||||
- can OpenClaw discover the package metadata and `openclaw.plugin.json`
|
||||
manifest?
|
||||
- which hooks, registration calls, manifest contracts, and SDK imports does the
|
||||
plugin use?
|
||||
- does the plugin still look compatible without local OpenClaw internals?
|
||||
- if CI finds a breakage, which JSON, Markdown, SARIF, JUnit, and summary
|
||||
artifacts should downstream automation read?
|
||||
- when a fixture-suite harness such as Crabpot runs many plugins, which findings
|
||||
are hard breakages, known warnings, live issues, deprecations, or inspector
|
||||
proof gaps?
|
||||
|
||||
The default path is static, offline, and credential-free. Runtime capture exists,
|
||||
but it is opt-in because it imports plugin code.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Node.js 22 or newer.
|
||||
- A plugin package root with `package.json`.
|
||||
- `openclaw.plugin.json` when the plugin uses the OpenClaw manifest contract.
|
||||
- No OpenClaw checkout, credentials, network service, or live provider access for
|
||||
default inspection.
|
||||
|
||||
Pass `--no-openclaw` when CI should not compare against a local OpenClaw
|
||||
checkout. If an OpenClaw checkout is supplied with `--openclaw <path>`, the
|
||||
inspector only reads public compatibility surfaces such as compat records, SDK
|
||||
exports, hook names, manifest fields, and registrar metadata.
|
||||
|
||||
## Quick Start
|
||||
|
||||
From a plugin package directory:
|
||||
Run this from a plugin package root:
|
||||
|
||||
```bash
|
||||
npx @openclaw/plugin-inspector
|
||||
npx @openclaw/plugin-inspector inspect --no-openclaw
|
||||
```
|
||||
|
||||
That runs `check`, writes report artifacts to `reports/`, and exits non-zero
|
||||
when compatibility breakages are found.
|
||||
|
||||
Add a local config and GitHub Actions workflow:
|
||||
Equivalent one-off runners:
|
||||
|
||||
```bash
|
||||
npx @openclaw/plugin-inspector init --ci
|
||||
pnpm dlx @openclaw/plugin-inspector inspect --no-openclaw
|
||||
yarn dlx @openclaw/plugin-inspector inspect --no-openclaw
|
||||
bunx @openclaw/plugin-inspector inspect --no-openclaw
|
||||
```
|
||||
|
||||
Or install it as a dev dependency:
|
||||
|
||||
```bash
|
||||
npm install --save-dev @openclaw/plugin-inspector
|
||||
npx plugin-inspector check
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
```bash
|
||||
npx @openclaw/plugin-inspector check
|
||||
npx @openclaw/plugin-inspector check --plugin-root ./plugins/weather
|
||||
npx @openclaw/plugin-inspector init --ci --package-manager pnpm
|
||||
```
|
||||
|
||||
`check` reads the current directory as one plugin unless `--plugin-root` is set.
|
||||
It writes:
|
||||
The command writes:
|
||||
|
||||
- `reports/plugin-inspector-report.json`
|
||||
- `reports/plugin-inspector-report.md`
|
||||
- `reports/plugin-inspector-issues.md`
|
||||
|
||||
Use `--no-openclaw` when CI should not compare against a local OpenClaw
|
||||
checkout:
|
||||
It exits non-zero when hard compatibility breakages are found. Warnings,
|
||||
suggestions, issue classifications, and logs stay visible in the report without
|
||||
necessarily failing the command.
|
||||
|
||||
## Install In A Plugin Repo
|
||||
|
||||
Install the package when you want repeatable local scripts and CI:
|
||||
|
||||
```bash
|
||||
plugin-inspector check --no-openclaw
|
||||
npm install --save-dev @openclaw/plugin-inspector
|
||||
```
|
||||
|
||||
Use `plugin-inspector.config.json` when CI needs stable fixture metadata,
|
||||
expected seams, or runtime capture defaults:
|
||||
Add scripts:
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"plugin:check": "plugin-inspector inspect --no-openclaw",
|
||||
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
npm run plugin:check
|
||||
```
|
||||
|
||||
The initializer can write the starter config, package scripts, and GitHub
|
||||
Actions workflow:
|
||||
|
||||
```bash
|
||||
npx @openclaw/plugin-inspector init --ci --scripts --dry-run
|
||||
npx @openclaw/plugin-inspector init --ci --scripts
|
||||
```
|
||||
|
||||
`init` detects `packageManager` and common lockfiles. Override that with
|
||||
`--package-manager npm`, `--package-manager pnpm`, `--package-manager yarn`, or
|
||||
`--package-manager bun`. Existing files are protected unless you pass `--force`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Small plugin repos can keep configuration in `package.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"plugin:check": "plugin-inspector inspect --no-openclaw",
|
||||
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
|
||||
},
|
||||
"pluginInspector": {
|
||||
"version": 1,
|
||||
"plugin": {
|
||||
"id": "weather",
|
||||
"priority": "high",
|
||||
"seams": ["dynamic-tool"],
|
||||
"sourceRoot": "src",
|
||||
"expect": {
|
||||
"registrations": ["registerTool"]
|
||||
}
|
||||
},
|
||||
"capture": {
|
||||
"mockSdk": true
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Use `plugin-inspector.config.json` for a standalone config file:
|
||||
|
||||
```json
|
||||
{
|
||||
@ -77,57 +148,99 @@ expected seams, or runtime capture defaults:
|
||||
}
|
||||
```
|
||||
|
||||
Then run:
|
||||
Inspect the resolved config before wiring CI:
|
||||
|
||||
```bash
|
||||
plugin-inspector check --config plugin-inspector.config.json
|
||||
plugin-inspector config --json
|
||||
```
|
||||
|
||||
`init --ci` writes this shape for you, plus
|
||||
`.github/workflows/plugin-inspector.yml`. Copy-ready examples also live in
|
||||
`examples/plugin-inspector.config.json` and
|
||||
`examples/github-actions-plugin-inspector.yml`.
|
||||
Copy-ready examples live in:
|
||||
|
||||
- `examples/plugin-inspector.config.json`
|
||||
- `examples/package-json-plugin-inspector.json`
|
||||
|
||||
## Commands
|
||||
|
||||
| Command | Purpose |
|
||||
| --- | --- |
|
||||
| `plugin-inspector` | Default alias for `check`. |
|
||||
| `plugin-inspector check` | Script-friendly plugin-root check. |
|
||||
| `plugin-inspector inspect` | Plugin-root check unless `--config` is supplied; with `--config`, runs a fixture report. |
|
||||
| `plugin-inspector ci` | Compatibility report plus CI summary, SARIF, and JUnit outputs. |
|
||||
| `plugin-inspector config` | Print resolved plugin-root config as text or JSON. |
|
||||
| `plugin-inspector init` | Write starter config, scripts, and optional GitHub Actions workflow. |
|
||||
| `plugin-inspector report` | Run a fixture-suite config with many plugins. |
|
||||
| `plugin-inspector capture` | Runtime-capture one entrypoint directly. |
|
||||
|
||||
Common options:
|
||||
|
||||
| Option | Meaning |
|
||||
| --- | --- |
|
||||
| `--plugin-root <path>` / `--root <path>` | Check a plugin somewhere other than the current directory. |
|
||||
| `--config <path>` | Read a standalone config file. Required for fixture-suite `report`. |
|
||||
| `--out <dir>` | Write reports somewhere other than `reports/`. |
|
||||
| `--openclaw <path>` | Compare against a local OpenClaw checkout. |
|
||||
| `--no-openclaw` | Disable OpenClaw checkout comparison. |
|
||||
| `--runtime` / `--capture` | Add opt-in runtime registration capture. |
|
||||
| `--no-runtime` / `--no-capture` | Disable runtime capture even when config enables it. |
|
||||
| `--mock-sdk` / `--sdk mock` | Use generated SDK and external-package mocks for runtime capture. |
|
||||
| `--real-sdk` / `--sdk real` | Use installed real SDK dependencies instead of mocks. |
|
||||
| `--allow-execute` | Permit commands that import plugin code. |
|
||||
| `--json` | Print machine-readable JSON to stdout. |
|
||||
| `--sarif [path]` | Write SARIF from `check` or `inspect`; `ci` enables this by default. |
|
||||
| `--junit [path]` | Write JUnit XML from `check` or `inspect`; `ci` enables this by default. |
|
||||
| `--no-sarif` / `--no-junit` | Disable default `ci` outputs. |
|
||||
|
||||
Run the built-in help for the exact CLI surface:
|
||||
|
||||
```bash
|
||||
plugin-inspector --help
|
||||
```
|
||||
|
||||
## Runtime Capture
|
||||
|
||||
Runtime capture imports plugin entrypoints in an isolated subprocess and records
|
||||
the registrations made during `register(api)`. It is opt-in because it executes
|
||||
plugin code:
|
||||
what `register(api)` does. Use it when static inspection cannot prove the actual
|
||||
registrations made at runtime.
|
||||
|
||||
```bash
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 npx @openclaw/plugin-inspector check --runtime --mock-sdk
|
||||
plugin-inspector inspect --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
```
|
||||
|
||||
By default, runtime capture uses a generated mock for `openclaw/plugin-sdk` and
|
||||
common external packages so plugin code can load in clean CI without OpenClaw
|
||||
installed. Use `--real-sdk` only when the plugin workspace already has real SDK
|
||||
dependencies installed and you intentionally want to test that path.
|
||||
`--allow-execute` is the deliberate safety switch. Without it, modes that import
|
||||
plugin code fail closed. The older environment guard still works for custom
|
||||
harnesses:
|
||||
|
||||
```bash
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector inspect --no-openclaw --runtime --mock-sdk
|
||||
```
|
||||
|
||||
By default, runtime capture uses generated mocks for `openclaw/plugin-sdk`
|
||||
subpaths and unresolved external packages discovered in the plugin import graph.
|
||||
That keeps compatibility CI offline and credential-free. It does not call live
|
||||
services, launch OpenClaw, run provider SDKs, or emulate service lifecycle side
|
||||
effects.
|
||||
|
||||
Use `--real-sdk` only when the plugin workspace already has real SDK
|
||||
dependencies installed and you intentionally want that path.
|
||||
|
||||
Runtime capture writes:
|
||||
|
||||
- `reports/plugin-inspector-runtime-capture.json`
|
||||
- `reports/plugin-inspector-runtime-capture.md`
|
||||
|
||||
You can also capture one entrypoint directly:
|
||||
Capture one entrypoint directly:
|
||||
|
||||
```bash
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector capture ./dist/index.js --mock-sdk
|
||||
plugin-inspector capture ./dist/index.js --mock-sdk --allow-execute
|
||||
```
|
||||
|
||||
## CI
|
||||
|
||||
Minimal package scripts:
|
||||
`plugin-inspector ci` writes the normal compatibility report plus CI-native
|
||||
summary, SARIF, and JUnit artifacts.
|
||||
|
||||
```json
|
||||
{
|
||||
"scripts": {
|
||||
"plugin:check": "plugin-inspector check --no-openclaw",
|
||||
"plugin:check:runtime": "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector check --no-openclaw --runtime --mock-sdk"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
GitHub Actions without a local dev dependency:
|
||||
Minimal GitHub Actions workflow:
|
||||
|
||||
```yaml
|
||||
name: plugin-inspector
|
||||
@ -147,8 +260,7 @@ jobs:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npx @openclaw/plugin-inspector check --no-openclaw
|
||||
- run: PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 npx @openclaw/plugin-inspector check --no-openclaw --runtime --mock-sdk
|
||||
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
- uses: actions/upload-artifact@v5
|
||||
if: always()
|
||||
with:
|
||||
@ -156,38 +268,214 @@ jobs:
|
||||
path: reports/plugin-inspector-*
|
||||
```
|
||||
|
||||
Generated `ci` artifacts:
|
||||
|
||||
- `reports/plugin-inspector-report.json`
|
||||
- `reports/plugin-inspector-report.md`
|
||||
- `reports/plugin-inspector-issues.md`
|
||||
- `reports/plugin-inspector-ci-summary.json`
|
||||
- `reports/plugin-inspector-ci-summary.md`
|
||||
- `reports/plugin-inspector.sarif`
|
||||
- `reports/plugin-inspector.junit.xml`
|
||||
|
||||
CI examples:
|
||||
|
||||
- `examples/github-actions-plugin-inspector.yml`
|
||||
- `examples/github-actions-code-scanning.yml`
|
||||
- `examples/gitlab-ci-plugin-inspector.yml`
|
||||
- `examples/circleci-plugin-inspector.yml`
|
||||
|
||||
## Report Surfaces
|
||||
|
||||
The compatibility report is the primary contract. Preserve field names and
|
||||
finding codes because downstream CI and Crabpot reports may consume them.
|
||||
|
||||
Important report sections:
|
||||
|
||||
| Field | Meaning |
|
||||
| --- | --- |
|
||||
| `status` | `pass` unless hard breakages exist. |
|
||||
| `summary` | Counts for fixtures, breakages, warnings, suggestions, issues, issue classes, and contract probes. |
|
||||
| `targetOpenClaw` | Status and public compatibility data read from the optional OpenClaw checkout. |
|
||||
| `fixtures` | Per-plugin metadata, hooks, registrations, manifest contracts, package data, and SDK imports. |
|
||||
| `breakages` | Blocking compatibility failures. |
|
||||
| `warnings` / `suggestions` | Non-blocking compatibility findings. |
|
||||
| `issues` | Normalized issue rows with severity and class. |
|
||||
| `contractProbes` | Suggested synthetic probes derived from observed contracts. |
|
||||
| `logs` | Informational inventory and coverage rows. |
|
||||
| `decisions` | Maintainer-facing follow-up or compatibility-policy decisions. |
|
||||
|
||||
Issue classes currently flow through the reports as live issues, compat gaps,
|
||||
deprecation warnings, inspector gaps, upstream metadata, and fixture regressions.
|
||||
|
||||
## CI Policy And Shared Reporting Primitives
|
||||
|
||||
`plugin-inspector` owns the shared CI policy and report rendering primitives.
|
||||
Fixture-suite harnesses such as Crabpot should call these exports instead of
|
||||
reimplementing scoring, summaries, Markdown, SARIF, or JUnit handling.
|
||||
|
||||
The root API exposes grouped helpers:
|
||||
|
||||
```js
|
||||
import { ci } from "@openclaw/plugin-inspector";
|
||||
|
||||
const policyReport = ci.buildPolicyReport({
|
||||
policy,
|
||||
compatibilityReport,
|
||||
executionResults,
|
||||
strict: false,
|
||||
});
|
||||
|
||||
await ci.writePolicyReport(policyReport);
|
||||
```
|
||||
|
||||
CI policy reports default to:
|
||||
|
||||
- `reports/plugin-inspector-ci-policy.json`
|
||||
- `reports/plugin-inspector-ci-policy.md`
|
||||
|
||||
A policy must use `version: 1` and define:
|
||||
|
||||
- `allowedBlocked`
|
||||
- `expectedWarnings`
|
||||
- `thresholds`
|
||||
- `fixtureSets`
|
||||
|
||||
Policy scoring fails hard breakages, unknown blocked synthetic probes, hard ref
|
||||
diff regressions, failed execution results, strict live P0 issues, and strict
|
||||
classified blockers. Non-strict mode keeps classified blocked probes and live P0
|
||||
issues visible as warnings.
|
||||
|
||||
CI summary helpers read the known report set from `reports/` and render one
|
||||
machine-readable and one Markdown rollup:
|
||||
|
||||
- compatibility
|
||||
- runtime capture
|
||||
- synthetic probes
|
||||
- cold import readiness
|
||||
- workspace plan
|
||||
- platform probes
|
||||
- import-loop profile
|
||||
- execution results
|
||||
- runtime profile
|
||||
- ref diff
|
||||
- profile diff
|
||||
- CI policy
|
||||
|
||||
## Fixture Suites
|
||||
|
||||
Fixture-set configs are still supported for crabpot-style compatibility suites:
|
||||
Most plugin authors should use the plugin-root workflow. Use fixture suites when
|
||||
one repository intentionally checks many plugins or packages, as Crabpot does.
|
||||
|
||||
```bash
|
||||
plugin-inspector report --config crabpot.config.json --out reports
|
||||
plugin-inspector report --config crabpot.config.json --out reports --check
|
||||
plugin-inspector ci --config crabpot.config.json --out reports --no-openclaw
|
||||
```
|
||||
|
||||
Use fixture suites when one repo wants to inspect many plugins. Use plugin-root
|
||||
`check` for normal plugin CI.
|
||||
Fixture-suite configs are loaded through the explicit fixture helpers. That keeps
|
||||
normal plugin-root configuration simple while still supporting bulk compatibility
|
||||
harnesses.
|
||||
|
||||
## Mocking Model
|
||||
## Public API
|
||||
|
||||
Default inspection is static, offline, and credential-free. Runtime capture is
|
||||
the only mode that imports plugin code.
|
||||
Prefer the CLI for normal plugin repositories. Import the public API when a test
|
||||
harness needs to compose workflows directly:
|
||||
|
||||
When `--mock-sdk` is enabled, the inspector generates temporary modules for
|
||||
`openclaw/plugin-sdk` subpaths and unresolved external packages discovered in
|
||||
the plugin import graph. The mock SDK captures registrations; it does not call
|
||||
network services, launch OpenClaw, run provider SDKs, or emulate service
|
||||
lifecycle side effects.
|
||||
```js
|
||||
import { pluginRoot } from "@openclaw/plugin-inspector";
|
||||
|
||||
Use the mock lane for plugin compatibility CI. Keep live provider/service tests
|
||||
in the plugin repo behind their own credentials and explicit opt-in flags.
|
||||
const { report, paths } = await pluginRoot.runCheck({
|
||||
pluginRoot: process.cwd(),
|
||||
openclawPath: false,
|
||||
outDir: "reports",
|
||||
});
|
||||
|
||||
## Scope
|
||||
console.log(report.status, paths.jsonPath);
|
||||
```
|
||||
|
||||
Default inspection is offline and credential-free. It reads manifests, package
|
||||
metadata, and source files, then reports observed `api.on(...)`,
|
||||
`api.register*`, `define*`, SDK imports, and manifest contracts.
|
||||
OpenClaw target checkout parsing is limited to public compatibility registries,
|
||||
SDK package exports, manifest types, hooks, and captured registrar metadata.
|
||||
Stable grouped facades:
|
||||
|
||||
Cold import capture, synthetic contract probes, and runtime capture are explicit
|
||||
opt-in modes. Live lanes stay credential-gated and must never run in default CI.
|
||||
| Facade | Use |
|
||||
| --- | --- |
|
||||
| `pluginRoot` | Load config, inspect, run checks, capture entrypoints, or set up a plugin repo. |
|
||||
| `fixtureSuites` | Load fixture-suite configs, run reports, and build fixture-suite readiness plans. |
|
||||
| `staticInspection` | Inspect source text or fixture sets without the compatibility report layer. |
|
||||
| `reports` | Render/write reports and classify issue findings. |
|
||||
| `contracts` | Build, render, validate, and write contract captures and coverage. |
|
||||
| `ci` | Build summaries, policy reports, execution results, SARIF, and JUnit outputs. |
|
||||
| `runtime` | Build runtime profiles, profile diffs, ref diffs, and import-loop profiles. |
|
||||
| `synthetic` | Build and run synthetic probe plans. |
|
||||
|
||||
Named exports remain available for existing automation. Prefer the grouped
|
||||
facades for new code because they show ownership and keep downstream wrappers
|
||||
thin.
|
||||
|
||||
## Development
|
||||
|
||||
Repository checks are intentionally small and offline:
|
||||
|
||||
```bash
|
||||
npm test
|
||||
npm run release:contents
|
||||
npm run check
|
||||
```
|
||||
|
||||
`npm run check` runs the Node test suite and the package-contents guard. The
|
||||
contents guard shells through `npm pack --dry-run --json` and verifies the npm
|
||||
tarball includes package entrypoints, examples, README assets, and no private
|
||||
`test/`, `scripts/`, or `.github/` paths.
|
||||
|
||||
Useful release-prep commands:
|
||||
|
||||
```bash
|
||||
npm run release:local
|
||||
npm run release:readiness
|
||||
npm run release:notes
|
||||
npm run release:plan
|
||||
npm run release:crabpot -- --crabpot ../crabpot
|
||||
```
|
||||
|
||||
`release:readiness` proves the local package and verifies Crabpot follow-through.
|
||||
It does not publish.
|
||||
|
||||
Keep this package dependency-light. Do not add runtime dependencies unless they
|
||||
remove real complexity. Default checks must stay offline and credential-free.
|
||||
|
||||
## Release Notes
|
||||
|
||||
The package publishes from annotated `v*` tags through GitHub Actions. The
|
||||
release workflow runs the test suite, verifies the npm tarball, publishes the
|
||||
GitHub release, and publishes the public npm package through npm trusted
|
||||
publishing.
|
||||
|
||||
Before tagging a release:
|
||||
|
||||
1. Move `CHANGELOG.md` `Unreleased` notes into a versioned section.
|
||||
2. Update `package.json` to the same version.
|
||||
3. Update Crabpot's `pluginInspectorRef` to the release commit.
|
||||
4. Run `npm run release:readiness`.
|
||||
5. Run the Crabpot plugin-inspector smoke commands printed by
|
||||
`npm run release:crabpot -- --crabpot ../crabpot`.
|
||||
|
||||
After npm publish, update Crabpot's package pin and run:
|
||||
|
||||
```bash
|
||||
npm run release:crabpot -- --crabpot ../crabpot --published
|
||||
```
|
||||
|
||||
Do not publish npm packages without explicit owner approval.
|
||||
|
||||
## Contribution Notes
|
||||
|
||||
There is no `CONTRIBUTING.md` in this repository. Until one exists, use the repo
|
||||
scripts above as the local contract and follow these project rules:
|
||||
|
||||
- preserve stable report field names and finding codes;
|
||||
- prefer public OpenClaw plugin contracts over core internals;
|
||||
- isolate any OpenClaw source parsing behind explicit helpers;
|
||||
- keep runtime execution behind `--allow-execute` or
|
||||
`PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1`;
|
||||
- when behavior, entrypoints, release metadata, or the npm package version
|
||||
change, update Crabpot's `@openclaw/plugin-inspector` pin/docs/smoke path and
|
||||
run the Crabpot plugin-inspector smoke before calling the work done.
|
||||
|
||||
@ -55,9 +55,67 @@ npm trust github @openclaw/plugin-inspector --repo openclaw/plugin-inspector --f
|
||||
npm run release:local
|
||||
```
|
||||
|
||||
This runs tests and `npm pack --dry-run`. Once a version has been published,
|
||||
`npm publish --dry-run` rejects that same version, so the real publish check is
|
||||
the tag workflow.
|
||||
This runs tests and the package-contents guard, which shells through
|
||||
`npm pack --dry-run --json` and fails if exported files, README assets, or
|
||||
expected examples are missing from the npm tarball. Once a version has been
|
||||
published, `npm publish --dry-run` rejects that same version, so the real
|
||||
publish check is the tag workflow.
|
||||
|
||||
For normal patch prep before tagging, run the combined local readiness gate:
|
||||
|
||||
```bash
|
||||
npm run release:readiness
|
||||
```
|
||||
|
||||
That proves the package tarball locally and verifies Crabpot source-ref
|
||||
follow-through. It does not publish anything.
|
||||
|
||||
Before creating a tag, move the `CHANGELOG.md` `Unreleased` notes into a
|
||||
versioned section like `## 0.3.1 - 2026-04-28`, update `package.json` to the
|
||||
same version, and update Crabpot's `pluginInspectorRef` to the exact release
|
||||
commit.
|
||||
|
||||
Draft release notes from the current `Unreleased` section with:
|
||||
|
||||
```bash
|
||||
npm run release:notes
|
||||
```
|
||||
|
||||
Preview the exact patch-release edits without changing files:
|
||||
|
||||
```bash
|
||||
npm run release:plan
|
||||
```
|
||||
|
||||
That prints the next patch version, release tag, changelog heading, Crabpot
|
||||
source ref, and post-publish Crabpot package pin.
|
||||
|
||||
## Crabpot follow-through
|
||||
|
||||
Before tagging a release, update Crabpot's `pluginInspectorRef` to the
|
||||
plugin-inspector commit being released and run:
|
||||
|
||||
```bash
|
||||
npm run release:crabpot -- --crabpot ../crabpot
|
||||
```
|
||||
|
||||
The checklist verifies the source ref and prints the required Crabpot smoke
|
||||
commands:
|
||||
|
||||
```bash
|
||||
CRABPOT_PLUGIN_INSPECTOR_CLI=source npm run plugin-inspector:smoke
|
||||
npm run plugin-inspector:smoke
|
||||
```
|
||||
|
||||
After the npm package is published, update Crabpot's `pluginInspectorPackage` to
|
||||
the released version and run the stricter post-publish check:
|
||||
|
||||
```bash
|
||||
npm run release:crabpot -- --crabpot ../crabpot --published
|
||||
```
|
||||
|
||||
Do not consider a release complete until the Crabpot source ref, package pin,
|
||||
local smoke proof, and Crabpot CI proof are all current.
|
||||
|
||||
## Publish
|
||||
|
||||
|
||||
19
examples/circleci-plugin-inspector.yml
Normal file
19
examples/circleci-plugin-inspector.yml
Normal file
@ -0,0 +1,19 @@
|
||||
version: 2.1
|
||||
|
||||
jobs:
|
||||
plugin-inspector:
|
||||
docker:
|
||||
- image: cimg/node:24.0
|
||||
steps:
|
||||
- checkout
|
||||
- run: npm ci
|
||||
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
- store_test_results:
|
||||
path: reports
|
||||
- store_artifacts:
|
||||
path: reports
|
||||
|
||||
workflows:
|
||||
plugin-inspector:
|
||||
jobs:
|
||||
- plugin-inspector
|
||||
31
examples/github-actions-code-scanning.yml
Normal file
31
examples/github-actions-code-scanning.yml
Normal file
@ -0,0 +1,31 @@
|
||||
name: plugin-inspector
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
security-events: write
|
||||
|
||||
jobs:
|
||||
check:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v5
|
||||
- uses: actions/setup-node@v5
|
||||
with:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
- uses: github/codeql-action/upload-sarif@v3
|
||||
if: always()
|
||||
with:
|
||||
sarif_file: reports/plugin-inspector.sarif
|
||||
- uses: actions/upload-artifact@v5
|
||||
if: always()
|
||||
with:
|
||||
name: plugin-inspector-reports
|
||||
path: reports/plugin-inspector-*
|
||||
@ -15,8 +15,7 @@ jobs:
|
||||
node-version: 24
|
||||
cache: npm
|
||||
- run: npm ci
|
||||
- run: npx @openclaw/plugin-inspector check --no-openclaw
|
||||
- run: PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 npx @openclaw/plugin-inspector check --no-openclaw --runtime --mock-sdk
|
||||
- run: npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
- uses: actions/upload-artifact@v5
|
||||
if: always()
|
||||
with:
|
||||
|
||||
11
examples/gitlab-ci-plugin-inspector.yml
Normal file
11
examples/gitlab-ci-plugin-inspector.yml
Normal file
@ -0,0 +1,11 @@
|
||||
plugin_inspector:
|
||||
image: node:24
|
||||
script:
|
||||
- npm ci
|
||||
- npx @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
artifacts:
|
||||
when: always
|
||||
paths:
|
||||
- reports/plugin-inspector-*
|
||||
reports:
|
||||
junit: reports/plugin-inspector.junit.xml
|
||||
21
examples/package-json-plugin-inspector.json
Normal file
21
examples/package-json-plugin-inspector.json
Normal file
@ -0,0 +1,21 @@
|
||||
{
|
||||
"scripts": {
|
||||
"plugin:check": "plugin-inspector inspect --no-openclaw",
|
||||
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute"
|
||||
},
|
||||
"pluginInspector": {
|
||||
"version": 1,
|
||||
"plugin": {
|
||||
"id": "weather",
|
||||
"priority": "high",
|
||||
"seams": ["dynamic-tool"],
|
||||
"sourceRoot": "src",
|
||||
"expect": {
|
||||
"registrations": ["registerTool"]
|
||||
}
|
||||
},
|
||||
"capture": {
|
||||
"mockSdk": true
|
||||
}
|
||||
}
|
||||
}
|
||||
12
package.json
12
package.json
@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@openclaw/plugin-inspector",
|
||||
"version": "0.1.3",
|
||||
"version": "0.3.10",
|
||||
"private": false,
|
||||
"description": "Offline compatibility inspector for OpenClaw plugins.",
|
||||
"type": "module",
|
||||
@ -36,18 +36,26 @@
|
||||
"./ref-diff": "./src/ref-diff.js",
|
||||
"./runtime-capture-report": "./src/runtime-capture-report.js",
|
||||
"./runtime-profile": "./src/runtime-profile.js",
|
||||
"./synthetic-probe-suite": "./src/synthetic-probe-suite.js",
|
||||
"./synthetic-probes": "./src/synthetic-probes.js",
|
||||
"./workspace-plan": "./src/workspace-plan.js"
|
||||
},
|
||||
"files": [
|
||||
"src",
|
||||
"examples",
|
||||
"docs/plugin-inspector-banner.jpg",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"LICENSE"
|
||||
],
|
||||
"scripts": {
|
||||
"check": "npm test && npm pack --dry-run",
|
||||
"check": "npm test && npm run release:contents",
|
||||
"release:crabpot": "node scripts/check-crabpot-followthrough.mjs",
|
||||
"release:contents": "node scripts/check-package-contents.mjs",
|
||||
"release:plan": "node scripts/release-plan.mjs",
|
||||
"release:readiness": "npm run release:local && npm run release:crabpot",
|
||||
"release:local": "npm run check",
|
||||
"release:notes": "node scripts/release-notes.mjs --unreleased",
|
||||
"test": "node --test test/*.test.js"
|
||||
},
|
||||
"keywords": [
|
||||
|
||||
194
scripts/check-crabpot-followthrough.mjs
Normal file
194
scripts/check-crabpot-followthrough.mjs
Normal file
@ -0,0 +1,194 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, readFileSync, readdirSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const pluginInspectorRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const result = buildCrabpotFollowthroughChecklist({
|
||||
pluginInspectorRoot,
|
||||
crabpotRoot: options.crabpotRoot,
|
||||
expectedRef: options.expectedRef,
|
||||
expectedVersion: options.expectedVersion,
|
||||
requirePublishedPin: options.requirePublishedPin,
|
||||
});
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify(result, null, 2));
|
||||
} else {
|
||||
printChecklist(result);
|
||||
}
|
||||
|
||||
if (result.status === "fail") {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildCrabpotFollowthroughChecklist(options = {}) {
|
||||
const root = options.pluginInspectorRoot ?? pluginInspectorRoot;
|
||||
const crabpotRoot = path.resolve(root, options.crabpotRoot ?? "../crabpot");
|
||||
const expectedRef = options.expectedRef ?? gitHead(root);
|
||||
const expectedVersion = options.expectedVersion ?? packageVersion(root);
|
||||
const sourcePath = path.join(crabpotRoot, "scripts", "plugin-inspector-source.mjs");
|
||||
const pins = existsSync(sourcePath) ? readCrabpotPins(sourcePath) : {};
|
||||
const advancedConsumers = findAdvancedConsumers(crabpotRoot);
|
||||
const expectedPackage = `@openclaw/plugin-inspector@${expectedVersion}`;
|
||||
const checks = [
|
||||
{
|
||||
id: "crabpot-source-ref",
|
||||
status: pins.pluginInspectorRef === expectedRef ? "pass" : "fail",
|
||||
message: `crabpot source ref points at plugin-inspector ${expectedRef}`,
|
||||
expected: expectedRef,
|
||||
actual: pins.pluginInspectorRef ?? "missing",
|
||||
fix: `update ${path.relative(process.cwd(), sourcePath)} pluginInspectorRef to ${expectedRef}`,
|
||||
},
|
||||
{
|
||||
id: "crabpot-public-api-migration",
|
||||
status: advancedConsumers.length === 0 ? "pass" : "fail",
|
||||
message: "crabpot scripts use the plugin-inspector root public API",
|
||||
expected: "no advanced bundle consumers",
|
||||
actual: advancedConsumers.length === 0 ? "none" : advancedConsumers.join(", "),
|
||||
fix: "switch listed crabpot scripts from loadPluginInspector() to loadPluginInspectorPublicApi() or local helpers",
|
||||
},
|
||||
{
|
||||
id: "crabpot-package-pin",
|
||||
status: pins.pluginInspectorPackage === expectedPackage ? "pass" : options.requirePublishedPin ? "fail" : "manual",
|
||||
message: `crabpot package smoke pin uses ${expectedPackage}`,
|
||||
expected: expectedPackage,
|
||||
actual: pins.pluginInspectorPackage ?? "missing",
|
||||
fix: `after npm publish, update ${path.relative(process.cwd(), sourcePath)} pluginInspectorPackage to ${expectedPackage}`,
|
||||
},
|
||||
{
|
||||
id: "crabpot-source-smoke",
|
||||
status: "manual",
|
||||
message: "run crabpot source-mode plugin-inspector smoke",
|
||||
command: "CRABPOT_PLUGIN_INSPECTOR_CLI=source npm run plugin-inspector:smoke",
|
||||
},
|
||||
{
|
||||
id: "crabpot-package-smoke",
|
||||
status: "manual",
|
||||
message: "run crabpot package-mode plugin-inspector smoke",
|
||||
command: "npm run plugin-inspector:smoke",
|
||||
},
|
||||
];
|
||||
|
||||
return {
|
||||
status: checks.some((check) => check.status === "fail") ? "fail" : "pass",
|
||||
pluginInspectorRef: expectedRef,
|
||||
pluginInspectorVersion: expectedVersion,
|
||||
crabpotRoot,
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
crabpotRoot: "../crabpot",
|
||||
expectedRef: undefined,
|
||||
expectedVersion: undefined,
|
||||
json: false,
|
||||
requirePublishedPin: false,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--crabpot") {
|
||||
options.crabpotRoot = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--expected-ref") {
|
||||
options.expectedRef = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--expected-version") {
|
||||
options.expectedVersion = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--json") {
|
||||
options.json = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--published") {
|
||||
options.requirePublishedPin = true;
|
||||
}
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function readCrabpotPins(sourcePath) {
|
||||
const text = readFileSync(sourcePath, "utf8");
|
||||
return {
|
||||
pluginInspectorRef: text.match(/pluginInspectorRef\s*=\s*"([^"]+)"/)?.[1],
|
||||
pluginInspectorPackage: text.match(/pluginInspectorPackage\s*=\s*"([^"]+)"/)?.[1],
|
||||
};
|
||||
}
|
||||
|
||||
function findAdvancedConsumers(crabpotRoot) {
|
||||
const scriptsDir = path.join(crabpotRoot, "scripts");
|
||||
if (!existsSync(scriptsDir)) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const consumers = [];
|
||||
for (const filePath of walkFiles(scriptsDir)) {
|
||||
if (!filePath.endsWith(".mjs") || path.basename(filePath) === "plugin-inspector-source.mjs") {
|
||||
continue;
|
||||
}
|
||||
const text = readFileSync(filePath, "utf8");
|
||||
if (/\bloadPluginInspector\s*\(/.test(text) || /src["']\s*,\s*["']advanced\.js/.test(text)) {
|
||||
consumers.push(path.relative(crabpotRoot, filePath));
|
||||
}
|
||||
}
|
||||
return consumers.sort();
|
||||
}
|
||||
|
||||
function walkFiles(dir) {
|
||||
const files = [];
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const filePath = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) {
|
||||
files.push(...walkFiles(filePath));
|
||||
} else if (entry.isFile()) {
|
||||
files.push(filePath);
|
||||
}
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
function gitHead(root) {
|
||||
return execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim();
|
||||
}
|
||||
|
||||
function packageVersion(root) {
|
||||
return JSON.parse(readFileSync(path.join(root, "package.json"), "utf8")).version;
|
||||
}
|
||||
|
||||
function printChecklist(result) {
|
||||
console.log(`crabpot follow-through: ${result.status}`);
|
||||
console.log(`plugin-inspector ref: ${result.pluginInspectorRef}`);
|
||||
console.log(`plugin-inspector version: ${result.pluginInspectorVersion}`);
|
||||
for (const check of result.checks) {
|
||||
console.log(`- ${check.status.toUpperCase()} ${check.id}: ${check.message}`);
|
||||
if (check.actual && check.actual !== check.expected) {
|
||||
console.log(` expected: ${check.expected}`);
|
||||
console.log(` actual: ${check.actual}`);
|
||||
}
|
||||
if (check.command) {
|
||||
console.log(` command: ${check.command}`);
|
||||
}
|
||||
if (check.status === "fail" && check.fix) {
|
||||
console.log(` fix: ${check.fix}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
138
scripts/check-package-contents.mjs
Normal file
138
scripts/check-package-contents.mjs
Normal file
@ -0,0 +1,138 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const root = path.resolve(process.argv[2] ?? ".");
|
||||
const packageJson = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
|
||||
const readmeText = readFileSync(path.join(root, "README.md"), "utf8");
|
||||
const filePaths = npmPackFilePaths(root);
|
||||
const result = buildPackageContentsChecklist({ filePaths, packageJson, readmeText });
|
||||
|
||||
printChecklist(result);
|
||||
if (result.status === "fail") {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildPackageContentsChecklist({ filePaths, packageJson, readmeText = "" }) {
|
||||
const files = new Set(filePaths);
|
||||
const checks = [
|
||||
...requiredFileChecks(files),
|
||||
...entrypointChecks(files, packageJson),
|
||||
...readmeAssetChecks(files, readmeText),
|
||||
...forbiddenPathChecks(filePaths),
|
||||
];
|
||||
|
||||
return {
|
||||
status: checks.some((check) => check.status === "fail") ? "fail" : "pass",
|
||||
entryCount: filePaths.length,
|
||||
checks,
|
||||
};
|
||||
}
|
||||
|
||||
function requiredFileChecks(files) {
|
||||
return [
|
||||
"package.json",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"LICENSE",
|
||||
"src/cli.js",
|
||||
"src/index.js",
|
||||
"examples/github-actions-plugin-inspector.yml",
|
||||
"examples/plugin-inspector.config.json",
|
||||
].map((filePath) => presenceCheck(files, "package-required-file", filePath));
|
||||
}
|
||||
|
||||
function entrypointChecks(files, packageJson) {
|
||||
const checks = [];
|
||||
for (const [name, filePath] of Object.entries(packageJson.bin ?? {})) {
|
||||
checks.push(presenceCheck(files, "package-bin-entry", normalizePackagePath(filePath), name));
|
||||
}
|
||||
for (const [name, target] of Object.entries(packageJson.exports ?? {})) {
|
||||
for (const filePath of exportTargets(target)) {
|
||||
checks.push(presenceCheck(files, "package-export-entry", normalizePackagePath(filePath), name));
|
||||
}
|
||||
}
|
||||
return checks;
|
||||
}
|
||||
|
||||
function readmeAssetChecks(files, readmeText) {
|
||||
const assetPaths = new Set();
|
||||
for (const match of readmeText.matchAll(/<img\s+[^>]*src=["']([^"']+)["']/gi)) {
|
||||
assetPaths.add(match[1]);
|
||||
}
|
||||
for (const match of readmeText.matchAll(/!\[[^\]]*]\(([^)]+)\)/g)) {
|
||||
assetPaths.add(match[1]);
|
||||
}
|
||||
|
||||
return [...assetPaths]
|
||||
.filter((assetPath) => isPackageRelativeAsset(assetPath))
|
||||
.map((assetPath) => presenceCheck(files, "package-readme-asset", normalizePackagePath(assetPath)));
|
||||
}
|
||||
|
||||
function forbiddenPathChecks(filePaths) {
|
||||
const forbiddenPrefixes = ["test/", "scripts/", ".github/"];
|
||||
return filePaths
|
||||
.filter((filePath) => forbiddenPrefixes.some((prefix) => filePath.startsWith(prefix)))
|
||||
.map((filePath) => ({
|
||||
id: "package-forbidden-path",
|
||||
status: "fail",
|
||||
message: `${filePath} should not be published in the npm package`,
|
||||
}));
|
||||
}
|
||||
|
||||
function presenceCheck(files, id, filePath, detail = filePath) {
|
||||
return {
|
||||
id,
|
||||
status: files.has(filePath) ? "pass" : "fail",
|
||||
message: `${detail} includes ${filePath}`,
|
||||
expected: filePath,
|
||||
actual: files.has(filePath) ? filePath : "missing",
|
||||
};
|
||||
}
|
||||
|
||||
function exportTargets(target) {
|
||||
if (typeof target === "string") {
|
||||
return [target];
|
||||
}
|
||||
if (target && typeof target === "object") {
|
||||
return Object.values(target).flatMap((value) => exportTargets(value));
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function isPackageRelativeAsset(assetPath) {
|
||||
return !assetPath.startsWith("http://")
|
||||
&& !assetPath.startsWith("https://")
|
||||
&& !assetPath.startsWith("#")
|
||||
&& !assetPath.startsWith("/");
|
||||
}
|
||||
|
||||
function normalizePackagePath(filePath) {
|
||||
return filePath.replace(/^\.\//, "");
|
||||
}
|
||||
|
||||
function npmPackFilePaths(root) {
|
||||
const output = execFileSync("npm", ["pack", "--dry-run", "--json"], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "inherit"],
|
||||
});
|
||||
const pack = JSON.parse(output)[0];
|
||||
return pack.files.map((file) => file.path).sort();
|
||||
}
|
||||
|
||||
function printChecklist(result) {
|
||||
console.log(`package contents: ${result.status}`);
|
||||
console.log(`package entries: ${result.entryCount}`);
|
||||
for (const check of result.checks) {
|
||||
console.log(`- ${check.status.toUpperCase()} ${check.id}: ${check.message}`);
|
||||
if (check.status === "fail" && check.actual !== check.expected) {
|
||||
console.log(` expected: ${check.expected}`);
|
||||
console.log(` actual: ${check.actual}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
66
scripts/release-notes.mjs
Normal file
66
scripts/release-notes.mjs
Normal file
@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const changelogText = readFileSync(options.changelogPath, "utf8");
|
||||
process.stdout.write(extractReleaseNotes({ changelogText, version: options.version }));
|
||||
}
|
||||
|
||||
export function extractReleaseNotes({ changelogText, version }) {
|
||||
const lines = changelogText.split(/\r?\n/);
|
||||
const start = findReleaseStart(lines, version);
|
||||
if (start === -1) {
|
||||
throw new Error(`CHANGELOG.md is missing a ${version} release section`);
|
||||
}
|
||||
|
||||
const end = lines.findIndex((line, index) => index > start && line.startsWith("## "));
|
||||
const body = lines.slice(start + 1, end === -1 ? lines.length : end).join("\n").trim();
|
||||
const title = version === "Unreleased" ? "plugin-inspector unreleased" : `plugin-inspector v${version}`;
|
||||
return `## ${title}\n\n${body}\n`;
|
||||
}
|
||||
|
||||
function findReleaseStart(lines, version) {
|
||||
if (version === "Unreleased") {
|
||||
return lines.findIndex((line) => line.trim() === "## Unreleased");
|
||||
}
|
||||
return lines.findIndex((line) => line.startsWith(`## ${version} - `));
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
changelogPath: path.resolve("CHANGELOG.md"),
|
||||
version: undefined,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--changelog") {
|
||||
options.changelogPath = path.resolve(argv[index + 1]);
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--unreleased") {
|
||||
options.version = "Unreleased";
|
||||
continue;
|
||||
}
|
||||
if (arg === "--version") {
|
||||
options.version = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (!options.version) {
|
||||
options.version = arg;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
if (!options.version) {
|
||||
throw new Error("usage: release-notes.mjs <version>|--unreleased [--changelog CHANGELOG.md]");
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
203
scripts/release-plan.mjs
Normal file
203
scripts/release-plan.mjs
Normal file
@ -0,0 +1,203 @@
|
||||
#!/usr/bin/env node
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { readFileSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { extractReleaseNotes } from "./release-notes.mjs";
|
||||
|
||||
if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
const root = path.resolve(options.root);
|
||||
const packageJson = JSON.parse(readFileSync(path.join(root, "package.json"), "utf8"));
|
||||
const changelogText = readFileSync(path.join(root, "CHANGELOG.md"), "utf8");
|
||||
const plan = buildReleasePlan({
|
||||
packageName: packageJson.name,
|
||||
packageVersion: packageJson.version,
|
||||
changelogText,
|
||||
releaseRef: options.releaseRef ?? gitHead(root),
|
||||
releaseDate: options.date,
|
||||
nextVersion: options.version,
|
||||
});
|
||||
|
||||
if (options.json) {
|
||||
console.log(JSON.stringify(plan, null, 2));
|
||||
} else {
|
||||
printReleasePlan(plan);
|
||||
}
|
||||
|
||||
if (plan.status === "fail") {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildReleasePlan({
|
||||
packageName = "@openclaw/plugin-inspector",
|
||||
packageVersion,
|
||||
changelogText,
|
||||
releaseRef,
|
||||
releaseDate = today(),
|
||||
nextVersion,
|
||||
}) {
|
||||
const version = nextVersion ?? bumpPatch(packageVersion);
|
||||
const checks = [
|
||||
{
|
||||
id: "version-advance",
|
||||
status: compareVersions(version, packageVersion) > 0 ? "pass" : "fail",
|
||||
message: `${version} is newer than ${packageVersion}`,
|
||||
expected: `>${packageVersion}`,
|
||||
actual: version,
|
||||
},
|
||||
{
|
||||
id: "changelog-unreleased",
|
||||
status: hasUnreleasedNotes(changelogText) ? "pass" : "fail",
|
||||
message: "CHANGELOG.md has non-empty Unreleased notes",
|
||||
},
|
||||
];
|
||||
|
||||
return {
|
||||
status: checks.some((check) => check.status === "fail") ? "fail" : "pass",
|
||||
packageName,
|
||||
currentVersion: packageVersion,
|
||||
nextVersion: version,
|
||||
releaseDate,
|
||||
releaseRef,
|
||||
tagName: `v${version}`,
|
||||
changelogHeading: `## ${version} - ${releaseDate}`,
|
||||
crabpotSourceRef: releaseRef,
|
||||
crabpotPackagePin: `${packageName}@${version}`,
|
||||
releaseNotes: safeReleaseNotes(changelogText),
|
||||
checks,
|
||||
steps: [
|
||||
`update package.json version to ${version}`,
|
||||
`move CHANGELOG.md Unreleased notes to "${`## ${version} - ${releaseDate}`}"`,
|
||||
`set crabpot pluginInspectorRef to ${releaseRef}`,
|
||||
"run npm run release:readiness",
|
||||
`create and push annotated tag ${`v${version}`}`,
|
||||
`after npm publish, set crabpot pluginInspectorPackage to ${packageName}@${version}`,
|
||||
"run npm run release:crabpot -- --published",
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function hasUnreleasedNotes(changelogText) {
|
||||
try {
|
||||
return extractReleaseNotes({ changelogText, version: "Unreleased" })
|
||||
.split(/\r?\n/)
|
||||
.some((line) => line.trim().startsWith("- "));
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function safeReleaseNotes(changelogText) {
|
||||
try {
|
||||
return extractReleaseNotes({ changelogText, version: "Unreleased" });
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
function bumpPatch(version) {
|
||||
const match = version?.match(/^(\d+)\.(\d+)\.(\d+)$/);
|
||||
if (!match) {
|
||||
throw new Error(`cannot infer next patch version from ${version}`);
|
||||
}
|
||||
return `${match[1]}.${match[2]}.${Number(match[3]) + 1}`;
|
||||
}
|
||||
|
||||
function compareVersions(left, right) {
|
||||
const leftParts = parseVersion(left);
|
||||
const rightParts = parseVersion(right);
|
||||
for (let index = 0; index < 3; index += 1) {
|
||||
if (leftParts[index] !== rightParts[index]) {
|
||||
return leftParts[index] - rightParts[index];
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
function parseVersion(version) {
|
||||
const match = version?.match(/^(\d+)\.(\d+)\.(\d+)$/);
|
||||
if (!match) {
|
||||
throw new Error(`invalid semver version: ${version}`);
|
||||
}
|
||||
return match.slice(1).map(Number);
|
||||
}
|
||||
|
||||
function today() {
|
||||
return new Date().toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function gitHead(root) {
|
||||
return execFileSync("git", ["rev-parse", "HEAD"], {
|
||||
cwd: root,
|
||||
encoding: "utf8",
|
||||
stdio: ["ignore", "pipe", "ignore"],
|
||||
}).trim();
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = {
|
||||
root: ".",
|
||||
date: today(),
|
||||
json: false,
|
||||
releaseRef: undefined,
|
||||
version: undefined,
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--root") {
|
||||
options.root = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--date") {
|
||||
options.date = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--json") {
|
||||
options.json = true;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--ref") {
|
||||
options.releaseRef = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (arg === "--version") {
|
||||
options.version = argv[index + 1];
|
||||
index += 1;
|
||||
continue;
|
||||
}
|
||||
if (!options.version) {
|
||||
options.version = arg;
|
||||
continue;
|
||||
}
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
|
||||
return options;
|
||||
}
|
||||
|
||||
function printReleasePlan(plan) {
|
||||
console.log(`release plan: ${plan.status}`);
|
||||
console.log(`package: ${plan.packageName}`);
|
||||
console.log(`current version: ${plan.currentVersion}`);
|
||||
console.log(`next version: ${plan.nextVersion}`);
|
||||
console.log(`release date: ${plan.releaseDate}`);
|
||||
console.log(`release ref: ${plan.releaseRef}`);
|
||||
console.log(`tag: ${plan.tagName}`);
|
||||
for (const check of plan.checks) {
|
||||
console.log(`- ${check.status.toUpperCase()} ${check.id}: ${check.message}`);
|
||||
if (check.status === "fail" && check.actual !== check.expected) {
|
||||
console.log(` expected: ${check.expected}`);
|
||||
console.log(` actual: ${check.actual}`);
|
||||
}
|
||||
}
|
||||
console.log("steps:");
|
||||
for (const step of plan.steps) {
|
||||
console.log(`- ${step}`);
|
||||
}
|
||||
}
|
||||
@ -33,8 +33,17 @@ export {
|
||||
renderCiSummaryMarkdown,
|
||||
writeCiSummary,
|
||||
} from "./ci-summary.js";
|
||||
export {
|
||||
buildSarifReport,
|
||||
defaultJunitPath,
|
||||
defaultSarifPath,
|
||||
renderJunitXml,
|
||||
reportFindings,
|
||||
writeCiOutputArtifacts,
|
||||
} from "./ci-outputs.js";
|
||||
export {
|
||||
buildContractProbes,
|
||||
compatRecordForIssueCode,
|
||||
contractProbeRules,
|
||||
probePriority,
|
||||
} from "./contract-probes.js";
|
||||
@ -86,6 +95,7 @@ export {
|
||||
classifyTargetOpenClawCoverage,
|
||||
readPackageSummaries,
|
||||
readPluginManifests,
|
||||
readSecurityManifests,
|
||||
summarizePackage,
|
||||
} from "./fixture-summary.js";
|
||||
export {
|
||||
@ -122,13 +132,16 @@ export {
|
||||
loadPluginRootConfig,
|
||||
normalizeInspectorConfig,
|
||||
normalizePluginRootConfig,
|
||||
packageJsonConfigKeys,
|
||||
packageId,
|
||||
validateInspectorConfig,
|
||||
} from "./config.js";
|
||||
export {
|
||||
buildPluginInspectorConfig,
|
||||
defaultInitPackageScripts,
|
||||
defaultInitConfigPath,
|
||||
defaultInitWorkflowPath,
|
||||
detectPackageManager,
|
||||
renderGithubActionsWorkflow,
|
||||
writePluginInspectorInit,
|
||||
} from "./init.js";
|
||||
@ -159,6 +172,7 @@ export {
|
||||
classifyCompatRecordCoverage,
|
||||
renderMarkdownReport,
|
||||
renderTextSummary,
|
||||
sanitizeReportArtifact,
|
||||
writeCompatibilityReport,
|
||||
writeReport,
|
||||
} from "./report.js";
|
||||
@ -170,6 +184,10 @@ export {
|
||||
validateRuntimeProfile,
|
||||
writeRuntimeProfile,
|
||||
} from "./runtime-profile.js";
|
||||
export {
|
||||
applyRuntimeExecutionCoverage,
|
||||
buildRuntimeExecutionCoverage,
|
||||
} from "./runtime-reconciliation.js";
|
||||
export {
|
||||
buildRuntimeCaptureReport,
|
||||
renderRuntimeCaptureMarkdown,
|
||||
@ -187,6 +205,7 @@ export {
|
||||
validateSyntheticProbePlan,
|
||||
writeSyntheticProbePlan,
|
||||
} from "./synthetic-probes.js";
|
||||
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
|
||||
export {
|
||||
buildWorkspacePlan,
|
||||
defaultWorkspacePlanOptions,
|
||||
@ -194,3 +213,10 @@ export {
|
||||
validateWorkspacePlan,
|
||||
writeWorkspacePlan,
|
||||
} from "./workspace-plan.js";
|
||||
export {
|
||||
inspectCompatibilityFixtureSetConfig,
|
||||
renderFixtureSetIssuesReport,
|
||||
renderFixtureSetMarkdownReport,
|
||||
runFixtureSetReport,
|
||||
writeFixtureSetReports,
|
||||
} from "./api.js";
|
||||
|
||||
177
src/api.js
177
src/api.js
@ -1,11 +1,31 @@
|
||||
import path from "node:path";
|
||||
import { createCaptureApi } from "./capture-api.js";
|
||||
import { loadInspectorConfig, loadPluginRootConfig } from "./config.js";
|
||||
import { renderCompatibilityIssuesReport, renderCompatibilityMarkdownReport } from "./compatibility-report.js";
|
||||
import { writePluginInspectorInit } from "./init.js";
|
||||
import { captureEntrypoint } from "./inspector.js";
|
||||
import { renderTextSummary, writeCompatibilityReport } from "./report.js";
|
||||
import { writeCiOutputArtifacts } from "./ci-outputs.js";
|
||||
import { buildRuntimeCaptureReport, writeRuntimeCaptureReport } from "./runtime-capture-report.js";
|
||||
import { inspectCompatibilityFixtureSet, inspectFixtureSet } from "./inspector.js";
|
||||
import {
|
||||
buildColdImportReadiness,
|
||||
renderColdImportReadinessMarkdown,
|
||||
validateColdImportReadiness,
|
||||
writeColdImportReadiness,
|
||||
} from "./cold-import-readiness.js";
|
||||
import {
|
||||
buildWorkspacePlan,
|
||||
renderWorkspacePlanMarkdown,
|
||||
validateWorkspacePlan,
|
||||
writeWorkspacePlan,
|
||||
} from "./workspace-plan.js";
|
||||
import {
|
||||
buildPlatformProbes,
|
||||
renderPlatformProbesMarkdown,
|
||||
validatePlatformProbes,
|
||||
writePlatformProbes,
|
||||
} from "./platform-probes.js";
|
||||
|
||||
export async function loadPluginConfig(options = {}) {
|
||||
if (options.config) {
|
||||
@ -25,6 +45,7 @@ export async function inspectPluginRoot(options = {}) {
|
||||
return inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
});
|
||||
}
|
||||
@ -34,6 +55,16 @@ export async function inspectFixtureSetConfig(options = {}) {
|
||||
return inspectFixtureSet(config, { generatedAt: options.generatedAt });
|
||||
}
|
||||
|
||||
export async function inspectCompatibilityFixtureSetConfig(options = {}) {
|
||||
const config = await loadFixtureSetConfig(options);
|
||||
return inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
});
|
||||
}
|
||||
|
||||
export async function writePluginReports(report, options = {}) {
|
||||
return writeCompatibilityReport(report, {
|
||||
basename: options.basename,
|
||||
@ -44,6 +75,132 @@ export async function writePluginReports(report, options = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
export async function writeFixtureSetReports(report, options = {}) {
|
||||
return writeCompatibilityReport(report, {
|
||||
basename: options.basename,
|
||||
check: options.check,
|
||||
cwd: options.cwd,
|
||||
formatEvidence: options.formatEvidence,
|
||||
issuesBasename: options.issuesBasename,
|
||||
issuesPath: options.issuesPath,
|
||||
issuesTitle: options.issuesTitle,
|
||||
jsonPath: options.jsonPath,
|
||||
markdownPath: options.markdownPath,
|
||||
markdownTitle: options.markdownTitle,
|
||||
outDir: options.outDir,
|
||||
severityLabels: options.severityLabels,
|
||||
title: options.title,
|
||||
});
|
||||
}
|
||||
|
||||
export function renderFixtureSetMarkdownReport(report, options = {}) {
|
||||
return renderCompatibilityMarkdownReport(report, options);
|
||||
}
|
||||
|
||||
export function renderFixtureSetIssuesReport(report, options = {}) {
|
||||
return renderCompatibilityIssuesReport(report, options);
|
||||
}
|
||||
|
||||
export async function runFixtureSetReport(options = {}) {
|
||||
const report = await inspectCompatibilityFixtureSetConfig(options);
|
||||
const paths = options.write === false ? null : await writeFixtureSetReports(report, options);
|
||||
return { report, paths };
|
||||
}
|
||||
|
||||
export async function buildFixtureSetColdImportReadiness(options = {}) {
|
||||
const config = options.report ? null : await loadFixtureSetConfig(options);
|
||||
const report =
|
||||
options.report ??
|
||||
(await inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
}));
|
||||
|
||||
return buildColdImportReadiness({
|
||||
...options,
|
||||
report,
|
||||
rootDir: options.rootDir ?? config?.rootDir ?? options.cwd,
|
||||
});
|
||||
}
|
||||
|
||||
export function renderFixtureSetColdImportReadinessMarkdown(readiness, options = {}) {
|
||||
return renderColdImportReadinessMarkdown(readiness, options);
|
||||
}
|
||||
|
||||
export async function writeFixtureSetColdImportReadiness(readiness, options = {}) {
|
||||
return writeColdImportReadiness(readiness, options);
|
||||
}
|
||||
|
||||
export async function runFixtureSetColdImportReadiness(options = {}) {
|
||||
const readiness = await buildFixtureSetColdImportReadiness(options);
|
||||
const paths = options.write === false ? null : await writeFixtureSetColdImportReadiness(readiness, options);
|
||||
return { readiness, paths };
|
||||
}
|
||||
|
||||
export async function buildFixtureSetWorkspacePlan(options = {}) {
|
||||
const config = options.report ? null : await loadFixtureSetConfig(options);
|
||||
const report =
|
||||
options.report ??
|
||||
(await inspectCompatibilityFixtureSet(config, {
|
||||
generatedAt: options.generatedAt,
|
||||
openclawPath: options.openclawPath,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw: options.targetOpenClaw,
|
||||
}));
|
||||
const rootDir = options.rootDir ?? config?.rootDir ?? options.cwd;
|
||||
const readiness = options.readiness ?? buildColdImportReadiness({ ...options, report, rootDir });
|
||||
|
||||
return buildWorkspacePlan({
|
||||
...options,
|
||||
report,
|
||||
readiness,
|
||||
rootDir,
|
||||
});
|
||||
}
|
||||
|
||||
export function renderFixtureSetWorkspacePlanMarkdown(plan, options = {}) {
|
||||
return renderWorkspacePlanMarkdown(plan, options);
|
||||
}
|
||||
|
||||
export function validateFixtureSetWorkspacePlan(plan, options = {}) {
|
||||
return validateWorkspacePlan(plan, options);
|
||||
}
|
||||
|
||||
export async function writeFixtureSetWorkspacePlan(plan, options = {}) {
|
||||
return writeWorkspacePlan(plan, options);
|
||||
}
|
||||
|
||||
export async function runFixtureSetWorkspacePlan(options = {}) {
|
||||
const plan = await buildFixtureSetWorkspacePlan(options);
|
||||
const paths = options.write === false ? null : await writeFixtureSetWorkspacePlan(plan, options);
|
||||
return { plan, paths };
|
||||
}
|
||||
|
||||
export async function buildFixtureSetPlatformProbes(options = {}) {
|
||||
const plan = options.plan ?? (await buildFixtureSetWorkspacePlan(options));
|
||||
return buildPlatformProbes({ ...options, plan });
|
||||
}
|
||||
|
||||
export function renderFixtureSetPlatformProbesMarkdown(report, options = {}) {
|
||||
return renderPlatformProbesMarkdown(report, options);
|
||||
}
|
||||
|
||||
export function validateFixtureSetPlatformProbes(report, options = {}) {
|
||||
return validatePlatformProbes(report, options);
|
||||
}
|
||||
|
||||
export async function writeFixtureSetPlatformProbes(report, options = {}) {
|
||||
return writePlatformProbes(report, options);
|
||||
}
|
||||
|
||||
export async function runFixtureSetPlatformProbes(options = {}) {
|
||||
const report = await buildFixtureSetPlatformProbes(options);
|
||||
const paths = options.write === false ? null : await writeFixtureSetPlatformProbes(report, options);
|
||||
return { report, paths };
|
||||
}
|
||||
|
||||
export async function runPluginCheck(options = {}) {
|
||||
const outDir = options.outDir ?? "reports";
|
||||
const config = await loadPluginConfig(options);
|
||||
@ -54,8 +211,8 @@ export async function runPluginCheck(options = {}) {
|
||||
const mockSdk = options.mockSdk ?? config.capture?.mockSdk ?? true;
|
||||
|
||||
if (capture === true) {
|
||||
if (process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
|
||||
throw new Error("runtime capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 in an isolated workspace");
|
||||
if (!executionAllowed(options)) {
|
||||
throw new Error("runtime capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 or --allow-execute in an isolated workspace");
|
||||
}
|
||||
const runtimeCapture = await buildRuntimeCaptureReport({
|
||||
mockSdk,
|
||||
@ -88,4 +245,18 @@ export async function setupPluginInspector(options = {}) {
|
||||
return writePluginInspectorInit(options);
|
||||
}
|
||||
|
||||
export { createCaptureApi, renderTextSummary };
|
||||
export { createCaptureApi, renderTextSummary, validateColdImportReadiness, writeCiOutputArtifacts };
|
||||
|
||||
function executionAllowed(options) {
|
||||
return options.allowExecution === true || process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED === "1";
|
||||
}
|
||||
|
||||
async function loadFixtureSetConfig(options) {
|
||||
if (options.config) {
|
||||
return {
|
||||
...options.config,
|
||||
rootDir: options.config.rootDir ?? options.rootDir ?? options.cwd ?? process.cwd(),
|
||||
};
|
||||
}
|
||||
return loadInspectorConfig(options.configPath, { cwd: options.cwd ?? options.rootDir });
|
||||
}
|
||||
|
||||
@ -95,7 +95,7 @@ export function renderPaddedMarkdownTable(rows, headers, options = {}) {
|
||||
}
|
||||
|
||||
export function escapeMarkdownTableCell(value) {
|
||||
return value.replace(/\|/g, "\\|").replace(/\n/g, "<br>");
|
||||
return value.replace(/\\/g, "\\\\").replace(/\|/g, "\\|").replace(/\n/g, "<br>");
|
||||
}
|
||||
|
||||
async function assertFileMatches(filePath, expected) {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
export const defaultCaptureApiRegistrarProfiles = {
|
||||
registerChannel: {
|
||||
returnValue: ({ args }) => registrationObject(args, { id: "channel" }),
|
||||
returnValue: ({ args }) => channelRegistrationObject(args),
|
||||
},
|
||||
registerCli: {
|
||||
returnValue: ({ args }) => registrationObject(args, { name: "cli" }),
|
||||
@ -12,7 +12,7 @@ export const defaultCaptureApiRegistrarProfiles = {
|
||||
returnValue: ({ args }) => registrationObject(args, { id: "context-engine" }),
|
||||
},
|
||||
registerGatewayMethod: {
|
||||
returnValue: ({ args }) => registrationObject(args, { name: "gateway.method" }),
|
||||
returnValue: ({ args }) => gatewayMethodRegistrationObject(args),
|
||||
},
|
||||
registerHook: {
|
||||
returnValue: ({ api }) => api,
|
||||
@ -37,9 +37,10 @@ export const defaultCaptureApiRegistrarProfiles = {
|
||||
},
|
||||
registerService: {
|
||||
returnValue: ({ args }) => ({
|
||||
...registrationObject(args, { name: "service" }),
|
||||
...registrationObject(args, { id: "service", name: "service" }),
|
||||
start: async () => undefined,
|
||||
stop: async () => undefined,
|
||||
dispose: async () => undefined,
|
||||
}),
|
||||
},
|
||||
registerSpeechProvider: {
|
||||
@ -81,6 +82,24 @@ export function createCaptureApi(options = {}) {
|
||||
}
|
||||
return api;
|
||||
},
|
||||
onConversationBindingResolved(handler) {
|
||||
const captureIndex =
|
||||
captured.push({
|
||||
kind: "hook",
|
||||
name: "onConversationBindingResolved",
|
||||
handlerType: typeof handler,
|
||||
arguments: summarizeArguments([handler]),
|
||||
}) - 1;
|
||||
if (retainHandlers) {
|
||||
retained.push({
|
||||
kind: "hook",
|
||||
name: "onConversationBindingResolved",
|
||||
handler,
|
||||
captureIndex,
|
||||
});
|
||||
}
|
||||
return api;
|
||||
},
|
||||
},
|
||||
{
|
||||
get(target, property) {
|
||||
@ -125,9 +144,11 @@ export function createCaptureApi(options = {}) {
|
||||
|
||||
export function createCaptureContext(options = {}) {
|
||||
return {
|
||||
registrationMode: options.registrationMode ?? "full",
|
||||
config: options.config ?? {},
|
||||
logger: options.logger ?? console,
|
||||
pluginConfig: options.pluginConfig ?? {},
|
||||
resolvePath: options.resolvePath ?? ((value) => value),
|
||||
runtime: options.runtime ?? createRuntimeContext(options),
|
||||
secrets: options.secrets ?? createSecretContext(options),
|
||||
store: options.store ?? createStoreContext(options),
|
||||
@ -142,6 +163,12 @@ export function createCaptureContext(options = {}) {
|
||||
},
|
||||
gateway: options.gateway ?? {
|
||||
baseUrl: "http://127.0.0.1:0",
|
||||
async call(method, params) {
|
||||
return { ok: true, method, params };
|
||||
},
|
||||
respond(ok, result, error) {
|
||||
return { ok, result, ...(error ? { error } : {}) };
|
||||
},
|
||||
registerRoute(route) {
|
||||
return {
|
||||
...route,
|
||||
@ -223,6 +250,7 @@ function createStoreContext() {
|
||||
|
||||
function registrationObject(args, defaults) {
|
||||
const first = args[0];
|
||||
const callable = firstCallable(args);
|
||||
if (first && typeof first === "object") {
|
||||
return {
|
||||
...defaults,
|
||||
@ -232,13 +260,59 @@ function registrationObject(args, defaults) {
|
||||
};
|
||||
}
|
||||
if (typeof first === "string") {
|
||||
return {
|
||||
return withCallableDefaults({
|
||||
...defaults,
|
||||
name: first,
|
||||
id: defaults.id,
|
||||
}, callable);
|
||||
}
|
||||
return withCallableDefaults({ ...defaults }, callable);
|
||||
}
|
||||
|
||||
function channelRegistrationObject(args) {
|
||||
const first = args[0];
|
||||
const registration = registrationObject(args, { id: "channel" });
|
||||
if (first?.plugin && typeof first.plugin === "object") {
|
||||
return {
|
||||
...registration,
|
||||
id: objectId(first.plugin) ?? registration.id,
|
||||
plugin: first.plugin,
|
||||
};
|
||||
}
|
||||
return { ...defaults };
|
||||
return registration;
|
||||
}
|
||||
|
||||
function gatewayMethodRegistrationObject(args) {
|
||||
const [method, handler, options] = args;
|
||||
const registration = registrationObject(args, { name: "gateway.method" });
|
||||
if (typeof method !== "string") {
|
||||
return registration;
|
||||
}
|
||||
return {
|
||||
...registration,
|
||||
name: method,
|
||||
method,
|
||||
handler: typeof handler === "function" ? handler : registration.handler,
|
||||
run: typeof handler === "function" ? handler : registration.run,
|
||||
execute: typeof handler === "function" ? handler : registration.execute,
|
||||
scope: options?.scope ?? registration.scope,
|
||||
};
|
||||
}
|
||||
|
||||
function firstCallable(args) {
|
||||
return args.find((arg) => typeof arg === "function");
|
||||
}
|
||||
|
||||
function withCallableDefaults(value, callable) {
|
||||
if (!callable) {
|
||||
return value;
|
||||
}
|
||||
return {
|
||||
...value,
|
||||
handler: typeof value.handler === "function" ? value.handler : callable,
|
||||
run: typeof value.run === "function" ? value.run : callable,
|
||||
execute: typeof value.execute === "function" ? value.execute : callable,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeArguments(args) {
|
||||
|
||||
126
src/capture-config.js
Normal file
126
src/capture-config.js
Normal file
@ -0,0 +1,126 @@
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
export async function captureApiOptionsForPlugin(apiOptions = {}, options = {}) {
|
||||
if (apiOptions.pluginConfig !== undefined || !options.pluginRoot) {
|
||||
return apiOptions;
|
||||
}
|
||||
|
||||
const pluginConfig = await readSamplePluginConfig(options.pluginRoot);
|
||||
if (pluginConfig === undefined) {
|
||||
return apiOptions;
|
||||
}
|
||||
return {
|
||||
...apiOptions,
|
||||
pluginConfig,
|
||||
};
|
||||
}
|
||||
|
||||
async function readSamplePluginConfig(pluginRoot) {
|
||||
const manifestPath = path.join(pluginRoot, "openclaw.plugin.json");
|
||||
let manifest;
|
||||
try {
|
||||
manifest = JSON.parse(await readFile(manifestPath, "utf8"));
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const sample = sampleJsonSchema(manifest.configSchema, { key: "config" });
|
||||
return isPlainObject(sample) && Object.keys(sample).length > 0 ? sample : undefined;
|
||||
}
|
||||
|
||||
function sampleJsonSchema(schema, context = {}) {
|
||||
if (!isPlainObject(schema)) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
if (Array.isArray(schema.enum) && schema.enum.length > 0) {
|
||||
return schema.enum[0];
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(schema, "const")) {
|
||||
return schema.const;
|
||||
}
|
||||
if (Object.prototype.hasOwnProperty.call(schema, "default")) {
|
||||
return schema.default;
|
||||
}
|
||||
|
||||
const type = Array.isArray(schema.type) ? schema.type.find((item) => item !== "null") : schema.type;
|
||||
if (type === "object" || schema.properties) {
|
||||
return sampleObjectSchema(schema);
|
||||
}
|
||||
if (type === "array") {
|
||||
return [];
|
||||
}
|
||||
if (type === "boolean") {
|
||||
return false;
|
||||
}
|
||||
if (type === "number" || type === "integer") {
|
||||
return typeof schema.minimum === "number" ? schema.minimum : 1;
|
||||
}
|
||||
if (type === "string" || !type) {
|
||||
return sampleString(context.key);
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function sampleObjectSchema(schema) {
|
||||
const properties = isPlainObject(schema.properties) ? schema.properties : {};
|
||||
const required = new Set(Array.isArray(schema.required) ? schema.required : []);
|
||||
const output = {};
|
||||
|
||||
for (const key of Object.keys(properties)) {
|
||||
if (required.has(key)) {
|
||||
const value = sampleJsonSchema(properties[key], { key });
|
||||
if (value !== undefined) {
|
||||
output[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for (const key of Object.keys(properties)) {
|
||||
if (properties[key]?.type === "boolean") {
|
||||
output[key] = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (Object.keys(output).length === 0 && Number(schema.minProperties ?? 0) > 0) {
|
||||
const key = preferredSamplePropertyKey(properties);
|
||||
if (key) {
|
||||
const value = sampleJsonSchema(properties[key], { key });
|
||||
if (value !== undefined) {
|
||||
output[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
|
||||
function preferredSamplePropertyKey(properties) {
|
||||
for (const key of ["provider", "model", "apiKey", "id", "name", ...Object.keys(properties)]) {
|
||||
if (Object.prototype.hasOwnProperty.call(properties, key)) {
|
||||
return key;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function sampleString(key = "") {
|
||||
if (key === "provider") {
|
||||
return "openai";
|
||||
}
|
||||
if (key === "model") {
|
||||
return "text-embedding-3-small";
|
||||
}
|
||||
if (key === "apiKey") {
|
||||
return "fixture-api-key";
|
||||
}
|
||||
if (key === "dbPath") {
|
||||
return ".plugin-inspector/state/lancedb";
|
||||
}
|
||||
return "fixture";
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
235
src/ci-outputs.js
Normal file
235
src/ci-outputs.js
Normal file
@ -0,0 +1,235 @@
|
||||
import path from "node:path";
|
||||
import { writeArtifacts } from "./artifacts.js";
|
||||
|
||||
export const defaultSarifPath = "plugin-inspector.sarif";
|
||||
export const defaultJunitPath = "plugin-inspector.junit.xml";
|
||||
|
||||
export async function writeCiOutputArtifacts(report, options = {}) {
|
||||
const outDir = path.resolve(options.cwd ?? process.cwd(), options.outDir ?? "reports");
|
||||
const artifacts = [];
|
||||
|
||||
if (options.sarifPath) {
|
||||
artifacts.push({
|
||||
name: "sarifPath",
|
||||
path: path.resolve(outDir, options.sarifPath),
|
||||
json: buildSarifReport(report),
|
||||
});
|
||||
}
|
||||
|
||||
if (options.junitPath) {
|
||||
artifacts.push({
|
||||
name: "junitPath",
|
||||
path: path.resolve(outDir, options.junitPath),
|
||||
content: renderJunitXml(report),
|
||||
});
|
||||
}
|
||||
|
||||
if (artifacts.length === 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
return writeArtifacts(artifacts, { check: options.check });
|
||||
}
|
||||
|
||||
export function buildSarifReport(report) {
|
||||
const findings = reportFindings(report);
|
||||
const rules = [...new Map(findings.map((finding) => [finding.code, sarifRule(finding)])).values()];
|
||||
const fixtureById = new Map((report.fixtures ?? []).map((fixture) => [fixture.id, fixture]));
|
||||
|
||||
return {
|
||||
$schema: "https://json.schemastore.org/sarif-2.1.0.json",
|
||||
version: "2.1.0",
|
||||
runs: [
|
||||
{
|
||||
tool: {
|
||||
driver: {
|
||||
name: "plugin-inspector",
|
||||
informationUri: "https://github.com/openclaw/plugin-inspector",
|
||||
rules,
|
||||
},
|
||||
},
|
||||
results: findings.map((finding) => sarifResult(finding, fixtureById)),
|
||||
},
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
export function renderJunitXml(report) {
|
||||
const findings = reportFindings(report);
|
||||
const testcases = findings.length > 0 ? findings.map(junitFindingTestcase) : [junitPassingTestcase(report)];
|
||||
const failures = findings.filter(isBlockingFinding).length;
|
||||
const tests = testcases.length;
|
||||
|
||||
return [
|
||||
'<?xml version="1.0" encoding="UTF-8"?>',
|
||||
`<testsuite name="plugin-inspector" tests="${tests}" failures="${failures}" errors="0" skipped="0">`,
|
||||
...testcases,
|
||||
"</testsuite>",
|
||||
"",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function reportFindings(report) {
|
||||
const findings = new Map();
|
||||
for (const finding of [...(report.breakages ?? []), ...(report.warnings ?? []), ...(report.suggestions ?? [])]) {
|
||||
findings.set(findingKey(finding), finding);
|
||||
}
|
||||
for (const issue of report.issues ?? []) {
|
||||
const finding = issueToFinding(issue);
|
||||
findings.set(findingKey(finding), {
|
||||
...findings.get(findingKey(finding)),
|
||||
...finding,
|
||||
});
|
||||
}
|
||||
return [...findings.values()];
|
||||
}
|
||||
|
||||
function issueToFinding(issue) {
|
||||
return {
|
||||
fixture: issue.fixture,
|
||||
code: issue.code,
|
||||
level: issue.status === "blocking" ? "breakage" : "warning",
|
||||
message: issue.title,
|
||||
evidence: issue.evidence ?? [],
|
||||
severity: issue.severity,
|
||||
issueClass: issue.issueClass,
|
||||
};
|
||||
}
|
||||
|
||||
function findingKey(finding) {
|
||||
return [
|
||||
finding.fixture ?? "",
|
||||
finding.code ?? "",
|
||||
...normalizeEvidence(finding.evidence),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function sarifRule(finding) {
|
||||
return {
|
||||
id: finding.code,
|
||||
shortDescription: {
|
||||
text: finding.code,
|
||||
},
|
||||
fullDescription: {
|
||||
text: finding.message ?? finding.code,
|
||||
},
|
||||
defaultConfiguration: {
|
||||
level: sarifLevel(finding),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sarifResult(finding, fixtureById) {
|
||||
return {
|
||||
ruleId: finding.code,
|
||||
level: sarifLevel(finding),
|
||||
message: {
|
||||
text: finding.message ?? finding.code,
|
||||
},
|
||||
locations: [sarifLocation(finding, fixtureById)],
|
||||
properties: {
|
||||
fixture: finding.fixture,
|
||||
severity: finding.severity ?? finding.level,
|
||||
issueClass: finding.issueClass,
|
||||
evidence: normalizeEvidence(finding.evidence),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function sarifLocation(finding, fixtureById) {
|
||||
const parsed = parseEvidenceLocation(normalizeEvidence(finding.evidence)[0]);
|
||||
const fixture = fixtureById.get(finding.fixture);
|
||||
const uri = parsed?.uri ?? fixture?.path ?? ".";
|
||||
return {
|
||||
physicalLocation: {
|
||||
artifactLocation: {
|
||||
uri: normalizeUri(uri),
|
||||
},
|
||||
region: {
|
||||
startLine: parsed?.line ?? 1,
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function parseEvidenceLocation(evidence) {
|
||||
if (!evidence) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const ref = evidence.includes(" @ ") ? evidence.split(" @ ").pop() : evidence;
|
||||
const match = /^(?<uri>.+?):(?<line>\d+)(?::\d+)?$/.exec(ref);
|
||||
if (!match?.groups?.uri) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
uri: match.groups.uri,
|
||||
line: Number(match.groups.line),
|
||||
};
|
||||
}
|
||||
|
||||
function junitFindingTestcase(finding) {
|
||||
const classname = `plugin-inspector.${xmlName(finding.fixture ?? "unknown")}`;
|
||||
const name = `${finding.level ?? "finding"}:${finding.code}`;
|
||||
const output = normalizeEvidence(finding.evidence).join("\n");
|
||||
if (!isBlockingFinding(finding)) {
|
||||
return [
|
||||
` <testcase classname="${escapeXml(classname)}" name="${escapeXml(name)}">`,
|
||||
output ? ` <system-out>${escapeXml(output)}</system-out>` : "",
|
||||
" </testcase>",
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
return [
|
||||
` <testcase classname="${escapeXml(classname)}" name="${escapeXml(name)}">`,
|
||||
` <failure message="${escapeXml(finding.message ?? finding.code)}">${escapeXml(output || finding.message || finding.code)}</failure>`,
|
||||
" </testcase>",
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function junitPassingTestcase(report) {
|
||||
return ` <testcase classname="plugin-inspector" name="status:${escapeXml(report.status ?? "pass")}"/>`;
|
||||
}
|
||||
|
||||
function isBlockingFinding(finding) {
|
||||
return finding.level === "breakage" || finding.status === "blocking" || finding.severity === "P0";
|
||||
}
|
||||
|
||||
function sarifLevel(finding) {
|
||||
if (isBlockingFinding(finding) || finding.severity === "P1") {
|
||||
return "error";
|
||||
}
|
||||
if (finding.level === "warning" || finding.severity === "P2") {
|
||||
return "warning";
|
||||
}
|
||||
return "note";
|
||||
}
|
||||
|
||||
function normalizeEvidence(evidence) {
|
||||
if (Array.isArray(evidence)) {
|
||||
return evidence.map(String);
|
||||
}
|
||||
if (evidence == null) {
|
||||
return [];
|
||||
}
|
||||
return [String(evidence)];
|
||||
}
|
||||
|
||||
function normalizeUri(uri) {
|
||||
return String(uri).replaceAll(path.sep, "/");
|
||||
}
|
||||
|
||||
function xmlName(value) {
|
||||
return String(value).replace(/[^a-zA-Z0-9_.-]/g, "_");
|
||||
}
|
||||
|
||||
function escapeXml(value) {
|
||||
return String(value)
|
||||
.replace(/&/g, "&")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/"/g, """)
|
||||
.replace(/'/g, "'");
|
||||
}
|
||||
@ -241,7 +241,9 @@ function executionChecks(executionResults, policy, options) {
|
||||
}
|
||||
|
||||
function findPolicyMatch(rules, item) {
|
||||
return rules.find((rule) => item.seam === rule.seam && item.reason?.includes(rule.reasonIncludes));
|
||||
return rules.find(
|
||||
(rule) => (rule.seam === "*" || item.seam === rule.seam) && item.reason?.includes(rule.reasonIncludes),
|
||||
);
|
||||
}
|
||||
|
||||
function failedExecutionEvidence(executionResults) {
|
||||
|
||||
@ -56,8 +56,14 @@ export async function buildCiSummary(options = {}) {
|
||||
loaderJitiCandidates: reports.platform?.summary?.jitiAlternativeCount ?? 0,
|
||||
importLoopP50Ms: reports.importLoop?.summary?.p50WallMs ?? 0,
|
||||
importLoopP95Ms: reports.importLoop?.summary?.p95WallMs ?? 0,
|
||||
importLoopMaxRssMb: reports.importLoop?.summary?.maxPeakRssMb ?? 0,
|
||||
importLoopMaxCpuMs: reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
|
||||
importLoopOpenClawLifecycleCount: reports.importLoop?.summary?.openClawLifecycleCount ?? 0,
|
||||
importLoopOpenClawImportP50Ms: reports.importLoop?.summary?.p50OpenClawImportMs ?? 0,
|
||||
importLoopOpenClawActivationP50Ms: reports.importLoop?.summary?.p50OpenClawActivationMs ?? 0,
|
||||
importLoopMetricBasis: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb === undefined ? "raw" : "baseline-adjusted",
|
||||
importLoopMaxRssMb: reports.importLoop?.summary?.maxPluginPeakRssDeltaMb ?? reports.importLoop?.summary?.maxPeakRssMb ?? 0,
|
||||
importLoopMaxCpuMs: reports.importLoop?.summary?.maxPluginCpuDeltaMsEstimate ?? reports.importLoop?.summary?.maxCpuMsEstimate ?? 0,
|
||||
importLoopRssSampleCount: metricSampleCount(reports.importLoop, "rss", "maxPeakRssMb"),
|
||||
importLoopCpuSampleCount: metricSampleCount(reports.importLoop, "cpu", "maxCpuMsEstimate"),
|
||||
},
|
||||
topIssues: topIssues(reports.compatibility),
|
||||
refRegressions: (reports.refDiff?.regressions ?? []).slice(0, 20),
|
||||
@ -148,7 +154,7 @@ export function renderCiSummaryMarkdown(summary) {
|
||||
["Jiti loader candidates", summary.summary.loaderJitiCandidates],
|
||||
[
|
||||
"Import loop",
|
||||
`p50 ${summary.summary.importLoopP50Ms} ms / p95 ${summary.summary.importLoopP95Ms} ms / max RSS ${summary.summary.importLoopMaxRssMb} MB / CPU ${summary.summary.importLoopMaxCpuMs} ms`,
|
||||
importLoopSummaryLabel(summary.summary),
|
||||
],
|
||||
],
|
||||
["Metric", "Value"],
|
||||
@ -221,3 +227,44 @@ function topIssues(report) {
|
||||
function markdownTable(rows, headers) {
|
||||
return renderPaddedMarkdownTable(rows, headers, { nullValue: "-" });
|
||||
}
|
||||
|
||||
function metricSampleCount(report, kind, maxMetric) {
|
||||
const summaryKey = kind === "rss" ? "rssSampleCount" : "cpuSampleCount";
|
||||
const summaryCount = report?.summary?.[summaryKey];
|
||||
if (Number.isFinite(summaryCount)) {
|
||||
return summaryCount;
|
||||
}
|
||||
const sampleCount = inferSampleCount(report?.samples, kind);
|
||||
if (sampleCount > 0) {
|
||||
return sampleCount;
|
||||
}
|
||||
return (report?.summary?.[maxMetric] ?? 0) > 0 ? 1 : 0;
|
||||
}
|
||||
|
||||
function inferSampleCount(samples = [], kind) {
|
||||
if (!Array.isArray(samples)) {
|
||||
return 0;
|
||||
}
|
||||
return samples.reduce((sum, sample) => {
|
||||
if (kind === "rss") {
|
||||
return sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0));
|
||||
}
|
||||
return sum + (sample.cpuSampleCount ?? 0);
|
||||
}, 0);
|
||||
}
|
||||
|
||||
function importLoopSummaryLabel(summary) {
|
||||
const metricLabel = summary.importLoopMetricBasis === "baseline-adjusted" ? "plugin delta" : "raw";
|
||||
const lifecycle =
|
||||
summary.importLoopOpenClawLifecycleCount > 0
|
||||
? ` / OpenClaw import ${summary.importLoopOpenClawImportP50Ms} ms / activate ${summary.importLoopOpenClawActivationP50Ms} ms`
|
||||
: "";
|
||||
return `p50 ${summary.importLoopP50Ms} ms / p95 ${summary.importLoopP95Ms} ms / ${metricLabel} RSS ${formatSampledMetric(summary.importLoopMaxRssMb, summary.importLoopRssSampleCount)} / ${metricLabel} CPU ${formatSampledMetric(summary.importLoopMaxCpuMs, summary.importLoopCpuSampleCount, "ms")}${lifecycle}`;
|
||||
}
|
||||
|
||||
function formatSampledMetric(value, count, unit = "MB") {
|
||||
if ((count ?? 0) <= 0) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
151
src/cli.js
151
src/cli.js
@ -1,15 +1,20 @@
|
||||
#!/usr/bin/env node
|
||||
import path from "node:path";
|
||||
import {
|
||||
loadPluginConfig,
|
||||
renderTextSummary,
|
||||
sanitizeReportArtifact,
|
||||
runPluginCheck,
|
||||
} from "./index.js";
|
||||
import {
|
||||
buildCiSummary,
|
||||
captureEntrypoint,
|
||||
defaultJunitPath,
|
||||
defaultSarifPath,
|
||||
inspectCompatibilityFixtureSet,
|
||||
inspectFixtureSet,
|
||||
loadInspectorConfig,
|
||||
writeCiOutputArtifacts,
|
||||
writeCiSummary,
|
||||
writeCompatibilityReport,
|
||||
writePluginInspectorInit,
|
||||
@ -28,8 +33,14 @@ try {
|
||||
await runCheck(commandArgs);
|
||||
} else if (command === "init") {
|
||||
await runInit(commandArgs);
|
||||
} else if (command === "config") {
|
||||
await runConfig(commandArgs);
|
||||
} else if (command === "inspect" || command === "report") {
|
||||
await runReport(command, commandArgs);
|
||||
if (command === "inspect" && !commandArgs.includes("--config")) {
|
||||
await runCheck(commandArgs);
|
||||
} else {
|
||||
await runReport(command, commandArgs);
|
||||
}
|
||||
} else if (command === "ci") {
|
||||
await runCi(commandArgs);
|
||||
} else if (command === "capture") {
|
||||
@ -42,6 +53,18 @@ try {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
async function runConfig(commandArgs) {
|
||||
const configPath = readFlag(commandArgs, "--config");
|
||||
const pluginRoot = readFlag(commandArgs, "--plugin-root") ?? readFlag(commandArgs, "--root");
|
||||
const config = await loadPluginConfig({ configPath, pluginRoot });
|
||||
|
||||
if (commandArgs.includes("--json")) {
|
||||
console.log(JSON.stringify(config, null, 2));
|
||||
} else {
|
||||
console.log(renderConfigTextSummary(config));
|
||||
}
|
||||
}
|
||||
|
||||
async function runCheck(commandArgs) {
|
||||
const configPath = readFlag(commandArgs, "--config");
|
||||
const pluginRoot = readFlag(commandArgs, "--plugin-root") ?? readFlag(commandArgs, "--root");
|
||||
@ -50,12 +73,27 @@ async function runCheck(commandArgs) {
|
||||
const json = commandArgs.includes("--json");
|
||||
const capture = readRuntimeFlag(commandArgs);
|
||||
const mockSdk = readMockSdkFlag(commandArgs);
|
||||
const { report } = await runPluginCheck({ configPath, pluginRoot, outDir, openclawPath, capture, mockSdk });
|
||||
const allowExecution = readAllowExecutionFlag(commandArgs);
|
||||
const ciOutputs = readCiOutputFlags(commandArgs);
|
||||
const { report, paths } = await runPluginCheck({
|
||||
allowExecution,
|
||||
capture,
|
||||
configPath,
|
||||
mockSdk,
|
||||
openclawPath,
|
||||
outDir,
|
||||
pluginRoot,
|
||||
});
|
||||
await writeCiOutputArtifacts(report, {
|
||||
...ciOutputs,
|
||||
cwd: path.dirname(paths.jsonPath),
|
||||
outDir: ".",
|
||||
});
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
console.log(JSON.stringify(sanitizeReportArtifact(report), null, 2));
|
||||
} else {
|
||||
console.log(renderTextSummary(report));
|
||||
console.log(renderTextSummary(report, { artifacts: paths }));
|
||||
}
|
||||
|
||||
if (report.status !== "pass") {
|
||||
@ -67,19 +105,27 @@ async function runInit(commandArgs) {
|
||||
const pluginRoot = readFlag(commandArgs, "--plugin-root") ?? readFlag(commandArgs, "--root");
|
||||
const configPath = readFlag(commandArgs, "--config") ?? undefined;
|
||||
const workflowPath = readFlag(commandArgs, "--workflow") ?? undefined;
|
||||
const packageManager = readFlag(commandArgs, "--package-manager") ?? "npm";
|
||||
const packageManager = readFlag(commandArgs, "--package-manager") ?? undefined;
|
||||
const result = await writePluginInspectorInit({
|
||||
pluginRoot,
|
||||
configPath,
|
||||
workflowPath,
|
||||
packageManager,
|
||||
ci: commandArgs.includes("--ci"),
|
||||
dryRun: commandArgs.includes("--dry-run"),
|
||||
scripts: commandArgs.includes("--scripts"),
|
||||
force: commandArgs.includes("--force"),
|
||||
});
|
||||
|
||||
for (const filePath of result.written) {
|
||||
console.log(`wrote ${filePath}`);
|
||||
if (commandArgs.includes("--json")) {
|
||||
console.log(JSON.stringify(initCommandSummary(result), null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
for (const filePath of result.written) {
|
||||
console.log(`${result.dryRun ? "would write" : "wrote"} ${path.relative(result.pluginRoot, filePath)}`);
|
||||
}
|
||||
console.log(`package manager: ${result.packageManager}`);
|
||||
}
|
||||
|
||||
async function runReport(command, commandArgs) {
|
||||
@ -87,14 +133,20 @@ async function runReport(command, commandArgs) {
|
||||
const outDir = readFlag(commandArgs, "--out") ?? "reports";
|
||||
const check = commandArgs.includes("--check") || command === "ci";
|
||||
const json = commandArgs.includes("--json");
|
||||
const ciOutputs = readCiOutputFlags(commandArgs);
|
||||
const config = await loadInspectorConfig(configPath);
|
||||
const report = await inspectFixtureSet(config);
|
||||
await writeReport(report, { outDir });
|
||||
const paths = await writeReport(report, { outDir });
|
||||
await writeCiOutputArtifacts(report, {
|
||||
...ciOutputs,
|
||||
cwd: path.dirname(paths.jsonPath),
|
||||
outDir: ".",
|
||||
});
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
console.log(renderTextSummary(report));
|
||||
console.log(renderTextSummary(report, { artifacts: paths }));
|
||||
}
|
||||
|
||||
if (check && report.status !== "pass") {
|
||||
@ -108,8 +160,15 @@ async function runCi(commandArgs) {
|
||||
const outDir = readFlag(commandArgs, "--out") ?? "reports";
|
||||
const openclawPath = commandArgs.includes("--no-openclaw") ? false : readFlag(commandArgs, "--openclaw");
|
||||
const json = commandArgs.includes("--json");
|
||||
const capture = readRuntimeFlag(commandArgs);
|
||||
const mockSdk = readMockSdkFlag(commandArgs);
|
||||
const allowExecution = readAllowExecutionFlag(commandArgs);
|
||||
const ciOutputs = readCiOutputFlags(commandArgs, { defaultEnabled: true });
|
||||
const { report, reportDir } = await runCiCompatibilityReport({
|
||||
allowExecution,
|
||||
capture,
|
||||
configPath,
|
||||
mockSdk,
|
||||
openclawPath,
|
||||
outDir,
|
||||
pluginRoot,
|
||||
@ -128,6 +187,11 @@ async function runCi(commandArgs) {
|
||||
jsonPath: path.join(reportDir, "plugin-inspector-ci-summary.json"),
|
||||
markdownPath: path.join(reportDir, "plugin-inspector-ci-summary.md"),
|
||||
});
|
||||
await writeCiOutputArtifacts(report, {
|
||||
...ciOutputs,
|
||||
cwd: reportDir,
|
||||
outDir: ".",
|
||||
});
|
||||
|
||||
if (json) {
|
||||
console.log(JSON.stringify(summary, null, 2));
|
||||
@ -140,7 +204,7 @@ async function runCi(commandArgs) {
|
||||
}
|
||||
}
|
||||
|
||||
async function runCiCompatibilityReport({ configPath, openclawPath, outDir, pluginRoot }) {
|
||||
async function runCiCompatibilityReport({ allowExecution, capture, configPath, mockSdk, openclawPath, outDir, pluginRoot }) {
|
||||
if (configPath) {
|
||||
const config = await loadInspectorConfig(configPath, { cwd: pluginRoot });
|
||||
const report = await inspectCompatibilityFixtureSet(config, { openclawPath });
|
||||
@ -151,7 +215,7 @@ async function runCiCompatibilityReport({ configPath, openclawPath, outDir, plug
|
||||
};
|
||||
}
|
||||
|
||||
const { report } = await runPluginCheck({ pluginRoot, outDir, openclawPath });
|
||||
const { report } = await runPluginCheck({ allowExecution, capture, mockSdk, openclawPath, outDir, pluginRoot });
|
||||
return {
|
||||
report,
|
||||
reportDir: path.resolve(pluginRoot ?? process.cwd(), outDir),
|
||||
@ -163,11 +227,12 @@ async function runCapture(commandArgs) {
|
||||
const outputPath = readFlag(commandArgs, "--output");
|
||||
const pluginRoot = readFlag(commandArgs, "--plugin-root");
|
||||
const mockSdk = readMockSdkFlag(commandArgs) ?? commandArgs.includes("--mock-sdk");
|
||||
const allowExecution = readAllowExecutionFlag(commandArgs);
|
||||
if (!entrypoint) {
|
||||
throw new Error("capture requires an entrypoint path");
|
||||
}
|
||||
if (process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
|
||||
throw new Error("capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 in an isolated workspace");
|
||||
if (!allowExecution && process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED !== "1") {
|
||||
throw new Error("capture imports plugin code; rerun with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 or --allow-execute in an isolated workspace");
|
||||
}
|
||||
|
||||
const result = await captureEntrypoint(entrypoint, { mockSdk, pluginRoot });
|
||||
@ -187,6 +252,26 @@ function readFlag(commandArgs, name) {
|
||||
return commandArgs[index + 1] ?? null;
|
||||
}
|
||||
|
||||
function readOptionalPathFlag(commandArgs, name, defaultPath) {
|
||||
const index = commandArgs.indexOf(name);
|
||||
if (index === -1) {
|
||||
return null;
|
||||
}
|
||||
const value = commandArgs[index + 1];
|
||||
return value && !value.startsWith("-") ? value : defaultPath;
|
||||
}
|
||||
|
||||
function readCiOutputFlags(commandArgs, options = {}) {
|
||||
return {
|
||||
sarifPath: commandArgs.includes("--no-sarif")
|
||||
? null
|
||||
: (readOptionalPathFlag(commandArgs, "--sarif", defaultSarifPath) ?? (options.defaultEnabled ? defaultSarifPath : null)),
|
||||
junitPath: commandArgs.includes("--no-junit")
|
||||
? null
|
||||
: (readOptionalPathFlag(commandArgs, "--junit", defaultJunitPath) ?? (options.defaultEnabled ? defaultJunitPath : null)),
|
||||
};
|
||||
}
|
||||
|
||||
function readRuntimeFlag(commandArgs) {
|
||||
if (commandArgs.includes("--runtime") || commandArgs.includes("--capture")) {
|
||||
return true;
|
||||
@ -217,6 +302,10 @@ function readMockSdkFlag(commandArgs) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function readAllowExecutionFlag(commandArgs) {
|
||||
return commandArgs.includes("--allow-execute");
|
||||
}
|
||||
|
||||
function renderCiTextSummary(summary) {
|
||||
return [
|
||||
`Status: ${summary.status.toUpperCase()}`,
|
||||
@ -226,19 +315,43 @@ function renderCiTextSummary(summary) {
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function initCommandSummary(result) {
|
||||
return {
|
||||
dryRun: result.dryRun,
|
||||
packageManager: result.packageManager,
|
||||
pluginRoot: result.pluginRoot,
|
||||
files: result.written.map((filePath) => path.relative(result.pluginRoot, filePath)),
|
||||
};
|
||||
}
|
||||
|
||||
function renderConfigTextSummary(config) {
|
||||
const fixture = config.fixtures[0];
|
||||
return [
|
||||
`Plugin: ${fixture.id}`,
|
||||
`Root: ${config.rootDir}`,
|
||||
`Config: ${config.configPath ?? "auto"}`,
|
||||
`Priority: ${fixture.priority}`,
|
||||
`Seams: ${fixture.seams.join(", ")}`,
|
||||
`Runtime capture: ${config.capture?.runtime === true ? "on" : "off"}`,
|
||||
`Mock SDK: ${config.capture?.mockSdk === false ? "off" : "on"}`,
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log(`plugin-inspector
|
||||
|
||||
Usage:
|
||||
plugin-inspector
|
||||
plugin-inspector check [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--runtime] [--mock-sdk|--real-sdk] [--json]
|
||||
plugin-inspector init [--plugin-root <path>] [--config <path>] [--ci] [--package-manager npm|pnpm|yarn|bun] [--force]
|
||||
plugin-inspector check [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--runtime] [--mock-sdk|--real-sdk] [--allow-execute] [--json]
|
||||
plugin-inspector config [--plugin-root <path>] [--config <path>] [--json]
|
||||
plugin-inspector init [--plugin-root <path>] [--config <path>] [--ci] [--scripts] [--package-manager npm|pnpm|yarn|bun] [--dry-run] [--json] [--force]
|
||||
plugin-inspector report --config <path> [--out <dir>] [--check] [--json]
|
||||
plugin-inspector inspect --config <path> [--out <dir>] [--check] [--json]
|
||||
plugin-inspector ci [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--json]
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 plugin-inspector capture <entrypoint> [--mock-sdk|--real-sdk] [--plugin-root <path>] [--output <path>]
|
||||
plugin-inspector inspect [--plugin-root <path>] [--config <path>] [--out <dir>] [--check] [--json] [--sarif [path]] [--junit [path]] [--allow-execute]
|
||||
plugin-inspector ci [--plugin-root <path>] [--config <path>] [--out <dir>] [--openclaw <path>] [--no-openclaw] [--runtime] [--mock-sdk|--real-sdk] [--allow-execute] [--json] [--no-sarif] [--no-junit]
|
||||
plugin-inspector capture <entrypoint> [--mock-sdk|--real-sdk] [--allow-execute] [--plugin-root <path>] [--output <path>]
|
||||
|
||||
Default check runs from the current plugin root and writes reports/ unless --out is set.
|
||||
Runtime capture is opt-in because it imports plugin code; use --runtime with PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1.
|
||||
CI writes SARIF and JUnit artifacts by default; check/inspect can write them with --sarif and --junit.
|
||||
Runtime capture is opt-in because it imports plugin code; use --runtime with --allow-execute or PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1.
|
||||
`);
|
||||
}
|
||||
|
||||
@ -3,6 +3,8 @@ import path from "node:path";
|
||||
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
||||
import { slugForArtifact } from "./path-utils.js";
|
||||
|
||||
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
|
||||
|
||||
export function buildColdImportReadiness(options = {}) {
|
||||
const report = options.report;
|
||||
if (!report) {
|
||||
@ -182,7 +184,7 @@ function classifyEntrypointReadiness({ fixture, packageSummary, entrypoint, root
|
||||
...(packageSummary.dependencies ?? []),
|
||||
...(packageSummary.peerDependencies ?? []),
|
||||
...(packageSummary.optionalDependencies ?? []),
|
||||
]);
|
||||
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
|
||||
if (entrypoint.exists && runtimeDependencies.length > 0) {
|
||||
blockers.push({
|
||||
code: "dependency-install-required",
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { renderPaddedMarkdownTable } from "./artifacts.js";
|
||||
import { sanitizeReportArtifact } from "./report-sanitizer.js";
|
||||
|
||||
const defaultSeverityLabels = {
|
||||
P0: "P0",
|
||||
@ -8,6 +9,7 @@ const defaultSeverityLabels = {
|
||||
};
|
||||
|
||||
export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
report = sanitizeReportArtifact(report, options);
|
||||
return [
|
||||
`# ${options.title ?? "OpenClaw Plugin Compatibility Report"}`,
|
||||
"",
|
||||
@ -24,13 +26,20 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
["Warnings", report.summary.warningCount],
|
||||
["Compatibility suggestions", report.summary.suggestionCount],
|
||||
["Issue findings", report.summary.issueCount],
|
||||
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
|
||||
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
|
||||
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
|
||||
["P0 issues", report.summary.p0IssueCount],
|
||||
["P1 issues", report.summary.p1IssueCount],
|
||||
["Open P0 issues", report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
|
||||
["Open P1 issues", report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
|
||||
["Live issues", report.summary.liveIssueCount],
|
||||
["Live P0 issues", report.summary.liveP0IssueCount],
|
||||
["Compat gaps", report.summary.compatGapCount],
|
||||
["Deprecation warnings", report.summary.deprecationWarningCount],
|
||||
["Inspector gaps", report.summary.inspectorGapCount],
|
||||
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
|
||||
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
|
||||
["Upstream metadata", report.summary.upstreamIssueCount],
|
||||
["Contract probes", report.summary.contractProbeCount],
|
||||
["Decision rows", report.summary.decisionCount],
|
||||
@ -49,9 +58,9 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
options,
|
||||
),
|
||||
"",
|
||||
"## Live Issues",
|
||||
"## Other Live Issues",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
|
||||
"",
|
||||
"## Compat Gaps",
|
||||
"",
|
||||
@ -63,7 +72,11 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
"",
|
||||
"## Inspector Proof Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
|
||||
"",
|
||||
"## Runtime-Covered Inspector Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
|
||||
"",
|
||||
"## Upstream Metadata Issues",
|
||||
"",
|
||||
@ -127,6 +140,7 @@ export function renderCompatibilityMarkdownReport(report, options = {}) {
|
||||
}
|
||||
|
||||
export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
report = sanitizeReportArtifact(report, options);
|
||||
return [
|
||||
`# ${options.title ?? "OpenClaw Plugin Issue Findings"}`,
|
||||
"",
|
||||
@ -138,13 +152,20 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
markdownTable(
|
||||
[
|
||||
["Issue findings", report.summary.issueCount],
|
||||
["Open issue findings", report.summary.openIssueCount ?? report.summary.issueCount],
|
||||
["Runtime-covered findings", report.summary.runtimeCoveredIssueCount ?? 0],
|
||||
["Runtime-partial findings", report.summary.runtimePartiallyCoveredIssueCount ?? 0],
|
||||
[severityLabel("P0", options), report.summary.p0IssueCount],
|
||||
[severityLabel("P1", options), report.summary.p1IssueCount],
|
||||
[`Open ${severityLabel("P0", options)}`, report.summary.openP0IssueCount ?? report.summary.p0IssueCount],
|
||||
[`Open ${severityLabel("P1", options)}`, report.summary.openP1IssueCount ?? report.summary.p1IssueCount],
|
||||
["Live issues", report.summary.liveIssueCount],
|
||||
["Live P0 issues", report.summary.liveP0IssueCount],
|
||||
["Compat gaps", report.summary.compatGapCount],
|
||||
["Deprecation warnings", report.summary.deprecationWarningCount],
|
||||
["Inspector gaps", report.summary.inspectorGapCount],
|
||||
["Open inspector gaps", report.summary.openInspectorGapCount ?? report.summary.inspectorGapCount],
|
||||
["Runtime coverage artifacts", report.summary.runtimeCoverageArtifactCount ?? 0],
|
||||
["Upstream metadata", report.summary.upstreamIssueCount],
|
||||
["Contract probes", report.summary.contractProbeCount],
|
||||
],
|
||||
@ -162,9 +183,9 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
options,
|
||||
),
|
||||
"",
|
||||
"## Live Issues",
|
||||
"## Other Live Issues",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity !== "P0"), options),
|
||||
"",
|
||||
"## Compat Gaps",
|
||||
"",
|
||||
@ -176,7 +197,11 @@ export function renderCompatibilityIssuesReport(report, options = {}) {
|
||||
"",
|
||||
"## Inspector Proof Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap"), options),
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status !== "runtime-covered"), options),
|
||||
"",
|
||||
"## Runtime-Covered Inspector Gaps",
|
||||
"",
|
||||
issuesTable(report.issues.filter((issue) => issue.issueClass === "inspector-gap" && issue.status === "runtime-covered"), options),
|
||||
"",
|
||||
"## Upstream Metadata Issues",
|
||||
"",
|
||||
@ -225,6 +250,7 @@ function issueBlock(issue, options) {
|
||||
` - state: ${issueState(issue)}`,
|
||||
" - evidence:",
|
||||
...evidenceList(issue.evidence, options).map((item) => ` - ${item}`),
|
||||
...runtimeCoverageList(issue, options),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
@ -232,6 +258,7 @@ function issueState(issue) {
|
||||
const flags = [
|
||||
issue.status,
|
||||
`compat:${issue.compatStatus ?? "none"}`,
|
||||
issue.runtimeCoverage?.status ? `runtime:${issue.runtimeCoverage.status}` : null,
|
||||
issue.live ? "live" : null,
|
||||
issue.deprecated ? "deprecated" : null,
|
||||
].filter(Boolean);
|
||||
@ -263,7 +290,7 @@ function triageOverview(report) {
|
||||
"inspector-gap",
|
||||
report.summary.inspectorGapCount,
|
||||
"-",
|
||||
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments.",
|
||||
"Plugin Inspector needs stronger capture/probe evidence before making contract judgments. Runtime-covered rows are proof-backed and not open report work.",
|
||||
],
|
||||
[
|
||||
"upstream-metadata",
|
||||
@ -354,3 +381,15 @@ function evidenceList(evidence, options) {
|
||||
const formatEvidence = options.formatEvidence ?? ((item) => item);
|
||||
return items.map((item) => formatEvidence(item));
|
||||
}
|
||||
|
||||
function runtimeCoverageList(issue, options) {
|
||||
const runtimeCoverage = issue.runtimeCoverage;
|
||||
if (!runtimeCoverage) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
" - runtime coverage:",
|
||||
...evidenceList(runtimeCoverage.captured, options).map((item) => ` - captured ${item}`),
|
||||
...evidenceList(runtimeCoverage.artifacts, options).map((item) => ` - ${item}`),
|
||||
];
|
||||
}
|
||||
|
||||
@ -4,6 +4,7 @@ import path from "node:path";
|
||||
|
||||
export const npmPackagePayloadDir = ".crabpot-package";
|
||||
export const defaultPluginRootConfigFiles = ["plugin-inspector.config.json", ".plugin-inspector.json"];
|
||||
export const packageJsonConfigKeys = ["pluginInspector", "plugin-inspector"];
|
||||
|
||||
export async function loadInspectorConfig(configPath, options = {}) {
|
||||
if (!configPath) {
|
||||
@ -24,16 +25,23 @@ export async function loadInspectorConfig(configPath, options = {}) {
|
||||
export async function loadPluginRootConfig(configPath = null, options = {}) {
|
||||
const rootDir = path.resolve(options.cwd ?? process.cwd());
|
||||
const resolvedPath = configPath ? path.resolve(rootDir, configPath) : findPluginRootConfigPath(rootDir);
|
||||
if (!resolvedPath && !existsSync(path.join(rootDir, "package.json")) && !existsSync(path.join(rootDir, "openclaw.plugin.json"))) {
|
||||
const packageJsonPath = path.join(rootDir, "package.json");
|
||||
const packageJson = await readJsonIfExists(packageJsonPath);
|
||||
const packageConfig = packageJsonConfig(packageJson);
|
||||
|
||||
if (!resolvedPath && !packageJson && !existsSync(path.join(rootDir, "openclaw.plugin.json"))) {
|
||||
throw new Error("run from a plugin root with package.json/openclaw.plugin.json, or pass --config");
|
||||
}
|
||||
const config = resolvedPath ? JSON.parse(await readFile(resolvedPath, "utf8")) : { version: 1 };
|
||||
|
||||
const config = resolvedPath
|
||||
? JSON.parse(await readFile(resolvedPath, "utf8"))
|
||||
: (packageConfig.config ?? { version: 1 });
|
||||
const normalizedConfig = await normalizePluginRootConfig(config, { rootDir });
|
||||
validateInspectorConfig(normalizedConfig);
|
||||
return {
|
||||
...normalizedConfig,
|
||||
rootDir,
|
||||
configPath: resolvedPath,
|
||||
configPath: resolvedPath ?? packageConfig.configPath,
|
||||
};
|
||||
}
|
||||
|
||||
@ -164,6 +172,25 @@ function findPluginRootConfigPath(rootDir) {
|
||||
return defaultPluginRootConfigFiles.map((file) => path.join(rootDir, file)).find(existsSync) ?? null;
|
||||
}
|
||||
|
||||
function packageJsonConfig(packageJson) {
|
||||
if (!packageJson) {
|
||||
return { config: null, configPath: null };
|
||||
}
|
||||
for (const key of packageJsonConfigKeys) {
|
||||
if (packageJson[key] === undefined) {
|
||||
continue;
|
||||
}
|
||||
if (!packageJson[key] || typeof packageJson[key] !== "object" || Array.isArray(packageJson[key])) {
|
||||
throw new Error(`package.json ${key} must be an object`);
|
||||
}
|
||||
return {
|
||||
config: packageJson[key],
|
||||
configPath: `package.json#${key}`,
|
||||
};
|
||||
}
|
||||
return { config: null, configPath: null };
|
||||
}
|
||||
|
||||
async function readJsonIfExists(filePath) {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
@ -175,13 +202,43 @@ export function packageId(packageName) {
|
||||
if (!packageName) {
|
||||
return null;
|
||||
}
|
||||
return packageName
|
||||
const packageBase = packageName
|
||||
.split("/")
|
||||
.pop()
|
||||
.replace(/^openclaw-/, "")
|
||||
.replace(/[^a-zA-Z0-9]+/g, "-")
|
||||
.replace(/^-+|-+$/g, "")
|
||||
.toLowerCase();
|
||||
.replace(/^openclaw-/, "");
|
||||
return trimHyphenEdges(collapsePackageIdSeparators(packageBase)).toLowerCase();
|
||||
}
|
||||
|
||||
function collapsePackageIdSeparators(value) {
|
||||
let result = "";
|
||||
let previousWasHyphen = false;
|
||||
for (const char of value) {
|
||||
if (isAsciiAlphaNumeric(char)) {
|
||||
result += char;
|
||||
previousWasHyphen = false;
|
||||
} else if (!previousWasHyphen) {
|
||||
result += "-";
|
||||
previousWasHyphen = true;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isAsciiAlphaNumeric(char) {
|
||||
const code = char.charCodeAt(0);
|
||||
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
|
||||
}
|
||||
|
||||
function trimHyphenEdges(value) {
|
||||
let start = 0;
|
||||
let end = value.length;
|
||||
while (start < end && value[start] === "-") {
|
||||
start += 1;
|
||||
}
|
||||
while (end > start && value[end - 1] === "-") {
|
||||
end -= 1;
|
||||
}
|
||||
return value.slice(start, end);
|
||||
}
|
||||
|
||||
export function inferPluginSeams(pluginManifest, packageJson) {
|
||||
|
||||
@ -7,6 +7,7 @@ import {
|
||||
} from "./synthetic-probes.js";
|
||||
|
||||
export const defaultRegistrationAssertions = {
|
||||
createChatChannelPlugin: ["channel plugin id is stable", "channel factory metadata is captured"],
|
||||
defineChannelPluginEntry: ["channel id is stable", "setup/config schema can be read", "message envelope metadata is preserved"],
|
||||
definePluginEntry: ["entrypoint register function is callable", "entrypoint metadata is preserved"],
|
||||
registerChannel: ["channel id is stable", "inbound/outbound envelope shape is captured", "sender metadata is preserved"],
|
||||
|
||||
@ -157,10 +157,15 @@ function requireCompatRecordReconciliation(report, errors) {
|
||||
.filter((finding) => finding.code === "missing-compat-record")
|
||||
.map((finding) => `${finding.fixture}:${finding.compatRecord}`),
|
||||
);
|
||||
const compatGapRecords = new Set(
|
||||
report.issues
|
||||
.filter((issue) => issue.issueClass === "compat-gap" && issue.compatRecord)
|
||||
.map((issue) => `${issue.fixture}:${issue.compatRecord}`),
|
||||
);
|
||||
|
||||
for (const finding of [...report.warnings, ...report.suggestions].filter((item) => item.compatRecord)) {
|
||||
const key = `${finding.fixture}:${finding.compatRecord}`;
|
||||
if (!presentRecords.has(key) && !missingRecords.has(key)) {
|
||||
if (!presentRecords.has(key) && !missingRecords.has(key) && !compatGapRecords.has(key)) {
|
||||
errors.push(`${finding.fixture}: compat record ${finding.compatRecord} was not reconciled`);
|
||||
}
|
||||
}
|
||||
|
||||
@ -74,6 +74,31 @@ export const contractProbeRules = {
|
||||
contract: "OpenClaw package entrypoints resolve to files in the published or built plugin package.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-install-metadata-incomplete": {
|
||||
id: "package.metadata.install-release",
|
||||
contract: "Release publishing metadata declares canonical ClawHub and npm install specs.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-min-host-version-drift": {
|
||||
id: "package.metadata.min-host-version",
|
||||
contract: "Install minimum host version matches the OpenClaw package surface targeted by the plugin.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-npm-pack-entrypoint-missing": {
|
||||
id: "package.npm-pack.entrypoints",
|
||||
contract: "Advertised npm artifacts include every declared OpenClaw package entrypoint.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-npm-pack-metadata-missing": {
|
||||
id: "package.npm-pack.metadata",
|
||||
contract: "Advertised npm artifacts include OpenClaw manifest and package metadata.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-npm-pack-unavailable": {
|
||||
id: "package.npm-pack.available",
|
||||
contract: "Packages that advertise npm install support can produce an npm pack artifact.",
|
||||
target: "package-loader",
|
||||
},
|
||||
"package-openclaw-entry-missing": {
|
||||
id: "package.entrypoint.openclaw-metadata",
|
||||
contract: "OpenClaw package metadata declares entrypoints for cold import and registration capture.",
|
||||
@ -106,6 +131,20 @@ export const contractProbeRules = {
|
||||
},
|
||||
};
|
||||
|
||||
const openClawOwnedProbeIssueCodes = new Set([
|
||||
"before-tool-call-probe",
|
||||
"channel-contract-probe",
|
||||
"conversation-access-hook",
|
||||
"registration-capture-gap",
|
||||
]);
|
||||
|
||||
export function compatRecordForIssueCode(code) {
|
||||
if (!openClawOwnedProbeIssueCodes.has(code)) {
|
||||
return undefined;
|
||||
}
|
||||
return contractProbeRules[code]?.id;
|
||||
}
|
||||
|
||||
export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
|
||||
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
|
||||
const probes = [];
|
||||
@ -136,7 +175,6 @@ export function probePriority(code, fixturePriority) {
|
||||
"before-tool-call-probe",
|
||||
"conversation-access-hook",
|
||||
"missing-compat-record",
|
||||
"registration-capture-gap",
|
||||
"sdk-export-missing",
|
||||
].includes(code)
|
||||
) {
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { compatRecordForIssueCode } from "./contract-probes.js";
|
||||
import { readJsonFile } from "./json-file.js";
|
||||
|
||||
const conversationAccessHooks = new Set(["agent_end", "llm_input", "llm_output"]);
|
||||
@ -17,9 +18,13 @@ const channelRegistrations = new Set([
|
||||
"defineChannelPluginEntry",
|
||||
"registerChannel",
|
||||
]);
|
||||
const hostLinkedRuntimeDependencies = new Set(["openclaw"]);
|
||||
const unsupportedSecurityManifestName = "openclaw.security.json";
|
||||
const unavailableSecurityManifestSchema = "https://openclaw.ai/schemas/plugin-security.json";
|
||||
|
||||
export async function buildCompatibilityFixtureReport({ fixture, inspection, checkoutPath, sourceRoot, rootDir = process.cwd() }) {
|
||||
const pluginManifests = await readPluginManifests({ checkoutPath, sourceRoot, rootDir });
|
||||
const securityManifests = await readSecurityManifests({ checkoutPath, sourceRoot, rootDir });
|
||||
const packageSummaries = await readPackageSummaries({ checkoutPath, sourceRoot, rootDir });
|
||||
const packageJson = selectPrimaryPackage(packageSummaries);
|
||||
const sdkImports = unique((inspection.sdkImports ?? []).map((sdkImport) => sdkImport.specifier));
|
||||
@ -39,6 +44,7 @@ export async function buildCompatibilityFixtureReport({ fixture, inspection, che
|
||||
manifestFiles: inspection.manifestFiles ?? [],
|
||||
sourceFiles: inspection.sourceFiles ?? [],
|
||||
pluginManifests,
|
||||
securityManifests,
|
||||
package: packageJson,
|
||||
packages: packageSummaries,
|
||||
sdkImports,
|
||||
@ -72,6 +78,46 @@ export async function readPluginManifests({ checkoutPath, sourceRoot, rootDir =
|
||||
return manifests;
|
||||
}
|
||||
|
||||
export async function readSecurityManifests({ checkoutPath, sourceRoot, rootDir = process.cwd() }) {
|
||||
const candidates = unique(
|
||||
[
|
||||
path.join(sourceRoot, unsupportedSecurityManifestName),
|
||||
path.join(checkoutPath, unsupportedSecurityManifestName),
|
||||
].filter(existsSync),
|
||||
);
|
||||
const manifests = [];
|
||||
|
||||
for (const manifestPath of candidates) {
|
||||
const relativePath = path.relative(rootDir, manifestPath);
|
||||
try {
|
||||
const manifest = await readJsonFile(manifestPath);
|
||||
manifests.push({
|
||||
path: relativePath,
|
||||
schema: typeof manifest.$schema === "string" ? manifest.$schema : null,
|
||||
version: typeof manifest.version === "string" ? manifest.version : null,
|
||||
plugin: typeof manifest.plugin === "string" ? manifest.plugin : null,
|
||||
expectedBehaviorCount: Array.isArray(manifest.expectedBehaviors)
|
||||
? manifest.expectedBehaviors.length
|
||||
: 0,
|
||||
securityNoteCount: Array.isArray(manifest.securityNotes) ? manifest.securityNotes.length : 0,
|
||||
validJson: true,
|
||||
});
|
||||
} catch {
|
||||
manifests.push({
|
||||
path: relativePath,
|
||||
schema: null,
|
||||
version: null,
|
||||
plugin: null,
|
||||
expectedBehaviorCount: 0,
|
||||
securityNoteCount: 0,
|
||||
validJson: false,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return manifests;
|
||||
}
|
||||
|
||||
export async function readPackageSummaries({ checkoutPath, sourceRoot, rootDir = process.cwd(), maxDepth = 3 }) {
|
||||
const candidates = unique([
|
||||
path.join(sourceRoot, "package.json"),
|
||||
@ -106,6 +152,9 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
|
||||
typeof packageJson.openclaw.build?.pluginSdkVersion === "string"
|
||||
? packageJson.openclaw.build.pluginSdkVersion
|
||||
: null,
|
||||
install: summarizeOpenClawInstall(packageJson.openclaw.install),
|
||||
release: summarizeOpenClawRelease(packageJson.openclaw.release),
|
||||
unsupportedMetadata: unsupportedOpenClawPackageMetadata(packageJson.openclaw),
|
||||
}
|
||||
: null;
|
||||
|
||||
@ -119,6 +168,7 @@ export function summarizePackage(packagePath, packageJson, options = {}) {
|
||||
version: packageJson.version ?? null,
|
||||
type: packageJson.type ?? null,
|
||||
main: typeof packageJson.main === "string" ? packageJson.main : null,
|
||||
npmPack: summarizeNpmPack(packageJson, openclaw),
|
||||
dependencies: Object.keys(packageJson.dependencies ?? {}).sort(),
|
||||
peerDependencies: Object.keys(packageJson.peerDependencies ?? {}).sort(),
|
||||
optionalDependencies: Object.keys(packageJson.optionalDependencies ?? {}).sort(),
|
||||
@ -200,6 +250,79 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
|
||||
});
|
||||
}
|
||||
|
||||
if ((packageSummary.openclaw?.unsupportedMetadata ?? []).length > 0) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "package-openclaw-unsupported-metadata",
|
||||
level: "warning",
|
||||
message: "package declares unsupported OpenClaw metadata",
|
||||
evidence: packageSummary.openclaw.unsupportedMetadata,
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-metadata",
|
||||
action: "Remove unsupported OpenClaw metadata; native plugins use openclaw.plugin.json plus supported package openclaw fields.",
|
||||
evidence: packageSummary.openclaw.unsupportedMetadata.join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
const installMetadataIssues = packageInstallMetadataIssues(packageSummary);
|
||||
if (installMetadataIssues.length > 0) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "package-install-metadata-incomplete",
|
||||
level: "warning",
|
||||
message: "package OpenClaw install metadata does not match advertised release targets",
|
||||
evidence: installMetadataIssues,
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-metadata",
|
||||
action: "Ask the plugin to align openclaw.install metadata with openclaw.release publishing targets.",
|
||||
evidence: installMetadataIssues.join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
if (packageMinHostVersionDrift(packageSummary)) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "package-min-host-version-drift",
|
||||
level: "warning",
|
||||
message: "package openclaw.install.minHostVersion is not a semver floor for the target OpenClaw build version",
|
||||
evidence: [
|
||||
`minHostVersion:${packageSummary.openclaw.install.minHostVersion}`,
|
||||
`buildOpenClawVersion:${packageSummary.openclaw.buildOpenClawVersion}`,
|
||||
],
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-metadata",
|
||||
action: "Ask the plugin to publish install.minHostVersion as a semver floor for the OpenClaw package surface it targets.",
|
||||
evidence: packageSummary.path,
|
||||
});
|
||||
}
|
||||
|
||||
const npmPackIssues = packageNpmPackIssues(packageSummary, fixtureReport);
|
||||
for (const finding of npmPackIssues) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: finding.code,
|
||||
level: "warning",
|
||||
message: finding.message,
|
||||
evidence: finding.evidence,
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "package-artifact",
|
||||
action: "Ask the plugin to make its advertised npm install artifact match the published OpenClaw metadata.",
|
||||
evidence: finding.evidence.join(", "),
|
||||
});
|
||||
}
|
||||
|
||||
if (packageSummary.openclaw && packageSummary.openclaw.entrypoints.length === 0) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
@ -217,9 +340,12 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
|
||||
});
|
||||
}
|
||||
|
||||
const missingEntrypoints = packageSummary.openclaw?.entrypoints.filter((entrypoint) => !entrypoint.exists) ?? [];
|
||||
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
|
||||
const missingEntrypoints = entrypoints.filter((entrypoint) => !entrypoint.exists);
|
||||
const buildEntrypoints = missingEntrypoints.filter((entrypoint) => entrypoint.requiresBuild);
|
||||
const plainMissingEntrypoints = missingEntrypoints.filter((entrypoint) => !entrypoint.requiresBuild);
|
||||
const plainMissingEntrypoints = missingEntrypoints.filter(
|
||||
(entrypoint) => !entrypoint.requiresBuild && !hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
|
||||
);
|
||||
|
||||
if (buildEntrypoints.length > 0) {
|
||||
suggestions.push({
|
||||
@ -278,7 +404,7 @@ export function classifyPackageContracts({ fixture, inspection, fixtureReport })
|
||||
...packageSummary.dependencies,
|
||||
...packageSummary.peerDependencies,
|
||||
...packageSummary.optionalDependencies,
|
||||
]);
|
||||
]).filter((dependency) => !hostLinkedRuntimeDependencies.has(dependency));
|
||||
if (packageSummary.openclaw?.entrypoints.length > 0 && runtimeDependencies.length > 0) {
|
||||
suggestions.push({
|
||||
fixture: fixture.id,
|
||||
@ -349,6 +475,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
suggestions.push(...packageContracts.suggestions);
|
||||
logs.push(...packageContracts.logs);
|
||||
decisions.push(...packageContracts.decisions);
|
||||
classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions });
|
||||
|
||||
for (const pluginManifest of fixtureReport.pluginManifests) {
|
||||
const providerAuthKeys = Object.keys(pluginManifest.providerAuthEnvVars ?? {});
|
||||
@ -399,6 +526,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "warning",
|
||||
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
|
||||
evidence: detailEvidence(conversationHookDetails),
|
||||
compatRecord: compatRecordForIssueCode("conversation-access-hook"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -459,6 +587,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
|
||||
evidence: detailEvidence(captureGapRegistrationDetails),
|
||||
compatRecord: compatRecordForIssueCode("registration-capture-gap"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -477,6 +606,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
|
||||
evidence: detailEvidence(hookDetails),
|
||||
compatRecord: compatRecordForIssueCode("before-tool-call-probe"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -500,6 +630,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "add channel envelope, config-schema, and runtime metadata probes",
|
||||
evidence: detailEvidence(channelRegistrationDetails),
|
||||
compatRecord: compatRecordForIssueCode("channel-contract-probe"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -551,6 +682,45 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
return { warnings, suggestions, logs, decisions };
|
||||
}
|
||||
|
||||
function classifySecurityManifestCoverage({ fixture, fixtureReport, warnings, decisions }) {
|
||||
for (const securityManifest of fixtureReport.securityManifests ?? []) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "unrecognized-security-manifest",
|
||||
level: "warning",
|
||||
message:
|
||||
"openclaw.security.json is not a supported OpenClaw or ClawHub security contract and is ignored by install safety checks",
|
||||
evidence: [securityManifest.path],
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "security-metadata",
|
||||
action:
|
||||
"Remove the advisory security manifest or replace it with a supported, versioned OpenClaw/ClawHub security contract once one exists.",
|
||||
evidence: securityManifest.path,
|
||||
});
|
||||
|
||||
if (securityManifest.schema === unavailableSecurityManifestSchema) {
|
||||
warnings.push({
|
||||
fixture: fixture.id,
|
||||
code: "security-manifest-schema-unavailable",
|
||||
level: "warning",
|
||||
message: "openclaw.security.json references an OpenClaw schema URL that is not currently published",
|
||||
evidence: [`${securityManifest.path}:$schema=${securityManifest.schema}`],
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
decision: "plugin-upstream-fix",
|
||||
seam: "security-metadata",
|
||||
action:
|
||||
"Do not rely on the schema URL until OpenClaw publishes and documents a real plugin security metadata schema.",
|
||||
evidence: securityManifest.schema,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function registrationCaptureGapDetails(inspection, targetOpenClaw) {
|
||||
const apiRegistrationDetails = inspection.registrationDetails.filter((registration) =>
|
||||
registration.name.startsWith("register"),
|
||||
@ -753,6 +923,46 @@ function collectOpenClawEntrypoints(packageDir, openclaw, options) {
|
||||
});
|
||||
}
|
||||
|
||||
function hasUsablePackageRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
|
||||
if (!isSourceEntrypoint(entrypoint.specifier)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
|
||||
if (
|
||||
entrypoints.some(
|
||||
(candidate) =>
|
||||
candidate.exists &&
|
||||
candidate.requiresBuild &&
|
||||
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (entrypoint.kind === "extension" && entrypoints.some((candidate) => candidate.kind === "runtimeExtension" && candidate.exists)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const packageDir = path.dirname(packageSummary.path);
|
||||
return existsSync(path.resolve(packageDir, runtimeBuildSpecifier));
|
||||
}
|
||||
|
||||
function isSourceEntrypoint(specifier) {
|
||||
return /\.(?:ts|tsx)$/.test(specifier);
|
||||
}
|
||||
|
||||
function runtimeBuildSpecifierFor(specifier) {
|
||||
const normalized = normalizeEntrypointSpecifier(specifier);
|
||||
const basename = path.posix.basename(normalized).replace(/\.(?:ts|tsx)$/, ".js");
|
||||
return `./dist/${basename}`;
|
||||
}
|
||||
|
||||
function normalizeEntrypointSpecifier(specifier) {
|
||||
const normalized = specifier.replaceAll("\\", "/");
|
||||
return normalized.startsWith("./") ? normalized : `./${normalized}`;
|
||||
}
|
||||
|
||||
async function findPackageFiles(root, options, depth = 0) {
|
||||
if (!existsSync(root) || depth > options.maxDepth) {
|
||||
return [];
|
||||
@ -781,6 +991,269 @@ function selectPrimaryPackage(packages) {
|
||||
return packages[0] ?? null;
|
||||
}
|
||||
|
||||
function summarizeOpenClawInstall(install) {
|
||||
if (!install || typeof install !== "object") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
clawhubSpec: stringOrNull(install.clawhubSpec),
|
||||
npmSpec: stringOrNull(install.npmSpec),
|
||||
defaultChoice: stringOrNull(install.defaultChoice),
|
||||
minHostVersion: stringOrNull(install.minHostVersion),
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeOpenClawRelease(release) {
|
||||
if (!release || typeof release !== "object") {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
publishToClawHub: booleanOrNull(release.publishToClawHub),
|
||||
publishToNpm: booleanOrNull(release.publishToNpm),
|
||||
};
|
||||
}
|
||||
|
||||
function unsupportedOpenClawPackageMetadata(openclaw) {
|
||||
if (!openclaw || typeof openclaw !== "object") {
|
||||
return [];
|
||||
}
|
||||
return Object.keys(openclaw)
|
||||
.filter((key) => key === "bundle")
|
||||
.map((key) => `openclaw.${key}`);
|
||||
}
|
||||
|
||||
function summarizeNpmPack(packageJson, openclaw) {
|
||||
const files = arrayValues(packageJson.files).map(normalizePackagePath).filter((item) => item.length > 0);
|
||||
return {
|
||||
advertised: openclaw?.release?.publishToNpm === true || nonEmptyString(openclaw?.install?.npmSpec),
|
||||
private: packageJson.private === true,
|
||||
filesMode: files.length > 0 ? "allowlist" : "implicit",
|
||||
files,
|
||||
invalidFileSpecs: files.filter((item) => invalidPackageFileSpec(item)),
|
||||
};
|
||||
}
|
||||
|
||||
function packageInstallMetadataIssues(packageSummary) {
|
||||
const openclaw = packageSummary.openclaw;
|
||||
if (!openclaw) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const issues = [];
|
||||
const install = openclaw.install;
|
||||
const release = openclaw.release;
|
||||
const publishToClawHub = release?.publishToClawHub === true;
|
||||
const publishToNpm = release?.publishToNpm === true;
|
||||
|
||||
if (publishToClawHub && !nonEmptyString(install?.clawhubSpec)) {
|
||||
issues.push("openclaw.release.publishToClawHub requires openclaw.install.clawhubSpec");
|
||||
}
|
||||
if (publishToNpm && !nonEmptyString(install?.npmSpec)) {
|
||||
issues.push("openclaw.release.publishToNpm requires openclaw.install.npmSpec");
|
||||
}
|
||||
if (publishToNpm && nonEmptyString(install?.npmSpec) && nonEmptyString(packageSummary.name) && install.npmSpec !== packageSummary.name) {
|
||||
issues.push(`openclaw.install.npmSpec:${install.npmSpec} does not match package name:${packageSummary.name}`);
|
||||
}
|
||||
if (nonEmptyString(install?.defaultChoice) && !["clawhub", "npm"].includes(install.defaultChoice)) {
|
||||
issues.push(`openclaw.install.defaultChoice:${install.defaultChoice} must be clawhub or npm`);
|
||||
}
|
||||
if (install?.defaultChoice === "clawhub" && !nonEmptyString(install.clawhubSpec)) {
|
||||
issues.push("openclaw.install.defaultChoice clawhub requires openclaw.install.clawhubSpec");
|
||||
}
|
||||
if (install?.defaultChoice === "npm" && !nonEmptyString(install.npmSpec)) {
|
||||
issues.push("openclaw.install.defaultChoice npm requires openclaw.install.npmSpec");
|
||||
}
|
||||
|
||||
return issues;
|
||||
}
|
||||
|
||||
function packageNpmPackIssues(packageSummary, fixtureReport) {
|
||||
if (!packageSummary.npmPack?.advertised) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const findings = [];
|
||||
const unavailable = [];
|
||||
if (packageSummary.npmPack.private) {
|
||||
unavailable.push("package.json private:true");
|
||||
}
|
||||
if (!nonEmptyString(packageSummary.name)) {
|
||||
unavailable.push("package.json name missing");
|
||||
}
|
||||
if (!nonEmptyString(packageSummary.version)) {
|
||||
unavailable.push("package.json version missing");
|
||||
}
|
||||
unavailable.push(...packageSummary.npmPack.invalidFileSpecs.map((item) => `invalid files entry:${item}`));
|
||||
|
||||
if (unavailable.length > 0) {
|
||||
findings.push({
|
||||
code: "package-npm-pack-unavailable",
|
||||
message: "package advertises npm install or publish metadata but cannot produce a usable npm pack artifact",
|
||||
evidence: unavailable,
|
||||
});
|
||||
}
|
||||
|
||||
const missingMetadata = packageNpmPackMissingMetadata(packageSummary, fixtureReport);
|
||||
if (missingMetadata.length > 0) {
|
||||
findings.push({
|
||||
code: "package-npm-pack-metadata-missing",
|
||||
message: "advertised npm artifact would not include required OpenClaw package metadata",
|
||||
evidence: missingMetadata,
|
||||
});
|
||||
}
|
||||
|
||||
const entrypoints = packageSummary.openclaw?.entrypoints ?? [];
|
||||
const missingEntrypoints = entrypoints
|
||||
.filter(
|
||||
(entrypoint) =>
|
||||
!repoPathIncludedInNpmPack(packageSummary, entrypoint.relativePath) &&
|
||||
!hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints),
|
||||
)
|
||||
.map((entrypoint) => `${entrypoint.kind}:${entrypoint.specifier} -> ${entrypoint.relativePath}`) ?? [];
|
||||
if (missingEntrypoints.length > 0) {
|
||||
findings.push({
|
||||
code: "package-npm-pack-entrypoint-missing",
|
||||
message: "advertised npm artifact would not include declared OpenClaw entrypoints",
|
||||
evidence: missingEntrypoints,
|
||||
});
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
function packageNpmPackMissingMetadata(packageSummary, fixtureReport) {
|
||||
const missing = [];
|
||||
if (!repoPathIncludedInNpmPack(packageSummary, packageSummary.path)) {
|
||||
missing.push(packageSummary.path);
|
||||
}
|
||||
|
||||
for (const manifest of fixtureReport.pluginManifests ?? []) {
|
||||
if (repoPathWithinPackage(packageSummary, manifest.path) && !repoPathIncludedInNpmPack(packageSummary, manifest.path)) {
|
||||
missing.push(manifest.path);
|
||||
}
|
||||
}
|
||||
|
||||
return missing;
|
||||
}
|
||||
|
||||
function hasPackagedRuntimeEntrypoint(entrypoint, packageSummary, entrypoints) {
|
||||
if (!isSourceEntrypoint(entrypoint.specifier)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const runtimeBuildSpecifier = runtimeBuildSpecifierFor(entrypoint.specifier);
|
||||
const matchingRuntimeEntrypoint = entrypoints.find(
|
||||
(candidate) =>
|
||||
candidate.requiresBuild &&
|
||||
normalizeEntrypointSpecifier(candidate.specifier) === normalizeEntrypointSpecifier(runtimeBuildSpecifier),
|
||||
);
|
||||
if (matchingRuntimeEntrypoint && repoPathIncludedInNpmPack(packageSummary, matchingRuntimeEntrypoint.relativePath)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (
|
||||
entrypoint.kind === "extension" &&
|
||||
entrypoints.some(
|
||||
(candidate) => candidate.kind === "runtimeExtension" && repoPathIncludedInNpmPack(packageSummary, candidate.relativePath),
|
||||
)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
|
||||
const runtimeBuildPath = path.posix.join(packageDir === "." ? "" : packageDir, normalizeEntrypointSpecifier(runtimeBuildSpecifier));
|
||||
return repoPathIncludedInNpmPack(packageSummary, runtimeBuildPath);
|
||||
}
|
||||
|
||||
function packageMinHostVersionDrift(packageSummary) {
|
||||
const openclaw = packageSummary.openclaw;
|
||||
if (!nonEmptyString(openclaw?.install?.minHostVersion) || !nonEmptyString(openclaw?.buildOpenClawVersion)) {
|
||||
return false;
|
||||
}
|
||||
return parseMinHostVersionFloor(openclaw.install.minHostVersion) !== openclaw.buildOpenClawVersion;
|
||||
}
|
||||
|
||||
function parseMinHostVersionFloor(value) {
|
||||
const match = /^>=([0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?)$/.exec(value);
|
||||
return match?.[1] ?? null;
|
||||
}
|
||||
|
||||
function repoPathIncludedInNpmPack(packageSummary, repoPath) {
|
||||
const packageRelativePath = packageRelativeRepoPath(packageSummary, repoPath);
|
||||
if (!packageRelativePath) {
|
||||
return false;
|
||||
}
|
||||
if (npmAlwaysPacksPath(packageRelativePath)) {
|
||||
return true;
|
||||
}
|
||||
if (packageSummary.npmPack?.filesMode !== "allowlist") {
|
||||
return true;
|
||||
}
|
||||
return packageSummary.npmPack.files.some((spec) => packageFileSpecIncludesPath(spec, packageRelativePath));
|
||||
}
|
||||
|
||||
function repoPathWithinPackage(packageSummary, repoPath) {
|
||||
return packageRelativeRepoPath(packageSummary, repoPath) !== null;
|
||||
}
|
||||
|
||||
function packageRelativeRepoPath(packageSummary, repoPath) {
|
||||
const packageDir = path.posix.dirname(normalizeRepoPath(packageSummary.path));
|
||||
const normalized = normalizeRepoPath(repoPath);
|
||||
if (packageDir === ".") {
|
||||
return normalized;
|
||||
}
|
||||
if (normalized === packageDir) {
|
||||
return "";
|
||||
}
|
||||
return normalized.startsWith(`${packageDir}/`) ? normalized.slice(packageDir.length + 1) : null;
|
||||
}
|
||||
|
||||
function npmAlwaysPacksPath(packageRelativePath) {
|
||||
const base = path.posix.basename(packageRelativePath).toLowerCase();
|
||||
return packageRelativePath === "package.json" || /^readme(?:\..*)?$/u.test(base) || /^licen[cs]e(?:\..*)?$/u.test(base);
|
||||
}
|
||||
|
||||
function packageFileSpecIncludesPath(spec, packageRelativePath) {
|
||||
if (spec === "." || spec === packageRelativePath) {
|
||||
return true;
|
||||
}
|
||||
if (spec.includes("*")) {
|
||||
return globLikeSpecIncludesPath(spec, packageRelativePath);
|
||||
}
|
||||
return packageRelativePath.startsWith(`${spec.replace(/\/$/u, "")}/`);
|
||||
}
|
||||
|
||||
function globLikeSpecIncludesPath(spec, packageRelativePath) {
|
||||
return globSegmentsIncludePath(spec.split("/"), packageRelativePath.split("/"));
|
||||
}
|
||||
|
||||
function globSegmentsIncludePath(specSegments, packageSegments) {
|
||||
if (specSegments.length === 0) {
|
||||
return packageSegments.length === 0;
|
||||
}
|
||||
const [head, ...tail] = specSegments;
|
||||
if (head === "**") {
|
||||
return globSegmentsIncludePath(tail, packageSegments) || (packageSegments.length > 0 && globSegmentsIncludePath(specSegments, packageSegments.slice(1)));
|
||||
}
|
||||
if (packageSegments.length === 0) {
|
||||
return false;
|
||||
}
|
||||
return globSegmentIncludesPath(head, packageSegments[0]) && globSegmentsIncludePath(tail, packageSegments.slice(1));
|
||||
}
|
||||
|
||||
function globSegmentIncludesPath(specSegment, packageSegment) {
|
||||
const escaped = specSegment.replace(/[.+?^${}()|[\]\\]/gu, "\\$&").replaceAll("*", ".*");
|
||||
return new RegExp(`^${escaped}$`, "u").test(packageSegment);
|
||||
}
|
||||
|
||||
function invalidPackageFileSpec(spec) {
|
||||
return spec.startsWith("/") || spec === ".." || spec.startsWith("../") || spec.includes("/../");
|
||||
}
|
||||
|
||||
function normalizePackagePath(value) {
|
||||
return normalizeRepoPath(value).replace(/^\.\/+/u, "").replace(/\/$/u, "");
|
||||
}
|
||||
|
||||
function packageRank(packageSummary) {
|
||||
if (packageSummary.openclaw?.entrypoints.length > 0) {
|
||||
return 0;
|
||||
@ -795,6 +1268,22 @@ function arrayValues(value) {
|
||||
return Array.isArray(value) ? value.filter((item) => typeof item === "string") : [];
|
||||
}
|
||||
|
||||
function stringOrNull(value) {
|
||||
return typeof value === "string" ? value : null;
|
||||
}
|
||||
|
||||
function booleanOrNull(value) {
|
||||
return typeof value === "boolean" ? value : null;
|
||||
}
|
||||
|
||||
function nonEmptyString(value) {
|
||||
return typeof value === "string" && value.trim().length > 0;
|
||||
}
|
||||
|
||||
function normalizeRepoPath(value) {
|
||||
return String(value ?? "").replaceAll("\\", "/").replace(/^\.\/+/u, "");
|
||||
}
|
||||
|
||||
function detailEvidence(details, key = "name") {
|
||||
return unique(details.map((detail) => `${detail[key]} @ ${detail.ref}`));
|
||||
}
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
import { mkdir } from "node:fs/promises";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
||||
@ -24,22 +24,48 @@ export async function buildImportLoopProfile(options = {}) {
|
||||
const entrypoint = options.entrypoint ?? defaultImportLoopProfileOptions.entrypoint;
|
||||
assertRunCount(runs, 20);
|
||||
|
||||
const baseline = await buildBaselineProfile({ ...options, rootDir, runs });
|
||||
const samples = [];
|
||||
for (let index = 0; index < runs; index += 1) {
|
||||
samples.push(await runCaptureSample({ ...options, entrypoint, index, rootDir }));
|
||||
const sample = await runCaptureSample({ ...options, entrypoint, index, rootDir });
|
||||
samples.push(applyBaselineAdjustment(sample, baseline));
|
||||
}
|
||||
|
||||
const wallMs = samples.map((sample) => sample.wallMs).sort((left, right) => left - right);
|
||||
const pluginWallDeltaMs = samples.map((sample) => sample.pluginWallDeltaMs).sort((left, right) => left - right);
|
||||
const openClawImportMs = openClawLifecycleMetric(samples, "importMs");
|
||||
const openClawActivationMs = openClawLifecycleMetric(samples, "activationMs");
|
||||
const rssSampleCount = samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)), 0);
|
||||
const cpuSampleCount = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
|
||||
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
|
||||
return {
|
||||
generatedAt: options.generatedAt ?? defaultImportLoopProfileOptions.generatedAt,
|
||||
mode: options.mode ?? "subprocess-cold-import-loop",
|
||||
mode: options.mode ?? "baseline-adjusted-cold-capture-loop",
|
||||
entrypoint,
|
||||
baseline,
|
||||
summary: {
|
||||
runs,
|
||||
baselineRuns: baseline.runs,
|
||||
baselineFailCount: baseline.failCount,
|
||||
p50WallMs: percentile(wallMs, 0.5),
|
||||
p95WallMs: percentile(wallMs, 0.95),
|
||||
p50PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.5),
|
||||
p95PluginWallDeltaMs: percentile(pluginWallDeltaMs, 0.95),
|
||||
openClawLifecycleCount: openClawImportMs.length,
|
||||
p50OpenClawImportMs: percentile(openClawImportMs, 0.5),
|
||||
p95OpenClawImportMs: percentile(openClawImportMs, 0.95),
|
||||
p50OpenClawActivationMs: percentile(openClawActivationMs, 0.5),
|
||||
p95OpenClawActivationMs: percentile(openClawActivationMs, 0.95),
|
||||
maxPeakRssMb: Math.max(0, ...samples.map((sample) => sample.peakRssMb)),
|
||||
maxCpuMsEstimate: Math.max(0, ...samples.map((sample) => sample.cpuMsEstimate)),
|
||||
maxPluginPeakRssDeltaMb: Math.max(0, ...samples.map((sample) => sample.pluginPeakRssDeltaMb)),
|
||||
maxPluginCpuDeltaMsEstimate: Math.max(0, ...samples.map((sample) => sample.pluginCpuDeltaMsEstimate)),
|
||||
baselineReferenceWallMs: baseline.reference.wallMs,
|
||||
baselineReferencePeakRssMb: baseline.reference.peakRssMb,
|
||||
baselineReferenceCpuMsEstimate: baseline.reference.cpuMsEstimate,
|
||||
statSampleCount,
|
||||
rssSampleCount,
|
||||
cpuSampleCount,
|
||||
capturedCount: samples.reduce((sum, sample) => sum + sample.capturedCount, 0),
|
||||
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
|
||||
},
|
||||
@ -52,6 +78,9 @@ export function validateImportLoopProfile(report) {
|
||||
if (report.summary.failCount > 0) {
|
||||
errors.push(`import loop has ${report.summary.failCount} failed sample(s)`);
|
||||
}
|
||||
if ((report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0) > 0) {
|
||||
errors.push("import loop baseline capture failed");
|
||||
}
|
||||
if (report.summary.capturedCount < report.summary.runs) {
|
||||
errors.push("import loop did not capture at least one contract per run");
|
||||
}
|
||||
@ -85,7 +114,11 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
|
||||
"",
|
||||
"## Summary",
|
||||
"",
|
||||
markdownTable(Object.entries(report.summary).map(([key, value]) => [key, value]), ["Metric", "Value"]),
|
||||
markdownTable(summaryRows(report), ["Metric", "Value"]),
|
||||
"",
|
||||
"## Harness Baseline",
|
||||
"",
|
||||
markdownTable(baselineRows(report), ["Metric", "Value"]),
|
||||
"",
|
||||
"## Samples",
|
||||
"",
|
||||
@ -94,22 +127,132 @@ export function renderImportLoopProfileMarkdown(report, options = {}) {
|
||||
sample.index,
|
||||
sample.status,
|
||||
sample.capturedCount,
|
||||
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.importMs),
|
||||
formatOpenClawLifecycleMetric(sample.openClawLifecycle?.activationMs),
|
||||
formatOptionalMetric(sample.pluginWallDeltaMs, "ms"),
|
||||
formatSampledMetric(sample.pluginPeakRssDeltaMb, sample.rssSampleCount),
|
||||
formatSampledMetric(sample.pluginCpuDeltaMsEstimate, sample.cpuSampleCount, "ms"),
|
||||
`${sample.wallMs} ms`,
|
||||
`${sample.peakRssMb} MB`,
|
||||
`${sample.cpuMsEstimate} ms`,
|
||||
formatSampledMetric(sample.peakRssMb, sample.rssSampleCount),
|
||||
formatSampledMetric(sample.cpuMsEstimate, sample.cpuSampleCount, "ms"),
|
||||
`${sample.rssSampleCount ?? 0}/${sample.cpuSampleCount ?? 0}`,
|
||||
sample.exitCode,
|
||||
]),
|
||||
["Run", "Status", "Captured", "Wall", "Peak RSS", "CPU Estimate", "Exit"],
|
||||
[
|
||||
"Run",
|
||||
"Status",
|
||||
"Captured",
|
||||
"OpenClaw Import",
|
||||
"OpenClaw Activate",
|
||||
"Plugin Wall Delta",
|
||||
"Plugin RSS Delta",
|
||||
"Plugin CPU Delta",
|
||||
"Raw Wall",
|
||||
"Raw Peak RSS",
|
||||
"Raw CPU Estimate",
|
||||
"RSS/CPU samples",
|
||||
"Exit",
|
||||
],
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
async function buildBaselineProfile(options) {
|
||||
const baselineRuns = options.baseline === false ? 0 : options.baselineRuns ?? Math.min(options.runs, 3);
|
||||
if (baselineRuns <= 0) {
|
||||
return emptyBaseline();
|
||||
}
|
||||
|
||||
const entrypoint = await writeBaselineEntrypoint(options);
|
||||
const samples = [];
|
||||
for (let index = 0; index < baselineRuns; index += 1) {
|
||||
samples.push(
|
||||
await runCaptureSample({
|
||||
...options,
|
||||
entrypoint,
|
||||
index,
|
||||
sampleName: "baseline",
|
||||
rootDir: options.rootDir,
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
const wallMs = sortedMetric(samples, "wallMs");
|
||||
const peakRssMb = sortedMetric(samples, "peakRssMb");
|
||||
const cpuMsEstimate = sortedMetric(samples, "cpuMsEstimate");
|
||||
return {
|
||||
mode: "minimal-plugin-capture",
|
||||
runs: baselineRuns,
|
||||
entrypoint: path.relative(options.rootDir, entrypoint),
|
||||
reference: {
|
||||
wallMs: percentile(wallMs, 0.5),
|
||||
peakRssMb: percentile(peakRssMb, 0.5),
|
||||
cpuMsEstimate: percentile(cpuMsEstimate, 0.5),
|
||||
},
|
||||
max: {
|
||||
wallMs: wallMs.at(-1) ?? 0,
|
||||
peakRssMb: peakRssMb.at(-1) ?? 0,
|
||||
cpuMsEstimate: cpuMsEstimate.at(-1) ?? 0,
|
||||
},
|
||||
statSampleCount: samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0),
|
||||
rssSampleCount: samples.reduce((sum, sample) => sum + (sample.rssSampleCount ?? 0), 0),
|
||||
cpuSampleCount: samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0),
|
||||
failCount: samples.filter((sample) => sample.exitCode !== 0 || sample.status !== "captured").length,
|
||||
samples,
|
||||
};
|
||||
}
|
||||
|
||||
function emptyBaseline() {
|
||||
return {
|
||||
mode: "disabled",
|
||||
runs: 0,
|
||||
entrypoint: null,
|
||||
reference: {
|
||||
wallMs: 0,
|
||||
peakRssMb: 0,
|
||||
cpuMsEstimate: 0,
|
||||
},
|
||||
max: {
|
||||
wallMs: 0,
|
||||
peakRssMb: 0,
|
||||
cpuMsEstimate: 0,
|
||||
},
|
||||
statSampleCount: 0,
|
||||
rssSampleCount: 0,
|
||||
cpuSampleCount: 0,
|
||||
failCount: 0,
|
||||
samples: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function writeBaselineEntrypoint(options) {
|
||||
const outputDir = resolveFromRoot(
|
||||
options.rootDir,
|
||||
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
|
||||
);
|
||||
const baselinePath = path.join(outputDir, "baseline-plugin.mjs");
|
||||
await mkdir(path.dirname(baselinePath), { recursive: true });
|
||||
await writeFile(
|
||||
baselinePath,
|
||||
[
|
||||
"export default {",
|
||||
" register(api) {",
|
||||
" api.registerTool({ name: 'baseline_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
" },",
|
||||
"};",
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
return baselinePath;
|
||||
}
|
||||
|
||||
async function runCaptureSample(options) {
|
||||
const outputDir = resolveFromRoot(
|
||||
options.rootDir,
|
||||
options.outputDir ?? defaultImportLoopProfileOptions.outputDir,
|
||||
);
|
||||
const outputPath = path.join(outputDir, `capture-${options.index}.json`);
|
||||
const outputPath = path.join(outputDir, `${options.sampleName ?? "capture"}-${options.index}.json`);
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
|
||||
const command = buildCaptureCommand({ ...options, outputPath });
|
||||
@ -126,14 +269,125 @@ async function runCaptureSample(options) {
|
||||
exitCode: profile.exitCode,
|
||||
status: output?.status ?? "failed",
|
||||
capturedCount: output?.captured?.length ?? 0,
|
||||
openClawLifecycle: output?.openClawLifecycle ?? null,
|
||||
wallMs: profile.wallMs,
|
||||
peakRssMb: profile.peakRssMb,
|
||||
peakCpuPercent: profile.peakCpuPercent,
|
||||
cpuMsEstimate: profile.cpuMsEstimate,
|
||||
statSampleCount: profile.statSampleCount,
|
||||
rssSampleCount: profile.rssSampleCount,
|
||||
cpuSampleCount: profile.cpuSampleCount,
|
||||
stderrPreview: profile.stderrPreview,
|
||||
};
|
||||
}
|
||||
|
||||
function summaryRows(report) {
|
||||
return [
|
||||
["runs", report.summary.runs],
|
||||
["baselineRuns", report.summary.baselineRuns ?? report.baseline?.runs ?? 0],
|
||||
["baselineFailCount", report.summary.baselineFailCount ?? report.baseline?.failCount ?? 0],
|
||||
["p50WallMs", report.summary.p50WallMs],
|
||||
["p95WallMs", report.summary.p95WallMs],
|
||||
...(Number.isFinite(report.summary.p50PluginWallDeltaMs)
|
||||
? [
|
||||
["p50PluginWallDeltaMs", report.summary.p50PluginWallDeltaMs],
|
||||
["p95PluginWallDeltaMs", report.summary.p95PluginWallDeltaMs],
|
||||
["maxPluginPeakRssDeltaMb", formatSampledMetric(report.summary.maxPluginPeakRssDeltaMb, report.summary.rssSampleCount)],
|
||||
[
|
||||
"maxPluginCpuDeltaMsEstimate",
|
||||
formatSampledMetric(report.summary.maxPluginCpuDeltaMsEstimate, report.summary.cpuSampleCount, "ms"),
|
||||
],
|
||||
]
|
||||
: []),
|
||||
...((report.summary.openClawLifecycleCount ?? 0) > 0
|
||||
? [
|
||||
["openClawLifecycleCount", report.summary.openClawLifecycleCount],
|
||||
["p50OpenClawImportMs", `${report.summary.p50OpenClawImportMs} ms`],
|
||||
["p95OpenClawImportMs", `${report.summary.p95OpenClawImportMs} ms`],
|
||||
["p50OpenClawActivationMs", `${report.summary.p50OpenClawActivationMs} ms`],
|
||||
["p95OpenClawActivationMs", `${report.summary.p95OpenClawActivationMs} ms`],
|
||||
]
|
||||
: []),
|
||||
["maxPeakRssMb", formatSampledMetric(report.summary.maxPeakRssMb, report.summary.rssSampleCount)],
|
||||
["maxCpuMsEstimate", formatSampledMetric(report.summary.maxCpuMsEstimate, report.summary.cpuSampleCount, "ms")],
|
||||
...(Number.isFinite(report.summary.baselineReferenceWallMs)
|
||||
? [
|
||||
["baselineReferenceWallMs", `${report.summary.baselineReferenceWallMs} ms`],
|
||||
["baselineReferencePeakRssMb", formatSampledMetric(report.summary.baselineReferencePeakRssMb, report.baseline?.rssSampleCount ?? 0)],
|
||||
[
|
||||
"baselineReferenceCpuMsEstimate",
|
||||
formatSampledMetric(report.summary.baselineReferenceCpuMsEstimate, report.baseline?.cpuSampleCount ?? 0, "ms"),
|
||||
],
|
||||
]
|
||||
: []),
|
||||
["statSampleCount", report.summary.statSampleCount ?? 0],
|
||||
["rssSampleCount", report.summary.rssSampleCount ?? 0],
|
||||
["cpuSampleCount", report.summary.cpuSampleCount ?? 0],
|
||||
["capturedCount", report.summary.capturedCount],
|
||||
["failCount", report.summary.failCount],
|
||||
];
|
||||
}
|
||||
|
||||
function baselineRows(report) {
|
||||
const baseline = report.baseline ?? emptyBaseline();
|
||||
return [
|
||||
["mode", baseline.mode],
|
||||
["runs", baseline.runs],
|
||||
["entrypoint", baseline.entrypoint ?? "-"],
|
||||
["referenceWallMs", `${baseline.reference?.wallMs ?? 0} ms`],
|
||||
["referencePeakRssMb", formatSampledMetric(baseline.reference?.peakRssMb ?? 0, baseline.rssSampleCount)],
|
||||
["referenceCpuMsEstimate", formatSampledMetric(baseline.reference?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
|
||||
["maxWallMs", `${baseline.max?.wallMs ?? 0} ms`],
|
||||
["maxPeakRssMb", formatSampledMetric(baseline.max?.peakRssMb ?? 0, baseline.rssSampleCount)],
|
||||
["maxCpuMsEstimate", formatSampledMetric(baseline.max?.cpuMsEstimate ?? 0, baseline.cpuSampleCount, "ms")],
|
||||
["statSampleCount", baseline.statSampleCount ?? 0],
|
||||
["failCount", baseline.failCount ?? 0],
|
||||
];
|
||||
}
|
||||
|
||||
function formatSampledMetric(value, count, unit = "MB") {
|
||||
if ((count ?? 0) <= 0) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
function formatOptionalMetric(value, unit) {
|
||||
if (!Number.isFinite(value)) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
function formatOpenClawLifecycleMetric(value) {
|
||||
return Number.isFinite(value) ? `${value} ms` : "n/a";
|
||||
}
|
||||
|
||||
function openClawLifecycleMetric(samples, field) {
|
||||
return samples
|
||||
.map((sample) => sample.openClawLifecycle?.[field])
|
||||
.filter((value) => Number.isFinite(value))
|
||||
.sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
function applyBaselineAdjustment(sample, baseline) {
|
||||
return {
|
||||
...sample,
|
||||
pluginWallDeltaMs: roundNonNegative(sample.wallMs - baseline.reference.wallMs, 0),
|
||||
pluginPeakRssDeltaMb: roundNonNegative(sample.peakRssMb - baseline.reference.peakRssMb, 1),
|
||||
pluginCpuDeltaMsEstimate: roundNonNegative(sample.cpuMsEstimate - baseline.reference.cpuMsEstimate, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function sortedMetric(samples, field) {
|
||||
return samples.map((sample) => sample[field]).sort((left, right) => left - right);
|
||||
}
|
||||
|
||||
function roundNonNegative(value, digits) {
|
||||
const scale = 10 ** digits;
|
||||
return Math.max(0, Math.round(value * scale) / scale);
|
||||
}
|
||||
|
||||
function buildCaptureCommand(options) {
|
||||
if (typeof options.captureCommand === "function") {
|
||||
return options.captureCommand({
|
||||
|
||||
237
src/index.js
237
src/index.js
@ -1,11 +1,248 @@
|
||||
import * as pluginApi from "./api.js";
|
||||
import * as ciPolicyApi from "./ci-policy.js";
|
||||
import * as ciSummaryApi from "./ci-summary.js";
|
||||
import * as configApi from "./config.js";
|
||||
import * as contractCaptureApi from "./contract-capture.js";
|
||||
import * as contractCoverageApi from "./contract-coverage.js";
|
||||
import * as executionResultsApi from "./execution-results.js";
|
||||
import * as importLoopProfileApi from "./import-loop-profile.js";
|
||||
import * as inspectorApi from "./inspector.js";
|
||||
import * as issuesApi from "./issues.js";
|
||||
import * as openClawTargetApi from "./openclaw-target.js";
|
||||
import * as profileDiffApi from "./profile-diff.js";
|
||||
import * as refDiffApi from "./ref-diff.js";
|
||||
import * as reportApi from "./report.js";
|
||||
import * as runtimeProfileApi from "./runtime-profile.js";
|
||||
import * as runtimeReconciliationApi from "./runtime-reconciliation.js";
|
||||
import * as syntheticProbeSuiteApi from "./synthetic-probe-suite.js";
|
||||
import * as syntheticProbesApi from "./synthetic-probes.js";
|
||||
|
||||
export const pluginRoot = Object.freeze({
|
||||
loadConfig: pluginApi.loadPluginConfig,
|
||||
inspect: pluginApi.inspectPluginRoot,
|
||||
runCheck: pluginApi.runPluginCheck,
|
||||
captureEntrypoint: pluginApi.capturePluginEntrypoint,
|
||||
setup: pluginApi.setupPluginInspector,
|
||||
});
|
||||
|
||||
export const fixtureSuites = Object.freeze({
|
||||
loadConfig: configApi.loadInspectorConfig,
|
||||
inspect: pluginApi.inspectCompatibilityFixtureSetConfig,
|
||||
inspectStatic: pluginApi.inspectFixtureSetConfig,
|
||||
runReport: pluginApi.runFixtureSetReport,
|
||||
writeReports: pluginApi.writeFixtureSetReports,
|
||||
renderReport: pluginApi.renderFixtureSetMarkdownReport,
|
||||
renderIssues: pluginApi.renderFixtureSetIssuesReport,
|
||||
buildColdImportReadiness: pluginApi.buildFixtureSetColdImportReadiness,
|
||||
runColdImportReadiness: pluginApi.runFixtureSetColdImportReadiness,
|
||||
writeColdImportReadiness: pluginApi.writeFixtureSetColdImportReadiness,
|
||||
buildWorkspacePlan: pluginApi.buildFixtureSetWorkspacePlan,
|
||||
runWorkspacePlan: pluginApi.runFixtureSetWorkspacePlan,
|
||||
writeWorkspacePlan: pluginApi.writeFixtureSetWorkspacePlan,
|
||||
buildPlatformProbes: pluginApi.buildFixtureSetPlatformProbes,
|
||||
runPlatformProbes: pluginApi.runFixtureSetPlatformProbes,
|
||||
writePlatformProbes: pluginApi.writeFixtureSetPlatformProbes,
|
||||
});
|
||||
|
||||
export const staticInspection = Object.freeze({
|
||||
loadConfig: configApi.loadInspectorConfig,
|
||||
inspectSourceText: inspectorApi.inspectSourceText,
|
||||
inspectPlugin: inspectorApi.inspectPlugin,
|
||||
inspectFixtureSet: inspectorApi.inspectFixtureSet,
|
||||
});
|
||||
|
||||
export const reports = Object.freeze({
|
||||
renderMarkdown: reportApi.renderMarkdownReport,
|
||||
renderTextSummary: pluginApi.renderTextSummary,
|
||||
sanitizeArtifact: reportApi.sanitizeReportArtifact,
|
||||
write: reportApi.writeReport,
|
||||
issueId: issuesApi.issueId,
|
||||
classifyIssueFinding: issuesApi.classifyIssueFinding,
|
||||
knownIssueCodes: issuesApi.knownIssueCodes,
|
||||
openClawTargetPathCandidates: openClawTargetApi.openClawTargetPathCandidates,
|
||||
readOpenClawTargetSurface: openClawTargetApi.readOpenClawTargetSurface,
|
||||
});
|
||||
|
||||
export const contracts = Object.freeze({
|
||||
buildCapture: contractCaptureApi.buildContractCapture,
|
||||
writeCapture: contractCaptureApi.writeContractCapture,
|
||||
renderCapture: contractCaptureApi.renderContractCaptureMarkdown,
|
||||
validateCapture: contractCaptureApi.validateContractCapture,
|
||||
validateCoverage: contractCoverageApi.validateContractCoverage,
|
||||
defaults: Object.freeze({
|
||||
registrationAssertions: contractCaptureApi.defaultRegistrationAssertions,
|
||||
registrationArguments: contractCaptureApi.defaultRegistrationArguments,
|
||||
hookAssertions: contractCaptureApi.defaultHookAssertions,
|
||||
hookEvents: contractCaptureApi.defaultHookEvents,
|
||||
hookContexts: contractCaptureApi.defaultHookContexts,
|
||||
}),
|
||||
});
|
||||
|
||||
export const ci = Object.freeze({
|
||||
buildSummary: ciSummaryApi.buildCiSummary,
|
||||
writeSummary: ciSummaryApi.writeCiSummary,
|
||||
renderSummary: ciSummaryApi.renderCiSummaryMarkdown,
|
||||
readReports: ciSummaryApi.readCiReports,
|
||||
deriveStatus: ciSummaryApi.deriveCiStatus,
|
||||
buildPolicyReport: ciPolicyApi.buildCiPolicyReport,
|
||||
writePolicyReport: ciPolicyApi.writeCiPolicyReport,
|
||||
renderPolicyReport: ciPolicyApi.renderCiPolicyMarkdown,
|
||||
validatePolicy: ciPolicyApi.validateCiPolicy,
|
||||
validatePolicyReport: ciPolicyApi.validateCiPolicyReport,
|
||||
buildExecutionResults: executionResultsApi.buildExecutionResultsReport,
|
||||
writeExecutionResults: executionResultsApi.writeExecutionResultsReport,
|
||||
renderExecutionResults: executionResultsApi.renderExecutionResultsMarkdown,
|
||||
writeOutputs: pluginApi.writeCiOutputArtifacts,
|
||||
});
|
||||
|
||||
export const runtime = Object.freeze({
|
||||
buildProfile: runtimeProfileApi.buildRuntimeProfile,
|
||||
writeProfile: runtimeProfileApi.writeRuntimeProfile,
|
||||
renderProfile: runtimeProfileApi.renderRuntimeProfileMarkdown,
|
||||
validateProfile: runtimeProfileApi.validateRuntimeProfile,
|
||||
buildProfileDiff: profileDiffApi.buildProfileDiff,
|
||||
writeProfileDiff: profileDiffApi.writeProfileDiff,
|
||||
renderProfileDiff: profileDiffApi.renderProfileDiffMarkdown,
|
||||
validateProfileDiff: profileDiffApi.validateProfileDiff,
|
||||
buildRefDiff: refDiffApi.buildRefDiff,
|
||||
writeRefDiff: refDiffApi.writeRefDiff,
|
||||
renderRefDiff: refDiffApi.renderRefDiffMarkdown,
|
||||
validateRefDiff: refDiffApi.validateRefDiff,
|
||||
buildImportLoopProfile: importLoopProfileApi.buildImportLoopProfile,
|
||||
writeImportLoopProfile: importLoopProfileApi.writeImportLoopProfile,
|
||||
renderImportLoopProfile: importLoopProfileApi.renderImportLoopProfileMarkdown,
|
||||
validateImportLoopProfile: importLoopProfileApi.validateImportLoopProfile,
|
||||
applyExecutionCoverage: runtimeReconciliationApi.applyRuntimeExecutionCoverage,
|
||||
buildExecutionCoverage: runtimeReconciliationApi.buildRuntimeExecutionCoverage,
|
||||
});
|
||||
|
||||
export const synthetic = Object.freeze({
|
||||
buildPlan: syntheticProbesApi.buildSyntheticProbePlan,
|
||||
buildPlanFromReport: syntheticProbeSuiteApi.buildSyntheticProbePlanFromReport,
|
||||
writePlan: syntheticProbesApi.writeSyntheticProbePlan,
|
||||
renderPlan: syntheticProbesApi.renderSyntheticProbeMarkdown,
|
||||
validatePlan: syntheticProbesApi.validateSyntheticProbePlan,
|
||||
runCaptured: syntheticProbesApi.runCapturedSyntheticProbes,
|
||||
registrationExecutionProfiles: syntheticProbesApi.syntheticRegistrationExecutionProfiles,
|
||||
defaultHookEvents: syntheticProbesApi.defaultSyntheticHookEvents,
|
||||
defaultHookContexts: syntheticProbesApi.defaultSyntheticHookContexts,
|
||||
defaultRegistrationArguments: syntheticProbesApi.defaultSyntheticRegistrationArguments,
|
||||
});
|
||||
|
||||
export {
|
||||
capturePluginEntrypoint,
|
||||
buildFixtureSetColdImportReadiness,
|
||||
buildFixtureSetPlatformProbes,
|
||||
buildFixtureSetWorkspacePlan,
|
||||
createCaptureApi,
|
||||
inspectCompatibilityFixtureSetConfig,
|
||||
inspectFixtureSetConfig,
|
||||
inspectPluginRoot,
|
||||
loadPluginConfig,
|
||||
renderFixtureSetColdImportReadinessMarkdown,
|
||||
renderFixtureSetIssuesReport,
|
||||
renderFixtureSetMarkdownReport,
|
||||
renderFixtureSetPlatformProbesMarkdown,
|
||||
renderFixtureSetWorkspacePlanMarkdown,
|
||||
renderTextSummary,
|
||||
runFixtureSetColdImportReadiness,
|
||||
runFixtureSetPlatformProbes,
|
||||
runFixtureSetReport,
|
||||
runFixtureSetWorkspacePlan,
|
||||
runPluginCheck,
|
||||
setupPluginInspector,
|
||||
validateColdImportReadiness,
|
||||
validateFixtureSetPlatformProbes,
|
||||
validateFixtureSetWorkspacePlan,
|
||||
writeCiOutputArtifacts,
|
||||
writeFixtureSetColdImportReadiness,
|
||||
writeFixtureSetPlatformProbes,
|
||||
writeFixtureSetReports,
|
||||
writeFixtureSetWorkspacePlan,
|
||||
writePluginReports,
|
||||
} from "./api.js";
|
||||
export {
|
||||
buildContractCapture,
|
||||
defaultHookAssertions,
|
||||
defaultHookContexts,
|
||||
defaultHookEvents,
|
||||
defaultRegistrationArguments,
|
||||
defaultRegistrationAssertions,
|
||||
renderContractCaptureMarkdown,
|
||||
validateContractCapture,
|
||||
writeContractCapture,
|
||||
} from "./contract-capture.js";
|
||||
export { validateContractCoverage } from "./contract-coverage.js";
|
||||
export {
|
||||
buildCiPolicyReport,
|
||||
defaultCiPolicyReportOptions,
|
||||
renderCiPolicyMarkdown,
|
||||
validateCiPolicy,
|
||||
validateCiPolicyReport,
|
||||
writeCiPolicyReport,
|
||||
} from "./ci-policy.js";
|
||||
export {
|
||||
buildCiSummary,
|
||||
defaultCiReportPaths,
|
||||
deriveCiStatus,
|
||||
readCiReports,
|
||||
renderCiSummaryMarkdown,
|
||||
writeCiSummary,
|
||||
} from "./ci-summary.js";
|
||||
export { loadInspectorConfig } from "./config.js";
|
||||
export {
|
||||
buildExecutionResultsReport,
|
||||
defaultExecutionResultsOptions,
|
||||
renderExecutionResultsMarkdown,
|
||||
writeExecutionResultsReport,
|
||||
} from "./execution-results.js";
|
||||
export {
|
||||
buildImportLoopProfile,
|
||||
defaultImportLoopProfileOptions,
|
||||
renderImportLoopProfileMarkdown,
|
||||
validateImportLoopProfile,
|
||||
writeImportLoopProfile,
|
||||
} from "./import-loop-profile.js";
|
||||
export { classifyIssueFinding, issueId, knownIssueCodes } from "./issues.js";
|
||||
export { inspectFixtureSet, inspectPlugin, inspectSourceText } from "./inspector.js";
|
||||
export { openClawTargetPathCandidates, readOpenClawTargetSurface } from "./openclaw-target.js";
|
||||
export {
|
||||
buildProfileDiff,
|
||||
defaultProfileDiffOptions,
|
||||
renderProfileDiffMarkdown,
|
||||
validateProfileDiff,
|
||||
writeProfileDiff,
|
||||
} from "./profile-diff.js";
|
||||
export {
|
||||
buildRefDiff,
|
||||
defaultRefDiffDimensions,
|
||||
defaultRefDiffOptions,
|
||||
renderRefDiffMarkdown,
|
||||
validateRefDiff,
|
||||
writeRefDiff,
|
||||
} from "./ref-diff.js";
|
||||
export { renderMarkdownReport, sanitizeReportArtifact, writeReport } from "./report.js";
|
||||
export {
|
||||
buildRuntimeProfile,
|
||||
defaultRuntimeProfileCommands,
|
||||
defaultRuntimeProfileOptions,
|
||||
renderRuntimeProfileMarkdown,
|
||||
validateRuntimeProfile,
|
||||
writeRuntimeProfile,
|
||||
} from "./runtime-profile.js";
|
||||
export {
|
||||
applyRuntimeExecutionCoverage,
|
||||
buildRuntimeExecutionCoverage,
|
||||
} from "./runtime-reconciliation.js";
|
||||
export { buildSyntheticProbePlanFromReport } from "./synthetic-probe-suite.js";
|
||||
export {
|
||||
buildSyntheticProbePlan,
|
||||
defaultSyntheticHookContexts,
|
||||
defaultSyntheticHookEvents,
|
||||
defaultSyntheticRegistrationArguments,
|
||||
renderSyntheticProbeMarkdown,
|
||||
runCapturedSyntheticProbes,
|
||||
syntheticRegistrationExecutionProfiles,
|
||||
validateSyntheticProbePlan,
|
||||
writeSyntheticProbePlan,
|
||||
} from "./synthetic-probes.js";
|
||||
|
||||
104
src/init.js
104
src/init.js
@ -5,32 +5,66 @@ import { inferPluginSeams, packageId } from "./config.js";
|
||||
|
||||
export const defaultInitConfigPath = "plugin-inspector.config.json";
|
||||
export const defaultInitWorkflowPath = ".github/workflows/plugin-inspector.yml";
|
||||
export const defaultInitPackageScripts = {
|
||||
"plugin:check": "plugin-inspector inspect --no-openclaw",
|
||||
"plugin:ci": "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute",
|
||||
};
|
||||
|
||||
export async function writePluginInspectorInit(options = {}) {
|
||||
const pluginRoot = path.resolve(options.pluginRoot ?? options.cwd ?? process.cwd());
|
||||
const configPath = path.resolve(pluginRoot, options.configPath ?? defaultInitConfigPath);
|
||||
const workflowPath = options.ci === true ? path.resolve(pluginRoot, options.workflowPath ?? defaultInitWorkflowPath) : null;
|
||||
const packageManager = options.packageManager ?? (await detectPackageManager(pluginRoot));
|
||||
const dryRun = options.dryRun === true;
|
||||
const written = [];
|
||||
|
||||
if (existsSync(configPath) && options.force !== true) {
|
||||
if (!dryRun && existsSync(configPath) && options.force !== true) {
|
||||
throw new Error(`${path.relative(pluginRoot, configPath)} already exists; pass --force to overwrite it`);
|
||||
}
|
||||
if (!dryRun && workflowPath && existsSync(workflowPath) && options.force !== true) {
|
||||
throw new Error(`${path.relative(pluginRoot, workflowPath)} already exists; pass --force to overwrite it`);
|
||||
}
|
||||
const packageJsonPath = path.join(pluginRoot, "package.json");
|
||||
const packageJson = options.scripts === true ? await readJsonIfExists(packageJsonPath) : null;
|
||||
if (options.scripts === true) {
|
||||
if (!packageJson) {
|
||||
throw new Error("package.json is required to write plugin-inspector package scripts");
|
||||
}
|
||||
for (const name of Object.keys(defaultInitPackageScripts)) {
|
||||
if (!dryRun && packageJson.scripts?.[name] && options.force !== true) {
|
||||
throw new Error(`package.json scripts.${name} already exists; pass --force to overwrite it`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const config = await buildPluginInspectorConfig({ pluginRoot });
|
||||
await mkdir(path.dirname(configPath), { recursive: true });
|
||||
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
|
||||
if (!dryRun) {
|
||||
await mkdir(path.dirname(configPath), { recursive: true });
|
||||
await writeFile(configPath, `${JSON.stringify(config, null, 2)}\n`, "utf8");
|
||||
}
|
||||
written.push(configPath);
|
||||
|
||||
if (options.ci === true) {
|
||||
const workflowPath = path.resolve(pluginRoot, options.workflowPath ?? defaultInitWorkflowPath);
|
||||
if (existsSync(workflowPath) && options.force !== true) {
|
||||
throw new Error(`${path.relative(pluginRoot, workflowPath)} already exists; pass --force to overwrite it`);
|
||||
if (workflowPath) {
|
||||
if (!dryRun) {
|
||||
await mkdir(path.dirname(workflowPath), { recursive: true });
|
||||
await writeFile(workflowPath, renderGithubActionsWorkflow({ packageManager }), "utf8");
|
||||
}
|
||||
await mkdir(path.dirname(workflowPath), { recursive: true });
|
||||
await writeFile(workflowPath, renderGithubActionsWorkflow({ packageManager: options.packageManager }), "utf8");
|
||||
written.push(workflowPath);
|
||||
}
|
||||
|
||||
return { pluginRoot, configPath, written };
|
||||
if (options.scripts === true) {
|
||||
const existingScripts = packageJson.scripts ?? {};
|
||||
packageJson.scripts = {
|
||||
...existingScripts,
|
||||
...defaultInitPackageScripts,
|
||||
};
|
||||
if (!dryRun) {
|
||||
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
|
||||
}
|
||||
written.push(packageJsonPath);
|
||||
}
|
||||
|
||||
return { pluginRoot, configPath, dryRun, packageManager, written };
|
||||
}
|
||||
|
||||
export async function buildPluginInspectorConfig(options = {}) {
|
||||
@ -79,8 +113,7 @@ jobs:
|
||||
node-version: 24
|
||||
cache: ${setup.cache}
|
||||
${setup.corepack ? " - run: corepack enable\n" : ""} - run: ${setup.install}
|
||||
- run: ${setup.exec} @openclaw/plugin-inspector check --no-openclaw
|
||||
- run: PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 ${setup.exec} @openclaw/plugin-inspector check --no-openclaw --runtime --mock-sdk
|
||||
- run: ${setup.exec} @openclaw/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute
|
||||
- uses: actions/upload-artifact@v5
|
||||
if: always()
|
||||
with:
|
||||
@ -89,19 +122,62 @@ ${setup.corepack ? " - run: corepack enable\n" : ""} - run: ${setup.in
|
||||
`;
|
||||
}
|
||||
|
||||
export async function detectPackageManager(pluginRoot) {
|
||||
const root = path.resolve(pluginRoot ?? process.cwd());
|
||||
const packageJson = await readJsonIfExists(path.join(root, "package.json"));
|
||||
const packageManager = packageJson?.packageManager;
|
||||
if (typeof packageManager === "string") {
|
||||
const [name] = packageManager.split("@");
|
||||
if (["npm", "pnpm", "yarn", "bun"].includes(name)) {
|
||||
return name;
|
||||
}
|
||||
}
|
||||
|
||||
if (existsSync(path.join(root, "pnpm-lock.yaml"))) {
|
||||
return "pnpm";
|
||||
}
|
||||
if (existsSync(path.join(root, "yarn.lock"))) {
|
||||
return "yarn";
|
||||
}
|
||||
if (existsSync(path.join(root, "bun.lockb")) || existsSync(path.join(root, "bun.lock"))) {
|
||||
return "bun";
|
||||
}
|
||||
return "npm";
|
||||
}
|
||||
|
||||
function inferSourceRoot(packageJson) {
|
||||
const entrypoints = [
|
||||
packageJson?.openclaw?.entrypoint,
|
||||
...(packageJson?.openclaw?.extensions ?? []),
|
||||
...(packageJson?.openclaw?.runtimeExtensions ?? []),
|
||||
...entrypointStrings(packageJson?.exports?.["."]),
|
||||
...entrypointStrings(packageJson?.exports),
|
||||
packageJson?.module,
|
||||
packageJson?.main,
|
||||
].filter((value) => typeof value === "string");
|
||||
const entrypoint = entrypoints[0] ?? packageJson?.exports?.["."] ?? packageJson?.main ?? "src/index.js";
|
||||
if (typeof entrypoint === "string" && entrypoint.startsWith("src/")) {
|
||||
const entrypoint = entrypoints[0] ?? "src/index.js";
|
||||
if (stripRelativePrefix(entrypoint).startsWith("src/")) {
|
||||
return "src";
|
||||
}
|
||||
return ".";
|
||||
}
|
||||
|
||||
function entrypointStrings(value) {
|
||||
if (typeof value === "string") {
|
||||
return [value];
|
||||
}
|
||||
if (!value || typeof value !== "object" || Array.isArray(value)) {
|
||||
return [];
|
||||
}
|
||||
return ["import", "default", "require", "node", "module"]
|
||||
.map((key) => value[key])
|
||||
.filter((item) => typeof item === "string");
|
||||
}
|
||||
|
||||
function stripRelativePrefix(filePath) {
|
||||
return filePath.replace(/^\.\//, "");
|
||||
}
|
||||
|
||||
async function readJsonIfExists(filePath) {
|
||||
if (!existsSync(filePath)) {
|
||||
return null;
|
||||
|
||||
@ -5,12 +5,16 @@ import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { promisify } from "node:util";
|
||||
import { createCaptureApi } from "./capture-api.js";
|
||||
import { captureApiOptionsForPlugin } from "./capture-config.js";
|
||||
import { fixtureCheckoutPath, fixtureSourceRoot } from "./config.js";
|
||||
import { buildCompatibilityFixtureReport } from "./fixture-summary.js";
|
||||
import { readOpenClawTargetSurface } from "./openclaw-target.js";
|
||||
import { buildCompatibilityReport, buildReport } from "./report.js";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const registrationEquivalents = new Map([
|
||||
["registerChannel", new Set(["createChatChannelPlugin", "defineBundledChannelEntry", "defineChannelPluginEntry", "registerChannel"])],
|
||||
]);
|
||||
|
||||
export async function inspectFixtureSet(config, options = {}) {
|
||||
const { inspections, failures } = await inspectConfiguredFixtures(config, options);
|
||||
@ -32,6 +36,7 @@ export async function inspectCompatibilityFixtureSet(config, options = {}) {
|
||||
inspections,
|
||||
failures,
|
||||
generatedAt: options.generatedAt,
|
||||
executionResults: options.executionResults,
|
||||
targetOpenClaw,
|
||||
buildFixtureReport: ({ fixture, inspection }) =>
|
||||
buildCompatibilityFixtureReport({
|
||||
@ -58,7 +63,7 @@ async function inspectConfiguredFixtures(config, options = {}) {
|
||||
["manifestContracts", inspection.manifestContracts],
|
||||
]) {
|
||||
const expected = fixture.expect?.[key] ?? [];
|
||||
const missing = expected.filter((value) => !observed.includes(value));
|
||||
const missing = expected.filter((value) => !satisfiesExpectedSeam(key, value, observed));
|
||||
if (missing.length > 0) {
|
||||
failures.push(`${fixture.id}: missing ${key}: ${missing.join(", ")}`);
|
||||
}
|
||||
@ -68,6 +73,17 @@ async function inspectConfiguredFixtures(config, options = {}) {
|
||||
return { inspections, failures };
|
||||
}
|
||||
|
||||
function satisfiesExpectedSeam(key, expected, observed) {
|
||||
if (observed.includes(expected)) {
|
||||
return true;
|
||||
}
|
||||
if (key !== "registrations") {
|
||||
return false;
|
||||
}
|
||||
const equivalents = registrationEquivalents.get(expected);
|
||||
return Boolean(equivalents && observed.some((value) => equivalents.has(value)));
|
||||
}
|
||||
|
||||
export async function inspectPlugin(fixture, options = {}) {
|
||||
const config = options.config ?? { rootDir: options.rootDir ?? process.cwd() };
|
||||
const checkoutPath = fixtureCheckoutPath(config, fixture);
|
||||
@ -132,6 +148,7 @@ export function inspectSourceText(text, filePath = "source.js") {
|
||||
const hooks = collectDetailedMatches(searchableText, /\bapi\.on\(\s*["'`]([^"'`]+)["'`]/g, filePath, "name");
|
||||
const registrations = [
|
||||
...collectDetailedMatches(searchableText, /\bapi\.(register[A-Za-z0-9]+)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(defineBundledChannelEntry)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(defineChannelPluginEntry)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(createChatChannelPlugin)\s*\(/g, filePath, "name"),
|
||||
...collectDetailedMatches(searchableText, /\b(definePluginEntry)\s*\(/g, filePath, "name"),
|
||||
@ -172,7 +189,12 @@ export async function captureEntrypoint(entrypoint, options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
const api = createCaptureApi(options.apiOptions);
|
||||
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
|
||||
pluginRoot: options.pluginRoot
|
||||
? path.resolve(options.cwd ?? process.cwd(), options.pluginRoot)
|
||||
: path.dirname(resolvedEntrypoint),
|
||||
});
|
||||
const api = createCaptureApi(apiOptions);
|
||||
try {
|
||||
await register(api);
|
||||
} catch (error) {
|
||||
@ -183,7 +205,7 @@ export async function captureEntrypoint(entrypoint, options = {}) {
|
||||
entrypoint: resolvedEntrypoint,
|
||||
captured: api.getCapturedContracts(),
|
||||
};
|
||||
if (options.apiOptions?.retainHandlers === true) {
|
||||
if (apiOptions?.retainHandlers === true) {
|
||||
result.retained = api.getRetainedContracts();
|
||||
}
|
||||
return result;
|
||||
@ -212,10 +234,34 @@ export async function captureEntrypointWithMockSdk(entrypoint, options = {}) {
|
||||
);
|
||||
return JSON.parse(stdout);
|
||||
} catch (error) {
|
||||
const captured = parseCaptureResultFromStdout(error?.stdout);
|
||||
if (captured) {
|
||||
return captured;
|
||||
}
|
||||
throw classifyMockSdkCaptureError(error);
|
||||
}
|
||||
}
|
||||
|
||||
function parseCaptureResultFromStdout(stdout) {
|
||||
if (!stdout) {
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(stdout);
|
||||
if (
|
||||
parsed &&
|
||||
typeof parsed === "object" &&
|
||||
typeof parsed.status === "string" &&
|
||||
Array.isArray(parsed.captured)
|
||||
) {
|
||||
return parsed;
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function classifyMockSdkCaptureError(error) {
|
||||
const rawMessage = [error?.stderr, error?.stdout, error?.message].filter(Boolean).join("\n");
|
||||
const missingExport = rawMessage.match(/does not provide an export named ['"]([^'"]+)['"]/)?.[1];
|
||||
@ -457,9 +503,38 @@ function lineForOffset(text, offset) {
|
||||
}
|
||||
|
||||
function stripComments(text) {
|
||||
return text
|
||||
.replace(/\/\*[\s\S]*?\*\//g, (comment) => comment.replace(/[^\n]/g, " "))
|
||||
.replace(/\/\/.*$/gm, (comment) => " ".repeat(comment.length));
|
||||
let result = "";
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
const next = text[index + 1];
|
||||
if (char === "/" && next === "*") {
|
||||
result += " ";
|
||||
index += 2;
|
||||
while (index < text.length && !(text[index] === "*" && text[index + 1] === "/")) {
|
||||
result += blankCommentChar(text[index]);
|
||||
index += 1;
|
||||
}
|
||||
if (index < text.length) {
|
||||
result += " ";
|
||||
index += 1;
|
||||
}
|
||||
} else if (char === "/" && next === "/") {
|
||||
result += " ";
|
||||
index += 2;
|
||||
while (index < text.length && text[index] !== "\n" && text[index] !== "\r") {
|
||||
result += " ";
|
||||
index += 1;
|
||||
}
|
||||
index -= 1;
|
||||
} else {
|
||||
result += char;
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function blankCommentChar(char) {
|
||||
return char === "\n" || char === "\r" ? char : " ";
|
||||
}
|
||||
|
||||
function sortDetails(details) {
|
||||
|
||||
@ -23,17 +23,25 @@ export const knownIssueCodes = new Set([
|
||||
"package-build-artifact-entrypoint",
|
||||
"package-dependency-install-required",
|
||||
"package-entrypoint-missing",
|
||||
"package-install-metadata-incomplete",
|
||||
"package-json-missing",
|
||||
"package-manifest-version-drift",
|
||||
"package-min-host-version-drift",
|
||||
"package-npm-pack-entrypoint-missing",
|
||||
"package-npm-pack-metadata-missing",
|
||||
"package-npm-pack-unavailable",
|
||||
"package-openclaw-entry-missing",
|
||||
"package-openclaw-metadata-missing",
|
||||
"package-openclaw-unsupported-metadata",
|
||||
"package-plugin-api-compat-missing",
|
||||
"package-typescript-source-entrypoint",
|
||||
"provider-auth-env-vars",
|
||||
"registration-capture-gap",
|
||||
"runtime-tool-capture",
|
||||
"reserved-sdk-import",
|
||||
"security-manifest-schema-unavailable",
|
||||
"sdk-export-missing",
|
||||
"unrecognized-security-manifest",
|
||||
]);
|
||||
|
||||
export const issueMetadataByCode = {
|
||||
@ -85,6 +93,12 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "plugin imports reserved bundled-plugin SDK compatibility subpaths",
|
||||
},
|
||||
"security-manifest-schema-unavailable": {
|
||||
severity: "P3",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "plugin security manifest references an unavailable schema",
|
||||
},
|
||||
"missing-compat-record": {
|
||||
severity: "P1",
|
||||
owner: "core",
|
||||
@ -119,7 +133,7 @@ export const issueMetadataByCode = {
|
||||
severity: "P2",
|
||||
owner: "inspector",
|
||||
decision: "inspector-follow-up",
|
||||
title: "cold import requires isolated dependency installation",
|
||||
title: "cold import requires dependency installation in an isolated workspace",
|
||||
},
|
||||
"package-entrypoint-missing": {
|
||||
severity: "P1",
|
||||
@ -127,6 +141,12 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package entrypoint is missing",
|
||||
},
|
||||
"package-install-metadata-incomplete": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package install metadata is incomplete",
|
||||
},
|
||||
"package-json-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
@ -139,6 +159,30 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "package and manifest versions drift",
|
||||
},
|
||||
"package-min-host-version-drift": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package minimum host version drifts from build target",
|
||||
},
|
||||
"package-npm-pack-entrypoint-missing": {
|
||||
severity: "P1",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "advertised npm artifact is missing OpenClaw entrypoints",
|
||||
},
|
||||
"package-npm-pack-metadata-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "advertised npm artifact is missing OpenClaw metadata",
|
||||
},
|
||||
"package-npm-pack-unavailable": {
|
||||
severity: "P1",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "advertised npm artifact cannot be packed",
|
||||
},
|
||||
"package-openclaw-entry-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
@ -151,6 +195,12 @@ export const issueMetadataByCode = {
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "OpenClaw package metadata is missing",
|
||||
},
|
||||
"package-openclaw-unsupported-metadata": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "package declares unsupported OpenClaw metadata",
|
||||
},
|
||||
"package-plugin-api-compat-missing": {
|
||||
severity: "P2",
|
||||
owner: "plugin",
|
||||
@ -170,10 +220,10 @@ export const issueMetadataByCode = {
|
||||
title: "providerAuthEnvVars legacy manifest metadata must stay covered",
|
||||
},
|
||||
"registration-capture-gap": {
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
owner: "inspector",
|
||||
decision: "inspector-follow-up",
|
||||
title: "runtime registrations need capture before contract judgment",
|
||||
title: "runtime registrations need capture evidence before final contract judgment",
|
||||
},
|
||||
"runtime-tool-capture": {
|
||||
severity: "P2",
|
||||
@ -193,6 +243,12 @@ export const issueMetadataByCode = {
|
||||
decision: "core-compat-adapter",
|
||||
title: "fixture calls a registrar missing from target OpenClaw",
|
||||
},
|
||||
"unrecognized-security-manifest": {
|
||||
severity: "P3",
|
||||
owner: "plugin",
|
||||
decision: "plugin-upstream-fix",
|
||||
title: "plugin ships an unsupported security manifest",
|
||||
},
|
||||
};
|
||||
|
||||
export function buildIssues({ breakages = [], warnings = [], suggestions = [], targetOpenClaw, idPrefix = "CRABPOT" }) {
|
||||
@ -212,7 +268,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
|
||||
owner: finding.owner,
|
||||
code: finding.code,
|
||||
decision: finding.decision,
|
||||
status: finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open",
|
||||
status: finding.status ?? (finding.severity === "P0" || finding.level === "breakage" ? "blocking" : "open"),
|
||||
issueClass: finding.issueClass,
|
||||
live: finding.live,
|
||||
deprecated: finding.deprecated,
|
||||
@ -220,6 +276,7 @@ export function buildIssues({ breakages = [], warnings = [], suggestions = [], t
|
||||
title: issueTitle(finding),
|
||||
evidence: finding.evidence ?? [],
|
||||
compatRecord: finding.compatRecord ?? null,
|
||||
runtimeCoverage: finding.runtimeCoverage ?? null,
|
||||
}));
|
||||
}
|
||||
|
||||
@ -285,7 +342,10 @@ export function summarizeIssueClasses(issues) {
|
||||
}
|
||||
|
||||
function issueClassFor(code, options) {
|
||||
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)) {
|
||||
if (code === "sdk-export-missing" && options.compatRecord) {
|
||||
return "compat-gap";
|
||||
}
|
||||
if (["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)) {
|
||||
return "live-issue";
|
||||
}
|
||||
if (code === "missing-compat-record") {
|
||||
@ -314,10 +374,18 @@ function issueClassFor(code, options) {
|
||||
"manifest-unknown-fields",
|
||||
"package-json-missing",
|
||||
"package-manifest-version-drift",
|
||||
"package-min-host-version-drift",
|
||||
"package-npm-pack-entrypoint-missing",
|
||||
"package-npm-pack-metadata-missing",
|
||||
"package-npm-pack-unavailable",
|
||||
"package-openclaw-entry-missing",
|
||||
"package-openclaw-metadata-missing",
|
||||
"package-openclaw-unsupported-metadata",
|
||||
"package-plugin-api-compat-missing",
|
||||
"package-install-metadata-incomplete",
|
||||
"reserved-sdk-import",
|
||||
"security-manifest-schema-unavailable",
|
||||
"unrecognized-security-manifest",
|
||||
].includes(code)
|
||||
) {
|
||||
return "upstream-metadata";
|
||||
@ -332,7 +400,7 @@ function severityForClass(code, defaultSeverity, options) {
|
||||
if (
|
||||
options.issueClass === "live-issue" &&
|
||||
["none", "untracked"].includes(options.compatStatus) &&
|
||||
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing", "sdk-export-missing"].includes(code)
|
||||
["unknown-hook-name", "unknown-registration-name", "package-entrypoint-missing"].includes(code)
|
||||
) {
|
||||
return "P0";
|
||||
}
|
||||
|
||||
@ -1,10 +1,12 @@
|
||||
#!/usr/bin/env node
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { rmSync } from "node:fs";
|
||||
import { mkdtemp } from "node:fs/promises";
|
||||
import { register } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { createCaptureApi } from "./capture-api.js";
|
||||
import { captureApiOptionsForPlugin } from "./capture-config.js";
|
||||
import { createMockSdkPackage } from "./sdk-mock.js";
|
||||
|
||||
const options = JSON.parse(process.argv[2] ?? "{}");
|
||||
@ -26,13 +28,16 @@ async function run(options) {
|
||||
const pluginRoot = path.resolve(options.cwd ?? process.cwd(), options.pluginRoot ?? path.dirname(entrypoint));
|
||||
const workspace = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-sdk-"));
|
||||
|
||||
try {
|
||||
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
|
||||
register(pathToFileURL(loaderPath));
|
||||
return await captureLinkedEntrypoint(entrypoint, options);
|
||||
} finally {
|
||||
await rm(workspace, { force: true, recursive: true });
|
||||
}
|
||||
cleanupTempDirOnExit(workspace);
|
||||
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
|
||||
register(pathToFileURL(loaderPath));
|
||||
return await captureLinkedEntrypoint(entrypoint, { ...options, pluginRoot });
|
||||
}
|
||||
|
||||
function cleanupTempDirOnExit(dir) {
|
||||
process.once("exit", () => {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
});
|
||||
}
|
||||
|
||||
async function captureLinkedEntrypoint(entrypoint, options) {
|
||||
@ -61,7 +66,10 @@ async function captureLinkedEntrypoint(entrypoint, options) {
|
||||
);
|
||||
}
|
||||
|
||||
const api = createCaptureApi(options.apiOptions);
|
||||
const apiOptions = await captureApiOptionsForPlugin(options.apiOptions, {
|
||||
pluginRoot: options.pluginRoot,
|
||||
});
|
||||
const api = createCaptureApi(apiOptions);
|
||||
try {
|
||||
await register(api);
|
||||
} catch (error) {
|
||||
@ -76,7 +84,7 @@ async function captureLinkedEntrypoint(entrypoint, options) {
|
||||
mockSdk: true,
|
||||
captured: api.getCapturedContracts(),
|
||||
};
|
||||
if (options.apiOptions?.retainHandlers === true) {
|
||||
if (apiOptions?.retainHandlers === true) {
|
||||
result.retained = api.getRetainedContracts();
|
||||
}
|
||||
return withProcessOutput(result, outputCapture);
|
||||
|
||||
@ -106,12 +106,89 @@ export function openClawTargetPathCandidates(manifest, configuredPath) {
|
||||
|
||||
export function parseCompatRecordEntries(source) {
|
||||
const entries = [];
|
||||
for (const match of source.matchAll(/\{[\s\S]*?\bcode:\s*["'`]([^"'`]+)["'`][\s\S]*?\bstatus:\s*["'`]([^"'`]+)["'`][\s\S]*?\}/g)) {
|
||||
entries.push({ code: match[1], status: match[2] });
|
||||
let cursor = 0;
|
||||
while (cursor < source.length) {
|
||||
const codeProperty = readStringProperty(source, "code", cursor);
|
||||
if (!codeProperty) {
|
||||
break;
|
||||
}
|
||||
|
||||
const statusProperty = readStringProperty(source, "status", codeProperty.end);
|
||||
if (statusProperty) {
|
||||
entries.push({ code: codeProperty.value, status: statusProperty.value });
|
||||
cursor = statusProperty.end;
|
||||
} else {
|
||||
cursor = codeProperty.end;
|
||||
}
|
||||
}
|
||||
return dedupeBy(entries, (entry) => entry.code).sort((left, right) => left.code.localeCompare(right.code));
|
||||
}
|
||||
|
||||
function readStringProperty(source, property, fromIndex) {
|
||||
const propertyIndex = findProperty(source, property, fromIndex);
|
||||
if (propertyIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
const colonIndex = source.indexOf(":", propertyIndex + property.length);
|
||||
if (colonIndex === -1) {
|
||||
return null;
|
||||
}
|
||||
let quoteIndex = colonIndex + 1;
|
||||
while (quoteIndex < source.length && isWhitespace(source[quoteIndex])) {
|
||||
quoteIndex += 1;
|
||||
}
|
||||
if (!isQuote(source[quoteIndex])) {
|
||||
return null;
|
||||
}
|
||||
return readQuotedValue(source, quoteIndex);
|
||||
}
|
||||
|
||||
function findProperty(source, property, fromIndex) {
|
||||
let index = source.indexOf(property, fromIndex);
|
||||
while (index !== -1) {
|
||||
const previous = index === 0 ? "" : source[index - 1];
|
||||
const next = source[index + property.length] ?? "";
|
||||
if (!isIdentifierChar(previous) && !isIdentifierChar(next)) {
|
||||
return index;
|
||||
}
|
||||
index = source.indexOf(property, index + property.length);
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
function readQuotedValue(source, quoteIndex) {
|
||||
const quote = source[quoteIndex];
|
||||
let value = "";
|
||||
for (let index = quoteIndex + 1; index < source.length; index += 1) {
|
||||
const char = source[index];
|
||||
if (char === "\\") {
|
||||
value += source[index + 1] ?? "";
|
||||
index += 1;
|
||||
} else if (char === quote) {
|
||||
return { value, end: index + 1 };
|
||||
} else {
|
||||
value += char;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function isQuote(char) {
|
||||
return char === '"' || char === "'" || char === "`";
|
||||
}
|
||||
|
||||
function isIdentifierChar(char) {
|
||||
if (char === "_" || char === "$") {
|
||||
return true;
|
||||
}
|
||||
const code = char.charCodeAt(0);
|
||||
return (code >= 48 && code <= 57) || (code >= 65 && code <= 90) || (code >= 97 && code <= 122);
|
||||
}
|
||||
|
||||
function isWhitespace(char) {
|
||||
return char === " " || char === "\n" || char === "\r" || char === "\t";
|
||||
}
|
||||
|
||||
export function parsePluginSdkExports(packageJson) {
|
||||
return Object.keys(packageJson.exports ?? {})
|
||||
.filter((specifier) => specifier === "./plugin-sdk" || specifier.startsWith("./plugin-sdk/"))
|
||||
|
||||
@ -12,13 +12,11 @@ export function buildPlatformProbes(options = {}) {
|
||||
const entrypoints = plan.fixtures.flatMap((fixture) =>
|
||||
fixture.entrypoints.map((entrypoint) => summarizeEntrypoint(fixture.id, entrypoint)),
|
||||
);
|
||||
const portabilityFindings = plan.fixtures.flatMap((fixture) =>
|
||||
fixture.entrypoints.flatMap((entrypoint) =>
|
||||
entrypoint.steps
|
||||
.map((step) => summarizeStep(fixture.id, entrypoint, step))
|
||||
.filter((finding) => finding.riskCodes.length > 0),
|
||||
),
|
||||
const stepFindings = plan.fixtures.flatMap((fixture) =>
|
||||
fixture.entrypoints.flatMap((entrypoint) => entrypoint.steps.map((step) => summarizeStep(fixture.id, entrypoint, step, options.stepCoverage))),
|
||||
);
|
||||
const portabilityFindings = stepFindings.flatMap((finding) => (finding.residual ? [finding.residual] : []));
|
||||
const coveredPortabilityFindings = stepFindings.flatMap((finding) => (finding.covered ? [finding.covered] : []));
|
||||
|
||||
return {
|
||||
generatedAt: plan.generatedAt,
|
||||
@ -31,6 +29,7 @@ export function buildPlatformProbes(options = {}) {
|
||||
jitiAlternativeCount: entrypoints.filter((entrypoint) => entrypoint.loaderAlternatives.includes("jiti")).length,
|
||||
lazyImportProbeCount: entrypoints.filter((entrypoint) => entrypoint.capturePlanned && entrypoint.syntheticProbePlanned).length,
|
||||
portabilityFindingCount: portabilityFindings.length,
|
||||
coveredPortabilityFindingCount: coveredPortabilityFindings.length,
|
||||
windowsRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("windows")).length,
|
||||
macosRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("macos")).length,
|
||||
linuxRiskStepCount: portabilityFindings.filter((finding) => finding.platforms.includes("linux")).length,
|
||||
@ -38,6 +37,7 @@ export function buildPlatformProbes(options = {}) {
|
||||
},
|
||||
entrypoints,
|
||||
portabilityFindings,
|
||||
coveredPortabilityFindings,
|
||||
recommendations: buildRecommendations(portabilityFindings, entrypoints),
|
||||
};
|
||||
}
|
||||
@ -55,8 +55,11 @@ export function validatePlatformProbes(report, options = {}) {
|
||||
errors.push("all TypeScript loader entrypoints must track a Jiti fallback candidate");
|
||||
}
|
||||
for (const entrypoint of report.entrypoints) {
|
||||
if (entrypoint.loaderPrimary === "tsx" && (!entrypoint.captureUsesTsx || !entrypoint.syntheticUsesTsx)) {
|
||||
errors.push(`${entrypoint.id}: tsx loader strategy is not reflected in capture and synthetic commands`);
|
||||
if (
|
||||
entrypoint.loaderPrimary === "tsx" &&
|
||||
(!entrypoint.captureUsesTypeScriptLoader || !entrypoint.syntheticUsesTypeScriptLoader)
|
||||
) {
|
||||
errors.push(`${entrypoint.id}: TypeScript loader strategy is not reflected in capture and synthetic commands`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
@ -94,9 +97,21 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
||||
entrypoint.loaderAlternatives.join(", ") || "-",
|
||||
entrypoint.captureUsesTsx ? "yes" : "no",
|
||||
entrypoint.syntheticUsesTsx ? "yes" : "no",
|
||||
entrypoint.captureUsesMockSdk ? "yes" : "no",
|
||||
entrypoint.syntheticUsesMockSdk ? "yes" : "no",
|
||||
entrypoint.entrypoint,
|
||||
]),
|
||||
["Fixture", "Status", "Primary", "Alternatives", "Capture TSX", "Synthetic TSX", "Entrypoint"],
|
||||
[
|
||||
"Fixture",
|
||||
"Status",
|
||||
"Primary",
|
||||
"Alternatives",
|
||||
"Capture TSX",
|
||||
"Synthetic TSX",
|
||||
"Capture Mock SDK",
|
||||
"Synthetic Mock SDK",
|
||||
"Entrypoint",
|
||||
],
|
||||
),
|
||||
"",
|
||||
"## Portability Findings",
|
||||
@ -112,6 +127,19 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
||||
["Fixture", "Step", "Platforms", "Risks", "Mitigation"],
|
||||
),
|
||||
"",
|
||||
"## Covered Portability Findings",
|
||||
"",
|
||||
markdownTable(
|
||||
(report.coveredPortabilityFindings ?? []).map((finding) => [
|
||||
finding.fixture,
|
||||
finding.kind,
|
||||
finding.platforms.join(", ") || "-",
|
||||
finding.riskCodes.join(", "),
|
||||
finding.coverage,
|
||||
]),
|
||||
["Fixture", "Step", "Platforms", "Covered Risks", "Coverage"],
|
||||
),
|
||||
"",
|
||||
"## Recommendations",
|
||||
"",
|
||||
markdownTable(
|
||||
@ -124,6 +152,10 @@ export function renderPlatformProbesMarkdown(report, options = {}) {
|
||||
function summarizeEntrypoint(fixtureId, entrypoint) {
|
||||
const captureStep = entrypoint.steps.find((step) => step.kind === "capture");
|
||||
const syntheticStep = entrypoint.steps.find((step) => step.kind === "synthetic-probe");
|
||||
const captureUsesTsx = Boolean(captureStep?.command.includes("--import tsx"));
|
||||
const syntheticUsesTsx = Boolean(syntheticStep?.command.includes("--import tsx"));
|
||||
const captureUsesMockSdk = Boolean(captureStep?.command.includes("--mock-sdk"));
|
||||
const syntheticUsesMockSdk = Boolean(syntheticStep?.command.includes("--mock-sdk"));
|
||||
return {
|
||||
fixture: fixtureId,
|
||||
id: entrypoint.id,
|
||||
@ -135,21 +167,57 @@ function summarizeEntrypoint(fixtureId, entrypoint) {
|
||||
loaderAlternatives: entrypoint.loaderStrategy.alternatives,
|
||||
capturePlanned: Boolean(captureStep),
|
||||
syntheticProbePlanned: Boolean(syntheticStep),
|
||||
captureUsesTsx: Boolean(captureStep?.command.includes("--import tsx")),
|
||||
syntheticUsesTsx: Boolean(syntheticStep?.command.includes("--import tsx")),
|
||||
captureUsesTsx,
|
||||
syntheticUsesTsx,
|
||||
captureUsesMockSdk,
|
||||
syntheticUsesMockSdk,
|
||||
captureUsesTypeScriptLoader: captureUsesTsx || captureUsesMockSdk,
|
||||
syntheticUsesTypeScriptLoader: syntheticUsesTsx || syntheticUsesMockSdk,
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeStep(fixtureId, entrypoint, step) {
|
||||
function summarizeStep(fixtureId, entrypoint, step, stepCoverage) {
|
||||
const riskCodes = stepRiskCodes(step);
|
||||
return {
|
||||
const coverage = normalizeStepCoverage(stepCoverage?.({ fixture: fixtureId, entrypoint, step, riskCodes }), riskCodes);
|
||||
const residualRiskCodes = riskCodes.filter((code) => !coverage.riskCodes.includes(code));
|
||||
const common = {
|
||||
fixture: fixtureId,
|
||||
entrypoint: entrypoint.id,
|
||||
kind: step.kind,
|
||||
platforms: platformsForRiskCodes(riskCodes),
|
||||
riskCodes,
|
||||
command: step.command,
|
||||
mitigation: mitigationForRiskCodes(riskCodes),
|
||||
};
|
||||
return {
|
||||
residual:
|
||||
residualRiskCodes.length > 0
|
||||
? {
|
||||
...common,
|
||||
coveredRiskCodes: coverage.riskCodes,
|
||||
platforms: platformsForRiskCodes(residualRiskCodes),
|
||||
riskCodes: residualRiskCodes,
|
||||
mitigation: mitigationForRiskCodes(residualRiskCodes),
|
||||
}
|
||||
: null,
|
||||
covered:
|
||||
coverage.riskCodes.length > 0
|
||||
? {
|
||||
...common,
|
||||
coverage: coverage.reason,
|
||||
platforms: platformsForRiskCodes(coverage.riskCodes),
|
||||
riskCodes: coverage.riskCodes,
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeStepCoverage(coverage, riskCodes) {
|
||||
if (!coverage) {
|
||||
return { reason: "", riskCodes: [] };
|
||||
}
|
||||
const requested = coverage === true ? riskCodes : coverage.coveredRiskCodes ?? coverage.handledRiskCodes ?? coverage.riskCodes ?? coverage;
|
||||
const covered = Array.isArray(requested) ? requested : [];
|
||||
return {
|
||||
reason: typeof coverage.reason === "string" && coverage.reason ? coverage.reason : "covered by isolated workspace executor",
|
||||
riskCodes: covered.filter((code) => riskCodes.includes(code)).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
@ -215,7 +283,7 @@ function buildRecommendations(portabilityFindings, entrypoints) {
|
||||
if (entrypoints.some((entrypoint) => entrypoint.loaderPrimary === "tsx")) {
|
||||
recommendations.push({
|
||||
area: "loader",
|
||||
action: "keep tsx as the source-entrypoint smoke path, add a Jiti execution lane before treating TS plugin source compatibility as covered",
|
||||
action: "keep mock-SDK TypeScript capture green, add a real host-loader/Jiti lane before treating TS plugin source compatibility as covered",
|
||||
});
|
||||
}
|
||||
if (portabilityFindings.some((finding) => finding.riskCodes.includes("rsync-required"))) {
|
||||
|
||||
@ -7,8 +7,12 @@ export async function runProfiledProcess(options) {
|
||||
let firstRssKb = 0;
|
||||
let peakRssKb = 0;
|
||||
let peakCpuPercent = 0;
|
||||
let statSampleCount = 0;
|
||||
let rssSampleCount = 0;
|
||||
let cpuSampleCount = 0;
|
||||
const cpuSamples = [];
|
||||
let pollInFlight = false;
|
||||
const pendingStats = new Set();
|
||||
|
||||
const child = spawn(options.command, options.args ?? [], {
|
||||
cwd: options.cwd,
|
||||
@ -21,27 +25,43 @@ export async function runProfiledProcess(options) {
|
||||
child.stderr?.on("data", (chunk) => stderr.push(chunk));
|
||||
|
||||
const recordStats = (stats) => {
|
||||
if (stats.rssKb > 0 && firstRssKb === 0) {
|
||||
if (stats.rssAvailable || stats.cpuAvailable) {
|
||||
statSampleCount += 1;
|
||||
}
|
||||
if (stats.rssAvailable) {
|
||||
rssSampleCount += 1;
|
||||
}
|
||||
if (stats.cpuAvailable) {
|
||||
cpuSampleCount += 1;
|
||||
}
|
||||
if (stats.rssAvailable && stats.rssKb > 0 && firstRssKb === 0) {
|
||||
firstRssKb = stats.rssKb;
|
||||
}
|
||||
peakRssKb = Math.max(peakRssKb, stats.rssKb);
|
||||
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
|
||||
if (stats.cpuPercent > 0) {
|
||||
if (stats.rssAvailable) {
|
||||
peakRssKb = Math.max(peakRssKb, stats.rssKb);
|
||||
}
|
||||
if (stats.cpuAvailable) {
|
||||
peakCpuPercent = Math.max(peakCpuPercent, stats.cpuPercent);
|
||||
cpuSamples.push(stats.cpuPercent);
|
||||
}
|
||||
};
|
||||
|
||||
const poll = setInterval(() => {
|
||||
const sampleStats = () => {
|
||||
if (pollInFlight) {
|
||||
return;
|
||||
}
|
||||
pollInFlight = true;
|
||||
readProcessStats(child.pid)
|
||||
const pending = readProcessStats(child.pid)
|
||||
.then(recordStats)
|
||||
.finally(() => {
|
||||
pollInFlight = false;
|
||||
pendingStats.delete(pending);
|
||||
});
|
||||
}, options.pollMs ?? 100);
|
||||
pendingStats.add(pending);
|
||||
};
|
||||
|
||||
sampleStats();
|
||||
const poll = setInterval(sampleStats, options.pollMs ?? 25);
|
||||
|
||||
const exitCode = await new Promise((resolve, reject) => {
|
||||
child.on("error", (error) => {
|
||||
@ -51,16 +71,10 @@ export async function runProfiledProcess(options) {
|
||||
child.on("exit", (code) => resolve(code ?? 1));
|
||||
});
|
||||
clearInterval(poll);
|
||||
await Promise.allSettled([...pendingStats]);
|
||||
|
||||
const finalStats = await readProcessStats(child.pid);
|
||||
if (finalStats.rssKb > 0 && firstRssKb === 0) {
|
||||
firstRssKb = finalStats.rssKb;
|
||||
}
|
||||
peakRssKb = Math.max(peakRssKb, finalStats.rssKb);
|
||||
peakCpuPercent = Math.max(peakCpuPercent, finalStats.cpuPercent);
|
||||
if (finalStats.cpuPercent > 0) {
|
||||
cpuSamples.push(finalStats.cpuPercent);
|
||||
}
|
||||
recordStats(finalStats);
|
||||
|
||||
const wallMs = Math.round(performance.now() - start);
|
||||
const averageCpuPercent =
|
||||
@ -79,6 +93,9 @@ export async function runProfiledProcess(options) {
|
||||
peakCpuPercent: Math.round(peakCpuPercent * 10) / 10,
|
||||
cpuMsEstimate: Math.round((wallMs * cpuPercentForEstimate) / 100),
|
||||
harnessHeapDeltaMb: Math.round((heapUsedMb() - heapStartMb) * 10) / 10,
|
||||
statSampleCount,
|
||||
rssSampleCount,
|
||||
cpuSampleCount,
|
||||
exitCode,
|
||||
stdoutPreview: previewLines(stdout),
|
||||
stderrPreview: previewLines(stderr),
|
||||
@ -87,7 +104,7 @@ export async function runProfiledProcess(options) {
|
||||
|
||||
async function readProcessStats(pid) {
|
||||
if (!pid || process.platform === "win32") {
|
||||
return { rssKb: 0, cpuPercent: 0 };
|
||||
return { rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 };
|
||||
}
|
||||
return new Promise((resolve) => {
|
||||
const ps = spawn("ps", ["-o", "rss=", "-o", "%cpu=", "-p", String(pid)], {
|
||||
@ -95,14 +112,18 @@ async function readProcessStats(pid) {
|
||||
});
|
||||
const chunks = [];
|
||||
ps.stdout.on("data", (chunk) => chunks.push(chunk));
|
||||
ps.on("error", () => resolve({ rssKb: 0, cpuPercent: 0 }));
|
||||
ps.on("error", () => resolve({ rssAvailable: false, rssKb: 0, cpuAvailable: false, cpuPercent: 0 }));
|
||||
ps.on("exit", () => {
|
||||
const [rssRaw, cpuRaw] = Buffer.concat(chunks).toString("utf8").trim().split(/\s+/);
|
||||
const rssKb = Number.parseInt(rssRaw, 10);
|
||||
const cpuPercent = Number.parseFloat(cpuRaw);
|
||||
const rssAvailable = Number.isFinite(rssKb);
|
||||
const cpuAvailable = Number.isFinite(cpuPercent);
|
||||
resolve({
|
||||
rssKb: Number.isFinite(rssKb) ? rssKb : 0,
|
||||
cpuPercent: Number.isFinite(cpuPercent) ? cpuPercent : 0,
|
||||
rssAvailable,
|
||||
rssKb: rssAvailable ? rssKb : 0,
|
||||
cpuAvailable,
|
||||
cpuPercent: cpuAvailable ? cpuPercent : 0,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
23
src/prune-workspace-dev-deps-cli.js
Normal file
23
src/prune-workspace-dev-deps-cli.js
Normal file
@ -0,0 +1,23 @@
|
||||
#!/usr/bin/env node
|
||||
import { readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const packageJsonPath = path.resolve(process.cwd(), "package.json");
|
||||
const packageJson = JSON.parse(await readFile(packageJsonPath, "utf8"));
|
||||
let changed = false;
|
||||
|
||||
for (const [name, specifier] of Object.entries(packageJson.devDependencies ?? {})) {
|
||||
if (typeof specifier === "string" && specifier.startsWith("workspace:")) {
|
||||
delete packageJson.devDependencies[name];
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (packageJson.devDependencies && Object.keys(packageJson.devDependencies).length === 0) {
|
||||
delete packageJson.devDependencies;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
await writeFile(packageJsonPath, `${JSON.stringify(packageJson, null, 2)}\n`, "utf8");
|
||||
}
|
||||
42
src/report-sanitizer.js
Normal file
42
src/report-sanitizer.js
Normal file
@ -0,0 +1,42 @@
|
||||
import path from "node:path";
|
||||
|
||||
export function sanitizeReportArtifact(report, options = {}) {
|
||||
const sensitivePaths = sensitiveOpenClawPaths(report);
|
||||
if (sensitivePaths.length === 0) {
|
||||
return report;
|
||||
}
|
||||
const placeholder = options.openclawPathPlaceholder ?? "<OPENCLAW_PATH>";
|
||||
return sanitizeValue(report, sensitivePaths, placeholder);
|
||||
}
|
||||
|
||||
function sensitiveOpenClawPaths(report) {
|
||||
const targetOpenClaw = report?.targetOpenClaw;
|
||||
return unique(
|
||||
[targetOpenClaw?.configuredPath, ...(targetOpenClaw?.searchedPaths ?? [])]
|
||||
.filter((value) => typeof value === "string" && isAbsolutePath(value))
|
||||
.sort((left, right) => right.length - left.length),
|
||||
);
|
||||
}
|
||||
|
||||
function sanitizeValue(value, sensitivePaths, placeholder) {
|
||||
if (typeof value === "string") {
|
||||
return sensitivePaths.reduce((result, sensitivePath) => result.replaceAll(sensitivePath, placeholder), value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((item) => sanitizeValue(item, sensitivePaths, placeholder));
|
||||
}
|
||||
if (value && typeof value === "object") {
|
||||
return Object.fromEntries(
|
||||
Object.entries(value).map(([key, entryValue]) => [key, sanitizeValue(entryValue, sensitivePaths, placeholder)]),
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function isAbsolutePath(value) {
|
||||
return path.isAbsolute(value) || /^[A-Za-z]:[\\/]/u.test(value) || value.startsWith("\\\\");
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
101
src/report.js
101
src/report.js
@ -4,6 +4,8 @@ import { renderCompatibilityIssuesReport, renderCompatibilityMarkdownReport } fr
|
||||
import { buildContractProbes } from "./contract-probes.js";
|
||||
import { classifyCompatibilityFixture } from "./fixture-summary.js";
|
||||
import { buildIssues, summarizeIssueClasses } from "./issues.js";
|
||||
import { sanitizeReportArtifact } from "./report-sanitizer.js";
|
||||
import { applyRuntimeExecutionCoverage } from "./runtime-reconciliation.js";
|
||||
|
||||
export function buildReport({ config, inspections, failures = [], generatedAt = "deterministic" }) {
|
||||
const inspectionById = new Map(inspections.map((inspection) => [inspection.id, inspection]));
|
||||
@ -139,6 +141,10 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
decisions,
|
||||
});
|
||||
|
||||
const runtimeCoverage = applyRuntimeExecutionCoverage({
|
||||
findings: [...warnings, ...suggestions],
|
||||
executionResults: options.executionResults,
|
||||
});
|
||||
const issues = buildIssues({
|
||||
breakages,
|
||||
warnings,
|
||||
@ -148,6 +154,8 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
});
|
||||
const contractProbes = buildContractProbes({ warnings, suggestions, fixtures: fixtureReports });
|
||||
const issueSummary = summarizeIssueClasses(issues);
|
||||
const openIssues = issues.filter((issue) => issue.status !== "runtime-covered");
|
||||
const openIssueSummary = summarizeIssueClasses(openIssues);
|
||||
|
||||
return {
|
||||
generatedAt: options.generatedAt ?? "deterministic",
|
||||
@ -162,8 +170,11 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
decisionCount: decisions.length,
|
||||
logCount: logs.length,
|
||||
issueCount: issues.length,
|
||||
openIssueCount: openIssues.length,
|
||||
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
|
||||
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
|
||||
openP0IssueCount: openIssues.filter((issue) => issue.severity === "P0").length,
|
||||
openP1IssueCount: openIssues.filter((issue) => issue.severity === "P1").length,
|
||||
liveIssueCount: issueSummary["live-issue"],
|
||||
liveP0IssueCount: issues.filter((issue) => issue.issueClass === "live-issue" && issue.severity === "P0").length,
|
||||
compatGapCount: issueSummary["compat-gap"],
|
||||
@ -171,6 +182,10 @@ export async function buildCompatibilityReport(options = {}) {
|
||||
inspectorGapCount: issueSummary["inspector-gap"],
|
||||
upstreamIssueCount: issueSummary["upstream-metadata"],
|
||||
fixtureRegressionCount: issueSummary["fixture-regression"],
|
||||
openInspectorGapCount: openIssueSummary["inspector-gap"],
|
||||
runtimeCoveredIssueCount: runtimeCoverage.coveredFindingCount,
|
||||
runtimePartiallyCoveredIssueCount: runtimeCoverage.partiallyCoveredFindingCount,
|
||||
runtimeCoverageArtifactCount: runtimeCoverage.coverage.artifactCount,
|
||||
contractProbeCount: contractProbes.length,
|
||||
},
|
||||
fixtures: fixtureReports,
|
||||
@ -210,6 +225,10 @@ export function classifyCompatRecordCoverage({ targetOpenClaw, findings, suggest
|
||||
continue;
|
||||
}
|
||||
|
||||
if (finding.code === "sdk-export-missing") {
|
||||
continue;
|
||||
}
|
||||
|
||||
suggestions.push({
|
||||
fixture: finding.fixture,
|
||||
code: "missing-compat-record",
|
||||
@ -233,12 +252,13 @@ export async function writeReport(report, options = {}) {
|
||||
const basename = options.basename ?? "plugin-inspector-report";
|
||||
const jsonPath = path.join(outDir, `${basename}.json`);
|
||||
const markdownPath = path.join(outDir, `${basename}.md`);
|
||||
const artifactReport = sanitizeReportArtifact(report, options);
|
||||
|
||||
return writeJsonMarkdownArtifacts({
|
||||
jsonPath,
|
||||
markdownPath,
|
||||
json: report,
|
||||
markdown: renderMarkdownReport(report),
|
||||
json: artifactReport,
|
||||
markdown: renderMarkdownReport(artifactReport),
|
||||
check: options.check,
|
||||
});
|
||||
}
|
||||
@ -246,27 +266,86 @@ export async function writeReport(report, options = {}) {
|
||||
export async function writeCompatibilityReport(report, options = {}) {
|
||||
const outDir = path.resolve(options.cwd ?? process.cwd(), options.outDir ?? "reports");
|
||||
const basename = options.basename ?? "plugin-inspector-report";
|
||||
const jsonPath = path.join(outDir, `${basename}.json`);
|
||||
const markdownPath = path.join(outDir, `${basename}.md`);
|
||||
const issuesPath = path.join(outDir, options.issuesBasename ?? "plugin-inspector-issues.md");
|
||||
const jsonPath = options.jsonPath ?? path.join(outDir, `${basename}.json`);
|
||||
const markdownPath = options.markdownPath ?? path.join(outDir, `${basename}.md`);
|
||||
const issuesPath = options.issuesPath ?? path.join(outDir, options.issuesBasename ?? "plugin-inspector-issues.md");
|
||||
const artifactReport = sanitizeReportArtifact(report, options);
|
||||
const markdownOptions = compatibilityRenderOptions(options, {
|
||||
title: options.markdownTitle ?? options.title,
|
||||
...options.markdownOptions,
|
||||
});
|
||||
const issuesOptions = compatibilityRenderOptions(options, {
|
||||
title: options.issuesTitle ?? options.title,
|
||||
...options.issuesOptions,
|
||||
});
|
||||
|
||||
return writeArtifacts(
|
||||
[
|
||||
{ name: "jsonPath", path: jsonPath, json: report },
|
||||
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(report) },
|
||||
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(report) },
|
||||
{ name: "jsonPath", path: jsonPath, json: artifactReport },
|
||||
{ name: "markdownPath", path: markdownPath, markdown: renderCompatibilityMarkdownReport(artifactReport, markdownOptions) },
|
||||
{ name: "issuesPath", path: issuesPath, markdown: renderCompatibilityIssuesReport(artifactReport, issuesOptions) },
|
||||
],
|
||||
{ check: options.check },
|
||||
);
|
||||
}
|
||||
|
||||
export function renderTextSummary(report) {
|
||||
return [
|
||||
export { sanitizeReportArtifact };
|
||||
|
||||
function compatibilityRenderOptions(options, overrides) {
|
||||
const renderOptions = {
|
||||
formatEvidence: options.formatEvidence,
|
||||
severityLabels: options.severityLabels,
|
||||
...overrides,
|
||||
};
|
||||
return Object.fromEntries(Object.entries(renderOptions).filter(([, value]) => value !== undefined));
|
||||
}
|
||||
|
||||
export function renderTextSummary(report, options = {}) {
|
||||
const lines = [
|
||||
`Status: ${report.status.toUpperCase()}`,
|
||||
`Fixtures: ${report.summary.fixtureCount}`,
|
||||
`Breakages: ${report.summary.breakageCount}`,
|
||||
...(typeof report.summary.issueCount === "number" ? [`Issues: ${report.summary.issueCount}`] : []),
|
||||
`Logs: ${report.summary.logCount}`,
|
||||
].join("\n");
|
||||
];
|
||||
const artifacts = Object.entries(options.artifacts ?? {}).filter(([, filePath]) => Boolean(filePath));
|
||||
if (artifacts.length > 0) {
|
||||
lines.push("", "Reports:", ...artifacts.map(([name, filePath]) => `- ${artifactLabel(name)}: ${filePath}`));
|
||||
}
|
||||
const findings = topTextFindings(report, options.topFindings ?? 3);
|
||||
if (findings.length > 0) {
|
||||
lines.push("", "Top findings:", ...findings.map((finding) => `- ${finding}`));
|
||||
}
|
||||
return lines.join("\n");
|
||||
}
|
||||
|
||||
function topTextFindings(report, limit) {
|
||||
if (report.status === "pass" || limit <= 0) {
|
||||
return [];
|
||||
}
|
||||
return [
|
||||
...(report.breakages ?? []).map((finding) => formatTextFinding(finding, "breakage")),
|
||||
...(report.issues ?? [])
|
||||
.filter(
|
||||
(issue) =>
|
||||
issue.status !== "runtime-covered" &&
|
||||
(issue.status === "blocking" || issue.severity === "P0" || issue.severity === "P1"),
|
||||
)
|
||||
.map((issue) => formatTextFinding(issue, issue.severity ?? "issue")),
|
||||
...(report.warnings ?? []).map((finding) => formatTextFinding(finding, "warning")),
|
||||
].slice(0, limit);
|
||||
}
|
||||
|
||||
function formatTextFinding(finding, fallbackLevel) {
|
||||
const level = finding.level ?? finding.severity ?? fallbackLevel;
|
||||
const code = finding.code ? ` ${finding.code}` : "";
|
||||
const message = finding.message ?? finding.title ?? "see report";
|
||||
const evidence = Array.isArray(finding.evidence) && finding.evidence.length > 0 ? ` (${finding.evidence[0]})` : "";
|
||||
return `${String(level).toUpperCase()} ${finding.fixture ?? "unknown"}${code}: ${message}${evidence}`;
|
||||
}
|
||||
|
||||
function artifactLabel(name) {
|
||||
return String(name).replace(/Path$/u, "");
|
||||
}
|
||||
|
||||
export function renderMarkdownReport(report) {
|
||||
|
||||
@ -46,8 +46,8 @@ export async function buildRuntimeProfile(options = {}) {
|
||||
os: process.platform,
|
||||
arch: process.arch,
|
||||
node: process.version,
|
||||
rssSampler: process.platform === "win32" ? "unavailable" : "ps",
|
||||
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent",
|
||||
rssSampler: process.platform === "win32" ? "unavailable" : "ps-immediate-25ms",
|
||||
cpuSampler: process.platform === "win32" ? "unavailable" : "ps-percent-immediate-25ms",
|
||||
},
|
||||
summary: summarizeProfile(commands),
|
||||
groups: summarizeCommandGroups(commands),
|
||||
@ -65,8 +65,8 @@ export function validateRuntimeProfile(profile) {
|
||||
errors.push(`${command.id}: missing wall time`);
|
||||
}
|
||||
}
|
||||
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => command.peakRssMb.max <= 0)) {
|
||||
errors.push("all commands are missing peak RSS");
|
||||
if (profile.platform?.rssSampler !== "unavailable" && profile.commands.every((command) => !hasRssSample(command))) {
|
||||
errors.push("all commands are missing peak RSS samples");
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
@ -98,10 +98,17 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
||||
[
|
||||
["Commands", profile.summary.commandCount],
|
||||
["P50 wall time", `${profile.summary.p50WallMs} ms`],
|
||||
["P95 wall time", `${profile.summary.p95WallMs} ms`],
|
||||
["Max peak RSS", `${profile.summary.maxPeakRssMb} MB`],
|
||||
["Max RSS delta", `${profile.summary.maxRssDeltaMb} MB`],
|
||||
["Max CPU estimate", `${profile.summary.maxCpuMsEstimate} ms`],
|
||||
["Command P95 wall time", `${profile.summary.p95WallMs} ms`],
|
||||
["Wall time basis", profile.summary.wallTimeBasis ?? "command-median-p95"],
|
||||
["Profile samples", profile.summary.sampleCount ?? sampleCount(profile.commands)],
|
||||
["RSS samples", profile.summary.rssSampleCount ?? rssSampleCount(profile.commands)],
|
||||
["CPU samples", profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands)],
|
||||
["Max peak RSS", formatSampledMetric(profile.summary.maxPeakRssMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
|
||||
["Max RSS delta", formatSampledMetric(profile.summary.maxRssDeltaMb, profile.summary.rssSampleCount ?? rssSampleCount(profile.commands))],
|
||||
[
|
||||
"Max CPU estimate",
|
||||
formatSampledMetric(profile.summary.maxCpuMsEstimate, profile.summary.cpuSampleCount ?? cpuSampleCount(profile.commands), "ms"),
|
||||
],
|
||||
["Max harness heap delta", `${profile.summary.maxHarnessHeapDeltaMb} MB`],
|
||||
],
|
||||
["Metric", "Value"],
|
||||
@ -129,13 +136,14 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
||||
command.label,
|
||||
`${command.wallMs.median} ms`,
|
||||
`${command.wallMs.max} ms`,
|
||||
`${command.peakRssMb.max} MB`,
|
||||
`${command.rssDeltaMb.max} MB`,
|
||||
`${command.cpuMsEstimate.max} ms`,
|
||||
formatSampledMetric(command.peakRssMb.max, command.rssSampleCount),
|
||||
formatSampledMetric(command.rssDeltaMb.max, command.rssSampleCount),
|
||||
formatSampledMetric(command.cpuMsEstimate.max, command.cpuSampleCount, "ms"),
|
||||
`${command.harnessHeapDeltaMb.max} MB`,
|
||||
`${command.rssSampleCount ?? 0}/${command.cpuSampleCount ?? 0}`,
|
||||
command.exitCodes.join(", "),
|
||||
]),
|
||||
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "Exit codes"],
|
||||
["ID", "Label", "Median wall", "Max wall", "Max peak RSS", "Max RSS delta", "CPU estimate", "Heap delta", "RSS/CPU samples", "Exit codes"],
|
||||
),
|
||||
"",
|
||||
"## Category Rollups",
|
||||
@ -146,11 +154,12 @@ export function renderRuntimeProfileMarkdown(profile, options = {}) {
|
||||
group.commandCount,
|
||||
`${group.p50WallMs} ms`,
|
||||
`${group.p95WallMs} ms`,
|
||||
`${group.maxPeakRssMb} MB`,
|
||||
`${group.maxCpuMsEstimate} ms`,
|
||||
formatSampledMetric(group.maxPeakRssMb, group.rssSampleCount),
|
||||
formatSampledMetric(group.maxCpuMsEstimate, group.cpuSampleCount, "ms"),
|
||||
`${group.rssSampleCount ?? 0}/${group.cpuSampleCount ?? 0}`,
|
||||
group.commands.join(", "),
|
||||
]),
|
||||
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "Command IDs"],
|
||||
["Category", "Commands", "P50 wall", "P95 wall", "Max peak RSS", "CPU estimate", "RSS/CPU samples", "Command IDs"],
|
||||
),
|
||||
].join("\n");
|
||||
}
|
||||
@ -189,8 +198,15 @@ function summarizeProfile(commands) {
|
||||
const maxRssDeltaMb = Math.max(0, ...commands.map((command) => command.rssDeltaMb.max));
|
||||
const maxCpuMsEstimate = Math.max(0, ...commands.map((command) => command.cpuMsEstimate.max));
|
||||
const maxHarnessHeapDeltaMb = Math.max(0, ...commands.map((command) => command.harnessHeapDeltaMb.max));
|
||||
const totalSampleCount = sampleCount(commands);
|
||||
const totalRssSampleCount = rssSampleCount(commands);
|
||||
const totalCpuSampleCount = cpuSampleCount(commands);
|
||||
return {
|
||||
commandCount: commands.length,
|
||||
sampleCount: totalSampleCount,
|
||||
rssSampleCount: totalRssSampleCount,
|
||||
cpuSampleCount: totalCpuSampleCount,
|
||||
wallTimeBasis: "command-median-p95",
|
||||
p50WallMs: percentile(wallTimes, 0.5),
|
||||
p95WallMs: percentile(wallTimes, 0.95),
|
||||
maxPeakRssMb,
|
||||
@ -206,6 +222,12 @@ function summarizeCommand(command, samples) {
|
||||
const rssDeltaMb = samples.map((sample) => sample.rssDeltaMb).sort((left, right) => left - right);
|
||||
const peakCpuPercent = samples.map((sample) => sample.peakCpuPercent).sort((left, right) => left - right);
|
||||
const cpuMsEstimate = samples.map((sample) => sample.cpuMsEstimate).sort((left, right) => left - right);
|
||||
const statSampleCount = samples.reduce((sum, sample) => sum + (sample.statSampleCount ?? 0), 0);
|
||||
const rssSampleTotal = samples.reduce(
|
||||
(sum, sample) => sum + (sample.rssSampleCount ?? (sample.peakRssMb > 0 ? 1 : 0)),
|
||||
0,
|
||||
);
|
||||
const cpuSampleTotal = samples.reduce((sum, sample) => sum + (sample.cpuSampleCount ?? 0), 0);
|
||||
const harnessHeapDeltaMb = samples
|
||||
.map((sample) => sample.harnessHeapDeltaMb)
|
||||
.sort((left, right) => left - right);
|
||||
@ -222,6 +244,9 @@ function summarizeCommand(command, samples) {
|
||||
peakCpuPercent: summarizeNumbers(peakCpuPercent),
|
||||
cpuMsEstimate: summarizeNumbers(cpuMsEstimate),
|
||||
harnessHeapDeltaMb: summarizeNumbers(harnessHeapDeltaMb),
|
||||
statSampleCount,
|
||||
rssSampleCount: rssSampleTotal,
|
||||
cpuSampleCount: cpuSampleTotal,
|
||||
exitCodes: [...new Set(samples.map((sample) => sample.exitCode))].sort(),
|
||||
};
|
||||
}
|
||||
@ -244,6 +269,8 @@ function summarizeCommandGroups(commands) {
|
||||
const cpuMs = categoryCommands
|
||||
.flatMap((command) => command.samples.map((sample) => sample.cpuMsEstimate))
|
||||
.sort((left, right) => left - right);
|
||||
const groupRssSampleCount = rssSampleCount(categoryCommands);
|
||||
const groupCpuSampleCount = cpuSampleCount(categoryCommands);
|
||||
return {
|
||||
category,
|
||||
commandCount: categoryCommands.length,
|
||||
@ -251,11 +278,36 @@ function summarizeCommandGroups(commands) {
|
||||
p95WallMs: percentile(wallTimes, 0.95),
|
||||
maxPeakRssMb: peakRss.at(-1) ?? 0,
|
||||
maxCpuMsEstimate: cpuMs.at(-1) ?? 0,
|
||||
rssSampleCount: groupRssSampleCount,
|
||||
cpuSampleCount: groupCpuSampleCount,
|
||||
commands: categoryCommands.map((command) => command.id),
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
function hasRssSample(command) {
|
||||
return (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)) > 0;
|
||||
}
|
||||
|
||||
function sampleCount(commands) {
|
||||
return commands.reduce((sum, command) => sum + (command.samples?.length ?? 0), 0);
|
||||
}
|
||||
|
||||
function rssSampleCount(commands) {
|
||||
return commands.reduce((sum, command) => sum + (command.rssSampleCount ?? (command.peakRssMb?.max > 0 ? 1 : 0)), 0);
|
||||
}
|
||||
|
||||
function cpuSampleCount(commands) {
|
||||
return commands.reduce((sum, command) => sum + (command.cpuSampleCount ?? 0), 0);
|
||||
}
|
||||
|
||||
function formatSampledMetric(value, count, unit = "MB") {
|
||||
if ((count ?? 0) <= 0) {
|
||||
return "n/a";
|
||||
}
|
||||
return `${value} ${unit}`;
|
||||
}
|
||||
|
||||
function summarizeNumbers(values) {
|
||||
return {
|
||||
min: values[0],
|
||||
|
||||
124
src/runtime-reconciliation.js
Normal file
124
src/runtime-reconciliation.js
Normal file
@ -0,0 +1,124 @@
|
||||
export function applyRuntimeExecutionCoverage({ findings = [], executionResults } = {}) {
|
||||
const coverage = buildRuntimeExecutionCoverage(executionResults);
|
||||
let coveredFindingCount = 0;
|
||||
let partiallyCoveredFindingCount = 0;
|
||||
|
||||
for (const finding of findings) {
|
||||
const findingCoverage = runtimeCoverageForFinding(finding, coverage);
|
||||
if (!findingCoverage) {
|
||||
continue;
|
||||
}
|
||||
|
||||
finding.runtimeCoverage = findingCoverage;
|
||||
if (findingCoverage.status === "covered") {
|
||||
finding.status = "runtime-covered";
|
||||
coveredFindingCount += 1;
|
||||
} else {
|
||||
partiallyCoveredFindingCount += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
coverage,
|
||||
coveredFindingCount,
|
||||
partiallyCoveredFindingCount,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildRuntimeExecutionCoverage(executionResults) {
|
||||
const fixtures = new Map();
|
||||
for (const artifact of executionResults?.artifacts ?? []) {
|
||||
if (artifact.kind !== "capture") {
|
||||
continue;
|
||||
}
|
||||
|
||||
const fixture = String(artifact.fixture ?? "unknown");
|
||||
const fixtureCoverage = ensureFixtureCoverage(fixtures, fixture);
|
||||
if (artifact.artifactPath) {
|
||||
fixtureCoverage.artifacts.add(artifact.artifactPath);
|
||||
}
|
||||
|
||||
for (const captured of normalizeCaptured(artifact.captured)) {
|
||||
fixtureCoverage.captured.add(captured);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
fixtures,
|
||||
artifactCount: [...fixtures.values()].reduce((sum, fixture) => sum + fixture.artifacts.size, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeCoverageForFinding(finding, coverage) {
|
||||
const fixtureCoverage = coverage.fixtures.get(finding.fixture);
|
||||
if (!fixtureCoverage || fixtureCoverage.captured.size === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const expected = expectedRuntimeCaptureKeys(finding);
|
||||
if (expected.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const captured = expected.filter((item) => fixtureCoverage.captured.has(item));
|
||||
if (captured.length === 0) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
status: captured.length === expected.length ? "covered" : "partial",
|
||||
captured,
|
||||
expected,
|
||||
artifacts: [...fixtureCoverage.artifacts].sort(),
|
||||
};
|
||||
}
|
||||
|
||||
function expectedRuntimeCaptureKeys(finding) {
|
||||
const names = evidenceNames(finding.evidence);
|
||||
if (finding.code === "registration-capture-gap") {
|
||||
return names.map((name) => `registration:${name}`);
|
||||
}
|
||||
if (finding.code === "runtime-tool-capture") {
|
||||
return ["registration:registerTool"];
|
||||
}
|
||||
if (finding.code === "conversation-access-hook") {
|
||||
return names.map((name) => `hook:${name}`);
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
function normalizeCaptured(captured) {
|
||||
return (captured ?? [])
|
||||
.map((item) => {
|
||||
if (typeof item === "string") {
|
||||
return item;
|
||||
}
|
||||
if (item && typeof item === "object" && item.kind && item.name) {
|
||||
return `${item.kind}:${item.name}`;
|
||||
}
|
||||
return "";
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function evidenceNames(evidence) {
|
||||
return [
|
||||
...new Set(
|
||||
(evidence ?? [])
|
||||
.map((item) => String(item).split(" @ ")[0]?.trim())
|
||||
.filter(Boolean),
|
||||
),
|
||||
];
|
||||
}
|
||||
|
||||
function ensureFixtureCoverage(fixtures, fixture) {
|
||||
let fixtureCoverage = fixtures.get(fixture);
|
||||
if (!fixtureCoverage) {
|
||||
fixtureCoverage = {
|
||||
artifacts: new Set(),
|
||||
captured: new Set(),
|
||||
};
|
||||
fixtures.set(fixture, fixtureCoverage);
|
||||
}
|
||||
return fixtureCoverage;
|
||||
}
|
||||
418
src/sdk-mock.js
418
src/sdk-mock.js
@ -11,8 +11,11 @@ export const mockSdkSubpathExports = {
|
||||
"emptyPluginConfigSchema",
|
||||
],
|
||||
core: [
|
||||
"buildChannelOutboundSessionRoute",
|
||||
"buildChannelConfigSchema",
|
||||
"buildPluginConfigSchema",
|
||||
"createActionGate",
|
||||
"createChannelPluginBase",
|
||||
"createChatChannelPlugin",
|
||||
"createDedupeCache",
|
||||
"defineChannelPluginEntry",
|
||||
@ -22,12 +25,24 @@ export const mockSdkSubpathExports = {
|
||||
"emptyPluginConfigSchema",
|
||||
"jsonResult",
|
||||
"readNumberParam",
|
||||
"readReactionParams",
|
||||
"readStringArrayParam",
|
||||
"readStringParam",
|
||||
],
|
||||
"channel-actions": [
|
||||
"createActionGate",
|
||||
"jsonResult",
|
||||
"readNumberParam",
|
||||
"readReactionParams",
|
||||
"readStringArrayParam",
|
||||
"readStringParam",
|
||||
],
|
||||
"channel-core": [
|
||||
"buildChannelConfigSchema",
|
||||
"buildChannelOutboundSessionRoute",
|
||||
"buildThreadAwareOutboundSessionRoute",
|
||||
"clearAccountEntryFields",
|
||||
"createChannelPluginBase",
|
||||
"createChatChannelPlugin",
|
||||
"defineChannelPluginEntry",
|
||||
"defineSetupPluginEntry",
|
||||
@ -239,9 +254,20 @@ export async function createMockSdkPackage(rootDir, options = {}) {
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(), "utf8");
|
||||
const rootExportNames = new Set([
|
||||
...mockSdkExportNames,
|
||||
...(imports.bySpecifier.get("openclaw/plugin-sdk") ?? []),
|
||||
]);
|
||||
await writeFile(path.join(pluginSdkDir, "index.js"), mockSdkSource(rootExportNames), "utf8");
|
||||
for (const [subpath, exportNames] of Object.entries(mockSdkSubpathExports)) {
|
||||
await writeFile(path.join(pluginSdkDir, `${subpath}.js`), mockSdkSubpathSource(exportNames), "utf8");
|
||||
const specifier = `openclaw/plugin-sdk/${subpath}`;
|
||||
await writeFile(
|
||||
path.join(pluginSdkDir, `${subpath}.js`),
|
||||
mockSdkSubpathSource(exportNames, imports.bySpecifier.get(specifier) ?? new Set(), {
|
||||
zod: subpath === "zod",
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
}
|
||||
for (const specifier of imports.openclawSdkSpecifiers) {
|
||||
if (specifier === "openclaw/plugin-sdk") {
|
||||
@ -413,6 +439,9 @@ export async function resolve(specifier, context, nextResolve) {
|
||||
const subpath = specifier.slice("openclaw/plugin-sdk/".length);
|
||||
return moduleUrl(path.join(pluginSdkDir, \`\${subpath}.js\`));
|
||||
}
|
||||
if (externalMap.has(specifier)) {
|
||||
return moduleUrl(externalMap.get(specifier));
|
||||
}
|
||||
try {
|
||||
return await nextResolve(specifier, context);
|
||||
} catch (error) {
|
||||
@ -516,6 +545,15 @@ function genericExportStatement(name) {
|
||||
if (["createChatChannelPlugin", "createPlugin", "defineChannelPluginEntry", "definePlugin", "definePluginEntry", "defineSetupPluginEntry"].includes(name)) {
|
||||
return name === "definePluginEntry" ? "export { definePluginEntry };" : `export const ${name} = definePluginEntry;`;
|
||||
}
|
||||
if (name === "defineBundledChannelEntry") {
|
||||
return "export { defineBundledChannelEntry };";
|
||||
}
|
||||
if (name === "defineBundledChannelSetupEntry") {
|
||||
return "export { defineBundledChannelSetupEntry };";
|
||||
}
|
||||
if (name === "loadBundledEntryExportSync") {
|
||||
return "export { loadBundledEntryExportSync };";
|
||||
}
|
||||
if (/^[A-Z].*Schema$/u.test(name)) {
|
||||
return `export const ${name} = createSchema();`;
|
||||
}
|
||||
@ -523,7 +561,13 @@ function genericExportStatement(name) {
|
||||
}
|
||||
|
||||
function genericMockRuntimeSource(options = {}) {
|
||||
return `${options.includeSdkRuntime ? `function definePluginEntry(entry) {
|
||||
return `${options.includeSdkRuntime ? `import { existsSync } from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
|
||||
const pendingBundledEntryLoads = new Set();
|
||||
|
||||
function definePluginEntry(entry) {
|
||||
if (entry && typeof entry === "object" && typeof entry.register === "function") {
|
||||
return entry;
|
||||
}
|
||||
@ -532,9 +576,86 @@ function genericMockRuntimeSource(options = {}) {
|
||||
}
|
||||
return typeof entry === "function" ? { register: entry } : entry;
|
||||
}
|
||||
|
||||
function defineBundledChannelEntry(entry = {}) {
|
||||
return {
|
||||
...entry,
|
||||
kind: "bundled-channel-entry",
|
||||
async register(api) {
|
||||
if (api?.registrationMode === "cli-metadata") {
|
||||
return entry.registerCliMetadata?.(api);
|
||||
}
|
||||
if (api?.registrationMode !== "tool-discovery") {
|
||||
api?.registerChannel?.({
|
||||
id: entry.id,
|
||||
name: entry.name,
|
||||
description: entry.description,
|
||||
plugin: { id: entry.id, name: entry.name },
|
||||
});
|
||||
}
|
||||
entry.registerCliMetadata?.(api);
|
||||
const result = entry.registerFull?.(api);
|
||||
if (result && typeof result.then === "function") {
|
||||
await result;
|
||||
}
|
||||
await drainBundledEntryLoads();
|
||||
return result;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function defineBundledChannelSetupEntry(entry = {}) {
|
||||
return {
|
||||
...entry,
|
||||
kind: "bundled-channel-setup-entry",
|
||||
};
|
||||
}
|
||||
|
||||
function loadBundledEntryExportSync(importMetaUrl, options = {}) {
|
||||
return (...args) => {
|
||||
const promise = import(resolveBundledEntryUrl(importMetaUrl, options.specifier)).then((module) => {
|
||||
const loaded = module[options.exportName] ?? module.default;
|
||||
return typeof loaded === "function" ? loaded(...args) : loaded;
|
||||
});
|
||||
pendingBundledEntryLoads.add(promise);
|
||||
promise.finally(() => pendingBundledEntryLoads.delete(promise));
|
||||
return promise;
|
||||
};
|
||||
}
|
||||
|
||||
async function drainBundledEntryLoads() {
|
||||
while (pendingBundledEntryLoads.size > 0) {
|
||||
await Promise.all([...pendingBundledEntryLoads]);
|
||||
}
|
||||
}
|
||||
|
||||
function resolveBundledEntryUrl(importMetaUrl, specifier) {
|
||||
const basePath = fileURLToPath(importMetaUrl);
|
||||
const target = specifier ? path.resolve(path.dirname(basePath), specifier) : basePath;
|
||||
const resolved = resolveExistingSourcePath(target);
|
||||
return pathToFileURL(resolved).href;
|
||||
}
|
||||
|
||||
function resolveExistingSourcePath(target) {
|
||||
if (existsSync(target)) {
|
||||
return target;
|
||||
}
|
||||
const parsed = path.parse(target);
|
||||
const withoutJsExtension = [".js", ".mjs", ".cjs"].includes(parsed.ext) ? path.join(parsed.dir, parsed.name) : null;
|
||||
const candidates = [
|
||||
...(withoutJsExtension ? [\`\${withoutJsExtension}.ts\`, \`\${withoutJsExtension}.mts\`, \`\${withoutJsExtension}.cts\`] : []),
|
||||
\`\${target}.js\`,
|
||||
\`\${target}.mjs\`,
|
||||
\`\${target}.ts\`,
|
||||
];
|
||||
return candidates.find((candidate) => existsSync(candidate)) ?? target;
|
||||
}
|
||||
` : ""}
|
||||
function createMockValue(name) {
|
||||
function fn(...args) {
|
||||
if (name === "resolvePreferredOpenClawTmpDir") {
|
||||
return process.env.TMPDIR || "/tmp";
|
||||
}
|
||||
if (name.startsWith("normalize")) {
|
||||
return typeof args[0] === "string" ? args[0] : "";
|
||||
}
|
||||
@ -713,25 +834,185 @@ function createTypeNamespace() {
|
||||
`;
|
||||
}
|
||||
|
||||
function mockSdkSource() {
|
||||
function mockSdkSource(exportNames = mockSdkExportNames) {
|
||||
const dynamicExportNames = [...exportNames].filter((name) => !mockSdkExportNames.includes(name));
|
||||
return `function normalizeEntry(entry) {
|
||||
return typeof entry === "function" ? { register: entry } : entry;
|
||||
}
|
||||
|
||||
function normalizeRegistrationMode(api) {
|
||||
return api?.registrationMode ?? "full";
|
||||
}
|
||||
|
||||
function isPlainObject(value) {
|
||||
return value !== null && typeof value === "object" && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function parseWithSchema(schema, value) {
|
||||
return schema && typeof schema.parse === "function" ? schema.parse(value) : value;
|
||||
}
|
||||
|
||||
function createConfigSchema(schema = {}) {
|
||||
if (schema && typeof schema.parse === "function") {
|
||||
return schema;
|
||||
}
|
||||
const shape = isPlainObject(schema?.shape) ? schema.shape : isPlainObject(schema?.properties) ? schema.properties : schema;
|
||||
return {
|
||||
...schema,
|
||||
parse(value = {}) {
|
||||
if (!isPlainObject(shape)) {
|
||||
return isPlainObject(value) ? value : {};
|
||||
}
|
||||
const source = isPlainObject(value) ? value : {};
|
||||
const output = { ...source };
|
||||
for (const [key, fieldSchema] of Object.entries(shape)) {
|
||||
if (Object.prototype.hasOwnProperty.call(source, key)) {
|
||||
output[key] = parseWithSchema(fieldSchema, source[key]);
|
||||
}
|
||||
}
|
||||
return output;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function definePluginEntry(entry) {
|
||||
return normalizeEntry(entry);
|
||||
}
|
||||
|
||||
export function defineChannelPluginEntry(entry) {
|
||||
return normalizeEntry(entry);
|
||||
if (!isPlainObject(entry) || !entry.plugin) {
|
||||
return normalizeEntry(entry);
|
||||
}
|
||||
const resolved = {
|
||||
id: entry.id,
|
||||
name: entry.name,
|
||||
description: entry.description,
|
||||
configSchema: createConfigSchema(entry.configSchema),
|
||||
channelPlugin: entry.plugin,
|
||||
register(api) {
|
||||
const mode = normalizeRegistrationMode(api);
|
||||
if (mode === "cli-metadata") {
|
||||
entry.registerCliMetadata?.(api);
|
||||
return;
|
||||
}
|
||||
api.registerChannel?.({ plugin: entry.plugin });
|
||||
entry.setRuntime?.(api.runtime);
|
||||
if (mode === "discovery") {
|
||||
entry.registerCliMetadata?.(api);
|
||||
return;
|
||||
}
|
||||
if (mode !== "full") {
|
||||
return;
|
||||
}
|
||||
entry.registerCliMetadata?.(api);
|
||||
entry.registerFull?.(api);
|
||||
},
|
||||
};
|
||||
if (entry.setRuntime) {
|
||||
resolved.setChannelRuntime = entry.setRuntime;
|
||||
}
|
||||
return resolved;
|
||||
}
|
||||
|
||||
export function defineSetupPluginEntry(entry) {
|
||||
return normalizeEntry(entry);
|
||||
return isPlainObject(entry) && entry.plugin ? entry : { plugin: entry };
|
||||
}
|
||||
|
||||
export function createChatChannelPlugin(entry) {
|
||||
return normalizeEntry(entry);
|
||||
if (!isPlainObject(entry) || !entry.base) {
|
||||
return normalizeEntry(entry);
|
||||
}
|
||||
return {
|
||||
...entry.base,
|
||||
conversationBindings: {
|
||||
supportsCurrentConversationBinding: true,
|
||||
...(entry.base.conversationBindings ?? {}),
|
||||
},
|
||||
...(entry.security ? { security: resolveChannelSecurity(entry.security) } : {}),
|
||||
...(entry.pairing ? { pairing: resolveChannelPairing(entry.pairing) } : {}),
|
||||
...(entry.threading ? { threading: resolveChannelThreading(entry.threading) } : {}),
|
||||
...(entry.outbound ? { outbound: resolveChannelOutbound(entry.outbound) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function createChannelPluginBase(params = {}) {
|
||||
return {
|
||||
id: params.id ?? "fixture-channel",
|
||||
meta: { id: params.id ?? "fixture-channel", ...(params.meta ?? {}) },
|
||||
...(params.setupWizard ? { setupWizard: params.setupWizard } : {}),
|
||||
...(params.capabilities ? { capabilities: params.capabilities } : {}),
|
||||
...(params.commands ? { commands: params.commands } : {}),
|
||||
...(params.doctor ? { doctor: params.doctor } : {}),
|
||||
...(params.agentPrompt ? { agentPrompt: params.agentPrompt } : {}),
|
||||
...(params.streaming ? { streaming: params.streaming } : {}),
|
||||
...(params.reload ? { reload: params.reload } : {}),
|
||||
...(params.gatewayMethods ? { gatewayMethods: params.gatewayMethods } : {}),
|
||||
...(params.configSchema ? { configSchema: createConfigSchema(params.configSchema) } : {}),
|
||||
...(params.config ? { config: params.config } : {}),
|
||||
...(params.security ? { security: params.security } : {}),
|
||||
...(params.groups ? { groups: params.groups } : {}),
|
||||
setup: params.setup ?? (() => ({})),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveChannelSecurity(security) {
|
||||
if (!isPlainObject(security) || !security.dm) {
|
||||
return security;
|
||||
}
|
||||
return {
|
||||
resolveDmPolicy: ({ account } = {}) => ({
|
||||
policy: security.dm.resolvePolicy?.(account ?? {}) ?? security.dm.defaultPolicy ?? "allow",
|
||||
allowFrom: security.dm.resolveAllowFrom?.(account ?? {}) ?? [],
|
||||
}),
|
||||
...(security.collectWarnings ? { collectWarnings: security.collectWarnings } : {}),
|
||||
...(security.collectAuditFindings ? { collectAuditFindings: security.collectAuditFindings } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveChannelPairing(pairing) {
|
||||
if (!isPlainObject(pairing) || !pairing.text) {
|
||||
return pairing;
|
||||
}
|
||||
return {
|
||||
idLabel: pairing.text.idLabel,
|
||||
normalizeAllowEntry: pairing.text.normalizeAllowEntry,
|
||||
notifyApproval: (ctx) => pairing.text.notify?.({ ...ctx, message: pairing.text.message }),
|
||||
};
|
||||
}
|
||||
|
||||
function resolveChannelThreading(threading) {
|
||||
if (!isPlainObject(threading)) {
|
||||
return threading;
|
||||
}
|
||||
if (threading.resolveReplyToMode) {
|
||||
return threading;
|
||||
}
|
||||
return {
|
||||
...threading,
|
||||
resolveReplyToMode: () =>
|
||||
threading.topLevelReplyToMode ??
|
||||
threading.scopedAccountReplyToMode?.fallback ??
|
||||
"thread",
|
||||
};
|
||||
}
|
||||
|
||||
function resolveChannelOutbound(outbound) {
|
||||
if (!isPlainObject(outbound) || !outbound.attachedResults) {
|
||||
return outbound;
|
||||
}
|
||||
const { base = {}, attachedResults } = outbound;
|
||||
return {
|
||||
...base,
|
||||
...(attachedResults.sendText
|
||||
? { sendText: async (ctx) => ({ channel: attachedResults.channel, ...(await attachedResults.sendText(ctx)) }) }
|
||||
: {}),
|
||||
...(attachedResults.sendMedia
|
||||
? { sendMedia: async (ctx) => ({ channel: attachedResults.channel, ...(await attachedResults.sendMedia(ctx)) }) }
|
||||
: {}),
|
||||
...(attachedResults.sendPoll
|
||||
? { sendPoll: async (ctx) => ({ channel: attachedResults.channel, ...(await attachedResults.sendPoll(ctx)) }) }
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function definePlugin(entry) {
|
||||
@ -759,13 +1040,13 @@ export function defineSingleProviderPluginEntry(options) {
|
||||
}
|
||||
|
||||
export function buildPluginConfigSchema(schema = {}) {
|
||||
return schema;
|
||||
return createConfigSchema(schema);
|
||||
}
|
||||
|
||||
export const emptyPluginConfigSchema = { type: "object", properties: {}, additionalProperties: false };
|
||||
export const emptyPluginConfigSchema = createConfigSchema({ type: "object", properties: {}, additionalProperties: false });
|
||||
|
||||
export function buildChannelConfigSchema(schema = {}) {
|
||||
return schema;
|
||||
return createConfigSchema(schema);
|
||||
}
|
||||
|
||||
export const emptyChannelConfigSchema = emptyPluginConfigSchema;
|
||||
@ -774,13 +1055,63 @@ export function jsonResult(value) {
|
||||
return { content: [{ type: "text", text: JSON.stringify(value) }] };
|
||||
}
|
||||
|
||||
export function readNumberParam(value, fallback = 0) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
export function readNumberParam(value, keyOrFallback = 0, options = {}) {
|
||||
const raw = isPlainObject(value) ? value[keyOrFallback] : value;
|
||||
const parsed = Number(raw);
|
||||
if (Number.isFinite(parsed)) {
|
||||
return options.integer ? Math.trunc(parsed) : parsed;
|
||||
}
|
||||
return isPlainObject(value) ? undefined : keyOrFallback;
|
||||
}
|
||||
|
||||
export function readStringParam(value, fallback = "") {
|
||||
return typeof value === "string" ? value : fallback;
|
||||
export function readStringParam(value, keyOrFallback = "") {
|
||||
if (isPlainObject(value)) {
|
||||
const raw = value[keyOrFallback];
|
||||
return typeof raw === "string" ? raw : undefined;
|
||||
}
|
||||
return typeof value === "string" ? value : keyOrFallback;
|
||||
}
|
||||
|
||||
export function readStringArrayParam(value, key) {
|
||||
const raw = isPlainObject(value) ? value[key] : value;
|
||||
if (Array.isArray(raw)) {
|
||||
return raw.map((entry) => String(entry));
|
||||
}
|
||||
return typeof raw === "string" && raw ? [raw] : [];
|
||||
}
|
||||
|
||||
export function readReactionParams(value = {}) {
|
||||
return {
|
||||
messageId: value.messageId ?? value.id ?? "",
|
||||
reaction: value.reaction ?? value.emoji ?? "",
|
||||
};
|
||||
}
|
||||
|
||||
export function createActionGate(actions = {}) {
|
||||
return (key, defaultValue = true) => {
|
||||
const value = actions?.[key];
|
||||
return value === undefined ? defaultValue : value !== false;
|
||||
};
|
||||
}
|
||||
|
||||
export function buildChannelOutboundSessionRoute(params = {}) {
|
||||
const peer = params.peer ?? { kind: params.chatType ?? "direct", id: params.to ?? "fixture-peer" };
|
||||
const baseSessionKey = [
|
||||
params.agentId ?? "agent",
|
||||
params.channel ?? "channel",
|
||||
params.accountId ?? "default",
|
||||
peer.kind,
|
||||
peer.id,
|
||||
].filter(Boolean).join(":");
|
||||
return {
|
||||
sessionKey: baseSessionKey,
|
||||
baseSessionKey,
|
||||
peer,
|
||||
chatType: params.chatType ?? peer.kind ?? "direct",
|
||||
from: params.from ?? "fixture-source",
|
||||
to: params.to ?? peer.id,
|
||||
...(params.threadId !== undefined ? { threadId: params.threadId } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export function createDedupeCache() {
|
||||
@ -845,16 +1176,36 @@ export function normalizeSecretInputString(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function createSimpleSchema(defaultValue) {
|
||||
return {
|
||||
parse(value) {
|
||||
return value === undefined ? defaultValue : value;
|
||||
},
|
||||
default(value) {
|
||||
return createSimpleSchema(value);
|
||||
},
|
||||
optional() {
|
||||
return this;
|
||||
},
|
||||
nullable() {
|
||||
return this;
|
||||
},
|
||||
nullish() {
|
||||
return this;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSecretInputSchema() {
|
||||
return { type: "string" };
|
||||
return createSimpleSchema();
|
||||
}
|
||||
|
||||
export function buildOptionalSecretInputSchema() {
|
||||
return { anyOf: [buildSecretInputSchema(), { type: "undefined" }] };
|
||||
return createSimpleSchema();
|
||||
}
|
||||
|
||||
export function buildSecretInputArraySchema() {
|
||||
return { type: "array", items: buildSecretInputSchema() };
|
||||
return createSimpleSchema([]);
|
||||
}
|
||||
|
||||
export function registerPluginHttpRoute(options = {}) {
|
||||
@ -951,14 +1302,28 @@ export function createAuthRateLimiter() {
|
||||
}
|
||||
|
||||
export function createProviderApiKeyAuthMethod(options = {}) {
|
||||
return { type: "apiKey", ...options };
|
||||
return {
|
||||
id: options.id ?? "apiKey",
|
||||
type: "apiKey",
|
||||
...options,
|
||||
async resolve(ctx = {}) {
|
||||
return ctx.apiKey ?? ctx.key ?? ctx.token ?? null;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSingleProviderApiKeyCatalog(options = {}) {
|
||||
const auth = options.auth ?? createProviderApiKeyAuthMethod(options.authOptions);
|
||||
return {
|
||||
auth,
|
||||
order: "simple",
|
||||
async run(ctx) {
|
||||
return { provider: await options.buildProvider?.(ctx) };
|
||||
const provider = (await options.buildProvider?.(ctx)) ?? options.provider ?? { id: options.id ?? "provider", auth };
|
||||
return {
|
||||
provider,
|
||||
providers: [provider],
|
||||
models: (await options.buildModels?.(ctx)) ?? options.models ?? [],
|
||||
};
|
||||
},
|
||||
};
|
||||
}
|
||||
@ -1084,7 +1449,7 @@ export function createSubsystemLogger() {
|
||||
}
|
||||
|
||||
export function buildThreadAwareOutboundSessionRoute(route = {}) {
|
||||
return route;
|
||||
return route.route ?? route;
|
||||
}
|
||||
|
||||
export function clearAccountEntryFields(entry = {}) {
|
||||
@ -1300,15 +1665,20 @@ export const OPENAI_RESPONSES_STREAM_HOOKS = buildProviderStreamFamilyHooks("ope
|
||||
export const OPENROUTER_THINKING_STREAM_HOOKS = buildProviderStreamFamilyHooks("openrouter-thinking");
|
||||
export const TOOL_STREAM_DEFAULT_ON_HOOKS = buildProviderStreamFamilyHooks("tool-stream-default");
|
||||
export const pluginSdkMock = true;
|
||||
${dynamicExportNames.map(genericExportStatement).join("\n")}
|
||||
|
||||
export default {
|
||||
${mockSdkExportNames.map((name) => ` ${name},`).join("\n")}
|
||||
${[...exportNames].map((name) => ` ${name},`).join("\n")}
|
||||
};
|
||||
`;
|
||||
}
|
||||
|
||||
function mockSdkSubpathSource(exportNames) {
|
||||
return `${exportNames.map((name) => `export { ${name} } from "./index.js";`).join("\n")}
|
||||
function mockSdkSubpathSource(staticExportNames, importedExportNames, options = {}) {
|
||||
const staticNames = new Set(staticExportNames);
|
||||
const dynamicNames = [...importedExportNames].filter((name) => !staticNames.has(name));
|
||||
return `${[...staticNames].map((name) => `export { ${name} } from "./index.js";`).join("\n")}
|
||||
${dynamicNames.length > 0 ? genericMockRuntimeSource({ includeSdkRuntime: true, zod: options.zod }) : ""}
|
||||
${dynamicNames.map(genericExportStatement).join("\n")}
|
||||
export { default } from "./index.js";
|
||||
`;
|
||||
}
|
||||
|
||||
19
src/synthetic-probe-suite.js
Normal file
19
src/synthetic-probe-suite.js
Normal file
@ -0,0 +1,19 @@
|
||||
import { buildContractCapture } from "./contract-capture.js";
|
||||
import { buildSyntheticProbePlan } from "./synthetic-probes.js";
|
||||
|
||||
export function buildSyntheticProbePlanFromReport(report, options = {}) {
|
||||
const capture = options.capture ?? buildContractCapture({
|
||||
report,
|
||||
hookAssertions: options.hookAssertions,
|
||||
hookContexts: options.hookContexts,
|
||||
hookEvents: options.hookEvents,
|
||||
registrationArguments: options.registrationArguments,
|
||||
registrationAssertions: options.registrationAssertions,
|
||||
});
|
||||
return buildSyntheticProbePlan({
|
||||
capture,
|
||||
hookContexts: options.hookContexts,
|
||||
hookEvents: options.hookEvents,
|
||||
registrationArguments: options.registrationArguments,
|
||||
});
|
||||
}
|
||||
@ -1,5 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
import { mkdtemp, rm } from "node:fs/promises";
|
||||
import { rmSync } from "node:fs";
|
||||
import { mkdtemp } from "node:fs/promises";
|
||||
import { register } from "node:module";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
@ -59,17 +60,20 @@ async function captureForSyntheticProbes(entrypoint, options) {
|
||||
const resolvedEntrypoint = path.resolve(process.cwd(), entrypoint);
|
||||
const pluginRoot = path.resolve(process.cwd(), options.pluginRoot ?? path.dirname(resolvedEntrypoint));
|
||||
const workspace = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-synthetic-mock-sdk-"));
|
||||
try {
|
||||
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
|
||||
register(pathToFileURL(loaderPath));
|
||||
return captureEntrypoint(entrypoint, {
|
||||
...options,
|
||||
mockSdk: false,
|
||||
pluginRoot,
|
||||
});
|
||||
} finally {
|
||||
await rm(workspace, { force: true, recursive: true });
|
||||
}
|
||||
cleanupTempDirOnExit(workspace);
|
||||
const { loaderPath } = await createMockSdkPackage(workspace, { pluginRoot });
|
||||
register(pathToFileURL(loaderPath));
|
||||
return captureEntrypoint(entrypoint, {
|
||||
...options,
|
||||
mockSdk: false,
|
||||
pluginRoot,
|
||||
});
|
||||
}
|
||||
|
||||
function cleanupTempDirOnExit(dir) {
|
||||
process.once("exit", () => {
|
||||
rmSync(dir, { force: true, recursive: true });
|
||||
});
|
||||
}
|
||||
|
||||
function readFlag(commandArgs, name) {
|
||||
|
||||
@ -1,11 +1,21 @@
|
||||
import { renderPaddedMarkdownTable, writeJsonMarkdownArtifacts } from "./artifacts.js";
|
||||
|
||||
export const syntheticRegistrationExecutionProfiles = {
|
||||
createChatChannelPlugin: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "channel plugin factory metadata is captured before channel runtime execution",
|
||||
},
|
||||
defineChannelPluginEntry: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "entry wrapper metadata is captured before channel runtime execution",
|
||||
},
|
||||
defineBundledChannelEntry: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "bundled channel entry metadata is captured before channel runtime execution",
|
||||
},
|
||||
definePluginEntry: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
@ -13,25 +23,80 @@ export const syntheticRegistrationExecutionProfiles = {
|
||||
},
|
||||
registerChannel: {
|
||||
mode: "channel-opt-in",
|
||||
callableProperties: ["send", "receive"],
|
||||
callableProperties: ["send", "sendMessage", "receive", "handleMessage"],
|
||||
option: "includeChannelRuntime",
|
||||
},
|
||||
registerAgentHarness: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "agent harness factories are captured as registration metadata; agent runtime execution remains isolated opt-in",
|
||||
},
|
||||
registerAgentEventSubscription: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "agent event subscriptions are captured as registration metadata before agent event dispatch",
|
||||
},
|
||||
registerAgentToolResultMiddleware: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "agent tool-result middleware is captured as registration metadata before tool-result pipeline execution",
|
||||
},
|
||||
registerAutoEnableProbe: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "auto-enable probes are captured as registration metadata before runtime activation checks",
|
||||
},
|
||||
registerCli: {
|
||||
mode: "direct",
|
||||
callableProperties: ["handler", "run", "execute"],
|
||||
},
|
||||
registerCliBackend: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "CLI backend descriptors are captured as registration metadata before backend process execution",
|
||||
},
|
||||
registerCommand: {
|
||||
mode: "direct",
|
||||
callableProperties: ["handler", "run", "execute"],
|
||||
},
|
||||
registerCodexAppServerExtensionFactory: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "Codex app server extension factories are captured as registration metadata before host UI execution",
|
||||
},
|
||||
registerCompactionProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "compaction providers are captured as registration metadata before compaction runtime execution",
|
||||
},
|
||||
registerConfigMigration: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "config migrations are captured as registration metadata before mutating stored plugin config",
|
||||
},
|
||||
registerContextEngine: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "context engine factories are captured as registration metadata; engine startup remains isolated opt-in",
|
||||
},
|
||||
registerControlUiDescriptor: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "control UI descriptors are captured as registration metadata before UI composition",
|
||||
},
|
||||
registerDetachedTaskRuntime: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "detached task runtimes are captured as registration metadata before async task execution",
|
||||
},
|
||||
registerGatewayDiscoveryService: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "gateway discovery services are captured as registration metadata before network discovery execution",
|
||||
},
|
||||
registerGatewayMethod: {
|
||||
mode: "direct",
|
||||
callableProperties: ["handler", "run", "execute"],
|
||||
callableProperties: ["handler", "run", "execute", "invoke"],
|
||||
},
|
||||
registerHttpRoute: {
|
||||
mode: "direct",
|
||||
@ -46,30 +111,155 @@ export const syntheticRegistrationExecutionProfiles = {
|
||||
callableProperties: [],
|
||||
reason: "legacy hook registrar is captured as metadata; hook handlers are probed through hook events",
|
||||
},
|
||||
registerImageGenerationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "image generation providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerMemoryPromptSection: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory prompt section renderers are captured as metadata before prompt-runtime execution",
|
||||
},
|
||||
registerMediaUnderstandingProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "media understanding providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerMemoryCapability: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory capabilities are captured as registration metadata before memory runtime execution",
|
||||
},
|
||||
registerMemoryCorpusSupplement: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory corpus supplements are captured as registration metadata before memory runtime execution",
|
||||
},
|
||||
registerMemoryEmbeddingProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory embedding providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerMemoryFlushPlan: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory flush plans are captured as registration metadata before memory runtime execution",
|
||||
},
|
||||
registerMemoryRuntime: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory runtime factories are captured as metadata; external memory startup remains isolated opt-in",
|
||||
},
|
||||
registerMemoryPromptSupplement: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "memory prompt supplements are captured as registration metadata before prompt-runtime execution",
|
||||
},
|
||||
registerMigrationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "migration providers are captured as registration metadata before migration runtime execution",
|
||||
},
|
||||
registerMusicGenerationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "music generation providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerNodeHostCommand: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "node host commands are captured as registration metadata before host process execution",
|
||||
},
|
||||
registerNodeInvokePolicy: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "node invoke policies are captured as registration metadata before host authorization checks",
|
||||
},
|
||||
registerProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "provider descriptors are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerRealtimeTranscriptionProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "realtime transcription providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerRealtimeVoiceProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "realtime voice providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerReload: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "reload handlers are captured as registration metadata before runtime reload execution",
|
||||
},
|
||||
registerRuntimeLifecycle: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "runtime lifecycle handlers are captured as registration metadata before lifecycle dispatch",
|
||||
},
|
||||
registerSecurityAuditCollector: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "security audit collectors are captured as registration metadata before filesystem or policy scans",
|
||||
},
|
||||
registerService: {
|
||||
mode: "lifecycle-opt-in",
|
||||
callableProperties: ["start", "stop"],
|
||||
callableProperties: ["start", "stop", "dispose"],
|
||||
option: "includeLifecycle",
|
||||
},
|
||||
registerSessionExtension: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "session extensions are captured as registration metadata before session runtime execution",
|
||||
},
|
||||
registerSessionSchedulerJob: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "session scheduler jobs are captured as registration metadata before scheduler execution",
|
||||
},
|
||||
registerSpeechProvider: {
|
||||
mode: "provider-opt-in",
|
||||
callableProperties: ["speak", "synthesize"],
|
||||
callableProperties: ["speak", "synthesize", "tts"],
|
||||
option: "includeProviderCapabilities",
|
||||
},
|
||||
registerTool: {
|
||||
mode: "direct",
|
||||
callableProperties: ["run", "handler", "execute"],
|
||||
},
|
||||
registerToolMetadata: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "tool metadata descriptors are captured as registration metadata before tool runtime execution",
|
||||
},
|
||||
registerTextTransforms: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "text transforms are captured as registration metadata before content mutation execution",
|
||||
},
|
||||
registerVideoGenerationProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "video generation providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerTrustedToolPolicy: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "trusted tool policies are captured as registration metadata before trust-policy enforcement",
|
||||
},
|
||||
registerWebFetchProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "web fetch providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
registerWebSearchProvider: {
|
||||
mode: "metadata-only",
|
||||
callableProperties: [],
|
||||
reason: "web search providers are captured as registration metadata before provider runtime execution",
|
||||
},
|
||||
};
|
||||
|
||||
export const defaultSyntheticHookEvents = {
|
||||
@ -186,6 +376,7 @@ export const defaultSyntheticHookContexts = {
|
||||
};
|
||||
|
||||
export const defaultSyntheticRegistrationArguments = {
|
||||
createChatChannelPlugin: [{ base: { id: "fixture-channel" }, outbound: { sendText: "function" } }],
|
||||
defineChannelPluginEntry: [{ id: "fixture-channel", setup: "function", receive: "function" }],
|
||||
definePluginEntry: [{ id: "fixture-plugin", register: "function" }],
|
||||
registerChannel: [{ id: "fixture-channel", send: "function", receive: "function" }],
|
||||
@ -214,6 +405,12 @@ export const defaultSyntheticRegistrationProbeInputs = {
|
||||
handler: commandProbeArgs,
|
||||
run: commandProbeArgs,
|
||||
},
|
||||
registerChannel: {
|
||||
handleMessage: channelReceiveProbeArgs,
|
||||
receive: channelReceiveProbeArgs,
|
||||
send: channelSendProbeArgs,
|
||||
sendMessage: channelSendProbeArgs,
|
||||
},
|
||||
registerGatewayMethod: {
|
||||
execute: gatewayProbeArgs,
|
||||
handler: gatewayProbeArgs,
|
||||
@ -452,7 +649,8 @@ async function runRegistrationProbes(entry, retainedEntry, captureIndex, options
|
||||
return [metadataOnlyResult(entry, captureIndex, profile.reason)];
|
||||
}
|
||||
|
||||
const descriptor = retainedEntry.arguments?.[0] ?? retainedEntry.returnValue;
|
||||
const descriptor =
|
||||
retainedEntry.arguments?.find((value) => value && typeof value === "object") ?? retainedEntry.returnValue;
|
||||
if (!descriptor || typeof descriptor !== "object") {
|
||||
return [blockedResult(entry, captureIndex, "captured registration has no object descriptor")];
|
||||
}
|
||||
@ -522,6 +720,9 @@ function syntheticRegistrationEvent(registrar, property, options) {
|
||||
id: beforeToolCall.toolCallId,
|
||||
name: beforeToolCall.toolName,
|
||||
},
|
||||
respond(ok, result, error) {
|
||||
return { ok, result, ...(error ? { error } : {}) };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
@ -575,9 +776,11 @@ function commandProbeArgs(event) {
|
||||
function gatewayProbeArgs(event) {
|
||||
return [
|
||||
{
|
||||
...event,
|
||||
params: event.params,
|
||||
body: event.body,
|
||||
headers: event.headers,
|
||||
respond: event.respond,
|
||||
},
|
||||
{
|
||||
source: event.source,
|
||||
@ -586,6 +789,47 @@ function gatewayProbeArgs(event) {
|
||||
];
|
||||
}
|
||||
|
||||
function channelSendProbeArgs(event) {
|
||||
return [
|
||||
{
|
||||
source: event.source,
|
||||
channelId: "fixture-channel",
|
||||
accountId: "fixture-account",
|
||||
to: "fixture-recipient",
|
||||
text: "fixture message",
|
||||
replyToId: "fixture-reply",
|
||||
threadId: "fixture-thread",
|
||||
logger: console,
|
||||
signal: new AbortController().signal,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function channelReceiveProbeArgs(event) {
|
||||
return [
|
||||
{
|
||||
source: event.source,
|
||||
channelId: "fixture-channel",
|
||||
accountId: "fixture-account",
|
||||
message: {
|
||||
id: "message-fixture",
|
||||
text: "fixture inbound message",
|
||||
sender: { id: "sender-fixture", displayName: "Fixture Sender" },
|
||||
},
|
||||
route: {
|
||||
sessionKey: "fixture-session",
|
||||
baseSessionKey: "fixture-base-session",
|
||||
peer: { kind: "direct", id: "sender-fixture" },
|
||||
chatType: "direct",
|
||||
from: "sender-fixture",
|
||||
to: "fixture-channel",
|
||||
},
|
||||
logger: console,
|
||||
signal: new AbortController().signal,
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
function interactiveProbeArgs(event) {
|
||||
return [
|
||||
{
|
||||
@ -603,7 +847,10 @@ function lifecycleProbeArgs(event) {
|
||||
return [
|
||||
{
|
||||
source: event.source,
|
||||
config: {},
|
||||
logger: console,
|
||||
runtime: { env: {}, logger: console },
|
||||
secrets: { get: async () => null, has: async () => false },
|
||||
signal: new AbortController().signal,
|
||||
},
|
||||
];
|
||||
|
||||
@ -71,6 +71,7 @@ export async function buildWorkspacePlan(options = {}) {
|
||||
installStepCount: allSteps.filter((step) => step.kind === "install").length,
|
||||
auditStepCount: allSteps.filter((step) => step.kind === "audit").length,
|
||||
buildStepCount: allSteps.filter((step) => step.kind === "build").length,
|
||||
pruneDevWorkspaceDependencyStepCount: allSteps.filter((step) => step.kind === "prune-dev-workspace-deps").length,
|
||||
artifactStepCount: allSteps.filter((step) => step.kind === "prepare-artifacts").length,
|
||||
captureStepCount: allSteps.filter((step) => step.kind === "capture").length,
|
||||
syntheticProbeStepCount: allSteps.filter((step) => step.kind === "synthetic-probe").length,
|
||||
@ -179,6 +180,7 @@ export function renderWorkspacePlanMarkdown(plan, options = {}) {
|
||||
["Artifact dirs", plan.summary.artifactStepCount],
|
||||
["Install steps", plan.summary.installStepCount],
|
||||
["Audit steps", plan.summary.auditStepCount],
|
||||
["Prune dev workspace dependency steps", plan.summary.pruneDevWorkspaceDependencyStepCount],
|
||||
["Build steps", plan.summary.buildStepCount],
|
||||
["Capture steps", plan.summary.captureStepCount],
|
||||
["Synthetic probe steps", plan.summary.syntheticProbeStepCount],
|
||||
@ -218,7 +220,7 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
|
||||
const packageManager = detectPackageManager(settings.rootDir, packageDir, packageJson);
|
||||
const lockfile = findNearestLockfile(settings.rootDir, packageDir);
|
||||
const buildScript = packageJson.scripts?.build;
|
||||
const requiredCapabilities = requiredCapabilitiesFor(entrypoint);
|
||||
const requiredCapabilities = requiredCapabilitiesFor(entrypoint, packageSummary);
|
||||
const loaderStrategy = loaderStrategyFor(entrypoint);
|
||||
const blockers = [...entrypoint.blockers];
|
||||
const workspacePath = posixJoin(settings.workspaceRoot, fixtureId);
|
||||
@ -248,6 +250,14 @@ async function buildEntrypointPlan({ fixtureId, entrypoint, packageSummary, pack
|
||||
}
|
||||
|
||||
if (requiredCapabilities.includes("dependency-install")) {
|
||||
if (hasWorkspaceProtocolDevDependencies(packageJson)) {
|
||||
steps.push({
|
||||
kind: "prune-dev-workspace-deps",
|
||||
command: `node ${helperScript(settings, workspacePath, settings.pruneWorkspaceDevDepsScript, "prune-workspace-dev-deps-cli.js")}`,
|
||||
cwd: workspacePath,
|
||||
reason: "remove workspace: devDependencies from the isolated runtime install; the mock SDK supplies OpenClaw host imports",
|
||||
});
|
||||
}
|
||||
steps.push({
|
||||
kind: "install",
|
||||
command: installCommand(packageManager),
|
||||
@ -319,6 +329,7 @@ function workspaceSettings(options) {
|
||||
resultsRoot: repoRelative(options.resultsRoot ?? defaultWorkspacePlanOptions.resultsRoot),
|
||||
rootDir: path.resolve(options.rootDir ?? process.cwd()),
|
||||
syntheticProbeScript: options.syntheticProbeScript ?? defaultWorkspacePlanOptions.syntheticProbeScript,
|
||||
pruneWorkspaceDevDepsScript: options.pruneWorkspaceDevDepsScript,
|
||||
workspaceRoot: repoRelative(options.workspaceRoot ?? defaultWorkspacePlanOptions.workspaceRoot),
|
||||
};
|
||||
}
|
||||
@ -342,7 +353,7 @@ function loaderStrategyFor(entrypoint) {
|
||||
};
|
||||
}
|
||||
|
||||
function requiredCapabilitiesFor(entrypoint) {
|
||||
function requiredCapabilitiesFor(entrypoint, packageSummary = {}) {
|
||||
const capabilities = new Set();
|
||||
for (const blocker of entrypoint.blockers) {
|
||||
if (blocker.code === "dependency-install-required") {
|
||||
@ -361,7 +372,7 @@ function requiredCapabilitiesFor(entrypoint) {
|
||||
capabilities.add("side-effect-sandbox");
|
||||
}
|
||||
}
|
||||
if (entrypoint.blockers.some((blocker) => /\bopenclaw\b/.test(blocker.evidence ?? ""))) {
|
||||
if (hasHostLinkedOpenClawDependency(packageSummary)) {
|
||||
capabilities.add("target-openclaw-link");
|
||||
}
|
||||
capabilities.add("capture-shim");
|
||||
@ -369,6 +380,20 @@ function requiredCapabilitiesFor(entrypoint) {
|
||||
return [...capabilities].sort();
|
||||
}
|
||||
|
||||
function hasHostLinkedOpenClawDependency(packageSummary) {
|
||||
return [
|
||||
...(packageSummary.dependencies ?? []),
|
||||
...(packageSummary.peerDependencies ?? []),
|
||||
...(packageSummary.optionalDependencies ?? []),
|
||||
].includes("openclaw");
|
||||
}
|
||||
|
||||
function hasWorkspaceProtocolDevDependencies(packageJson) {
|
||||
return Object.values(packageJson.devDependencies ?? {}).some(
|
||||
(value) => typeof value === "string" && value.startsWith("workspace:"),
|
||||
);
|
||||
}
|
||||
|
||||
function detectPackageManager(rootDir, packageDir, packageJson) {
|
||||
const declared = typeof packageJson.packageManager === "string" ? packageJson.packageManager.split("@")[0] : null;
|
||||
if (declared) {
|
||||
@ -450,15 +475,13 @@ function runCommand(packageManager, script) {
|
||||
}
|
||||
|
||||
function captureCommand(settings, fixtureId, entrypoint, workspacePath) {
|
||||
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
|
||||
const script = helperScript(settings, workspacePath, settings.captureScript, "capture-cli.js");
|
||||
return `${settings.optInEnv} node${loader} ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
|
||||
return `${settings.optInEnv} node ${script} ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "capture")}`;
|
||||
}
|
||||
|
||||
function syntheticProbeCommand(settings, fixtureId, entrypoint, workspacePath) {
|
||||
const loader = entrypoint.blockers.some((blocker) => blocker.code === "ts-loader-required") ? " --import tsx" : "";
|
||||
const script = helperScript(settings, workspacePath, settings.syntheticProbeScript, "synthetic-probes-cli.js");
|
||||
return `${settings.optInEnv} node${loader} ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
|
||||
return `${settings.optInEnv} node ${script} --entrypoint ${entrypoint.specifier} --mock-sdk --output ${workspaceArtifactPath(settings, fixtureId, entrypoint, workspacePath, "synthetic")}`;
|
||||
}
|
||||
|
||||
function helperScript(settings, workspacePath, configuredScript, helperFileName) {
|
||||
|
||||
630
test/api.test.js
630
test/api.test.js
@ -3,15 +3,93 @@ import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { packageId } from "../src/config.js";
|
||||
import {
|
||||
buildCiPolicyReport,
|
||||
buildCiSummary,
|
||||
buildContractCapture,
|
||||
buildExecutionResultsReport,
|
||||
buildFixtureSetColdImportReadiness,
|
||||
buildImportLoopProfile,
|
||||
buildFixtureSetPlatformProbes,
|
||||
buildFixtureSetWorkspacePlan,
|
||||
buildProfileDiff,
|
||||
buildRefDiff,
|
||||
buildRuntimeProfile,
|
||||
buildSyntheticProbePlanFromReport,
|
||||
capturePluginEntrypoint,
|
||||
ci,
|
||||
classifyIssueFinding,
|
||||
contracts,
|
||||
fixtureSuites,
|
||||
inspectFixtureSet,
|
||||
inspectCompatibilityFixtureSetConfig,
|
||||
inspectFixtureSetConfig,
|
||||
inspectPluginRoot,
|
||||
inspectSourceText,
|
||||
issueId,
|
||||
knownIssueCodes,
|
||||
loadInspectorConfig,
|
||||
loadPluginConfig,
|
||||
openClawTargetPathCandidates,
|
||||
pluginRoot,
|
||||
renderCiPolicyMarkdown,
|
||||
renderCiSummaryMarkdown,
|
||||
renderContractCaptureMarkdown,
|
||||
renderExecutionResultsMarkdown,
|
||||
renderImportLoopProfileMarkdown,
|
||||
renderMarkdownReport,
|
||||
renderFixtureSetColdImportReadinessMarkdown,
|
||||
renderFixtureSetIssuesReport,
|
||||
renderFixtureSetPlatformProbesMarkdown,
|
||||
renderFixtureSetWorkspacePlanMarkdown,
|
||||
renderProfileDiffMarkdown,
|
||||
renderRefDiffMarkdown,
|
||||
renderRuntimeProfileMarkdown,
|
||||
renderSyntheticProbeMarkdown,
|
||||
reports,
|
||||
runtime,
|
||||
runCapturedSyntheticProbes,
|
||||
runFixtureSetColdImportReadiness,
|
||||
runFixtureSetPlatformProbes,
|
||||
runFixtureSetReport,
|
||||
runFixtureSetWorkspacePlan,
|
||||
runPluginCheck,
|
||||
setupPluginInspector,
|
||||
staticInspection,
|
||||
synthetic,
|
||||
validateCiPolicy,
|
||||
validateCiPolicyReport,
|
||||
validateContractCapture,
|
||||
validateContractCoverage,
|
||||
validateColdImportReadiness,
|
||||
validateFixtureSetPlatformProbes,
|
||||
validateFixtureSetWorkspacePlan,
|
||||
validateImportLoopProfile,
|
||||
validateProfileDiff,
|
||||
validateRefDiff,
|
||||
validateRuntimeProfile,
|
||||
validateSyntheticProbePlan,
|
||||
writeFixtureSetColdImportReadiness,
|
||||
writeFixtureSetPlatformProbes,
|
||||
writeReport,
|
||||
writeFixtureSetReports,
|
||||
writeFixtureSetWorkspacePlan,
|
||||
writeContractCapture,
|
||||
writeCiPolicyReport,
|
||||
writeCiSummary,
|
||||
writeExecutionResultsReport,
|
||||
writeImportLoopProfile,
|
||||
writeProfileDiff,
|
||||
writeRefDiff,
|
||||
writeRuntimeProfile,
|
||||
writeSyntheticProbePlan,
|
||||
} from "../src/index.js";
|
||||
|
||||
test("package ids collapse separators and trim hyphen edges", () => {
|
||||
assert.equal(packageId("@openclaw/openclaw---Weather_Plugin!!!"), "weather-plugin");
|
||||
});
|
||||
|
||||
test("public API runs the plugin-root check and writes reports", async () => {
|
||||
const pluginRoot = await createPluginRoot();
|
||||
|
||||
@ -30,6 +108,53 @@ test("public API runs the plugin-root check and writes reports", async () => {
|
||||
assert.equal(paths.jsonPath, path.join(pluginRoot, "reports", "plugin-inspector-report.json"));
|
||||
});
|
||||
|
||||
test("public API exposes grouped facades for common workflows", () => {
|
||||
assert.equal(pluginRoot.loadConfig, loadPluginConfig);
|
||||
assert.equal(pluginRoot.inspect, inspectPluginRoot);
|
||||
assert.equal(pluginRoot.runCheck, runPluginCheck);
|
||||
assert.equal(fixtureSuites.inspect, inspectCompatibilityFixtureSetConfig);
|
||||
assert.equal(fixtureSuites.runReport, runFixtureSetReport);
|
||||
assert.equal(staticInspection.inspectSourceText, inspectSourceText);
|
||||
assert.equal(reports.renderMarkdown, renderMarkdownReport);
|
||||
assert.equal(typeof reports.sanitizeArtifact, "function");
|
||||
assert.equal(typeof reports.readOpenClawTargetSurface, "function");
|
||||
assert.equal(contracts.buildCapture, buildContractCapture);
|
||||
assert.equal(contracts.validateCoverage, validateContractCoverage);
|
||||
assert.equal(ci.buildSummary, buildCiSummary);
|
||||
assert.equal(ci.buildPolicyReport, buildCiPolicyReport);
|
||||
assert.equal(runtime.buildProfile, buildRuntimeProfile);
|
||||
assert.equal(runtime.buildRefDiff, buildRefDiff);
|
||||
assert.equal(synthetic.buildPlanFromReport, buildSyntheticProbePlanFromReport);
|
||||
assert.equal(synthetic.runCaptured, runCapturedSyntheticProbes);
|
||||
});
|
||||
|
||||
test("public API reads plugin config from package.json", async () => {
|
||||
const pluginRoot = await createPluginRoot({
|
||||
packageConfig: {
|
||||
plugin: {
|
||||
id: "weather-pkg",
|
||||
priority: "medium",
|
||||
seams: ["channel"],
|
||||
sourceRoot: "src",
|
||||
expect: {
|
||||
registrations: ["definePluginEntry"],
|
||||
},
|
||||
},
|
||||
capture: {
|
||||
mockSdk: true,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
const config = await loadPluginConfig({ pluginRoot });
|
||||
|
||||
assert.equal(config.configPath, "package.json#pluginInspector");
|
||||
assert.equal(config.fixtures[0].id, "weather-pkg");
|
||||
assert.equal(config.fixtures[0].priority, "medium");
|
||||
assert.deepEqual(config.fixtures[0].seams, ["channel"]);
|
||||
assert.equal(config.capture.mockSdk, true);
|
||||
});
|
||||
|
||||
test("public API keeps crabpot-style fixture configs behind an explicit helper", async () => {
|
||||
const report = await inspectFixtureSetConfig({ configPath: "test/fixtures/inspector.config.json" });
|
||||
|
||||
@ -37,6 +162,154 @@ test("public API keeps crabpot-style fixture configs behind an explicit helper",
|
||||
assert.equal(report.summary.fixtureCount, 1);
|
||||
});
|
||||
|
||||
test("public API exposes static source and fixture-set inspection primitives", async () => {
|
||||
const sourceInspection = inspectSourceText(
|
||||
[
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk";',
|
||||
"export default definePluginEntry((api) => {",
|
||||
' api.on("before_tool_call", () => undefined);',
|
||||
' api.registerTool({ name: "weather" });',
|
||||
"});",
|
||||
].join("\n"),
|
||||
"plugins/weather/src/index.js",
|
||||
);
|
||||
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
|
||||
const report = await inspectFixtureSet(config);
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-root-report-"));
|
||||
const paths = await writeReport(report, { outDir, basename: "static-inspection" });
|
||||
|
||||
assert.deepEqual(sourceInspection.hooks.map((hook) => hook.name), ["before_tool_call"]);
|
||||
assert.deepEqual(sourceInspection.registrations.map((registration) => registration.name), [
|
||||
"registerTool",
|
||||
"definePluginEntry",
|
||||
]);
|
||||
assert.equal(report.status, "pass");
|
||||
assert.match(renderMarkdownReport(report), /# OpenClaw Plugin Inspector Report/);
|
||||
assert.equal(paths.jsonPath, path.join(outDir, "static-inspection.json"));
|
||||
});
|
||||
|
||||
test("public API writes compatibility fixture-set reports with custom render options", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-fixture-api-"));
|
||||
const report = await inspectCompatibilityFixtureSetConfig({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
});
|
||||
const paths = await writeFixtureSetReports(report, {
|
||||
jsonPath: path.join(outDir, "suite.json"),
|
||||
markdownPath: path.join(outDir, "suite.md"),
|
||||
issuesPath: path.join(outDir, "issues.md"),
|
||||
markdownTitle: "Fixture Suite Compatibility",
|
||||
issuesTitle: "Fixture Suite Issues",
|
||||
formatEvidence: (evidence) => `linked:${evidence}`,
|
||||
});
|
||||
const result = await runFixtureSetReport({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
outDir,
|
||||
basename: "run",
|
||||
});
|
||||
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(paths, {
|
||||
jsonPath: path.join(outDir, "suite.json"),
|
||||
markdownPath: path.join(outDir, "suite.md"),
|
||||
issuesPath: path.join(outDir, "issues.md"),
|
||||
});
|
||||
assert.match(await readFile(paths.markdownPath, "utf8"), /# Fixture Suite Compatibility/);
|
||||
assert.match(await readFile(paths.issuesPath, "utf8"), /# Fixture Suite Issues/);
|
||||
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
assert.equal(result.report.summary.fixtureCount, 1);
|
||||
assert.equal(result.paths.jsonPath, path.join(outDir, "run.json"));
|
||||
});
|
||||
|
||||
test("public API builds fixture-set cold import readiness from config", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-cold-import-api-"));
|
||||
const readiness = await buildFixtureSetColdImportReadiness({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
});
|
||||
const paths = await writeFixtureSetColdImportReadiness(readiness, {
|
||||
jsonPath: path.join(outDir, "cold-import.json"),
|
||||
markdownPath: path.join(outDir, "cold-import.md"),
|
||||
title: "Fixture Cold Import",
|
||||
});
|
||||
const result = await runFixtureSetColdImportReadiness({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
write: false,
|
||||
});
|
||||
|
||||
assert.equal(readiness.summary.fixtureCount, 1);
|
||||
assert.deepEqual(validateColdImportReadiness(readiness), []);
|
||||
assert.match(renderFixtureSetColdImportReadinessMarkdown(readiness), /## Entrypoints/);
|
||||
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
assert.equal(result.paths, null);
|
||||
assert.equal(result.readiness.summary.fixtureCount, 1);
|
||||
});
|
||||
|
||||
test("public API builds fixture-set workspace and platform plans from config", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-plan-api-"));
|
||||
const plan = await buildFixtureSetWorkspacePlan({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
});
|
||||
const planPaths = await writeFixtureSetWorkspacePlan(plan, {
|
||||
jsonPath: path.join(outDir, "workspace.json"),
|
||||
markdownPath: path.join(outDir, "workspace.md"),
|
||||
});
|
||||
const platform = await buildFixtureSetPlatformProbes({ plan });
|
||||
const coveredPlan = structuredClone(plan);
|
||||
coveredPlan.fixtures[0].entrypoints = [
|
||||
{
|
||||
id: "cold-import.extension:sample-plugin:index",
|
||||
status: "dependency-install-required",
|
||||
entrypoint: "plugins/sample-plugin/index.js",
|
||||
packageManager: "npm",
|
||||
loaderStrategy: {
|
||||
source: "javascript",
|
||||
primary: "node",
|
||||
alternatives: [],
|
||||
reason: "test",
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
kind: "prepare",
|
||||
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
const coveredPlatform = await buildFixtureSetPlatformProbes({
|
||||
plan: coveredPlan,
|
||||
stepCoverage({ riskCodes }) {
|
||||
return { riskCodes };
|
||||
},
|
||||
});
|
||||
const platformPaths = await writeFixtureSetPlatformProbes(platform, {
|
||||
jsonPath: path.join(outDir, "platform.json"),
|
||||
markdownPath: path.join(outDir, "platform.md"),
|
||||
});
|
||||
const planResult = await runFixtureSetWorkspacePlan({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
write: false,
|
||||
});
|
||||
const platformResult = await runFixtureSetPlatformProbes({ plan, write: false });
|
||||
|
||||
assert.equal(plan.summary.fixtureCount, 1);
|
||||
assert.deepEqual(validateFixtureSetWorkspacePlan(plan), []);
|
||||
assert.match(renderFixtureSetWorkspacePlanMarkdown(plan), /## Entrypoint Workspaces/);
|
||||
assert.equal(JSON.parse(await readFile(planPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
assert.equal(platform.summary.fixtureCount, 1);
|
||||
assert.equal(coveredPlatform.summary.portabilityFindingCount, 0);
|
||||
assert.ok(coveredPlatform.summary.coveredPortabilityFindingCount > 0);
|
||||
assert.deepEqual(validateFixtureSetPlatformProbes(platform), []);
|
||||
assert.match(renderFixtureSetPlatformProbesMarkdown(platform), /## Loader Probes/);
|
||||
assert.equal(JSON.parse(await readFile(platformPaths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
assert.equal(planResult.paths, null);
|
||||
assert.equal(platformResult.paths, null);
|
||||
});
|
||||
|
||||
test("public API exposes capture through an explicit entrypoint helper", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-api-capture-"));
|
||||
const entrypoint = path.join(dir, "fixture.mjs");
|
||||
@ -65,14 +338,265 @@ test("public API exposes capture through an explicit entrypoint helper", async (
|
||||
test("public API can initialize plugin inspector files", async () => {
|
||||
const pluginRoot = await createPluginRoot();
|
||||
|
||||
const result = await setupPluginInspector({ pluginRoot, ci: true, packageManager: "npm" });
|
||||
const result = await setupPluginInspector({ pluginRoot, ci: true, scripts: true, packageManager: "npm" });
|
||||
const config = JSON.parse(await readFile(path.join(pluginRoot, "plugin-inspector.config.json"), "utf8"));
|
||||
const packageJson = JSON.parse(await readFile(path.join(pluginRoot, "package.json"), "utf8"));
|
||||
const workflow = await readFile(path.join(pluginRoot, ".github", "workflows", "plugin-inspector.yml"), "utf8");
|
||||
|
||||
assert.equal(result.written.length, 2);
|
||||
assert.equal(result.written.length, 3);
|
||||
assert.equal(result.packageManager, "npm");
|
||||
assert.equal(config.plugin.id, "weather");
|
||||
assert.equal(config.capture.mockSdk, true);
|
||||
assert.match(workflow, /npx @openclaw\/plugin-inspector check --no-openclaw/);
|
||||
assert.equal(packageJson.scripts["plugin:check"], "plugin-inspector inspect --no-openclaw");
|
||||
assert.equal(packageJson.scripts["plugin:ci"], "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute");
|
||||
assert.match(workflow, /npx @openclaw\/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute/);
|
||||
});
|
||||
|
||||
test("public API initializes source root from package export maps", async () => {
|
||||
const pluginRoot = await createPluginRoot({
|
||||
packageJson: {
|
||||
openclaw: null,
|
||||
exports: {
|
||||
".": {
|
||||
import: "./src/plugin-entry.js",
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
await setupPluginInspector({ pluginRoot, force: true });
|
||||
const config = JSON.parse(await readFile(path.join(pluginRoot, "plugin-inspector.config.json"), "utf8"));
|
||||
|
||||
assert.equal(config.plugin.sourceRoot, "src");
|
||||
});
|
||||
|
||||
test("fixture-set issue renderer is available without advanced internals", () => {
|
||||
const markdown = renderFixtureSetIssuesReport(
|
||||
{
|
||||
generatedAt: "test",
|
||||
status: "pass",
|
||||
summary: {
|
||||
issueCount: 0,
|
||||
p0IssueCount: 0,
|
||||
p1IssueCount: 0,
|
||||
liveIssueCount: 0,
|
||||
liveP0IssueCount: 0,
|
||||
compatGapCount: 0,
|
||||
deprecationWarningCount: 0,
|
||||
inspectorGapCount: 0,
|
||||
upstreamIssueCount: 0,
|
||||
contractProbeCount: 0,
|
||||
},
|
||||
issues: [],
|
||||
contractProbes: [],
|
||||
},
|
||||
{ title: "Fixture Issues" },
|
||||
);
|
||||
|
||||
assert.match(markdown, /# Fixture Issues/);
|
||||
assert.match(markdown, /## Triage Summary/);
|
||||
});
|
||||
|
||||
test("public API exposes report issue metadata helpers", () => {
|
||||
assert.ok(knownIssueCodes.has("registration-capture-gap"));
|
||||
assert.match(issueId({ fixture: "weather", code: "registration-capture-gap" }), /^CRABPOT-[A-F0-9]{8}$/);
|
||||
assert.equal(classifyIssueFinding({ code: "registration-capture-gap" }).issueClass, "inspector-gap");
|
||||
assert.ok(openClawTargetPathCandidates().some((candidate) => candidate.includes("openclaw")));
|
||||
});
|
||||
|
||||
test("public API exposes contract capture and coverage helpers", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-contract-api-"));
|
||||
const report = await inspectCompatibilityFixtureSetConfig({
|
||||
configPath: "test/fixtures/inspector.config.json",
|
||||
openclawPath: false,
|
||||
});
|
||||
const capture = buildContractCapture({ report });
|
||||
const paths = await writeContractCapture(capture, {
|
||||
jsonPath: path.join(outDir, "capture.json"),
|
||||
markdownPath: path.join(outDir, "capture.md"),
|
||||
});
|
||||
|
||||
assert.equal(capture.summary.fixtureCount, 1);
|
||||
assert.deepEqual(validateContractCapture(capture), []);
|
||||
assert.deepEqual(validateContractCoverage(report), []);
|
||||
assert.match(renderContractCaptureMarkdown(capture), /## Registration Capture/);
|
||||
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.fixtureCount, 1);
|
||||
});
|
||||
|
||||
test("public API exposes execution and CI rollup helpers", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ci-api-"));
|
||||
const resultDir = path.join(rootDir, ".plugin-inspector", "results", "weather");
|
||||
const outDir = path.join(rootDir, "reports");
|
||||
await mkdir(resultDir, { recursive: true });
|
||||
await writeFile(
|
||||
path.join(resultDir, "entry.synthetic.json"),
|
||||
JSON.stringify({
|
||||
summary: { probeCount: 1, passCount: 1, failCount: 0, blockedCount: 0 },
|
||||
results: [{ kind: "hook", seam: "before_tool_call", label: "before_tool_call", status: "pass" }],
|
||||
}),
|
||||
"utf8",
|
||||
);
|
||||
const policy = {
|
||||
version: 1,
|
||||
allowedBlocked: [],
|
||||
expectedWarnings: [],
|
||||
thresholds: {
|
||||
wallP95RegressionPercent: 50,
|
||||
peakRssRegressionMb: 50,
|
||||
bootRegressionMs: 500,
|
||||
strictMinimumSamples: 3,
|
||||
},
|
||||
fixtureSets: { smoke: ["weather"] },
|
||||
};
|
||||
const compatibilityReport = {
|
||||
summary: { breakageCount: 0, p1IssueCount: 0 },
|
||||
breakages: [],
|
||||
issues: [],
|
||||
};
|
||||
|
||||
const execution = await buildExecutionResultsReport({ rootDir });
|
||||
const policyReport = buildCiPolicyReport({ policy, compatibilityReport, executionResults: execution });
|
||||
const summary = await buildCiSummary({
|
||||
reports: { compatibility: compatibilityReport, execution, ciPolicy: policyReport },
|
||||
});
|
||||
const executionPaths = await writeExecutionResultsReport(execution, {
|
||||
jsonPath: path.join(outDir, "execution.json"),
|
||||
markdownPath: path.join(outDir, "execution.md"),
|
||||
});
|
||||
const policyPaths = await writeCiPolicyReport(policyReport, {
|
||||
jsonPath: path.join(outDir, "policy.json"),
|
||||
markdownPath: path.join(outDir, "policy.md"),
|
||||
});
|
||||
const summaryPaths = await writeCiSummary(summary, {
|
||||
jsonPath: path.join(outDir, "summary.json"),
|
||||
markdownPath: path.join(outDir, "summary.md"),
|
||||
});
|
||||
|
||||
assert.equal(execution.summary.passCount, 1);
|
||||
assert.doesNotThrow(() => validateCiPolicy(policy));
|
||||
assert.deepEqual(validateCiPolicyReport(policyReport), []);
|
||||
assert.equal(summary.status, "pass");
|
||||
assert.match(renderExecutionResultsMarkdown(execution), /Execution Results/);
|
||||
assert.match(renderCiPolicyMarkdown(policyReport), /CI Policy/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /CI Summary/);
|
||||
assert.equal(JSON.parse(await readFile(executionPaths.jsonPath, "utf8")).summary.passCount, 1);
|
||||
assert.equal(JSON.parse(await readFile(policyPaths.jsonPath, "utf8")).status, "pass");
|
||||
assert.equal(JSON.parse(await readFile(summaryPaths.jsonPath, "utf8")).status, "pass");
|
||||
});
|
||||
|
||||
test("public API exposes runtime profile and diff helpers", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-profile-api-"));
|
||||
const runtimeProfile = await buildRuntimeProfile({ runs: 1 });
|
||||
const profileDiff = await buildProfileDiff({
|
||||
baseline: profileFixture({ p95WallMs: 100, maxPeakRssMb: 80, nodeBootMs: 25 }),
|
||||
current: profileFixture({ p95WallMs: 120, maxPeakRssMb: 90, nodeBootMs: 30 }),
|
||||
policy: {
|
||||
thresholds: {
|
||||
wallP95RegressionPercent: 50,
|
||||
peakRssRegressionMb: 50,
|
||||
bootRegressionMs: 500,
|
||||
strictMinimumSamples: 3,
|
||||
},
|
||||
},
|
||||
});
|
||||
const refDiff = await buildRefDiff({
|
||||
baseReport: diffReport({ hookNames: ["before_tool_call"], issues: [] }),
|
||||
headReport: diffReport({ hookNames: ["before_tool_call"], issues: [] }),
|
||||
});
|
||||
const importLoopProfile = {
|
||||
generatedAt: "deterministic",
|
||||
mode: "subprocess-cold-import-loop",
|
||||
entrypoint: "fixtures/plugin.mjs",
|
||||
summary: {
|
||||
runs: 1,
|
||||
p50WallMs: 5,
|
||||
p95WallMs: 5,
|
||||
maxPeakRssMb: 10,
|
||||
maxCpuMsEstimate: 2,
|
||||
capturedCount: 1,
|
||||
failCount: 0,
|
||||
},
|
||||
samples: [
|
||||
{
|
||||
index: 0,
|
||||
status: "captured",
|
||||
capturedCount: 1,
|
||||
wallMs: 5,
|
||||
peakRssMb: 10,
|
||||
cpuMsEstimate: 2,
|
||||
exitCode: 0,
|
||||
},
|
||||
],
|
||||
};
|
||||
|
||||
assert.equal(typeof buildImportLoopProfile, "function");
|
||||
const portableRuntimeProfile = {
|
||||
...runtimeProfile,
|
||||
platform: { ...runtimeProfile.platform, rssSampler: "unavailable" },
|
||||
};
|
||||
|
||||
assert.deepEqual(validateRuntimeProfile(portableRuntimeProfile), []);
|
||||
assert.deepEqual(validateProfileDiff(profileDiff), []);
|
||||
assert.deepEqual(validateRefDiff(refDiff), []);
|
||||
assert.deepEqual(validateImportLoopProfile(importLoopProfile), []);
|
||||
assert.match(renderRuntimeProfileMarkdown(portableRuntimeProfile), /Runtime Profile/);
|
||||
assert.match(renderProfileDiffMarkdown(profileDiff), /Runtime Profile Diff/);
|
||||
assert.match(renderRefDiffMarkdown(refDiff), /Ref Diff/);
|
||||
assert.match(renderImportLoopProfileMarkdown(importLoopProfile), /Import Loop Profile/);
|
||||
|
||||
const runtimePaths = await writeRuntimeProfile(portableRuntimeProfile, {
|
||||
jsonPath: path.join(outDir, "runtime.json"),
|
||||
markdownPath: path.join(outDir, "runtime.md"),
|
||||
});
|
||||
const profileDiffPaths = await writeProfileDiff(profileDiff, {
|
||||
jsonPath: path.join(outDir, "profile-diff.json"),
|
||||
markdownPath: path.join(outDir, "profile-diff.md"),
|
||||
});
|
||||
const refDiffPaths = await writeRefDiff(refDiff, {
|
||||
jsonPath: path.join(outDir, "ref-diff.json"),
|
||||
markdownPath: path.join(outDir, "ref-diff.md"),
|
||||
});
|
||||
const importLoopPaths = await writeImportLoopProfile(importLoopProfile, {
|
||||
jsonPath: path.join(outDir, "import-loop.json"),
|
||||
markdownPath: path.join(outDir, "import-loop.md"),
|
||||
});
|
||||
|
||||
assert.equal(JSON.parse(await readFile(runtimePaths.jsonPath, "utf8")).summary.commandCount, 1);
|
||||
assert.equal(JSON.parse(await readFile(profileDiffPaths.jsonPath, "utf8")).status, "pass");
|
||||
assert.equal(JSON.parse(await readFile(refDiffPaths.jsonPath, "utf8")).status, "pass");
|
||||
assert.equal(JSON.parse(await readFile(importLoopPaths.jsonPath, "utf8")).summary.runs, 1);
|
||||
});
|
||||
|
||||
test("public API exposes synthetic probe helpers", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-synthetic-api-"));
|
||||
const plan = buildSyntheticProbePlanFromReport({
|
||||
generatedAt: "test",
|
||||
targetOpenClaw: {
|
||||
capturedRegistrars: ["registerTool"],
|
||||
sdkExports: [],
|
||||
},
|
||||
summary: {},
|
||||
fixtures: [
|
||||
{
|
||||
id: "weather",
|
||||
priority: "high",
|
||||
hookDetails: [{ name: "before_tool_call", ref: "src/index.js:1" }],
|
||||
registrationDetails: [{ name: "registerTool", ref: "src/index.js:2" }],
|
||||
sdkImportDetails: [],
|
||||
packages: [],
|
||||
},
|
||||
],
|
||||
contractProbes: [],
|
||||
});
|
||||
const paths = await writeSyntheticProbePlan(plan, {
|
||||
jsonPath: path.join(outDir, "synthetic.json"),
|
||||
markdownPath: path.join(outDir, "synthetic.md"),
|
||||
});
|
||||
|
||||
assert.equal(typeof runCapturedSyntheticProbes, "function");
|
||||
assert.deepEqual(validateSyntheticProbePlan(plan), []);
|
||||
assert.match(renderSyntheticProbeMarkdown(plan), /registerTool/);
|
||||
assert.equal(JSON.parse(await readFile(paths.jsonPath, "utf8")).summary.probeCount, 2);
|
||||
});
|
||||
|
||||
test("public API honors config-driven runtime capture", async () => {
|
||||
@ -83,38 +607,86 @@ test("public API honors config-driven runtime capture", async () => {
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const previous = process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED;
|
||||
process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED = "1";
|
||||
try {
|
||||
const result = await runPluginCheck({ pluginRoot, outDir: "reports", openclawPath: false });
|
||||
assert.equal(result.runtimeCapture.summary.registrationCount, 1);
|
||||
} finally {
|
||||
if (previous === undefined) {
|
||||
delete process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED;
|
||||
} else {
|
||||
process.env.PLUGIN_INSPECTOR_EXECUTE_ISOLATED = previous;
|
||||
}
|
||||
}
|
||||
const result = await runPluginCheck({ pluginRoot, outDir: "reports", openclawPath: false, allowExecution: true });
|
||||
assert.equal(result.runtimeCapture.summary.registrationCount, 1);
|
||||
});
|
||||
|
||||
async function createPluginRoot() {
|
||||
function profileFixture({ p95WallMs, maxPeakRssMb, nodeBootMs }) {
|
||||
return {
|
||||
runs: 3,
|
||||
summary: { p95WallMs, maxPeakRssMb },
|
||||
targetOpenClaw: {
|
||||
compatRecords: 1,
|
||||
hookNames: 1,
|
||||
apiRegistrars: 1,
|
||||
capturedRegistrars: 1,
|
||||
sdkExports: 1,
|
||||
manifestFields: 1,
|
||||
manifestContractFields: 1,
|
||||
},
|
||||
fixtureInventory: {},
|
||||
commands: [{ id: "node-boot", wallMs: { median: nodeBootMs } }],
|
||||
};
|
||||
}
|
||||
|
||||
function diffReport({ hookNames, issues }) {
|
||||
return {
|
||||
summary: {
|
||||
fixtureCount: 1,
|
||||
breakageCount: 0,
|
||||
issueCount: issues.length,
|
||||
p0IssueCount: issues.filter((issue) => issue.severity === "P0").length,
|
||||
p1IssueCount: issues.filter((issue) => issue.severity === "P1").length,
|
||||
},
|
||||
targetOpenClaw: {
|
||||
status: "available",
|
||||
compatRecords: [],
|
||||
hookNames,
|
||||
apiRegistrars: ["registerTool"],
|
||||
capturedRegistrars: ["registerTool"],
|
||||
sdkExports: ["definePluginEntry"],
|
||||
manifestFields: ["name"],
|
||||
manifestContractFields: ["permissions"],
|
||||
},
|
||||
fixtures: [
|
||||
{
|
||||
id: "weather",
|
||||
hooks: hookNames,
|
||||
registrations: ["registerTool"],
|
||||
sdkImports: ["definePluginEntry"],
|
||||
pluginManifests: [{ name: "weather" }],
|
||||
manifestContracts: ["permissions"],
|
||||
},
|
||||
],
|
||||
issues,
|
||||
};
|
||||
}
|
||||
|
||||
async function createPluginRoot(options = {}) {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-api-root-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
const packageJson = {
|
||||
name: "@example/openclaw-weather",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.js"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
...(options.packageJson ?? {}),
|
||||
};
|
||||
if (packageJson.openclaw === null) {
|
||||
delete packageJson.openclaw;
|
||||
}
|
||||
if (options.packageConfig) {
|
||||
packageJson.pluginInspector = {
|
||||
version: 1,
|
||||
...options.packageConfig,
|
||||
};
|
||||
}
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "@example/openclaw-weather",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.js"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
`${JSON.stringify(packageJson, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
|
||||
@ -31,6 +31,35 @@ test("capture API defaults to known OpenClaw registrar profiles", () => {
|
||||
assert.ok(defaultCaptureApiRegistrarProfiles.registerTool);
|
||||
});
|
||||
|
||||
test("capture API returns useful channel, gateway, and lifecycle descriptors", () => {
|
||||
const api = createCaptureApi();
|
||||
const channel = api.registerChannel({ plugin: { id: "fixture-channel", outbound: { sendText() {} } } });
|
||||
const gatewayMethod = api.registerGatewayMethod(
|
||||
"fixture.ping",
|
||||
({ respond }) => respond(true, { ok: true }),
|
||||
{ scope: "operator.read" },
|
||||
);
|
||||
const service = api.registerService({ id: "fixture-service" });
|
||||
|
||||
assert.equal(api.registrationMode, "full");
|
||||
assert.equal(channel.id, "fixture-channel");
|
||||
assert.equal(channel.plugin.id, "fixture-channel");
|
||||
assert.equal(gatewayMethod.method, "fixture.ping");
|
||||
assert.equal(gatewayMethod.scope, "operator.read");
|
||||
assert.deepEqual(gatewayMethod.handler({ respond: api.gateway.respond }), { ok: true, result: { ok: true } });
|
||||
assert.equal(typeof service.dispose, "function");
|
||||
});
|
||||
|
||||
test("capture API records conversation binding resolved callbacks", () => {
|
||||
const api = createCaptureApi();
|
||||
|
||||
assert.equal(api.onConversationBindingResolved(() => undefined), api);
|
||||
assert.deepEqual(
|
||||
api.getCapturedContracts().map((entry) => `${entry.kind}:${entry.name}`),
|
||||
["hook:onConversationBindingResolved"],
|
||||
);
|
||||
});
|
||||
|
||||
test("capture API accepts custom registrar return profiles", () => {
|
||||
const api = createCaptureApi({
|
||||
registrarProfiles: {
|
||||
@ -48,6 +77,7 @@ test("capture API accepts custom registrar return profiles", () => {
|
||||
|
||||
test("capture API exposes mock context helpers", async () => {
|
||||
const api = createCaptureApi({
|
||||
resolvePath: (value) => `/fixture/${value}`,
|
||||
secretValues: {
|
||||
token: "redacted",
|
||||
},
|
||||
@ -61,6 +91,7 @@ test("capture API exposes mock context helpers", async () => {
|
||||
assert.deepEqual(await api.store.list(), ["key"]);
|
||||
assert.equal(api.agent.id, "plugin-inspector-agent");
|
||||
assert.equal(api.paths.dataDir, ".plugin-inspector/data");
|
||||
assert.equal(api.resolvePath("state"), "/fixture/state");
|
||||
});
|
||||
|
||||
test("capture API can retain handlers for probes", () => {
|
||||
|
||||
@ -63,6 +63,41 @@ test("ci policy allows known blocked probes but fails unknown blockers", () => {
|
||||
assert.match(renderCiPolicyMarkdown(report), /Plugin Inspector CI Policy/);
|
||||
});
|
||||
|
||||
test("ci policy supports wildcard seam rules for generated surface blockers", () => {
|
||||
const report = buildCiPolicyReport({
|
||||
policy: {
|
||||
...policy,
|
||||
allowedBlocked: [
|
||||
...policy.allowedBlocked,
|
||||
{
|
||||
id: "generated-surface-runtime-gap",
|
||||
seam: "*",
|
||||
reasonIncludes: "generated surface has no callable runtime",
|
||||
decision: "allowed-blocked",
|
||||
until: "generated surface runtime harness lands",
|
||||
},
|
||||
],
|
||||
},
|
||||
compatibilityReport: compatibilityReport(),
|
||||
executionResults: executionResults([
|
||||
{
|
||||
seam: "before_tool_call",
|
||||
reason: "generated surface has no callable runtime",
|
||||
},
|
||||
{
|
||||
seam: "registerCommand",
|
||||
reason: "generated surface has no callable runtime",
|
||||
},
|
||||
]),
|
||||
});
|
||||
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(
|
||||
report.checks.filter((check) => check.id.startsWith("execution-results.blocked.")).map((check) => check.action),
|
||||
["warn", "warn"],
|
||||
);
|
||||
});
|
||||
|
||||
test("ci policy fails ref diff hard regressions", () => {
|
||||
const report = buildCiPolicyReport({
|
||||
policy,
|
||||
@ -127,7 +162,7 @@ test("ci policy surfaces P0 live issues without blocking default lanes", () => {
|
||||
code: "legacy-before-agent-start",
|
||||
},
|
||||
{
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
issueClass: "inspector-gap",
|
||||
fixture: "wecom",
|
||||
code: "registration-capture-gap",
|
||||
@ -225,7 +260,7 @@ test("ci policy writer emits JSON and Markdown artifacts", async () => {
|
||||
function compatibilityReport(overrides = {}) {
|
||||
const issues = overrides.issues ?? [
|
||||
{
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
issueClass: "inspector-gap",
|
||||
fixture: "fixture",
|
||||
code: "registration-capture-gap",
|
||||
|
||||
@ -18,17 +18,17 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
|
||||
suggestionCount: 3,
|
||||
issueCount: 4,
|
||||
p0IssueCount: 1,
|
||||
p1IssueCount: 1,
|
||||
p1IssueCount: 0,
|
||||
liveIssueCount: 1,
|
||||
compatGapCount: 1,
|
||||
},
|
||||
issues: [
|
||||
{
|
||||
severity: "P1",
|
||||
severity: "P2",
|
||||
issueClass: "inspector-gap",
|
||||
fixture: "fixture",
|
||||
code: "registration-capture-gap",
|
||||
title: "runtime registrations need capture",
|
||||
title: "runtime registrations need capture evidence",
|
||||
decision: "inspector-follow-up",
|
||||
},
|
||||
],
|
||||
@ -97,6 +97,13 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
|
||||
p95WallMs: 75,
|
||||
maxPeakRssMb: 40,
|
||||
maxCpuMsEstimate: 30,
|
||||
maxPluginPeakRssDeltaMb: 8,
|
||||
maxPluginCpuDeltaMsEstimate: 6,
|
||||
openClawLifecycleCount: 2,
|
||||
p50OpenClawImportMs: 12,
|
||||
p50OpenClawActivationMs: 3,
|
||||
rssSampleCount: 2,
|
||||
cpuSampleCount: 2,
|
||||
},
|
||||
},
|
||||
},
|
||||
@ -108,9 +115,12 @@ test("ci summary rolls up compatibility, policy, ref diff, and profile findings"
|
||||
assert.equal(summary.summary.platformWindowsRisks, 3);
|
||||
assert.equal(summary.summary.loaderJitiCandidates, 1);
|
||||
assert.equal(summary.summary.importLoopP50Ms, 50);
|
||||
assert.equal(summary.summary.importLoopMetricBasis, "baseline-adjusted");
|
||||
assert.equal(summary.summary.importLoopMaxRssMb, 8);
|
||||
assert.equal(summary.summary.importLoopOpenClawImportP50Ms, 12);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /Crabpot CI Summary/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /Windows portability risks/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ max RSS 40 MB \/ CPU 30 ms/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /p50 50 ms \/ p95 75 ms \/ plugin delta RSS 8 MB \/ plugin delta CPU 6 ms \/ OpenClaw import 12 ms \/ activate 3 ms/);
|
||||
assert.match(renderCiSummaryMarkdown(summary), /\| P0 issues\s+\| 1\s+\|/);
|
||||
});
|
||||
|
||||
|
||||
191
test/cli.test.js
191
test/cli.test.js
@ -55,13 +55,11 @@ test("check command runs from a plugin root without fixture config", async () =>
|
||||
assert.ok(report.fixtures[0].package.openclaw.entrypoints.some((entrypoint) => entrypoint.exists));
|
||||
assert.match(issues, /# OpenClaw Plugin Issue Findings/);
|
||||
|
||||
await execFileAsync(process.execPath, [cliPath, "check", "--out", "capture-reports", "--no-openclaw", "--capture"], {
|
||||
cwd: rootDir,
|
||||
env: {
|
||||
...process.env,
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED: "1",
|
||||
},
|
||||
});
|
||||
await execFileAsync(
|
||||
process.execPath,
|
||||
[cliPath, "check", "--out", "capture-reports", "--no-openclaw", "--capture", "--allow-execute"],
|
||||
{ cwd: rootDir },
|
||||
);
|
||||
const capture = JSON.parse(
|
||||
await readFile(path.join(rootDir, "capture-reports", "plugin-inspector-runtime-capture.json"), "utf8"),
|
||||
);
|
||||
@ -75,13 +73,9 @@ test("check command can target a plugin root and use runtime aliases", async ()
|
||||
|
||||
await execFileAsync(
|
||||
process.execPath,
|
||||
[cliPath, "--plugin-root", rootDir, "--out", "reports", "--no-openclaw", "--runtime", "--mock-sdk"],
|
||||
[cliPath, "--plugin-root", rootDir, "--out", "reports", "--no-openclaw", "--runtime", "--mock-sdk", "--allow-execute"],
|
||||
{
|
||||
cwd: os.tmpdir(),
|
||||
env: {
|
||||
...process.env,
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED: "1",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
@ -93,6 +87,49 @@ test("check command can target a plugin root and use runtime aliases", async ()
|
||||
assert.equal(capture.summary.capturedCount, 1);
|
||||
});
|
||||
|
||||
test("check command sanitizes absolute OpenClaw paths in JSON output and artifacts", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-sanitize-");
|
||||
const openclawPath = await createTargetOpenClaw(rootDir);
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
const { stdout } = await execFileAsync(
|
||||
process.execPath,
|
||||
[cliPath, "check", "--out", "reports", "--openclaw", openclawPath, "--json"],
|
||||
{ cwd: rootDir },
|
||||
);
|
||||
const output = JSON.parse(stdout);
|
||||
const artifact = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector-report.json"), "utf8"));
|
||||
|
||||
assert.equal(output.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
|
||||
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
|
||||
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
|
||||
assert.doesNotMatch(stdout, new RegExp(escapeRegExp(openclawPath)));
|
||||
});
|
||||
|
||||
test("inspect command runs from a plugin root and can write CI outputs", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-inspect-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
const { stdout } = await execFileAsync(process.execPath, [
|
||||
cliPath,
|
||||
"inspect",
|
||||
"--out",
|
||||
"reports",
|
||||
"--no-openclaw",
|
||||
"--sarif",
|
||||
"--junit",
|
||||
], {
|
||||
cwd: rootDir,
|
||||
});
|
||||
|
||||
const sarif = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector.sarif"), "utf8"));
|
||||
const junit = await readFile(path.join(rootDir, "reports", "plugin-inspector.junit.xml"), "utf8");
|
||||
|
||||
assert.match(stdout, /Status: PASS/);
|
||||
assert.equal(sarif.version, "2.1.0");
|
||||
assert.match(junit, /<testsuite name="plugin-inspector"/);
|
||||
});
|
||||
|
||||
test("check command can enable runtime capture from plugin config", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-config-runtime-");
|
||||
await writeFile(
|
||||
@ -102,12 +139,8 @@ test("check command can enable runtime capture from plugin config", async () =>
|
||||
);
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
await execFileAsync(process.execPath, [cliPath, "check", "--out", "reports", "--no-openclaw"], {
|
||||
await execFileAsync(process.execPath, [cliPath, "check", "--out", "reports", "--no-openclaw", "--allow-execute"], {
|
||||
cwd: rootDir,
|
||||
env: {
|
||||
...process.env,
|
||||
PLUGIN_INSPECTOR_EXECUTE_ISOLATED: "1",
|
||||
},
|
||||
});
|
||||
|
||||
const capture = JSON.parse(
|
||||
@ -116,6 +149,26 @@ test("check command can enable runtime capture from plugin config", async () =>
|
||||
assert.equal(capture.summary.registrationCount, 1);
|
||||
});
|
||||
|
||||
test("config command prints resolved plugin root config", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-config-print-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
const { stdout } = await execFileAsync(process.execPath, [cliPath, "config", "--plugin-root", rootDir]);
|
||||
const { stdout: jsonStdout } = await execFileAsync(process.execPath, [
|
||||
cliPath,
|
||||
"config",
|
||||
"--plugin-root",
|
||||
rootDir,
|
||||
"--json",
|
||||
]);
|
||||
const config = JSON.parse(jsonStdout);
|
||||
|
||||
assert.match(stdout, /Plugin: weather/);
|
||||
assert.match(stdout, /Runtime capture: off/);
|
||||
assert.equal(config.fixtures[0].id, "weather");
|
||||
assert.equal(config.fixtures[0].subdir, "src");
|
||||
});
|
||||
|
||||
test("ci command writes CI summary artifacts", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-ci-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
@ -133,6 +186,8 @@ test("ci command writes CI summary artifacts", async () => {
|
||||
await readFile(path.join(rootDir, "reports", "plugin-inspector-ci-summary.json"), "utf8"),
|
||||
);
|
||||
const markdown = await readFile(path.join(rootDir, "reports", "plugin-inspector-ci-summary.md"), "utf8");
|
||||
const sarif = JSON.parse(await readFile(path.join(rootDir, "reports", "plugin-inspector.sarif"), "utf8"));
|
||||
const junit = await readFile(path.join(rootDir, "reports", "plugin-inspector.junit.xml"), "utf8");
|
||||
|
||||
assert.match(stdout, /Status: PASS/);
|
||||
assert.match(stdout, /Artifacts: 1/);
|
||||
@ -143,6 +198,8 @@ test("ci command writes CI summary artifacts", async () => {
|
||||
assert.equal(summary.summary.issues, report.summary.issueCount);
|
||||
assert.equal(summary.artifacts.compatibility, "plugin-inspector-report.json");
|
||||
assert.match(markdown, /# Plugin Inspector CI Summary/);
|
||||
assert.equal(sarif.runs[0].tool.driver.name, "plugin-inspector");
|
||||
assert.match(junit, /failures="0"/);
|
||||
});
|
||||
|
||||
test("init command writes plugin config and CI workflow", async () => {
|
||||
@ -156,12 +213,91 @@ test("init command writes plugin config and CI workflow", async () => {
|
||||
const config = JSON.parse(await readFile(path.join(rootDir, "plugin-inspector.config.json"), "utf8"));
|
||||
const workflow = await readFile(path.join(rootDir, ".github", "workflows", "plugin-inspector.yml"), "utf8");
|
||||
|
||||
assert.match(stdout, /plugin-inspector\.config\.json/);
|
||||
assert.match(stdout, /^wrote plugin-inspector\.config\.json$/m);
|
||||
assert.match(stdout, /^wrote \.github\/workflows\/plugin-inspector\.yml$/m);
|
||||
assert.match(stdout, /^package manager: pnpm$/m);
|
||||
assert.equal(stdout.includes(rootDir), false);
|
||||
assert.equal(config.plugin.id, "weather");
|
||||
assert.equal(config.plugin.sourceRoot, "src");
|
||||
assert.equal(config.capture.mockSdk, true);
|
||||
assert.match(workflow, /pnpm dlx @openclaw\/plugin-inspector check --no-openclaw/);
|
||||
assert.match(workflow, /--runtime --mock-sdk/);
|
||||
assert.match(workflow, /pnpm dlx @openclaw\/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute/);
|
||||
});
|
||||
|
||||
test("init command detects plugin package managers", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-pm-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
await writeFile(path.join(rootDir, "pnpm-lock.yaml"), "lockfileVersion: '9.0'\n", "utf8");
|
||||
|
||||
await execFileAsync(process.execPath, [cliPath, "init", "--plugin-root", rootDir, "--ci", "--force"]);
|
||||
const workflow = await readFile(path.join(rootDir, ".github", "workflows", "plugin-inspector.yml"), "utf8");
|
||||
|
||||
assert.match(workflow, /cache: pnpm/);
|
||||
assert.match(workflow, /corepack enable/);
|
||||
assert.match(workflow, /pnpm install --frozen-lockfile/);
|
||||
assert.match(workflow, /pnpm dlx @openclaw\/plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute/);
|
||||
});
|
||||
|
||||
test("init command can add package scripts", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-scripts-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
await execFileAsync(process.execPath, [cliPath, "init", "--plugin-root", rootDir, "--scripts", "--force"]);
|
||||
const packageJson = JSON.parse(await readFile(path.join(rootDir, "package.json"), "utf8"));
|
||||
|
||||
assert.equal(packageJson.scripts["plugin:check"], "plugin-inspector inspect --no-openclaw");
|
||||
assert.equal(packageJson.scripts["plugin:ci"], "plugin-inspector ci --no-openclaw --runtime --mock-sdk --allow-execute");
|
||||
});
|
||||
|
||||
test("init command can preview generated files", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-dry-run-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
const beforeConfig = await readFile(path.join(rootDir, "plugin-inspector.config.json"), "utf8");
|
||||
const beforePackageJson = await readFile(path.join(rootDir, "package.json"), "utf8");
|
||||
|
||||
const { stdout } = await execFileAsync(process.execPath, [
|
||||
cliPath,
|
||||
"init",
|
||||
"--plugin-root",
|
||||
rootDir,
|
||||
"--ci",
|
||||
"--scripts",
|
||||
"--dry-run",
|
||||
]);
|
||||
|
||||
assert.match(stdout, /^would write plugin-inspector\.config\.json$/m);
|
||||
assert.match(stdout, /^would write \.github\/workflows\/plugin-inspector\.yml$/m);
|
||||
assert.match(stdout, /^would write package\.json$/m);
|
||||
assert.equal(await readFile(path.join(rootDir, "plugin-inspector.config.json"), "utf8"), beforeConfig);
|
||||
assert.equal(await readFile(path.join(rootDir, "package.json"), "utf8"), beforePackageJson);
|
||||
await assert.rejects(readFile(path.join(rootDir, ".github", "workflows", "plugin-inspector.yml"), "utf8"), {
|
||||
code: "ENOENT",
|
||||
});
|
||||
});
|
||||
|
||||
test("init command can print a JSON summary", async () => {
|
||||
const rootDir = await createCliPluginRoot("plugin-inspector-cli-init-json-");
|
||||
const cliPath = path.resolve("src/cli.js");
|
||||
|
||||
const { stdout } = await execFileAsync(process.execPath, [
|
||||
cliPath,
|
||||
"init",
|
||||
"--plugin-root",
|
||||
rootDir,
|
||||
"--ci",
|
||||
"--scripts",
|
||||
"--dry-run",
|
||||
"--json",
|
||||
]);
|
||||
const summary = JSON.parse(stdout);
|
||||
|
||||
assert.equal(summary.dryRun, true);
|
||||
assert.equal(summary.packageManager, "npm");
|
||||
assert.equal(summary.pluginRoot, rootDir);
|
||||
assert.deepEqual(summary.files, [
|
||||
"plugin-inspector.config.json",
|
||||
".github/workflows/plugin-inspector.yml",
|
||||
"package.json",
|
||||
]);
|
||||
});
|
||||
|
||||
async function createCliPluginRoot(prefix) {
|
||||
@ -201,3 +337,18 @@ async function createCliPluginRoot(prefix) {
|
||||
);
|
||||
return rootDir;
|
||||
}
|
||||
|
||||
async function createTargetOpenClaw(rootDir) {
|
||||
const openclawPath = path.join(rootDir, "target-openclaw");
|
||||
await mkdir(path.join(openclawPath, "src/plugins/compat"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(openclawPath, "src/plugins/compat/registry.ts"),
|
||||
'export const records = [{ code: "legacy-root-sdk-import", status: "deprecated" }];\n',
|
||||
"utf8",
|
||||
);
|
||||
return openclawPath;
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
@ -71,6 +71,23 @@ test("cold import readiness preserves combined blocker evidence", () => {
|
||||
assert.deepEqual(validateColdImportReadiness(readiness), []);
|
||||
});
|
||||
|
||||
test("cold import readiness treats openclaw as a host-linked dependency", () => {
|
||||
const readiness = buildColdImportReadiness({
|
||||
report: readinessReport(
|
||||
[{ kind: "extension", specifier: "./index.js", relativePath: "index.js", exists: true }],
|
||||
{
|
||||
dependencies: ["openclaw"],
|
||||
},
|
||||
),
|
||||
});
|
||||
const entrypoint = readiness.fixtures[0].entrypoints[0];
|
||||
|
||||
assert.equal(entrypoint.status, "ready");
|
||||
assert.deepEqual(entrypoint.blockers, []);
|
||||
assert.equal(readiness.summary.dependencyInstallRequiredCount, 0);
|
||||
assert.deepEqual(validateColdImportReadiness(readiness), []);
|
||||
});
|
||||
|
||||
test("cold import readiness validation rejects incomplete entries", () => {
|
||||
const errors = validateColdImportReadiness({
|
||||
fixtures: [
|
||||
|
||||
@ -26,7 +26,7 @@ test("contract coverage fails missing evidence and P1 probe gaps", () => {
|
||||
fixture: "fixture",
|
||||
severity: "P1",
|
||||
issueClass: "inspector-gap",
|
||||
code: "registration-capture-gap",
|
||||
code: "conversation-access-hook",
|
||||
evidence: [],
|
||||
},
|
||||
],
|
||||
@ -165,4 +165,20 @@ test("contract coverage requires compat record reconciliation evidence", () => {
|
||||
compatRecord: "fixture.provider-auth-env-vars",
|
||||
});
|
||||
assert.deepEqual(validateContractCoverage(report), []);
|
||||
|
||||
report.logs = [];
|
||||
report.issues.push({
|
||||
id: "PLUGIN-COMPAT",
|
||||
fixture: "fixture",
|
||||
severity: "P1",
|
||||
issueClass: "compat-gap",
|
||||
code: "provider-auth-env-vars",
|
||||
evidence: ["fixture"],
|
||||
compatRecord: "fixture.provider-auth-env-vars",
|
||||
});
|
||||
report.contractProbes.push({
|
||||
fixture: "fixture",
|
||||
id: "compat.provider-auth-env-vars:fixture",
|
||||
});
|
||||
assert.deepEqual(validateContractCoverage(report), []);
|
||||
});
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { buildContractProbes, contractProbeRules, probePriority } from "../src/advanced.js";
|
||||
import { buildContractProbes, compatRecordForIssueCode, contractProbeRules, probePriority } from "../src/advanced.js";
|
||||
|
||||
test("contract probes map issue findings to executable backlog rows", () => {
|
||||
const probes = buildContractProbes({
|
||||
@ -41,11 +41,13 @@ test("contract probes map issue findings to executable backlog rows", () => {
|
||||
});
|
||||
|
||||
assert.ok(contractProbeRules["registration-capture-gap"]);
|
||||
assert.equal(compatRecordForIssueCode("registration-capture-gap"), "api.capture.runtime-registrars");
|
||||
assert.equal(compatRecordForIssueCode("package-dependency-install-required"), undefined);
|
||||
assert.deepEqual(
|
||||
probes.map((probe) => [probe.id, probe.priority, probe.target]),
|
||||
[
|
||||
["api.capture.runtime-registrars:wecom", "P1", "inspector-capture-api"],
|
||||
["sdk.import.package-export-cold-import:codex-app-server", "P1", "sdk-alias"],
|
||||
["api.capture.runtime-registrars:wecom", "P2", "inspector-capture-api"],
|
||||
["manifest.schema.top-level-fields:agentchat", "P3", "manifest-loader"],
|
||||
],
|
||||
);
|
||||
@ -53,6 +55,7 @@ test("contract probes map issue findings to executable backlog rows", () => {
|
||||
|
||||
test("contract probe priority escalates critical codes and high-priority fixtures", () => {
|
||||
assert.equal(probePriority("sdk-export-missing", "medium"), "P1");
|
||||
assert.equal(probePriority("registration-capture-gap", "high"), "P2");
|
||||
assert.equal(probePriority("manifest-unknown-fields", "high"), "P2");
|
||||
assert.equal(probePriority("manifest-unknown-fields", "medium"), "P3");
|
||||
});
|
||||
|
||||
@ -30,12 +30,19 @@ test("import loop profile measures repeated cold capture subprocesses", async ()
|
||||
|
||||
assert.deepEqual(validateImportLoopProfile(profile), []);
|
||||
assert.equal(profile.summary.runs, 2);
|
||||
assert.equal(profile.summary.baselineRuns, 2);
|
||||
assert.equal(profile.summary.baselineFailCount, 0);
|
||||
assert.equal(profile.summary.failCount, 0);
|
||||
assert.ok(profile.summary.capturedCount >= 2);
|
||||
assert.ok(profile.summary.p50WallMs > 0);
|
||||
assert.ok(profile.summary.p50PluginWallDeltaMs >= 0);
|
||||
assert.ok(profile.summary.maxPluginPeakRssDeltaMb >= 0);
|
||||
assert.ok(profile.baseline.reference.wallMs > 0);
|
||||
assert.ok(profile.samples.every((sample) => Number.isFinite(sample.pluginCpuDeltaMsEstimate)));
|
||||
assert.ok(profile.samples.every((sample) => sample.exitCode === 0));
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /Import Loop Profile/);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /CPU Estimate/);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /Harness Baseline/);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /Plugin CPU Delta/);
|
||||
});
|
||||
|
||||
test("import loop profile can use a custom capture script and opt-in env", async () => {
|
||||
@ -50,7 +57,7 @@ test("import loop profile can use a custom capture script and opt-in env", async
|
||||
"const [entrypoint,, outputPath] = process.argv.slice(2);",
|
||||
"if (process.env.CUSTOM_IMPORT_LOOP !== '1') throw new Error('missing opt-in');",
|
||||
"await mkdir(path.dirname(outputPath), { recursive: true });",
|
||||
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }] }));",
|
||||
"await writeFile(outputPath, JSON.stringify({ status: 'captured', entrypoint, captured: [{ kind: 'hook', name: 'before_tool_call' }], openClawLifecycle: { importMs: 12, activationMs: 3, importPhase: 'full', activationPhase: 'full:register' } }));",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
@ -64,7 +71,12 @@ test("import loop profile can use a custom capture script and opt-in env", async
|
||||
});
|
||||
|
||||
assert.equal(profile.summary.failCount, 0);
|
||||
assert.equal(profile.summary.baselineRuns, 1);
|
||||
assert.equal(profile.summary.capturedCount, 1);
|
||||
assert.equal(profile.summary.openClawLifecycleCount, 1);
|
||||
assert.equal(profile.summary.p50OpenClawImportMs, 12);
|
||||
assert.equal(profile.summary.p50OpenClawActivationMs, 3);
|
||||
assert.match(renderImportLoopProfileMarkdown(profile), /OpenClaw Import/);
|
||||
});
|
||||
|
||||
test("import loop profile validation rejects failed or empty captures", () => {
|
||||
|
||||
@ -43,6 +43,18 @@ test("source inspection records hook, registrar, and SDK import evidence", () =>
|
||||
]);
|
||||
});
|
||||
|
||||
test("source inspection strips long comments before matching registrations", () => {
|
||||
const inspection = inspectSourceText(
|
||||
[`/* ${"a/*".repeat(512)} */`, "api.registerTool({ name: 'weather' });"].join("\n"),
|
||||
"plugins/example/index.ts",
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
inspection.registrations.map((registration) => `${registration.name}@${registration.ref}`),
|
||||
["registerTool@plugins/example/index.ts:2"],
|
||||
);
|
||||
});
|
||||
|
||||
test("fixture set inspection produces a passing report", async () => {
|
||||
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
|
||||
const report = await inspectFixtureSet(config);
|
||||
@ -65,6 +77,44 @@ test("fixture set inspection reports missing expected seams", async () => {
|
||||
assert.match(report.breakages[0].message, /llm_output/);
|
||||
});
|
||||
|
||||
test("fixture set inspection treats channel factories as channel registration coverage", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-channel-factory-"));
|
||||
await mkdir(path.join(dir, "fixture"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(dir, "fixture", "index.js"),
|
||||
[
|
||||
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
|
||||
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
|
||||
"",
|
||||
"export const channel = createChatChannelPlugin({ id: 'fixture-channel' });",
|
||||
"export default defineBundledChannelEntry({ id: 'bundled-channel' });",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const report = await inspectFixtureSet({
|
||||
version: 1,
|
||||
submoduleRoot: ".",
|
||||
rootDir: dir,
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
path: "fixture",
|
||||
repo: "https://github.com/openclaw/fixture.git",
|
||||
priority: "high",
|
||||
seams: ["channel"],
|
||||
expect: {
|
||||
registrations: ["registerChannel"],
|
||||
},
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(report.breakages, []);
|
||||
assert.deepEqual(report.fixtures[0].registrations, ["createChatChannelPlugin", "defineBundledChannelEntry"]);
|
||||
});
|
||||
|
||||
test("capture entrypoint imports a local fixture and records registrations", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-capture-"));
|
||||
const entrypoint = path.join(dir, "fixture.mjs");
|
||||
@ -103,6 +153,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
'import { createChatChannelPlugin } from "openclaw/plugin-sdk/channel-core";',
|
||||
'import { defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";',
|
||||
'import { GPT5_BEHAVIOR_CONTRACT } from "openclaw/plugin-sdk/provider-model-shared";',
|
||||
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
|
||||
'import { registerPluginHttpRoute, resolveWebhookPath } from "openclaw/plugin-sdk/webhook-ingress";',
|
||||
"",
|
||||
@ -123,6 +174,7 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
|
||||
"",
|
||||
"export default definePluginEntry((api) => {",
|
||||
" if (!pluginSdkMock) throw new Error('expected mock SDK');",
|
||||
" if (!GPT5_BEHAVIOR_CONTRACT) throw new Error('expected dynamic subpath mock export');",
|
||||
" provider.register(api);",
|
||||
" api.registerHttpRoute({ path: resolveWebhookPath('hook'), handler() {} });",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
@ -144,3 +196,153 @@ test("capture entrypoint can mock OpenClaw plugin SDK imports", async () => {
|
||||
["registration:registerProvider", "registration:registerHttpRoute", "registration:registerTool"],
|
||||
);
|
||||
});
|
||||
|
||||
test("mock capture accepts valid output when plugin code dirties process exit code", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-exit-code-"));
|
||||
const entrypoint = path.join(dir, "index.mjs");
|
||||
await writeFile(
|
||||
entrypoint,
|
||||
[
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
"process.exitCode = 1;",
|
||||
"export default definePluginEntry({",
|
||||
" register(api) {",
|
||||
" api.registerProvider({ id: 'fixture-provider' });",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.mjs", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
|
||||
"registration:registerProvider",
|
||||
]);
|
||||
});
|
||||
|
||||
test("mock capture prefers discovered bare mocks over installed dependency exports", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-mock-bare-capture-"));
|
||||
await mkdir(path.join(dir, "node_modules/typebox"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(dir, "node_modules/typebox/package.json"),
|
||||
JSON.stringify({ name: "typebox", version: "0.0.0", type: "module", exports: "./index.js" }, null, 2),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(dir, "node_modules/typebox/index.js"), "export const Type = {};\n", "utf8");
|
||||
const entrypoint = path.join(dir, "index.mjs");
|
||||
await writeFile(
|
||||
entrypoint,
|
||||
[
|
||||
"import path from 'node:path';",
|
||||
'import { Static, Type } from "typebox";',
|
||||
'import { resolvePreferredOpenClawTmpDir } from "fixture-api";',
|
||||
"export function register(api) {",
|
||||
" if (!Static || !Type) throw new Error('expected mocked typebox exports');",
|
||||
" path.join(resolvePreferredOpenClawTmpDir(), 'fixture');",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: Type.Object({}), run() {} });",
|
||||
"}",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.mjs", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), ["registration:registerTool"]);
|
||||
});
|
||||
|
||||
test("mock capture expands bundled channel entry registration shells", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-capture-"));
|
||||
const entrypoint = path.join(dir, "index.mjs");
|
||||
await writeFile(
|
||||
entrypoint,
|
||||
[
|
||||
'import { defineBundledChannelEntry } from "openclaw/plugin-sdk/channel-entry-contract";',
|
||||
"export default defineBundledChannelEntry({",
|
||||
" id: 'fixture-channel',",
|
||||
" name: 'Fixture Channel',",
|
||||
" description: 'Fixture channel',",
|
||||
" plugin: { specifier: './channel.js', exportName: 'fixtureChannel' },",
|
||||
" registerFull(api) {",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.mjs", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
|
||||
"registration:registerChannel",
|
||||
"registration:registerTool",
|
||||
]);
|
||||
});
|
||||
|
||||
test("mock capture follows bundled channel linked registerFull exports", async () => {
|
||||
const dir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-bundled-channel-linked-capture-"));
|
||||
await writeFile(
|
||||
path.join(dir, "index.ts"),
|
||||
[
|
||||
'import { defineBundledChannelEntry, loadBundledEntryExportSync } from "openclaw/plugin-sdk/channel-entry-contract";',
|
||||
"",
|
||||
"function registerFull(api) {",
|
||||
" const register = loadBundledEntryExportSync(import.meta.url, {",
|
||||
" specifier: './api.js',",
|
||||
" exportName: 'registerFixtureFull',",
|
||||
" });",
|
||||
" register(api);",
|
||||
"}",
|
||||
"",
|
||||
"export default defineBundledChannelEntry({",
|
||||
" id: 'fixture-channel',",
|
||||
" name: 'Fixture Channel',",
|
||||
" description: 'Fixture channel',",
|
||||
" plugin: { specifier: './channel-plugin-api.js', exportName: 'fixtureChannel' },",
|
||||
" registerFull,",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(dir, "api.ts"),
|
||||
[
|
||||
'import { buildSecretInputSchema } from "openclaw/plugin-sdk/secret-input";',
|
||||
"",
|
||||
"const schema = buildSecretInputSchema().optional();",
|
||||
"",
|
||||
"export function registerFixtureFull(api) {",
|
||||
" schema.parse(undefined);",
|
||||
" api.registerCommand({ name: 'fixture.command', run() {} });",
|
||||
"}",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("index.ts", {
|
||||
cwd: dir,
|
||||
pluginRoot: dir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(result.captured.map((item) => `${item.kind}:${item.name}`), [
|
||||
"registration:registerChannel",
|
||||
"registration:registerCommand",
|
||||
]);
|
||||
});
|
||||
|
||||
@ -19,18 +19,18 @@ test("issue ids are stable fingerprints", () => {
|
||||
test("issue classification separates live breaks from compat and deprecation buckets", () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "untracked SDK alias is a blocking live issue",
|
||||
name: "untracked SDK alias is a compat gap",
|
||||
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
|
||||
targetOpenClaw: { compatRecordStatuses: {} },
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "live-issue", compatStatus: "untracked", severity: "P0", live: true },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "untracked", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "active SDK alias compat avoids false P0 escalation",
|
||||
name: "active SDK alias compat stays a compat row",
|
||||
finding: { code: "sdk-export-missing", compatRecord: "plugin-sdk-export-aliases" },
|
||||
targetOpenClaw: { compatRecordStatuses: { "plugin-sdk-export-aliases": "active" } },
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "live-issue", compatStatus: "active", severity: "P1", live: true },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "active", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "deprecated compat remains warning-class even when used",
|
||||
@ -46,6 +46,18 @@ test("issue classification separates live breaks from compat and deprecation buc
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "missing", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "active OpenClaw probe contract stays an inspector gap",
|
||||
finding: {
|
||||
code: "before-tool-call-probe",
|
||||
compatRecord: "hook.before_tool_call.terminal-block-approval",
|
||||
},
|
||||
targetOpenClaw: {
|
||||
compatRecordStatuses: { "hook.before_tool_call.terminal-block-approval": "active" },
|
||||
},
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "inspector-gap", compatStatus: "active", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "unknown untracked hook is P0 live break",
|
||||
finding: { code: "unknown-hook-name" },
|
||||
@ -100,17 +112,17 @@ test("issue builder applies metadata and class summaries", () => {
|
||||
assert.deepEqual(
|
||||
issues.map((issue) => [issue.fixture, issue.code, issue.severity, issue.issueClass, issue.status]),
|
||||
[
|
||||
["codex-app-server", "sdk-export-missing", "P0", "live-issue", "blocking"],
|
||||
["wecom", "registration-capture-gap", "P1", "inspector-gap", "open"],
|
||||
["codex-app-server", "sdk-export-missing", "P1", "compat-gap", "open"],
|
||||
["agentchat", "manifest-unknown-fields", "P2", "upstream-metadata", "open"],
|
||||
["wecom", "registration-capture-gap", "P2", "inspector-gap", "open"],
|
||||
],
|
||||
);
|
||||
assert.deepEqual(summarizeIssueClasses(issues), {
|
||||
"compat-gap": 0,
|
||||
"compat-gap": 1,
|
||||
"deprecation-warning": 0,
|
||||
"fixture-regression": 0,
|
||||
"inspector-gap": 1,
|
||||
"live-issue": 1,
|
||||
"live-issue": 0,
|
||||
"upstream-metadata": 1,
|
||||
});
|
||||
});
|
||||
|
||||
@ -136,6 +136,7 @@ test("OpenClaw target parsing helpers stay deterministic", () => {
|
||||
);
|
||||
assert.deepEqual(
|
||||
parseCompatRecordEntries(`
|
||||
${"{{".repeat(256)}
|
||||
{ code: "b", status: "supported" }
|
||||
{ code: "a", status: "deprecated" }
|
||||
{ code: "b", status: "supported" }
|
||||
|
||||
60
test/package-contents.test.js
Normal file
60
test/package-contents.test.js
Normal file
@ -0,0 +1,60 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { buildPackageContentsChecklist } from "../scripts/check-package-contents.mjs";
|
||||
|
||||
const packageJson = {
|
||||
bin: {
|
||||
"plugin-inspector": "src/cli.js",
|
||||
},
|
||||
exports: {
|
||||
".": "./src/index.js",
|
||||
"./capture-api": "./src/capture-api.js",
|
||||
},
|
||||
};
|
||||
|
||||
const requiredFiles = [
|
||||
"package.json",
|
||||
"README.md",
|
||||
"CHANGELOG.md",
|
||||
"LICENSE",
|
||||
"src/cli.js",
|
||||
"src/index.js",
|
||||
"src/capture-api.js",
|
||||
"examples/github-actions-plugin-inspector.yml",
|
||||
"examples/plugin-inspector.config.json",
|
||||
"docs/plugin-inspector-banner.jpg",
|
||||
];
|
||||
|
||||
test("package contents pass when entrypoints and README assets are packed", () => {
|
||||
const result = buildPackageContentsChecklist({
|
||||
filePaths: requiredFiles,
|
||||
packageJson,
|
||||
readmeText: '<img src="docs/plugin-inspector-banner.jpg" alt="banner"/>',
|
||||
});
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
});
|
||||
|
||||
test("package contents fail when README assets are missing", () => {
|
||||
const result = buildPackageContentsChecklist({
|
||||
filePaths: requiredFiles.filter((filePath) => filePath !== "docs/plugin-inspector-banner.jpg"),
|
||||
packageJson,
|
||||
readmeText: '<img src="docs/plugin-inspector-banner.jpg" alt="banner"/>',
|
||||
});
|
||||
|
||||
assert.equal(result.status, "fail");
|
||||
assert.equal(result.checks.find((check) => check.id === "package-readme-asset").actual, "missing");
|
||||
});
|
||||
|
||||
test("package contents fail when private release scripts are packed", () => {
|
||||
const result = buildPackageContentsChecklist({
|
||||
filePaths: [...requiredFiles, "scripts/release-notes.mjs"],
|
||||
packageJson,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "fail");
|
||||
assert.equal(
|
||||
result.checks.find((check) => check.id === "package-forbidden-path").message,
|
||||
"scripts/release-notes.mjs should not be published in the npm package",
|
||||
);
|
||||
});
|
||||
@ -40,11 +40,11 @@ test("platform probes classify loader and shell portability risks", () => {
|
||||
},
|
||||
{
|
||||
kind: "capture",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx capture.mjs ./src/index.ts",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./src/index.ts --mock-sdk",
|
||||
},
|
||||
{
|
||||
kind: "synthetic-probe",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node --import tsx synthetic.mjs --entrypoint ./src/index.ts",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node synthetic.mjs --entrypoint ./src/index.ts --mock-sdk",
|
||||
},
|
||||
],
|
||||
},
|
||||
@ -63,7 +63,68 @@ test("platform probes classify loader and shell portability risks", () => {
|
||||
assert.match(renderPlatformProbesMarkdown(report), /rsync/);
|
||||
});
|
||||
|
||||
test("platform probe validation requires jiti fallback and reflected tsx commands", () => {
|
||||
test("platform probes separate executor-covered portability risks from residual risks", () => {
|
||||
const report = buildPlatformProbes({
|
||||
plan: {
|
||||
generatedAt: "test",
|
||||
mode: "plan-only",
|
||||
summary: {
|
||||
fixtureCount: 1,
|
||||
},
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
entrypoints: [
|
||||
{
|
||||
id: "cold-import.extension:fixture:index",
|
||||
status: "dependency-install-required",
|
||||
entrypoint: "plugins/fixture/index.js",
|
||||
packageManager: "pnpm",
|
||||
loaderStrategy: {
|
||||
source: "javascript",
|
||||
primary: "node",
|
||||
alternatives: [],
|
||||
reason: "test",
|
||||
},
|
||||
steps: [
|
||||
{
|
||||
kind: "prepare",
|
||||
command: "mkdir -p .workspaces/fixture && rsync -a plugins/fixture/ .workspaces/fixture/",
|
||||
},
|
||||
{
|
||||
kind: "audit",
|
||||
command: "pnpm audit --json > ../../results/fixture/package-audit.json || true",
|
||||
},
|
||||
{
|
||||
kind: "capture",
|
||||
command: "PLUGIN_INSPECTOR_EXECUTE_ISOLATED=1 node capture.mjs ./index.js",
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
stepCoverage({ riskCodes }) {
|
||||
return {
|
||||
reason: "covered by Crabpot structured executor",
|
||||
riskCodes: riskCodes.filter((code) =>
|
||||
["posix-mkdir", "posix-env-prefix", "posix-null-failure", "rsync-required", "shell-redirection"].includes(code),
|
||||
),
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(report.summary.portabilityFindingCount, 1);
|
||||
assert.equal(report.summary.coveredPortabilityFindingCount, 3);
|
||||
assert.equal(report.summary.windowsRiskStepCount, 1);
|
||||
assert.deepEqual(report.portabilityFindings[0].riskCodes, ["package-manager-availability"]);
|
||||
assert.ok(report.coveredPortabilityFindings.every((finding) => finding.coverage === "covered by Crabpot structured executor"));
|
||||
assert.doesNotMatch(renderPlatformProbesMarkdown(report), /replace shell mkdir/);
|
||||
assert.match(renderPlatformProbesMarkdown(report), /Covered Portability Findings/);
|
||||
});
|
||||
|
||||
test("platform probe validation requires jiti fallback and reflected TypeScript loader commands", () => {
|
||||
const errors = validatePlatformProbes({
|
||||
mode: "plan-only",
|
||||
targets: ["linux", "macos", "windows", "container"],
|
||||
@ -76,11 +137,14 @@ test("platform probe validation requires jiti fallback and reflected tsx command
|
||||
id: "cold-import.extension:fixture:index",
|
||||
loaderPrimary: "tsx",
|
||||
captureUsesTsx: true,
|
||||
captureUsesTypeScriptLoader: true,
|
||||
syntheticUsesTsx: false,
|
||||
syntheticUsesMockSdk: false,
|
||||
syntheticUsesTypeScriptLoader: false,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
assert.ok(errors.some((error) => error.includes("Jiti fallback")));
|
||||
assert.ok(errors.some((error) => error.includes("tsx loader strategy")));
|
||||
assert.ok(errors.some((error) => error.includes("TypeScript loader strategy")));
|
||||
});
|
||||
|
||||
103
test/release-followthrough.test.js
Normal file
103
test/release-followthrough.test.js
Normal file
@ -0,0 +1,103 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdir, mkdtemp, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import { buildCrabpotFollowthroughChecklist } from "../scripts/check-crabpot-followthrough.mjs";
|
||||
|
||||
test("crabpot follow-through checklist passes matching source refs", async () => {
|
||||
const roots = await createFixtureRoots({
|
||||
ref: "abc123",
|
||||
packagePin: "@openclaw/plugin-inspector@1.2.3",
|
||||
});
|
||||
|
||||
const result = buildCrabpotFollowthroughChecklist({
|
||||
pluginInspectorRoot: roots.pluginInspectorRoot,
|
||||
crabpotRoot: roots.crabpotRoot,
|
||||
expectedRef: "abc123",
|
||||
expectedVersion: "1.2.3",
|
||||
});
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.checks.find((check) => check.id === "crabpot-source-ref").status, "pass");
|
||||
assert.equal(result.checks.find((check) => check.id === "crabpot-public-api-migration").status, "pass");
|
||||
assert.equal(result.checks.find((check) => check.id === "crabpot-package-pin").status, "pass");
|
||||
});
|
||||
|
||||
test("crabpot follow-through checklist fails stale refs and optionally stale package pins", async () => {
|
||||
const roots = await createFixtureRoots({
|
||||
ref: "old-ref",
|
||||
packagePin: "@openclaw/plugin-inspector@1.0.0",
|
||||
});
|
||||
|
||||
const preRelease = buildCrabpotFollowthroughChecklist({
|
||||
pluginInspectorRoot: roots.pluginInspectorRoot,
|
||||
crabpotRoot: roots.crabpotRoot,
|
||||
expectedRef: "new-ref",
|
||||
expectedVersion: "1.2.3",
|
||||
});
|
||||
const postRelease = buildCrabpotFollowthroughChecklist({
|
||||
pluginInspectorRoot: roots.pluginInspectorRoot,
|
||||
crabpotRoot: roots.crabpotRoot,
|
||||
expectedRef: "old-ref",
|
||||
expectedVersion: "1.2.3",
|
||||
requirePublishedPin: true,
|
||||
});
|
||||
|
||||
assert.equal(preRelease.status, "fail");
|
||||
assert.equal(preRelease.checks.find((check) => check.id === "crabpot-package-pin").status, "manual");
|
||||
assert.equal(postRelease.status, "fail");
|
||||
assert.equal(postRelease.checks.find((check) => check.id === "crabpot-package-pin").status, "fail");
|
||||
});
|
||||
|
||||
test("crabpot follow-through checklist fails advanced bundle script consumers", async () => {
|
||||
const roots = await createFixtureRoots({
|
||||
ref: "abc123",
|
||||
packagePin: "@openclaw/plugin-inspector@1.2.3",
|
||||
scripts: {
|
||||
"synthetic-probes.mjs": [
|
||||
'import { loadPluginInspector } from "./plugin-inspector-source.mjs";',
|
||||
"const pluginInspector = await loadPluginInspector();",
|
||||
"export const validateSyntheticProbePlan = pluginInspector.validateSyntheticProbePlan;",
|
||||
"",
|
||||
].join("\n"),
|
||||
},
|
||||
});
|
||||
|
||||
const result = buildCrabpotFollowthroughChecklist({
|
||||
pluginInspectorRoot: roots.pluginInspectorRoot,
|
||||
crabpotRoot: roots.crabpotRoot,
|
||||
expectedRef: "abc123",
|
||||
expectedVersion: "1.2.3",
|
||||
});
|
||||
|
||||
assert.equal(result.status, "fail");
|
||||
assert.equal(result.checks.find((check) => check.id === "crabpot-public-api-migration").status, "fail");
|
||||
assert.match(result.checks.find((check) => check.id === "crabpot-public-api-migration").actual, /synthetic-probes/);
|
||||
});
|
||||
|
||||
async function createFixtureRoots({ ref, packagePin, scripts = {} }) {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-release-followthrough-"));
|
||||
const pluginInspectorRoot = path.join(root, "plugin-inspector");
|
||||
const crabpotRoot = path.join(root, "crabpot");
|
||||
await mkdir(path.join(crabpotRoot, "scripts"), { recursive: true });
|
||||
await mkdir(pluginInspectorRoot, { recursive: true });
|
||||
await writeFile(
|
||||
path.join(pluginInspectorRoot, "package.json"),
|
||||
`${JSON.stringify({ version: "1.2.3" }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(crabpotRoot, "scripts", "plugin-inspector-source.mjs"),
|
||||
[
|
||||
`export const pluginInspectorRef = "${ref}";`,
|
||||
`export const pluginInspectorPackage = "${packagePin}";`,
|
||||
"",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
for (const [scriptPath, content] of Object.entries(scripts)) {
|
||||
await writeFile(path.join(crabpotRoot, "scripts", scriptPath), content, "utf8");
|
||||
}
|
||||
return { pluginInspectorRoot, crabpotRoot };
|
||||
}
|
||||
38
test/release-notes.test.js
Normal file
38
test/release-notes.test.js
Normal file
@ -0,0 +1,38 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { extractReleaseNotes } from "../scripts/release-notes.mjs";
|
||||
|
||||
const changelog = [
|
||||
"# Changelog",
|
||||
"",
|
||||
"## Unreleased",
|
||||
"",
|
||||
"### Added",
|
||||
"",
|
||||
"- Add grouped API helpers.",
|
||||
"",
|
||||
"## 0.3.0 - 2026-04-27",
|
||||
"",
|
||||
"### Changed",
|
||||
"",
|
||||
"- Improve setup.",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
test("release notes can render unreleased draft notes", () => {
|
||||
assert.equal(
|
||||
extractReleaseNotes({ changelogText: changelog, version: "Unreleased" }),
|
||||
["## plugin-inspector unreleased", "", "### Added", "", "- Add grouped API helpers.", ""].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
test("release notes can render versioned changelog sections", () => {
|
||||
assert.equal(
|
||||
extractReleaseNotes({ changelogText: changelog, version: "0.3.0" }),
|
||||
["## plugin-inspector v0.3.0", "", "### Changed", "", "- Improve setup.", ""].join("\n"),
|
||||
);
|
||||
});
|
||||
|
||||
test("release notes fail when a version section is missing", () => {
|
||||
assert.throws(() => extractReleaseNotes({ changelogText: changelog, version: "9.9.9" }), /missing a 9\.9\.9/);
|
||||
});
|
||||
57
test/release-plan.test.js
Normal file
57
test/release-plan.test.js
Normal file
@ -0,0 +1,57 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { buildReleasePlan } from "../scripts/release-plan.mjs";
|
||||
|
||||
const changelog = [
|
||||
"# Changelog",
|
||||
"",
|
||||
"## Unreleased",
|
||||
"",
|
||||
"### Changed",
|
||||
"",
|
||||
"- Tighten package contents.",
|
||||
"",
|
||||
"## 0.3.0 - 2026-04-27",
|
||||
"",
|
||||
].join("\n");
|
||||
|
||||
test("release plan infers the next patch version", () => {
|
||||
const plan = buildReleasePlan({
|
||||
packageVersion: "0.3.0",
|
||||
changelogText: changelog,
|
||||
releaseDate: "2026-04-28",
|
||||
releaseRef: "abc123",
|
||||
});
|
||||
|
||||
assert.equal(plan.status, "pass");
|
||||
assert.equal(plan.nextVersion, "0.3.1");
|
||||
assert.equal(plan.tagName, "v0.3.1");
|
||||
assert.equal(plan.changelogHeading, "## 0.3.1 - 2026-04-28");
|
||||
assert.equal(plan.crabpotSourceRef, "abc123");
|
||||
assert.equal(plan.crabpotPackagePin, "@openclaw/plugin-inspector@0.3.1");
|
||||
});
|
||||
|
||||
test("release plan rejects non-advancing versions", () => {
|
||||
const plan = buildReleasePlan({
|
||||
packageVersion: "0.3.0",
|
||||
nextVersion: "0.3.0",
|
||||
changelogText: changelog,
|
||||
releaseDate: "2026-04-28",
|
||||
releaseRef: "abc123",
|
||||
});
|
||||
|
||||
assert.equal(plan.status, "fail");
|
||||
assert.equal(plan.checks.find((check) => check.id === "version-advance").status, "fail");
|
||||
});
|
||||
|
||||
test("release plan requires unreleased changelog bullets", () => {
|
||||
const plan = buildReleasePlan({
|
||||
packageVersion: "0.3.0",
|
||||
changelogText: "# Changelog\n\n## Unreleased\n",
|
||||
releaseDate: "2026-04-28",
|
||||
releaseRef: "abc123",
|
||||
});
|
||||
|
||||
assert.equal(plan.status, "fail");
|
||||
assert.equal(plan.checks.find((check) => check.id === "changelog-unreleased").status, "fail");
|
||||
});
|
||||
@ -4,22 +4,34 @@ import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
buildSarifReport,
|
||||
buildCompatibilityReport,
|
||||
buildCompatibilityFixtureReport,
|
||||
buildIssues,
|
||||
classifyCompatibilityFixture,
|
||||
classifyCompatRecordCoverage,
|
||||
classifyPackageContracts,
|
||||
classifyTargetOpenClawCoverage,
|
||||
escapeMarkdownTableCell,
|
||||
inspectFixtureSet,
|
||||
loadInspectorConfig,
|
||||
renderCompatibilityIssuesReport,
|
||||
renderCompatibilityMarkdownReport,
|
||||
renderJunitXml,
|
||||
renderMarkdownReport,
|
||||
renderMarkdownTable,
|
||||
renderTextSummary,
|
||||
writeArtifacts,
|
||||
writeCompatibilityReport,
|
||||
writeCiOutputArtifacts,
|
||||
writeReport,
|
||||
} from "../src/advanced.js";
|
||||
|
||||
test("markdown table cell escaping preserves literal backslashes", () => {
|
||||
assert.equal(escapeMarkdownTableCell(String.raw`C:\tmp|next
|
||||
line`), String.raw`C:\\tmp\|next<br>line`);
|
||||
});
|
||||
|
||||
test("markdown report includes summary and inventory", async () => {
|
||||
const config = await loadInspectorConfig("test/fixtures/inspector.config.json");
|
||||
const report = await inspectFixtureSet(config);
|
||||
@ -29,6 +41,53 @@ test("markdown report includes summary and inventory", async () => {
|
||||
assert.match(markdown, /\| sample-plugin \| high \| native-tool \| before_tool_call \| definePluginEntry, registerTool \| tools \|/);
|
||||
});
|
||||
|
||||
test("text summary includes artifact paths and top blocking findings", () => {
|
||||
const summary = renderTextSummary(
|
||||
{
|
||||
status: "fail",
|
||||
summary: {
|
||||
fixtureCount: 1,
|
||||
breakageCount: 1,
|
||||
issueCount: 1,
|
||||
logCount: 0,
|
||||
},
|
||||
breakages: [
|
||||
{
|
||||
fixture: "weather",
|
||||
code: "missing-expected-seam",
|
||||
level: "breakage",
|
||||
message: "weather: missing expected registration registerTool",
|
||||
evidence: ["src/index.js:12"],
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
issues: [
|
||||
{
|
||||
fixture: "weather",
|
||||
code: "sdk-export-missing",
|
||||
severity: "P0",
|
||||
status: "blocking",
|
||||
title: "SDK export is missing",
|
||||
evidence: ["src/index.js:1"],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
artifacts: {
|
||||
jsonPath: "reports/plugin-inspector-report.json",
|
||||
markdownPath: "reports/plugin-inspector-report.md",
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
assert.match(summary, /Status: FAIL/);
|
||||
assert.match(summary, /Issues: 1/);
|
||||
assert.match(summary, /Reports:/);
|
||||
assert.match(summary, /json: reports\/plugin-inspector-report\.json/);
|
||||
assert.match(summary, /Top findings:/);
|
||||
assert.match(summary, /BREAKAGE weather missing-expected-seam/);
|
||||
});
|
||||
|
||||
test("compatibility report renderer supports issue metadata and evidence links", () => {
|
||||
const report = {
|
||||
generatedAt: "test",
|
||||
@ -134,6 +193,91 @@ test("compatibility report renderer supports issue metadata and evidence links",
|
||||
assert.match(issues, /\[linked\]\(plugins\/sample\/src\/index\.ts:1\)/);
|
||||
});
|
||||
|
||||
test("compatibility report artifacts sanitize absolute OpenClaw target paths", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-sanitized-report-"));
|
||||
const absoluteOpenClawPath = path.join(outDir, "openclaw");
|
||||
const report = {
|
||||
generatedAt: "test",
|
||||
status: "pass",
|
||||
targetOpenClaw: {
|
||||
status: "ok",
|
||||
configuredPath: absoluteOpenClawPath,
|
||||
searchedPaths: [absoluteOpenClawPath],
|
||||
compatRecords: [],
|
||||
compatRecordStatuses: {},
|
||||
},
|
||||
summary: {
|
||||
fixtureCount: 1,
|
||||
highPriorityFixtures: 1,
|
||||
breakageCount: 0,
|
||||
warningCount: 0,
|
||||
suggestionCount: 0,
|
||||
decisionCount: 0,
|
||||
issueCount: 1,
|
||||
p0IssueCount: 0,
|
||||
p1IssueCount: 0,
|
||||
liveIssueCount: 0,
|
||||
liveP0IssueCount: 0,
|
||||
compatGapCount: 0,
|
||||
deprecationWarningCount: 0,
|
||||
inspectorGapCount: 1,
|
||||
upstreamIssueCount: 0,
|
||||
fixtureRegressionCount: 0,
|
||||
contractProbeCount: 0,
|
||||
},
|
||||
fixtures: [
|
||||
{
|
||||
id: "sample-plugin",
|
||||
priority: "high",
|
||||
seams: ["native-tool"],
|
||||
hooks: [],
|
||||
registrations: [],
|
||||
manifestContracts: [],
|
||||
},
|
||||
],
|
||||
breakages: [],
|
||||
warnings: [],
|
||||
suggestions: [],
|
||||
issues: [
|
||||
{
|
||||
fixture: "sample-plugin",
|
||||
code: "package-dependency-install-required",
|
||||
issueClass: "inspector-gap",
|
||||
decision: "inspector-follow-up",
|
||||
severity: "P2",
|
||||
title: `sample-plugin: path ${absoluteOpenClawPath}`,
|
||||
status: "open",
|
||||
compatStatus: "none",
|
||||
live: false,
|
||||
evidence: [absoluteOpenClawPath],
|
||||
},
|
||||
],
|
||||
contractProbes: [],
|
||||
logs: [],
|
||||
decisions: [],
|
||||
};
|
||||
|
||||
const markdown = renderCompatibilityMarkdownReport(report);
|
||||
const issues = renderCompatibilityIssuesReport(report);
|
||||
const paths = await writeCompatibilityReport(report, {
|
||||
jsonPath: path.join(outDir, "report.json"),
|
||||
markdownPath: path.join(outDir, "report.md"),
|
||||
issuesPath: path.join(outDir, "issues.md"),
|
||||
});
|
||||
const artifact = JSON.parse(await readFile(paths.jsonPath, "utf8"));
|
||||
|
||||
assert.equal(report.targetOpenClaw.configuredPath, absoluteOpenClawPath);
|
||||
assert.equal(artifact.targetOpenClaw.configuredPath, "<OPENCLAW_PATH>");
|
||||
assert.deepEqual(artifact.targetOpenClaw.searchedPaths, ["<OPENCLAW_PATH>"]);
|
||||
assert.equal(artifact.issues[0].evidence[0], "<OPENCLAW_PATH>");
|
||||
assert.equal(artifact.issues[0].title, "sample-plugin: path <OPENCLAW_PATH>");
|
||||
assert.doesNotMatch(markdown, new RegExp(escapeRegExp(absoluteOpenClawPath)));
|
||||
assert.doesNotMatch(issues, new RegExp(escapeRegExp(absoluteOpenClawPath)));
|
||||
assert.match(markdown, /<OPENCLAW_PATH>/);
|
||||
assert.match(await readFile(paths.markdownPath, "utf8"), /<OPENCLAW_PATH>/);
|
||||
assert.match(await readFile(paths.issuesPath, "utf8"), /<OPENCLAW_PATH>/);
|
||||
});
|
||||
|
||||
test("compatibility report assembly classifies fixtures, issues, probes, and compat records", async () => {
|
||||
const report = await buildCompatibilityReport({
|
||||
generatedAt: "test",
|
||||
@ -226,6 +370,110 @@ test("compatibility report assembly classifies fixtures, issues, probes, and com
|
||||
assert.ok(report.decisions.some((decision) => decision.seam === "compat-registry"));
|
||||
});
|
||||
|
||||
test("compatibility report marks inspector gaps covered by runtime execution artifacts", async () => {
|
||||
const report = await buildCompatibilityReport({
|
||||
generatedAt: "test",
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
name: "Fixture",
|
||||
path: "plugins/fixture",
|
||||
priority: "high",
|
||||
seams: ["native-tool"],
|
||||
why: "covers runtime-only seams",
|
||||
},
|
||||
],
|
||||
inspections: [
|
||||
{
|
||||
id: "fixture",
|
||||
status: "ok",
|
||||
hooks: ["llm_input"],
|
||||
hookDetails: [{ name: "llm_input", ref: "plugins/fixture/src/index.ts:1" }],
|
||||
registrations: ["registerTool", "registerService", "registerCommand"],
|
||||
registrationDetails: [
|
||||
{ name: "registerTool", ref: "plugins/fixture/src/index.ts:2" },
|
||||
{ name: "registerService", ref: "plugins/fixture/src/index.ts:3" },
|
||||
{ name: "registerCommand", ref: "plugins/fixture/src/index.ts:4" },
|
||||
],
|
||||
manifestContracts: [],
|
||||
manifestFiles: [],
|
||||
sdkImports: [],
|
||||
sourceFiles: ["plugins/fixture/src/index.ts"],
|
||||
},
|
||||
],
|
||||
targetOpenClaw: {
|
||||
status: "ok",
|
||||
compatRecords: [],
|
||||
compatRecordStatuses: {},
|
||||
hookNames: ["llm_input"],
|
||||
apiRegistrars: ["registerTool", "registerService", "registerCommand"],
|
||||
capturedRegistrars: [],
|
||||
sdkExports: [],
|
||||
manifestFields: ["id"],
|
||||
manifestContractFields: [],
|
||||
},
|
||||
executionResults: {
|
||||
artifacts: [
|
||||
{
|
||||
fixture: "fixture",
|
||||
kind: "capture",
|
||||
status: "pass",
|
||||
artifactPath: ".crabpot/results/fixture/index.capture.json",
|
||||
captured: [
|
||||
"hook:llm_input",
|
||||
"registration:registerTool",
|
||||
"registration:registerService",
|
||||
"registration:registerCommand",
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
buildFixtureReport: ({ fixture, inspection }) => ({
|
||||
id: fixture.id,
|
||||
name: fixture.name,
|
||||
priority: fixture.priority,
|
||||
seams: fixture.seams,
|
||||
why: fixture.why,
|
||||
status: inspection.status,
|
||||
hooks: inspection.hooks,
|
||||
hookDetails: inspection.hookDetails,
|
||||
registrations: inspection.registrations,
|
||||
registrationDetails: inspection.registrationDetails,
|
||||
manifestContracts: inspection.manifestContracts,
|
||||
manifestFiles: [],
|
||||
sourceFiles: inspection.sourceFiles,
|
||||
pluginManifests: [],
|
||||
package: null,
|
||||
packages: [],
|
||||
sdkImports: [],
|
||||
sdkImportDetails: [],
|
||||
}),
|
||||
});
|
||||
|
||||
const coveredCodes = report.issues
|
||||
.filter((issue) => issue.status === "runtime-covered")
|
||||
.map((issue) => issue.code)
|
||||
.sort();
|
||||
assert.deepEqual(coveredCodes, [
|
||||
"conversation-access-hook",
|
||||
"registration-capture-gap",
|
||||
"runtime-tool-capture",
|
||||
]);
|
||||
assert.equal(report.summary.runtimeCoveredIssueCount, 3);
|
||||
assert.equal(report.summary.openInspectorGapCount, 0);
|
||||
assert.equal(report.summary.runtimeCoverageArtifactCount, 1);
|
||||
|
||||
const registrationIssue = report.issues.find((issue) => issue.code === "registration-capture-gap");
|
||||
assert.deepEqual(registrationIssue.runtimeCoverage.captured, [
|
||||
"registration:registerService",
|
||||
"registration:registerCommand",
|
||||
]);
|
||||
|
||||
const markdown = renderCompatibilityIssuesReport(report);
|
||||
assert.match(markdown, /## Runtime-Covered Inspector Gaps/);
|
||||
assert.match(markdown, /state: runtime-covered .* runtime:covered/);
|
||||
});
|
||||
|
||||
test("compat record coverage logs unavailable targets", () => {
|
||||
const logs = [];
|
||||
classifyCompatRecordCoverage({
|
||||
@ -254,6 +502,21 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
|
||||
`${JSON.stringify({ id: "fixture", name: "Fixture", version: "1.0.0", contracts: { tools: {} } }, null, 2)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(fixtureDir, "openclaw.security.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
$schema: "https://openclaw.ai/schemas/plugin-security.json",
|
||||
version: "1.0.0",
|
||||
plugin: "fixture",
|
||||
expectedBehaviors: [{ id: "api-key", description: "requires an API key" }],
|
||||
securityNotes: [{ id: "storage", description: "stores local state" }],
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(path.join(fixtureDir, "src", "index.js"), "export function register() {}\n", "utf8");
|
||||
await writeFile(
|
||||
path.join(fixtureDir, "package.json"),
|
||||
@ -262,10 +525,28 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
|
||||
name: "fixture-plugin",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
files: ["src", "openclaw.plugin.json"],
|
||||
dependencies: { zod: "^1.0.0" },
|
||||
openclaw: {
|
||||
extensions: ["src/index.js"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
build: {
|
||||
openclawVersion: "2026.5.2",
|
||||
pluginSdkVersion: "2026.5.2",
|
||||
},
|
||||
install: {
|
||||
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
defaultChoice: "clawhub",
|
||||
minHostVersion: ">=2026.5.2",
|
||||
},
|
||||
release: {
|
||||
publishToClawHub: true,
|
||||
publishToNpm: true,
|
||||
},
|
||||
bundle: {
|
||||
includeInCore: false,
|
||||
},
|
||||
},
|
||||
},
|
||||
null,
|
||||
@ -298,8 +579,35 @@ test("compatibility fixture summary reads manifests and OpenClaw package metadat
|
||||
});
|
||||
|
||||
assert.equal(report.pluginManifests[0].id, "fixture");
|
||||
assert.deepEqual(report.securityManifests[0], {
|
||||
path: "plugin/openclaw.security.json",
|
||||
schema: "https://openclaw.ai/schemas/plugin-security.json",
|
||||
version: "1.0.0",
|
||||
plugin: "fixture",
|
||||
expectedBehaviorCount: 1,
|
||||
securityNoteCount: 1,
|
||||
validJson: true,
|
||||
});
|
||||
assert.equal(report.package.name, "fixture-plugin");
|
||||
assert.deepEqual(report.package.npmPack, {
|
||||
advertised: true,
|
||||
private: false,
|
||||
filesMode: "allowlist",
|
||||
files: ["src", "openclaw.plugin.json"],
|
||||
invalidFileSpecs: [],
|
||||
});
|
||||
assert.equal(report.package.openclaw.compatPluginApi, "^1.0.0");
|
||||
assert.deepEqual(report.package.openclaw.install, {
|
||||
clawhubSpec: "clawhub:@openclaw/fixture-plugin",
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
defaultChoice: "clawhub",
|
||||
minHostVersion: ">=2026.5.2",
|
||||
});
|
||||
assert.deepEqual(report.package.openclaw.release, {
|
||||
publishToClawHub: true,
|
||||
publishToNpm: true,
|
||||
});
|
||||
assert.deepEqual(report.package.openclaw.unsupportedMetadata, ["openclaw.bundle"]);
|
||||
assert.deepEqual(report.package.openclaw.entrypoints[0], {
|
||||
kind: "extension",
|
||||
specifier: "src/index.js",
|
||||
@ -350,6 +658,284 @@ test("package contract classifier reports install and entrypoint blockers", () =
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "package-build-artifact-entrypoint"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "package-dependency-install-required"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "cold-import"));
|
||||
|
||||
const issues = buildIssues({
|
||||
suggestions: result.suggestions,
|
||||
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
|
||||
});
|
||||
assert.ok(
|
||||
issues.some(
|
||||
(issue) =>
|
||||
issue.code === "package-dependency-install-required" &&
|
||||
issue.title === "fixture: cold import requires dependency installation in an isolated workspace",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("package contract classifier treats openclaw as a host-linked dependency", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "fixture-plugin",
|
||||
version: "1.0.0",
|
||||
dependencies: ["openclaw"],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "dist/index.js",
|
||||
relativePath: "plugins/fixture/dist/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(
|
||||
result.suggestions.some((finding) => finding.code === "package-dependency-install-required"),
|
||||
false,
|
||||
);
|
||||
});
|
||||
|
||||
test("package contract classifier reports broken install and release metadata", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
dependencies: [],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
buildOpenClawVersion: "2026.5.2",
|
||||
install: {
|
||||
clawhubSpec: null,
|
||||
npmSpec: "fixture-plugin",
|
||||
defaultChoice: "clawhub",
|
||||
minHostVersion: ">=2026.5.1",
|
||||
},
|
||||
release: {
|
||||
publishToClawHub: true,
|
||||
publishToNpm: true,
|
||||
},
|
||||
unsupportedMetadata: ["openclaw.bundle"],
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "dist/index.js",
|
||||
relativePath: "plugins/fixture/dist/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-install-metadata-incomplete"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-min-host-version-drift"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-openclaw-unsupported-metadata"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "package-metadata"));
|
||||
});
|
||||
|
||||
test("package contract classifier reports advertised npm pack blockers", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
npmPack: {
|
||||
advertised: true,
|
||||
private: true,
|
||||
filesMode: "allowlist",
|
||||
files: ["README.md"],
|
||||
invalidFileSpecs: ["../secrets"],
|
||||
},
|
||||
dependencies: [],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
install: {
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
},
|
||||
release: {
|
||||
publishToNpm: true,
|
||||
},
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "src/index.js",
|
||||
relativePath: "plugins/fixture/src/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-unavailable"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-metadata-missing"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "package-artifact"));
|
||||
|
||||
const globResult = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
npmPack: {
|
||||
advertised: true,
|
||||
private: false,
|
||||
filesMode: "allowlist",
|
||||
files: ["src/**/*.js", "openclaw.plugin.json"],
|
||||
invalidFileSpecs: [],
|
||||
},
|
||||
dependencies: [],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
install: {
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
},
|
||||
release: {
|
||||
publishToNpm: true,
|
||||
},
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "src/index.js",
|
||||
relativePath: "plugins/fixture/src/index.js",
|
||||
exists: true,
|
||||
requiresBuild: false,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(globResult.warnings.some((finding) => finding.code.startsWith("package-npm-pack-")), false);
|
||||
});
|
||||
|
||||
test("package contract classifier accepts built runtime entries for source package metadata", () => {
|
||||
const result = classifyPackageContracts({
|
||||
fixture: {
|
||||
id: "fixture",
|
||||
path: "plugins/fixture",
|
||||
},
|
||||
inspection: {
|
||||
registrations: ["registerTool"],
|
||||
},
|
||||
fixtureReport: {
|
||||
pluginManifests: [{ path: "plugins/fixture/openclaw.plugin.json", version: "1.0.0" }],
|
||||
package: {
|
||||
path: "plugins/fixture/package.json",
|
||||
name: "@openclaw/fixture-plugin",
|
||||
version: "1.0.0",
|
||||
npmPack: {
|
||||
advertised: true,
|
||||
private: false,
|
||||
filesMode: "allowlist",
|
||||
files: ["dist/**", "openclaw.plugin.json"],
|
||||
invalidFileSpecs: [],
|
||||
},
|
||||
dependencies: [],
|
||||
peerDependencies: ["openclaw"],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
compatPluginApi: "^1.0.0",
|
||||
install: {
|
||||
npmSpec: "@openclaw/fixture-plugin",
|
||||
},
|
||||
release: {
|
||||
publishToNpm: true,
|
||||
},
|
||||
entrypoints: [
|
||||
{
|
||||
kind: "extension",
|
||||
specifier: "./index.ts",
|
||||
relativePath: "plugins/fixture/index.ts",
|
||||
exists: false,
|
||||
requiresBuild: false,
|
||||
},
|
||||
{
|
||||
kind: "runtimeExtension",
|
||||
specifier: "./dist/index.js",
|
||||
relativePath: "plugins/fixture/dist/index.js",
|
||||
exists: true,
|
||||
requiresBuild: true,
|
||||
},
|
||||
{
|
||||
kind: "setupEntry",
|
||||
specifier: "./setup-entry.ts",
|
||||
relativePath: "plugins/fixture/setup-entry.ts",
|
||||
exists: false,
|
||||
requiresBuild: false,
|
||||
},
|
||||
{
|
||||
kind: "runtimeExtension",
|
||||
specifier: "./dist/setup-entry.js",
|
||||
relativePath: "plugins/fixture/dist/setup-entry.js",
|
||||
exists: true,
|
||||
requiresBuild: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.warnings.some((finding) => finding.code === "package-entrypoint-missing"), false);
|
||||
assert.equal(result.warnings.some((finding) => finding.code === "package-npm-pack-entrypoint-missing"), false);
|
||||
assert.equal(result.decisions.some((decision) => decision.seam === "package-entrypoint"), false);
|
||||
});
|
||||
|
||||
test("target OpenClaw coverage classifier reports missing public surface", () => {
|
||||
@ -427,6 +1013,17 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
|
||||
channelEnvVars: { CHANNEL_ID: "channel id" },
|
||||
},
|
||||
],
|
||||
securityManifests: [
|
||||
{
|
||||
path: "plugins/fixture/openclaw.security.json",
|
||||
schema: "https://openclaw.ai/schemas/plugin-security.json",
|
||||
version: "1.0.0",
|
||||
plugin: "fixture",
|
||||
expectedBehaviorCount: 1,
|
||||
securityNoteCount: 1,
|
||||
validJson: true,
|
||||
},
|
||||
],
|
||||
package: null,
|
||||
},
|
||||
targetOpenClaw: {
|
||||
@ -442,13 +1039,48 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "provider-auth-env-vars"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "channel-env-vars"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "conversation-access-hook"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "unrecognized-security-manifest"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "security-manifest-schema-unavailable"));
|
||||
assert.ok(
|
||||
result.warnings.some(
|
||||
(finding) =>
|
||||
finding.code === "conversation-access-hook" &&
|
||||
finding.compatRecord === "hook.llm-observer.privacy-payload",
|
||||
),
|
||||
);
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "legacy-root-sdk-import"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-json-missing"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "registration-capture-gap"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "before-tool-call-probe"));
|
||||
assert.ok(
|
||||
result.suggestions.some(
|
||||
(finding) =>
|
||||
finding.code === "registration-capture-gap" &&
|
||||
finding.compatRecord === "api.capture.runtime-registrars",
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
result.suggestions.some(
|
||||
(finding) =>
|
||||
finding.code === "before-tool-call-probe" &&
|
||||
finding.compatRecord === "hook.before_tool_call.terminal-block-approval",
|
||||
),
|
||||
);
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "runtime-tool-capture"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "conversation-access"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "security-metadata"));
|
||||
|
||||
const issues = buildIssues({
|
||||
warnings: result.warnings,
|
||||
suggestions: result.suggestions,
|
||||
targetOpenClaw: { status: "ok", compatRecordStatuses: {} },
|
||||
});
|
||||
assert.ok(
|
||||
issues.some(
|
||||
(issue) =>
|
||||
issue.code === "unrecognized-security-manifest" &&
|
||||
issue.issueClass === "upstream-metadata" &&
|
||||
issue.severity === "P3",
|
||||
),
|
||||
);
|
||||
});
|
||||
|
||||
test("writeReport writes JSON and Markdown artifacts", async () => {
|
||||
@ -464,6 +1096,44 @@ test("writeReport writes JSON and Markdown artifacts", async () => {
|
||||
assert.match(markdown, /Status: PASS/);
|
||||
});
|
||||
|
||||
test("CI output helpers write SARIF and JUnit artifacts", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-ci-outputs-"));
|
||||
const report = {
|
||||
status: "fail",
|
||||
summary: { fixtureCount: 1, breakageCount: 1 },
|
||||
fixtures: [{ id: "weather", path: "." }],
|
||||
breakages: [
|
||||
{
|
||||
fixture: "weather",
|
||||
code: "missing-expected-seam",
|
||||
level: "breakage",
|
||||
message: "weather: missing expected registration registerTool",
|
||||
evidence: ["registerTool @ src/index.js:12:4"],
|
||||
},
|
||||
],
|
||||
warnings: [],
|
||||
suggestions: [],
|
||||
issues: [],
|
||||
};
|
||||
|
||||
const sarif = buildSarifReport(report);
|
||||
const junit = renderJunitXml(report);
|
||||
const paths = await writeCiOutputArtifacts(report, {
|
||||
outDir,
|
||||
sarifPath: "plugin-inspector.sarif",
|
||||
junitPath: "plugin-inspector.junit.xml",
|
||||
});
|
||||
|
||||
assert.equal(sarif.version, "2.1.0");
|
||||
assert.equal(sarif.runs[0].results[0].ruleId, "missing-expected-seam");
|
||||
assert.equal(sarif.runs[0].results[0].level, "error");
|
||||
assert.equal(sarif.runs[0].results[0].locations[0].physicalLocation.artifactLocation.uri, "src/index.js");
|
||||
assert.match(junit, /tests="1" failures="1"/);
|
||||
assert.match(junit, /missing expected registration registerTool/);
|
||||
assert.equal(JSON.parse(await readFile(paths.sarifPath, "utf8")).runs[0].results.length, 1);
|
||||
assert.match(await readFile(paths.junitPath, "utf8"), /<testsuite name="plugin-inspector"/);
|
||||
});
|
||||
|
||||
test("artifact helpers write stable CI files", async () => {
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-artifacts-"));
|
||||
const jsonPath = path.join(outDir, "summary.json");
|
||||
@ -496,3 +1166,7 @@ test("markdown table helper supports padded empty-table reports", () => {
|
||||
);
|
||||
assert.equal(renderMarkdownTable([], ["Name"], { empty: "_none_" }), "_none_");
|
||||
});
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
||||
}
|
||||
|
||||
@ -5,6 +5,7 @@ import path from "node:path";
|
||||
import { test } from "node:test";
|
||||
import {
|
||||
buildRuntimeCaptureReport,
|
||||
captureEntrypoint,
|
||||
inspectCompatibilityFixtureSet,
|
||||
loadPluginRootConfig,
|
||||
writeRuntimeCaptureReport,
|
||||
@ -61,14 +62,59 @@ test("runtime capture report imports plugin entrypoints with mocked SDK", async
|
||||
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
|
||||
});
|
||||
|
||||
test("runtime capture report classifies missing mocked SDK exports", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-"));
|
||||
test("runtime capture records conversation binding resolved callbacks", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-binding-resolved-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-missing-sdk-export",
|
||||
name: "openclaw-binding-resolved",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.mjs"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "src", "index.mjs"),
|
||||
[
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk";',
|
||||
"",
|
||||
"export default definePluginEntry((api) => {",
|
||||
" api.onConversationBindingResolved(() => undefined);",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const config = await loadPluginRootConfig(null, { cwd: rootDir });
|
||||
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
|
||||
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
|
||||
|
||||
assert.equal(captureReport.summary.failedCount, 0);
|
||||
assert.equal(captureReport.summary.capturedCount, 1);
|
||||
assert.equal(captureReport.summary.hookCount, 1);
|
||||
assert.deepEqual(
|
||||
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
|
||||
["hook:onConversationBindingResolved"],
|
||||
);
|
||||
});
|
||||
|
||||
test("runtime capture report synthesizes newly imported mocked SDK exports", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-dynamic-sdk-export",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
@ -86,9 +132,10 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
|
||||
[
|
||||
'import { definitelyMissing } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
"",
|
||||
"export default definitelyMissing({",
|
||||
" register() {},",
|
||||
"});",
|
||||
"export function register(api) {",
|
||||
" if (!definitelyMissing) throw new Error('expected dynamic mock export');",
|
||||
" api.registerTool({ name: 'fixture_tool', inputSchema: { type: 'object' }, run() {} });",
|
||||
"}",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
@ -97,17 +144,18 @@ test("runtime capture report classifies missing mocked SDK exports", async () =>
|
||||
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
|
||||
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
|
||||
|
||||
assert.equal(captureReport.summary.failedCount, 1);
|
||||
assert.equal(captureReport.results[0].status, "error");
|
||||
assert.equal(captureReport.results[0].failureClass, "missing-sdk-export");
|
||||
assert.equal(captureReport.results[0].missingExport, "definitelyMissing");
|
||||
assert.equal(captureReport.summary.failedCount, 0);
|
||||
assert.equal(captureReport.results[0].status, "captured");
|
||||
assert.deepEqual(captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`), [
|
||||
"registration:registerTool",
|
||||
]);
|
||||
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-missing-export-out-"));
|
||||
const outDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-capture-dynamic-export-out-"));
|
||||
await writeRuntimeCaptureReport(captureReport, {
|
||||
jsonPath: path.join(outDir, "capture.json"),
|
||||
markdownPath: path.join(outDir, "capture.md"),
|
||||
});
|
||||
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /missing-sdk-export/);
|
||||
assert.match(await readFile(path.join(outDir, "capture.md"), "utf8"), /registerTool/);
|
||||
});
|
||||
|
||||
test("runtime capture report classifies registration execution failures", async () => {
|
||||
@ -227,6 +275,83 @@ test("runtime capture supports TypeScript entrypoints, SDK subpaths, external mo
|
||||
assert.match(captureReport.results[0].processOutput.stdout, /late plugin noise/);
|
||||
});
|
||||
|
||||
test("runtime capture synthesizes manifest config before plugin registration", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-config-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-configured-memory",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.ts"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "openclaw.plugin.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
id: "configured-memory",
|
||||
configSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
embedding: {
|
||||
type: "object",
|
||||
minProperties: 1,
|
||||
properties: {
|
||||
provider: { type: "string" },
|
||||
model: { type: "string" },
|
||||
},
|
||||
},
|
||||
autoCapture: { type: "boolean" },
|
||||
autoRecall: { type: "boolean" },
|
||||
},
|
||||
required: ["embedding"],
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "src", "index.ts"),
|
||||
[
|
||||
'import { definePluginEntry } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
"export default definePluginEntry({",
|
||||
" register(api) {",
|
||||
" if (!api.pluginConfig?.embedding) {",
|
||||
" api.registerService({ id: 'configured-memory-disabled', start() {} });",
|
||||
" return;",
|
||||
" }",
|
||||
" api.on('agent_end', () => undefined);",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const result = await captureEntrypoint("src/index.ts", {
|
||||
cwd: rootDir,
|
||||
pluginRoot: rootDir,
|
||||
mockSdk: true,
|
||||
});
|
||||
|
||||
assert.equal(result.status, "captured");
|
||||
assert.deepEqual(
|
||||
result.captured.map((item) => `${item.kind}:${item.name}`),
|
||||
["hook:agent_end"],
|
||||
);
|
||||
});
|
||||
|
||||
test("runtime capture supports namespace imports from mocked externals", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-zod-namespace-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
@ -268,6 +393,134 @@ test("runtime capture supports namespace imports from mocked externals", async (
|
||||
assert.equal(captureReport.summary.registrationCount, 1);
|
||||
});
|
||||
|
||||
test("runtime capture mock SDK supports config schemas and provider catalogs", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-provider-catalog-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-provider-catalog",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.mjs"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "src", "index.mjs"),
|
||||
[
|
||||
'import { buildPluginConfigSchema } from "openclaw/plugin-sdk/plugin-entry";',
|
||||
'import { buildSingleProviderApiKeyCatalog, createProviderApiKeyAuthMethod, defineSingleProviderPluginEntry } from "openclaw/plugin-sdk/provider-entry";',
|
||||
"",
|
||||
"const config = buildPluginConfigSchema({ providerId: { parse: (value) => value ?? 'fixture-provider' } }).parse({});",
|
||||
"const auth = createProviderApiKeyAuthMethod({ id: 'fixture-key' });",
|
||||
"const catalog = buildSingleProviderApiKeyCatalog({",
|
||||
" id: config.providerId,",
|
||||
" auth,",
|
||||
" buildModels: async () => [{ id: 'fixture-model' }],",
|
||||
"});",
|
||||
"const listed = await catalog.run({ apiKey: 'redacted' });",
|
||||
"",
|
||||
"export default defineSingleProviderPluginEntry({",
|
||||
" id: config.providerId,",
|
||||
" provider: listed.provider,",
|
||||
" register(api) {",
|
||||
" api.registerProvider({ id: listed.provider.id, auth, catalog, modelCount: listed.models.length });",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const config = await loadPluginRootConfig(null, { cwd: rootDir });
|
||||
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
|
||||
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
|
||||
|
||||
assert.equal(captureReport.summary.failedCount, 0);
|
||||
assert.equal(captureReport.summary.registrationCount, 2);
|
||||
assert.deepEqual(
|
||||
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
|
||||
["registration:registerProvider", "registration:registerProvider"],
|
||||
);
|
||||
});
|
||||
|
||||
test("runtime capture mock SDK supports channel entries, gateway responders, and action gates", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-channel-entry-"));
|
||||
await mkdir(path.join(rootDir, "src"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(rootDir, "package.json"),
|
||||
`${JSON.stringify(
|
||||
{
|
||||
name: "openclaw-channel-entry",
|
||||
version: "1.0.0",
|
||||
type: "module",
|
||||
openclaw: {
|
||||
extensions: ["src/index.mjs"],
|
||||
compat: { pluginApi: "^1.0.0" },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
)}\n`,
|
||||
"utf8",
|
||||
);
|
||||
await writeFile(
|
||||
path.join(rootDir, "src", "index.mjs"),
|
||||
[
|
||||
'import { createActionGate } from "openclaw/plugin-sdk/channel-actions";',
|
||||
'import { buildChannelOutboundSessionRoute, createChannelPluginBase, createChatChannelPlugin, defineChannelPluginEntry } from "openclaw/plugin-sdk/channel-core";',
|
||||
"",
|
||||
"const gate = createActionGate({ gateway: true });",
|
||||
"const route = buildChannelOutboundSessionRoute({ agentId: 'agent', channel: 'fixture-channel', to: 'sender', chatType: 'direct' });",
|
||||
"const plugin = createChatChannelPlugin({",
|
||||
" base: createChannelPluginBase({",
|
||||
" id: 'fixture-channel',",
|
||||
" setup: () => ({ accountId: 'default' }),",
|
||||
" }),",
|
||||
" outbound: {",
|
||||
" base: { resolveOutboundSessionRoute: () => route },",
|
||||
" attachedResults: {",
|
||||
" channel: 'fixture-channel',",
|
||||
" sendText: async ({ text }) => ({ messageId: text }),",
|
||||
" },",
|
||||
" },",
|
||||
"});",
|
||||
"",
|
||||
"export default defineChannelPluginEntry({",
|
||||
" id: 'fixture-channel-entry',",
|
||||
" name: 'Fixture channel',",
|
||||
" description: 'Fixture channel',",
|
||||
" plugin,",
|
||||
" registerFull(api) {",
|
||||
" if (gate('gateway')) {",
|
||||
" api.registerGatewayMethod('fixture.ping', ({ respond }) => respond(true, { ok: true }), { scope: 'operator.read' });",
|
||||
" }",
|
||||
" api.registerService({ id: 'fixture-service', start() {}, stop() {} });",
|
||||
" },",
|
||||
"});",
|
||||
].join("\n"),
|
||||
"utf8",
|
||||
);
|
||||
|
||||
const config = await loadPluginRootConfig(null, { cwd: rootDir });
|
||||
const compatibilityReport = await inspectCompatibilityFixtureSet(config, { openclawPath: false });
|
||||
const captureReport = await buildRuntimeCaptureReport({ report: compatibilityReport, rootDir });
|
||||
|
||||
assert.equal(captureReport.summary.failedCount, 0);
|
||||
assert.deepEqual(
|
||||
captureReport.results[0].captured.map((entry) => `${entry.kind}:${entry.name}`),
|
||||
["registration:registerChannel", "registration:registerGatewayMethod", "registration:registerService"],
|
||||
);
|
||||
assert.equal(captureReport.results[0].captured[0].arguments[0].keys.includes("plugin"), true);
|
||||
});
|
||||
|
||||
test("runtime capture keeps dist chunk imports rooted at their original package", async () => {
|
||||
const rootDir = await mkdtemp(path.join(os.tmpdir(), "plugin-inspector-runtime-dist-"));
|
||||
await mkdir(path.join(rootDir, "dist", "extensions", "weather"), { recursive: true });
|
||||
|
||||
@ -10,6 +10,7 @@ import {
|
||||
runCapturedSyntheticProbes,
|
||||
validateSyntheticProbePlan,
|
||||
} from "../src/advanced.js";
|
||||
import { buildSyntheticProbePlanFromReport } from "../src/synthetic-probe-suite.js";
|
||||
|
||||
test("synthetic probe plan maps capture inventory to executable probes", () => {
|
||||
const plan = buildSyntheticProbePlan({
|
||||
@ -48,6 +49,33 @@ test("synthetic probe plan maps capture inventory to executable probes", () => {
|
||||
assert.match(renderSyntheticProbeMarkdown(plan), /registerTool/);
|
||||
});
|
||||
|
||||
test("synthetic probe plan can be built from a compatibility report", () => {
|
||||
const plan = buildSyntheticProbePlanFromReport({
|
||||
generatedAt: "test",
|
||||
targetOpenClaw: {
|
||||
capturedRegistrars: ["registerTool"],
|
||||
sdkExports: [],
|
||||
},
|
||||
summary: {},
|
||||
fixtures: [
|
||||
{
|
||||
id: "fixture",
|
||||
priority: "high",
|
||||
hookDetails: [{ name: "before_tool_call", ref: "src/index.js:1" }],
|
||||
registrationDetails: [{ name: "registerTool", ref: "src/index.js:2" }],
|
||||
sdkImportDetails: [],
|
||||
packages: [],
|
||||
},
|
||||
],
|
||||
contractProbes: [],
|
||||
});
|
||||
|
||||
assert.equal(plan.generatedAt, "test");
|
||||
assert.equal(plan.summary.probeCount, 2);
|
||||
assert.equal(plan.summary.readyCount, 2);
|
||||
assert.deepEqual(validateSyntheticProbePlan(plan), []);
|
||||
});
|
||||
|
||||
test("synthetic probe plan blocks unclassified registrars", () => {
|
||||
const plan = buildSyntheticProbePlan({
|
||||
capture: {
|
||||
@ -75,6 +103,85 @@ test("synthetic probe plan blocks unclassified registrars", () => {
|
||||
assert.match(validateSyntheticProbePlan(plan).join("\n"), /not been classified/);
|
||||
});
|
||||
|
||||
test("synthetic probe plan classifies generated kitchen-sink registrars", () => {
|
||||
const kitchenSinkRegistrars = [
|
||||
"createChatChannelPlugin",
|
||||
"registerAgentEventSubscription",
|
||||
"registerAgentHarness",
|
||||
"registerAgentToolResultMiddleware",
|
||||
"registerAutoEnableProbe",
|
||||
"registerChannel",
|
||||
"defineBundledChannelEntry",
|
||||
"registerCli",
|
||||
"registerCliBackend",
|
||||
"registerCodexAppServerExtensionFactory",
|
||||
"registerCommand",
|
||||
"registerCompactionProvider",
|
||||
"registerConfigMigration",
|
||||
"registerContextEngine",
|
||||
"registerControlUiDescriptor",
|
||||
"registerDetachedTaskRuntime",
|
||||
"registerGatewayDiscoveryService",
|
||||
"registerGatewayMethod",
|
||||
"registerHook",
|
||||
"registerHttpRoute",
|
||||
"registerImageGenerationProvider",
|
||||
"registerInteractiveHandler",
|
||||
"registerMediaUnderstandingProvider",
|
||||
"registerMemoryCapability",
|
||||
"registerMemoryCorpusSupplement",
|
||||
"registerMemoryEmbeddingProvider",
|
||||
"registerMemoryFlushPlan",
|
||||
"registerMemoryPromptSection",
|
||||
"registerMemoryPromptSupplement",
|
||||
"registerMemoryRuntime",
|
||||
"registerMigrationProvider",
|
||||
"registerMusicGenerationProvider",
|
||||
"registerNodeHostCommand",
|
||||
"registerNodeInvokePolicy",
|
||||
"registerProvider",
|
||||
"registerRealtimeTranscriptionProvider",
|
||||
"registerRealtimeVoiceProvider",
|
||||
"registerReload",
|
||||
"registerRuntimeLifecycle",
|
||||
"registerSecurityAuditCollector",
|
||||
"registerService",
|
||||
"registerSessionExtension",
|
||||
"registerSessionSchedulerJob",
|
||||
"registerSpeechProvider",
|
||||
"registerTextTransforms",
|
||||
"registerTool",
|
||||
"registerToolMetadata",
|
||||
"registerTrustedToolPolicy",
|
||||
"registerVideoGenerationProvider",
|
||||
"registerWebFetchProvider",
|
||||
"registerWebSearchProvider",
|
||||
];
|
||||
const plan = buildSyntheticProbePlan({
|
||||
capture: {
|
||||
generatedAt: "test",
|
||||
summary: { fixtureCount: 1 },
|
||||
fixtures: [
|
||||
{
|
||||
id: "kitchen-sink",
|
||||
hooks: [],
|
||||
registrations: kitchenSinkRegistrars.map((registrar) => ({
|
||||
id: `registration.${registrar}:kitchen-sink:index`,
|
||||
registrar,
|
||||
ref: "src/generated-registrars.js",
|
||||
assertions: [`${registrar} is classified`],
|
||||
syntheticArguments: [{}],
|
||||
})),
|
||||
},
|
||||
],
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(plan.summary.probeCount, kitchenSinkRegistrars.length);
|
||||
assert.equal(plan.summary.blockedCount, 0);
|
||||
assert.deepEqual(validateSyntheticProbePlan(plan), []);
|
||||
});
|
||||
|
||||
test("synthetic probes invoke retained hook and tool handlers", async () => {
|
||||
const capture = await captureLocalFixture([
|
||||
"export function register(api) {",
|
||||
@ -120,6 +227,57 @@ test("synthetic probes pass registrar-specific handler inputs", async () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("synthetic probes pass channel envelopes and gateway responders", async () => {
|
||||
const capture = await captureLocalFixture([
|
||||
"export function register(api) {",
|
||||
" api.registerChannel({",
|
||||
" id: 'fixture_channel',",
|
||||
" async send(ctx) { return { messageId: ctx.replyToId, to: ctx.to }; },",
|
||||
" async receive(ctx) { return { messageId: ctx.message.id, peer: ctx.route.peer.id }; },",
|
||||
" });",
|
||||
" api.registerGatewayMethod('fixture.ping', ({ respond, params }) => respond(true, { sawParams: typeof params === 'object' }));",
|
||||
"}",
|
||||
]);
|
||||
|
||||
const blocked = await runCapturedSyntheticProbes(capture);
|
||||
assert.equal(blocked.summary.blockedCount, 1);
|
||||
|
||||
const result = await runCapturedSyntheticProbes(capture, { includeChannelRuntime: true });
|
||||
|
||||
assert.equal(result.summary.failCount, 0);
|
||||
assert.deepEqual(
|
||||
result.results.map((item) => `${item.status}:${item.label}`),
|
||||
[
|
||||
"pass:registerChannel.send",
|
||||
"pass:registerChannel.receive",
|
||||
"pass:registerGatewayMethod.handler",
|
||||
"pass:registerGatewayMethod.run",
|
||||
"pass:registerGatewayMethod.execute",
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("synthetic probes can execute string plus handler registrations", async () => {
|
||||
const capture = await captureLocalFixture([
|
||||
"export function register(api) {",
|
||||
" api.registerGatewayMethod('fixture.ping', (event) => ({ method: event.registrar, property: event.property }));",
|
||||
"}",
|
||||
]);
|
||||
|
||||
const result = await runCapturedSyntheticProbes(capture);
|
||||
|
||||
assert.equal(result.summary.failCount, 0);
|
||||
assert.equal(result.summary.blockedCount, 0);
|
||||
assert.deepEqual(
|
||||
result.results.map((item) => `${item.status}:${item.label}`),
|
||||
[
|
||||
"pass:registerGatewayMethod.handler",
|
||||
"pass:registerGatewayMethod.run",
|
||||
"pass:registerGatewayMethod.execute",
|
||||
],
|
||||
);
|
||||
});
|
||||
|
||||
test("synthetic probes keep opt-in registrations guarded", async () => {
|
||||
const capture = await captureLocalFixture([
|
||||
"export function register(api) {",
|
||||
|
||||
@ -22,7 +22,8 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
|
||||
name: "fixture",
|
||||
packageManager: "npm@10.0.0",
|
||||
scripts: { build: "tsup" },
|
||||
dependencies: { "left-pad": "^1.3.0" },
|
||||
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
|
||||
devDependencies: { "@openclaw/plugin-sdk": "workspace:*" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
@ -57,10 +58,11 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
|
||||
assert.equal(plan.summary.artifactStepCount, 2);
|
||||
assert.equal(plan.summary.installStepCount, 1);
|
||||
assert.equal(plan.summary.auditStepCount, 1);
|
||||
assert.equal(plan.summary.pruneDevWorkspaceDependencyStepCount, 1);
|
||||
assert.equal(plan.summary.buildStepCount, 1);
|
||||
assert.equal(plan.summary.captureStepCount, 2);
|
||||
assert.equal(plan.summary.syntheticProbeStepCount, 2);
|
||||
assert.equal(plan.summary.targetOpenClawLinkStepCount, 2);
|
||||
assert.equal(plan.summary.targetOpenClawLinkStepCount, 1);
|
||||
assert.equal(plan.summary.tsLoaderEntrypointCount, 1);
|
||||
assert.equal(plan.summary.jitiAlternativeCount, 1);
|
||||
|
||||
@ -73,7 +75,14 @@ test("workspace plan maps blocked entrypoints to opt-in install/build/capture st
|
||||
assert.ok(entrypoint.requiredCapabilities.includes("sdk-alias-compat"));
|
||||
assert.ok(entrypoint.requiredCapabilities.includes("ts-loader"));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "install" && step.command === "npm install --ignore-scripts"));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node --import tsx capture.mjs")));
|
||||
assert.ok(
|
||||
entrypoint.steps.some(
|
||||
(step) => step.kind === "prune-dev-workspace-deps" && step.command.includes("prune-workspace-dev-deps-cli.js"),
|
||||
),
|
||||
);
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("node capture.mjs")));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "capture" && step.command.includes("--mock-sdk")));
|
||||
assert.ok(entrypoint.steps.every((step) => !step.command.includes("--import tsx")));
|
||||
assert.ok(entrypoint.steps.some((step) => step.kind === "synthetic-probe" && step.command.includes("synthetic.mjs")));
|
||||
const buildEntrypoint = plan.fixtures[0].entrypoints.find((item) => item.packageName === "build-fixture");
|
||||
assert.ok(buildEntrypoint);
|
||||
@ -94,7 +103,7 @@ test("workspace plan defaults point at packaged helper wrappers", async (t) => {
|
||||
name: "fixture",
|
||||
packageManager: "npm@10.0.0",
|
||||
scripts: { build: "tsup" },
|
||||
dependencies: { "left-pad": "^1.3.0" },
|
||||
dependencies: { "left-pad": "^1.3.0", openclaw: "^1.0.0" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
@ -226,7 +235,7 @@ function readinessReport() {
|
||||
{
|
||||
path: "plugins\\fixture\\package.json",
|
||||
name: "fixture",
|
||||
dependencies: ["left-pad"],
|
||||
dependencies: ["left-pad", "openclaw"],
|
||||
peerDependencies: [],
|
||||
optionalDependencies: [],
|
||||
openclaw: {
|
||||
|
||||
Loading…
Reference in New Issue
Block a user