From fbb02499809f8203afac958bd387b33bd95838d5 Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Sat, 25 Apr 2026 12:11:50 -0700 Subject: [PATCH] chore: initialize crabpot compat testbed --- .github/pull_request_template.md | 24 +++++++ .github/workflows/check.yml | 25 +++++++ .gitignore | 9 +++ .gitmodules | 44 ++++++++++++ AGENTS.md | 12 ++++ LICENSE | 22 ++++++ README.md | 58 ++++++++++++++++ crabpot.config.json | 112 +++++++++++++++++++++++++++++++ crabpot.schema.json | 44 ++++++++++++ docs/contract-matrix.md | 31 +++++++++ docs/operations.md | 39 +++++++++++ package.json | 17 +++++ plugins/a2a-gateway | 1 + plugins/agentchat | 1 + plugins/apify | 1 + plugins/clawmetry | 1 + plugins/codex-app-server | 1 + plugins/hasdata | 1 + plugins/inworld-tts | 1 + plugins/llm-trace-phoenix | 1 + plugins/mcp-adapter | 1 + plugins/web-search-plus | 1 + plugins/wecom | 1 + scripts/list-fixtures.mjs | 14 ++++ scripts/manifest-lib.mjs | 63 +++++++++++++++++ scripts/run-contract-smoke.mjs | 91 +++++++++++++++++++++++++ scripts/sync-fixtures.mjs | 64 ++++++++++++++++++ test/manifest.test.mjs | 35 ++++++++++ 28 files changed, 715 insertions(+) create mode 100644 .github/pull_request_template.md create mode 100644 .github/workflows/check.yml create mode 100644 .gitignore create mode 100644 .gitmodules create mode 100644 AGENTS.md create mode 100644 LICENSE create mode 100644 README.md create mode 100644 crabpot.config.json create mode 100644 crabpot.schema.json create mode 100644 docs/contract-matrix.md create mode 100644 docs/operations.md create mode 100644 package.json create mode 160000 plugins/a2a-gateway create mode 160000 plugins/agentchat create mode 160000 plugins/apify create mode 160000 plugins/clawmetry create mode 160000 plugins/codex-app-server create mode 160000 plugins/hasdata create mode 160000 plugins/inworld-tts create mode 160000 plugins/llm-trace-phoenix create mode 160000 plugins/mcp-adapter create mode 160000 plugins/web-search-plus create mode 160000 plugins/wecom create mode 100644 scripts/list-fixtures.mjs create mode 100644 scripts/manifest-lib.mjs create mode 100644 scripts/run-contract-smoke.mjs create mode 100644 scripts/sync-fixtures.mjs create mode 100644 test/manifest.test.mjs diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..34eb78a --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,24 @@ +## Summary + +- Problem: +- What changed: +- What this does not cover: + +## Fixture impact + +- [ ] Adds or updates fixture manifest entries +- [ ] Updates submodule pins +- [ ] Changes contract/smoke logic +- [ ] Docs only + +## Verification + +- [ ] `npm test` +- [ ] `node scripts/sync-fixtures.mjs --check` +- [ ] `node scripts/run-contract-smoke.mjs` +- [ ] strict fixture smoke, if submodules were materialized + +## Notes + +List any external services, secrets, or live checks intentionally skipped. + diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml new file mode 100644 index 0000000..481a3d7 --- /dev/null +++ b/.github/workflows/check.yml @@ -0,0 +1,25 @@ +name: Check + +on: + pull_request: + push: + branches: [main] + workflow_dispatch: + +jobs: + manifest: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + - uses: actions/checkout@v4 + with: + repository: openclaw/openclaw + path: openclaw + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm test + - run: node scripts/sync-fixtures.mjs --check + - run: node scripts/run-contract-smoke.mjs --strict --openclaw ./openclaw diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e94f3e --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +node_modules/ +.DS_Store +.env +.env.* +!.env.example +coverage/ +dist/ +tmp/ + diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..6e920c7 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,44 @@ +[submodule "plugins/agentchat"] + path = plugins/agentchat + url = https://github.com/agentchatme/agentchat.git + shallow = true +[submodule "plugins/wecom"] + path = plugins/wecom + url = https://github.com/sunnoy/openclaw-plugin-wecom.git + shallow = true +[submodule "plugins/a2a-gateway"] + path = plugins/a2a-gateway + url = https://github.com/win4r/openclaw-a2a-gateway.git + shallow = true +[submodule "plugins/hasdata"] + path = plugins/hasdata + url = https://github.com/HasData/hasdata-openclaw-plugin.git + shallow = true +[submodule "plugins/mcp-adapter"] + path = plugins/mcp-adapter + url = https://github.com/androidStern-personal/openclaw-mcp-adapter.git + shallow = true +[submodule "plugins/llm-trace-phoenix"] + path = plugins/llm-trace-phoenix + url = https://github.com/pingshian0131/openclaw-plugin-llm-trace-phoenix.git + shallow = true +[submodule "plugins/clawmetry"] + path = plugins/clawmetry + url = https://github.com/vivekchand/clawmetry.git + shallow = true +[submodule "plugins/codex-app-server"] + path = plugins/codex-app-server + url = https://github.com/pwrdrvr/openclaw-codex-app-server.git + shallow = true +[submodule "plugins/web-search-plus"] + path = plugins/web-search-plus + url = https://github.com/robbyczgw-cla/web-search-plus-plugin.git + shallow = true +[submodule "plugins/apify"] + path = plugins/apify + url = https://github.com/apify/apify-openclaw-plugin.git + shallow = true +[submodule "plugins/inworld-tts"] + path = plugins/inworld-tts + url = https://github.com/livingghost/openclaw-inworld-tts.git + shallow = true diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..1937eb9 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,12 @@ +# crabpot agent notes + +- Keep this repo fixture-driven. Add plugins to `crabpot.config.json`; do not + hardcode fixture lists in scripts. +- External plugin code lives under `plugins/` as git submodules. Do not vendor or + rewrite external plugin source here. +- Default checks must stay cheap and credential-free. Live tests require explicit + opt-in and secrets. +- Prefer seam labels over product categories: `dynamic-tool`, `llm-observer`, + `gateway-service`, `provider-capability`, and similar. +- When adding a fixture, explain the unique seam it covers in `why`. + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..b1956cd --- /dev/null +++ b/LICENSE @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2026 OpenClaw + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/README.md b/README.md new file mode 100644 index 0000000..71cfd8f --- /dev/null +++ b/README.md @@ -0,0 +1,58 @@ +# crabpot + +Compatibility trap for OpenClaw plugin contracts. + +`crabpot` keeps a curated set of real community plugins pinned under `plugins/` +and runs seam-focused compatibility checks against OpenClaw plugin APIs. The goal +is to catch contract drift before external plugin authors do. + +## What this tests + +- plugin manifests and install metadata +- native tool registration and dynamic tool schemas +- channel registration and message delivery seams +- lifecycle hooks such as `gateway_start`, `gateway_stop`, and `before_install` +- agent hooks such as `before_tool_call`, `before_prompt_build`, `llm_input`, + `llm_output`, and `agent_end` +- provider capability registration such as speech/TTS +- plugin-owned services, routes, subprocesses, and async job patterns + +## Layout + +```text +crabpot/ + crabpot.config.json fixture manifest and seam tags + plugins/ external plugin repositories as git submodules + scripts/ manifest and fixture helpers + test/ repo-level checks + docs/ operating notes and seam matrix +``` + +## Quick start + +```bash +npm test +node scripts/list-fixtures.mjs +node scripts/sync-fixtures.mjs --check +``` + +To materialize the fixture repos as submodules: + +```bash +node scripts/sync-fixtures.mjs --materialize +git submodule update --init --recursive +``` + +That command mutates `.gitmodules` and `plugins/*`. Commit those changes when +you intentionally pin or update fixture revisions. + +## Fixture policy + +Fixtures should earn their spot by covering a distinct seam. Popularity is a +useful signal, but a small plugin that exercises a rare hook is more valuable +than the fourth web-search wrapper. + +The first fixture set intentionally covers channels, dynamic tools, LLM +observation, diagnostics, gateway-owned services, async jobs, provider +capabilities, and security/policy hooks. + diff --git a/crabpot.config.json b/crabpot.config.json new file mode 100644 index 0000000..cc2ec3a --- /dev/null +++ b/crabpot.config.json @@ -0,0 +1,112 @@ +{ + "$schema": "./crabpot.schema.json", + "version": 1, + "submoduleRoot": "plugins", + "openclaw": { + "minimumNode": "22", + "defaultCheckoutPath": "../openclaw" + }, + "fixtures": [ + { + "id": "agentchat", + "name": "AgentChat OpenClaw Channel", + "repo": "https://github.com/agentchatme/agentchat.git", + "path": "plugins/agentchat", + "subdir": "integrations/openclaw-channel", + "priority": "high", + "seams": ["channel", "prompt-mutation", "config-schema", "websocket", "backpressure"], + "why": "Production-shaped channel plugin with WebSocket auth, reconnect, prompt hints, and typed runtime contracts." + }, + { + "id": "wecom", + "name": "WeCom Enhanced Channel", + "repo": "https://github.com/sunnoy/openclaw-plugin-wecom.git", + "path": "plugins/wecom", + "priority": "high", + "seams": ["channel", "streaming", "dynamic-agent-routing", "message-policy", "media"], + "why": "Large community channel fixture covering WS streaming, group/DM policy, command allowlists, media, and dynamic agents." + }, + { + "id": "a2a-gateway", + "name": "A2A Gateway", + "repo": "https://github.com/win4r/openclaw-a2a-gateway.git", + "path": "plugins/a2a-gateway", + "priority": "high", + "seams": ["gateway-service", "http-routes", "agent-routing", "async-job", "auth"], + "why": "Agent-to-agent server fixture with agent-card discovery, JSON-RPC/REST/gRPC transports, metrics, peer auth, and async task modes." + }, + { + "id": "hasdata", + "name": "HasData", + "repo": "https://github.com/HasData/hasdata-openclaw-plugin.git", + "path": "plugins/hasdata", + "priority": "high", + "seams": ["tool", "tool-schema", "external-api", "cli", "config-schema"], + "why": "Single native tool backed by a large OpenAPI-derived action catalog and strict per-action schemas." + }, + { + "id": "mcp-adapter", + "name": "MCP Adapter", + "repo": "https://github.com/androidStern-personal/openclaw-mcp-adapter.git", + "path": "plugins/mcp-adapter", + "priority": "high", + "seams": ["dynamic-tool", "json-schema", "stdio", "http", "reconnect"], + "why": "Dynamic tool discovery from MCP servers with stdio/http transports and schema passthrough." + }, + { + "id": "llm-trace-phoenix", + "name": "Phoenix LLM Trace", + "repo": "https://github.com/pingshian0131/openclaw-plugin-llm-trace-phoenix.git", + "path": "plugins/llm-trace-phoenix", + "priority": "high", + "seams": ["llm-observer", "conversation-access", "telemetry", "external-api"], + "why": "Small sharp fixture for llm_input/llm_output access and raw prompt/output privacy boundaries." + }, + { + "id": "clawmetry", + "name": "ClawMetry", + "repo": "https://github.com/vivekchand/clawmetry.git", + "path": "plugins/clawmetry", + "priority": "medium", + "seams": ["diagnostics", "log-transport", "gateway-service", "sidecar", "telemetry"], + "why": "Observability fixture for diagnostic events, log transport, telemetry buffering, and plugin-owned dashboard lifecycle." + }, + { + "id": "codex-app-server", + "name": "Codex App Server Bridge", + "repo": "https://github.com/pwrdrvr/openclaw-codex-app-server.git", + "path": "plugins/codex-app-server", + "priority": "medium", + "seams": ["process-spawn", "unsafe-install", "channel-bridge", "interactive-ui", "sdk-compat"], + "why": "Bridge fixture for process launch policy, install safety gates, and channel-specific SDK compatibility." + }, + { + "id": "web-search-plus", + "name": "Web Search Plus", + "repo": "https://github.com/robbyczgw-cla/web-search-plus-plugin.git", + "path": "plugins/web-search-plus", + "priority": "medium", + "seams": ["tool", "provider-routing", "external-api", "env-auth", "extraction"], + "why": "Multi-provider search/extract tool fixture with several auth/config permutations." + }, + { + "id": "apify", + "name": "Apify", + "repo": "https://github.com/apify/apify-openclaw-plugin.git", + "path": "plugins/apify", + "priority": "medium", + "seams": ["tool", "async-job", "polling", "external-api", "catalog-discovery"], + "why": "Async start/collect workflow fixture for remote jobs and huge third-party catalog discovery." + }, + { + "id": "inworld-tts", + "name": "Inworld TTS", + "repo": "https://github.com/livingghost/openclaw-inworld-tts.git", + "path": "plugins/inworld-tts", + "priority": "medium", + "seams": ["provider-capability", "speech", "tts", "env-auth", "runtime-overrides"], + "why": "Provider capability fixture for speech registration, config/env auth, and talk.speak request overrides." + } + ] +} + diff --git a/crabpot.schema.json b/crabpot.schema.json new file mode 100644 index 0000000..59b7b31 --- /dev/null +++ b/crabpot.schema.json @@ -0,0 +1,44 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "title": "Crabpot fixture manifest", + "type": "object", + "required": ["version", "submoduleRoot", "fixtures"], + "additionalProperties": false, + "properties": { + "version": { "type": "integer", "minimum": 1 }, + "submoduleRoot": { "type": "string", "pattern": "^[^/].*" }, + "openclaw": { + "type": "object", + "additionalProperties": false, + "properties": { + "minimumNode": { "type": "string" }, + "defaultCheckoutPath": { "type": "string" } + } + }, + "fixtures": { + "type": "array", + "minItems": 1, + "items": { + "type": "object", + "required": ["id", "name", "repo", "path", "priority", "seams", "why"], + "additionalProperties": false, + "properties": { + "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" }, + "name": { "type": "string", "minLength": 1 }, + "repo": { "type": "string", "pattern": "^https://github.com/.+\\.git$" }, + "path": { "type": "string", "pattern": "^plugins/[a-z0-9][a-z0-9-]*$" }, + "subdir": { "type": "string", "minLength": 1 }, + "priority": { "enum": ["high", "medium", "low"] }, + "seams": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]*$" }, + "uniqueItems": true + }, + "why": { "type": "string", "minLength": 1 } + } + } + } + } +} + diff --git a/docs/contract-matrix.md b/docs/contract-matrix.md new file mode 100644 index 0000000..e3b32b0 --- /dev/null +++ b/docs/contract-matrix.md @@ -0,0 +1,31 @@ +# Contract Matrix + +Crabpot should test contracts by seam, not by package popularity. + +| Seam | Fixture examples | What should fail fast | +| --- | --- | --- | +| Channel runtime | `agentchat`, `wecom` | registration drift, message envelope drift, sender/thread metadata drift | +| Native tools | `hasdata`, `web-search-plus`, `apify` | schema incompatibility, result-shape drift, auth/config drift | +| Dynamic tools | `mcp-adapter` | JSON Schema passthrough, tool namespacing, reconnect behavior | +| LLM observation | `llm-trace-phoenix` | conversation-access gating, prompt/output shape changes | +| Diagnostics/logs | `clawmetry` | diagnostic event shape, log transport registration | +| Gateway services | `a2a-gateway`, `clawmetry` | lifecycle start/stop, route registration, teardown | +| Provider capability | `inworld-tts` | provider registration, runtime helper compatibility | +| Unsafe/process policy | `codex-app-server` | install scan behavior, process-spawn declarations | +| Async jobs | `a2a-gateway`, `apify` | start/poll/collect result contracts and timeout handling | +| Prompt mutation | `agentchat` | prompt-injection opt-out and prompt builder contracts | + +## Test levels + +1. **Manifest smoke**: plugin metadata parses and declared OpenClaw compatibility + can be read without executing plugin code. +2. **SDK compile**: plugin TypeScript compiles against the target OpenClaw SDK. +3. **Registration capture**: plugin `register(api)` can run against a captured + API shim and records expected tools, hooks, providers, routes, and commands. +4. **Contract smoke**: selected hooks/tools run against fixture events with + synthetic inputs and validate stable return shapes. +5. **Live smoke**: opt-in, credential-gated checks for real remote services. + +Default CI should run levels 1-4. Live smoke belongs behind explicit labels, +manual workflow dispatch, or secret availability. + diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..76d50ec --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,39 @@ +# Operations + +## Adding a fixture + +1. Add an entry to `crabpot.config.json`. +2. Run `npm test`. +3. Run `node scripts/sync-fixtures.mjs --materialize`. +4. Review `.gitmodules` and the pinned submodule commit. +5. Commit the manifest and submodule pointer together. + +## Updating fixtures + +```bash +git submodule update --remote --recursive plugins/ +npm test +``` + +Then inspect the diff. A fixture update is only useful if it either: + +- tracks a plugin release we care about, +- adds coverage for a new seam, +- reproduces a compatibility break, +- or proves a contract migration path still works. + +## CI model + +Use a cheap default workflow first: + +- validate the manifest +- ensure `.gitmodules` agrees with `crabpot.config.json` +- initialize pinned submodules +- run registration-capture smoke checks + +Add heavier lanes later for SDK compilation against specific OpenClaw refs: + +- `openclaw@main` +- latest stable release tag +- active beta branch or release candidate + diff --git a/package.json b/package.json new file mode 100644 index 0000000..b46a0ff --- /dev/null +++ b/package.json @@ -0,0 +1,17 @@ +{ + "name": "crabpot", + "version": "0.1.0", + "private": true, + "description": "Compatibility testbed for OpenClaw community plugins and plugin seams.", + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "check": "npm test && node scripts/sync-fixtures.mjs --check && node scripts/run-contract-smoke.mjs", + "fixtures:list": "node scripts/list-fixtures.mjs", + "fixtures:sync": "node scripts/sync-fixtures.mjs --materialize", + "smoke": "node scripts/run-contract-smoke.mjs", + "test": "node --test test/*.test.mjs" + } +} diff --git a/plugins/a2a-gateway b/plugins/a2a-gateway new file mode 160000 index 0000000..a335e59 --- /dev/null +++ b/plugins/a2a-gateway @@ -0,0 +1 @@ +Subproject commit a335e59e926f7e1a8913e6cd7b1cbf2d44c33cb7 diff --git a/plugins/agentchat b/plugins/agentchat new file mode 160000 index 0000000..df6870b --- /dev/null +++ b/plugins/agentchat @@ -0,0 +1 @@ +Subproject commit df6870bcf325ae34e2d41b3cb479b0c8e88aba83 diff --git a/plugins/apify b/plugins/apify new file mode 160000 index 0000000..41f4979 --- /dev/null +++ b/plugins/apify @@ -0,0 +1 @@ +Subproject commit 41f49794d230f7ad092d1c699ee4d91fecf6ba91 diff --git a/plugins/clawmetry b/plugins/clawmetry new file mode 160000 index 0000000..b329bb3 --- /dev/null +++ b/plugins/clawmetry @@ -0,0 +1 @@ +Subproject commit b329bb3ed18b651d369bf35321ec58bd47dc33b4 diff --git a/plugins/codex-app-server b/plugins/codex-app-server new file mode 160000 index 0000000..4a87dce --- /dev/null +++ b/plugins/codex-app-server @@ -0,0 +1 @@ +Subproject commit 4a87dce5d620a8fb30842bb1b726390fe442247e diff --git a/plugins/hasdata b/plugins/hasdata new file mode 160000 index 0000000..bed32d8 --- /dev/null +++ b/plugins/hasdata @@ -0,0 +1 @@ +Subproject commit bed32d8a0359392b7c4628f12b909b6e204c8426 diff --git a/plugins/inworld-tts b/plugins/inworld-tts new file mode 160000 index 0000000..d2abaee --- /dev/null +++ b/plugins/inworld-tts @@ -0,0 +1 @@ +Subproject commit d2abaeea330ebef7530f43f8b395671f6f404aea diff --git a/plugins/llm-trace-phoenix b/plugins/llm-trace-phoenix new file mode 160000 index 0000000..05bc0f4 --- /dev/null +++ b/plugins/llm-trace-phoenix @@ -0,0 +1 @@ +Subproject commit 05bc0f4ba67281c10fad7be356d32a54b00c59fd diff --git a/plugins/mcp-adapter b/plugins/mcp-adapter new file mode 160000 index 0000000..5434ce2 --- /dev/null +++ b/plugins/mcp-adapter @@ -0,0 +1 @@ +Subproject commit 5434ce21ac780a46a493c8125e52e80a03dd2640 diff --git a/plugins/web-search-plus b/plugins/web-search-plus new file mode 160000 index 0000000..be6580d --- /dev/null +++ b/plugins/web-search-plus @@ -0,0 +1 @@ +Subproject commit be6580db68997a209e38cb43e978e9825f681d3e diff --git a/plugins/wecom b/plugins/wecom new file mode 160000 index 0000000..b7849ac --- /dev/null +++ b/plugins/wecom @@ -0,0 +1 @@ +Subproject commit b7849ac055c8fa699d01b48e83cf24028907307d diff --git a/scripts/list-fixtures.mjs b/scripts/list-fixtures.mjs new file mode 100644 index 0000000..a3b3239 --- /dev/null +++ b/scripts/list-fixtures.mjs @@ -0,0 +1,14 @@ +#!/usr/bin/env node +import { readManifest } from "./manifest-lib.mjs"; + +const manifest = await readManifest(); + +const rows = manifest.fixtures.map((fixture) => ({ + id: fixture.id, + priority: fixture.priority, + seams: fixture.seams.join(","), + path: fixture.path, +})); + +console.table(rows); + diff --git a/scripts/manifest-lib.mjs b/scripts/manifest-lib.mjs new file mode 100644 index 0000000..2eeb7b0 --- /dev/null +++ b/scripts/manifest-lib.mjs @@ -0,0 +1,63 @@ +import { readFile } from "node:fs/promises"; +import path from "node:path"; + +export const repoRoot = path.resolve(import.meta.dirname, ".."); +export const manifestPath = path.join(repoRoot, "crabpot.config.json"); + +export async function readManifest() { + const raw = await readFile(manifestPath, "utf8"); + const manifest = JSON.parse(raw); + validateManifest(manifest); + return manifest; +} + +export function validateManifest(manifest) { + const errors = []; + + if (manifest.version !== 1) { + errors.push("manifest.version must be 1"); + } + + if (manifest.submoduleRoot !== "plugins") { + errors.push('manifest.submoduleRoot must be "plugins"'); + } + + if (!Array.isArray(manifest.fixtures) || manifest.fixtures.length === 0) { + errors.push("manifest.fixtures must be a non-empty array"); + } + + const ids = new Set(); + const paths = new Set(); + for (const fixture of manifest.fixtures ?? []) { + if (!/^[a-z0-9][a-z0-9-]*$/.test(fixture.id ?? "")) { + errors.push(`invalid fixture id: ${fixture.id}`); + } + if (ids.has(fixture.id)) { + errors.push(`duplicate fixture id: ${fixture.id}`); + } + ids.add(fixture.id); + + if (!fixture.path?.startsWith("plugins/")) { + errors.push(`${fixture.id}: path must live under plugins/`); + } + if (paths.has(fixture.path)) { + errors.push(`duplicate fixture path: ${fixture.path}`); + } + paths.add(fixture.path); + + if (!fixture.repo?.startsWith("https://github.com/") || !fixture.repo.endsWith(".git")) { + errors.push(`${fixture.id}: repo must be a GitHub HTTPS .git URL`); + } + if (!["high", "medium", "low"].includes(fixture.priority)) { + errors.push(`${fixture.id}: priority must be high, medium, or low`); + } + if (!Array.isArray(fixture.seams) || fixture.seams.length === 0) { + errors.push(`${fixture.id}: seams must be non-empty`); + } + } + + if (errors.length > 0) { + throw new Error(errors.join("\n")); + } +} + diff --git a/scripts/run-contract-smoke.mjs b/scripts/run-contract-smoke.mjs new file mode 100644 index 0000000..64a6a5a --- /dev/null +++ b/scripts/run-contract-smoke.mjs @@ -0,0 +1,91 @@ +#!/usr/bin/env node +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { readManifest, repoRoot } from "./manifest-lib.mjs"; + +const args = process.argv.slice(2); +const strict = args.includes("--strict"); +const openclawArgIndex = args.indexOf("--openclaw"); +const openclawPath = + openclawArgIndex === -1 ? "../openclaw" : args[openclawArgIndex + 1]; + +const manifest = await readManifest(); +const openclawRoot = path.resolve(repoRoot, openclawPath); + +const openclawPackage = path.join(openclawRoot, "package.json"); +if (!existsSync(openclawPackage)) { + throw new Error(`OpenClaw checkout not found at ${openclawRoot}`); +} + +const rows = []; +const missing = []; + +for (const fixture of manifest.fixtures) { + const checkoutPath = path.join(repoRoot, fixture.path); + const pluginRoot = fixture.subdir ? path.join(checkoutPath, fixture.subdir) : checkoutPath; + + if (!existsSync(checkoutPath)) { + rows.push(row(fixture, "missing", "not materialized")); + missing.push(fixture.id); + continue; + } + + const manifestFile = await firstExisting([ + path.join(pluginRoot, "openclaw.plugin.json"), + path.join(checkoutPath, "openclaw.plugin.json"), + ]); + const packageFile = await firstExisting([ + path.join(pluginRoot, "package.json"), + path.join(checkoutPath, "package.json"), + ]); + + if (!manifestFile && !packageFile) { + rows.push(row(fixture, "incomplete", "no plugin manifest or package.json found")); + continue; + } + + const pluginId = manifestFile + ? await readPluginId(manifestFile) + : await readPackageName(packageFile); + rows.push(row(fixture, "ok", pluginId)); +} + +console.table(rows); + +if (strict && missing.length > 0) { + throw new Error(`missing fixture checkouts: ${missing.join(", ")}`); +} + +function row(fixture, status, detail) { + return { + id: fixture.id, + priority: fixture.priority, + status, + detail, + seams: fixture.seams.join(","), + }; +} + +async function firstExisting(paths) { + return paths.find((candidate) => existsSync(candidate)); +} + +async function readPluginId(filePath) { + try { + const json = JSON.parse(await readFile(filePath, "utf8")); + return json.id ?? json.name ?? path.basename(path.dirname(filePath)); + } catch { + return "invalid openclaw.plugin.json"; + } +} + +async function readPackageName(filePath) { + try { + const json = JSON.parse(await readFile(filePath, "utf8")); + return json.name ?? path.basename(path.dirname(filePath)); + } catch { + return "invalid package.json"; + } +} + diff --git a/scripts/sync-fixtures.mjs b/scripts/sync-fixtures.mjs new file mode 100644 index 0000000..efa32e5 --- /dev/null +++ b/scripts/sync-fixtures.mjs @@ -0,0 +1,64 @@ +#!/usr/bin/env node +import { existsSync } from "node:fs"; +import { readFile } from "node:fs/promises"; +import { spawnSync } from "node:child_process"; +import path from "node:path"; +import { readManifest, repoRoot } from "./manifest-lib.mjs"; + +const args = new Set(process.argv.slice(2)); +const materialize = args.has("--materialize"); +const check = args.has("--check") || !materialize; + +const manifest = await readManifest(); + +if (check) { + await checkGitmodules(manifest); + console.log(`crabpot: manifest ok (${manifest.fixtures.length} fixtures)`); + process.exit(0); +} + +for (const fixture of manifest.fixtures) { + const target = path.join(repoRoot, fixture.path); + if (existsSync(target)) { + run("git", ["submodule", "update", "--init", "--recursive", fixture.path]); + continue; + } + + run("git", ["submodule", "add", "--depth", "1", fixture.repo, fixture.path]); +} + +console.log("crabpot: fixtures materialized. review .gitmodules and commit pinned revisions."); + +async function checkGitmodules(manifest) { + const gitmodulesPath = path.join(repoRoot, ".gitmodules"); + if (!existsSync(gitmodulesPath)) { + return; + } + + const gitmodules = await readFile(gitmodulesPath, "utf8"); + const missing = []; + for (const fixture of manifest.fixtures) { + if (!gitmodules.includes(`path = ${fixture.path}`)) { + missing.push(fixture.path); + } + if (!gitmodules.includes(`url = ${fixture.repo}`)) { + missing.push(fixture.repo); + } + } + + if (missing.length > 0) { + throw new Error(`.gitmodules is missing manifest entries:\n${missing.join("\n")}`); + } +} + +function run(command, args) { + const result = spawnSync(command, args, { + cwd: repoRoot, + stdio: "inherit", + env: process.env, + }); + + if (result.status !== 0) { + throw new Error(`${command} ${args.join(" ")} failed with ${result.status}`); + } +} diff --git a/test/manifest.test.mjs b/test/manifest.test.mjs new file mode 100644 index 0000000..bede44c --- /dev/null +++ b/test/manifest.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { readManifest } from "../scripts/manifest-lib.mjs"; + +test("fixture manifest is valid and seam-rich", async () => { + const manifest = await readManifest(); + + assert.equal(manifest.submoduleRoot, "plugins"); + assert.ok(manifest.fixtures.length >= 10); + + const seams = new Set(manifest.fixtures.flatMap((fixture) => fixture.seams)); + for (const seam of [ + "channel", + "tool", + "dynamic-tool", + "llm-observer", + "diagnostics", + "gateway-service", + "provider-capability", + "async-job", + "prompt-mutation", + ]) { + assert.ok(seams.has(seam), `missing seam coverage: ${seam}`); + } +}); + +test("fixture paths are stable plugin submodule paths", async () => { + const manifest = await readManifest(); + + for (const fixture of manifest.fixtures) { + assert.match(fixture.path, /^plugins\/[a-z0-9][a-z0-9-]*$/); + assert.ok(!fixture.path.includes("..")); + } +}); +