openclaw/crabpot
openclaw/crabpot
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

chore(fixtures): refresh kitchen sink metadata reports (#27)

Browse Source 
* chore(fixtures): refresh kitchen sink metadata reports

* fix(ci): harden kitchen sink surface checks

* fix(ci): classify kitchen sink isolated probes

* fix(ci): pin merged plugin inspector source
This commit is contained in:
Vincent Koc 2026-05-02 18:32:03 -07:00 committed by GitHub
parent 7644287646
commit 2e8fac3fc7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
22 changed files with 17969 additions and 10253 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/check.yml vendored
View File

@ -197,7 +197,7 @@ jobs:
- name: Reconcile compatibility report with runtime evidence
if: always()
continue-on-error: true
run: node scripts/generate-report.mjs --openclaw ./openclaw --execution-results reports/crabpot-execution-results.json
run: node scripts/generate-report.mjs --openclaw ./openclaw --execution-results reports/crabpot-execution-results.json --fixture-set "${{ matrix.id }}"
- name: Run execution policy
id: policy
if: always()

2
.github/workflows/openclaw-ref-compat.yml vendored
View File

@ -308,7 +308,7 @@ jobs:
- name: Reconcile compatibility report with runtime evidence
if: always()
continue-on-error: true
run: node scripts/generate-report.mjs --openclaw ./openclaw --execution-results reports/crabpot-execution-results.json
run: node scripts/generate-report.mjs --openclaw ./openclaw --execution-results reports/crabpot-execution-results.json --fixture-set "${{ matrix.id }}"
- name: Run execution policy
id: policy

14
crabpot.ci-policy.json
View File

@ -29,6 +29,20 @@
"reasonIncludes": "live network access",
"decision": "allowed-blocked",
"until": "network-capability harness lands"
},
{
"id": "generated-surface-hook-stubs",
"seam": "*",
"reasonIncludes": "captured hook has no callable handler",
"decision": "allowed-blocked",
"until": "generated surface runtime harness lands"
},
{
"id": "generated-surface-registration-stubs",
"seam": "*",
"reasonIncludes": "captured registration has no supported callable probe",
"decision": "allowed-blocked",
"until": "generated surface runtime harness lands"
}
],
"expectedWarnings": [

2
plugins/kitchen-sink

@ -1 +1 @@
Subproject commit 89449910b4dfa70a09d5d1e421037f88ceb8589b
Subproject commit 998901973d5ce083764a77861a4839f4f184aaa3

83
reports/crabpot-ci-policy.json
View File

@ -3,10 +3,11 @@
"status": "pass",
"strict": false,
"policy": {
"allowedBlocked": 4,
"allowedBlocked": 6,
"expectedWarnings": 1,
"fixtureSets": [
"build",
"openclaw-beta",
"sdk-alias",
"side-effect-review",
"smoke"
@ -28,21 +29,12 @@
{
"id": "compatibility-report.live-p0-issues",
"action": "warn",
"message": "13 live P0 issues tracked",
"message": "4 live P0 issues tracked",
"evidence": [
"bluebubbles:sdk-export-missing:untracked",
"clawmetry:sdk-export-missing:untracked",
"diagnostics-otel:sdk-export-missing:untracked",
"feishu:sdk-export-missing:untracked",
"honcho:sdk-export-missing:untracked",
"lobster:sdk-export-missing:untracked",
"matrix:sdk-export-missing:untracked",
"msteams:sdk-export-missing:untracked",
"nextcloud-talk:sdk-export-missing:untracked",
"nostr:sdk-export-missing:untracked",
"voice-call:sdk-export-missing:untracked",
"yuanbao:sdk-export-missing:untracked",
"zalo:sdk-export-missing:untracked"
"yuanbao:sdk-export-missing:untracked"
]
},
{
@ -260,7 +252,7 @@
{
"id": "compatibility-report.deprecation-warnings",
"action": "pass",
"message": "33 deprecated compat seams tracked",
"message": "40 deprecated compat seams tracked",
"evidence": [
"a2a-gateway:legacy-root-sdk-import",
"agentchat:channel-env-vars",
@ -271,6 +263,8 @@
"connectclaw:legacy-before-agent-start",
"connectclaw:legacy-root-sdk-import",
"dingtalk-connector:legacy-root-sdk-import",
"discord:channel-env-vars",
"feishu:channel-env-vars",
"hasdata:provider-auth-env-vars",
"honcho:legacy-before-agent-start",
"honcho:legacy-root-sdk-import",
@ -284,26 +278,30 @@
"memos-cloud:legacy-before-agent-start",
"memu-engine:legacy-root-sdk-import",
"mocrane-wecom:legacy-root-sdk-import",
"msteams:channel-env-vars",
"nemoclaw:legacy-before-agent-start",
"nextcloud-talk:channel-env-vars",
"nostr:channel-env-vars",
"openclaw-qqbot:channel-env-vars",
"openclaw-telemetry:legacy-before-agent-start",
"openclaw-telemetry:legacy-root-sdk-import",
"opik-openclaw:legacy-root-sdk-import",
"qqbot:legacy-root-sdk-import",
"synology-chat:legacy-root-sdk-import",
"synology-chat:channel-env-vars",
"telnyx-sms:channel-env-vars",
"tlon:legacy-root-sdk-import",
"twitch:legacy-root-sdk-import",
"yuanbao:channel-env-vars"
"twitch:channel-env-vars",
"voice-call:channel-env-vars",
"yuanbao:channel-env-vars",
"zalo:channel-env-vars",
"zalouser:channel-env-vars"
]
},
{
"id": "compatibility-report.inspector-gaps",
"action": "pass",
"message": "203 inspector proof gaps tracked",
"message": "182 inspector proof gaps tracked",
"evidence": [
"a2a-gateway:registration-capture-gap",
"bluebubbles:registration-capture-gap",
"clawmetry:registration-capture-gap",
"clawrouter:registration-capture-gap",
"codex:registration-capture-gap",
@ -313,8 +311,6 @@
"diagnostics-prometheus:registration-capture-gap",
"diffs:registration-capture-gap",
"dingtalk-connector:registration-capture-gap",
"discord:registration-capture-gap",
"feishu:registration-capture-gap",
"google-meet:registration-capture-gap",
"honcho:conversation-access-hook",
"honcho:registration-capture-gap",
@ -334,10 +330,8 @@
"memos-cloud:conversation-access-hook",
"memos-cloud:registration-capture-gap",
"mocrane-wecom:registration-capture-gap",
"msteams:registration-capture-gap",
"nemoclaw:before-tool-call-probe",
"nemoclaw:registration-capture-gap",
"nextcloud-talk:registration-capture-gap",
"nostr:registration-capture-gap",
"openclaw-qqbot:registration-capture-gap",
"openclaw-telemetry:before-tool-call-probe",
@ -349,16 +343,11 @@
"opik-openclaw:registration-capture-gap",
"qqbot:registration-capture-gap",
"secureclaw:registration-capture-gap",
"synology-chat:registration-capture-gap",
"telnyx-sms:registration-capture-gap",
"tlon:registration-capture-gap",
"twitch:registration-capture-gap",
"voice-call:registration-capture-gap",
"wecom:before-tool-call-probe",
"wecom:registration-capture-gap",
"yuanbao:registration-capture-gap",
"zalo:registration-capture-gap",
"zalouser:registration-capture-gap",
"a2a-gateway:package-dependency-install-required",
"a2a-gateway:package-typescript-source-entrypoint",
"a2a-gateway:runtime-tool-capture",
@ -368,8 +357,6 @@
"apify:package-dependency-install-required",
"apify:package-typescript-source-entrypoint",
"apify:runtime-tool-capture",
"bluebubbles:channel-contract-probe",
"bluebubbles:package-dependency-install-required",
"bluebubbles:package-typescript-source-entrypoint",
"brave-plugin:package-typescript-source-entrypoint",
"clawmetry:package-build-artifact-entrypoint",
@ -394,19 +381,16 @@
"diagnostics-prometheus:package-typescript-source-entrypoint",
"diffs:package-dependency-install-required",
"diffs:package-typescript-source-entrypoint",
"diffs:runtime-tool-capture",
"dingtalk-connector:channel-contract-probe",
"dingtalk-connector:package-build-artifact-entrypoint",
"dingtalk-connector:package-dependency-install-required",
"discord:channel-contract-probe",
"discord:package-dependency-install-required",
"discord:package-typescript-source-entrypoint",
"feishu:channel-contract-probe",
"feishu:package-dependency-install-required",
"feishu:package-typescript-source-entrypoint",
"feishu:runtime-tool-capture",
"google-meet:package-dependency-install-required",
"google-meet:package-typescript-source-entrypoint",
"google-meet:runtime-tool-capture",
"hasdata:package-dependency-install-required",
"hasdata:package-typescript-source-entrypoint",
"honcho:package-build-artifact-entrypoint",
@ -423,7 +407,6 @@
"llm-trace-phoenix:package-typescript-source-entrypoint",
"lobster:package-dependency-install-required",
"lobster:package-typescript-source-entrypoint",
"lobster:runtime-tool-capture",
"lossless-claw:package-build-artifact-entrypoint",
"lossless-claw:package-dependency-install-required",
"lossless-claw:runtime-tool-capture",
@ -437,7 +420,6 @@
"mcp-adapter:runtime-tool-capture",
"memory-lancedb:package-dependency-install-required",
"memory-lancedb:package-typescript-source-entrypoint",
"memory-lancedb:runtime-tool-capture",
"memory-tencentdb:package-dependency-install-required",
"memory-tencentdb:package-typescript-source-entrypoint",
"memory-tencentdb:runtime-tool-capture",
@ -460,7 +442,6 @@
"nostr:package-typescript-source-entrypoint",
"openclaw-qqbot:package-dependency-install-required",
"openclaw-qqbot:package-typescript-source-entrypoint",
"openclaw-qqbot:runtime-tool-capture",
"openclaw-telemetry:package-typescript-source-entrypoint",
"openclaw-weixin:channel-contract-probe",
"openclaw-weixin:package-dependency-install-required",
@ -481,12 +462,10 @@
"tlon:channel-contract-probe",
"tlon:package-dependency-install-required",
"tlon:package-typescript-source-entrypoint",
"twitch:channel-contract-probe",
"twitch:package-dependency-install-required",
"twitch:package-typescript-source-entrypoint",
"voice-call:package-dependency-install-required",
"voice-call:package-typescript-source-entrypoint",
"voice-call:runtime-tool-capture",
"web-search-plus:package-typescript-source-entrypoint",
"web-search-plus:runtime-tool-capture",
"wecom:channel-contract-probe",
@ -503,18 +482,15 @@
"zalo:package-typescript-source-entrypoint",
"zalouser:channel-contract-probe",
"zalouser:package-dependency-install-required",
"zalouser:package-typescript-source-entrypoint",
"zalouser:runtime-tool-capture"
"zalouser:package-typescript-source-entrypoint"
]
},
{
"id": "compatibility-report.p1-issues",
"action": "pass",
"message": "71 P1 issues tracked",
"message": "52 P1 issues tracked",
"evidence": [
"a2a-gateway:registration-capture-gap",
"bluebubbles:missing-compat-record",
"bluebubbles:registration-capture-gap",
"clawmetry:missing-compat-record",
"clawmetry:package-npm-pack-entrypoint-missing",
"clawmetry:registration-capture-gap",
@ -522,14 +498,10 @@
"codex:registration-capture-gap",
"codex-app-server:registration-capture-gap",
"connectclaw:registration-capture-gap",
"diagnostics-otel:missing-compat-record",
"diagnostics-otel:registration-capture-gap",
"diagnostics-prometheus:registration-capture-gap",
"diffs:registration-capture-gap",
"dingtalk-connector:registration-capture-gap",
"discord:registration-capture-gap",
"feishu:missing-compat-record",
"feishu:registration-capture-gap",
"google-meet:registration-capture-gap",
"honcho:conversation-access-hook",
"honcho:missing-compat-record",
@ -540,7 +512,6 @@
"kitchen-sink:registration-capture-gap",
"lightclawbot:registration-capture-gap",
"llm-trace-phoenix:conversation-access-hook",
"lobster:missing-compat-record",
"lossless-claw:registration-capture-gap",
"matrix:missing-compat-record",
"matrix:registration-capture-gap",
@ -552,13 +523,8 @@
"memos-cloud:conversation-access-hook",
"memos-cloud:registration-capture-gap",
"mocrane-wecom:registration-capture-gap",
"msteams:missing-compat-record",
"msteams:registration-capture-gap",
"nemoclaw:before-tool-call-probe",
"nemoclaw:registration-capture-gap",
"nextcloud-talk:missing-compat-record",
"nextcloud-talk:registration-capture-gap",
"nostr:missing-compat-record",
"nostr:registration-capture-gap",
"openclaw-qqbot:registration-capture-gap",
"openclaw-telemetry:before-tool-call-probe",
@ -570,19 +536,12 @@
"opik-openclaw:registration-capture-gap",
"qqbot:registration-capture-gap",
"secureclaw:registration-capture-gap",
"synology-chat:registration-capture-gap",
"telnyx-sms:registration-capture-gap",
"tlon:registration-capture-gap",
"twitch:registration-capture-gap",
"voice-call:missing-compat-record",
"voice-call:registration-capture-gap",
"wecom:before-tool-call-probe",
"wecom:registration-capture-gap",
"yuanbao:missing-compat-record",
"yuanbao:registration-capture-gap",
"zalo:missing-compat-record",
"zalo:registration-capture-gap",
"zalouser:registration-capture-gap"
"yuanbao:registration-capture-gap"
]
},
{

72
reports/crabpot-ci-policy.md
View File

File diff suppressed because one or more lines are too long

213
reports/crabpot-ci-summary.json
View File

@ -2,21 +2,21 @@
"generatedAt": "deterministic",
"title": "Crabpot CI Summary",
"mode": "check",
"openclawLabel": "openclaw@latest (2026.4.29, a448042c2edd)",
"openclawLabel": "openclaw/openclaw@qa-kitchen-sink-crabbox",
"status": "pass",
"summary": {
"breakages": 0,
"warnings": 117,
"suggestions": 208,
"issues": 325,
"p0Issues": 13,
"p1Issues": 71,
"liveIssues": 13,
"liveP0Issues": 13,
"compatGaps": 13,
"deprecationWarnings": 33,
"inspectorGaps": 203,
"upstreamIssues": 63,
"warnings": 129,
"suggestions": 178,
"issues": 307,
"p0Issues": 4,
"p1Issues": 52,
"liveIssues": 4,
"liveP0Issues": 4,
"compatGaps": 4,
"deprecationWarnings": 40,
"inspectorGaps": 182,
"upstreamIssues": 77,
"refDiffFailures": 0,
"refDiffWarnings": 0,
"policyFailures": 0,
@ -41,14 +41,6 @@
"importLoopCpuSampleCount": 185
},
"topIssues": [
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "bluebubbles",
"code": "sdk-export-missing",
"title": "bluebubbles: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
@ -57,22 +49,6 @@
"title": "clawmetry: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "diagnostics-otel",
"code": "sdk-export-missing",
"title": "diagnostics-otel: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "feishu",
"code": "sdk-export-missing",
"title": "feishu: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
@ -81,14 +57,6 @@
"title": "honcho: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "lobster",
"code": "sdk-export-missing",
"title": "lobster: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
@ -97,38 +65,6 @@
"title": "matrix: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "msteams",
"code": "sdk-export-missing",
"title": "msteams: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "nextcloud-talk",
"code": "sdk-export-missing",
"title": "nextcloud-talk: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "nostr",
"code": "sdk-export-missing",
"title": "nostr: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "voice-call",
"code": "sdk-export-missing",
"title": "voice-call: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
@ -137,14 +73,6 @@
"title": "yuanbao: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P0",
"issueClass": "live-issue",
"fixture": "zalo",
"code": "sdk-export-missing",
"title": "zalo: plugin SDK import aliases are missing from target package exports",
"decision": "core-compat-adapter"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
@ -153,22 +81,6 @@
"title": "a2a-gateway: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "compat-gap",
"fixture": "bluebubbles",
"code": "missing-compat-record",
"title": "bluebubbles: compat-dependent behavior lacks registry coverage",
"decision": "core-compat-adapter"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "bluebubbles",
"code": "registration-capture-gap",
"title": "bluebubbles: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "compat-gap",
@ -200,6 +112,94 @@
"code": "registration-capture-gap",
"title": "clawrouter: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "codex",
"code": "registration-capture-gap",
"title": "codex: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "codex-app-server",
"code": "registration-capture-gap",
"title": "codex-app-server: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "connectclaw",
"code": "registration-capture-gap",
"title": "connectclaw: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "diagnostics-otel",
"code": "registration-capture-gap",
"title": "diagnostics-otel: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "diagnostics-prometheus",
"code": "registration-capture-gap",
"title": "diagnostics-prometheus: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "diffs",
"code": "registration-capture-gap",
"title": "diffs: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "dingtalk-connector",
"code": "registration-capture-gap",
"title": "dingtalk-connector: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "google-meet",
"code": "registration-capture-gap",
"title": "google-meet: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "honcho",
"code": "conversation-access-hook",
"title": "honcho: conversation-access hooks need privacy-boundary probes",
"decision": "inspector-follow-up"
},
{
"severity": "P1",
"issueClass": "compat-gap",
"fixture": "honcho",
"code": "missing-compat-record",
"title": "honcho: compat-dependent behavior lacks registry coverage",
"decision": "core-compat-adapter"
},
{
"severity": "P1",
"issueClass": "inspector-gap",
"fixture": "honcho",
"code": "registration-capture-gap",
"title": "honcho: runtime registrations need capture before contract judgment",
"decision": "inspector-follow-up"
}
],
"refRegressions": [],
@ -207,21 +207,12 @@
{
"id": "compatibility-report.live-p0-issues",
"action": "warn",
"message": "13 live P0 issues tracked",
"message": "4 live P0 issues tracked",
"evidence": [
"bluebubbles:sdk-export-missing:untracked",
"clawmetry:sdk-export-missing:untracked",
"diagnostics-otel:sdk-export-missing:untracked",
"feishu:sdk-export-missing:untracked",
"honcho:sdk-export-missing:untracked",
"lobster:sdk-export-missing:untracked",
"matrix:sdk-export-missing:untracked",
"msteams:sdk-export-missing:untracked",
"nextcloud-talk:sdk-export-missing:untracked",
"nostr:sdk-export-missing:untracked",
"voice-call:sdk-export-missing:untracked",
"yuanbao:sdk-export-missing:untracked",
"zalo:sdk-export-missing:untracked"
"yuanbao:sdk-export-missing:untracked"
]
},
{

112
reports/crabpot-ci-summary.md
View File

@ -2,7 +2,7 @@
Generated: deterministic
Mode: check
OpenClaw: openclaw@latest (2026.4.29, a448042c2edd)
OpenClaw: openclaw/openclaw@qa-kitchen-sink-crabbox
Status: PASS
## Counts
@ -10,17 +10,17 @@ Status: PASS
| Metric | Value |
| --------------------------- | --------------------------------------------------------------------------------------------------------------------- |
| Breakages | 0 |
| Warnings | 117 |
| Suggestions | 208 |
| Issues | 325 |
| P0 issues | 13 |
| P1 issues | 71 |
| Live issues | 13 |
| Live P0 issues | 13 |
| Compat gaps | 13 |
| Deprecation warnings | 33 |
| Inspector gaps | 203 |
| Upstream metadata | 63 |
| Warnings | 129 |
| Suggestions | 178 |
| Issues | 307 |
| P0 issues | 4 |
| P1 issues | 52 |
| Live issues | 4 |
| Live P0 issues | 4 |
| Compat gaps | 4 |
| Deprecation warnings | 40 |
| Inspector gaps | 182 |
| Upstream metadata | 77 |
| Ref diff failures | 0 |
| Ref diff warnings | 0 |
| Policy failures | 0 |
@ -37,28 +37,28 @@ Status: PASS
## Top Issues
| Severity | Class | Fixture | Code | Decision | Title |
| -------- | ----------------- | ---------------- | ----------------------------------- | ------------------- | ----------------------------------------------------------------------------------- |
| P0 | live-issue | bluebubbles | sdk-export-missing | core-compat-adapter | bluebubbles: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | clawmetry | sdk-export-missing | core-compat-adapter | clawmetry: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | diagnostics-otel | sdk-export-missing | core-compat-adapter | diagnostics-otel: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | feishu | sdk-export-missing | core-compat-adapter | feishu: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | honcho | sdk-export-missing | core-compat-adapter | honcho: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | lobster | sdk-export-missing | core-compat-adapter | lobster: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | matrix | sdk-export-missing | core-compat-adapter | matrix: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | msteams | sdk-export-missing | core-compat-adapter | msteams: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | nextcloud-talk | sdk-export-missing | core-compat-adapter | nextcloud-talk: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | nostr | sdk-export-missing | core-compat-adapter | nostr: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | voice-call | sdk-export-missing | core-compat-adapter | voice-call: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | yuanbao | sdk-export-missing | core-compat-adapter | yuanbao: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | zalo | sdk-export-missing | core-compat-adapter | zalo: plugin SDK import aliases are missing from target package exports |
| P1 | inspector-gap | a2a-gateway | registration-capture-gap | inspector-follow-up | a2a-gateway: runtime registrations need capture before contract judgment |
| P1 | compat-gap | bluebubbles | missing-compat-record | core-compat-adapter | bluebubbles: compat-dependent behavior lacks registry coverage |
| P1 | inspector-gap | bluebubbles | registration-capture-gap | inspector-follow-up | bluebubbles: runtime registrations need capture before contract judgment |
| P1 | compat-gap | clawmetry | missing-compat-record | core-compat-adapter | clawmetry: compat-dependent behavior lacks registry coverage |
| P1 | upstream-metadata | clawmetry | package-npm-pack-entrypoint-missing | plugin-upstream-fix | clawmetry: advertised npm artifact is missing OpenClaw entrypoints |
| P1 | inspector-gap | clawmetry | registration-capture-gap | inspector-follow-up | clawmetry: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | clawrouter | registration-capture-gap | inspector-follow-up | clawrouter: runtime registrations need capture before contract judgment |
| Severity | Class | Fixture | Code | Decision | Title |
| -------- | ----------------- | ---------------------- | ----------------------------------- | ------------------- | ----------------------------------------------------------------------------------- |
| P0 | live-issue | clawmetry | sdk-export-missing | core-compat-adapter | clawmetry: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | honcho | sdk-export-missing | core-compat-adapter | honcho: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | matrix | sdk-export-missing | core-compat-adapter | matrix: plugin SDK import aliases are missing from target package exports |
| P0 | live-issue | yuanbao | sdk-export-missing | core-compat-adapter | yuanbao: plugin SDK import aliases are missing from target package exports |
| P1 | inspector-gap | a2a-gateway | registration-capture-gap | inspector-follow-up | a2a-gateway: runtime registrations need capture before contract judgment |
| P1 | compat-gap | clawmetry | missing-compat-record | core-compat-adapter | clawmetry: compat-dependent behavior lacks registry coverage |
| P1 | upstream-metadata | clawmetry | package-npm-pack-entrypoint-missing | plugin-upstream-fix | clawmetry: advertised npm artifact is missing OpenClaw entrypoints |
| P1 | inspector-gap | clawmetry | registration-capture-gap | inspector-follow-up | clawmetry: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | clawrouter | registration-capture-gap | inspector-follow-up | clawrouter: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | codex | registration-capture-gap | inspector-follow-up | codex: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | codex-app-server | registration-capture-gap | inspector-follow-up | codex-app-server: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | connectclaw | registration-capture-gap | inspector-follow-up | connectclaw: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | diagnostics-otel | registration-capture-gap | inspector-follow-up | diagnostics-otel: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | diagnostics-prometheus | registration-capture-gap | inspector-follow-up | diagnostics-prometheus: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | diffs | registration-capture-gap | inspector-follow-up | diffs: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | dingtalk-connector | registration-capture-gap | inspector-follow-up | dingtalk-connector: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | google-meet | registration-capture-gap | inspector-follow-up | google-meet: runtime registrations need capture before contract judgment |
| P1 | inspector-gap | honcho | conversation-access-hook | inspector-follow-up | honcho: conversation-access hooks need privacy-boundary probes |
| P1 | compat-gap | honcho | missing-compat-record | core-compat-adapter | honcho: compat-dependent behavior lacks registry coverage |
| P1 | inspector-gap | honcho | registration-capture-gap | inspector-follow-up | honcho: runtime registrations need capture before contract judgment |
## Ref Regressions
@ -66,28 +66,28 @@ _none_
## Policy Findings
| Action | ID | Message | Evidence |
| ------ | ------------------------------------------------------- | --------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| warn | compatibility-report.live-p0-issues | 13 live P0 issues tracked | bluebubbles:sdk-export-missing:untracked, clawmetry:sdk-export-missing:untracked, diagnostics-otel:sdk-export-missing:untracked, feishu:sdk-export-missing:untracked, honcho:sdk-export-missing:untracked, lobster:sdk-export-missing:untracked, matrix:sdk-export-missing:untracked, msteams:sdk-export-missing:untracked, nextcloud-talk:sdk-export-missing:untracked, nostr:sdk-export-missing:untracked, voice-call:sdk-export-missing:untracked, yuanbao:sdk-export-missing:untracked, zalo:sdk-export-missing:untracked |
| warn | execution-results.audit-findings | 12 package audit findings | clawrouter:12 |
| warn | execution-results.blocked.clawrouter.registerService.29 | allowed-blocked: captured registration requires includeLifecycle=true | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerService, captured registration requires includeLifecycle=true, service-lifecycle-harness |
| warn | execution-results.blocked.clawrouter.registerTool.10 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.11 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.12 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.13 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.14 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.15 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.16 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.17 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.18 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.19 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.20 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.21 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.5 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.6 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.7 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.8 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.9 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| Action | ID | Message | Evidence |
| ------ | ------------------------------------------------------- | --------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| warn | compatibility-report.live-p0-issues | 4 live P0 issues tracked | clawmetry:sdk-export-missing:untracked, honcho:sdk-export-missing:untracked, matrix:sdk-export-missing:untracked, yuanbao:sdk-export-missing:untracked |
| warn | execution-results.audit-findings | 12 package audit findings | clawrouter:12 |
| warn | execution-results.blocked.clawrouter.registerService.29 | allowed-blocked: captured registration requires includeLifecycle=true | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerService, captured registration requires includeLifecycle=true, service-lifecycle-harness |
| warn | execution-results.blocked.clawrouter.registerTool.10 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.11 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.12 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.13 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.14 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.15 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.16 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.17 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.18 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.19 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.20 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.21 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.5 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.6 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.7 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.8 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
| warn | execution-results.blocked.clawrouter.registerTool.9 | allowed-blocked: captured tool requires live network access | .crabpot/results/clawrouter/cold-import-extension-clawrouter-plugins-clawrouter-dist-index-js.synthetic.json, registerTool, captured tool requires live network access, live-network-tool-probes |
## Profile Findings

3477
reports/crabpot-issues.md
View File

File diff suppressed because it is too large Load Diff

19878
reports/crabpot-report.json
View File

File diff suppressed because it is too large Load Diff

4266
reports/crabpot-report.md
View File

File diff suppressed because one or more lines are too long

5
scripts/check-openclaw-plugin-contracts.mjs
View File

@ -14,13 +14,14 @@ const retiredTerms = [
const ignoredPaths = new Set([path.relative(repoRoot, import.meta.filename)]);
const trackedFiles = execFileSync("git", ["ls-files", "-z"], {
const trackedFiles = execFileSync("git", ["-c", "safe.directory=*", "ls-files", "-z"], {
cwd: repoRoot,
encoding: "utf8",
})
.split("\0")
.filter(Boolean)
.filter((filePath) => !ignoredPaths.has(filePath));
.filter((filePath) => !ignoredPaths.has(filePath))
.filter((filePath) => !filePath.startsWith("reports/"));
const offenders = [];
for (const filePath of trackedFiles) {

9
scripts/generate-report.mjs
View File

@ -21,6 +21,7 @@ async function main() {
const report = await buildReport({
executionResultsPath: parsedArgs.executionResultsPath,
fixtureSet: parsedArgs.fixtureSet,
openclawPath: parsedArgs.openclawPath,
});
@ -51,6 +52,7 @@ function parseArgs(argv) {
const flags = [];
let openclawPath;
let executionResultsPath;
let fixtureSet;
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
@ -68,8 +70,13 @@ function parseArgs(argv) {
index += 1;
continue;
}
if (arg === "--fixture-set") {
fixtureSet = argv[index + 1];
index += 1;
continue;
}
flags.push(arg);
}
return { executionResultsPath, flags, openclawPath };
return { executionResultsPath, fixtureSet, flags, openclawPath };
}

9
scripts/manifest-lib.mjs
View File

@ -27,12 +27,9 @@ export async function selectManifestFixtures(manifest, options = {}) {
return manifest;
}
const policy = JSON.parse(await readFile(options.policyPath ?? defaultCiPolicyPath, "utf8"));
const ids = policy.fixtureSets?.[fixtureSet];
if (!Array.isArray(ids) || ids.length === 0) {
throw new Error(`fixture set ${fixtureSet} is not defined in ${path.relative(repoRoot, options.policyPath ?? defaultCiPolicyPath)}`);
}
const wanted = new Set(ids);
const fixtures = manifest.fixtures.filter((fixture) => wanted.has(fixture.id));
const ids = policy.fixtureSets?.[fixtureSet] ?? fixtureSet.split(",").map((value) => value.trim()).filter(Boolean);
const fixturesById = new Map(manifest.fixtures.map((fixture) => [fixture.id, fixture]));
const fixtures = ids.map((id) => fixturesById.get(id)).filter(Boolean);
const missing = ids.filter((id) => !fixtures.some((fixture) => fixture.id === id));
if (missing.length > 0) {
throw new Error(`fixture set ${fixtureSet} references unknown fixture(s): ${missing.join(", ")}`);

2
scripts/plugin-inspector-source.mjs
View File

@ -4,7 +4,7 @@ import path from "node:path";
import { pathToFileURL } from "node:url";
import { repoRoot } from "./manifest-lib.mjs";
export const pluginInspectorRef = "2eda65a8a9bb70ab872b51383abb775c9765192d";
export const pluginInspectorRef = "06cc55ce517a0c53c6a42ad6f3b00b58fd87879f";
export const pluginInspectorPackage = "@openclaw/plugin-inspector@0.3.5";
export async function loadPluginInspector() {

14
scripts/sync-fixtures.mjs
View File

@ -100,7 +100,12 @@ async function materializeNpmFixture(fixture, target) {
await mkdir(target, { recursive: true });
await rm(payloadDir, { recursive: true, force: true });
await mkdir(payloadDir, { recursive: true });
run("tar", ["-xzf", path.join(tempDir, packed.filename), "-C", payloadDir, "--strip-components", "1"]);
const tarballPath = path.join(tempDir, packed.filename);
const tarArgs = ["-xzf", tarPath(tarballPath), "-C", tarPath(payloadDir), "--strip-components", "1"];
if (process.platform === "win32") {
tarArgs.unshift("--force-local");
}
run("tar", tarArgs);
await writePackageSourceMetadata(payloadDir, {
gitHead: packed.gitHead || (await npmPackageGitHead(dependency.name, dependency.version)),
name: dependency.name,
@ -263,3 +268,10 @@ function run(command, args) {
throw new Error(`${command} ${args.join(" ")} failed with ${result.status}${detail}`);
}
}
function tarPath(filePath) {
if (process.platform !== "win32") {
return filePath;
}
return filePath.replaceAll("\\", "/").replace(/^([A-Za-z]):\//, (_, drive) => `/${drive.toLowerCase()}/`);
}

22
test/ci-policy.test.mjs
View File

@ -86,6 +86,28 @@ test("default ci policy classifies opt-in kitchen-sink blockers", async () => {
);
});
test("default ci policy classifies generated kitchen-sink callable stubs", async () => {
const report = await buildCiPolicyReport({
compatibilityReport: compatibilityReport(),
executionResults: executionResults([
{
seam: "before_tool_call",
reason: "captured hook has no callable handler",
},
{
seam: "registerCommand",
reason: "captured registration has no supported callable probe",
},
]),
});
assert.equal(report.status, "pass");
assert.deepEqual(
report.checks.filter((check) => check.id.startsWith("execution-results.blocked.")).map((check) => check.action),
["warn", "warn"],
);
});
test("ci policy fails ref diff hard regressions", async () => {
const report = await buildCiPolicyReport({
policy,

4
test/ci-workflows.test.mjs
View File

@ -121,7 +121,7 @@ test("default check workflow resolves changed submodules into an isolated fixtur
assert.match(workflow, /changed-isolated-fixture:/);
assert.match(workflow, /matrix: \$\{\{ fromJson\(needs\.changed-fixture-plan\.outputs\.matrix\) \}\}/);
assert.match(workflow, /npm run workspace:execute -- --fixture "\$\{\{ matrix\.id \}\}" --allow-empty/);
assert.match(workflow, /node scripts\/generate-report\.mjs --openclaw \.\/openclaw --execution-results reports\/crabpot-execution-results\.json/);
assert.match(workflow, /node scripts\/generate-report\.mjs --openclaw \.\/openclaw --execution-results reports\/crabpot-execution-results\.json --fixture-set "\$\{\{ matrix\.id \}\}"/);
assert.match(workflow, /Fail if isolated policy failed/);
assert.match(workflow, /steps\.policy\.outcome == 'failure'/);
assert.doesNotMatch(workflow, /steps\.execute\.outcome == 'failure' \|\| steps\.policy\.outcome == 'failure'/);
@ -204,7 +204,7 @@ test("manual workflow keeps isolated execution artifacts and failure policy wire
/id: execute[\s\S]*continue-on-error: true[\s\S]*CRABPOT_EXECUTE_ISOLATED: "1"[\s\S]*npm run workspace:execute -- --fixture "\$\{\{ matrix\.id \}\}" --allow-empty/,
);
assert.match(workflow, /id: policy[\s\S]*continue-on-error: true[\s\S]*node scripts\/check-ci-policy\.mjs/);
assert.match(workflow, /node scripts\/generate-report\.mjs --openclaw \.\/openclaw --execution-results reports\/crabpot-execution-results\.json/);
assert.match(workflow, /node scripts\/generate-report\.mjs --openclaw \.\/openclaw --execution-results reports\/crabpot-execution-results\.json --fixture-set "\$\{\{ matrix\.id \}\}"/);
assert.match(workflow, /path: \|[\s\S]*\.crabpot\/results\/[\s\S]*reports\/crabpot-execution-results\.\*[\s\S]*reports\/crabpot-ci-policy\.\*[\s\S]*reports\/crabpot-ci-summary\.\*/);
assert.match(workflow, /steps\.execute\.outcome == 'failure' \|\| steps\.policy\.outcome == 'failure'/);
});

4
test/cold-import-readiness.test.mjs
View File

@ -15,8 +15,8 @@ test("cold import readiness classifies entrypoint blockers", async () => {
assert.ok(readiness.summary.dependencyInstallRequiredCount > 0);
assertHasBlocker(readiness, "wecom", "dependency-install-required");
assertHasStatus(readiness, "agentchat", "build-required");
assertHasStatus(readiness, "a2a-gateway", "ts-loader-required");
assertHasBlocker(readiness, "agentchat", "build-required");
assertHasBlocker(readiness, "a2a-gateway", "ts-loader-required");
if (report.issues.some((issue) => issue.code === "sdk-export-missing")) {
assert.ok(readiness.summary.sdkAliasRequiredCount > 0);
assertHasAnyStatus(readiness, "sdk-alias-required");

14
test/fixture-inspection.test.mjs
View File

@ -16,7 +16,7 @@ test("fixture inspections satisfy expected hooks and registrations", async () =>
for (const registration of fixture.expect?.registrations ?? []) {
assert.ok(
inspection.registrations.includes(registration),
satisfiesRegistration(registration, inspection.registrations),
`${fixture.id} should use ${registration}`,
);
}
@ -29,3 +29,15 @@ test("fixture inspections satisfy expected hooks and registrations", async () =>
}
}
});
function satisfiesRegistration(expected, observed) {
if (observed.includes(expected)) {
return true;
}
if (expected !== "registerChannel") {
return false;
}
return observed.some((registration) =>
["createChatChannelPlugin", "defineBundledChannelEntry", "defineChannelPluginEntry"].includes(registration),
);
}

10
test/manifest.test.mjs
View File

@ -51,6 +51,16 @@ test("openclaw beta fixture set narrows to beta npm packages", async () => {
assert.ok(manifest.fixtures.every((fixture) => fixture.package?.tag === "beta"));
});
test("explicit fixture set narrows to named fixtures", async () => {
const manifest = await readConfiguredManifest({ fixtureSet: "kitchen-sink,wecom" });
assert.deepEqual(manifest.fixtures.map((fixture) => fixture.id), ["kitchen-sink", "wecom"]);
assert.deepEqual(manifest.fixtureSelection, {
fixtureSet: "kitchen-sink,wecom",
ids: ["kitchen-sink", "wecom"],
});
});
test("fixture paths are stable plugin submodule paths", async () => {
const manifest = await readManifest();

8
test/report.test.mjs
View File

@ -49,16 +49,18 @@ test("compatibility report classifies current fixture seams", async () => {
assertHasFinding(report.suggestions, "wecom", "package-dependency-install-required");
assertHasFinding(report.warnings, "honcho", "conversation-access-hook");
assertHasFinding(report.warnings, "composio", "package-plugin-api-compat-missing");
assertHasFinding(report.warnings, "memos-cloud", "manifest-unknown-fields");
assertHasFinding(report.suggestions, "secureclaw", "registration-capture-gap");
if (hasTargetOpenClaw) {
assertHasFindingCode(report.warnings, "sdk-export-missing");
assertHasFindingCode(report.suggestions, "missing-compat-record");
assertHasFinding(report.warnings, "memos-cloud", "manifest-unknown-fields");
}
assertHasDecision(report.decisions, "core-compat-adapter", "env-auth");
assertHasDecision(report.decisions, "inspector-follow-up", "registration-capture");
assertHasDecision(report.decisions, "core-compat-adapter", "compat-registry");
if (hasTargetOpenClaw) {
assertHasDecision(report.decisions, "core-compat-adapter", "compat-registry");
}
assertHasIssue(report.issues, "P1", "registration-capture-gap");
assertHasIssue(report.issues, "P1", "conversation-access-hook");
@ -198,7 +200,7 @@ test("issue report maps npm payload evidence to monorepo source links", () => {
assert.match(
markdown,
/https:\/\/github\.com\/openclaw\/openclaw\/blob\/2ce6b77205187c76ce7cde6cb0913de14d4452fa\/extensions\/bluebubbles\/index\.js#L12/,
/https:\/\/github\.com\/openclaw\/openclaw\/blob\/[0-9a-f]{40}\/extensions\/bluebubbles\/index\.js#L12/,
);
});