fix(reports): link probe gaps to compat records

This commit is contained in:
Vincent Koc 2026-04-28 21:13:15 -07:00
parent 332706b014
commit d91d596d90
No known key found for this signature in database
6 changed files with 56 additions and 4 deletions

View File

@ -43,6 +43,7 @@ export {
} from "./ci-outputs.js";
export {
buildContractProbes,
compatRecordForIssueCode,
contractProbeRules,
probePriority,
} from "./contract-probes.js";

View File

@ -106,6 +106,20 @@ export const contractProbeRules = {
},
};
const openClawOwnedProbeIssueCodes = new Set([
"before-tool-call-probe",
"channel-contract-probe",
"conversation-access-hook",
"registration-capture-gap",
]);
export function compatRecordForIssueCode(code) {
if (!openClawOwnedProbeIssueCodes.has(code)) {
return undefined;
}
return contractProbeRules[code]?.id;
}
export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
const probes = [];

View File

@ -1,6 +1,7 @@
import { existsSync } from "node:fs";
import { readdir } from "node:fs/promises";
import path from "node:path";
import { compatRecordForIssueCode } from "./contract-probes.js";
import { readJsonFile } from "./json-file.js";
const conversationAccessHooks = new Set(["agent_end", "llm_input", "llm_output"]);
@ -399,6 +400,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "warning",
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
evidence: detailEvidence(conversationHookDetails),
compatRecord: compatRecordForIssueCode("conversation-access-hook"),
});
decisions.push({
fixture: fixture.id,
@ -459,6 +461,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
evidence: detailEvidence(captureGapRegistrationDetails),
compatRecord: compatRecordForIssueCode("registration-capture-gap"),
});
decisions.push({
fixture: fixture.id,
@ -477,6 +480,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
evidence: detailEvidence(hookDetails),
compatRecord: compatRecordForIssueCode("before-tool-call-probe"),
});
decisions.push({
fixture: fixture.id,
@ -500,6 +504,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
level: "suggestion",
message: "add channel envelope, config-schema, and runtime metadata probes",
evidence: detailEvidence(channelRegistrationDetails),
compatRecord: compatRecordForIssueCode("channel-contract-probe"),
});
decisions.push({
fixture: fixture.id,

View File

@ -1,6 +1,6 @@
import assert from "node:assert/strict";
import { test } from "node:test";
import { buildContractProbes, contractProbeRules, probePriority } from "../src/advanced.js";
import { buildContractProbes, compatRecordForIssueCode, contractProbeRules, probePriority } from "../src/advanced.js";
test("contract probes map issue findings to executable backlog rows", () => {
const probes = buildContractProbes({
@ -41,6 +41,8 @@ test("contract probes map issue findings to executable backlog rows", () => {
});
assert.ok(contractProbeRules["registration-capture-gap"]);
assert.equal(compatRecordForIssueCode("registration-capture-gap"), "api.capture.runtime-registrars");
assert.equal(compatRecordForIssueCode("package-dependency-install-required"), undefined);
assert.deepEqual(
probes.map((probe) => [probe.id, probe.priority, probe.target]),
[

View File

@ -46,6 +46,18 @@ test("issue classification separates live breaks from compat and deprecation buc
metadata: { severity: "P1" },
expected: { issueClass: "compat-gap", compatStatus: "missing", severity: "P1", live: false },
},
{
name: "active OpenClaw probe contract stays an inspector gap",
finding: {
code: "before-tool-call-probe",
compatRecord: "hook.before_tool_call.terminal-block-approval",
},
targetOpenClaw: {
compatRecordStatuses: { "hook.before_tool_call.terminal-block-approval": "active" },
},
metadata: { severity: "P1" },
expected: { issueClass: "inspector-gap", compatStatus: "active", severity: "P1", live: false },
},
{
name: "unknown untracked hook is P0 live break",
finding: { code: "unknown-hook-name" },

View File

@ -592,11 +592,29 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
assert.ok(result.warnings.some((finding) => finding.code === "provider-auth-env-vars"));
assert.ok(result.warnings.some((finding) => finding.code === "channel-env-vars"));
assert.ok(result.warnings.some((finding) => finding.code === "conversation-access-hook"));
assert.ok(
result.warnings.some(
(finding) =>
finding.code === "conversation-access-hook" &&
finding.compatRecord === "hook.llm-observer.privacy-payload",
),
);
assert.ok(result.warnings.some((finding) => finding.code === "legacy-root-sdk-import"));
assert.ok(result.warnings.some((finding) => finding.code === "package-json-missing"));
assert.ok(result.suggestions.some((finding) => finding.code === "registration-capture-gap"));
assert.ok(result.suggestions.some((finding) => finding.code === "before-tool-call-probe"));
assert.ok(
result.suggestions.some(
(finding) =>
finding.code === "registration-capture-gap" &&
finding.compatRecord === "api.capture.runtime-registrars",
),
);
assert.ok(
result.suggestions.some(
(finding) =>
finding.code === "before-tool-call-probe" &&
finding.compatRecord === "hook.before_tool_call.terminal-block-approval",
),
);
assert.ok(result.suggestions.some((finding) => finding.code === "runtime-tool-capture"));
assert.ok(result.decisions.some((decision) => decision.seam === "conversation-access"));
});