fix(reports): link probe gaps to compat records
This commit is contained in:
parent
332706b014
commit
d91d596d90
@ -43,6 +43,7 @@ export {
|
||||
} from "./ci-outputs.js";
|
||||
export {
|
||||
buildContractProbes,
|
||||
compatRecordForIssueCode,
|
||||
contractProbeRules,
|
||||
probePriority,
|
||||
} from "./contract-probes.js";
|
||||
|
||||
@ -106,6 +106,20 @@ export const contractProbeRules = {
|
||||
},
|
||||
};
|
||||
|
||||
const openClawOwnedProbeIssueCodes = new Set([
|
||||
"before-tool-call-probe",
|
||||
"channel-contract-probe",
|
||||
"conversation-access-hook",
|
||||
"registration-capture-gap",
|
||||
]);
|
||||
|
||||
export function compatRecordForIssueCode(code) {
|
||||
if (!openClawOwnedProbeIssueCodes.has(code)) {
|
||||
return undefined;
|
||||
}
|
||||
return contractProbeRules[code]?.id;
|
||||
}
|
||||
|
||||
export function buildContractProbes({ warnings = [], suggestions = [], fixtures = [] }) {
|
||||
const fixtureById = new Map(fixtures.map((fixture) => [fixture.id, fixture]));
|
||||
const probes = [];
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
import { existsSync } from "node:fs";
|
||||
import { readdir } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { compatRecordForIssueCode } from "./contract-probes.js";
|
||||
import { readJsonFile } from "./json-file.js";
|
||||
|
||||
const conversationAccessHooks = new Set(["agent_end", "llm_input", "llm_output"]);
|
||||
@ -399,6 +400,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "warning",
|
||||
message: "fixture observes raw model or conversation content and needs privacy-boundary contract probes",
|
||||
evidence: detailEvidence(conversationHookDetails),
|
||||
compatRecord: compatRecordForIssueCode("conversation-access-hook"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -459,6 +461,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "future inspector capture API should record lifecycle, route, gateway, command, and interactive registrations",
|
||||
evidence: detailEvidence(captureGapRegistrationDetails),
|
||||
compatRecord: compatRecordForIssueCode("registration-capture-gap"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -477,6 +480,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "add contract probes for before_tool_call terminal, block, and approval semantics",
|
||||
evidence: detailEvidence(hookDetails),
|
||||
compatRecord: compatRecordForIssueCode("before-tool-call-probe"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
@ -500,6 +504,7 @@ export function classifyCompatibilityFixture({ fixture, inspection, fixtureRepor
|
||||
level: "suggestion",
|
||||
message: "add channel envelope, config-schema, and runtime metadata probes",
|
||||
evidence: detailEvidence(channelRegistrationDetails),
|
||||
compatRecord: compatRecordForIssueCode("channel-contract-probe"),
|
||||
});
|
||||
decisions.push({
|
||||
fixture: fixture.id,
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "node:test";
|
||||
import { buildContractProbes, contractProbeRules, probePriority } from "../src/advanced.js";
|
||||
import { buildContractProbes, compatRecordForIssueCode, contractProbeRules, probePriority } from "../src/advanced.js";
|
||||
|
||||
test("contract probes map issue findings to executable backlog rows", () => {
|
||||
const probes = buildContractProbes({
|
||||
@ -41,6 +41,8 @@ test("contract probes map issue findings to executable backlog rows", () => {
|
||||
});
|
||||
|
||||
assert.ok(contractProbeRules["registration-capture-gap"]);
|
||||
assert.equal(compatRecordForIssueCode("registration-capture-gap"), "api.capture.runtime-registrars");
|
||||
assert.equal(compatRecordForIssueCode("package-dependency-install-required"), undefined);
|
||||
assert.deepEqual(
|
||||
probes.map((probe) => [probe.id, probe.priority, probe.target]),
|
||||
[
|
||||
|
||||
@ -46,6 +46,18 @@ test("issue classification separates live breaks from compat and deprecation buc
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "compat-gap", compatStatus: "missing", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "active OpenClaw probe contract stays an inspector gap",
|
||||
finding: {
|
||||
code: "before-tool-call-probe",
|
||||
compatRecord: "hook.before_tool_call.terminal-block-approval",
|
||||
},
|
||||
targetOpenClaw: {
|
||||
compatRecordStatuses: { "hook.before_tool_call.terminal-block-approval": "active" },
|
||||
},
|
||||
metadata: { severity: "P1" },
|
||||
expected: { issueClass: "inspector-gap", compatStatus: "active", severity: "P1", live: false },
|
||||
},
|
||||
{
|
||||
name: "unknown untracked hook is P0 live break",
|
||||
finding: { code: "unknown-hook-name" },
|
||||
|
||||
@ -592,11 +592,29 @@ test("compatibility fixture classifier reports seam and metadata follow-ups", ()
|
||||
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "provider-auth-env-vars"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "channel-env-vars"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "conversation-access-hook"));
|
||||
assert.ok(
|
||||
result.warnings.some(
|
||||
(finding) =>
|
||||
finding.code === "conversation-access-hook" &&
|
||||
finding.compatRecord === "hook.llm-observer.privacy-payload",
|
||||
),
|
||||
);
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "legacy-root-sdk-import"));
|
||||
assert.ok(result.warnings.some((finding) => finding.code === "package-json-missing"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "registration-capture-gap"));
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "before-tool-call-probe"));
|
||||
assert.ok(
|
||||
result.suggestions.some(
|
||||
(finding) =>
|
||||
finding.code === "registration-capture-gap" &&
|
||||
finding.compatRecord === "api.capture.runtime-registrars",
|
||||
),
|
||||
);
|
||||
assert.ok(
|
||||
result.suggestions.some(
|
||||
(finding) =>
|
||||
finding.code === "before-tool-call-probe" &&
|
||||
finding.compatRecord === "hook.before_tool_call.terminal-block-approval",
|
||||
),
|
||||
);
|
||||
assert.ok(result.suggestions.some((finding) => finding.code === "runtime-tool-capture"));
|
||||
assert.ok(result.decisions.some((decision) => decision.seam === "conversation-access"));
|
||||
});
|
||||
|
||||
Loading…
Reference in New Issue
Block a user