feat: validate metric contracts

metrics/known.json

@@ -0,0 +1,73 @@
+{
+  "schemaVersion": "kova.metrics.v1",
+  "metrics": [
+    "agentCleanupMaxMs",
+    "agentCleanupMs",
+    "agentColdWarmDeltaMs",
+    "agentContainmentHealthFailures",
+    "agentPreProviderMaxMs",
+    "agentPreProviderP95Ms",
+    "agentProcessLeaks",
+    "agentProviderFinalP95Ms",
+    "agentTurnMaxMs",
+    "agentTurnMs",
+    "agentTurnP95Ms",
+    "coldAgentTurnMs",
+    "coldPreProviderMs",
+    "coldReadyMs",
+    "coldRuntimeDepsStagingMs",
+    "coldWarmDeltaMs",
+    "dashboardConnectMs",
+    "diagnosticPresent",
+    "doctorFixMs",
+    "eventLoopMaxMs",
+    "gatewayReadyHardTimeoutMs",
+    "gatewayReadyMs",
+    "gatewayResponsive",
+    "gatewayRssGrowthMb",
+    "gatewaySurvives",
+    "healthMs",
+    "healthP95Ms",
+    "inputLagMs",
+    "maxCpuPercent",
+    "missingDependencyErrors",
+    "modelsListMs",
+    "peakRssMb",
+    "pluginIndexPresent",
+    "pluginLoadFailures",
+    "pluginUpdateDryRunMs",
+    "pluginsListMs",
+    "preProviderDominanceRatio",
+    "preProviderMs",
+    "providerConcurrencyMin",
+    "providerFailureHealthFailures",
+    "providerFinalMs",
+    "providerRequestCountMin",
+    "providerSlowMinMs",
+    "providerTimeoutMentions",
+    "restartCount",
+    "restartReadyMs",
+    "rssGrowthMb",
+    "runtimeDepsStagingMs",
+    "runtimeDepsWarmReuseOk",
+    "soakCommandFailures",
+    "soakCommandP95Ms",
+    "soakDurationMs",
+    "soakHealthFailures",
+    "soakHealthP95Ms",
+    "soakIterations",
+    "soakMinDurationMs",
+    "statusAfterFailureMs",
+    "statusAfterModelsMs",
+    "statusMs",
+    "syncFsStallDetected",
+    "tuiSmokeMs",
+    "upgradeMs",
+    "warmAgentTurnMs",
+    "warmPreProviderMs",
+    "warmReadyMs",
+    "warmRuntimeDepsRestageCount",
+    "warmRuntimeDepsStagingMs",
+    "websocketDisconnects"
+  ]
+}

src/main.mjs

@@ -18,6 +18,7 @@ import { buildPerformanceSummary } from "./performance/stats.mjs";
 import { platformInfo } from "./platform.mjs";
 import { repoRoot, reportsDir } from "./paths.mjs";
 import { loadProcessRoles } from "./registries/process-roles.mjs";
+import { loadMetrics } from "./registries/metrics.mjs";
 import { loadProfile, loadProfiles } from "./registries/profiles.mjs";
 import { loadScenarios, validateScenarioRun } from "./registries/scenarios.mjs";
 import { loadState, loadStates } from "./registries/states.mjs";
@@ -98,15 +99,16 @@ async function versionCommand(flags = {}) {
 }
 
 async function loadRegistryContext() {
-  const [surfaces, processRoles, scenarios, states, profiles] = await Promise.all([
+  const [surfaces, processRoles, metrics, scenarios, states, profiles] = await Promise.all([
     loadSurfaces(),
     loadProcessRoles(),
+    loadMetrics(),
     loadScenarios(),
     loadStates(),
     loadProfiles()
   ]);
-  validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles });
-  return { surfaces, processRoles, scenarios, states, profiles };
+  validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles, metrics });
+  return { surfaces, processRoles, metrics, scenarios, states, profiles };
 }
 
 function filterRegistry(items, selectedId, kind) {
@@ -134,6 +136,7 @@ async function plan(flags) {
       platform: platformInfo(),
       surfaces: registry.surfaces,
       processRoles: registry.processRoles,
+      metrics: registry.metrics,
       scenarios,
       states,
       profiles: profiles.map(profileSummary),

src/paths.mjs

@@ -8,6 +8,7 @@ export const statesDir = join(repoRoot, "states");
 export const profilesDir = join(repoRoot, "profiles");
 export const surfacesDir = join(repoRoot, "surfaces");
 export const processRolesDir = join(repoRoot, "process-roles");
+export const metricsDir = join(repoRoot, "metrics");
 export const kovaHome = resolveKovaHome();
 export const credentialsDir = join(kovaHome, "credentials");
 export const providersPath = join(credentialsDir, "providers.json");

src/registries/metrics.mjs

@@ -0,0 +1,33 @@
+import { readFile } from "node:fs/promises";
+import { join } from "node:path";
+import { metricsDir } from "../paths.mjs";
+import { assertNoShapeErrors } from "./validate.mjs";
+
+export async function loadMetrics() {
+  const data = JSON.parse(await readFile(join(metricsDir, "known.json"), "utf8"));
+  validateMetricsShape(data, "metrics/known.json");
+  return data.metrics.map((id) => ({ id }));
+}
+
+export function validateMetricsShape(data, sourceName = "metrics") {
+  const errors = [];
+  if (data?.schemaVersion !== "kova.metrics.v1") {
+    errors.push("schemaVersion must be kova.metrics.v1");
+  }
+  if (!Array.isArray(data?.metrics)) {
+    errors.push("metrics must be an array");
+  } else {
+    const seen = new Set();
+    for (const [index, metric] of data.metrics.entries()) {
+      if (typeof metric !== "string" || !/^[A-Za-z][A-Za-z0-9]*$/.test(metric)) {
+        errors.push(`metrics[${index}] must be a camelCase metric id`);
+        continue;
+      }
+      if (seen.has(metric)) {
+        errors.push(`duplicate metric id '${metric}'`);
+      }
+      seen.add(metric);
+    }
+  }
+  assertNoShapeErrors(errors, sourceName);
+}

src/registries/validate.mjs

@@ -25,12 +25,13 @@ export async function loadJsonRegistry({ dir, kind, selectedId, validate }) {
   return filtered;
 }
 
-export function validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles }) {
+export function validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles, metrics = [] }) {
   const errors = [];
   const scenarioIds = idSet(scenarios);
   const stateIds = idSet(states);
   const surfaceIds = idSet(surfaces);
   const processRoleIds = idSet(processRoles);
+  const metricIds = idSet(metrics);
   const traitIds = new Set(states.flatMap((state) => state.traits ?? []));
   const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario]));
   const stateById = new Map(states.map((state) => [state.id, state]));
@@ -41,7 +42,7 @@ export function validateRegistryReferences({ scenarios, states, profiles, surfac
       errors.push(`scenario '${scenario.id}' references unknown surface '${scenario.surface}'`);
       continue;
     }
-    validateScenarioContract(scenario, surfaceById.get(scenario.surface), { stateIds, processRoleIds }, errors);
+    validateScenarioContract(scenario, surfaceById.get(scenario.surface), { stateIds, processRoleIds, metricIds }, errors);
   }
 
   for (const state of states) {
@@ -73,6 +74,11 @@ export function validateRegistryReferences({ scenarios, states, profiles, surfac
         errors.push(`surface '${surface.id}' references unknown required state '${state}'`);
       }
     }
+    validateMetricList(surface.requiredMetrics ?? [], metricIds, errors, `surface '${surface.id}' requiredMetrics`);
+    validateThresholdMetrics(surface.thresholds ?? {}, metricIds, errors, `surface '${surface.id}' thresholds`);
+    for (const [role, thresholds] of Object.entries(surface.roleThresholds ?? {})) {
+      validateThresholdMetrics(thresholds, metricIds, errors, `surface '${surface.id}' roleThresholds.${role}`);
+    }
   }
 
   for (const profile of profiles) {
@@ -101,6 +107,29 @@ function validateScenarioContract(scenario, surface, refs, errors) {
       errors.push(`scenario '${scenario.id}' targetKinds references '${targetKind}' which is not supported by surface '${surface.id}'`);
     }
   }
+  validateThresholdMetrics(scenario.thresholds ?? {}, refs.metricIds, errors, `scenario '${scenario.id}' thresholds`);
+}
+
+function validateMetricList(metrics, metricIds, errors, prefix) {
+  for (const metric of metrics) {
+    if (!metricIds.has(metric)) {
+      errors.push(`${prefix} references unknown metric '${metric}'`);
+    }
+  }
+}
+
+function validateThresholdMetrics(thresholds, metricIds, errors, prefix) {
+  for (const [metric, value] of Object.entries(thresholds ?? {})) {
+    if (metric === "roleThresholds") {
+      for (const [role, roleThresholds] of Object.entries(value ?? {})) {
+        validateThresholdMetrics(roleThresholds, metricIds, errors, `${prefix}.roleThresholds.${role}`);
+      }
+      continue;
+    }
+    if (!metricIds.has(metric)) {
+      errors.push(`${prefix} references unknown metric '${metric}'`);
+    }
+  }
 }
 
 function validateProfileReferences(profile, refs, errors) {

src/selfcheck.mjs

@@ -84,6 +84,7 @@ export async function runSelfCheck(flags = {}) {
       assertEqual(data.schemaVersion, "kova.plan.v1", "plan schema");
       assertArrayNotEmpty(data.surfaces, "plan surfaces");
       assertArrayNotEmpty(data.processRoles, "plan process roles");
+      assertArrayNotEmpty(data.metrics, "plan metrics");
       assertArrayNotEmpty(data.scenarios, "plan scenarios");
       assertArrayNotEmpty(data.states, "plan states");
       assertArrayNotEmpty(data.profiles, "profiles");
@@ -2721,6 +2722,35 @@ function stateRegistryValidationCheck() {
     }
     assertEqual(rejectedCoveragePair, true, "invalid coverage state/surface pair rejected");
 
+    let rejectedMetric = false;
+    try {
+      validateRegistryReferences({
+        scenarios: [{
+          id: "scenario",
+          surface: "known-surface",
+          thresholds: { madeUpMetric: 1 },
+          states: [],
+          targetKinds: [],
+          processRoles: []
+        }],
+        states: [],
+        profiles: [],
+        surfaces: [{
+          id: "known-surface",
+          processRoles: [],
+          requiredStates: [],
+          requiredMetrics: ["knownMetric"],
+          thresholds: { knownMetric: 1 },
+          targetKinds: []
+        }],
+        processRoles: [],
+        metrics: [{ id: "knownMetric" }]
+      });
+    } catch (error) {
+      rejectedMetric = /unknown metric 'madeUpMetric'/.test(error.message);
+    }
+    assertEqual(rejectedMetric, true, "unknown scenario metric rejected");
+
     return {
       id: "state-registry-validation",
       status: "PASS",