diff --git a/metrics/known.json b/metrics/known.json new file mode 100644 index 0000000..aeb0ccc --- /dev/null +++ b/metrics/known.json @@ -0,0 +1,73 @@ +{ + "schemaVersion": "kova.metrics.v1", + "metrics": [ + "agentCleanupMaxMs", + "agentCleanupMs", + "agentColdWarmDeltaMs", + "agentContainmentHealthFailures", + "agentPreProviderMaxMs", + "agentPreProviderP95Ms", + "agentProcessLeaks", + "agentProviderFinalP95Ms", + "agentTurnMaxMs", + "agentTurnMs", + "agentTurnP95Ms", + "coldAgentTurnMs", + "coldPreProviderMs", + "coldReadyMs", + "coldRuntimeDepsStagingMs", + "coldWarmDeltaMs", + "dashboardConnectMs", + "diagnosticPresent", + "doctorFixMs", + "eventLoopMaxMs", + "gatewayReadyHardTimeoutMs", + "gatewayReadyMs", + "gatewayResponsive", + "gatewayRssGrowthMb", + "gatewaySurvives", + "healthMs", + "healthP95Ms", + "inputLagMs", + "maxCpuPercent", + "missingDependencyErrors", + "modelsListMs", + "peakRssMb", + "pluginIndexPresent", + "pluginLoadFailures", + "pluginUpdateDryRunMs", + "pluginsListMs", + "preProviderDominanceRatio", + "preProviderMs", + "providerConcurrencyMin", + "providerFailureHealthFailures", + "providerFinalMs", + "providerRequestCountMin", + "providerSlowMinMs", + "providerTimeoutMentions", + "restartCount", + "restartReadyMs", + "rssGrowthMb", + "runtimeDepsStagingMs", + "runtimeDepsWarmReuseOk", + "soakCommandFailures", + "soakCommandP95Ms", + "soakDurationMs", + "soakHealthFailures", + "soakHealthP95Ms", + "soakIterations", + "soakMinDurationMs", + "statusAfterFailureMs", + "statusAfterModelsMs", + "statusMs", + "syncFsStallDetected", + "tuiSmokeMs", + "upgradeMs", + "warmAgentTurnMs", + "warmPreProviderMs", + "warmReadyMs", + "warmRuntimeDepsRestageCount", + "warmRuntimeDepsStagingMs", + "websocketDisconnects" + ] +} diff --git a/src/main.mjs b/src/main.mjs index d65d46e..bdb051a 100644 --- a/src/main.mjs +++ b/src/main.mjs @@ -18,6 +18,7 @@ import { buildPerformanceSummary } from "./performance/stats.mjs"; import { platformInfo } from "./platform.mjs"; import { repoRoot, reportsDir } from "./paths.mjs"; import { loadProcessRoles } from "./registries/process-roles.mjs"; +import { loadMetrics } from "./registries/metrics.mjs"; import { loadProfile, loadProfiles } from "./registries/profiles.mjs"; import { loadScenarios, validateScenarioRun } from "./registries/scenarios.mjs"; import { loadState, loadStates } from "./registries/states.mjs"; @@ -98,15 +99,16 @@ async function versionCommand(flags = {}) { } async function loadRegistryContext() { - const [surfaces, processRoles, scenarios, states, profiles] = await Promise.all([ + const [surfaces, processRoles, metrics, scenarios, states, profiles] = await Promise.all([ loadSurfaces(), loadProcessRoles(), + loadMetrics(), loadScenarios(), loadStates(), loadProfiles() ]); - validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles }); - return { surfaces, processRoles, scenarios, states, profiles }; + validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles, metrics }); + return { surfaces, processRoles, metrics, scenarios, states, profiles }; } function filterRegistry(items, selectedId, kind) { @@ -134,6 +136,7 @@ async function plan(flags) { platform: platformInfo(), surfaces: registry.surfaces, processRoles: registry.processRoles, + metrics: registry.metrics, scenarios, states, profiles: profiles.map(profileSummary), diff --git a/src/paths.mjs b/src/paths.mjs index f84c446..cf73794 100644 --- a/src/paths.mjs +++ b/src/paths.mjs @@ -8,6 +8,7 @@ export const statesDir = join(repoRoot, "states"); export const profilesDir = join(repoRoot, "profiles"); export const surfacesDir = join(repoRoot, "surfaces"); export const processRolesDir = join(repoRoot, "process-roles"); +export const metricsDir = join(repoRoot, "metrics"); export const kovaHome = resolveKovaHome(); export const credentialsDir = join(kovaHome, "credentials"); export const providersPath = join(credentialsDir, "providers.json"); diff --git a/src/registries/metrics.mjs b/src/registries/metrics.mjs new file mode 100644 index 0000000..a17db6d --- /dev/null +++ b/src/registries/metrics.mjs @@ -0,0 +1,33 @@ +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { metricsDir } from "../paths.mjs"; +import { assertNoShapeErrors } from "./validate.mjs"; + +export async function loadMetrics() { + const data = JSON.parse(await readFile(join(metricsDir, "known.json"), "utf8")); + validateMetricsShape(data, "metrics/known.json"); + return data.metrics.map((id) => ({ id })); +} + +export function validateMetricsShape(data, sourceName = "metrics") { + const errors = []; + if (data?.schemaVersion !== "kova.metrics.v1") { + errors.push("schemaVersion must be kova.metrics.v1"); + } + if (!Array.isArray(data?.metrics)) { + errors.push("metrics must be an array"); + } else { + const seen = new Set(); + for (const [index, metric] of data.metrics.entries()) { + if (typeof metric !== "string" || !/^[A-Za-z][A-Za-z0-9]*$/.test(metric)) { + errors.push(`metrics[${index}] must be a camelCase metric id`); + continue; + } + if (seen.has(metric)) { + errors.push(`duplicate metric id '${metric}'`); + } + seen.add(metric); + } + } + assertNoShapeErrors(errors, sourceName); +} diff --git a/src/registries/validate.mjs b/src/registries/validate.mjs index 8712b1d..ddbce39 100644 --- a/src/registries/validate.mjs +++ b/src/registries/validate.mjs @@ -25,12 +25,13 @@ export async function loadJsonRegistry({ dir, kind, selectedId, validate }) { return filtered; } -export function validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles }) { +export function validateRegistryReferences({ scenarios, states, profiles, surfaces, processRoles, metrics = [] }) { const errors = []; const scenarioIds = idSet(scenarios); const stateIds = idSet(states); const surfaceIds = idSet(surfaces); const processRoleIds = idSet(processRoles); + const metricIds = idSet(metrics); const traitIds = new Set(states.flatMap((state) => state.traits ?? [])); const scenarioById = new Map(scenarios.map((scenario) => [scenario.id, scenario])); const stateById = new Map(states.map((state) => [state.id, state])); @@ -41,7 +42,7 @@ export function validateRegistryReferences({ scenarios, states, profiles, surfac errors.push(`scenario '${scenario.id}' references unknown surface '${scenario.surface}'`); continue; } - validateScenarioContract(scenario, surfaceById.get(scenario.surface), { stateIds, processRoleIds }, errors); + validateScenarioContract(scenario, surfaceById.get(scenario.surface), { stateIds, processRoleIds, metricIds }, errors); } for (const state of states) { @@ -73,6 +74,11 @@ export function validateRegistryReferences({ scenarios, states, profiles, surfac errors.push(`surface '${surface.id}' references unknown required state '${state}'`); } } + validateMetricList(surface.requiredMetrics ?? [], metricIds, errors, `surface '${surface.id}' requiredMetrics`); + validateThresholdMetrics(surface.thresholds ?? {}, metricIds, errors, `surface '${surface.id}' thresholds`); + for (const [role, thresholds] of Object.entries(surface.roleThresholds ?? {})) { + validateThresholdMetrics(thresholds, metricIds, errors, `surface '${surface.id}' roleThresholds.${role}`); + } } for (const profile of profiles) { @@ -101,6 +107,29 @@ function validateScenarioContract(scenario, surface, refs, errors) { errors.push(`scenario '${scenario.id}' targetKinds references '${targetKind}' which is not supported by surface '${surface.id}'`); } } + validateThresholdMetrics(scenario.thresholds ?? {}, refs.metricIds, errors, `scenario '${scenario.id}' thresholds`); +} + +function validateMetricList(metrics, metricIds, errors, prefix) { + for (const metric of metrics) { + if (!metricIds.has(metric)) { + errors.push(`${prefix} references unknown metric '${metric}'`); + } + } +} + +function validateThresholdMetrics(thresholds, metricIds, errors, prefix) { + for (const [metric, value] of Object.entries(thresholds ?? {})) { + if (metric === "roleThresholds") { + for (const [role, roleThresholds] of Object.entries(value ?? {})) { + validateThresholdMetrics(roleThresholds, metricIds, errors, `${prefix}.roleThresholds.${role}`); + } + continue; + } + if (!metricIds.has(metric)) { + errors.push(`${prefix} references unknown metric '${metric}'`); + } + } } function validateProfileReferences(profile, refs, errors) { diff --git a/src/selfcheck.mjs b/src/selfcheck.mjs index 700e46a..0765f7a 100644 --- a/src/selfcheck.mjs +++ b/src/selfcheck.mjs @@ -84,6 +84,7 @@ export async function runSelfCheck(flags = {}) { assertEqual(data.schemaVersion, "kova.plan.v1", "plan schema"); assertArrayNotEmpty(data.surfaces, "plan surfaces"); assertArrayNotEmpty(data.processRoles, "plan process roles"); + assertArrayNotEmpty(data.metrics, "plan metrics"); assertArrayNotEmpty(data.scenarios, "plan scenarios"); assertArrayNotEmpty(data.states, "plan states"); assertArrayNotEmpty(data.profiles, "profiles"); @@ -2721,6 +2722,35 @@ function stateRegistryValidationCheck() { } assertEqual(rejectedCoveragePair, true, "invalid coverage state/surface pair rejected"); + let rejectedMetric = false; + try { + validateRegistryReferences({ + scenarios: [{ + id: "scenario", + surface: "known-surface", + thresholds: { madeUpMetric: 1 }, + states: [], + targetKinds: [], + processRoles: [] + }], + states: [], + profiles: [], + surfaces: [{ + id: "known-surface", + processRoles: [], + requiredStates: [], + requiredMetrics: ["knownMetric"], + thresholds: { knownMetric: 1 }, + targetKinds: [] + }], + processRoles: [], + metrics: [{ id: "knownMetric" }] + }); + } catch (error) { + rejectedMetric = /unknown metric 'madeUpMetric'/.test(error.message); + } + assertEqual(rejectedMetric, true, "unknown scenario metric rejected"); + return { id: "state-registry-validation", status: "PASS",