feat: attribute dashboard pre-provider spans

This commit is contained in:
Shakker 2026-05-06 14:46:54 +01:00
parent b67f5457d9
commit 2b5bddff4a
No known key found for this signature in database
9 changed files with 851 additions and 4 deletions

View File

@ -275,6 +275,51 @@ Gateway/session turn entries include:
}
```
Dashboard session turns also include pre-provider attribution when an OpenClaw
diagnostics timeline is available. Kova clips `gateway.chat_send*`,
`auto_reply*`, and `reply.*` spans to the active `sessions.send` pre-provider
window and reports the unioned known time so overlapping spans are not counted
twice. Provider work remains separate.
```json
{
"dashboardPreProviderAttribution": {
"schemaVersion": "kova.dashboardPreProviderAttribution.v1",
"available": true,
"label": "cold",
"timelineArtifacts": ["/tmp/kova/openclaw/timeline.jsonl"],
"window": {
"startEpochMs": 1777536000000,
"endEpochMs": 1777536000200,
"durationMs": 200
},
"provider": {
"totalDurationMs": 600,
"firstByteLatencyMs": 25,
"firstChunkLatencyMs": 30
},
"knownAttributedMs": 170,
"unattributedMs": 30,
"coverageRatio": 0.85,
"spanSummaries": [
{
"name": "auto_reply.finalize_context",
"count": 1,
"errorCount": 0,
"totalClippedDurationMs": 100,
"maxClippedDurationMs": 100
}
]
}
}
```
Repeat summaries expose machine-readable medians at
`records[*].measurements.dashboardPreProviderAttribution` plus flat comparison
metrics such as `coldPreProviderAttributedMs`,
`coldPreProviderUnattributedMs`, `warmPreProviderAttributedMs`, and
`warmPreProviderUnattributedMs`.
Aggregate fields are also exposed on `measurements` for comparison and
performance summaries:

View File

@ -29,7 +29,10 @@
"browserTabCountMin",
"browserTabsMs",
"coldAgentTurnMs",
"coldPreProviderAttributedMs",
"coldPreProviderAttributionCoverage",
"coldPreProviderMs",
"coldPreProviderUnattributedMs",
"coldReadyMs",
"coldRuntimeDepsStagingMs",
"coldWarmDeltaMs",
@ -105,7 +108,10 @@
"tuiSmokeMs",
"upgradeMs",
"warmAgentTurnMs",
"warmPreProviderAttributedMs",
"warmPreProviderAttributionCoverage",
"warmPreProviderMs",
"warmPreProviderUnattributedMs",
"warmReadyMs",
"warmRuntimeDepsRestageCount",
"warmRuntimeDepsStagingMs",

View File

@ -0,0 +1,419 @@
export const DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA = "kova.dashboardPreProviderAttribution.v1";
export const DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA = "kova.dashboardPreProviderAttributionSummary.v1";
export function buildDashboardPreProviderAttribution({
label,
phaseId,
activeStartedAtEpochMs,
activeFinishedAtEpochMs,
attribution,
timelineSummary
}) {
const artifacts = timelineArtifacts(timelineSummary);
const events = attributionEvents(timelineSummary);
const providerBoundaryEpochMs = numberOrNull(attribution?.firstProviderRequestAtEpochMs);
const windowStartEpochMs = numberOrNull(activeStartedAtEpochMs ?? attribution?.commandStartedAtEpochMs);
const activeEndEpochMs = numberOrNull(activeFinishedAtEpochMs ?? attribution?.commandFinishedAtEpochMs);
const windowEndEpochMs = providerBoundaryEpochMs;
const preProviderMs = numberOrNull(attribution?.preProviderMs) ??
durationBetween(windowStartEpochMs, windowEndEpochMs);
const base = {
schemaVersion: DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA,
available: false,
label: label ?? null,
phaseId: phaseId ?? null,
timelineAvailable: timelineSummary?.available === true,
timelineArtifacts: artifacts,
eventCount: events.length,
window: {
startEpochMs: windowStartEpochMs,
startAt: isoOrNull(windowStartEpochMs),
endEpochMs: windowEndEpochMs,
endAt: isoOrNull(windowEndEpochMs),
durationMs: preProviderMs
},
activeWindow: {
startEpochMs: windowStartEpochMs,
startAt: isoOrNull(windowStartEpochMs),
endEpochMs: activeEndEpochMs,
endAt: isoOrNull(activeEndEpochMs),
durationMs: durationBetween(windowStartEpochMs, activeEndEpochMs)
},
providerBoundary: {
firstRequestAtEpochMs: providerBoundaryEpochMs,
firstRequestAt: isoOrNull(providerBoundaryEpochMs),
source: providerBoundaryEpochMs === null ? null : "provider-evidence"
},
provider: summarizeProviderEvents(events, providerBoundaryEpochMs, activeEndEpochMs, attribution),
spanSummaries: [],
knownAttributedMs: null,
unattributedMs: preProviderMs,
coverageRatio: null,
error: null
};
if (timelineSummary?.available !== true) {
return {
...base,
error: "OpenClaw diagnostics timeline unavailable"
};
}
if (events.length === 0) {
return {
...base,
error: "timeline contains no dashboard turn attribution events"
};
}
if (windowStartEpochMs === null || windowEndEpochMs === null || preProviderMs === null || windowEndEpochMs < windowStartEpochMs) {
return {
...base,
error: "pre-provider window boundary unavailable"
};
}
const intervals = attributedSpanIntervals(events)
.map((span) => clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs))
.filter(Boolean);
const spanSummaries = summarizeAttributedSpans(intervals);
const knownAttributedMs = round(unionDuration(intervals));
const unattributedMs = round(Math.max(0, preProviderMs - knownAttributedMs));
return {
...base,
available: true,
spanSummaries,
knownAttributedMs,
unattributedMs,
coverageRatio: preProviderMs > 0 ? round(knownAttributedMs / preProviderMs) : null,
error: null
};
}
export function summarizeDashboardPreProviderAttributions(turns) {
const entries = (turns ?? [])
.map((turn) => turn.dashboardPreProviderAttribution)
.filter(Boolean);
const cold = summarizeLabeledAttribution(entries, "cold");
const warm = summarizeLabeledAttribution(entries, "warm");
return {
schemaVersion: DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA,
available: entries.some((entry) => entry.available === true),
count: entries.length,
cold,
warm,
spanMedians: summarizeSpanMedians(entries),
timelineArtifacts: unique(entries.flatMap((entry) => entry.timelineArtifacts ?? []))
};
}
export function dashboardPreProviderMarkdownRows(turns) {
const attributions = (turns ?? [])
.map((turn) => turn.dashboardPreProviderAttribution)
.filter(Boolean);
if (attributions.length === 0) {
return [];
}
const lines = [
"- Dashboard pre-provider attribution:",
"",
" | turn | pre-provider | known | unattributed | provider | timeline |",
" |---|---:|---:|---:|---:|---|"
];
for (const item of attributions) {
const timeline = item.timelineArtifacts?.[0] ?? (item.timelineAvailable ? "available" : "missing");
lines.push(
` | ${item.label ?? "turn"} | ${formatMs(item.window?.durationMs)} | ${formatMs(item.knownAttributedMs)} | ${formatMs(item.unattributedMs)} | ${formatMs(item.provider?.totalDurationMs)} | ${timeline} |`
);
}
const spanRows = attributions.flatMap((item) =>
(item.spanSummaries ?? []).slice(0, 6).map((span) => ({ turn: item.label ?? "turn", ...span }))
);
if (spanRows.length > 0) {
lines.push("");
lines.push(" | turn | span | count | errors | clipped | max |");
lines.push(" |---|---|---:|---:|---:|---:|");
for (const span of spanRows.slice(0, 12)) {
lines.push(
` | ${span.turn} | \`${span.name}\` | ${span.count} | ${span.errorCount} | ${formatMs(span.totalClippedDurationMs)} | ${formatMs(span.maxClippedDurationMs)} |`
);
}
}
return lines;
}
export function attributedSpanIntervals(events) {
const startsById = new Map();
const intervals = [];
for (const event of events ?? []) {
if (event?.type === "span.start" && isDashboardAttributedSpanName(event.name)) {
const key = spanKey(event);
if (key) {
startsById.set(key, event);
}
continue;
}
if ((event?.type === "span.end" || event?.type === "span.error") && isDashboardAttributedSpanName(event.name)) {
const terminal = spanIntervalFromTerminal(event, startsById.get(spanKey(event)));
if (terminal) {
intervals.push(terminal);
}
}
}
return intervals;
}
function spanIntervalFromTerminal(event, startEvent) {
const endEpochMs = eventEpochMs(event);
const durationMs = numberOrNull(event.durationMs) ?? durationBetween(eventEpochMs(startEvent), endEpochMs);
const startEpochMs = eventEpochMs(startEvent) ??
(endEpochMs !== null && durationMs !== null ? endEpochMs - durationMs : null);
if (startEpochMs === null || endEpochMs === null || endEpochMs < startEpochMs) {
return null;
}
return {
name: event.name,
type: event.type,
startEpochMs,
endEpochMs,
durationMs: round(endEpochMs - startEpochMs),
rawDurationMs: durationMs,
spanId: event.spanId ?? null,
phase: event.phase ?? startEvent?.phase ?? null,
errorName: event.errorName ?? null,
errorMessage: event.errorMessage ?? null
};
}
function clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs) {
const startEpochMs = Math.max(span.startEpochMs, windowStartEpochMs);
const endEpochMs = Math.min(span.endEpochMs, windowEndEpochMs);
if (endEpochMs <= startEpochMs) {
return null;
}
return {
...span,
clippedStartEpochMs: startEpochMs,
clippedEndEpochMs: endEpochMs,
clippedDurationMs: round(endEpochMs - startEpochMs)
};
}
function summarizeAttributedSpans(intervals) {
const byName = new Map();
for (const interval of intervals) {
const current = byName.get(interval.name) ?? {
name: interval.name,
count: 0,
errorCount: 0,
totalClippedDurationMs: 0,
maxClippedDurationMs: null,
totalRawDurationMs: 0,
maxRawDurationMs: null
};
current.count += 1;
if (interval.type === "span.error") {
current.errorCount += 1;
}
current.totalClippedDurationMs = round(current.totalClippedDurationMs + interval.clippedDurationMs);
current.maxClippedDurationMs = maxNullable(current.maxClippedDurationMs, interval.clippedDurationMs);
if (typeof interval.rawDurationMs === "number") {
current.totalRawDurationMs = round(current.totalRawDurationMs + interval.rawDurationMs);
current.maxRawDurationMs = maxNullable(current.maxRawDurationMs, interval.rawDurationMs);
}
byName.set(interval.name, current);
}
return [...byName.values()].toSorted((left, right) =>
(right.totalClippedDurationMs - left.totalClippedDurationMs) ||
left.name.localeCompare(right.name)
);
}
function summarizeProviderEvents(events, providerBoundaryEpochMs, activeFinishedAtEpochMs, attribution) {
const providerEvents = (events ?? [])
.filter((event) => event?.type === "provider.request" || event?.name === "provider.request")
.map((event) => {
const startEpochMs = numberOrNull(event.receivedAtEpochMs) ?? eventEpochMs(event);
const durationMs = numberOrNull(event.durationMs);
const endEpochMs = numberOrNull(event.respondedAtEpochMs) ??
(startEpochMs !== null && durationMs !== null ? startEpochMs + durationMs : null);
return { event, startEpochMs, endEpochMs, durationMs: durationMs ?? durationBetween(startEpochMs, endEpochMs) };
})
.filter((event) =>
event.startEpochMs !== null &&
(providerBoundaryEpochMs === null || event.startEpochMs >= providerBoundaryEpochMs) &&
(activeFinishedAtEpochMs === null || event.startEpochMs <= activeFinishedAtEpochMs)
);
const durations = providerEvents.map((event) => event.durationMs).filter(isNumber);
return {
requestCount: providerEvents.length,
timelineTotalDurationMs: round(durations.reduce((sum, value) => sum + value, 0)),
timelineMaxDurationMs: durations.length > 0 ? Math.max(...durations) : null,
totalDurationMs: numberOrNull(attribution?.providerFinalMs) ??
round(durations.reduce((sum, value) => sum + value, 0)),
firstByteLatencyMs: numberOrNull(attribution?.firstByteLatencyMs),
firstChunkLatencyMs: numberOrNull(attribution?.firstChunkLatencyMs)
};
}
function summarizeLabeledAttribution(entries, label) {
const values = entries.filter((entry) => entry.label === label);
return {
count: values.length,
preProviderMs: summarizeValues(values.map((entry) => entry.window?.durationMs)),
knownAttributedMs: summarizeValues(values.map((entry) => entry.knownAttributedMs)),
unattributedMs: summarizeValues(values.map((entry) => entry.unattributedMs)),
coverageRatio: summarizeValues(values.map((entry) => entry.coverageRatio))
};
}
function summarizeSpanMedians(entries) {
const byName = new Map();
for (const entry of entries) {
for (const span of entry.spanSummaries ?? []) {
const current = byName.get(span.name) ?? [];
current.push(span.totalClippedDurationMs);
byName.set(span.name, current);
}
}
return [...byName.entries()]
.map(([name, values]) => ({
name,
medianClippedDurationMs: summarizeValues(values).median,
sampleCount: values.length
}))
.toSorted((left, right) => (right.medianClippedDurationMs ?? 0) - (left.medianClippedDurationMs ?? 0));
}
function summarizeValues(values) {
const sorted = values.filter(isNumber).toSorted((left, right) => left - right);
if (sorted.length === 0) {
return { count: 0, median: null, min: null, max: null };
}
return {
count: sorted.length,
median: round(percentile(sorted, 50)),
min: round(sorted[0]),
max: round(sorted.at(-1))
};
}
function unionDuration(intervals) {
const sorted = intervals
.filter((interval) => isNumber(interval.clippedStartEpochMs) && isNumber(interval.clippedEndEpochMs))
.toSorted((left, right) => left.clippedStartEpochMs - right.clippedStartEpochMs);
let total = 0;
let currentStart = null;
let currentEnd = null;
for (const interval of sorted) {
if (currentStart === null) {
currentStart = interval.clippedStartEpochMs;
currentEnd = interval.clippedEndEpochMs;
continue;
}
if (interval.clippedStartEpochMs <= currentEnd) {
currentEnd = Math.max(currentEnd, interval.clippedEndEpochMs);
continue;
}
total += currentEnd - currentStart;
currentStart = interval.clippedStartEpochMs;
currentEnd = interval.clippedEndEpochMs;
}
if (currentStart !== null) {
total += currentEnd - currentStart;
}
return total;
}
function isDashboardAttributedSpanName(name) {
const text = String(name ?? "");
return text.startsWith("gateway.chat_send") ||
text.startsWith("auto_reply") ||
text.startsWith("reply.");
}
function attributionEvents(timelineSummary) {
return Array.isArray(timelineSummary?.turnAttributionEvents) && timelineSummary.turnAttributionEvents.length > 0
? timelineSummary.turnAttributionEvents
: (Array.isArray(timelineSummary?.events) ? timelineSummary.events : []);
}
function timelineArtifacts(timelineSummary) {
return unique([
...(timelineSummary?.artifacts ?? []),
...(timelineSummary?.timelineArtifacts ?? [])
].filter(Boolean));
}
function spanKey(event) {
return event?.spanId === undefined || event?.spanId === null || String(event.spanId).length === 0
? null
: String(event.spanId);
}
function eventEpochMs(event) {
const direct = numberOrNull(event?.timestampEpochMs ?? event?.timeEpochMs);
if (direct !== null) {
return direct;
}
const parsed = Date.parse(event?.timestamp ?? event?.time ?? "");
return Number.isFinite(parsed) ? parsed : null;
}
function durationBetween(startEpochMs, endEpochMs) {
return isNumber(startEpochMs) && isNumber(endEpochMs) && endEpochMs >= startEpochMs
? round(endEpochMs - startEpochMs)
: null;
}
function percentile(sortedValues, percentileValue) {
if (sortedValues.length === 1) {
return sortedValues[0];
}
const position = (percentileValue / 100) * (sortedValues.length - 1);
const lower = Math.floor(position);
const upper = Math.ceil(position);
if (lower === upper) {
return sortedValues[lower];
}
const weight = position - lower;
return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
}
function maxNullable(left, right) {
if (!isNumber(right)) {
return left;
}
return isNumber(left) ? Math.max(left, right) : right;
}
function numberOrNull(value) {
if (value === null || value === undefined || value === "") {
return null;
}
const number = Number(value);
return Number.isFinite(number) ? round(number) : null;
}
function isoOrNull(epochMs) {
return isNumber(epochMs) ? new Date(epochMs).toISOString() : null;
}
function formatMs(value) {
return isNumber(value) ? `${value} ms` : "unknown";
}
function unique(values) {
return [...new Set(values)];
}
function isNumber(value) {
return typeof value === "number" && Number.isFinite(value);
}
function round(value) {
return Math.round(value * 100) / 100;
}

View File

@ -6,6 +6,7 @@ export const TIMELINE_COLLECTOR_SCHEMA = "kova.timelineCollector.v1";
export const KEY_OPENCLAW_SPANS = [
"gateway.startup",
"gateway.ready",
"gateway.chat_send",
"config.normalize",
"plugins.metadata.scan",
"runtimeDeps.stage",
@ -19,7 +20,9 @@ export const KEY_OPENCLAW_SPANS = [
"channel.plugin.load",
"agent.prepare",
"agent.turn",
"agent.cleanup"
"agent.cleanup",
"auto_reply",
"reply"
];
export async function collectTimelineMetrics(artifactDir) {
@ -59,6 +62,7 @@ export async function collectTimelineMetrics(artifactDir) {
eventLoop: timeline.eventLoop,
providers: timeline.providers,
childProcesses: timeline.childProcesses,
turnAttributionEvents: timeline.turnAttributionEvents,
events: timeline.events,
artifacts: timeline.available ? [timelinePath] : [],
error: timeline.available ? null : (timeline.error ?? (timeline.missing ? "OpenClaw timeline not emitted" : null))
@ -141,6 +145,7 @@ export function summarizeTimeline(events, parseErrors = []) {
eventLoop: summarizeEventLoop(eventLoopSamples),
providers: summarizeTimedCollection(providerRequests),
childProcesses: summarizeChildProcesses(childProcesses),
turnAttributionEvents: events.filter(isTurnAttributionEvent).map(compactAttributionEvent),
events: events.slice(0, 200)
};
}
@ -184,6 +189,7 @@ function emptyTimeline(extra = {}) {
maxDurationMs: null,
slowest: null
},
turnAttributionEvents: [],
events: [],
...extra
};
@ -318,8 +324,8 @@ function summarizeOpenSpans({ starts, terminals, events }) {
function summarizeKeySpans({ spanEvents, openSpans }) {
const byName = {};
for (const name of KEY_OPENCLAW_SPANS) {
const spans = spanEvents.filter((event) => event.name === name);
const open = openSpans.filter((event) => event.name === name);
const spans = spanEvents.filter((event) => keySpanMatches(name, event.name));
const open = openSpans.filter((event) => keySpanMatches(name, event.name));
const durations = spans.map((event) => event.durationMs).filter(isNumber);
const slowest = spans
.filter((event) => typeof event.durationMs === "number")
@ -339,6 +345,19 @@ function summarizeKeySpans({ spanEvents, openSpans }) {
return byName;
}
function keySpanMatches(keyName, eventName) {
if (keyName === "gateway.chat_send") {
return eventName === keyName || eventName.startsWith("gateway.chat_send.");
}
if (keyName === "auto_reply") {
return eventName === keyName || eventName.startsWith("auto_reply.");
}
if (keyName === "reply") {
return eventName === keyName || eventName.startsWith("reply.");
}
return eventName === keyName;
}
function emptyKeySpans() {
return Object.fromEntries(KEY_OPENCLAW_SPANS.map((name) => [name, {
name,
@ -416,6 +435,56 @@ function compactTimedEvent(event) {
};
}
function isTurnAttributionEvent(event) {
if (event.type === "eventLoop.sample") {
return true;
}
if (event.type === "provider.request" || event.name === "provider.request") {
return true;
}
if (event.type !== "span.start" && event.type !== "span.end" && event.type !== "span.error") {
return false;
}
return event.name === "plugins.metadata.scan" ||
event.name === "provider.request" ||
event.name === "auto_reply" ||
event.name.startsWith("auto_reply.") ||
event.name.startsWith("gateway.chat_send") ||
event.name.startsWith("reply.");
}
function compactAttributionEvent(event) {
return {
type: event.type,
name: event.name,
timestamp: event.timestamp ?? null,
timestampEpochMs: numberOrNull(event.timestampEpochMs ?? event.timeEpochMs) ?? parsedTimestampMs(event.timestamp ?? event.time),
durationMs: event.durationMs ?? null,
spanId: event.spanId ?? null,
parentSpanId: event.parentSpanId ?? null,
phase: event.phase ?? null,
pid: event.pid ?? null,
provider: event.provider ?? event.attributes?.provider ?? null,
operation: event.operation ?? event.attributes?.operation ?? null,
pluginId: event.pluginId ?? event.attributes?.pluginId ?? null,
errorName: event.errorName ?? event.attributes?.errorName ?? null,
errorMessage: event.errorMessage ?? event.attributes?.errorMessage ?? null,
maxMs: numberOrNull(event.maxMs ?? event.eventLoopDelayMs),
p95Ms: numberOrNull(event.p95Ms),
p99Ms: numberOrNull(event.p99Ms),
receivedAtEpochMs: numberOrNull(event.receivedAtEpochMs),
respondedAtEpochMs: numberOrNull(event.respondedAtEpochMs),
status: numberOrNull(event.status),
route: event.route ?? event.path ?? null,
model: event.model ?? event.modelId ?? event.attributes?.model ?? null
};
}
function parsedTimestampMs(value) {
const parsed = Date.parse(value ?? "");
return Number.isFinite(parsed) ? parsed : null;
}
function spanIdentity(event) {
if (event.spanId !== undefined && event.spanId !== null && String(event.spanId).length > 0) {
return `id:${event.spanId}`;

View File

@ -1,4 +1,8 @@
import { buildAgentTurnBreakdown } from "./collectors/agent-turns.mjs";
import {
buildDashboardPreProviderAttribution,
summarizeDashboardPreProviderAttributions
} from "./collectors/dashboard-turn-attribution.mjs";
import { computeProviderTurnAttribution } from "./collectors/provider.mjs";
import { summarizeRuntimeDepsLogs } from "./collectors/logs.mjs";
import { buildHealthMeasurement, healthReadinessClassification } from "./health.mjs";
@ -77,6 +81,7 @@ export function evaluateRecord(record, scenario, options = {}) {
const providerTurn = collectSlowestProviderTurn(agentTurns);
const agentTurnStats = summarizeAgentTurnStats(agentTurns);
const agentTurnDiagnostics = summarizeAgentTurnDiagnostics(agentTurns);
const dashboardPreProviderAttribution = summarizeDashboardPreProviderAttributions(agentTurns);
const agentTurnMs = maxTurnDuration(agentTurns);
const agentResponseOk = agentTurns.length === 0 ? null : agentTurns.every((turn) => turn.responseOk === true);
const agentProviderSimulation = evaluateProviderSimulation({ turns: agentTurns, scenario, record, thresholds });
@ -754,6 +759,13 @@ export function evaluateRecord(record, scenario, options = {}) {
agentEventLoopSampleCount: agentTurnDiagnostics.eventLoopSampleCount,
agentSessionPollCount: agentTurnDiagnostics.sessionPollCount,
agentSessionPollErrorCount: agentTurnDiagnostics.sessionPollErrorCount,
dashboardPreProviderAttribution,
coldPreProviderAttributedMs: dashboardPreProviderAttribution.cold.knownAttributedMs.median,
warmPreProviderAttributedMs: dashboardPreProviderAttribution.warm.knownAttributedMs.median,
coldPreProviderUnattributedMs: dashboardPreProviderAttribution.cold.unattributedMs.median,
warmPreProviderUnattributedMs: dashboardPreProviderAttribution.warm.unattributedMs.median,
coldPreProviderAttributionCoverage: dashboardPreProviderAttribution.cold.coverageRatio.median,
warmPreProviderAttributionCoverage: dashboardPreProviderAttribution.warm.coverageRatio.median,
coldAgentTurnMs: coldAgentTurn?.totalTurnMs ?? null,
warmAgentTurnMs: warmAgentTurn?.totalTurnMs ?? null,
agentColdWarmDeltaMs: delta(coldAgentTurn?.totalTurnMs, warmAgentTurn?.totalTurnMs),
@ -976,6 +988,16 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
activeFinishedAtEpochMs: timingResult.finishedAtEpochMs,
gatewaySession
});
const dashboardPreProviderAttribution = gatewaySession
? buildDashboardPreProviderAttribution({
label: agentTurnLabel(phase.id, index),
phaseId: phase.id,
activeStartedAtEpochMs: timingResult.startedAtEpochMs,
activeFinishedAtEpochMs: timingResult.finishedAtEpochMs,
attribution,
timelineSummary
})
: null;
turns.push({
schemaVersion: "kova.agentTurnEvidence.v1",
index,
@ -1022,6 +1044,7 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
providerLateByMs: attribution?.providerLateByMs ?? null,
phaseBreakdown,
turnDiagnostics,
dashboardPreProviderAttribution,
metadataScanCount: turnDiagnostics.metadataScan.count,
metadataScanTotalMs: turnDiagnostics.metadataScan.totalDurationMs,
metadataScanMaxMs: turnDiagnostics.metadataScan.maxDurationMs,
@ -1092,7 +1115,9 @@ function resultForActiveTurnWindow(result, gatewaySession) {
}
function summarizeActiveTurnDiagnostics({ timelineSummary, activeStartedAtEpochMs, activeFinishedAtEpochMs, gatewaySession }) {
const events = Array.isArray(timelineSummary?.events) ? timelineSummary.events : [];
const events = Array.isArray(timelineSummary?.turnAttributionEvents) && timelineSummary.turnAttributionEvents.length > 0
? timelineSummary.turnAttributionEvents
: (Array.isArray(timelineSummary?.events) ? timelineSummary.events : []);
const windowEvents = events.filter((event) =>
eventEpochMs(event) !== null &&
eventEpochMs(event) >= activeStartedAtEpochMs &&
@ -2637,6 +2662,8 @@ function collectTimelineSummary(record) {
let openSpans = [];
let latestEventCount = -1;
let events = [];
let turnAttributionEvents = [];
const artifacts = new Set();
const keySpans = {};
const spanTotals = {};
@ -2644,6 +2671,10 @@ function collectTimelineSummary(record) {
if ((timeline.eventCount ?? 0) >= latestEventCount && Array.isArray(timeline.events)) {
latestEventCount = timeline.eventCount ?? 0;
events = timeline.events;
turnAttributionEvents = Array.isArray(timeline.turnAttributionEvents) ? timeline.turnAttributionEvents : [];
}
for (const artifact of timeline.artifacts ?? []) {
artifacts.add(artifact);
}
eventCount = Math.max(eventCount, timeline.eventCount ?? 0);
parseErrorCount = Math.max(parseErrorCount, timeline.parseErrorCount ?? 0);
@ -2684,7 +2715,10 @@ function collectTimelineSummary(record) {
repeatedSpanCount,
openSpanCount,
openSpans,
artifacts: [...artifacts],
timelineArtifacts: [...artifacts],
events,
turnAttributionEvents,
keySpans,
spanTotals,
eventLoopMaxMs,

View File

@ -21,6 +21,10 @@ export const PERFORMANCE_METRICS = [
{ id: "agentCleanupMaxMs", title: "Agent Cleanup Max", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "coldPreProviderMs", title: "Cold Pre-Provider", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "warmPreProviderMs", title: "Warm Pre-Provider", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "coldPreProviderAttributedMs", title: "Cold Pre-Provider Attributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "warmPreProviderAttributedMs", title: "Warm Pre-Provider Attributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "coldPreProviderUnattributedMs", title: "Cold Pre-Provider Unattributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "warmPreProviderUnattributedMs", title: "Warm Pre-Provider Unattributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "agentMetadataScanCount", title: "Agent Metadata Scans", unit: "count", regressionKey: "agentLatencyRegressionPercent" },
{ id: "agentMetadataScanTotalMs", title: "Agent Metadata Scan Total", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
{ id: "agentEventLoopMaxMs", title: "Agent Event Loop Max", unit: "ms", regressionKey: "eventLoopRegressionPercent" },

View File

@ -414,6 +414,12 @@ function metricDeltas(baseline, current) {
"coldPreProviderMs",
"warmPreProviderMs",
"agentColdWarmPreProviderDeltaMs",
"coldPreProviderAttributedMs",
"warmPreProviderAttributedMs",
"coldPreProviderUnattributedMs",
"warmPreProviderUnattributedMs",
"coldPreProviderAttributionCoverage",
"warmPreProviderAttributionCoverage",
"coldProviderFinalMs",
"warmProviderFinalMs",
"agentMetadataScanCount",

View File

@ -1,4 +1,5 @@
import { summarizeAgentTurnBreakdownForMarkdown } from "../collectors/agent-turns.mjs";
import { dashboardPreProviderMarkdownRows } from "../collectors/dashboard-turn-attribution.mjs";
import { healthTotalFailures } from "../health.mjs";
export function summarizeRecords(records) {
@ -182,6 +183,9 @@ export function renderMarkdownReport(report) {
if (record.measurements.agentTurnCount > 0) {
lines.push(`- Agent cold/warm: cold ${record.measurements.coldAgentTurnMs ?? "unknown"} ms; warm ${record.measurements.warmAgentTurnMs ?? "unknown"} ms; delta ${record.measurements.agentColdWarmDeltaMs ?? "unknown"} ms`);
lines.push(`- Agent pre-provider: cold ${record.measurements.coldPreProviderMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderMs ?? "unknown"} ms; delta ${record.measurements.agentColdWarmPreProviderDeltaMs ?? "unknown"} ms`);
if (record.measurements.dashboardPreProviderAttribution?.count > 0) {
lines.push(`- Dashboard pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
}
lines.push(`- Agent provider final: cold ${record.measurements.coldProviderFinalMs ?? "unknown"} ms; warm ${record.measurements.warmProviderFinalMs ?? "unknown"} ms`);
lines.push(`- Agent turn stats: count ${record.measurements.agentTurnCount}; p95 ${record.measurements.agentTurnP95Ms ?? "unknown"} ms; max ${record.measurements.agentTurnMaxMs ?? "unknown"} ms; pre-provider p95 ${record.measurements.agentPreProviderP95Ms ?? "unknown"} ms`);
lines.push(`- Agent active-turn diagnostics: metadata scans ${record.measurements.agentMetadataScanCount ?? "unknown"} (${record.measurements.agentMetadataScanTotalMs ?? "unknown"} ms total); event-loop max ${record.measurements.agentEventLoopMaxMs ?? "unknown"} ms; session polls ${record.measurements.agentSessionPollCount ?? "unknown"} (${record.measurements.agentSessionPollErrorCount ?? "unknown"} errors)`);
@ -239,6 +243,7 @@ export function renderMarkdownReport(report) {
lines.push(` - breakdown: ${breakdown}`);
}
}
lines.push(...dashboardPreProviderMarkdownRows(record.measurements.agentTurns));
}
lines.push(`- Profiling: ${record.profiling?.enabled ? "enabled" : "off"} (${record.profiling?.interpretation ?? "unknown"})`);
lines.push(`- V8 reports / heap snapshots: ${record.measurements.v8ReportCount ?? "unknown"} / ${record.measurements.heapSnapshotCount ?? "unknown"}`);
@ -730,6 +735,13 @@ function summarizeMeasurements(measurements) {
agentEventLoopSampleCount: measurements.agentEventLoopSampleCount ?? null,
agentSessionPollCount: measurements.agentSessionPollCount ?? null,
agentSessionPollErrorCount: measurements.agentSessionPollErrorCount ?? null,
dashboardPreProviderAttribution: measurements.dashboardPreProviderAttribution ?? null,
coldPreProviderAttributedMs: measurements.coldPreProviderAttributedMs ?? null,
warmPreProviderAttributedMs: measurements.warmPreProviderAttributedMs ?? null,
coldPreProviderUnattributedMs: measurements.coldPreProviderUnattributedMs ?? null,
warmPreProviderUnattributedMs: measurements.warmPreProviderUnattributedMs ?? null,
coldPreProviderAttributionCoverage: measurements.coldPreProviderAttributionCoverage ?? null,
warmPreProviderAttributionCoverage: measurements.warmPreProviderAttributionCoverage ?? null,
coldAgentTurnMs: measurements.coldAgentTurnMs ?? null,
warmAgentTurnMs: measurements.warmAgentTurnMs ?? null,
agentColdWarmDeltaMs: measurements.agentColdWarmDeltaMs ?? null,
@ -1149,6 +1161,8 @@ function compactPerformanceMetrics(metrics = {}) {
"warmAgentTurnMs",
"agentColdWarmDeltaMs",
"coldPreProviderMs",
"coldPreProviderAttributedMs",
"coldPreProviderUnattributedMs",
"runtimeDepsStagingMs"
];
const byId = new Map(Object.entries(metrics).map(([id, metric]) => [id, { id, ...metric }]));
@ -1186,6 +1200,9 @@ function pushMeasurementBrief(lines, measurements, { compact }) {
lines.push(`- health: startup p95 ${valueMs(measurements.health?.startupSamples?.p95Ms)}; post-ready p95 ${valueMs(measurements.health?.postReadySamples?.p95Ms)}; failures ${totalHealthFailures ?? "unknown"}; final failures ${measurements.health?.final?.failureCount ?? "unknown"}${healthSlowestText(measurements)}`);
lines.push(`- resources: peak RSS ${valueMb(measurements.peakRssMb)}; max CPU ${valuePercent(measurements.cpuPercentMax)}; samples ${measurements.resourceSampleCount ?? "unknown"}; roles ${rolePeakText(measurements)}`);
lines.push(`- agent: turn ${valueMs(measurements.agentTurnMs, "not-run")}; cold/warm ${valueMs(measurements.coldAgentTurnMs)}/${valueMs(measurements.warmAgentTurnMs)}; cold-warm delta ${valueMs(measurements.agentColdWarmDeltaMs)}; pre-provider ${valueMs(measurements.agentPreProviderMs)}; provider ${valueMs(measurements.agentProviderFinalMs)}; metadata scans ${measurements.agentMetadataScanCount ?? "unknown"} (${valueMs(measurements.agentMetadataScanTotalMs)}); event-loop ${valueMs(measurements.agentEventLoopMaxMs)}; polls ${measurements.agentSessionPollCount ?? "unknown"}; cleanup ${valueMs(measurements.agentCleanupMaxMs)}; diagnosis ${measurements.agentLatencyDiagnosis?.kind ?? "unknown"}; leaks ${measurements.agentProcessLeakCount ?? "unknown"}`);
if (measurements.dashboardPreProviderAttribution?.count > 0) {
lines.push(`- dashboard attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
}
lines.push(`- plugins/runtime: missing deps ${measurements.missingDependencyErrors ?? "unknown"}; plugin failures ${measurements.pluginLoadFailures ?? "unknown"}; runtime deps ${valueMs(measurements.runtimeDepsStagingMs)}${runtimeDepsPluginText(measurements)}; warm restages ${measurements.warmRuntimeDepsRestageCount ?? "unknown"}; warm reuse ${measurements.runtimeDepsWarmReuseOk ?? "unknown"}`);
if (!compact || hasDiagnosticSignal(measurements)) {

View File

@ -31,6 +31,10 @@ import {
buildAgentTurnBreakdown,
summarizeAgentTurnBreakdownForMarkdown
} from "./collectors/agent-turns.mjs";
import {
attributedSpanIntervals,
buildDashboardPreProviderAttribution
} from "./collectors/dashboard-turn-attribution.mjs";
import {
computeProviderTurnAttribution,
parseProviderRequestLog,
@ -373,6 +377,7 @@ export async function runSelfCheck(flags = {}) {
checks.push(await providerEvidenceParserCheck());
checks.push(agentTurnBreakdownCheck());
checks.push(gatewaySessionTurnEvaluationCheck());
checks.push(dashboardPreProviderAttributionCheck());
checks.push(await mockProviderBehaviorCheck(tmp));
checks.push(providerFailureEvaluationCheck());
checks.push(agentColdWarmEvaluationCheck());
@ -2305,6 +2310,248 @@ function gatewaySessionTurnEvaluationCheck() {
}
}
function dashboardPreProviderAttributionCheck() {
try {
const base = 1777536000000;
const timelineText = [
timelineEvent({ type: "span.start", name: "gateway.chat_send.load_session", timestamp: base + 1010, spanId: "cold-load" }),
timelineEvent({ type: "span.end", name: "gateway.chat_send.load_session", timestamp: base + 1070, spanId: "cold-load", durationMs: 60 }),
timelineEvent({ type: "span.start", name: "auto_reply.finalize_context", timestamp: base + 1060, spanId: "cold-finalize" }),
timelineEvent({ type: "span.end", name: "auto_reply.finalize_context", timestamp: base + 1160, spanId: "cold-finalize", durationMs: 100 }),
timelineEvent({ type: "span.start", name: "reply.ensure_workspace", timestamp: base + 1180, spanId: "cold-workspace" }),
timelineEvent({ type: "span.error", name: "reply.ensure_workspace", timestamp: base + 1230, spanId: "cold-workspace", durationMs: 50, errorName: "SyntheticError" }),
timelineEvent({ type: "span.end", name: "plugins.metadata.scan", timestamp: base + 1150, spanId: "cold-scan", durationMs: 33 }),
timelineEvent({ type: "provider.request", name: "provider.request", timestamp: base + 1200, receivedAtEpochMs: base + 1200, respondedAtEpochMs: base + 1800, durationMs: 600 }),
timelineEvent({ type: "eventLoop.sample", name: "eventLoop.sample", timestamp: base + 1250, maxMs: 9 }),
timelineEvent({ type: "span.start", name: "gateway.chat_send.dispatch_inbound", timestamp: base + 11025, spanId: "warm-dispatch" }),
timelineEvent({ type: "span.end", name: "gateway.chat_send.dispatch_inbound", timestamp: base + 11125, spanId: "warm-dispatch", durationMs: 100 }),
timelineEvent({ type: "span.start", name: "reply.load_runtime_plugins", timestamp: base + 11120, spanId: "warm-plugins" }),
timelineEvent({ type: "span.end", name: "reply.load_runtime_plugins", timestamp: base + 11220, spanId: "warm-plugins", durationMs: 100 }),
timelineEvent({ type: "span.end", name: "plugins.metadata.scan", timestamp: base + 11100, spanId: "warm-scan", durationMs: 11 }),
timelineEvent({ type: "provider.request", name: "provider.request", timestamp: base + 11250, receivedAtEpochMs: base + 11250, respondedAtEpochMs: base + 11600, durationMs: 350 }),
timelineEvent({ type: "eventLoop.sample", name: "eventLoop.sample", timestamp: base + 11200, maxMs: 7 })
].join("\n");
const parsed = parseTimelineText(timelineText);
assertEqual(parsed.turnAttributionEvents.length, 16, "turn attribution events retained");
const parsedIntervals = attributedSpanIntervals(parsed.turnAttributionEvents);
assertEqual(parsedIntervals.length, 5, "span parser includes error terminal");
assertEqual(parsedIntervals.some((span) => span.type === "span.error" && span.name === "reply.ensure_workspace"), true, "span error included");
const coldAttribution = buildDashboardPreProviderAttribution({
label: "cold",
phaseId: "cold-dashboard-session-turn",
activeStartedAtEpochMs: base + 1000,
activeFinishedAtEpochMs: base + 2500,
attribution: {
firstProviderRequestAtEpochMs: base + 1200,
preProviderMs: 200,
providerFinalMs: 600,
firstByteLatencyMs: 25,
firstChunkLatencyMs: 30
},
timelineSummary: {
available: true,
turnAttributionEvents: parsed.turnAttributionEvents,
artifacts: ["/tmp/kova/openclaw/timeline.jsonl"]
}
});
assertEqual(coldAttribution.available, true, "cold attribution available");
assertEqual(coldAttribution.knownAttributedMs, 170, "overlap-safe cold known attribution");
assertEqual(coldAttribution.unattributedMs, 30, "cold unattributed remainder");
assertEqual(coldAttribution.spanSummaries.find((span) => span.name === "reply.ensure_workspace")?.errorCount, 1, "error span summary");
assertEqual(coldAttribution.provider.totalDurationMs, 600, "provider duration stays separate");
assertEqual(coldAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "timeline artifact path");
const missingAttribution = buildDashboardPreProviderAttribution({
label: "cold",
phaseId: "cold-dashboard-session-turn",
activeStartedAtEpochMs: base + 1000,
activeFinishedAtEpochMs: base + 2500,
attribution: { firstProviderRequestAtEpochMs: base + 1200, preProviderMs: 200 },
timelineSummary: { available: false, artifacts: [] }
});
assertEqual(missingAttribution.available, false, "missing timeline unavailable");
assertEqual(missingAttribution.unattributedMs, 200, "missing timeline preserves full remainder");
const record = syntheticDashboardSessionRecord({ base, timeline: parsed });
evaluateRecord(record, {
id: "dashboard-session-send-turn",
agent: { expectedText: "KOVA_AGENT_OK" },
thresholds: { agentTurnMs: 2000, coldAgentTurnMs: 2000, warmAgentTurnMs: 1000 }
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
assertEqual(record.measurements.coldPreProviderAttributedMs, 170, "record cold attributed metric");
assertEqual(record.measurements.warmPreProviderAttributedMs, 195, "record warm attributed metric");
assertEqual(record.measurements.warmPreProviderUnattributedMs, 55, "record warm unattributed metric");
assertEqual(record.measurements.dashboardPreProviderAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "record timeline artifact");
const rendered = renderMarkdownReport({
generatedAt: "2026-05-01T00:00:00.000Z",
runId: "self-check-dashboard-pre-provider",
mode: "self-check",
target: "runtime:stable",
platform: { os: "test", release: "test", arch: "test", node: "test" },
records: [record],
summary: { statuses: { PASS: 1 } }
});
assertEqual(rendered.includes("Dashboard pre-provider attribution:"), true, "markdown includes dashboard attribution table");
assertEqual(rendered.includes("`reply.ensure_workspace`"), true, "markdown includes span table");
return {
id: "dashboard-pre-provider-attribution",
status: "PASS",
command: "evaluate synthetic dashboard pre-provider timeline attribution",
durationMs: 0
};
} catch (error) {
return {
id: "dashboard-pre-provider-attribution",
status: "FAIL",
command: "evaluate synthetic dashboard pre-provider timeline attribution",
durationMs: 0,
message: error.message
};
}
}
function timelineEvent(event) {
const timestamp = typeof event.timestamp === "number" ? new Date(event.timestamp).toISOString() : event.timestamp;
return JSON.stringify({
schemaVersion: "openclaw.diagnostics.v1",
...event,
timestamp
});
}
function syntheticDashboardSessionRecord({ base, timeline }) {
const coldPayload = {
ok: true,
surface: "dashboard-session-send-turn",
method: "sessions.send",
createSession: true,
minAssistantCount: 1,
sessionKey: "kova-dashboard-session-send",
runId: "cold-run",
activeStartedAtEpochMs: base + 1000,
activeFinishedAtEpochMs: base + 2500,
activeTurnMs: 1500,
sendStartedAtEpochMs: base + 1000,
sendFinishedAtEpochMs: base + 1040,
sendDurationMs: 40,
assistantFirstSeenAtEpochMs: base + 2200,
assistantMatchedAtEpochMs: base + 2500,
timeToFirstAssistantMs: 1200,
timeToMatchedAssistantMs: 1500,
historyPollCount: 3,
historyErrorCount: 0,
assistantMessageCount: 1,
finalAssistantVisibleText: "KOVA_AGENT_OK",
expectedTextPresent: true
};
const warmPayload = {
...coldPayload,
createSession: false,
minAssistantCount: 2,
runId: "warm-run",
activeStartedAtEpochMs: base + 11000,
activeFinishedAtEpochMs: base + 11800,
activeTurnMs: 800,
sendStartedAtEpochMs: base + 11000,
sendFinishedAtEpochMs: base + 11050,
sendDurationMs: 50,
assistantFirstSeenAtEpochMs: base + 11600,
assistantMatchedAtEpochMs: base + 11800,
timeToFirstAssistantMs: 600,
timeToMatchedAssistantMs: 800,
historyPollCount: 2,
assistantMessageCount: 2
};
return {
scenario: "dashboard-session-send-turn",
surface: "dashboard-session-send-turn",
title: "Gateway session cold/warm",
status: "PASS",
cleanup: "done",
auth: { mode: "mock" },
phases: [
syntheticDashboardTurnPhase({
id: "cold-dashboard-session-turn",
command: "node support/run-dashboard-session-send-turn.mjs --create-session true",
startedAtEpochMs: base,
finishedAtEpochMs: base + 5000,
payload: coldPayload
}),
syntheticDashboardTurnPhase({
id: "warm-dashboard-session-turn",
command: "node support/run-dashboard-session-send-turn.mjs --create-session false",
startedAtEpochMs: base + 10000,
finishedAtEpochMs: base + 14000,
payload: warmPayload
})
],
providerEvidence: {
available: true,
requestCount: 2,
requests: [
{
requestId: "cold-provider",
receivedAt: new Date(base + 1200).toISOString(),
receivedAtEpochMs: base + 1200,
respondedAt: new Date(base + 1800).toISOString(),
respondedAtEpochMs: base + 1800,
firstByteLatencyMs: 25,
firstChunkLatencyMs: 30,
route: "/v1/responses",
model: "gpt-5.5",
status: 200
},
{
requestId: "warm-provider",
receivedAt: new Date(base + 11250).toISOString(),
receivedAtEpochMs: base + 11250,
respondedAt: new Date(base + 11600).toISOString(),
respondedAtEpochMs: base + 11600,
firstByteLatencyMs: 20,
firstChunkLatencyMs: 22,
route: "/v1/responses",
model: "gpt-5.5",
status: 200
}
]
},
finalMetrics: {
service: { gatewayState: "running" },
logs: zeroLogMetrics(),
timeline: {
...timeline,
artifacts: ["/tmp/kova/openclaw/timeline.jsonl"]
}
}
};
}
function syntheticDashboardTurnPhase({ id, command, startedAtEpochMs, finishedAtEpochMs, payload }) {
return {
id,
title: id,
intent: "Synthetic dashboard session turn",
commands: [command],
evidence: [],
results: [{
command,
status: 0,
timedOut: false,
startedAt: new Date(startedAtEpochMs).toISOString(),
startedAtEpochMs,
finishedAt: new Date(finishedAtEpochMs).toISOString(),
finishedAtEpochMs,
durationMs: finishedAtEpochMs - startedAtEpochMs,
stdout: JSON.stringify(payload),
stderr: ""
}],
metrics: { logs: zeroLogMetrics(), health: { ok: true } }
};
}
function syntheticTurn({
startedAtEpochMs,
firstProviderRequestAtEpochMs,