feat: attribute dashboard pre-provider spans
This commit is contained in:
parent
b67f5457d9
commit
2b5bddff4a
@ -275,6 +275,51 @@ Gateway/session turn entries include:
|
||||
}
|
||||
```
|
||||
|
||||
Dashboard session turns also include pre-provider attribution when an OpenClaw
|
||||
diagnostics timeline is available. Kova clips `gateway.chat_send*`,
|
||||
`auto_reply*`, and `reply.*` spans to the active `sessions.send` pre-provider
|
||||
window and reports the unioned known time so overlapping spans are not counted
|
||||
twice. Provider work remains separate.
|
||||
|
||||
```json
|
||||
{
|
||||
"dashboardPreProviderAttribution": {
|
||||
"schemaVersion": "kova.dashboardPreProviderAttribution.v1",
|
||||
"available": true,
|
||||
"label": "cold",
|
||||
"timelineArtifacts": ["/tmp/kova/openclaw/timeline.jsonl"],
|
||||
"window": {
|
||||
"startEpochMs": 1777536000000,
|
||||
"endEpochMs": 1777536000200,
|
||||
"durationMs": 200
|
||||
},
|
||||
"provider": {
|
||||
"totalDurationMs": 600,
|
||||
"firstByteLatencyMs": 25,
|
||||
"firstChunkLatencyMs": 30
|
||||
},
|
||||
"knownAttributedMs": 170,
|
||||
"unattributedMs": 30,
|
||||
"coverageRatio": 0.85,
|
||||
"spanSummaries": [
|
||||
{
|
||||
"name": "auto_reply.finalize_context",
|
||||
"count": 1,
|
||||
"errorCount": 0,
|
||||
"totalClippedDurationMs": 100,
|
||||
"maxClippedDurationMs": 100
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Repeat summaries expose machine-readable medians at
|
||||
`records[*].measurements.dashboardPreProviderAttribution` plus flat comparison
|
||||
metrics such as `coldPreProviderAttributedMs`,
|
||||
`coldPreProviderUnattributedMs`, `warmPreProviderAttributedMs`, and
|
||||
`warmPreProviderUnattributedMs`.
|
||||
|
||||
Aggregate fields are also exposed on `measurements` for comparison and
|
||||
performance summaries:
|
||||
|
||||
|
||||
@ -29,7 +29,10 @@
|
||||
"browserTabCountMin",
|
||||
"browserTabsMs",
|
||||
"coldAgentTurnMs",
|
||||
"coldPreProviderAttributedMs",
|
||||
"coldPreProviderAttributionCoverage",
|
||||
"coldPreProviderMs",
|
||||
"coldPreProviderUnattributedMs",
|
||||
"coldReadyMs",
|
||||
"coldRuntimeDepsStagingMs",
|
||||
"coldWarmDeltaMs",
|
||||
@ -105,7 +108,10 @@
|
||||
"tuiSmokeMs",
|
||||
"upgradeMs",
|
||||
"warmAgentTurnMs",
|
||||
"warmPreProviderAttributedMs",
|
||||
"warmPreProviderAttributionCoverage",
|
||||
"warmPreProviderMs",
|
||||
"warmPreProviderUnattributedMs",
|
||||
"warmReadyMs",
|
||||
"warmRuntimeDepsRestageCount",
|
||||
"warmRuntimeDepsStagingMs",
|
||||
|
||||
419
src/collectors/dashboard-turn-attribution.mjs
Normal file
419
src/collectors/dashboard-turn-attribution.mjs
Normal file
@ -0,0 +1,419 @@
|
||||
export const DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA = "kova.dashboardPreProviderAttribution.v1";
|
||||
export const DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA = "kova.dashboardPreProviderAttributionSummary.v1";
|
||||
|
||||
export function buildDashboardPreProviderAttribution({
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary
|
||||
}) {
|
||||
const artifacts = timelineArtifacts(timelineSummary);
|
||||
const events = attributionEvents(timelineSummary);
|
||||
const providerBoundaryEpochMs = numberOrNull(attribution?.firstProviderRequestAtEpochMs);
|
||||
const windowStartEpochMs = numberOrNull(activeStartedAtEpochMs ?? attribution?.commandStartedAtEpochMs);
|
||||
const activeEndEpochMs = numberOrNull(activeFinishedAtEpochMs ?? attribution?.commandFinishedAtEpochMs);
|
||||
const windowEndEpochMs = providerBoundaryEpochMs;
|
||||
const preProviderMs = numberOrNull(attribution?.preProviderMs) ??
|
||||
durationBetween(windowStartEpochMs, windowEndEpochMs);
|
||||
const base = {
|
||||
schemaVersion: DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA,
|
||||
available: false,
|
||||
label: label ?? null,
|
||||
phaseId: phaseId ?? null,
|
||||
timelineAvailable: timelineSummary?.available === true,
|
||||
timelineArtifacts: artifacts,
|
||||
eventCount: events.length,
|
||||
window: {
|
||||
startEpochMs: windowStartEpochMs,
|
||||
startAt: isoOrNull(windowStartEpochMs),
|
||||
endEpochMs: windowEndEpochMs,
|
||||
endAt: isoOrNull(windowEndEpochMs),
|
||||
durationMs: preProviderMs
|
||||
},
|
||||
activeWindow: {
|
||||
startEpochMs: windowStartEpochMs,
|
||||
startAt: isoOrNull(windowStartEpochMs),
|
||||
endEpochMs: activeEndEpochMs,
|
||||
endAt: isoOrNull(activeEndEpochMs),
|
||||
durationMs: durationBetween(windowStartEpochMs, activeEndEpochMs)
|
||||
},
|
||||
providerBoundary: {
|
||||
firstRequestAtEpochMs: providerBoundaryEpochMs,
|
||||
firstRequestAt: isoOrNull(providerBoundaryEpochMs),
|
||||
source: providerBoundaryEpochMs === null ? null : "provider-evidence"
|
||||
},
|
||||
provider: summarizeProviderEvents(events, providerBoundaryEpochMs, activeEndEpochMs, attribution),
|
||||
spanSummaries: [],
|
||||
knownAttributedMs: null,
|
||||
unattributedMs: preProviderMs,
|
||||
coverageRatio: null,
|
||||
error: null
|
||||
};
|
||||
|
||||
if (timelineSummary?.available !== true) {
|
||||
return {
|
||||
...base,
|
||||
error: "OpenClaw diagnostics timeline unavailable"
|
||||
};
|
||||
}
|
||||
if (events.length === 0) {
|
||||
return {
|
||||
...base,
|
||||
error: "timeline contains no dashboard turn attribution events"
|
||||
};
|
||||
}
|
||||
if (windowStartEpochMs === null || windowEndEpochMs === null || preProviderMs === null || windowEndEpochMs < windowStartEpochMs) {
|
||||
return {
|
||||
...base,
|
||||
error: "pre-provider window boundary unavailable"
|
||||
};
|
||||
}
|
||||
|
||||
const intervals = attributedSpanIntervals(events)
|
||||
.map((span) => clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs))
|
||||
.filter(Boolean);
|
||||
const spanSummaries = summarizeAttributedSpans(intervals);
|
||||
const knownAttributedMs = round(unionDuration(intervals));
|
||||
const unattributedMs = round(Math.max(0, preProviderMs - knownAttributedMs));
|
||||
|
||||
return {
|
||||
...base,
|
||||
available: true,
|
||||
spanSummaries,
|
||||
knownAttributedMs,
|
||||
unattributedMs,
|
||||
coverageRatio: preProviderMs > 0 ? round(knownAttributedMs / preProviderMs) : null,
|
||||
error: null
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizeDashboardPreProviderAttributions(turns) {
|
||||
const entries = (turns ?? [])
|
||||
.map((turn) => turn.dashboardPreProviderAttribution)
|
||||
.filter(Boolean);
|
||||
const cold = summarizeLabeledAttribution(entries, "cold");
|
||||
const warm = summarizeLabeledAttribution(entries, "warm");
|
||||
return {
|
||||
schemaVersion: DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA,
|
||||
available: entries.some((entry) => entry.available === true),
|
||||
count: entries.length,
|
||||
cold,
|
||||
warm,
|
||||
spanMedians: summarizeSpanMedians(entries),
|
||||
timelineArtifacts: unique(entries.flatMap((entry) => entry.timelineArtifacts ?? []))
|
||||
};
|
||||
}
|
||||
|
||||
export function dashboardPreProviderMarkdownRows(turns) {
|
||||
const attributions = (turns ?? [])
|
||||
.map((turn) => turn.dashboardPreProviderAttribution)
|
||||
.filter(Boolean);
|
||||
if (attributions.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const lines = [
|
||||
"- Dashboard pre-provider attribution:",
|
||||
"",
|
||||
" | turn | pre-provider | known | unattributed | provider | timeline |",
|
||||
" |---|---:|---:|---:|---:|---|"
|
||||
];
|
||||
for (const item of attributions) {
|
||||
const timeline = item.timelineArtifacts?.[0] ?? (item.timelineAvailable ? "available" : "missing");
|
||||
lines.push(
|
||||
` | ${item.label ?? "turn"} | ${formatMs(item.window?.durationMs)} | ${formatMs(item.knownAttributedMs)} | ${formatMs(item.unattributedMs)} | ${formatMs(item.provider?.totalDurationMs)} | ${timeline} |`
|
||||
);
|
||||
}
|
||||
|
||||
const spanRows = attributions.flatMap((item) =>
|
||||
(item.spanSummaries ?? []).slice(0, 6).map((span) => ({ turn: item.label ?? "turn", ...span }))
|
||||
);
|
||||
if (spanRows.length > 0) {
|
||||
lines.push("");
|
||||
lines.push(" | turn | span | count | errors | clipped | max |");
|
||||
lines.push(" |---|---|---:|---:|---:|---:|");
|
||||
for (const span of spanRows.slice(0, 12)) {
|
||||
lines.push(
|
||||
` | ${span.turn} | \`${span.name}\` | ${span.count} | ${span.errorCount} | ${formatMs(span.totalClippedDurationMs)} | ${formatMs(span.maxClippedDurationMs)} |`
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function attributedSpanIntervals(events) {
|
||||
const startsById = new Map();
|
||||
const intervals = [];
|
||||
|
||||
for (const event of events ?? []) {
|
||||
if (event?.type === "span.start" && isDashboardAttributedSpanName(event.name)) {
|
||||
const key = spanKey(event);
|
||||
if (key) {
|
||||
startsById.set(key, event);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((event?.type === "span.end" || event?.type === "span.error") && isDashboardAttributedSpanName(event.name)) {
|
||||
const terminal = spanIntervalFromTerminal(event, startsById.get(spanKey(event)));
|
||||
if (terminal) {
|
||||
intervals.push(terminal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return intervals;
|
||||
}
|
||||
|
||||
function spanIntervalFromTerminal(event, startEvent) {
|
||||
const endEpochMs = eventEpochMs(event);
|
||||
const durationMs = numberOrNull(event.durationMs) ?? durationBetween(eventEpochMs(startEvent), endEpochMs);
|
||||
const startEpochMs = eventEpochMs(startEvent) ??
|
||||
(endEpochMs !== null && durationMs !== null ? endEpochMs - durationMs : null);
|
||||
if (startEpochMs === null || endEpochMs === null || endEpochMs < startEpochMs) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
name: event.name,
|
||||
type: event.type,
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
durationMs: round(endEpochMs - startEpochMs),
|
||||
rawDurationMs: durationMs,
|
||||
spanId: event.spanId ?? null,
|
||||
phase: event.phase ?? startEvent?.phase ?? null,
|
||||
errorName: event.errorName ?? null,
|
||||
errorMessage: event.errorMessage ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs) {
|
||||
const startEpochMs = Math.max(span.startEpochMs, windowStartEpochMs);
|
||||
const endEpochMs = Math.min(span.endEpochMs, windowEndEpochMs);
|
||||
if (endEpochMs <= startEpochMs) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...span,
|
||||
clippedStartEpochMs: startEpochMs,
|
||||
clippedEndEpochMs: endEpochMs,
|
||||
clippedDurationMs: round(endEpochMs - startEpochMs)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeAttributedSpans(intervals) {
|
||||
const byName = new Map();
|
||||
for (const interval of intervals) {
|
||||
const current = byName.get(interval.name) ?? {
|
||||
name: interval.name,
|
||||
count: 0,
|
||||
errorCount: 0,
|
||||
totalClippedDurationMs: 0,
|
||||
maxClippedDurationMs: null,
|
||||
totalRawDurationMs: 0,
|
||||
maxRawDurationMs: null
|
||||
};
|
||||
current.count += 1;
|
||||
if (interval.type === "span.error") {
|
||||
current.errorCount += 1;
|
||||
}
|
||||
current.totalClippedDurationMs = round(current.totalClippedDurationMs + interval.clippedDurationMs);
|
||||
current.maxClippedDurationMs = maxNullable(current.maxClippedDurationMs, interval.clippedDurationMs);
|
||||
if (typeof interval.rawDurationMs === "number") {
|
||||
current.totalRawDurationMs = round(current.totalRawDurationMs + interval.rawDurationMs);
|
||||
current.maxRawDurationMs = maxNullable(current.maxRawDurationMs, interval.rawDurationMs);
|
||||
}
|
||||
byName.set(interval.name, current);
|
||||
}
|
||||
return [...byName.values()].toSorted((left, right) =>
|
||||
(right.totalClippedDurationMs - left.totalClippedDurationMs) ||
|
||||
left.name.localeCompare(right.name)
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeProviderEvents(events, providerBoundaryEpochMs, activeFinishedAtEpochMs, attribution) {
|
||||
const providerEvents = (events ?? [])
|
||||
.filter((event) => event?.type === "provider.request" || event?.name === "provider.request")
|
||||
.map((event) => {
|
||||
const startEpochMs = numberOrNull(event.receivedAtEpochMs) ?? eventEpochMs(event);
|
||||
const durationMs = numberOrNull(event.durationMs);
|
||||
const endEpochMs = numberOrNull(event.respondedAtEpochMs) ??
|
||||
(startEpochMs !== null && durationMs !== null ? startEpochMs + durationMs : null);
|
||||
return { event, startEpochMs, endEpochMs, durationMs: durationMs ?? durationBetween(startEpochMs, endEpochMs) };
|
||||
})
|
||||
.filter((event) =>
|
||||
event.startEpochMs !== null &&
|
||||
(providerBoundaryEpochMs === null || event.startEpochMs >= providerBoundaryEpochMs) &&
|
||||
(activeFinishedAtEpochMs === null || event.startEpochMs <= activeFinishedAtEpochMs)
|
||||
);
|
||||
const durations = providerEvents.map((event) => event.durationMs).filter(isNumber);
|
||||
return {
|
||||
requestCount: providerEvents.length,
|
||||
timelineTotalDurationMs: round(durations.reduce((sum, value) => sum + value, 0)),
|
||||
timelineMaxDurationMs: durations.length > 0 ? Math.max(...durations) : null,
|
||||
totalDurationMs: numberOrNull(attribution?.providerFinalMs) ??
|
||||
round(durations.reduce((sum, value) => sum + value, 0)),
|
||||
firstByteLatencyMs: numberOrNull(attribution?.firstByteLatencyMs),
|
||||
firstChunkLatencyMs: numberOrNull(attribution?.firstChunkLatencyMs)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeLabeledAttribution(entries, label) {
|
||||
const values = entries.filter((entry) => entry.label === label);
|
||||
return {
|
||||
count: values.length,
|
||||
preProviderMs: summarizeValues(values.map((entry) => entry.window?.durationMs)),
|
||||
knownAttributedMs: summarizeValues(values.map((entry) => entry.knownAttributedMs)),
|
||||
unattributedMs: summarizeValues(values.map((entry) => entry.unattributedMs)),
|
||||
coverageRatio: summarizeValues(values.map((entry) => entry.coverageRatio))
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeSpanMedians(entries) {
|
||||
const byName = new Map();
|
||||
for (const entry of entries) {
|
||||
for (const span of entry.spanSummaries ?? []) {
|
||||
const current = byName.get(span.name) ?? [];
|
||||
current.push(span.totalClippedDurationMs);
|
||||
byName.set(span.name, current);
|
||||
}
|
||||
}
|
||||
return [...byName.entries()]
|
||||
.map(([name, values]) => ({
|
||||
name,
|
||||
medianClippedDurationMs: summarizeValues(values).median,
|
||||
sampleCount: values.length
|
||||
}))
|
||||
.toSorted((left, right) => (right.medianClippedDurationMs ?? 0) - (left.medianClippedDurationMs ?? 0));
|
||||
}
|
||||
|
||||
function summarizeValues(values) {
|
||||
const sorted = values.filter(isNumber).toSorted((left, right) => left - right);
|
||||
if (sorted.length === 0) {
|
||||
return { count: 0, median: null, min: null, max: null };
|
||||
}
|
||||
return {
|
||||
count: sorted.length,
|
||||
median: round(percentile(sorted, 50)),
|
||||
min: round(sorted[0]),
|
||||
max: round(sorted.at(-1))
|
||||
};
|
||||
}
|
||||
|
||||
function unionDuration(intervals) {
|
||||
const sorted = intervals
|
||||
.filter((interval) => isNumber(interval.clippedStartEpochMs) && isNumber(interval.clippedEndEpochMs))
|
||||
.toSorted((left, right) => left.clippedStartEpochMs - right.clippedStartEpochMs);
|
||||
let total = 0;
|
||||
let currentStart = null;
|
||||
let currentEnd = null;
|
||||
for (const interval of sorted) {
|
||||
if (currentStart === null) {
|
||||
currentStart = interval.clippedStartEpochMs;
|
||||
currentEnd = interval.clippedEndEpochMs;
|
||||
continue;
|
||||
}
|
||||
if (interval.clippedStartEpochMs <= currentEnd) {
|
||||
currentEnd = Math.max(currentEnd, interval.clippedEndEpochMs);
|
||||
continue;
|
||||
}
|
||||
total += currentEnd - currentStart;
|
||||
currentStart = interval.clippedStartEpochMs;
|
||||
currentEnd = interval.clippedEndEpochMs;
|
||||
}
|
||||
if (currentStart !== null) {
|
||||
total += currentEnd - currentStart;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function isDashboardAttributedSpanName(name) {
|
||||
const text = String(name ?? "");
|
||||
return text.startsWith("gateway.chat_send") ||
|
||||
text.startsWith("auto_reply") ||
|
||||
text.startsWith("reply.");
|
||||
}
|
||||
|
||||
function attributionEvents(timelineSummary) {
|
||||
return Array.isArray(timelineSummary?.turnAttributionEvents) && timelineSummary.turnAttributionEvents.length > 0
|
||||
? timelineSummary.turnAttributionEvents
|
||||
: (Array.isArray(timelineSummary?.events) ? timelineSummary.events : []);
|
||||
}
|
||||
|
||||
function timelineArtifacts(timelineSummary) {
|
||||
return unique([
|
||||
...(timelineSummary?.artifacts ?? []),
|
||||
...(timelineSummary?.timelineArtifacts ?? [])
|
||||
].filter(Boolean));
|
||||
}
|
||||
|
||||
function spanKey(event) {
|
||||
return event?.spanId === undefined || event?.spanId === null || String(event.spanId).length === 0
|
||||
? null
|
||||
: String(event.spanId);
|
||||
}
|
||||
|
||||
function eventEpochMs(event) {
|
||||
const direct = numberOrNull(event?.timestampEpochMs ?? event?.timeEpochMs);
|
||||
if (direct !== null) {
|
||||
return direct;
|
||||
}
|
||||
const parsed = Date.parse(event?.timestamp ?? event?.time ?? "");
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function durationBetween(startEpochMs, endEpochMs) {
|
||||
return isNumber(startEpochMs) && isNumber(endEpochMs) && endEpochMs >= startEpochMs
|
||||
? round(endEpochMs - startEpochMs)
|
||||
: null;
|
||||
}
|
||||
|
||||
function percentile(sortedValues, percentileValue) {
|
||||
if (sortedValues.length === 1) {
|
||||
return sortedValues[0];
|
||||
}
|
||||
const position = (percentileValue / 100) * (sortedValues.length - 1);
|
||||
const lower = Math.floor(position);
|
||||
const upper = Math.ceil(position);
|
||||
if (lower === upper) {
|
||||
return sortedValues[lower];
|
||||
}
|
||||
const weight = position - lower;
|
||||
return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
|
||||
}
|
||||
|
||||
function maxNullable(left, right) {
|
||||
if (!isNumber(right)) {
|
||||
return left;
|
||||
}
|
||||
return isNumber(left) ? Math.max(left, right) : right;
|
||||
}
|
||||
|
||||
function numberOrNull(value) {
|
||||
if (value === null || value === undefined || value === "") {
|
||||
return null;
|
||||
}
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? round(number) : null;
|
||||
}
|
||||
|
||||
function isoOrNull(epochMs) {
|
||||
return isNumber(epochMs) ? new Date(epochMs).toISOString() : null;
|
||||
}
|
||||
|
||||
function formatMs(value) {
|
||||
return isNumber(value) ? `${value} ms` : "unknown";
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function isNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 100) / 100;
|
||||
}
|
||||
@ -6,6 +6,7 @@ export const TIMELINE_COLLECTOR_SCHEMA = "kova.timelineCollector.v1";
|
||||
export const KEY_OPENCLAW_SPANS = [
|
||||
"gateway.startup",
|
||||
"gateway.ready",
|
||||
"gateway.chat_send",
|
||||
"config.normalize",
|
||||
"plugins.metadata.scan",
|
||||
"runtimeDeps.stage",
|
||||
@ -19,7 +20,9 @@ export const KEY_OPENCLAW_SPANS = [
|
||||
"channel.plugin.load",
|
||||
"agent.prepare",
|
||||
"agent.turn",
|
||||
"agent.cleanup"
|
||||
"agent.cleanup",
|
||||
"auto_reply",
|
||||
"reply"
|
||||
];
|
||||
|
||||
export async function collectTimelineMetrics(artifactDir) {
|
||||
@ -59,6 +62,7 @@ export async function collectTimelineMetrics(artifactDir) {
|
||||
eventLoop: timeline.eventLoop,
|
||||
providers: timeline.providers,
|
||||
childProcesses: timeline.childProcesses,
|
||||
turnAttributionEvents: timeline.turnAttributionEvents,
|
||||
events: timeline.events,
|
||||
artifacts: timeline.available ? [timelinePath] : [],
|
||||
error: timeline.available ? null : (timeline.error ?? (timeline.missing ? "OpenClaw timeline not emitted" : null))
|
||||
@ -141,6 +145,7 @@ export function summarizeTimeline(events, parseErrors = []) {
|
||||
eventLoop: summarizeEventLoop(eventLoopSamples),
|
||||
providers: summarizeTimedCollection(providerRequests),
|
||||
childProcesses: summarizeChildProcesses(childProcesses),
|
||||
turnAttributionEvents: events.filter(isTurnAttributionEvent).map(compactAttributionEvent),
|
||||
events: events.slice(0, 200)
|
||||
};
|
||||
}
|
||||
@ -184,6 +189,7 @@ function emptyTimeline(extra = {}) {
|
||||
maxDurationMs: null,
|
||||
slowest: null
|
||||
},
|
||||
turnAttributionEvents: [],
|
||||
events: [],
|
||||
...extra
|
||||
};
|
||||
@ -318,8 +324,8 @@ function summarizeOpenSpans({ starts, terminals, events }) {
|
||||
function summarizeKeySpans({ spanEvents, openSpans }) {
|
||||
const byName = {};
|
||||
for (const name of KEY_OPENCLAW_SPANS) {
|
||||
const spans = spanEvents.filter((event) => event.name === name);
|
||||
const open = openSpans.filter((event) => event.name === name);
|
||||
const spans = spanEvents.filter((event) => keySpanMatches(name, event.name));
|
||||
const open = openSpans.filter((event) => keySpanMatches(name, event.name));
|
||||
const durations = spans.map((event) => event.durationMs).filter(isNumber);
|
||||
const slowest = spans
|
||||
.filter((event) => typeof event.durationMs === "number")
|
||||
@ -339,6 +345,19 @@ function summarizeKeySpans({ spanEvents, openSpans }) {
|
||||
return byName;
|
||||
}
|
||||
|
||||
function keySpanMatches(keyName, eventName) {
|
||||
if (keyName === "gateway.chat_send") {
|
||||
return eventName === keyName || eventName.startsWith("gateway.chat_send.");
|
||||
}
|
||||
if (keyName === "auto_reply") {
|
||||
return eventName === keyName || eventName.startsWith("auto_reply.");
|
||||
}
|
||||
if (keyName === "reply") {
|
||||
return eventName === keyName || eventName.startsWith("reply.");
|
||||
}
|
||||
return eventName === keyName;
|
||||
}
|
||||
|
||||
function emptyKeySpans() {
|
||||
return Object.fromEntries(KEY_OPENCLAW_SPANS.map((name) => [name, {
|
||||
name,
|
||||
@ -416,6 +435,56 @@ function compactTimedEvent(event) {
|
||||
};
|
||||
}
|
||||
|
||||
function isTurnAttributionEvent(event) {
|
||||
if (event.type === "eventLoop.sample") {
|
||||
return true;
|
||||
}
|
||||
if (event.type === "provider.request" || event.name === "provider.request") {
|
||||
return true;
|
||||
}
|
||||
if (event.type !== "span.start" && event.type !== "span.end" && event.type !== "span.error") {
|
||||
return false;
|
||||
}
|
||||
return event.name === "plugins.metadata.scan" ||
|
||||
event.name === "provider.request" ||
|
||||
event.name === "auto_reply" ||
|
||||
event.name.startsWith("auto_reply.") ||
|
||||
event.name.startsWith("gateway.chat_send") ||
|
||||
event.name.startsWith("reply.");
|
||||
}
|
||||
|
||||
function compactAttributionEvent(event) {
|
||||
return {
|
||||
type: event.type,
|
||||
name: event.name,
|
||||
timestamp: event.timestamp ?? null,
|
||||
timestampEpochMs: numberOrNull(event.timestampEpochMs ?? event.timeEpochMs) ?? parsedTimestampMs(event.timestamp ?? event.time),
|
||||
durationMs: event.durationMs ?? null,
|
||||
spanId: event.spanId ?? null,
|
||||
parentSpanId: event.parentSpanId ?? null,
|
||||
phase: event.phase ?? null,
|
||||
pid: event.pid ?? null,
|
||||
provider: event.provider ?? event.attributes?.provider ?? null,
|
||||
operation: event.operation ?? event.attributes?.operation ?? null,
|
||||
pluginId: event.pluginId ?? event.attributes?.pluginId ?? null,
|
||||
errorName: event.errorName ?? event.attributes?.errorName ?? null,
|
||||
errorMessage: event.errorMessage ?? event.attributes?.errorMessage ?? null,
|
||||
maxMs: numberOrNull(event.maxMs ?? event.eventLoopDelayMs),
|
||||
p95Ms: numberOrNull(event.p95Ms),
|
||||
p99Ms: numberOrNull(event.p99Ms),
|
||||
receivedAtEpochMs: numberOrNull(event.receivedAtEpochMs),
|
||||
respondedAtEpochMs: numberOrNull(event.respondedAtEpochMs),
|
||||
status: numberOrNull(event.status),
|
||||
route: event.route ?? event.path ?? null,
|
||||
model: event.model ?? event.modelId ?? event.attributes?.model ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function parsedTimestampMs(value) {
|
||||
const parsed = Date.parse(value ?? "");
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function spanIdentity(event) {
|
||||
if (event.spanId !== undefined && event.spanId !== null && String(event.spanId).length > 0) {
|
||||
return `id:${event.spanId}`;
|
||||
|
||||
@ -1,4 +1,8 @@
|
||||
import { buildAgentTurnBreakdown } from "./collectors/agent-turns.mjs";
|
||||
import {
|
||||
buildDashboardPreProviderAttribution,
|
||||
summarizeDashboardPreProviderAttributions
|
||||
} from "./collectors/dashboard-turn-attribution.mjs";
|
||||
import { computeProviderTurnAttribution } from "./collectors/provider.mjs";
|
||||
import { summarizeRuntimeDepsLogs } from "./collectors/logs.mjs";
|
||||
import { buildHealthMeasurement, healthReadinessClassification } from "./health.mjs";
|
||||
@ -77,6 +81,7 @@ export function evaluateRecord(record, scenario, options = {}) {
|
||||
const providerTurn = collectSlowestProviderTurn(agentTurns);
|
||||
const agentTurnStats = summarizeAgentTurnStats(agentTurns);
|
||||
const agentTurnDiagnostics = summarizeAgentTurnDiagnostics(agentTurns);
|
||||
const dashboardPreProviderAttribution = summarizeDashboardPreProviderAttributions(agentTurns);
|
||||
const agentTurnMs = maxTurnDuration(agentTurns);
|
||||
const agentResponseOk = agentTurns.length === 0 ? null : agentTurns.every((turn) => turn.responseOk === true);
|
||||
const agentProviderSimulation = evaluateProviderSimulation({ turns: agentTurns, scenario, record, thresholds });
|
||||
@ -754,6 +759,13 @@ export function evaluateRecord(record, scenario, options = {}) {
|
||||
agentEventLoopSampleCount: agentTurnDiagnostics.eventLoopSampleCount,
|
||||
agentSessionPollCount: agentTurnDiagnostics.sessionPollCount,
|
||||
agentSessionPollErrorCount: agentTurnDiagnostics.sessionPollErrorCount,
|
||||
dashboardPreProviderAttribution,
|
||||
coldPreProviderAttributedMs: dashboardPreProviderAttribution.cold.knownAttributedMs.median,
|
||||
warmPreProviderAttributedMs: dashboardPreProviderAttribution.warm.knownAttributedMs.median,
|
||||
coldPreProviderUnattributedMs: dashboardPreProviderAttribution.cold.unattributedMs.median,
|
||||
warmPreProviderUnattributedMs: dashboardPreProviderAttribution.warm.unattributedMs.median,
|
||||
coldPreProviderAttributionCoverage: dashboardPreProviderAttribution.cold.coverageRatio.median,
|
||||
warmPreProviderAttributionCoverage: dashboardPreProviderAttribution.warm.coverageRatio.median,
|
||||
coldAgentTurnMs: coldAgentTurn?.totalTurnMs ?? null,
|
||||
warmAgentTurnMs: warmAgentTurn?.totalTurnMs ?? null,
|
||||
agentColdWarmDeltaMs: delta(coldAgentTurn?.totalTurnMs, warmAgentTurn?.totalTurnMs),
|
||||
@ -976,6 +988,16 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
activeFinishedAtEpochMs: timingResult.finishedAtEpochMs,
|
||||
gatewaySession
|
||||
});
|
||||
const dashboardPreProviderAttribution = gatewaySession
|
||||
? buildDashboardPreProviderAttribution({
|
||||
label: agentTurnLabel(phase.id, index),
|
||||
phaseId: phase.id,
|
||||
activeStartedAtEpochMs: timingResult.startedAtEpochMs,
|
||||
activeFinishedAtEpochMs: timingResult.finishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary
|
||||
})
|
||||
: null;
|
||||
turns.push({
|
||||
schemaVersion: "kova.agentTurnEvidence.v1",
|
||||
index,
|
||||
@ -1022,6 +1044,7 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
providerLateByMs: attribution?.providerLateByMs ?? null,
|
||||
phaseBreakdown,
|
||||
turnDiagnostics,
|
||||
dashboardPreProviderAttribution,
|
||||
metadataScanCount: turnDiagnostics.metadataScan.count,
|
||||
metadataScanTotalMs: turnDiagnostics.metadataScan.totalDurationMs,
|
||||
metadataScanMaxMs: turnDiagnostics.metadataScan.maxDurationMs,
|
||||
@ -1092,7 +1115,9 @@ function resultForActiveTurnWindow(result, gatewaySession) {
|
||||
}
|
||||
|
||||
function summarizeActiveTurnDiagnostics({ timelineSummary, activeStartedAtEpochMs, activeFinishedAtEpochMs, gatewaySession }) {
|
||||
const events = Array.isArray(timelineSummary?.events) ? timelineSummary.events : [];
|
||||
const events = Array.isArray(timelineSummary?.turnAttributionEvents) && timelineSummary.turnAttributionEvents.length > 0
|
||||
? timelineSummary.turnAttributionEvents
|
||||
: (Array.isArray(timelineSummary?.events) ? timelineSummary.events : []);
|
||||
const windowEvents = events.filter((event) =>
|
||||
eventEpochMs(event) !== null &&
|
||||
eventEpochMs(event) >= activeStartedAtEpochMs &&
|
||||
@ -2637,6 +2662,8 @@ function collectTimelineSummary(record) {
|
||||
let openSpans = [];
|
||||
let latestEventCount = -1;
|
||||
let events = [];
|
||||
let turnAttributionEvents = [];
|
||||
const artifacts = new Set();
|
||||
const keySpans = {};
|
||||
const spanTotals = {};
|
||||
|
||||
@ -2644,6 +2671,10 @@ function collectTimelineSummary(record) {
|
||||
if ((timeline.eventCount ?? 0) >= latestEventCount && Array.isArray(timeline.events)) {
|
||||
latestEventCount = timeline.eventCount ?? 0;
|
||||
events = timeline.events;
|
||||
turnAttributionEvents = Array.isArray(timeline.turnAttributionEvents) ? timeline.turnAttributionEvents : [];
|
||||
}
|
||||
for (const artifact of timeline.artifacts ?? []) {
|
||||
artifacts.add(artifact);
|
||||
}
|
||||
eventCount = Math.max(eventCount, timeline.eventCount ?? 0);
|
||||
parseErrorCount = Math.max(parseErrorCount, timeline.parseErrorCount ?? 0);
|
||||
@ -2684,7 +2715,10 @@ function collectTimelineSummary(record) {
|
||||
repeatedSpanCount,
|
||||
openSpanCount,
|
||||
openSpans,
|
||||
artifacts: [...artifacts],
|
||||
timelineArtifacts: [...artifacts],
|
||||
events,
|
||||
turnAttributionEvents,
|
||||
keySpans,
|
||||
spanTotals,
|
||||
eventLoopMaxMs,
|
||||
|
||||
@ -21,6 +21,10 @@ export const PERFORMANCE_METRICS = [
|
||||
{ id: "agentCleanupMaxMs", title: "Agent Cleanup Max", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "coldPreProviderMs", title: "Cold Pre-Provider", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "warmPreProviderMs", title: "Warm Pre-Provider", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "coldPreProviderAttributedMs", title: "Cold Pre-Provider Attributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "warmPreProviderAttributedMs", title: "Warm Pre-Provider Attributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "coldPreProviderUnattributedMs", title: "Cold Pre-Provider Unattributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "warmPreProviderUnattributedMs", title: "Warm Pre-Provider Unattributed", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "agentMetadataScanCount", title: "Agent Metadata Scans", unit: "count", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "agentMetadataScanTotalMs", title: "Agent Metadata Scan Total", unit: "ms", regressionKey: "agentLatencyRegressionPercent" },
|
||||
{ id: "agentEventLoopMaxMs", title: "Agent Event Loop Max", unit: "ms", regressionKey: "eventLoopRegressionPercent" },
|
||||
|
||||
@ -414,6 +414,12 @@ function metricDeltas(baseline, current) {
|
||||
"coldPreProviderMs",
|
||||
"warmPreProviderMs",
|
||||
"agentColdWarmPreProviderDeltaMs",
|
||||
"coldPreProviderAttributedMs",
|
||||
"warmPreProviderAttributedMs",
|
||||
"coldPreProviderUnattributedMs",
|
||||
"warmPreProviderUnattributedMs",
|
||||
"coldPreProviderAttributionCoverage",
|
||||
"warmPreProviderAttributionCoverage",
|
||||
"coldProviderFinalMs",
|
||||
"warmProviderFinalMs",
|
||||
"agentMetadataScanCount",
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { summarizeAgentTurnBreakdownForMarkdown } from "../collectors/agent-turns.mjs";
|
||||
import { dashboardPreProviderMarkdownRows } from "../collectors/dashboard-turn-attribution.mjs";
|
||||
import { healthTotalFailures } from "../health.mjs";
|
||||
|
||||
export function summarizeRecords(records) {
|
||||
@ -182,6 +183,9 @@ export function renderMarkdownReport(report) {
|
||||
if (record.measurements.agentTurnCount > 0) {
|
||||
lines.push(`- Agent cold/warm: cold ${record.measurements.coldAgentTurnMs ?? "unknown"} ms; warm ${record.measurements.warmAgentTurnMs ?? "unknown"} ms; delta ${record.measurements.agentColdWarmDeltaMs ?? "unknown"} ms`);
|
||||
lines.push(`- Agent pre-provider: cold ${record.measurements.coldPreProviderMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderMs ?? "unknown"} ms; delta ${record.measurements.agentColdWarmPreProviderDeltaMs ?? "unknown"} ms`);
|
||||
if (record.measurements.dashboardPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- Dashboard pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
|
||||
}
|
||||
lines.push(`- Agent provider final: cold ${record.measurements.coldProviderFinalMs ?? "unknown"} ms; warm ${record.measurements.warmProviderFinalMs ?? "unknown"} ms`);
|
||||
lines.push(`- Agent turn stats: count ${record.measurements.agentTurnCount}; p95 ${record.measurements.agentTurnP95Ms ?? "unknown"} ms; max ${record.measurements.agentTurnMaxMs ?? "unknown"} ms; pre-provider p95 ${record.measurements.agentPreProviderP95Ms ?? "unknown"} ms`);
|
||||
lines.push(`- Agent active-turn diagnostics: metadata scans ${record.measurements.agentMetadataScanCount ?? "unknown"} (${record.measurements.agentMetadataScanTotalMs ?? "unknown"} ms total); event-loop max ${record.measurements.agentEventLoopMaxMs ?? "unknown"} ms; session polls ${record.measurements.agentSessionPollCount ?? "unknown"} (${record.measurements.agentSessionPollErrorCount ?? "unknown"} errors)`);
|
||||
@ -239,6 +243,7 @@ export function renderMarkdownReport(report) {
|
||||
lines.push(` - breakdown: ${breakdown}`);
|
||||
}
|
||||
}
|
||||
lines.push(...dashboardPreProviderMarkdownRows(record.measurements.agentTurns));
|
||||
}
|
||||
lines.push(`- Profiling: ${record.profiling?.enabled ? "enabled" : "off"} (${record.profiling?.interpretation ?? "unknown"})`);
|
||||
lines.push(`- V8 reports / heap snapshots: ${record.measurements.v8ReportCount ?? "unknown"} / ${record.measurements.heapSnapshotCount ?? "unknown"}`);
|
||||
@ -730,6 +735,13 @@ function summarizeMeasurements(measurements) {
|
||||
agentEventLoopSampleCount: measurements.agentEventLoopSampleCount ?? null,
|
||||
agentSessionPollCount: measurements.agentSessionPollCount ?? null,
|
||||
agentSessionPollErrorCount: measurements.agentSessionPollErrorCount ?? null,
|
||||
dashboardPreProviderAttribution: measurements.dashboardPreProviderAttribution ?? null,
|
||||
coldPreProviderAttributedMs: measurements.coldPreProviderAttributedMs ?? null,
|
||||
warmPreProviderAttributedMs: measurements.warmPreProviderAttributedMs ?? null,
|
||||
coldPreProviderUnattributedMs: measurements.coldPreProviderUnattributedMs ?? null,
|
||||
warmPreProviderUnattributedMs: measurements.warmPreProviderUnattributedMs ?? null,
|
||||
coldPreProviderAttributionCoverage: measurements.coldPreProviderAttributionCoverage ?? null,
|
||||
warmPreProviderAttributionCoverage: measurements.warmPreProviderAttributionCoverage ?? null,
|
||||
coldAgentTurnMs: measurements.coldAgentTurnMs ?? null,
|
||||
warmAgentTurnMs: measurements.warmAgentTurnMs ?? null,
|
||||
agentColdWarmDeltaMs: measurements.agentColdWarmDeltaMs ?? null,
|
||||
@ -1149,6 +1161,8 @@ function compactPerformanceMetrics(metrics = {}) {
|
||||
"warmAgentTurnMs",
|
||||
"agentColdWarmDeltaMs",
|
||||
"coldPreProviderMs",
|
||||
"coldPreProviderAttributedMs",
|
||||
"coldPreProviderUnattributedMs",
|
||||
"runtimeDepsStagingMs"
|
||||
];
|
||||
const byId = new Map(Object.entries(metrics).map(([id, metric]) => [id, { id, ...metric }]));
|
||||
@ -1186,6 +1200,9 @@ function pushMeasurementBrief(lines, measurements, { compact }) {
|
||||
lines.push(`- health: startup p95 ${valueMs(measurements.health?.startupSamples?.p95Ms)}; post-ready p95 ${valueMs(measurements.health?.postReadySamples?.p95Ms)}; failures ${totalHealthFailures ?? "unknown"}; final failures ${measurements.health?.final?.failureCount ?? "unknown"}${healthSlowestText(measurements)}`);
|
||||
lines.push(`- resources: peak RSS ${valueMb(measurements.peakRssMb)}; max CPU ${valuePercent(measurements.cpuPercentMax)}; samples ${measurements.resourceSampleCount ?? "unknown"}; roles ${rolePeakText(measurements)}`);
|
||||
lines.push(`- agent: turn ${valueMs(measurements.agentTurnMs, "not-run")}; cold/warm ${valueMs(measurements.coldAgentTurnMs)}/${valueMs(measurements.warmAgentTurnMs)}; cold-warm delta ${valueMs(measurements.agentColdWarmDeltaMs)}; pre-provider ${valueMs(measurements.agentPreProviderMs)}; provider ${valueMs(measurements.agentProviderFinalMs)}; metadata scans ${measurements.agentMetadataScanCount ?? "unknown"} (${valueMs(measurements.agentMetadataScanTotalMs)}); event-loop ${valueMs(measurements.agentEventLoopMaxMs)}; polls ${measurements.agentSessionPollCount ?? "unknown"}; cleanup ${valueMs(measurements.agentCleanupMaxMs)}; diagnosis ${measurements.agentLatencyDiagnosis?.kind ?? "unknown"}; leaks ${measurements.agentProcessLeakCount ?? "unknown"}`);
|
||||
if (measurements.dashboardPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- dashboard attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
|
||||
}
|
||||
lines.push(`- plugins/runtime: missing deps ${measurements.missingDependencyErrors ?? "unknown"}; plugin failures ${measurements.pluginLoadFailures ?? "unknown"}; runtime deps ${valueMs(measurements.runtimeDepsStagingMs)}${runtimeDepsPluginText(measurements)}; warm restages ${measurements.warmRuntimeDepsRestageCount ?? "unknown"}; warm reuse ${measurements.runtimeDepsWarmReuseOk ?? "unknown"}`);
|
||||
|
||||
if (!compact || hasDiagnosticSignal(measurements)) {
|
||||
|
||||
@ -31,6 +31,10 @@ import {
|
||||
buildAgentTurnBreakdown,
|
||||
summarizeAgentTurnBreakdownForMarkdown
|
||||
} from "./collectors/agent-turns.mjs";
|
||||
import {
|
||||
attributedSpanIntervals,
|
||||
buildDashboardPreProviderAttribution
|
||||
} from "./collectors/dashboard-turn-attribution.mjs";
|
||||
import {
|
||||
computeProviderTurnAttribution,
|
||||
parseProviderRequestLog,
|
||||
@ -373,6 +377,7 @@ export async function runSelfCheck(flags = {}) {
|
||||
checks.push(await providerEvidenceParserCheck());
|
||||
checks.push(agentTurnBreakdownCheck());
|
||||
checks.push(gatewaySessionTurnEvaluationCheck());
|
||||
checks.push(dashboardPreProviderAttributionCheck());
|
||||
checks.push(await mockProviderBehaviorCheck(tmp));
|
||||
checks.push(providerFailureEvaluationCheck());
|
||||
checks.push(agentColdWarmEvaluationCheck());
|
||||
@ -2305,6 +2310,248 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
}
|
||||
}
|
||||
|
||||
function dashboardPreProviderAttributionCheck() {
|
||||
try {
|
||||
const base = 1777536000000;
|
||||
const timelineText = [
|
||||
timelineEvent({ type: "span.start", name: "gateway.chat_send.load_session", timestamp: base + 1010, spanId: "cold-load" }),
|
||||
timelineEvent({ type: "span.end", name: "gateway.chat_send.load_session", timestamp: base + 1070, spanId: "cold-load", durationMs: 60 }),
|
||||
timelineEvent({ type: "span.start", name: "auto_reply.finalize_context", timestamp: base + 1060, spanId: "cold-finalize" }),
|
||||
timelineEvent({ type: "span.end", name: "auto_reply.finalize_context", timestamp: base + 1160, spanId: "cold-finalize", durationMs: 100 }),
|
||||
timelineEvent({ type: "span.start", name: "reply.ensure_workspace", timestamp: base + 1180, spanId: "cold-workspace" }),
|
||||
timelineEvent({ type: "span.error", name: "reply.ensure_workspace", timestamp: base + 1230, spanId: "cold-workspace", durationMs: 50, errorName: "SyntheticError" }),
|
||||
timelineEvent({ type: "span.end", name: "plugins.metadata.scan", timestamp: base + 1150, spanId: "cold-scan", durationMs: 33 }),
|
||||
timelineEvent({ type: "provider.request", name: "provider.request", timestamp: base + 1200, receivedAtEpochMs: base + 1200, respondedAtEpochMs: base + 1800, durationMs: 600 }),
|
||||
timelineEvent({ type: "eventLoop.sample", name: "eventLoop.sample", timestamp: base + 1250, maxMs: 9 }),
|
||||
timelineEvent({ type: "span.start", name: "gateway.chat_send.dispatch_inbound", timestamp: base + 11025, spanId: "warm-dispatch" }),
|
||||
timelineEvent({ type: "span.end", name: "gateway.chat_send.dispatch_inbound", timestamp: base + 11125, spanId: "warm-dispatch", durationMs: 100 }),
|
||||
timelineEvent({ type: "span.start", name: "reply.load_runtime_plugins", timestamp: base + 11120, spanId: "warm-plugins" }),
|
||||
timelineEvent({ type: "span.end", name: "reply.load_runtime_plugins", timestamp: base + 11220, spanId: "warm-plugins", durationMs: 100 }),
|
||||
timelineEvent({ type: "span.end", name: "plugins.metadata.scan", timestamp: base + 11100, spanId: "warm-scan", durationMs: 11 }),
|
||||
timelineEvent({ type: "provider.request", name: "provider.request", timestamp: base + 11250, receivedAtEpochMs: base + 11250, respondedAtEpochMs: base + 11600, durationMs: 350 }),
|
||||
timelineEvent({ type: "eventLoop.sample", name: "eventLoop.sample", timestamp: base + 11200, maxMs: 7 })
|
||||
].join("\n");
|
||||
const parsed = parseTimelineText(timelineText);
|
||||
assertEqual(parsed.turnAttributionEvents.length, 16, "turn attribution events retained");
|
||||
const parsedIntervals = attributedSpanIntervals(parsed.turnAttributionEvents);
|
||||
assertEqual(parsedIntervals.length, 5, "span parser includes error terminal");
|
||||
assertEqual(parsedIntervals.some((span) => span.type === "span.error" && span.name === "reply.ensure_workspace"), true, "span error included");
|
||||
|
||||
const coldAttribution = buildDashboardPreProviderAttribution({
|
||||
label: "cold",
|
||||
phaseId: "cold-dashboard-session-turn",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 2500,
|
||||
attribution: {
|
||||
firstProviderRequestAtEpochMs: base + 1200,
|
||||
preProviderMs: 200,
|
||||
providerFinalMs: 600,
|
||||
firstByteLatencyMs: 25,
|
||||
firstChunkLatencyMs: 30
|
||||
},
|
||||
timelineSummary: {
|
||||
available: true,
|
||||
turnAttributionEvents: parsed.turnAttributionEvents,
|
||||
artifacts: ["/tmp/kova/openclaw/timeline.jsonl"]
|
||||
}
|
||||
});
|
||||
assertEqual(coldAttribution.available, true, "cold attribution available");
|
||||
assertEqual(coldAttribution.knownAttributedMs, 170, "overlap-safe cold known attribution");
|
||||
assertEqual(coldAttribution.unattributedMs, 30, "cold unattributed remainder");
|
||||
assertEqual(coldAttribution.spanSummaries.find((span) => span.name === "reply.ensure_workspace")?.errorCount, 1, "error span summary");
|
||||
assertEqual(coldAttribution.provider.totalDurationMs, 600, "provider duration stays separate");
|
||||
assertEqual(coldAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "timeline artifact path");
|
||||
|
||||
const missingAttribution = buildDashboardPreProviderAttribution({
|
||||
label: "cold",
|
||||
phaseId: "cold-dashboard-session-turn",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 2500,
|
||||
attribution: { firstProviderRequestAtEpochMs: base + 1200, preProviderMs: 200 },
|
||||
timelineSummary: { available: false, artifacts: [] }
|
||||
});
|
||||
assertEqual(missingAttribution.available, false, "missing timeline unavailable");
|
||||
assertEqual(missingAttribution.unattributedMs, 200, "missing timeline preserves full remainder");
|
||||
|
||||
const record = syntheticDashboardSessionRecord({ base, timeline: parsed });
|
||||
evaluateRecord(record, {
|
||||
id: "dashboard-session-send-turn",
|
||||
agent: { expectedText: "KOVA_AGENT_OK" },
|
||||
thresholds: { agentTurnMs: 2000, coldAgentTurnMs: 2000, warmAgentTurnMs: 1000 }
|
||||
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
|
||||
assertEqual(record.measurements.coldPreProviderAttributedMs, 170, "record cold attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderAttributedMs, 195, "record warm attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderUnattributedMs, 55, "record warm unattributed metric");
|
||||
assertEqual(record.measurements.dashboardPreProviderAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "record timeline artifact");
|
||||
|
||||
const rendered = renderMarkdownReport({
|
||||
generatedAt: "2026-05-01T00:00:00.000Z",
|
||||
runId: "self-check-dashboard-pre-provider",
|
||||
mode: "self-check",
|
||||
target: "runtime:stable",
|
||||
platform: { os: "test", release: "test", arch: "test", node: "test" },
|
||||
records: [record],
|
||||
summary: { statuses: { PASS: 1 } }
|
||||
});
|
||||
assertEqual(rendered.includes("Dashboard pre-provider attribution:"), true, "markdown includes dashboard attribution table");
|
||||
assertEqual(rendered.includes("`reply.ensure_workspace`"), true, "markdown includes span table");
|
||||
|
||||
return {
|
||||
id: "dashboard-pre-provider-attribution",
|
||||
status: "PASS",
|
||||
command: "evaluate synthetic dashboard pre-provider timeline attribution",
|
||||
durationMs: 0
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "dashboard-pre-provider-attribution",
|
||||
status: "FAIL",
|
||||
command: "evaluate synthetic dashboard pre-provider timeline attribution",
|
||||
durationMs: 0,
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function timelineEvent(event) {
|
||||
const timestamp = typeof event.timestamp === "number" ? new Date(event.timestamp).toISOString() : event.timestamp;
|
||||
return JSON.stringify({
|
||||
schemaVersion: "openclaw.diagnostics.v1",
|
||||
...event,
|
||||
timestamp
|
||||
});
|
||||
}
|
||||
|
||||
function syntheticDashboardSessionRecord({ base, timeline }) {
|
||||
const coldPayload = {
|
||||
ok: true,
|
||||
surface: "dashboard-session-send-turn",
|
||||
method: "sessions.send",
|
||||
createSession: true,
|
||||
minAssistantCount: 1,
|
||||
sessionKey: "kova-dashboard-session-send",
|
||||
runId: "cold-run",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 2500,
|
||||
activeTurnMs: 1500,
|
||||
sendStartedAtEpochMs: base + 1000,
|
||||
sendFinishedAtEpochMs: base + 1040,
|
||||
sendDurationMs: 40,
|
||||
assistantFirstSeenAtEpochMs: base + 2200,
|
||||
assistantMatchedAtEpochMs: base + 2500,
|
||||
timeToFirstAssistantMs: 1200,
|
||||
timeToMatchedAssistantMs: 1500,
|
||||
historyPollCount: 3,
|
||||
historyErrorCount: 0,
|
||||
assistantMessageCount: 1,
|
||||
finalAssistantVisibleText: "KOVA_AGENT_OK",
|
||||
expectedTextPresent: true
|
||||
};
|
||||
const warmPayload = {
|
||||
...coldPayload,
|
||||
createSession: false,
|
||||
minAssistantCount: 2,
|
||||
runId: "warm-run",
|
||||
activeStartedAtEpochMs: base + 11000,
|
||||
activeFinishedAtEpochMs: base + 11800,
|
||||
activeTurnMs: 800,
|
||||
sendStartedAtEpochMs: base + 11000,
|
||||
sendFinishedAtEpochMs: base + 11050,
|
||||
sendDurationMs: 50,
|
||||
assistantFirstSeenAtEpochMs: base + 11600,
|
||||
assistantMatchedAtEpochMs: base + 11800,
|
||||
timeToFirstAssistantMs: 600,
|
||||
timeToMatchedAssistantMs: 800,
|
||||
historyPollCount: 2,
|
||||
assistantMessageCount: 2
|
||||
};
|
||||
return {
|
||||
scenario: "dashboard-session-send-turn",
|
||||
surface: "dashboard-session-send-turn",
|
||||
title: "Gateway session cold/warm",
|
||||
status: "PASS",
|
||||
cleanup: "done",
|
||||
auth: { mode: "mock" },
|
||||
phases: [
|
||||
syntheticDashboardTurnPhase({
|
||||
id: "cold-dashboard-session-turn",
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session true",
|
||||
startedAtEpochMs: base,
|
||||
finishedAtEpochMs: base + 5000,
|
||||
payload: coldPayload
|
||||
}),
|
||||
syntheticDashboardTurnPhase({
|
||||
id: "warm-dashboard-session-turn",
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session false",
|
||||
startedAtEpochMs: base + 10000,
|
||||
finishedAtEpochMs: base + 14000,
|
||||
payload: warmPayload
|
||||
})
|
||||
],
|
||||
providerEvidence: {
|
||||
available: true,
|
||||
requestCount: 2,
|
||||
requests: [
|
||||
{
|
||||
requestId: "cold-provider",
|
||||
receivedAt: new Date(base + 1200).toISOString(),
|
||||
receivedAtEpochMs: base + 1200,
|
||||
respondedAt: new Date(base + 1800).toISOString(),
|
||||
respondedAtEpochMs: base + 1800,
|
||||
firstByteLatencyMs: 25,
|
||||
firstChunkLatencyMs: 30,
|
||||
route: "/v1/responses",
|
||||
model: "gpt-5.5",
|
||||
status: 200
|
||||
},
|
||||
{
|
||||
requestId: "warm-provider",
|
||||
receivedAt: new Date(base + 11250).toISOString(),
|
||||
receivedAtEpochMs: base + 11250,
|
||||
respondedAt: new Date(base + 11600).toISOString(),
|
||||
respondedAtEpochMs: base + 11600,
|
||||
firstByteLatencyMs: 20,
|
||||
firstChunkLatencyMs: 22,
|
||||
route: "/v1/responses",
|
||||
model: "gpt-5.5",
|
||||
status: 200
|
||||
}
|
||||
]
|
||||
},
|
||||
finalMetrics: {
|
||||
service: { gatewayState: "running" },
|
||||
logs: zeroLogMetrics(),
|
||||
timeline: {
|
||||
...timeline,
|
||||
artifacts: ["/tmp/kova/openclaw/timeline.jsonl"]
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function syntheticDashboardTurnPhase({ id, command, startedAtEpochMs, finishedAtEpochMs, payload }) {
|
||||
return {
|
||||
id,
|
||||
title: id,
|
||||
intent: "Synthetic dashboard session turn",
|
||||
commands: [command],
|
||||
evidence: [],
|
||||
results: [{
|
||||
command,
|
||||
status: 0,
|
||||
timedOut: false,
|
||||
startedAt: new Date(startedAtEpochMs).toISOString(),
|
||||
startedAtEpochMs,
|
||||
finishedAt: new Date(finishedAtEpochMs).toISOString(),
|
||||
finishedAtEpochMs,
|
||||
durationMs: finishedAtEpochMs - startedAtEpochMs,
|
||||
stdout: JSON.stringify(payload),
|
||||
stderr: ""
|
||||
}],
|
||||
metrics: { logs: zeroLogMetrics(), health: { ok: true } }
|
||||
};
|
||||
}
|
||||
|
||||
function syntheticTurn({
|
||||
startedAtEpochMs,
|
||||
firstProviderRequestAtEpochMs,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user