feat: attribute agent cli pre-provider spans
This commit is contained in:
Shakker
parent
2b5bddff4a
commit
75f60628ff
57
src/collectors/agent-cli-attribution.mjs
Normal file
57
src/collectors/agent-cli-attribution.mjs
Normal file
@ -0,0 +1,57 @@
|
||||
import {
|
||||
buildPreProviderAttribution,
|
||||
preProviderMarkdownRows,
|
||||
summarizePreProviderAttributions
|
||||
} from "./pre-provider-attribution.mjs";
|
||||
|
||||
export const AGENT_CLI_PRE_PROVIDER_ATTRIBUTION_SCHEMA = "kova.agentCliPreProviderAttribution.v1";
|
||||
export const AGENT_CLI_PRE_PROVIDER_SUMMARY_SCHEMA = "kova.agentCliPreProviderAttributionSummary.v1";
|
||||
|
||||
export function buildAgentCliPreProviderAttribution({
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary
|
||||
}) {
|
||||
return buildPreProviderAttribution({
|
||||
schemaVersion: AGENT_CLI_PRE_PROVIDER_ATTRIBUTION_SCHEMA,
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary,
|
||||
isAttributedSpanName: isAgentCliAttributedSpanName,
|
||||
missingEventsError: "timeline contains no agent CLI attribution events"
|
||||
});
|
||||
}
|
||||
|
||||
export function summarizeAgentCliPreProviderAttributions(turns) {
|
||||
return summarizePreProviderAttributions({
|
||||
schemaVersion: AGENT_CLI_PRE_PROVIDER_SUMMARY_SCHEMA,
|
||||
turns,
|
||||
fieldName: "agentCliPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function agentCliPreProviderMarkdownRows(turns) {
|
||||
return preProviderMarkdownRows({
|
||||
title: "Agent CLI pre-provider attribution",
|
||||
turns,
|
||||
fieldName: "agentCliPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
function isAgentCliAttributedSpanName(name) {
|
||||
const text = String(name ?? "");
|
||||
return text === "agent.prepare" ||
|
||||
text === "plugins.metadata.scan" ||
|
||||
text === "runtimeDeps.stage" ||
|
||||
text === "channel.capabilities" ||
|
||||
text === "models.catalog" ||
|
||||
text.startsWith("models.catalog.") ||
|
||||
text.startsWith("models.discovery") ||
|
||||
text.startsWith("channel.plugin.");
|
||||
}
|
||||
@ -1,3 +1,10 @@
|
||||
import {
|
||||
attributedSpanIntervals as collectAttributedSpanIntervals,
|
||||
buildPreProviderAttribution,
|
||||
preProviderMarkdownRows,
|
||||
summarizePreProviderAttributions
|
||||
} from "./pre-provider-attribution.mjs";
|
||||
|
||||
export const DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA = "kova.dashboardPreProviderAttribution.v1";
|
||||
export const DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA = "kova.dashboardPreProviderAttributionSummary.v1";
|
||||
|
||||
@ -9,323 +16,37 @@ export function buildDashboardPreProviderAttribution({
|
||||
attribution,
|
||||
timelineSummary
|
||||
}) {
|
||||
const artifacts = timelineArtifacts(timelineSummary);
|
||||
const events = attributionEvents(timelineSummary);
|
||||
const providerBoundaryEpochMs = numberOrNull(attribution?.firstProviderRequestAtEpochMs);
|
||||
const windowStartEpochMs = numberOrNull(activeStartedAtEpochMs ?? attribution?.commandStartedAtEpochMs);
|
||||
const activeEndEpochMs = numberOrNull(activeFinishedAtEpochMs ?? attribution?.commandFinishedAtEpochMs);
|
||||
const windowEndEpochMs = providerBoundaryEpochMs;
|
||||
const preProviderMs = numberOrNull(attribution?.preProviderMs) ??
|
||||
durationBetween(windowStartEpochMs, windowEndEpochMs);
|
||||
const base = {
|
||||
return buildPreProviderAttribution({
|
||||
schemaVersion: DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA,
|
||||
available: false,
|
||||
label: label ?? null,
|
||||
phaseId: phaseId ?? null,
|
||||
timelineAvailable: timelineSummary?.available === true,
|
||||
timelineArtifacts: artifacts,
|
||||
eventCount: events.length,
|
||||
window: {
|
||||
startEpochMs: windowStartEpochMs,
|
||||
startAt: isoOrNull(windowStartEpochMs),
|
||||
endEpochMs: windowEndEpochMs,
|
||||
endAt: isoOrNull(windowEndEpochMs),
|
||||
durationMs: preProviderMs
|
||||
},
|
||||
activeWindow: {
|
||||
startEpochMs: windowStartEpochMs,
|
||||
startAt: isoOrNull(windowStartEpochMs),
|
||||
endEpochMs: activeEndEpochMs,
|
||||
endAt: isoOrNull(activeEndEpochMs),
|
||||
durationMs: durationBetween(windowStartEpochMs, activeEndEpochMs)
|
||||
},
|
||||
providerBoundary: {
|
||||
firstRequestAtEpochMs: providerBoundaryEpochMs,
|
||||
firstRequestAt: isoOrNull(providerBoundaryEpochMs),
|
||||
source: providerBoundaryEpochMs === null ? null : "provider-evidence"
|
||||
},
|
||||
provider: summarizeProviderEvents(events, providerBoundaryEpochMs, activeEndEpochMs, attribution),
|
||||
spanSummaries: [],
|
||||
knownAttributedMs: null,
|
||||
unattributedMs: preProviderMs,
|
||||
coverageRatio: null,
|
||||
error: null
|
||||
};
|
||||
|
||||
if (timelineSummary?.available !== true) {
|
||||
return {
|
||||
...base,
|
||||
error: "OpenClaw diagnostics timeline unavailable"
|
||||
};
|
||||
}
|
||||
if (events.length === 0) {
|
||||
return {
|
||||
...base,
|
||||
error: "timeline contains no dashboard turn attribution events"
|
||||
};
|
||||
}
|
||||
if (windowStartEpochMs === null || windowEndEpochMs === null || preProviderMs === null || windowEndEpochMs < windowStartEpochMs) {
|
||||
return {
|
||||
...base,
|
||||
error: "pre-provider window boundary unavailable"
|
||||
};
|
||||
}
|
||||
|
||||
const intervals = attributedSpanIntervals(events)
|
||||
.map((span) => clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs))
|
||||
.filter(Boolean);
|
||||
const spanSummaries = summarizeAttributedSpans(intervals);
|
||||
const knownAttributedMs = round(unionDuration(intervals));
|
||||
const unattributedMs = round(Math.max(0, preProviderMs - knownAttributedMs));
|
||||
|
||||
return {
|
||||
...base,
|
||||
available: true,
|
||||
spanSummaries,
|
||||
knownAttributedMs,
|
||||
unattributedMs,
|
||||
coverageRatio: preProviderMs > 0 ? round(knownAttributedMs / preProviderMs) : null,
|
||||
error: null
|
||||
};
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary,
|
||||
isAttributedSpanName: isDashboardAttributedSpanName,
|
||||
missingEventsError: "timeline contains no dashboard turn attribution events"
|
||||
});
|
||||
}
|
||||
|
||||
export function summarizeDashboardPreProviderAttributions(turns) {
|
||||
const entries = (turns ?? [])
|
||||
.map((turn) => turn.dashboardPreProviderAttribution)
|
||||
.filter(Boolean);
|
||||
const cold = summarizeLabeledAttribution(entries, "cold");
|
||||
const warm = summarizeLabeledAttribution(entries, "warm");
|
||||
return {
|
||||
return summarizePreProviderAttributions({
|
||||
schemaVersion: DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA,
|
||||
available: entries.some((entry) => entry.available === true),
|
||||
count: entries.length,
|
||||
cold,
|
||||
warm,
|
||||
spanMedians: summarizeSpanMedians(entries),
|
||||
timelineArtifacts: unique(entries.flatMap((entry) => entry.timelineArtifacts ?? []))
|
||||
};
|
||||
turns,
|
||||
fieldName: "dashboardPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function dashboardPreProviderMarkdownRows(turns) {
|
||||
const attributions = (turns ?? [])
|
||||
.map((turn) => turn.dashboardPreProviderAttribution)
|
||||
.filter(Boolean);
|
||||
if (attributions.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const lines = [
|
||||
"- Dashboard pre-provider attribution:",
|
||||
"",
|
||||
" | turn | pre-provider | known | unattributed | provider | timeline |",
|
||||
" |---|---:|---:|---:|---:|---|"
|
||||
];
|
||||
for (const item of attributions) {
|
||||
const timeline = item.timelineArtifacts?.[0] ?? (item.timelineAvailable ? "available" : "missing");
|
||||
lines.push(
|
||||
` | ${item.label ?? "turn"} | ${formatMs(item.window?.durationMs)} | ${formatMs(item.knownAttributedMs)} | ${formatMs(item.unattributedMs)} | ${formatMs(item.provider?.totalDurationMs)} | ${timeline} |`
|
||||
);
|
||||
}
|
||||
|
||||
const spanRows = attributions.flatMap((item) =>
|
||||
(item.spanSummaries ?? []).slice(0, 6).map((span) => ({ turn: item.label ?? "turn", ...span }))
|
||||
);
|
||||
if (spanRows.length > 0) {
|
||||
lines.push("");
|
||||
lines.push(" | turn | span | count | errors | clipped | max |");
|
||||
lines.push(" |---|---|---:|---:|---:|---:|");
|
||||
for (const span of spanRows.slice(0, 12)) {
|
||||
lines.push(
|
||||
` | ${span.turn} | \`${span.name}\` | ${span.count} | ${span.errorCount} | ${formatMs(span.totalClippedDurationMs)} | ${formatMs(span.maxClippedDurationMs)} |`
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
return preProviderMarkdownRows({
|
||||
title: "Dashboard pre-provider attribution",
|
||||
turns,
|
||||
fieldName: "dashboardPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function attributedSpanIntervals(events) {
|
||||
const startsById = new Map();
|
||||
const intervals = [];
|
||||
|
||||
for (const event of events ?? []) {
|
||||
if (event?.type === "span.start" && isDashboardAttributedSpanName(event.name)) {
|
||||
const key = spanKey(event);
|
||||
if (key) {
|
||||
startsById.set(key, event);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((event?.type === "span.end" || event?.type === "span.error") && isDashboardAttributedSpanName(event.name)) {
|
||||
const terminal = spanIntervalFromTerminal(event, startsById.get(spanKey(event)));
|
||||
if (terminal) {
|
||||
intervals.push(terminal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return intervals;
|
||||
}
|
||||
|
||||
function spanIntervalFromTerminal(event, startEvent) {
|
||||
const endEpochMs = eventEpochMs(event);
|
||||
const durationMs = numberOrNull(event.durationMs) ?? durationBetween(eventEpochMs(startEvent), endEpochMs);
|
||||
const startEpochMs = eventEpochMs(startEvent) ??
|
||||
(endEpochMs !== null && durationMs !== null ? endEpochMs - durationMs : null);
|
||||
if (startEpochMs === null || endEpochMs === null || endEpochMs < startEpochMs) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
name: event.name,
|
||||
type: event.type,
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
durationMs: round(endEpochMs - startEpochMs),
|
||||
rawDurationMs: durationMs,
|
||||
spanId: event.spanId ?? null,
|
||||
phase: event.phase ?? startEvent?.phase ?? null,
|
||||
errorName: event.errorName ?? null,
|
||||
errorMessage: event.errorMessage ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs) {
|
||||
const startEpochMs = Math.max(span.startEpochMs, windowStartEpochMs);
|
||||
const endEpochMs = Math.min(span.endEpochMs, windowEndEpochMs);
|
||||
if (endEpochMs <= startEpochMs) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...span,
|
||||
clippedStartEpochMs: startEpochMs,
|
||||
clippedEndEpochMs: endEpochMs,
|
||||
clippedDurationMs: round(endEpochMs - startEpochMs)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeAttributedSpans(intervals) {
|
||||
const byName = new Map();
|
||||
for (const interval of intervals) {
|
||||
const current = byName.get(interval.name) ?? {
|
||||
name: interval.name,
|
||||
count: 0,
|
||||
errorCount: 0,
|
||||
totalClippedDurationMs: 0,
|
||||
maxClippedDurationMs: null,
|
||||
totalRawDurationMs: 0,
|
||||
maxRawDurationMs: null
|
||||
};
|
||||
current.count += 1;
|
||||
if (interval.type === "span.error") {
|
||||
current.errorCount += 1;
|
||||
}
|
||||
current.totalClippedDurationMs = round(current.totalClippedDurationMs + interval.clippedDurationMs);
|
||||
current.maxClippedDurationMs = maxNullable(current.maxClippedDurationMs, interval.clippedDurationMs);
|
||||
if (typeof interval.rawDurationMs === "number") {
|
||||
current.totalRawDurationMs = round(current.totalRawDurationMs + interval.rawDurationMs);
|
||||
current.maxRawDurationMs = maxNullable(current.maxRawDurationMs, interval.rawDurationMs);
|
||||
}
|
||||
byName.set(interval.name, current);
|
||||
}
|
||||
return [...byName.values()].toSorted((left, right) =>
|
||||
(right.totalClippedDurationMs - left.totalClippedDurationMs) ||
|
||||
left.name.localeCompare(right.name)
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeProviderEvents(events, providerBoundaryEpochMs, activeFinishedAtEpochMs, attribution) {
|
||||
const providerEvents = (events ?? [])
|
||||
.filter((event) => event?.type === "provider.request" || event?.name === "provider.request")
|
||||
.map((event) => {
|
||||
const startEpochMs = numberOrNull(event.receivedAtEpochMs) ?? eventEpochMs(event);
|
||||
const durationMs = numberOrNull(event.durationMs);
|
||||
const endEpochMs = numberOrNull(event.respondedAtEpochMs) ??
|
||||
(startEpochMs !== null && durationMs !== null ? startEpochMs + durationMs : null);
|
||||
return { event, startEpochMs, endEpochMs, durationMs: durationMs ?? durationBetween(startEpochMs, endEpochMs) };
|
||||
})
|
||||
.filter((event) =>
|
||||
event.startEpochMs !== null &&
|
||||
(providerBoundaryEpochMs === null || event.startEpochMs >= providerBoundaryEpochMs) &&
|
||||
(activeFinishedAtEpochMs === null || event.startEpochMs <= activeFinishedAtEpochMs)
|
||||
);
|
||||
const durations = providerEvents.map((event) => event.durationMs).filter(isNumber);
|
||||
return {
|
||||
requestCount: providerEvents.length,
|
||||
timelineTotalDurationMs: round(durations.reduce((sum, value) => sum + value, 0)),
|
||||
timelineMaxDurationMs: durations.length > 0 ? Math.max(...durations) : null,
|
||||
totalDurationMs: numberOrNull(attribution?.providerFinalMs) ??
|
||||
round(durations.reduce((sum, value) => sum + value, 0)),
|
||||
firstByteLatencyMs: numberOrNull(attribution?.firstByteLatencyMs),
|
||||
firstChunkLatencyMs: numberOrNull(attribution?.firstChunkLatencyMs)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeLabeledAttribution(entries, label) {
|
||||
const values = entries.filter((entry) => entry.label === label);
|
||||
return {
|
||||
count: values.length,
|
||||
preProviderMs: summarizeValues(values.map((entry) => entry.window?.durationMs)),
|
||||
knownAttributedMs: summarizeValues(values.map((entry) => entry.knownAttributedMs)),
|
||||
unattributedMs: summarizeValues(values.map((entry) => entry.unattributedMs)),
|
||||
coverageRatio: summarizeValues(values.map((entry) => entry.coverageRatio))
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeSpanMedians(entries) {
|
||||
const byName = new Map();
|
||||
for (const entry of entries) {
|
||||
for (const span of entry.spanSummaries ?? []) {
|
||||
const current = byName.get(span.name) ?? [];
|
||||
current.push(span.totalClippedDurationMs);
|
||||
byName.set(span.name, current);
|
||||
}
|
||||
}
|
||||
return [...byName.entries()]
|
||||
.map(([name, values]) => ({
|
||||
name,
|
||||
medianClippedDurationMs: summarizeValues(values).median,
|
||||
sampleCount: values.length
|
||||
}))
|
||||
.toSorted((left, right) => (right.medianClippedDurationMs ?? 0) - (left.medianClippedDurationMs ?? 0));
|
||||
}
|
||||
|
||||
function summarizeValues(values) {
|
||||
const sorted = values.filter(isNumber).toSorted((left, right) => left - right);
|
||||
if (sorted.length === 0) {
|
||||
return { count: 0, median: null, min: null, max: null };
|
||||
}
|
||||
return {
|
||||
count: sorted.length,
|
||||
median: round(percentile(sorted, 50)),
|
||||
min: round(sorted[0]),
|
||||
max: round(sorted.at(-1))
|
||||
};
|
||||
}
|
||||
|
||||
function unionDuration(intervals) {
|
||||
const sorted = intervals
|
||||
.filter((interval) => isNumber(interval.clippedStartEpochMs) && isNumber(interval.clippedEndEpochMs))
|
||||
.toSorted((left, right) => left.clippedStartEpochMs - right.clippedStartEpochMs);
|
||||
let total = 0;
|
||||
let currentStart = null;
|
||||
let currentEnd = null;
|
||||
for (const interval of sorted) {
|
||||
if (currentStart === null) {
|
||||
currentStart = interval.clippedStartEpochMs;
|
||||
currentEnd = interval.clippedEndEpochMs;
|
||||
continue;
|
||||
}
|
||||
if (interval.clippedStartEpochMs <= currentEnd) {
|
||||
currentEnd = Math.max(currentEnd, interval.clippedEndEpochMs);
|
||||
continue;
|
||||
}
|
||||
total += currentEnd - currentStart;
|
||||
currentStart = interval.clippedStartEpochMs;
|
||||
currentEnd = interval.clippedEndEpochMs;
|
||||
}
|
||||
if (currentStart !== null) {
|
||||
total += currentEnd - currentStart;
|
||||
}
|
||||
return total;
|
||||
return collectAttributedSpanIntervals(events, isDashboardAttributedSpanName);
|
||||
}
|
||||
|
||||
function isDashboardAttributedSpanName(name) {
|
||||
@ -334,86 +55,3 @@ function isDashboardAttributedSpanName(name) {
|
||||
text.startsWith("auto_reply") ||
|
||||
text.startsWith("reply.");
|
||||
}
|
||||
|
||||
function attributionEvents(timelineSummary) {
|
||||
return Array.isArray(timelineSummary?.turnAttributionEvents) && timelineSummary.turnAttributionEvents.length > 0
|
||||
? timelineSummary.turnAttributionEvents
|
||||
: (Array.isArray(timelineSummary?.events) ? timelineSummary.events : []);
|
||||
}
|
||||
|
||||
function timelineArtifacts(timelineSummary) {
|
||||
return unique([
|
||||
...(timelineSummary?.artifacts ?? []),
|
||||
...(timelineSummary?.timelineArtifacts ?? [])
|
||||
].filter(Boolean));
|
||||
}
|
||||
|
||||
function spanKey(event) {
|
||||
return event?.spanId === undefined || event?.spanId === null || String(event.spanId).length === 0
|
||||
? null
|
||||
: String(event.spanId);
|
||||
}
|
||||
|
||||
function eventEpochMs(event) {
|
||||
const direct = numberOrNull(event?.timestampEpochMs ?? event?.timeEpochMs);
|
||||
if (direct !== null) {
|
||||
return direct;
|
||||
}
|
||||
const parsed = Date.parse(event?.timestamp ?? event?.time ?? "");
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function durationBetween(startEpochMs, endEpochMs) {
|
||||
return isNumber(startEpochMs) && isNumber(endEpochMs) && endEpochMs >= startEpochMs
|
||||
? round(endEpochMs - startEpochMs)
|
||||
: null;
|
||||
}
|
||||
|
||||
function percentile(sortedValues, percentileValue) {
|
||||
if (sortedValues.length === 1) {
|
||||
return sortedValues[0];
|
||||
}
|
||||
const position = (percentileValue / 100) * (sortedValues.length - 1);
|
||||
const lower = Math.floor(position);
|
||||
const upper = Math.ceil(position);
|
||||
if (lower === upper) {
|
||||
return sortedValues[lower];
|
||||
}
|
||||
const weight = position - lower;
|
||||
return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
|
||||
}
|
||||
|
||||
function maxNullable(left, right) {
|
||||
if (!isNumber(right)) {
|
||||
return left;
|
||||
}
|
||||
return isNumber(left) ? Math.max(left, right) : right;
|
||||
}
|
||||
|
||||
function numberOrNull(value) {
|
||||
if (value === null || value === undefined || value === "") {
|
||||
return null;
|
||||
}
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? round(number) : null;
|
||||
}
|
||||
|
||||
function isoOrNull(epochMs) {
|
||||
return isNumber(epochMs) ? new Date(epochMs).toISOString() : null;
|
||||
}
|
||||
|
||||
function formatMs(value) {
|
||||
return isNumber(value) ? `${value} ms` : "unknown";
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function isNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 100) / 100;
|
||||
}
|
||||
|
||||
412
src/collectors/pre-provider-attribution.mjs
Normal file
412
src/collectors/pre-provider-attribution.mjs
Normal file
@ -0,0 +1,412 @@
|
||||
export function buildPreProviderAttribution({
|
||||
schemaVersion,
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary,
|
||||
isAttributedSpanName,
|
||||
missingEventsError
|
||||
}) {
|
||||
const artifacts = timelineArtifacts(timelineSummary);
|
||||
const events = attributionEvents(timelineSummary);
|
||||
const providerBoundaryEpochMs = numberOrNull(attribution?.firstProviderRequestAtEpochMs);
|
||||
const windowStartEpochMs = numberOrNull(activeStartedAtEpochMs ?? attribution?.commandStartedAtEpochMs);
|
||||
const activeEndEpochMs = numberOrNull(activeFinishedAtEpochMs ?? attribution?.commandFinishedAtEpochMs);
|
||||
const windowEndEpochMs = providerBoundaryEpochMs;
|
||||
const preProviderMs = numberOrNull(attribution?.preProviderMs) ??
|
||||
durationBetween(windowStartEpochMs, windowEndEpochMs);
|
||||
const base = {
|
||||
schemaVersion,
|
||||
available: false,
|
||||
label: label ?? null,
|
||||
phaseId: phaseId ?? null,
|
||||
timelineAvailable: timelineSummary?.available === true,
|
||||
timelineArtifacts: artifacts,
|
||||
eventCount: events.length,
|
||||
window: {
|
||||
startEpochMs: windowStartEpochMs,
|
||||
startAt: isoOrNull(windowStartEpochMs),
|
||||
endEpochMs: windowEndEpochMs,
|
||||
endAt: isoOrNull(windowEndEpochMs),
|
||||
durationMs: preProviderMs
|
||||
},
|
||||
activeWindow: {
|
||||
startEpochMs: windowStartEpochMs,
|
||||
startAt: isoOrNull(windowStartEpochMs),
|
||||
endEpochMs: activeEndEpochMs,
|
||||
endAt: isoOrNull(activeEndEpochMs),
|
||||
durationMs: durationBetween(windowStartEpochMs, activeEndEpochMs)
|
||||
},
|
||||
providerBoundary: {
|
||||
firstRequestAtEpochMs: providerBoundaryEpochMs,
|
||||
firstRequestAt: isoOrNull(providerBoundaryEpochMs),
|
||||
source: providerBoundaryEpochMs === null ? null : "provider-evidence"
|
||||
},
|
||||
provider: summarizeProviderEvents(events, providerBoundaryEpochMs, activeEndEpochMs, attribution),
|
||||
spanSummaries: [],
|
||||
knownAttributedMs: null,
|
||||
unattributedMs: preProviderMs,
|
||||
coverageRatio: null,
|
||||
error: null
|
||||
};
|
||||
|
||||
if (timelineSummary?.available !== true) {
|
||||
return {
|
||||
...base,
|
||||
error: "OpenClaw diagnostics timeline unavailable"
|
||||
};
|
||||
}
|
||||
if (events.length === 0) {
|
||||
return {
|
||||
...base,
|
||||
error: missingEventsError ?? "timeline contains no pre-provider attribution events"
|
||||
};
|
||||
}
|
||||
if (windowStartEpochMs === null || windowEndEpochMs === null || preProviderMs === null || windowEndEpochMs < windowStartEpochMs) {
|
||||
return {
|
||||
...base,
|
||||
error: "pre-provider window boundary unavailable"
|
||||
};
|
||||
}
|
||||
|
||||
const intervals = attributedSpanIntervals(events, isAttributedSpanName)
|
||||
.map((span) => clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs))
|
||||
.filter(Boolean);
|
||||
const spanSummaries = summarizeAttributedSpans(intervals);
|
||||
const knownAttributedMs = round(unionDuration(intervals));
|
||||
const unattributedMs = round(Math.max(0, preProviderMs - knownAttributedMs));
|
||||
|
||||
return {
|
||||
...base,
|
||||
available: true,
|
||||
spanSummaries,
|
||||
knownAttributedMs,
|
||||
unattributedMs,
|
||||
coverageRatio: preProviderMs > 0 ? round(knownAttributedMs / preProviderMs) : null,
|
||||
error: null
|
||||
};
|
||||
}
|
||||
|
||||
export function summarizePreProviderAttributions({ schemaVersion, turns, fieldName }) {
|
||||
const entries = (turns ?? [])
|
||||
.map((turn) => turn?.[fieldName])
|
||||
.filter(Boolean);
|
||||
const cold = summarizeLabeledAttribution(entries, "cold");
|
||||
const warm = summarizeLabeledAttribution(entries, "warm");
|
||||
return {
|
||||
schemaVersion,
|
||||
available: entries.some((entry) => entry.available === true),
|
||||
count: entries.length,
|
||||
cold,
|
||||
warm,
|
||||
spanMedians: summarizeSpanMedians(entries),
|
||||
timelineArtifacts: unique(entries.flatMap((entry) => entry.timelineArtifacts ?? []))
|
||||
};
|
||||
}
|
||||
|
||||
export function preProviderMarkdownRows({ title, turns, fieldName }) {
|
||||
const attributions = (turns ?? [])
|
||||
.map((turn) => turn?.[fieldName])
|
||||
.filter(Boolean);
|
||||
if (attributions.length === 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
const lines = [
|
||||
`- ${title}:`,
|
||||
"",
|
||||
" | turn | pre-provider | known | unattributed | provider | timeline |",
|
||||
" |---|---:|---:|---:|---:|---|"
|
||||
];
|
||||
for (const item of attributions) {
|
||||
const timeline = item.timelineArtifacts?.[0] ?? (item.timelineAvailable ? "available" : "missing");
|
||||
lines.push(
|
||||
` | ${item.label ?? "turn"} | ${formatMs(item.window?.durationMs)} | ${formatMs(item.knownAttributedMs)} | ${formatMs(item.unattributedMs)} | ${formatMs(item.provider?.totalDurationMs)} | ${timeline} |`
|
||||
);
|
||||
}
|
||||
|
||||
const spanRows = attributions.flatMap((item) =>
|
||||
(item.spanSummaries ?? []).slice(0, 6).map((span) => ({ turn: item.label ?? "turn", ...span }))
|
||||
);
|
||||
if (spanRows.length > 0) {
|
||||
lines.push("");
|
||||
lines.push(" | turn | span | count | errors | clipped | max |");
|
||||
lines.push(" |---|---|---:|---:|---:|---:|");
|
||||
for (const span of spanRows.slice(0, 12)) {
|
||||
lines.push(
|
||||
` | ${span.turn} | \`${span.name}\` | ${span.count} | ${span.errorCount} | ${formatMs(span.totalClippedDurationMs)} | ${formatMs(span.maxClippedDurationMs)} |`
|
||||
);
|
||||
}
|
||||
}
|
||||
return lines;
|
||||
}
|
||||
|
||||
export function attributedSpanIntervals(events, isAttributedSpanName) {
|
||||
const startsById = new Map();
|
||||
const intervals = [];
|
||||
|
||||
for (const event of events ?? []) {
|
||||
if (event?.type === "span.start" && isAttributedSpanName(event.name)) {
|
||||
const key = spanKey(event);
|
||||
if (key) {
|
||||
startsById.set(key, event);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if ((event?.type === "span.end" || event?.type === "span.error") && isAttributedSpanName(event.name)) {
|
||||
const terminal = spanIntervalFromTerminal(event, startsById.get(spanKey(event)));
|
||||
if (terminal) {
|
||||
intervals.push(terminal);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return intervals;
|
||||
}
|
||||
|
||||
function spanIntervalFromTerminal(event, startEvent) {
|
||||
const endEpochMs = eventEpochMs(event);
|
||||
const durationMs = numberOrNull(event.durationMs) ?? durationBetween(eventEpochMs(startEvent), endEpochMs);
|
||||
const startEpochMs = eventEpochMs(startEvent) ??
|
||||
(endEpochMs !== null && durationMs !== null ? endEpochMs - durationMs : null);
|
||||
if (startEpochMs === null || endEpochMs === null || endEpochMs < startEpochMs) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
name: event.name,
|
||||
type: event.type,
|
||||
startEpochMs,
|
||||
endEpochMs,
|
||||
durationMs: round(endEpochMs - startEpochMs),
|
||||
rawDurationMs: durationMs,
|
||||
spanId: event.spanId ?? null,
|
||||
phase: event.phase ?? startEvent?.phase ?? null,
|
||||
errorName: event.errorName ?? null,
|
||||
errorMessage: event.errorMessage ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function clipSpanToWindow(span, windowStartEpochMs, windowEndEpochMs) {
|
||||
const startEpochMs = Math.max(span.startEpochMs, windowStartEpochMs);
|
||||
const endEpochMs = Math.min(span.endEpochMs, windowEndEpochMs);
|
||||
if (endEpochMs <= startEpochMs) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
...span,
|
||||
clippedStartEpochMs: startEpochMs,
|
||||
clippedEndEpochMs: endEpochMs,
|
||||
clippedDurationMs: round(endEpochMs - startEpochMs)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeAttributedSpans(intervals) {
|
||||
const byName = new Map();
|
||||
for (const interval of intervals) {
|
||||
const current = byName.get(interval.name) ?? {
|
||||
name: interval.name,
|
||||
count: 0,
|
||||
errorCount: 0,
|
||||
totalClippedDurationMs: 0,
|
||||
maxClippedDurationMs: null,
|
||||
totalRawDurationMs: 0,
|
||||
maxRawDurationMs: null
|
||||
};
|
||||
current.count += 1;
|
||||
if (interval.type === "span.error") {
|
||||
current.errorCount += 1;
|
||||
}
|
||||
current.totalClippedDurationMs = round(current.totalClippedDurationMs + interval.clippedDurationMs);
|
||||
current.maxClippedDurationMs = maxNullable(current.maxClippedDurationMs, interval.clippedDurationMs);
|
||||
if (typeof interval.rawDurationMs === "number") {
|
||||
current.totalRawDurationMs = round(current.totalRawDurationMs + interval.rawDurationMs);
|
||||
current.maxRawDurationMs = maxNullable(current.maxRawDurationMs, interval.rawDurationMs);
|
||||
}
|
||||
byName.set(interval.name, current);
|
||||
}
|
||||
return [...byName.values()].toSorted((left, right) =>
|
||||
(right.totalClippedDurationMs - left.totalClippedDurationMs) ||
|
||||
left.name.localeCompare(right.name)
|
||||
);
|
||||
}
|
||||
|
||||
function summarizeProviderEvents(events, providerBoundaryEpochMs, activeFinishedAtEpochMs, attribution) {
|
||||
const providerEvents = (events ?? [])
|
||||
.filter((event) => event?.type === "provider.request" || event?.name === "provider.request")
|
||||
.map((event) => {
|
||||
const startEpochMs = numberOrNull(event.receivedAtEpochMs) ?? eventEpochMs(event);
|
||||
const durationMs = numberOrNull(event.durationMs);
|
||||
const endEpochMs = numberOrNull(event.respondedAtEpochMs) ??
|
||||
(startEpochMs !== null && durationMs !== null ? startEpochMs + durationMs : null);
|
||||
return { event, startEpochMs, endEpochMs, durationMs: durationMs ?? durationBetween(startEpochMs, endEpochMs) };
|
||||
})
|
||||
.filter((event) =>
|
||||
event.startEpochMs !== null &&
|
||||
(providerBoundaryEpochMs === null || event.startEpochMs >= providerBoundaryEpochMs) &&
|
||||
(activeFinishedAtEpochMs === null || event.startEpochMs <= activeFinishedAtEpochMs)
|
||||
);
|
||||
const durations = providerEvents.map((event) => event.durationMs).filter(isNumber);
|
||||
return {
|
||||
requestCount: providerEvents.length,
|
||||
timelineTotalDurationMs: round(durations.reduce((sum, value) => sum + value, 0)),
|
||||
timelineMaxDurationMs: durations.length > 0 ? Math.max(...durations) : null,
|
||||
totalDurationMs: numberOrNull(attribution?.providerFinalMs) ??
|
||||
round(durations.reduce((sum, value) => sum + value, 0)),
|
||||
firstByteLatencyMs: numberOrNull(attribution?.firstByteLatencyMs),
|
||||
firstChunkLatencyMs: numberOrNull(attribution?.firstChunkLatencyMs)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeLabeledAttribution(entries, label) {
|
||||
const values = entries.filter((entry) => entry.label === label);
|
||||
return {
|
||||
count: values.length,
|
||||
preProviderMs: summarizeValues(values.map((entry) => entry.window?.durationMs)),
|
||||
knownAttributedMs: summarizeValues(values.map((entry) => entry.knownAttributedMs)),
|
||||
unattributedMs: summarizeValues(values.map((entry) => entry.unattributedMs)),
|
||||
coverageRatio: summarizeValues(values.map((entry) => entry.coverageRatio))
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeSpanMedians(entries) {
|
||||
const byName = new Map();
|
||||
for (const entry of entries) {
|
||||
for (const span of entry.spanSummaries ?? []) {
|
||||
const current = byName.get(span.name) ?? [];
|
||||
current.push(span.totalClippedDurationMs);
|
||||
byName.set(span.name, current);
|
||||
}
|
||||
}
|
||||
return [...byName.entries()]
|
||||
.map(([name, values]) => ({
|
||||
name,
|
||||
medianClippedDurationMs: summarizeValues(values).median,
|
||||
sampleCount: values.length
|
||||
}))
|
||||
.toSorted((left, right) => (right.medianClippedDurationMs ?? 0) - (left.medianClippedDurationMs ?? 0));
|
||||
}
|
||||
|
||||
function summarizeValues(values) {
|
||||
const sorted = values.filter(isNumber).toSorted((left, right) => left - right);
|
||||
if (sorted.length === 0) {
|
||||
return { count: 0, median: null, min: null, max: null };
|
||||
}
|
||||
return {
|
||||
count: sorted.length,
|
||||
median: round(percentile(sorted, 50)),
|
||||
min: round(sorted[0]),
|
||||
max: round(sorted.at(-1))
|
||||
};
|
||||
}
|
||||
|
||||
function unionDuration(intervals) {
|
||||
const sorted = intervals
|
||||
.filter((interval) => isNumber(interval.clippedStartEpochMs) && isNumber(interval.clippedEndEpochMs))
|
||||
.toSorted((left, right) => left.clippedStartEpochMs - right.clippedStartEpochMs);
|
||||
let total = 0;
|
||||
let currentStart = null;
|
||||
let currentEnd = null;
|
||||
for (const interval of sorted) {
|
||||
if (currentStart === null) {
|
||||
currentStart = interval.clippedStartEpochMs;
|
||||
currentEnd = interval.clippedEndEpochMs;
|
||||
continue;
|
||||
}
|
||||
if (interval.clippedStartEpochMs <= currentEnd) {
|
||||
currentEnd = Math.max(currentEnd, interval.clippedEndEpochMs);
|
||||
continue;
|
||||
}
|
||||
total += currentEnd - currentStart;
|
||||
currentStart = interval.clippedStartEpochMs;
|
||||
currentEnd = interval.clippedEndEpochMs;
|
||||
}
|
||||
if (currentStart !== null) {
|
||||
total += currentEnd - currentStart;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function attributionEvents(timelineSummary) {
|
||||
return Array.isArray(timelineSummary?.turnAttributionEvents) && timelineSummary.turnAttributionEvents.length > 0
|
||||
? timelineSummary.turnAttributionEvents
|
||||
: (Array.isArray(timelineSummary?.events) ? timelineSummary.events : []);
|
||||
}
|
||||
|
||||
function timelineArtifacts(timelineSummary) {
|
||||
return unique([
|
||||
...(timelineSummary?.artifacts ?? []),
|
||||
...(timelineSummary?.timelineArtifacts ?? [])
|
||||
].filter(Boolean));
|
||||
}
|
||||
|
||||
function spanKey(event) {
|
||||
return event?.spanId === undefined || event?.spanId === null || String(event.spanId).length === 0
|
||||
? null
|
||||
: String(event.spanId);
|
||||
}
|
||||
|
||||
function eventEpochMs(event) {
|
||||
const direct = numberOrNull(event?.timestampEpochMs ?? event?.timeEpochMs);
|
||||
if (direct !== null) {
|
||||
return direct;
|
||||
}
|
||||
const parsed = Date.parse(event?.timestamp ?? event?.time ?? "");
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
function durationBetween(startEpochMs, endEpochMs) {
|
||||
return isNumber(startEpochMs) && isNumber(endEpochMs) && endEpochMs >= startEpochMs
|
||||
? round(endEpochMs - startEpochMs)
|
||||
: null;
|
||||
}
|
||||
|
||||
function percentile(sortedValues, percentileValue) {
|
||||
if (sortedValues.length === 1) {
|
||||
return sortedValues[0];
|
||||
}
|
||||
const position = (percentileValue / 100) * (sortedValues.length - 1);
|
||||
const lower = Math.floor(position);
|
||||
const upper = Math.ceil(position);
|
||||
if (lower === upper) {
|
||||
return sortedValues[lower];
|
||||
}
|
||||
const weight = position - lower;
|
||||
return sortedValues[lower] * (1 - weight) + sortedValues[upper] * weight;
|
||||
}
|
||||
|
||||
function maxNullable(left, right) {
|
||||
if (!isNumber(right)) {
|
||||
return left;
|
||||
}
|
||||
return isNumber(left) ? Math.max(left, right) : right;
|
||||
}
|
||||
|
||||
function numberOrNull(value) {
|
||||
if (value === null || value === undefined || value === "") {
|
||||
return null;
|
||||
}
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? round(number) : null;
|
||||
}
|
||||
|
||||
function isoOrNull(epochMs) {
|
||||
return isNumber(epochMs) ? new Date(epochMs).toISOString() : null;
|
||||
}
|
||||
|
||||
function formatMs(value) {
|
||||
return isNumber(value) ? `${value} ms` : "unknown";
|
||||
}
|
||||
|
||||
function unique(values) {
|
||||
return [...new Set(values)];
|
||||
}
|
||||
|
||||
function isNumber(value) {
|
||||
return typeof value === "number" && Number.isFinite(value);
|
||||
}
|
||||
|
||||
function round(value) {
|
||||
return Math.round(value * 100) / 100;
|
||||
}
|
||||
@ -447,9 +447,18 @@ function isTurnAttributionEvent(event) {
|
||||
}
|
||||
return event.name === "plugins.metadata.scan" ||
|
||||
event.name === "provider.request" ||
|
||||
event.name === "agent.prepare" ||
|
||||
event.name === "agent.turn" ||
|
||||
event.name === "agent.cleanup" ||
|
||||
event.name === "runtimeDeps.stage" ||
|
||||
event.name === "channel.capabilities" ||
|
||||
event.name === "models.catalog" ||
|
||||
event.name === "auto_reply" ||
|
||||
event.name.startsWith("auto_reply.") ||
|
||||
event.name.startsWith("gateway.chat_send") ||
|
||||
event.name.startsWith("models.catalog.") ||
|
||||
event.name.startsWith("models.discovery") ||
|
||||
event.name.startsWith("channel.plugin.") ||
|
||||
event.name.startsWith("reply.");
|
||||
}
|
||||
|
||||
|
||||
@ -1,4 +1,8 @@
|
||||
import { buildAgentTurnBreakdown } from "./collectors/agent-turns.mjs";
|
||||
import {
|
||||
buildAgentCliPreProviderAttribution,
|
||||
summarizeAgentCliPreProviderAttributions
|
||||
} from "./collectors/agent-cli-attribution.mjs";
|
||||
import {
|
||||
buildDashboardPreProviderAttribution,
|
||||
summarizeDashboardPreProviderAttributions
|
||||
@ -82,6 +86,11 @@ export function evaluateRecord(record, scenario, options = {}) {
|
||||
const agentTurnStats = summarizeAgentTurnStats(agentTurns);
|
||||
const agentTurnDiagnostics = summarizeAgentTurnDiagnostics(agentTurns);
|
||||
const dashboardPreProviderAttribution = summarizeDashboardPreProviderAttributions(agentTurns);
|
||||
const agentCliPreProviderAttribution = summarizeAgentCliPreProviderAttributions(agentTurns);
|
||||
const turnPreProviderAttribution = preferredPreProviderAttributionSummary(
|
||||
dashboardPreProviderAttribution,
|
||||
agentCliPreProviderAttribution
|
||||
);
|
||||
const agentTurnMs = maxTurnDuration(agentTurns);
|
||||
const agentResponseOk = agentTurns.length === 0 ? null : agentTurns.every((turn) => turn.responseOk === true);
|
||||
const agentProviderSimulation = evaluateProviderSimulation({ turns: agentTurns, scenario, record, thresholds });
|
||||
@ -760,12 +769,13 @@ export function evaluateRecord(record, scenario, options = {}) {
|
||||
agentSessionPollCount: agentTurnDiagnostics.sessionPollCount,
|
||||
agentSessionPollErrorCount: agentTurnDiagnostics.sessionPollErrorCount,
|
||||
dashboardPreProviderAttribution,
|
||||
coldPreProviderAttributedMs: dashboardPreProviderAttribution.cold.knownAttributedMs.median,
|
||||
warmPreProviderAttributedMs: dashboardPreProviderAttribution.warm.knownAttributedMs.median,
|
||||
coldPreProviderUnattributedMs: dashboardPreProviderAttribution.cold.unattributedMs.median,
|
||||
warmPreProviderUnattributedMs: dashboardPreProviderAttribution.warm.unattributedMs.median,
|
||||
coldPreProviderAttributionCoverage: dashboardPreProviderAttribution.cold.coverageRatio.median,
|
||||
warmPreProviderAttributionCoverage: dashboardPreProviderAttribution.warm.coverageRatio.median,
|
||||
agentCliPreProviderAttribution,
|
||||
coldPreProviderAttributedMs: turnPreProviderAttribution.cold.knownAttributedMs.median,
|
||||
warmPreProviderAttributedMs: turnPreProviderAttribution.warm.knownAttributedMs.median,
|
||||
coldPreProviderUnattributedMs: turnPreProviderAttribution.cold.unattributedMs.median,
|
||||
warmPreProviderUnattributedMs: turnPreProviderAttribution.warm.unattributedMs.median,
|
||||
coldPreProviderAttributionCoverage: turnPreProviderAttribution.cold.coverageRatio.median,
|
||||
warmPreProviderAttributionCoverage: turnPreProviderAttribution.warm.coverageRatio.median,
|
||||
coldAgentTurnMs: coldAgentTurn?.totalTurnMs ?? null,
|
||||
warmAgentTurnMs: warmAgentTurn?.totalTurnMs ?? null,
|
||||
agentColdWarmDeltaMs: delta(coldAgentTurn?.totalTurnMs, warmAgentTurn?.totalTurnMs),
|
||||
@ -981,6 +991,7 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
: null;
|
||||
const expectedFailureObserved = expectedFailure === true && result.status === 0 && result.timedOut !== true;
|
||||
const normalResponseOk = result.status === 0 && result.timedOut !== true && response.usable === true && (expectedTextPresent !== false);
|
||||
const isAgentCliTurn = isAgentCliMessageCommand(result.command);
|
||||
const phaseBreakdown = buildAgentTurnBreakdown({ result: timingResult, attribution, timelineSummary, logSummary });
|
||||
const turnDiagnostics = summarizeActiveTurnDiagnostics({
|
||||
timelineSummary,
|
||||
@ -998,6 +1009,16 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
timelineSummary
|
||||
})
|
||||
: null;
|
||||
const agentCliPreProviderAttribution = isAgentCliTurn
|
||||
? buildAgentCliPreProviderAttribution({
|
||||
label: agentTurnLabel(phase.id, index),
|
||||
phaseId: phase.id,
|
||||
activeStartedAtEpochMs: timingResult.startedAtEpochMs,
|
||||
activeFinishedAtEpochMs: timingResult.finishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary
|
||||
})
|
||||
: null;
|
||||
turns.push({
|
||||
schemaVersion: "kova.agentTurnEvidence.v1",
|
||||
index,
|
||||
@ -1045,6 +1066,7 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
phaseBreakdown,
|
||||
turnDiagnostics,
|
||||
dashboardPreProviderAttribution,
|
||||
agentCliPreProviderAttribution,
|
||||
metadataScanCount: turnDiagnostics.metadataScan.count,
|
||||
metadataScanTotalMs: turnDiagnostics.metadataScan.totalDurationMs,
|
||||
metadataScanMaxMs: turnDiagnostics.metadataScan.maxDurationMs,
|
||||
@ -1063,6 +1085,10 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
return turns;
|
||||
}
|
||||
|
||||
function preferredPreProviderAttributionSummary(...summaries) {
|
||||
return summaries.find((summary) => summary?.count > 0) ?? summaries[0];
|
||||
}
|
||||
|
||||
function extractGatewaySessionTurn(result) {
|
||||
if (!result?.command?.includes("run-dashboard-session-send-turn.mjs")) {
|
||||
return null;
|
||||
@ -3255,6 +3281,10 @@ function isAgentMessageCommand(command) {
|
||||
command.includes("run-openai-compatible-turn.mjs");
|
||||
}
|
||||
|
||||
function isAgentCliMessageCommand(command) {
|
||||
return command.includes(" -- agent ") && command.includes("--message");
|
||||
}
|
||||
|
||||
function extractAgentResponse(result) {
|
||||
if (result.status !== 0 || result.timedOut) {
|
||||
return { usable: false, text: null };
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import { summarizeAgentTurnBreakdownForMarkdown } from "../collectors/agent-turns.mjs";
|
||||
import { agentCliPreProviderMarkdownRows } from "../collectors/agent-cli-attribution.mjs";
|
||||
import { dashboardPreProviderMarkdownRows } from "../collectors/dashboard-turn-attribution.mjs";
|
||||
import { healthTotalFailures } from "../health.mjs";
|
||||
|
||||
@ -186,6 +187,9 @@ export function renderMarkdownReport(report) {
|
||||
if (record.measurements.dashboardPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- Dashboard pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
|
||||
}
|
||||
if (record.measurements.agentCliPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- Agent CLI pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
|
||||
}
|
||||
lines.push(`- Agent provider final: cold ${record.measurements.coldProviderFinalMs ?? "unknown"} ms; warm ${record.measurements.warmProviderFinalMs ?? "unknown"} ms`);
|
||||
lines.push(`- Agent turn stats: count ${record.measurements.agentTurnCount}; p95 ${record.measurements.agentTurnP95Ms ?? "unknown"} ms; max ${record.measurements.agentTurnMaxMs ?? "unknown"} ms; pre-provider p95 ${record.measurements.agentPreProviderP95Ms ?? "unknown"} ms`);
|
||||
lines.push(`- Agent active-turn diagnostics: metadata scans ${record.measurements.agentMetadataScanCount ?? "unknown"} (${record.measurements.agentMetadataScanTotalMs ?? "unknown"} ms total); event-loop max ${record.measurements.agentEventLoopMaxMs ?? "unknown"} ms; session polls ${record.measurements.agentSessionPollCount ?? "unknown"} (${record.measurements.agentSessionPollErrorCount ?? "unknown"} errors)`);
|
||||
@ -244,6 +248,7 @@ export function renderMarkdownReport(report) {
|
||||
}
|
||||
}
|
||||
lines.push(...dashboardPreProviderMarkdownRows(record.measurements.agentTurns));
|
||||
lines.push(...agentCliPreProviderMarkdownRows(record.measurements.agentTurns));
|
||||
}
|
||||
lines.push(`- Profiling: ${record.profiling?.enabled ? "enabled" : "off"} (${record.profiling?.interpretation ?? "unknown"})`);
|
||||
lines.push(`- V8 reports / heap snapshots: ${record.measurements.v8ReportCount ?? "unknown"} / ${record.measurements.heapSnapshotCount ?? "unknown"}`);
|
||||
@ -736,6 +741,7 @@ function summarizeMeasurements(measurements) {
|
||||
agentSessionPollCount: measurements.agentSessionPollCount ?? null,
|
||||
agentSessionPollErrorCount: measurements.agentSessionPollErrorCount ?? null,
|
||||
dashboardPreProviderAttribution: measurements.dashboardPreProviderAttribution ?? null,
|
||||
agentCliPreProviderAttribution: measurements.agentCliPreProviderAttribution ?? null,
|
||||
coldPreProviderAttributedMs: measurements.coldPreProviderAttributedMs ?? null,
|
||||
warmPreProviderAttributedMs: measurements.warmPreProviderAttributedMs ?? null,
|
||||
coldPreProviderUnattributedMs: measurements.coldPreProviderUnattributedMs ?? null,
|
||||
@ -1203,6 +1209,9 @@ function pushMeasurementBrief(lines, measurements, { compact }) {
|
||||
if (measurements.dashboardPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- dashboard attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
|
||||
}
|
||||
if (measurements.agentCliPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- agent CLI attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
|
||||
}
|
||||
lines.push(`- plugins/runtime: missing deps ${measurements.missingDependencyErrors ?? "unknown"}; plugin failures ${measurements.pluginLoadFailures ?? "unknown"}; runtime deps ${valueMs(measurements.runtimeDepsStagingMs)}${runtimeDepsPluginText(measurements)}; warm restages ${measurements.warmRuntimeDepsRestageCount ?? "unknown"}; warm reuse ${measurements.runtimeDepsWarmReuseOk ?? "unknown"}`);
|
||||
|
||||
if (!compact || hasDiagnosticSignal(measurements)) {
|
||||
|
||||
@ -31,6 +31,7 @@ import {
|
||||
buildAgentTurnBreakdown,
|
||||
summarizeAgentTurnBreakdownForMarkdown
|
||||
} from "./collectors/agent-turns.mjs";
|
||||
import { buildAgentCliPreProviderAttribution } from "./collectors/agent-cli-attribution.mjs";
|
||||
import {
|
||||
attributedSpanIntervals,
|
||||
buildDashboardPreProviderAttribution
|
||||
@ -378,6 +379,7 @@ export async function runSelfCheck(flags = {}) {
|
||||
checks.push(agentTurnBreakdownCheck());
|
||||
checks.push(gatewaySessionTurnEvaluationCheck());
|
||||
checks.push(dashboardPreProviderAttributionCheck());
|
||||
checks.push(agentCliPreProviderAttributionCheck());
|
||||
checks.push(await mockProviderBehaviorCheck(tmp));
|
||||
checks.push(providerFailureEvaluationCheck());
|
||||
checks.push(agentColdWarmEvaluationCheck());
|
||||
@ -2413,6 +2415,105 @@ function dashboardPreProviderAttributionCheck() {
|
||||
}
|
||||
}
|
||||
|
||||
function agentCliPreProviderAttributionCheck() {
|
||||
try {
|
||||
const base = 1777536000000;
|
||||
const timelineText = [
|
||||
timelineEvent({ type: "span.start", name: "agent.turn", timestamp: base + 1000, spanId: "cold-turn" }),
|
||||
timelineEvent({ type: "span.start", name: "agent.prepare", timestamp: base + 1020, spanId: "cold-prepare" }),
|
||||
timelineEvent({ type: "span.end", name: "agent.prepare", timestamp: base + 1120, spanId: "cold-prepare", durationMs: 100 }),
|
||||
timelineEvent({ type: "span.start", name: "models.catalog.gateway", timestamp: base + 1080, spanId: "cold-models" }),
|
||||
timelineEvent({ type: "span.end", name: "models.catalog.gateway", timestamp: base + 1180, spanId: "cold-models", durationMs: 100 }),
|
||||
timelineEvent({ type: "span.start", name: "channel.plugin.load", timestamp: base + 1150, spanId: "cold-channel" }),
|
||||
timelineEvent({ type: "span.error", name: "channel.plugin.load", timestamp: base + 1170, spanId: "cold-channel", durationMs: 20, errorName: "SyntheticError" }),
|
||||
timelineEvent({ type: "span.end", name: "plugins.metadata.scan", timestamp: base + 1190, spanId: "cold-scan", durationMs: 30 }),
|
||||
timelineEvent({ type: "provider.request", name: "provider.request", timestamp: base + 1200, receivedAtEpochMs: base + 1200, respondedAtEpochMs: base + 1700, durationMs: 500 }),
|
||||
timelineEvent({ type: "span.end", name: "agent.turn", timestamp: base + 1900, spanId: "cold-turn", durationMs: 900 }),
|
||||
timelineEvent({ type: "span.start", name: "runtimeDeps.stage", timestamp: base + 11020, spanId: "warm-runtime" }),
|
||||
timelineEvent({ type: "span.end", name: "runtimeDeps.stage", timestamp: base + 11070, spanId: "warm-runtime", durationMs: 50 }),
|
||||
timelineEvent({ type: "span.start", name: "channel.capabilities", timestamp: base + 11080, spanId: "warm-channel" }),
|
||||
timelineEvent({ type: "span.end", name: "channel.capabilities", timestamp: base + 11110, spanId: "warm-channel", durationMs: 30 }),
|
||||
timelineEvent({ type: "provider.request", name: "provider.request", timestamp: base + 11200, receivedAtEpochMs: base + 11200, respondedAtEpochMs: base + 11500, durationMs: 300 }),
|
||||
timelineEvent({ type: "eventLoop.sample", name: "eventLoop.sample", timestamp: base + 11250, maxMs: 6 })
|
||||
].join("\n");
|
||||
const parsed = parseTimelineText(timelineText);
|
||||
assertEqual(parsed.turnAttributionEvents.length, 16, "agent CLI turn attribution events retained");
|
||||
|
||||
const coldAttribution = buildAgentCliPreProviderAttribution({
|
||||
label: "cold",
|
||||
phaseId: "cold-agent-turn",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 1900,
|
||||
attribution: {
|
||||
firstProviderRequestAtEpochMs: base + 1200,
|
||||
preProviderMs: 200,
|
||||
providerFinalMs: 500
|
||||
},
|
||||
timelineSummary: {
|
||||
available: true,
|
||||
turnAttributionEvents: parsed.turnAttributionEvents,
|
||||
artifacts: ["/tmp/kova/openclaw/timeline.jsonl"]
|
||||
}
|
||||
});
|
||||
assertEqual(coldAttribution.available, true, "agent CLI cold attribution available");
|
||||
assertEqual(coldAttribution.knownAttributedMs, 170, "agent CLI overlap-safe cold known attribution");
|
||||
assertEqual(coldAttribution.unattributedMs, 30, "agent CLI cold unattributed remainder");
|
||||
assertEqual(coldAttribution.spanSummaries.some((span) => span.name === "agent.turn"), false, "agent.turn parent span is not counted as pre-provider work");
|
||||
assertEqual(coldAttribution.spanSummaries.find((span) => span.name === "channel.plugin.load")?.errorCount, 1, "agent CLI error span summary");
|
||||
|
||||
const missingAttribution = buildAgentCliPreProviderAttribution({
|
||||
label: "cold",
|
||||
phaseId: "cold-agent-turn",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 1900,
|
||||
attribution: { firstProviderRequestAtEpochMs: base + 1200, preProviderMs: 200 },
|
||||
timelineSummary: { available: false, artifacts: [] }
|
||||
});
|
||||
assertEqual(missingAttribution.available, false, "agent CLI missing timeline unavailable");
|
||||
assertEqual(missingAttribution.unattributedMs, 200, "agent CLI missing timeline preserves full remainder");
|
||||
|
||||
const record = syntheticAgentCliRecord({ base, timeline: parsed });
|
||||
evaluateRecord(record, {
|
||||
id: "agent-cold-warm-message",
|
||||
agent: { expectedText: "KOVA_AGENT_OK" },
|
||||
thresholds: { agentTurnMs: 2000, coldAgentTurnMs: 2000, warmAgentTurnMs: 1000 }
|
||||
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
|
||||
assertEqual(record.measurements.agentCliPreProviderAttribution.count, 2, "record agent CLI attribution count");
|
||||
assertEqual(record.measurements.dashboardPreProviderAttribution.count, 0, "record dashboard attribution stays empty for CLI turns");
|
||||
assertEqual(record.measurements.coldPreProviderAttributedMs, 170, "record agent CLI cold attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderAttributedMs, 80, "record agent CLI warm attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderUnattributedMs, 120, "record agent CLI warm unattributed metric");
|
||||
assertEqual(record.measurements.agentTurns[0].agentCliPreProviderAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "record agent CLI timeline artifact");
|
||||
|
||||
const rendered = renderMarkdownReport({
|
||||
generatedAt: "2026-05-01T00:00:00.000Z",
|
||||
runId: "self-check-agent-cli-pre-provider",
|
||||
mode: "self-check",
|
||||
target: "runtime:stable",
|
||||
platform: { os: "test", release: "test", arch: "test", node: "test" },
|
||||
records: [record],
|
||||
summary: { statuses: { PASS: 1 } }
|
||||
});
|
||||
assertEqual(rendered.includes("Agent CLI pre-provider attribution:"), true, "markdown includes agent CLI attribution table");
|
||||
assertEqual(rendered.includes("`channel.plugin.load`"), true, "markdown includes agent CLI span table");
|
||||
|
||||
return {
|
||||
id: "agent-cli-pre-provider-attribution",
|
||||
status: "PASS",
|
||||
command: "evaluate synthetic agent CLI pre-provider timeline attribution",
|
||||
durationMs: 0
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "agent-cli-pre-provider-attribution",
|
||||
status: "FAIL",
|
||||
command: "evaluate synthetic agent CLI pre-provider timeline attribution",
|
||||
durationMs: 0,
|
||||
message: error.message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function timelineEvent(event) {
|
||||
const timestamp = typeof event.timestamp === "number" ? new Date(event.timestamp).toISOString() : event.timestamp;
|
||||
return JSON.stringify({
|
||||
@ -2529,6 +2630,91 @@ function syntheticDashboardSessionRecord({ base, timeline }) {
|
||||
};
|
||||
}
|
||||
|
||||
function syntheticAgentCliRecord({ base, timeline }) {
|
||||
return {
|
||||
scenario: "agent-cold-warm-message",
|
||||
surface: "agent-cold-warm-message",
|
||||
title: "Agent CLI cold/warm",
|
||||
status: "PASS",
|
||||
cleanup: "done",
|
||||
auth: { mode: "mock" },
|
||||
phases: [
|
||||
syntheticAgentCliTurnPhase({
|
||||
id: "cold-agent-turn",
|
||||
startedAtEpochMs: base + 1000,
|
||||
finishedAtEpochMs: base + 1900
|
||||
}),
|
||||
syntheticAgentCliTurnPhase({
|
||||
id: "warm-agent-turn",
|
||||
startedAtEpochMs: base + 11000,
|
||||
finishedAtEpochMs: base + 11600
|
||||
})
|
||||
],
|
||||
providerEvidence: {
|
||||
available: true,
|
||||
requestCount: 2,
|
||||
requests: [
|
||||
{
|
||||
requestId: "cold-provider",
|
||||
receivedAt: new Date(base + 1200).toISOString(),
|
||||
receivedAtEpochMs: base + 1200,
|
||||
respondedAt: new Date(base + 1700).toISOString(),
|
||||
respondedAtEpochMs: base + 1700,
|
||||
firstByteLatencyMs: 20,
|
||||
firstChunkLatencyMs: 25,
|
||||
route: "/v1/responses",
|
||||
model: "gpt-5.5",
|
||||
status: 200
|
||||
},
|
||||
{
|
||||
requestId: "warm-provider",
|
||||
receivedAt: new Date(base + 11200).toISOString(),
|
||||
receivedAtEpochMs: base + 11200,
|
||||
respondedAt: new Date(base + 11500).toISOString(),
|
||||
respondedAtEpochMs: base + 11500,
|
||||
firstByteLatencyMs: 18,
|
||||
firstChunkLatencyMs: 20,
|
||||
route: "/v1/responses",
|
||||
model: "gpt-5.5",
|
||||
status: 200
|
||||
}
|
||||
]
|
||||
},
|
||||
finalMetrics: {
|
||||
service: { gatewayState: "running" },
|
||||
logs: zeroLogMetrics(),
|
||||
timeline: {
|
||||
...timeline,
|
||||
artifacts: ["/tmp/kova/openclaw/timeline.jsonl"]
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function syntheticAgentCliTurnPhase({ id, startedAtEpochMs, finishedAtEpochMs }) {
|
||||
const command = "ocm @kova -- agent --local --agent main --session-id kova-agent-cold-warm --message hi --json";
|
||||
return {
|
||||
id,
|
||||
title: id,
|
||||
intent: "Synthetic agent CLI turn",
|
||||
commands: [command],
|
||||
evidence: [],
|
||||
results: [{
|
||||
command,
|
||||
status: 0,
|
||||
timedOut: false,
|
||||
startedAt: new Date(startedAtEpochMs).toISOString(),
|
||||
startedAtEpochMs,
|
||||
finishedAt: new Date(finishedAtEpochMs).toISOString(),
|
||||
finishedAtEpochMs,
|
||||
durationMs: finishedAtEpochMs - startedAtEpochMs,
|
||||
stdout: "{\"finalAssistantVisibleText\":\"KOVA_AGENT_OK\"}",
|
||||
stderr: ""
|
||||
}],
|
||||
metrics: { logs: zeroLogMetrics(), health: { ok: true } }
|
||||
};
|
||||
}
|
||||
|
||||
function syntheticDashboardTurnPhase({ id, command, startedAtEpochMs, finishedAtEpochMs, payload }) {
|
||||
return {
|
||||
id,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user