refactor: rename gateway session turn surface
This commit is contained in:
Shakker
parent
e18582996a
commit
8f4e1511d2
@ -51,7 +51,7 @@ release-runtime-startup/fresh
|
||||
build-tooling peak RSS: 2409 MB
|
||||
missing dependency: @homebridge/ciao from bundled bonjour
|
||||
|
||||
dashboard-session-send-turn/mock-openai-provider
|
||||
gateway-session-send-turn/mock-openai-provider
|
||||
agent turn: 9.2s
|
||||
pre-provider OpenClaw time: 8.9s
|
||||
provider time: 1ms
|
||||
@ -60,7 +60,7 @@ dashboard-session-send-turn/mock-openai-provider
|
||||
health: gateway had post-command health failures
|
||||
|
||||
Fixer brief:
|
||||
Area: plugins/runtime deps, dashboard session agent path
|
||||
Area: plugins/runtime deps, Gateway session agent path
|
||||
Why it matters: users can start successfully but hit plugin dependency errors
|
||||
and slow first replies unrelated to provider latency.
|
||||
```
|
||||
@ -122,7 +122,7 @@ durable artifact bundle for failed gates.
|
||||
```sh
|
||||
node bin/kova.mjs run \
|
||||
--target local-build:/path/to/openclaw \
|
||||
--scenario dashboard-session-send-turn \
|
||||
--scenario gateway-session-send-turn \
|
||||
--execute \
|
||||
--json
|
||||
```
|
||||
|
||||
@ -245,7 +245,7 @@ Gateway/session turn entries include:
|
||||
"schemaVersion": "kova.gatewaySessionTurn.v1",
|
||||
"method": "sessions.send",
|
||||
"createSession": true,
|
||||
"sessionKey": "kova-dashboard-session-send",
|
||||
"sessionKey": "kova-gateway-session-send",
|
||||
"activeStartedAtEpochMs": 1777536000000,
|
||||
"activeFinishedAtEpochMs": 1777536001260,
|
||||
"activeTurnMs": 1260,
|
||||
@ -275,7 +275,7 @@ Gateway/session turn entries include:
|
||||
}
|
||||
```
|
||||
|
||||
Dashboard session turns also include pre-provider attribution when an OpenClaw
|
||||
Gateway session turns also include pre-provider attribution when an OpenClaw
|
||||
diagnostics timeline is available. Kova clips `gateway.chat_send*`,
|
||||
`auto_reply*`, and `reply.*` spans to the active `sessions.send` pre-provider
|
||||
window and reports the unioned known time so overlapping spans are not counted
|
||||
@ -283,8 +283,8 @@ twice. Provider work remains separate.
|
||||
|
||||
```json
|
||||
{
|
||||
"dashboardPreProviderAttribution": {
|
||||
"schemaVersion": "kova.dashboardPreProviderAttribution.v1",
|
||||
"gatewaySessionPreProviderAttribution": {
|
||||
"schemaVersion": "kova.gatewaySessionPreProviderAttribution.v1",
|
||||
"available": true,
|
||||
"label": "cold",
|
||||
"timelineArtifacts": ["/tmp/kova/openclaw/timeline.jsonl"],
|
||||
@ -315,7 +315,7 @@ twice. Provider work remains separate.
|
||||
```
|
||||
|
||||
Repeat summaries expose machine-readable medians at
|
||||
`records[*].measurements.dashboardPreProviderAttribution` plus flat comparison
|
||||
`records[*].measurements.gatewaySessionPreProviderAttribution` plus flat comparison
|
||||
metrics such as `coldPreProviderAttributedMs`,
|
||||
`coldPreProviderUnattributedMs`, `warmPreProviderAttributedMs`, and
|
||||
`warmPreProviderUnattributedMs`.
|
||||
|
||||
@ -2,6 +2,6 @@
|
||||
"id": "dashboard-cli",
|
||||
"title": "Dashboard CLI",
|
||||
"description": "OpenClaw dashboard command paths that produce or validate dashboard URLs and websocket configuration.",
|
||||
"commandPatterns": ["dashboard", "dashboard --no-open", "run-dashboard-session-send-turn.mjs", "sessions.send"],
|
||||
"processPatterns": ["openclaw.*dashboard", "node\\s+.*run-dashboard-session-send-turn\\.mjs"]
|
||||
"commandPatterns": ["dashboard", "dashboard --no-open"],
|
||||
"processPatterns": ["openclaw.*dashboard"]
|
||||
}
|
||||
|
||||
7
process-roles/gateway-session-client.json
Normal file
7
process-roles/gateway-session-client.json
Normal file
@ -0,0 +1,7 @@
|
||||
{
|
||||
"id": "gateway-session-client",
|
||||
"title": "Gateway Session Client",
|
||||
"description": "Kova helper process that drives the shared Gateway sessions.create, sessions.send, and chat.history path directly over Gateway RPC.",
|
||||
"commandPatterns": ["run-gateway-session-send-turn.mjs", "sessions.create", "sessions.send", "chat.history"],
|
||||
"processPatterns": ["node\\s+.*run-gateway-session-send-turn\\.mjs"]
|
||||
}
|
||||
@ -78,7 +78,7 @@
|
||||
"openclawSlowestSpanMs": 45000
|
||||
}
|
||||
},
|
||||
"dashboard-session-send-turn": {
|
||||
"gateway-session-send-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 60000,
|
||||
"preProviderMs": 15000,
|
||||
@ -132,7 +132,7 @@
|
||||
"bundled-runtime-deps:baseline",
|
||||
"agent-cli-local-turn:baseline",
|
||||
"agent-gateway-rpc-turn:baseline",
|
||||
"dashboard-session-send-turn:baseline",
|
||||
"gateway-session-send-turn:baseline",
|
||||
"tui-message-turn:baseline",
|
||||
"openai-compatible-turn:baseline"
|
||||
]
|
||||
@ -160,7 +160,7 @@
|
||||
"state": "mock-openai-provider"
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider"
|
||||
},
|
||||
{
|
||||
@ -200,7 +200,7 @@
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
|
||||
@ -109,7 +109,7 @@
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
|
||||
@ -48,6 +48,10 @@
|
||||
"peakRssMb": 650,
|
||||
"maxCpuPercent": 250
|
||||
},
|
||||
"gateway-session-client": {
|
||||
"peakRssMb": 650,
|
||||
"maxCpuPercent": 250
|
||||
},
|
||||
"openai-compatible-client": {
|
||||
"peakRssMb": 650,
|
||||
"maxCpuPercent": 250
|
||||
@ -104,7 +108,7 @@
|
||||
"agentProcessLeaks": 0
|
||||
}
|
||||
},
|
||||
"dashboard-session-send-turn": {
|
||||
"gateway-session-send-turn": {
|
||||
"thresholds": {
|
||||
"agentTurnMs": 45000,
|
||||
"preProviderMs": 10000,
|
||||
@ -243,7 +247,7 @@
|
||||
"provider-models:baseline",
|
||||
"agent-cli-local-turn:baseline",
|
||||
"agent-gateway-rpc-turn:baseline",
|
||||
"dashboard-session-send-turn:baseline",
|
||||
"gateway-session-send-turn:baseline",
|
||||
"tui-message-turn:baseline",
|
||||
"openai-compatible-turn:baseline",
|
||||
"dashboard:baseline",
|
||||
@ -340,7 +344,7 @@
|
||||
"state": "mock-openai-provider"
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider"
|
||||
},
|
||||
{
|
||||
@ -518,7 +522,7 @@
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
|
||||
@ -38,7 +38,7 @@
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
|
||||
@ -44,7 +44,7 @@
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
{
|
||||
"scenario": "dashboard-session-send-turn",
|
||||
"scenario": "gateway-session-send-turn",
|
||||
"state": "mock-openai-provider",
|
||||
"timeoutMs": 180000
|
||||
},
|
||||
|
||||
@ -1,14 +1,15 @@
|
||||
{
|
||||
"id": "dashboard-session-send-turn-existing-user",
|
||||
"surface": "dashboard-session-send-turn",
|
||||
"title": "Dashboard Session Send Existing User Turn",
|
||||
"objective": "Clone an existing OpenClaw env, move the clone to the target runtime, send a dashboard-style user message through Gateway sessions.send, and measure response latency without mutating the durable source env.",
|
||||
"id": "gateway-session-send-turn-existing-user",
|
||||
"surface": "gateway-session-send-turn",
|
||||
"title": "Gateway Session Send Existing User Turn",
|
||||
"objective": "Clone an existing OpenClaw env, move the clone to the target runtime, send a user message through Gateway sessions.send, and measure response latency without mutating the durable source env.",
|
||||
"tags": [
|
||||
"agent",
|
||||
"message",
|
||||
"dashboard",
|
||||
"sessions",
|
||||
"gateway",
|
||||
"control-ui",
|
||||
"channels",
|
||||
"providers",
|
||||
"existing-user",
|
||||
"live"
|
||||
@ -62,7 +63,7 @@
|
||||
{
|
||||
"id": "gateway-start",
|
||||
"title": "Start Gateway",
|
||||
"intent": "Start the cloned gateway after upgrade and wait for readiness before sending a dashboard-style message.",
|
||||
"intent": "Start the cloned gateway after upgrade and wait for readiness before sending a Gateway session message.",
|
||||
"commands": [
|
||||
"ocm service install {env} --json",
|
||||
"ocm service start {env} --json"
|
||||
@ -75,11 +76,11 @@
|
||||
"healthScope": "readiness"
|
||||
},
|
||||
{
|
||||
"id": "dashboard-session-turn",
|
||||
"title": "Dashboard Session Message",
|
||||
"id": "gateway-session-turn",
|
||||
"title": "Gateway Session Message",
|
||||
"intent": "Exercise Gateway sessions.send on cloned user state and verify final assistant text appears in chat history.",
|
||||
"commands": [
|
||||
"node {kovaRoot}/support/run-dashboard-session-send-turn.mjs --env {env} --session-key kova-dashboard-existing-user --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
"node {kovaRoot}/support/run-gateway-session-send-turn.mjs --env {env} --session-key kova-gateway-existing-user --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
],
|
||||
"evidence": [
|
||||
"sessions.create timing",
|
||||
@ -92,9 +93,9 @@
|
||||
"healthScope": "post-ready"
|
||||
},
|
||||
{
|
||||
"id": "post-dashboard-health",
|
||||
"title": "Post-Dashboard Gateway Health",
|
||||
"intent": "Verify the cloned gateway remains responsive after the dashboard-style turn and collect logs for embedded-run/liveness evidence.",
|
||||
"id": "post-gateway-session-health",
|
||||
"title": "Post-Gateway-Session Health",
|
||||
"intent": "Verify the cloned gateway remains responsive after the Gateway session turn and collect logs for embedded-run/liveness evidence.",
|
||||
"commands": [
|
||||
"ocm @{env} -- gateway status --json --require-rpc",
|
||||
"ocm logs {env} --tail 300 --raw"
|
||||
@ -104,7 +105,7 @@
|
||||
"embedded-run traces",
|
||||
"liveness warnings",
|
||||
"plugin errors",
|
||||
"memory after dashboard turn"
|
||||
"memory after Gateway session turn"
|
||||
],
|
||||
"healthScope": "post-ready"
|
||||
}
|
||||
@ -1,14 +1,15 @@
|
||||
{
|
||||
"id": "dashboard-session-send-turn",
|
||||
"surface": "dashboard-session-send-turn",
|
||||
"id": "gateway-session-send-turn",
|
||||
"surface": "gateway-session-send-turn",
|
||||
"title": "Gateway Session Cold/Warm Turns",
|
||||
"objective": "Start a normal gateway, send cold and warm user messages through the Gateway session API, and wait for final assistant text in chat history.",
|
||||
"tags": [
|
||||
"agent",
|
||||
"message",
|
||||
"dashboard",
|
||||
"sessions",
|
||||
"gateway",
|
||||
"control-ui",
|
||||
"channels",
|
||||
"providers",
|
||||
"cold-warm"
|
||||
],
|
||||
@ -34,7 +35,7 @@
|
||||
"phases": [
|
||||
{
|
||||
"id": "provision",
|
||||
"title": "Provision Dashboard Env",
|
||||
"title": "Provision Gateway Session Env",
|
||||
"intent": "Create a disposable OpenClaw env without starting the gateway yet so Kova auth is applied before the gateway process boots.",
|
||||
"commands": [
|
||||
"ocm start {env} {startSelector} --no-service --json"
|
||||
@ -62,11 +63,11 @@
|
||||
"healthScope": "readiness"
|
||||
},
|
||||
{
|
||||
"id": "cold-dashboard-session-turn",
|
||||
"id": "cold-gateway-session-turn",
|
||||
"title": "Cold Gateway Session Turn",
|
||||
"intent": "Create the Gateway session, send the first message through `sessions.send`, and verify the final assistant response is present in chat history.",
|
||||
"commands": [
|
||||
"node {kovaRoot}/support/run-dashboard-session-send-turn.mjs --env {env} --session-key kova-dashboard-session-send --create-session true --min-assistant-count 1 --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
"node {kovaRoot}/support/run-gateway-session-send-turn.mjs --env {env} --session-key kova-gateway-session-send --create-session true --min-assistant-count 1 --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
],
|
||||
"evidence": [
|
||||
"cold sessions.send active turn duration",
|
||||
@ -81,11 +82,11 @@
|
||||
"healthScope": "post-ready"
|
||||
},
|
||||
{
|
||||
"id": "warm-dashboard-session-turn",
|
||||
"id": "warm-gateway-session-turn",
|
||||
"title": "Warm Gateway Session Turn",
|
||||
"intent": "Reuse the same Gateway session, send a second message through `sessions.send`, and prove whether cold discovery/cache work disappears.",
|
||||
"commands": [
|
||||
"node {kovaRoot}/support/run-dashboard-session-send-turn.mjs --env {env} --session-key kova-dashboard-session-send --create-session false --min-assistant-count 2 --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
"node {kovaRoot}/support/run-gateway-session-send-turn.mjs --env {env} --session-key kova-gateway-session-send --create-session false --min-assistant-count 2 --message 'Reply with exact ASCII text KOVA_AGENT_OK only.' --expected-text KOVA_AGENT_OK --timeout 120000"
|
||||
],
|
||||
"evidence": [
|
||||
"warm sessions.send active turn duration",
|
||||
@ -100,9 +101,9 @@
|
||||
"healthScope": "post-ready"
|
||||
},
|
||||
{
|
||||
"id": "post-dashboard-health",
|
||||
"title": "Post-Dashboard Gateway Health",
|
||||
"intent": "Verify the gateway remains responsive after dashboard-style cold and warm message turns.",
|
||||
"id": "post-gateway-session-health",
|
||||
"title": "Post-Gateway-Session Health",
|
||||
"intent": "Verify the gateway remains responsive after cold and warm Gateway session message turns.",
|
||||
"commands": [
|
||||
"ocm @{env} -- gateway status --json --require-rpc",
|
||||
"ocm logs {env} --tail 300 --raw"
|
||||
@ -111,7 +112,7 @@
|
||||
"gateway status probe",
|
||||
"provider logs",
|
||||
"plugin errors",
|
||||
"memory after dashboard turn"
|
||||
"memory after Gateway session turn"
|
||||
],
|
||||
"healthScope": "post-ready"
|
||||
}
|
||||
@ -1,66 +0,0 @@
|
||||
import {
|
||||
attributedSpanIntervals as collectAttributedSpanIntervals,
|
||||
buildPreProviderAttribution,
|
||||
preProviderMarkdownRows,
|
||||
summarizePreProviderAttributions
|
||||
} from "./pre-provider-attribution.mjs";
|
||||
|
||||
export const DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA = "kova.dashboardPreProviderAttribution.v1";
|
||||
export const DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA = "kova.dashboardPreProviderAttributionSummary.v1";
|
||||
|
||||
export function buildDashboardPreProviderAttribution({
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary
|
||||
}) {
|
||||
return buildPreProviderAttribution({
|
||||
schemaVersion: DASHBOARD_PRE_PROVIDER_ATTRIBUTION_SCHEMA,
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary,
|
||||
isAttributedSpanName: isDashboardAttributedSpanName,
|
||||
shouldIncludeSpan: includeDashboardSpanInWindow,
|
||||
missingEventsError: "timeline contains no dashboard turn attribution events"
|
||||
});
|
||||
}
|
||||
|
||||
export function summarizeDashboardPreProviderAttributions(turns) {
|
||||
return summarizePreProviderAttributions({
|
||||
schemaVersion: DASHBOARD_PRE_PROVIDER_SUMMARY_SCHEMA,
|
||||
turns,
|
||||
fieldName: "dashboardPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function dashboardPreProviderMarkdownRows(turns) {
|
||||
return preProviderMarkdownRows({
|
||||
title: "Dashboard pre-provider attribution",
|
||||
turns,
|
||||
fieldName: "dashboardPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function attributedSpanIntervals(events) {
|
||||
return collectAttributedSpanIntervals(events, isDashboardAttributedSpanName);
|
||||
}
|
||||
|
||||
function isDashboardAttributedSpanName(name) {
|
||||
const text = String(name ?? "");
|
||||
return text === "plugins.metadata.scan" ||
|
||||
text.startsWith("gateway.chat_send") ||
|
||||
text.startsWith("auto_reply") ||
|
||||
text.startsWith("reply.");
|
||||
}
|
||||
|
||||
function includeDashboardSpanInWindow(span, { windowStartEpochMs, windowEndEpochMs }) {
|
||||
if (span.name !== "plugins.metadata.scan") {
|
||||
return true;
|
||||
}
|
||||
return span.endEpochMs >= windowStartEpochMs && span.endEpochMs <= windowEndEpochMs;
|
||||
}
|
||||
66
src/collectors/gateway-session-turn-attribution.mjs
Normal file
66
src/collectors/gateway-session-turn-attribution.mjs
Normal file
@ -0,0 +1,66 @@
|
||||
import {
|
||||
attributedSpanIntervals as collectAttributedSpanIntervals,
|
||||
buildPreProviderAttribution,
|
||||
preProviderMarkdownRows,
|
||||
summarizePreProviderAttributions
|
||||
} from "./pre-provider-attribution.mjs";
|
||||
|
||||
export const GATEWAY_SESSION_PRE_PROVIDER_ATTRIBUTION_SCHEMA = "kova.gatewaySessionPreProviderAttribution.v1";
|
||||
export const GATEWAY_SESSION_PRE_PROVIDER_SUMMARY_SCHEMA = "kova.gatewaySessionPreProviderAttributionSummary.v1";
|
||||
|
||||
export function buildGatewaySessionPreProviderAttribution({
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary
|
||||
}) {
|
||||
return buildPreProviderAttribution({
|
||||
schemaVersion: GATEWAY_SESSION_PRE_PROVIDER_ATTRIBUTION_SCHEMA,
|
||||
label,
|
||||
phaseId,
|
||||
activeStartedAtEpochMs,
|
||||
activeFinishedAtEpochMs,
|
||||
attribution,
|
||||
timelineSummary,
|
||||
isAttributedSpanName: isGatewaySessionAttributedSpanName,
|
||||
shouldIncludeSpan: includeGatewaySessionSpanInWindow,
|
||||
missingEventsError: "timeline contains no Gateway session turn attribution events"
|
||||
});
|
||||
}
|
||||
|
||||
export function summarizeGatewaySessionPreProviderAttributions(turns) {
|
||||
return summarizePreProviderAttributions({
|
||||
schemaVersion: GATEWAY_SESSION_PRE_PROVIDER_SUMMARY_SCHEMA,
|
||||
turns,
|
||||
fieldName: "gatewaySessionPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function gatewaySessionPreProviderMarkdownRows(turns) {
|
||||
return preProviderMarkdownRows({
|
||||
title: "Gateway session pre-provider attribution",
|
||||
turns,
|
||||
fieldName: "gatewaySessionPreProviderAttribution"
|
||||
});
|
||||
}
|
||||
|
||||
export function attributedSpanIntervals(events) {
|
||||
return collectAttributedSpanIntervals(events, isGatewaySessionAttributedSpanName);
|
||||
}
|
||||
|
||||
function isGatewaySessionAttributedSpanName(name) {
|
||||
const text = String(name ?? "");
|
||||
return text === "plugins.metadata.scan" ||
|
||||
text.startsWith("gateway.chat_send") ||
|
||||
text.startsWith("auto_reply") ||
|
||||
text.startsWith("reply.");
|
||||
}
|
||||
|
||||
function includeGatewaySessionSpanInWindow(span, { windowStartEpochMs, windowEndEpochMs }) {
|
||||
if (span.name !== "plugins.metadata.scan") {
|
||||
return true;
|
||||
}
|
||||
return span.endEpochMs >= windowStartEpochMs && span.endEpochMs <= windowEndEpochMs;
|
||||
}
|
||||
@ -4,9 +4,9 @@ import {
|
||||
summarizeAgentCliPreProviderAttributions
|
||||
} from "./collectors/agent-cli-attribution.mjs";
|
||||
import {
|
||||
buildDashboardPreProviderAttribution,
|
||||
summarizeDashboardPreProviderAttributions
|
||||
} from "./collectors/dashboard-turn-attribution.mjs";
|
||||
buildGatewaySessionPreProviderAttribution,
|
||||
summarizeGatewaySessionPreProviderAttributions
|
||||
} from "./collectors/gateway-session-turn-attribution.mjs";
|
||||
import { computeProviderTurnAttribution } from "./collectors/provider.mjs";
|
||||
import { summarizeRuntimeDepsLogs } from "./collectors/logs.mjs";
|
||||
import { buildHealthMeasurement, healthReadinessClassification } from "./health.mjs";
|
||||
@ -92,10 +92,10 @@ export function evaluateRecord(record, scenario, options = {}) {
|
||||
const providerTurn = collectSlowestProviderTurn(agentTurns);
|
||||
const agentTurnStats = summarizeAgentTurnStats(agentTurns);
|
||||
const agentTurnDiagnostics = summarizeAgentTurnDiagnostics(agentTurns);
|
||||
const dashboardPreProviderAttribution = summarizeDashboardPreProviderAttributions(agentTurns);
|
||||
const gatewaySessionPreProviderAttribution = summarizeGatewaySessionPreProviderAttributions(agentTurns);
|
||||
const agentCliPreProviderAttribution = summarizeAgentCliPreProviderAttributions(agentTurns);
|
||||
const turnPreProviderAttribution = preferredPreProviderAttributionSummary(
|
||||
dashboardPreProviderAttribution,
|
||||
gatewaySessionPreProviderAttribution,
|
||||
agentCliPreProviderAttribution
|
||||
);
|
||||
const agentTurnMs = maxTurnDuration(agentTurns);
|
||||
@ -792,7 +792,7 @@ export function evaluateRecord(record, scenario, options = {}) {
|
||||
agentEventLoopSampleCount: agentTurnDiagnostics.eventLoopSampleCount,
|
||||
agentSessionPollCount: agentTurnDiagnostics.sessionPollCount,
|
||||
agentSessionPollErrorCount: agentTurnDiagnostics.sessionPollErrorCount,
|
||||
dashboardPreProviderAttribution,
|
||||
gatewaySessionPreProviderAttribution,
|
||||
agentCliPreProviderAttribution,
|
||||
coldPreProviderAttributedMs: turnPreProviderAttribution.cold.knownAttributedMs.median,
|
||||
warmPreProviderAttributedMs: turnPreProviderAttribution.warm.knownAttributedMs.median,
|
||||
@ -1025,8 +1025,8 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
activeFinishedAtEpochMs: timingResult.finishedAtEpochMs,
|
||||
gatewaySession
|
||||
});
|
||||
const dashboardPreProviderAttribution = gatewaySession
|
||||
? buildDashboardPreProviderAttribution({
|
||||
const gatewaySessionPreProviderAttribution = gatewaySession
|
||||
? buildGatewaySessionPreProviderAttribution({
|
||||
label: agentTurnLabel(phase.id, index),
|
||||
phaseId: phase.id,
|
||||
activeStartedAtEpochMs: timingResult.startedAtEpochMs,
|
||||
@ -1091,7 +1091,7 @@ function collectAgentTurns(record, providerEvidence, scenario, timelineSummary,
|
||||
providerLateByMs: attribution?.providerLateByMs ?? null,
|
||||
phaseBreakdown,
|
||||
turnDiagnostics,
|
||||
dashboardPreProviderAttribution,
|
||||
gatewaySessionPreProviderAttribution,
|
||||
agentCliPreProviderAttribution,
|
||||
metadataScanCount: turnDiagnostics.metadataScan.count,
|
||||
metadataScanTotalMs: turnDiagnostics.metadataScan.totalDurationMs,
|
||||
@ -1116,7 +1116,7 @@ function preferredPreProviderAttributionSummary(...summaries) {
|
||||
}
|
||||
|
||||
function checkGatewaySessionTransport(violations, agentTurns, scenario) {
|
||||
if (scenario.id !== "dashboard-session-send-turn") {
|
||||
if (scenario.id !== "gateway-session-send-turn" && scenario.surface !== "gateway-session-send-turn") {
|
||||
return;
|
||||
}
|
||||
for (const turn of agentTurns) {
|
||||
@ -1133,17 +1133,17 @@ function checkGatewaySessionTransport(violations, agentTurns, scenario) {
|
||||
expected: "direct-gateway-rpc",
|
||||
actual: transport ?? "unknown",
|
||||
phaseId: turn.phaseId,
|
||||
message: `dashboard session benchmark used ${transport ?? "unknown"} transport; direct Gateway RPC is required for Gateway product measurement${turn.gatewaySession.gatewayTransportFallbackReason ? ` (${turn.gatewaySession.gatewayTransportFallbackReason})` : ""}`
|
||||
message: `Gateway session benchmark used ${transport ?? "unknown"} transport; direct Gateway RPC is required for Gateway product measurement${turn.gatewaySession.gatewayTransportFallbackReason ? ` (${turn.gatewaySession.gatewayTransportFallbackReason})` : ""}`
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
function extractGatewaySessionTurn(result) {
|
||||
if (!result?.command?.includes("run-dashboard-session-send-turn.mjs")) {
|
||||
if (!result?.command?.includes("run-gateway-session-send-turn.mjs")) {
|
||||
return null;
|
||||
}
|
||||
const payload = parseJsonObject(result.stdout);
|
||||
if (!payload || payload.surface !== "dashboard-session-send-turn") {
|
||||
if (!payload || payload.surface !== "gateway-session-send-turn") {
|
||||
return null;
|
||||
}
|
||||
const activeStartedAtEpochMs = numberOrNull(payload.activeStartedAtEpochMs ?? payload.sendStartedAtEpochMs);
|
||||
@ -2015,12 +2015,12 @@ function agentTurnLabel(phaseId, index) {
|
||||
if (phaseId?.includes("warm")) {
|
||||
return "warm";
|
||||
}
|
||||
if (phaseId?.includes("gateway-session")) {
|
||||
return "gateway-session";
|
||||
}
|
||||
if (phaseId?.includes("gateway")) {
|
||||
return "gateway-rpc";
|
||||
}
|
||||
if (phaseId?.includes("dashboard")) {
|
||||
return "dashboard-session";
|
||||
}
|
||||
if (phaseId?.includes("tui")) {
|
||||
return "tui";
|
||||
}
|
||||
@ -3417,7 +3417,7 @@ function countDiagnosticMetric(record, key) {
|
||||
function isAgentMessageCommand(command) {
|
||||
return (command.includes(" -- agent ") && command.includes("--message")) ||
|
||||
command.includes("run-concurrent-agent-turns.mjs") ||
|
||||
command.includes("run-dashboard-session-send-turn.mjs") ||
|
||||
command.includes("run-gateway-session-send-turn.mjs") ||
|
||||
command.includes("run-tui-message-turn.mjs") ||
|
||||
command.includes("run-openai-compatible-turn.mjs");
|
||||
}
|
||||
|
||||
@ -26,12 +26,12 @@ const commandSurfaceMap = new Map([
|
||||
["agents", ["agent-cli-local-turn", "agent-gateway-rpc-turn", "workspace-scan"]],
|
||||
["browser", ["browser-automation"]],
|
||||
["capability", ["openai-compatible-turn", "provider-models"]],
|
||||
["chat", ["tui", "tui-message-turn"]],
|
||||
["chat", ["gateway-session-send-turn", "tui", "tui-message-turn"]],
|
||||
["configure", ["fresh-install", "provider-models"]],
|
||||
["daemon", ["release-runtime-startup", "gateway-performance"]],
|
||||
["dashboard", ["dashboard", "dashboard-session-send-turn"]],
|
||||
["dashboard", ["dashboard", "gateway-session-send-turn"]],
|
||||
["doctor", ["failure-containment", "release-runtime-startup"]],
|
||||
["gateway", ["release-runtime-startup", "gateway-performance"]],
|
||||
["gateway", ["release-runtime-startup", "gateway-performance", "gateway-session-send-turn"]],
|
||||
["health", ["release-runtime-startup", "gateway-performance"]],
|
||||
["infer", ["openai-compatible-turn", "provider-models"]],
|
||||
["logs", ["release-runtime-startup", "gateway-performance"]],
|
||||
@ -559,8 +559,11 @@ function matchPackageScriptSurfaceIds(capability) {
|
||||
if (/tui|chat|terminal/.test(searchableName)) {
|
||||
add("tui", "tui-message-turn");
|
||||
}
|
||||
if (/chat|session|channel/.test(searchableName)) {
|
||||
add("gateway-session-send-turn");
|
||||
}
|
||||
if (/dashboard|(^|[: -])ui[: -]/.test(searchableName)) {
|
||||
add("dashboard", "dashboard-session-send-turn");
|
||||
add("dashboard", "gateway-session-send-turn");
|
||||
}
|
||||
if (/plugin-sdk|plugins?|plugin-update/.test(searchableName)) {
|
||||
add("plugin-lifecycle", "official-plugin-install");
|
||||
|
||||
@ -29,7 +29,7 @@ export function measurementScopeForPhase(phase) {
|
||||
|
||||
export function driverKindForCommand(command) {
|
||||
const text = String(command ?? "");
|
||||
if (text.includes("run-dashboard-session-send-turn.mjs")) {
|
||||
if (text.includes("run-gateway-session-send-turn.mjs")) {
|
||||
return "gateway-rpc";
|
||||
}
|
||||
if (text.includes("run-openai-compatible-turn.mjs")) {
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
import { summarizeAgentTurnBreakdownForMarkdown } from "../collectors/agent-turns.mjs";
|
||||
import { agentCliPreProviderMarkdownRows } from "../collectors/agent-cli-attribution.mjs";
|
||||
import { dashboardPreProviderMarkdownRows } from "../collectors/dashboard-turn-attribution.mjs";
|
||||
import { gatewaySessionPreProviderMarkdownRows } from "../collectors/gateway-session-turn-attribution.mjs";
|
||||
import { healthTotalFailures } from "../health.mjs";
|
||||
|
||||
export function summarizeRecords(records) {
|
||||
@ -184,8 +184,8 @@ export function renderMarkdownReport(report) {
|
||||
if (record.measurements.agentTurnCount > 0) {
|
||||
lines.push(`- Agent cold/warm: cold ${record.measurements.coldAgentTurnMs ?? "unknown"} ms; warm ${record.measurements.warmAgentTurnMs ?? "unknown"} ms; delta ${record.measurements.agentColdWarmDeltaMs ?? "unknown"} ms`);
|
||||
lines.push(`- Agent pre-provider: cold ${record.measurements.coldPreProviderMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderMs ?? "unknown"} ms; delta ${record.measurements.agentColdWarmPreProviderDeltaMs ?? "unknown"} ms`);
|
||||
if (record.measurements.dashboardPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- Dashboard pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
|
||||
if (record.measurements.gatewaySessionPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- Gateway session pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
|
||||
}
|
||||
if (record.measurements.agentCliPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- Agent CLI pre-provider known: cold ${record.measurements.coldPreProviderAttributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderAttributedMs ?? "unknown"} ms; unattributed cold ${record.measurements.coldPreProviderUnattributedMs ?? "unknown"} ms; warm ${record.measurements.warmPreProviderUnattributedMs ?? "unknown"} ms`);
|
||||
@ -249,7 +249,7 @@ export function renderMarkdownReport(report) {
|
||||
lines.push(` - breakdown: ${breakdown}`);
|
||||
}
|
||||
}
|
||||
lines.push(...dashboardPreProviderMarkdownRows(record.measurements.agentTurns));
|
||||
lines.push(...gatewaySessionPreProviderMarkdownRows(record.measurements.agentTurns));
|
||||
lines.push(...agentCliPreProviderMarkdownRows(record.measurements.agentTurns));
|
||||
}
|
||||
lines.push(`- Profiling: ${record.profiling?.enabled ? "enabled" : "off"} (${record.profiling?.interpretation ?? "unknown"})`);
|
||||
@ -749,7 +749,7 @@ function summarizeMeasurements(measurements) {
|
||||
agentEventLoopSampleCount: measurements.agentEventLoopSampleCount ?? null,
|
||||
agentSessionPollCount: measurements.agentSessionPollCount ?? null,
|
||||
agentSessionPollErrorCount: measurements.agentSessionPollErrorCount ?? null,
|
||||
dashboardPreProviderAttribution: measurements.dashboardPreProviderAttribution ?? null,
|
||||
gatewaySessionPreProviderAttribution: measurements.gatewaySessionPreProviderAttribution ?? null,
|
||||
agentCliPreProviderAttribution: measurements.agentCliPreProviderAttribution ?? null,
|
||||
coldPreProviderAttributedMs: measurements.coldPreProviderAttributedMs ?? null,
|
||||
warmPreProviderAttributedMs: measurements.warmPreProviderAttributedMs ?? null,
|
||||
@ -1216,8 +1216,8 @@ function pushMeasurementBrief(lines, measurements, { compact }) {
|
||||
lines.push(`- health: startup p95 ${valueMs(measurements.health?.startupSamples?.p95Ms)}; post-ready p95 ${valueMs(measurements.health?.postReadySamples?.p95Ms)}; failures ${totalHealthFailures ?? "unknown"}; final failures ${measurements.health?.final?.failureCount ?? "unknown"}${healthSlowestText(measurements)}`);
|
||||
lines.push(`- resources: peak RSS ${valueMb(measurements.peakRssMb)}; max CPU ${valuePercent(measurements.cpuPercentMax)}; samples ${measurements.resourceSampleCount ?? "unknown"}; roles ${rolePeakText(measurements)}`);
|
||||
lines.push(`- agent: turn ${valueMs(measurements.agentTurnMs, "not-run")}; cold/warm ${valueMs(measurements.coldAgentTurnMs)}/${valueMs(measurements.warmAgentTurnMs)}; cold-warm delta ${valueMs(measurements.agentColdWarmDeltaMs)}; pre-provider ${valueMs(measurements.agentPreProviderMs)}; provider ${valueMs(measurements.agentProviderFinalMs)}; metadata scans ${measurements.agentMetadataScanCount ?? "unknown"} (${valueMs(measurements.agentMetadataScanTotalMs)}); event-loop ${valueMs(measurements.agentEventLoopMaxMs)}; polls ${measurements.agentSessionPollCount ?? "unknown"}; cleanup ${valueMs(measurements.agentCleanupMaxMs)}; diagnosis ${measurements.agentLatencyDiagnosis?.kind ?? "unknown"}; leaks ${measurements.agentProcessLeakCount ?? "unknown"}`);
|
||||
if (measurements.dashboardPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- dashboard attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
|
||||
if (measurements.gatewaySessionPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- gateway session attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
|
||||
}
|
||||
if (measurements.agentCliPreProviderAttribution?.count > 0) {
|
||||
lines.push(`- agent CLI attribution: cold known ${valueMs(measurements.coldPreProviderAttributedMs)} / unattributed ${valueMs(measurements.coldPreProviderUnattributedMs)}; warm known ${valueMs(measurements.warmPreProviderAttributedMs)} / unattributed ${valueMs(measurements.warmPreProviderUnattributedMs)}`);
|
||||
|
||||
@ -682,7 +682,7 @@ function tagCommandResult(result, phaseId) {
|
||||
function isAgentMessageCommand(command) {
|
||||
return (command.includes(" -- agent ") && command.includes("--message")) ||
|
||||
command.includes("run-concurrent-agent-turns.mjs") ||
|
||||
command.includes("run-dashboard-session-send-turn.mjs") ||
|
||||
command.includes("run-gateway-session-send-turn.mjs") ||
|
||||
command.includes("run-tui-message-turn.mjs") ||
|
||||
command.includes("run-openai-compatible-turn.mjs");
|
||||
}
|
||||
|
||||
@ -34,8 +34,8 @@ import {
|
||||
import { buildAgentCliPreProviderAttribution } from "./collectors/agent-cli-attribution.mjs";
|
||||
import {
|
||||
attributedSpanIntervals,
|
||||
buildDashboardPreProviderAttribution
|
||||
} from "./collectors/dashboard-turn-attribution.mjs";
|
||||
buildGatewaySessionPreProviderAttribution
|
||||
} from "./collectors/gateway-session-turn-attribution.mjs";
|
||||
import {
|
||||
computeProviderTurnAttribution,
|
||||
parseProviderRequestLog,
|
||||
@ -211,7 +211,7 @@ export async function runSelfCheck(flags = {}) {
|
||||
throw new Error(`missing auth override should not inject auth phases: ${phaseIds.join(", ")}`);
|
||||
}
|
||||
}));
|
||||
checks.push(await jsonCommandCheck("run-auth-live-source-env-json", `node bin/kova.mjs run --auth live --target runtime:stable --scenario dashboard-session-send-turn-existing-user --source-env 'Team Env' --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
checks.push(await jsonCommandCheck("run-auth-live-source-env-json", `node bin/kova.mjs run --auth live --target runtime:stable --scenario gateway-session-send-turn-existing-user --source-env 'Team Env' --report-dir ${quoteShell(tmp)} --json`, async (data) => {
|
||||
const report = JSON.parse(await readFile(data.jsonPath, "utf8"));
|
||||
const record = report.records?.[0];
|
||||
assertEqual(record?.auth?.mode, "live", "source-env live auth mode");
|
||||
@ -223,11 +223,11 @@ export async function runSelfCheck(flags = {}) {
|
||||
}
|
||||
const commands = record?.phases?.flatMap((phase) => phase.commands ?? []) ?? [];
|
||||
assertEqual(commands.some((command) => command.includes("ocm env clone 'Team Env'")), true, "source env clone command present");
|
||||
assertEqual(commands.some((command) => command.includes("run-dashboard-session-send-turn.mjs")), true, "dashboard session helper command present");
|
||||
assertEqual(commands.some((command) => command.includes("run-gateway-session-send-turn.mjs")), true, "gateway session helper command present");
|
||||
}));
|
||||
for (const item of [
|
||||
["agent-gateway-rpc-turn", "agent-gateway-rpc-turn", "ocm @"],
|
||||
["dashboard-session-send-turn", "dashboard-session-send-turn", "run-dashboard-session-send-turn.mjs"],
|
||||
["gateway-session-send-turn", "gateway-session-send-turn", "run-gateway-session-send-turn.mjs"],
|
||||
["tui-message-turn", "tui-message-turn", "run-tui-message-turn.mjs"],
|
||||
["openai-compatible-turn", "openai-compatible-turn", "run-openai-compatible-turn.mjs"]
|
||||
]) {
|
||||
@ -380,7 +380,7 @@ export async function runSelfCheck(flags = {}) {
|
||||
checks.push(await providerEvidenceParserCheck());
|
||||
checks.push(agentTurnBreakdownCheck());
|
||||
checks.push(gatewaySessionTurnEvaluationCheck());
|
||||
checks.push(dashboardPreProviderAttributionCheck());
|
||||
checks.push(gatewaySessionPreProviderAttributionCheck());
|
||||
checks.push(agentCliPreProviderAttributionCheck());
|
||||
checks.push(await mockProviderBehaviorCheck(tmp));
|
||||
checks.push(providerFailureEvaluationCheck());
|
||||
@ -2185,11 +2185,11 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
const base = 1777536000000;
|
||||
const coldPayload = {
|
||||
ok: true,
|
||||
surface: "dashboard-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
method: "sessions.send",
|
||||
createSession: true,
|
||||
minAssistantCount: 1,
|
||||
sessionKey: "kova-dashboard-session-send",
|
||||
sessionKey: "kova-gateway-session-send",
|
||||
runId: "cold-run",
|
||||
gatewayTransport: { kind: "direct-gateway-rpc", fallbackReason: null },
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
@ -2211,11 +2211,11 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
};
|
||||
const warmPayload = {
|
||||
ok: true,
|
||||
surface: "dashboard-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
method: "sessions.send",
|
||||
createSession: false,
|
||||
minAssistantCount: 2,
|
||||
sessionKey: "kova-dashboard-session-send",
|
||||
sessionKey: "kova-gateway-session-send",
|
||||
runId: "warm-run",
|
||||
gatewayTransport: { kind: "direct-gateway-rpc", fallbackReason: null },
|
||||
activeStartedAtEpochMs: base + 11000,
|
||||
@ -2236,21 +2236,21 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
expectedTextPresent: true
|
||||
};
|
||||
const record = {
|
||||
scenario: "dashboard-session-send-turn",
|
||||
surface: "dashboard-session-send-turn",
|
||||
scenario: "gateway-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
title: "Gateway session cold/warm",
|
||||
status: "PASS",
|
||||
cleanup: "done",
|
||||
auth: { mode: "mock" },
|
||||
phases: [
|
||||
{
|
||||
id: "cold-dashboard-session-turn",
|
||||
id: "cold-gateway-session-turn",
|
||||
title: "Cold Gateway Session Turn",
|
||||
intent: "Synthetic cold Gateway session turn",
|
||||
commands: ["node support/run-dashboard-session-send-turn.mjs --create-session true"],
|
||||
commands: ["node support/run-gateway-session-send-turn.mjs --create-session true"],
|
||||
evidence: [],
|
||||
results: [{
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session true",
|
||||
command: "node support/run-gateway-session-send-turn.mjs --create-session true",
|
||||
status: 0,
|
||||
timedOut: false,
|
||||
startedAt: new Date(base).toISOString(),
|
||||
@ -2264,13 +2264,13 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
metrics: { logs: zeroLogMetrics(), health: { ok: true } }
|
||||
},
|
||||
{
|
||||
id: "warm-dashboard-session-turn",
|
||||
id: "warm-gateway-session-turn",
|
||||
title: "Warm Gateway Session Turn",
|
||||
intent: "Synthetic warm Gateway session turn",
|
||||
commands: ["node support/run-dashboard-session-send-turn.mjs --create-session false"],
|
||||
commands: ["node support/run-gateway-session-send-turn.mjs --create-session false"],
|
||||
evidence: [],
|
||||
results: [{
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session false",
|
||||
command: "node support/run-gateway-session-send-turn.mjs --create-session false",
|
||||
status: 0,
|
||||
timedOut: false,
|
||||
startedAt: new Date(base + 10000).toISOString(),
|
||||
@ -2335,7 +2335,7 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
};
|
||||
|
||||
evaluateRecord(record, {
|
||||
id: "dashboard-session-send-turn",
|
||||
id: "gateway-session-send-turn",
|
||||
agent: { expectedText: "KOVA_AGENT_OK" },
|
||||
thresholds: { agentTurnMs: 2000, coldAgentTurnMs: 2000, warmAgentTurnMs: 1000 }
|
||||
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
|
||||
@ -2352,7 +2352,7 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
assertEqual(record.measurements.agentEventLoopMaxMs, 9, "active-window event-loop max");
|
||||
assertEqual(record.measurements.agentSessionPollCount, 5, "session polling total");
|
||||
assertEqual(record.measurements.agentTurns[1].gatewaySession.createSession, false, "warm turn reuses session");
|
||||
assertEqual(record.measurements.agentTurns[0].gatewaySession.gatewayTransportKind, "direct-gateway-rpc", "dashboard turn direct Gateway transport");
|
||||
assertEqual(record.measurements.agentTurns[0].gatewaySession.gatewayTransportKind, "direct-gateway-rpc", "Gateway session direct Gateway transport");
|
||||
|
||||
const rendered = renderMarkdownReport({
|
||||
generatedAt: "2026-05-01T00:00:00.000Z",
|
||||
@ -2372,18 +2372,18 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
gatewayTransport: { kind: "shell", fallbackReason: "gateway-token-unavailable" }
|
||||
};
|
||||
const fallbackRecord = {
|
||||
scenario: "dashboard-session-send-turn",
|
||||
surface: "dashboard-session-send-turn",
|
||||
scenario: "gateway-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
title: "Gateway session shell fallback",
|
||||
status: "PASS",
|
||||
phases: [{
|
||||
id: "cold-dashboard-session-turn",
|
||||
id: "cold-gateway-session-turn",
|
||||
title: "Cold Gateway Session Turn",
|
||||
intent: "Synthetic shell fallback",
|
||||
commands: ["node support/run-dashboard-session-send-turn.mjs --create-session true"],
|
||||
commands: ["node support/run-gateway-session-send-turn.mjs --create-session true"],
|
||||
evidence: [],
|
||||
results: [{
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session true",
|
||||
command: "node support/run-gateway-session-send-turn.mjs --create-session true",
|
||||
status: 0,
|
||||
timedOut: false,
|
||||
startedAt: new Date(base).toISOString(),
|
||||
@ -2404,15 +2404,15 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
finalMetrics: { service: { gatewayState: "running" }, logs: zeroLogMetrics() }
|
||||
};
|
||||
evaluateRecord(fallbackRecord, {
|
||||
id: "dashboard-session-send-turn",
|
||||
id: "gateway-session-send-turn",
|
||||
agent: { expectedText: "KOVA_AGENT_OK" },
|
||||
thresholds: {}
|
||||
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
|
||||
assertEqual(fallbackRecord.status, "FAIL", "dashboard session shell fallback rejected");
|
||||
assertEqual(fallbackRecord.status, "FAIL", "gateway session shell fallback rejected");
|
||||
assertEqual(
|
||||
fallbackRecord.violations.some((violation) => violation.metric === "gatewayTransport.kind"),
|
||||
true,
|
||||
"dashboard session shell fallback violation"
|
||||
"gateway session shell fallback violation"
|
||||
);
|
||||
|
||||
return {
|
||||
@ -2432,7 +2432,7 @@ function gatewaySessionTurnEvaluationCheck() {
|
||||
}
|
||||
}
|
||||
|
||||
function dashboardPreProviderAttributionCheck() {
|
||||
function gatewaySessionPreProviderAttributionCheck() {
|
||||
try {
|
||||
const base = 1777536000000;
|
||||
const timelineText = [
|
||||
@ -2460,9 +2460,9 @@ function dashboardPreProviderAttributionCheck() {
|
||||
assertEqual(parsedIntervals.length, 8, "span parser includes error terminal and metadata scans");
|
||||
assertEqual(parsedIntervals.some((span) => span.type === "span.error" && span.name === "reply.ensure_workspace"), true, "span error included");
|
||||
|
||||
const coldAttribution = buildDashboardPreProviderAttribution({
|
||||
const coldAttribution = buildGatewaySessionPreProviderAttribution({
|
||||
label: "cold",
|
||||
phaseId: "cold-dashboard-session-turn",
|
||||
phaseId: "cold-gateway-session-turn",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 2500,
|
||||
attribution: {
|
||||
@ -2482,16 +2482,16 @@ function dashboardPreProviderAttributionCheck() {
|
||||
assertEqual(coldAttribution.knownAttributedMs, 180, "overlap-safe cold known attribution includes active-turn metadata scan");
|
||||
assertEqual(coldAttribution.unattributedMs, 20, "cold unattributed remainder");
|
||||
const coldScanSummary = coldAttribution.spanSummaries.find((span) => span.name === "plugins.metadata.scan");
|
||||
assertEqual(coldScanSummary?.count, 2, "dashboard attribution includes active-turn metadata scans");
|
||||
assertEqual(coldScanSummary?.count, 2, "gateway session attribution includes active-turn metadata scans");
|
||||
assertEqual(coldScanSummary?.phases?.some((phase) => phase.phase === "startup"), true, "startup phase scan inside active window is counted");
|
||||
assertEqual(coldScanSummary?.phases?.some((phase) => phase.phase === "agent-turn"), true, "agent-turn phase scan inside active window is counted");
|
||||
assertEqual(coldAttribution.spanSummaries.find((span) => span.name === "reply.ensure_workspace")?.errorCount, 1, "error span summary");
|
||||
assertEqual(coldAttribution.provider.totalDurationMs, 600, "provider duration stays separate");
|
||||
assertEqual(coldAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "timeline artifact path");
|
||||
|
||||
const missingAttribution = buildDashboardPreProviderAttribution({
|
||||
const missingAttribution = buildGatewaySessionPreProviderAttribution({
|
||||
label: "cold",
|
||||
phaseId: "cold-dashboard-session-turn",
|
||||
phaseId: "cold-gateway-session-turn",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 2500,
|
||||
attribution: { firstProviderRequestAtEpochMs: base + 1200, preProviderMs: 200 },
|
||||
@ -2500,42 +2500,42 @@ function dashboardPreProviderAttributionCheck() {
|
||||
assertEqual(missingAttribution.available, false, "missing timeline unavailable");
|
||||
assertEqual(missingAttribution.unattributedMs, 200, "missing timeline preserves full remainder");
|
||||
|
||||
const record = syntheticDashboardSessionRecord({ base, timeline: parsed });
|
||||
const record = syntheticGatewaySessionRecord({ base, timeline: parsed });
|
||||
evaluateRecord(record, {
|
||||
id: "dashboard-session-send-turn",
|
||||
id: "gateway-session-send-turn",
|
||||
agent: { expectedText: "KOVA_AGENT_OK" },
|
||||
thresholds: { agentTurnMs: 2000, coldAgentTurnMs: 2000, warmAgentTurnMs: 1000 }
|
||||
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
|
||||
assertEqual(record.measurements.coldPreProviderAttributedMs, 180, "record cold attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderAttributedMs, 195, "record warm attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderUnattributedMs, 55, "record warm unattributed metric");
|
||||
assertEqual(record.measurements.dashboardPreProviderAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "record timeline artifact");
|
||||
assertEqual(record.measurements.gatewaySessionPreProviderAttribution.timelineArtifacts[0], "/tmp/kova/openclaw/timeline.jsonl", "record timeline artifact");
|
||||
|
||||
const rendered = renderMarkdownReport({
|
||||
generatedAt: "2026-05-01T00:00:00.000Z",
|
||||
runId: "self-check-dashboard-pre-provider",
|
||||
runId: "self-check-gateway-session-pre-provider",
|
||||
mode: "self-check",
|
||||
target: "runtime:stable",
|
||||
platform: { os: "test", release: "test", arch: "test", node: "test" },
|
||||
records: [record],
|
||||
summary: { statuses: { PASS: 1 } }
|
||||
});
|
||||
assertEqual(rendered.includes("Dashboard pre-provider attribution:"), true, "markdown includes dashboard attribution table");
|
||||
assertEqual(rendered.includes("Gateway session pre-provider attribution:"), true, "markdown includes gateway session attribution table");
|
||||
assertEqual(rendered.includes("Spans are selected by active turn timestamp window"), true, "markdown describes timestamp-window attribution");
|
||||
assertEqual(rendered.includes("`agent-turn`"), true, "markdown includes metadata scan phase as descriptive context");
|
||||
assertEqual(rendered.includes("`reply.ensure_workspace`"), true, "markdown includes span table");
|
||||
|
||||
return {
|
||||
id: "dashboard-pre-provider-attribution",
|
||||
id: "gateway-session-pre-provider-attribution",
|
||||
status: "PASS",
|
||||
command: "evaluate synthetic dashboard pre-provider timeline attribution",
|
||||
command: "evaluate synthetic Gateway session pre-provider timeline attribution",
|
||||
durationMs: 0
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
id: "dashboard-pre-provider-attribution",
|
||||
id: "gateway-session-pre-provider-attribution",
|
||||
status: "FAIL",
|
||||
command: "evaluate synthetic dashboard pre-provider timeline attribution",
|
||||
command: "evaluate synthetic Gateway session pre-provider timeline attribution",
|
||||
durationMs: 0,
|
||||
message: error.message
|
||||
};
|
||||
@ -2606,7 +2606,7 @@ function agentCliPreProviderAttributionCheck() {
|
||||
thresholds: { agentTurnMs: 2000, coldAgentTurnMs: 2000, warmAgentTurnMs: 1000 }
|
||||
}, { surface: { thresholds: {} }, targetPlan: { kind: "runtime" } });
|
||||
assertEqual(record.measurements.agentCliPreProviderAttribution.count, 2, "record agent CLI attribution count");
|
||||
assertEqual(record.measurements.dashboardPreProviderAttribution.count, 0, "record dashboard attribution stays empty for CLI turns");
|
||||
assertEqual(record.measurements.gatewaySessionPreProviderAttribution.count, 0, "record gateway session attribution stays empty for CLI turns");
|
||||
assertEqual(record.measurements.coldPreProviderAttributedMs, 170, "record agent CLI cold attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderAttributedMs, 80, "record agent CLI warm attributed metric");
|
||||
assertEqual(record.measurements.warmPreProviderUnattributedMs, 120, "record agent CLI warm unattributed metric");
|
||||
@ -2650,14 +2650,14 @@ function timelineEvent(event) {
|
||||
});
|
||||
}
|
||||
|
||||
function syntheticDashboardSessionRecord({ base, timeline }) {
|
||||
function syntheticGatewaySessionRecord({ base, timeline }) {
|
||||
const coldPayload = {
|
||||
ok: true,
|
||||
surface: "dashboard-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
method: "sessions.send",
|
||||
createSession: true,
|
||||
minAssistantCount: 1,
|
||||
sessionKey: "kova-dashboard-session-send",
|
||||
sessionKey: "kova-gateway-session-send",
|
||||
runId: "cold-run",
|
||||
activeStartedAtEpochMs: base + 1000,
|
||||
activeFinishedAtEpochMs: base + 2500,
|
||||
@ -2694,23 +2694,23 @@ function syntheticDashboardSessionRecord({ base, timeline }) {
|
||||
assistantMessageCount: 2
|
||||
};
|
||||
return {
|
||||
scenario: "dashboard-session-send-turn",
|
||||
surface: "dashboard-session-send-turn",
|
||||
scenario: "gateway-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
title: "Gateway session cold/warm",
|
||||
status: "PASS",
|
||||
cleanup: "done",
|
||||
auth: { mode: "mock" },
|
||||
phases: [
|
||||
syntheticDashboardTurnPhase({
|
||||
id: "cold-dashboard-session-turn",
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session true",
|
||||
syntheticGatewayTurnPhase({
|
||||
id: "cold-gateway-session-turn",
|
||||
command: "node support/run-gateway-session-send-turn.mjs --create-session true",
|
||||
startedAtEpochMs: base,
|
||||
finishedAtEpochMs: base + 5000,
|
||||
payload: coldPayload
|
||||
}),
|
||||
syntheticDashboardTurnPhase({
|
||||
id: "warm-dashboard-session-turn",
|
||||
command: "node support/run-dashboard-session-send-turn.mjs --create-session false",
|
||||
syntheticGatewayTurnPhase({
|
||||
id: "warm-gateway-session-turn",
|
||||
command: "node support/run-gateway-session-send-turn.mjs --create-session false",
|
||||
startedAtEpochMs: base + 10000,
|
||||
finishedAtEpochMs: base + 14000,
|
||||
payload: warmPayload
|
||||
@ -2842,11 +2842,11 @@ function syntheticAgentCliTurnPhase({ id, startedAtEpochMs, finishedAtEpochMs })
|
||||
};
|
||||
}
|
||||
|
||||
function syntheticDashboardTurnPhase({ id, command, startedAtEpochMs, finishedAtEpochMs, payload }) {
|
||||
function syntheticGatewayTurnPhase({ id, command, startedAtEpochMs, finishedAtEpochMs, payload }) {
|
||||
return {
|
||||
id,
|
||||
title: id,
|
||||
intent: "Synthetic dashboard session turn",
|
||||
intent: "Synthetic Gateway session turn",
|
||||
commands: [command],
|
||||
evidence: [],
|
||||
results: [{
|
||||
@ -4432,7 +4432,7 @@ function embeddedRunLogParserCheck() {
|
||||
|
||||
const breakdown = buildAgentTurnBreakdown({
|
||||
result: {
|
||||
command: "node support/run-dashboard-session-send-turn.mjs",
|
||||
command: "node support/run-gateway-session-send-turn.mjs",
|
||||
startedAtEpochMs: 1000,
|
||||
finishedAtEpochMs: 63000,
|
||||
durationMs: 62000
|
||||
@ -4922,7 +4922,7 @@ function healthReadinessModelCheck() {
|
||||
function agentContainmentHealthScopeCheck() {
|
||||
try {
|
||||
const record = {
|
||||
scenario: "dashboard-session-send-turn",
|
||||
scenario: "gateway-session-send-turn",
|
||||
status: "PASS",
|
||||
auth: { mode: "mock", source: "mock", providerId: "openai" },
|
||||
phases: [
|
||||
@ -4960,7 +4960,7 @@ function agentContainmentHealthScopeCheck() {
|
||||
}
|
||||
},
|
||||
{
|
||||
id: "cold-dashboard-session-turn",
|
||||
id: "cold-gateway-session-turn",
|
||||
results: [{
|
||||
command: "ocm @kova -- agent --local --agent main --session-id kova --message hi --json",
|
||||
status: 0,
|
||||
@ -5030,10 +5030,10 @@ function agentContainmentHealthScopeCheck() {
|
||||
};
|
||||
|
||||
evaluateRecord(record, {
|
||||
id: "dashboard-session-send-turn",
|
||||
id: "gateway-session-send-turn",
|
||||
phases: [
|
||||
{ id: "gateway-start", healthScope: "readiness" },
|
||||
{ id: "cold-dashboard-session-turn", healthScope: "post-ready" }
|
||||
{ id: "cold-gateway-session-turn", healthScope: "post-ready" }
|
||||
],
|
||||
agent: { expectedText: "KOVA_AGENT_OK" },
|
||||
thresholds: {
|
||||
|
||||
@ -21,13 +21,13 @@ try {
|
||||
const message = args.message ?? "Reply with exact ASCII text KOVA_AGENT_OK only.";
|
||||
const expectedText = args["expected-text"] ?? "KOVA_AGENT_OK";
|
||||
const timeoutMs = readTimeoutMs(args.timeout, 120000);
|
||||
const sessionKey = args["session-key"] ?? `kova-dashboard-${randomUUID()}`;
|
||||
const sessionKey = args["session-key"] ?? `kova-gateway-session-${randomUUID()}`;
|
||||
const createSession = readBoolean(args["create-session"], true);
|
||||
const minAssistantCount = readPositiveInteger(args["min-assistant-count"], 1);
|
||||
const allowShellFallback = readBoolean(args["allow-shell-fallback"], false);
|
||||
const gatewayTransport = await openDirectGatewayRpcClient(runtimeContext);
|
||||
if (!gatewayTransport.client && !allowShellFallback) {
|
||||
throw new Error(`direct Gateway RPC is required for dashboard-session-send-turn; fallback=${gatewayTransport.transport}; reason=${gatewayTransport.fallbackReason ?? "unknown"}`);
|
||||
throw new Error(`direct Gateway RPC is required for gateway-session-send-turn; fallback=${gatewayTransport.transport}; reason=${gatewayTransport.fallbackReason ?? "unknown"}`);
|
||||
}
|
||||
|
||||
try {
|
||||
@ -39,7 +39,7 @@ try {
|
||||
created = await gatewayCall(runtimeContext, gatewayTransport, "sessions.create", {
|
||||
agentId: "main",
|
||||
key: sessionKey,
|
||||
label: "Kova Dashboard Session Send"
|
||||
label: "Kova Gateway Session Send"
|
||||
}, Math.min(timeoutMs, 60000));
|
||||
sessionCreateFinishedAtEpochMs = Date.now();
|
||||
}
|
||||
@ -50,7 +50,7 @@ try {
|
||||
message,
|
||||
thinking: "off",
|
||||
timeoutMs,
|
||||
idempotencyKey: `kova-dashboard-${randomUUID()}`
|
||||
idempotencyKey: `kova-gateway-session-${randomUUID()}`
|
||||
}, Math.min(timeoutMs, 60000));
|
||||
const sendFinishedAtEpochMs = Date.now();
|
||||
const runId = typeof sent?.runId === "string" ? sent.runId : null;
|
||||
@ -68,7 +68,7 @@ try {
|
||||
|
||||
finishJson({
|
||||
ok: true,
|
||||
surface: "dashboard-session-send-turn",
|
||||
surface: "gateway-session-send-turn",
|
||||
method: "sessions.send",
|
||||
createSession,
|
||||
minAssistantCount,
|
||||
@ -109,7 +109,7 @@ try {
|
||||
gatewayTransport.client?.close();
|
||||
}
|
||||
} catch (error) {
|
||||
failJson(error, { surface: "dashboard-session-send-turn", finishedAtEpochMs: Date.now() });
|
||||
failJson(error, { surface: "gateway-session-send-turn", finishedAtEpochMs: Date.now() });
|
||||
}
|
||||
|
||||
async function waitForAssistantText({ runtimeContext, gatewayTransport, sessionKey, expectedText, timeoutMs, minAssistantCount }) {
|
||||
@ -151,7 +151,7 @@ async function waitForAssistantText({ runtimeContext, gatewayTransport, sessionK
|
||||
await sleep(500);
|
||||
}
|
||||
throw new Error(
|
||||
`timed out waiting for dashboard assistant text ${JSON.stringify(expectedText)}; last=${JSON.stringify(lastAssistantText)}; lastHistoryError=${JSON.stringify(lastHistoryError?.message ?? null)}`
|
||||
`timed out waiting for Gateway session assistant text ${JSON.stringify(expectedText)}; last=${JSON.stringify(lastAssistantText)}; lastHistoryError=${JSON.stringify(lastHistoryError?.message ?? null)}`
|
||||
);
|
||||
}
|
||||
|
||||
@ -2,7 +2,7 @@
|
||||
"id": "agent-gateway-rpc-turn",
|
||||
"title": "Agent CLI Gateway RPC Turn",
|
||||
"ownerArea": "agent-cli-gateway-client",
|
||||
"description": "Send messages through `openclaw agent` without `--local`, measuring the CLI client surface that crosses the Gateway agent RPC boundary. Direct Gateway/session user-path turns are measured by `dashboard-session-send-turn`.",
|
||||
"description": "Send messages through `openclaw agent` without `--local`, measuring the CLI client surface that crosses the Gateway agent RPC boundary. Direct Gateway/session user-path turns are measured by `gateway-session-send-turn`.",
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
|
||||
@ -1,12 +1,12 @@
|
||||
{
|
||||
"id": "dashboard-session-send-turn",
|
||||
"id": "gateway-session-send-turn",
|
||||
"title": "Gateway Session Cold/Warm Turns",
|
||||
"ownerArea": "gateway-chat-session-runtime",
|
||||
"description": "Create or reuse a real Gateway session, call Gateway `sessions.send` for cold and warm turns, and validate the same session API path dashboard and channel users exercise.",
|
||||
"processRoles": [
|
||||
"gateway",
|
||||
"command-tree",
|
||||
"dashboard-cli",
|
||||
"gateway-session-client",
|
||||
"agent-process",
|
||||
"mock-provider"
|
||||
],
|
||||
@ -28,8 +28,8 @@
|
||||
"peakRssMb": 700,
|
||||
"maxCpuPercent": 250
|
||||
},
|
||||
"dashboard-cli": {
|
||||
"peakRssMb": 700,
|
||||
"gateway-session-client": {
|
||||
"peakRssMb": 650,
|
||||
"maxCpuPercent": 250
|
||||
},
|
||||
"mock-provider": {
|
||||
Loading…
Reference in New Issue
Block a user