From a2fc069d0fc1a7abc96c0ea7b0e583bc8c544efc Mon Sep 17 00:00:00 2001 From: Vincent Koc Date: Tue, 28 Apr 2026 20:49:43 -0700 Subject: [PATCH] feat: expand kitchen sink provider fixtures --- README.md | 15 +- openclaw.plugin.json | 64 +++++- scripts/check-kitchen-runtime.mjs | 103 ++++++++++ scripts/sync-surface.mjs | 30 ++- src/kitchen-runtime.js | 311 +++++++++++++++++++++++++++++- src/scenarios.js | 222 +++++++++++++++++++++ 6 files changed, 732 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 5a0a2ca..776d542 100644 --- a/README.md +++ b/README.md @@ -11,8 +11,9 @@ This repo is both: The generated runtime probes are credential-free. The hand-owned Kitchen Sink runtime also registers deterministic direct commands, tools, image generation, -media understanding, web search, web fetch, channel, hook, detached-task, and -text-provider catalog surfaces. +speech, realtime transcription/voice, video, music, media understanding, web +search, web fetch, memory, compaction, gateway/service/CLI, channel, hook, +detached-task, and text-provider catalog surfaces. It should not call external services, read secrets, spawn processes, or require live credentials. @@ -42,12 +43,22 @@ It also exposes provider and tool surfaces for live model routing: `rate limit`, `timeout`, or `fail` exercise deterministic provider error paths. - `kitchen-sink-media` describes images with deterministic fixture text. +- `kitchen-sink-speech`, `kitchen-sink-realtime-transcription`, + `kitchen-sink-realtime-voice`, `kitchen-sink-video`, and + `kitchen-sink-music` expose credential-free media provider fixtures with + deterministic WAV, transcript, bridge, storyboard, and track payloads. - `kitchen-sink-search` and `kitchen-sink-fetch` provide credential-free web tool fixtures with realistic status codes, request ids, result metadata, redirects, headers, cache metadata, links, and markdown content. +- `kitchen-sink-memory-embedding`, `kitchen-sink-memory-corpus`, and + `kitchen-sink-compaction` provide deterministic memory vectors, corpus + results, reads, and transcript summaries. - `kitchen-sink-channel` is a credential-free channel fixture that can resolve local ready/disabled/misconfigured accounts, route outbound sessions, and deliver deterministic text/media records. +- `kitchen.status`, `/kitchen-sink/status`, `kitchen-sink-service`, and the + lazy CLI descriptor exercise gateway method, HTTP route, service, and CLI + registration surfaces. - `kitchen-sink-llm` exposes a deterministic text-provider catalog row, provider-owned stream function, and prompt guidance so live LLM providers can discover the Kitchen Sink routes; responses describe which real plugin diff --git a/openclaw.plugin.json b/openclaw.plugin.json index cbf2e3f..796380e 100644 --- a/openclaw.plugin.json +++ b/openclaw.plugin.json @@ -15,7 +15,11 @@ ], "providers": [ "kitchen-sink-provider", - "kitchen-sink-llm" + "kitchen-sink-llm", + "kitchen-sink-image", + "kitchen-sink-speech", + "kitchen-sink-video", + "kitchen-sink-music" ], "cliBackends": [ "kitchen-sink-cli-backend" @@ -34,7 +38,10 @@ "onProviders": [ "kitchen-sink-provider", "kitchen-sink-llm", - "kitchen-sink-image" + "kitchen-sink-image", + "kitchen-sink-speech", + "kitchen-sink-video", + "kitchen-sink-music" ], "onChannels": [ "kitchen-sink-channel" @@ -72,6 +79,41 @@ "none" ], "envVars": [] + }, + { + "id": "kitchen-sink-speech", + "authMethods": [ + "none" + ], + "envVars": [] + }, + { + "id": "kitchen-sink-realtime-transcription", + "authMethods": [ + "none" + ], + "envVars": [] + }, + { + "id": "kitchen-sink-realtime-voice", + "authMethods": [ + "none" + ], + "envVars": [] + }, + { + "id": "kitchen-sink-video", + "authMethods": [ + "none" + ], + "envVars": [] + }, + { + "id": "kitchen-sink-music", + "authMethods": [ + "none" + ], + "envVars": [] } ], "cliBackends": [ @@ -104,22 +146,27 @@ "kitchen-sink-media" ], "memoryEmbeddingProviders": [ - "kitchen-sink-memory-embedding-providers" + "kitchen-sink-memory-embedding-providers", + "kitchen-sink-memory-embedding" ], "migrationProviders": [ "kitchen-sink-migration-providers" ], "musicGenerationProviders": [ - "kitchen-sink-music-generation-providers" + "kitchen-sink-music-generation-providers", + "kitchen-sink-music" ], "realtimeTranscriptionProviders": [ - "kitchen-sink-realtime-transcription-providers" + "kitchen-sink-realtime-transcription-providers", + "kitchen-sink-realtime-transcription" ], "realtimeVoiceProviders": [ - "kitchen-sink-realtime-voice-providers" + "kitchen-sink-realtime-voice-providers", + "kitchen-sink-realtime-voice" ], "speechProviders": [ - "kitchen-sink-speech-providers" + "kitchen-sink-speech-providers", + "kitchen-sink-speech" ], "tools": [ "kitchen-sink-tools", @@ -128,7 +175,8 @@ "kitchen_sink_search" ], "videoGenerationProviders": [ - "kitchen-sink-video-generation-providers" + "kitchen-sink-video-generation-providers", + "kitchen-sink-video" ], "webContentExtractors": [ "kitchen-sink-web-content-extractors" diff --git a/scripts/check-kitchen-runtime.mjs b/scripts/check-kitchen-runtime.mjs index 41b3a18..17810d6 100644 --- a/scripts/check-kitchen-runtime.mjs +++ b/scripts/check-kitchen-runtime.mjs @@ -175,6 +175,57 @@ const mediaResult = await mediaProvider.describeImage({ model: "kitchen-sink-vision-v1", }); assert.match(mediaResult.text, /Kitchen Sink media fixture/); +const audioDescription = await mediaProvider.transcribeAudio({ + audio: Buffer.from("audio fixture"), + prompt: "transcribe this kitchen audio", +}); +assert.match(audioDescription.text, /Kitchen Sink transcript/); +assert.equal(audioDescription.segments.length, 2); +const videoDescription = await mediaProvider.describeVideo({ prompt: "describe kitchen video" }); +assert.match(videoDescription.text, /three deterministic frames/); + +const speechProvider = findRegistration("registerSpeechProvider", "kitchen-sink-speech"); +const speechResult = await speechProvider.synthesize({ text: "say kitchen sink" }); +assert.equal(speechResult.mimeType, "audio/wav"); +assert.equal(speechResult.audioBuffer.subarray(0, 4).toString("ascii"), "RIFF"); +assert.equal(speechResult.metadata.providerId, "kitchen-sink-speech"); + +const realtimeTranscriptionProvider = findRegistration( + "registerRealtimeTranscriptionProvider", + "kitchen-sink-realtime-transcription", +); +const realtimeTranscripts = []; +const realtimeSession = realtimeTranscriptionProvider.createSession({ + onTranscript: (text) => realtimeTranscripts.push(text), +}); +await realtimeSession.connect(); +realtimeSession.sendAudio(Buffer.from("abc")); +const realtimeFinal = await realtimeSession.close(); +assert.match(realtimeFinal.text, /Kitchen Sink transcript/); +assert.ok(realtimeTranscripts.some((text) => /partial transcript/.test(text))); + +const realtimeVoiceProvider = findRegistration("registerRealtimeVoiceProvider", "kitchen-sink-realtime-voice"); +const realtimeVoiceEvents = []; +const realtimeBridge = realtimeVoiceProvider.createBridge({ + onEvent: (event) => realtimeVoiceEvents.push(event.type), +}); +await realtimeBridge.connect(); +assert.equal(realtimeBridge.isConnected(), true); +realtimeBridge.setMediaTimestamp(123); +realtimeBridge.submitToolResult({ ok: true }); +realtimeBridge.close(); +assert.equal(realtimeBridge.isConnected(), false); +assert.deepEqual(realtimeVoiceEvents, ["connected", "media_timestamp", "tool_result", "closed"]); + +const videoProvider = findRegistration("registerVideoGenerationProvider", "kitchen-sink-video"); +const videoResult = await videoProvider.generateVideo({ prompt: "kitchen video" }); +assert.equal(videoResult.videos[0].mimeType, "application/vnd.openclaw.kitchen-video+json"); +assert.equal(videoResult.job.status, "completed"); + +const musicProvider = findRegistration("registerMusicGenerationProvider", "kitchen-sink-music"); +const musicResult = await musicProvider.generateMusic({ prompt: "kitchen song" }); +assert.equal(musicResult.tracks[0].mimeType, "audio/wav"); +assert.equal(musicResult.tracks[0].audioBuffer.subarray(0, 4).toString("ascii"), "RIFF"); const searchProvider = findRegistration("registerWebSearchProvider", "kitchen-sink-search"); const searchTool = searchProvider.createTool({}); @@ -208,6 +259,58 @@ assert.deepEqual(streamEvents, ["start", "text_start", "text_delta", "text_end", assert.match(streamMessage.content[0].text, /kitchen explain text inference/); assert.ok(streamMessage.usage.totalTokens > 0); +const embeddingProvider = findRegistration("registerMemoryEmbeddingProvider", "kitchen-sink-memory-embedding"); +const embeddingResult = await embeddingProvider.embed({ text: "kitchen memory" }); +assert.equal(embeddingResult.embedding.length, 8); +assert.equal(embeddingResult.model, "kitchen-sink-embed-v1"); +const embeddingBatch = await embeddingProvider.embedMany({ texts: ["one", "two"] }); +assert.equal(embeddingBatch.embeddings.length, 2); + +const memoryCorpus = findRegistration("registerMemoryCorpusSupplement", "kitchen-sink-memory-corpus"); +const memorySearch = await memoryCorpus.search({ query: "runtime surfaces" }); +assert.equal(memorySearch.results[0].id, "ks-memory-runtime-surfaces"); +const memoryRead = await memoryCorpus.read("ks-memory-runtime-surfaces"); +assert.match(memoryRead.text, /providers, channels, hooks/); + +const compactionProvider = findRegistration("registerCompactionProvider", "kitchen-sink-compaction"); +const compacted = await compactionProvider.compact({ + messages: [{ role: "user", content: "remember job ks_image_1f8a5a98 and the image fixture" }], +}); +assert.match(compacted.summary, /Kitchen Sink compacted/); +assert.deepEqual(compacted.preservedIdentifiers, ["ks_image_1f8a5a98"]); + +const middleware = registrations.registerAgentToolResultMiddleware.at(-1); +assert.equal(typeof middleware?.[0], "function"); +assert.deepEqual(middleware[1].runtimes, ["pi", "codex", "cli"]); +const middlewareResult = await middleware[0]({ result: { content: "tool output" } }); +assert.equal(middlewareResult.metadata.kitchenSinkToolResultMiddleware, true); + +const service = registrations.registerService.map(([value]) => value).at(-1); +assert.equal(service.id, "kitchen-sink-service"); +assert.equal((await service.probe()).state, "ready"); +assert.equal((await service.start()).state, "started"); + +const httpRoute = findRegistration("registerHttpRoute", "kitchen-sink-http-status"); +let httpBody = ""; +const httpResult = await httpRoute.handler({}, { + setHeader: () => {}, + end: (body) => { + httpBody = body; + }, +}); +assert.equal(httpRoute.path, "/kitchen-sink/status"); +assert.equal(httpResult.ok, true); +assert.match(httpBody, /openclaw-kitchen-sink-fixture/); + +const gatewayMethod = registrations.registerGatewayMethod.find(([name]) => name === "kitchen.status"); +assert.ok(gatewayMethod, "registers kitchen.status gateway method"); +const gatewayResult = await gatewayMethod[1]({}); +assert.ok(gatewayResult.providerIds.includes("kitchen-sink-video")); + +const cliRegistration = registrations.registerCli.at(-1); +assert.equal(typeof cliRegistration?.[0], "function"); +assert.equal(cliRegistration[1].descriptors[0].name, "kitchen-sink"); + const imageTool = findRegistration("registerTool", "kitchen_sink_image_job"); assert.equal(typeof imageTool.execute, "function"); diff --git a/scripts/sync-surface.mjs b/scripts/sync-surface.mjs index 60e725b..b75eb94 100644 --- a/scripts/sync-surface.mjs +++ b/scripts/sync-surface.mjs @@ -156,6 +156,13 @@ function renderManifest({ manifestContracts, packageVersion }) { const contracts = Object.fromEntries(manifestContracts.map((field) => [field, [`kitchen-sink-${kebab(field)}`]])); appendContract(contracts, "imageGenerationProviders", "kitchen-sink-image"); appendContract(contracts, "mediaUnderstandingProviders", "kitchen-sink-media"); + appendContract(contracts, "speechProviders", "kitchen-sink-speech"); + appendContract(contracts, "realtimeTranscriptionProviders", "kitchen-sink-realtime-transcription"); + appendContract(contracts, "realtimeVoiceProviders", "kitchen-sink-realtime-voice"); + appendContract(contracts, "videoGenerationProviders", "kitchen-sink-video"); + appendContract(contracts, "musicGenerationProviders", "kitchen-sink-music"); + appendContract(contracts, "memoryEmbeddingProviders", "kitchen-sink-memory-embedding"); + appendContract(contracts, "agentToolResultMiddleware", "kitchen-sink-agent-tool-result-middleware"); appendContract(contracts, "webSearchProviders", "kitchen-sink-search"); appendContract(contracts, "webFetchProviders", "kitchen-sink-fetch"); appendContract(contracts, "tools", "kitchen_sink_image_job"); @@ -169,14 +176,28 @@ function renderManifest({ manifestContracts, packageVersion }) { enabledByDefault: false, kind: ["tool", "hook", "channel", "provider"], channels: ["kitchen-sink-channel"], - providers: ["kitchen-sink-provider", "kitchen-sink-llm"], + providers: [ + "kitchen-sink-provider", + "kitchen-sink-llm", + "kitchen-sink-image", + "kitchen-sink-speech", + "kitchen-sink-video", + "kitchen-sink-music", + ], cliBackends: ["kitchen-sink-cli-backend"], commandAliases: [ { command: "kitchen", pluginId: "openclaw-kitchen-sink-fixture" }, { command: "kitchen-sink", pluginId: "openclaw-kitchen-sink-fixture" }, ], activation: { - onProviders: ["kitchen-sink-provider", "kitchen-sink-llm", "kitchen-sink-image"], + onProviders: [ + "kitchen-sink-provider", + "kitchen-sink-llm", + "kitchen-sink-image", + "kitchen-sink-speech", + "kitchen-sink-video", + "kitchen-sink-music", + ], onChannels: ["kitchen-sink-channel"], onCommands: ["kitchen", "kitchen-sink"], onCapabilities: ["provider", "channel", "tool", "hook"], @@ -186,6 +207,11 @@ function renderManifest({ manifestContracts, packageVersion }) { { id: "kitchen-sink-provider", authMethods: ["none"], envVars: [] }, { id: "kitchen-sink-llm", authMethods: ["none"], envVars: [] }, { id: "kitchen-sink-image", authMethods: ["none"], envVars: [] }, + { id: "kitchen-sink-speech", authMethods: ["none"], envVars: [] }, + { id: "kitchen-sink-realtime-transcription", authMethods: ["none"], envVars: [] }, + { id: "kitchen-sink-realtime-voice", authMethods: ["none"], envVars: [] }, + { id: "kitchen-sink-video", authMethods: ["none"], envVars: [] }, + { id: "kitchen-sink-music", authMethods: ["none"], envVars: [] }, ], cliBackends: ["kitchen-sink-cli-backend"], configMigrations: ["kitchen-sink-config-migration"], diff --git a/src/kitchen-runtime.js b/src/kitchen-runtime.js index bd363eb..c2d196a 100644 --- a/src/kitchen-runtime.js +++ b/src/kitchen-runtime.js @@ -4,16 +4,31 @@ import { DEFAULT_TEXT_MODEL, CHANNEL_ACCOUNT_ID, CHANNEL_ID, + COMPACTION_PROVIDER_ID, + DEFAULT_EMBEDDING_MODEL, IMAGE_PROVIDER_ID, MEDIA_PROVIDER_ID, + MEMORY_EMBEDDING_PROVIDER_ID, + MUSIC_PROVIDER_ID, PLUGIN_ID, + REALTIME_TRANSCRIPTION_PROVIDER_ID, + REALTIME_VOICE_PROVIDER_ID, + SPEECH_PROVIDER_ID, TEXT_PROVIDER_ID, + VIDEO_PROVIDER_ID, WEB_FETCH_PROVIDER_ID, WEB_SEARCH_PROVIDER_ID, + createKitchenCompaction, + createKitchenEmbedding, + createKitchenMemorySearch, createKitchenChannelDelivery, + createKitchenMusicResult, createKitchenScenarioRuntime, createKitchenSinkImageAsset, + createKitchenSpeechAsset, createKitchenTextStream, + createKitchenTranscription, + createKitchenVideoResult, extractInteractiveText, kitchenChannelAccount, kitchenImageDescription, @@ -55,6 +70,19 @@ export function registerKitchenSinkRuntime(api, options = {}) { optionalRegister(api, "registerMediaUnderstandingProvider", () => api.registerMediaUnderstandingProvider(buildKitchenMediaProvider()), ); + optionalRegister(api, "registerSpeechProvider", () => api.registerSpeechProvider(buildKitchenSpeechProvider())); + optionalRegister(api, "registerRealtimeTranscriptionProvider", () => + api.registerRealtimeTranscriptionProvider(buildKitchenRealtimeTranscriptionProvider()), + ); + optionalRegister(api, "registerRealtimeVoiceProvider", () => + api.registerRealtimeVoiceProvider(buildKitchenRealtimeVoiceProvider()), + ); + optionalRegister(api, "registerVideoGenerationProvider", () => + api.registerVideoGenerationProvider(buildKitchenVideoProvider()), + ); + optionalRegister(api, "registerMusicGenerationProvider", () => + api.registerMusicGenerationProvider(buildKitchenMusicProvider()), + ); optionalRegister(api, "registerWebSearchProvider", () => api.registerWebSearchProvider(buildKitchenWebSearchProvider()), ); @@ -64,6 +92,26 @@ export function registerKitchenSinkRuntime(api, options = {}) { optionalRegister(api, "registerDetachedTaskRuntime", () => api.registerDetachedTaskRuntime(buildKitchenDetachedTaskRuntime()), ); + optionalRegister(api, "registerMemoryEmbeddingProvider", () => + api.registerMemoryEmbeddingProvider(buildKitchenMemoryEmbeddingProvider()), + ); + optionalRegister(api, "registerMemoryCorpusSupplement", () => + api.registerMemoryCorpusSupplement(buildKitchenMemoryCorpusSupplement()), + ); + optionalRegister(api, "registerCompactionProvider", () => + api.registerCompactionProvider(buildKitchenCompactionProvider()), + ); + optionalRegister(api, "registerAgentToolResultMiddleware", () => + api.registerAgentToolResultMiddleware(buildKitchenToolResultMiddleware(), { + runtimes: ["pi", "codex", "cli"], + }), + ); + optionalRegister(api, "registerService", () => api.registerService(buildKitchenService())); + optionalRegister(api, "registerHttpRoute", () => api.registerHttpRoute(buildKitchenHttpRoute())); + optionalRegister(api, "registerGatewayMethod", () => + api.registerGatewayMethod("kitchen.status", buildKitchenGatewayMethod()), + ); + optionalRegister(api, "registerCli", () => api.registerCli(buildKitchenCliRegistrar(), buildKitchenCliMetadata())); optionalRegister(api, "registerMemoryPromptSupplement", () => api.registerMemoryPromptSupplement(async () => kitchenPromptGuidance().join("\n")), ); @@ -296,7 +344,7 @@ function buildKitchenImageProvider(runtime) { function buildKitchenMediaProvider() { return { id: MEDIA_PROVIDER_ID, - capabilities: ["image"], + capabilities: ["image", "audio", "video"], defaultModels: { image: DEFAULT_MEDIA_MODEL }, autoPriority: { image: 5 }, describeImage: async (req) => ({ @@ -307,6 +355,127 @@ function buildKitchenMediaProvider() { text: kitchenImageDescription(req?.prompt, Array.isArray(req?.images) ? req.images.length : 0), model: req?.model || DEFAULT_MEDIA_MODEL, }), + transcribeAudio: async (req) => createKitchenTranscription({ audio: req?.audio, prompt: req?.prompt }), + describeVideo: async (req) => ({ + text: "Kitchen Sink video fixture: three deterministic frames show the office sink asset, a close-up, and a fixture badge.", + model: req?.model || DEFAULT_MEDIA_MODEL, + metadata: { kitchenSink: true, provider: MEDIA_PROVIDER_ID, scenarioId: "media.video-describe" }, + }), + }; +} + +function buildKitchenSpeechProvider() { + return { + id: SPEECH_PROVIDER_ID, + label: "Kitchen Sink Speech", + voices: ["kitchen-neutral", "kitchen-robot"], + defaultVoice: "kitchen-neutral", + isConfigured: () => true, + synthesize: async (req) => createKitchenSpeechAsset({ + text: req?.text, + voice: req?.voice, + model: req?.model, + }), + speak: async (req) => createKitchenSpeechAsset({ + text: req?.text, + voice: req?.voice, + model: req?.model, + }), + }; +} + +function buildKitchenRealtimeTranscriptionProvider() { + return { + id: REALTIME_TRANSCRIPTION_PROVIDER_ID, + label: "Kitchen Sink Realtime Transcription", + isConfigured: () => true, + createSession: (req = {}) => { + const chunks = []; + return { + provider: REALTIME_TRANSCRIPTION_PROVIDER_ID, + async connect() { + req.onReady?.({ provider: REALTIME_TRANSCRIPTION_PROVIDER_ID }); + return { ok: true, provider: REALTIME_TRANSCRIPTION_PROVIDER_ID }; + }, + sendAudio(audio) { + chunks.push(audio); + req.onTranscript?.(`Kitchen Sink partial transcript ${chunks.length}.`); + }, + async close() { + const result = createKitchenTranscription({ audio: Buffer.concat(chunks.map(toBuffer)) }); + req.onTranscript?.(result.text); + req.onClose?.({ code: 1000, reason: "kitchen sink complete" }); + return result; + }, + }; + }, + }; +} + +function buildKitchenRealtimeVoiceProvider() { + return { + id: REALTIME_VOICE_PROVIDER_ID, + label: "Kitchen Sink Realtime Voice", + isConfigured: () => true, + createBridge: (req = {}) => { + let connected = false; + const audio = []; + return { + supportsToolResultContinuation: true, + async connect() { + connected = true; + req.onEvent?.({ type: "connected", provider: REALTIME_VOICE_PROVIDER_ID }); + }, + sendAudio(chunk) { + audio.push(chunk); + req.onTranscript?.("Kitchen Sink realtime voice heard audio."); + }, + setMediaTimestamp(timestampMs) { + req.onEvent?.({ type: "media_timestamp", timestampMs }); + }, + submitToolResult(result) { + req.onEvent?.({ type: "tool_result", result }); + }, + acknowledgeMark(mark) { + req.onEvent?.({ type: "mark", mark }); + }, + close() { + connected = false; + req.onEvent?.({ type: "closed", audioChunks: audio.length }); + }, + isConnected: () => connected, + }; + }, + }; +} + +function buildKitchenVideoProvider() { + return { + id: VIDEO_PROVIDER_ID, + label: "Kitchen Sink Video", + defaultModel: "kitchen-sink-video-v1", + capabilities: { + generate: { maxVideos: 1, maxDurationSeconds: 3, supportsResolution: true }, + imageToVideo: { enabled: true, maxVideos: 1, maxInputImages: 1, maxDurationSeconds: 3 }, + videoToVideo: { enabled: false }, + }, + isConfigured: () => true, + generateVideo: async (req) => createKitchenVideoResult({ prompt: req?.prompt, model: req?.model }), + }; +} + +function buildKitchenMusicProvider() { + return { + id: MUSIC_PROVIDER_ID, + label: "Kitchen Sink Music", + defaultModel: "kitchen-sink-music-v1", + capabilities: { + generate: { maxTracks: 1, maxDurationSeconds: 1 }, + edit: { enabled: true, maxInputAudio: 1, maxTracks: 1 }, + }, + isConfigured: () => true, + generateMusic: async (req) => createKitchenMusicResult({ prompt: req?.prompt, model: req?.model }), + generate: async (req) => createKitchenMusicResult({ prompt: req?.prompt, model: req?.model }), }; } @@ -419,6 +588,133 @@ function buildKitchenWebFetchProvider() { }; } +function buildKitchenMemoryEmbeddingProvider() { + return { + id: MEMORY_EMBEDDING_PROVIDER_ID, + label: "Kitchen Sink Memory Embeddings", + model: DEFAULT_EMBEDDING_MODEL, + dimensions: 8, + isConfigured: () => true, + embed: async (input) => ({ + provider: MEMORY_EMBEDDING_PROVIDER_ID, + model: DEFAULT_EMBEDDING_MODEL, + embedding: createKitchenEmbedding(typeof input === "string" ? input : input?.text), + }), + embedMany: async (input) => { + const texts = Array.isArray(input) ? input : Array.isArray(input?.texts) ? input.texts : [input?.text ?? ""]; + return { + provider: MEMORY_EMBEDDING_PROVIDER_ID, + model: DEFAULT_EMBEDDING_MODEL, + embeddings: texts.map((text) => createKitchenEmbedding(text)), + }; + }, + }; +} + +function buildKitchenMemoryCorpusSupplement() { + return { + id: "kitchen-sink-memory-corpus", + label: "Kitchen Sink Memory Corpus", + search: async (query) => createKitchenMemorySearch(typeof query === "string" ? query : query?.query), + read: async (id = "ks-memory-runtime-surfaces") => ({ + id, + title: "Kitchen Sink runtime surfaces", + text: "Kitchen Sink memory corpus fixture covering providers, channels, hooks, compaction, and tasks.", + metadata: { kitchenSink: true, pluginId: PLUGIN_ID, scenarioId: "memory.read" }, + }), + list: async () => ({ + items: [{ id: "ks-memory-runtime-surfaces", title: "Kitchen Sink runtime surfaces" }], + }), + }; +} + +function buildKitchenCompactionProvider() { + return { + id: COMPACTION_PROVIDER_ID, + label: "Kitchen Sink Compaction", + compact: async (input) => createKitchenCompaction(input), + summarize: async (input) => createKitchenCompaction(input), + }; +} + +function buildKitchenToolResultMiddleware() { + return async (event = {}) => ({ + ...event, + kitchenSink: true, + pluginId: PLUGIN_ID, + scenarioId: "tool-result.middleware", + result: event.result, + metadata: { + ...(event.metadata || {}), + kitchenSinkToolResultMiddleware: true, + }, + }); +} + +function buildKitchenService() { + return { + id: "kitchen-sink-service", + name: "Kitchen Sink Service", + description: "Credential-free background service fixture.", + start: async () => ({ ok: true, service: "kitchen-sink-service", state: "started" }), + stop: async () => ({ ok: true, service: "kitchen-sink-service", state: "stopped" }), + probe: async () => ({ ok: true, service: "kitchen-sink-service", state: "ready" }), + }; +} + +function buildKitchenHttpRoute() { + return { + id: "kitchen-sink-http-status", + path: "/kitchen-sink/status", + auth: "gateway", + match: "exact", + handler: async (_req, res) => { + const body = JSON.stringify({ ok: true, pluginId: PLUGIN_ID, scenarioId: "http.status" }); + if (res && typeof res === "object") { + res.statusCode = 200; + res.setHeader?.("content-type", "application/json"); + res.end?.(body); + } + return { ok: true, body }; + }, + }; +} + +function buildKitchenGatewayMethod() { + return async () => ({ + ok: true, + pluginId: PLUGIN_ID, + providerIds: [ + SPEECH_PROVIDER_ID, + REALTIME_TRANSCRIPTION_PROVIDER_ID, + REALTIME_VOICE_PROVIDER_ID, + VIDEO_PROVIDER_ID, + MUSIC_PROVIDER_ID, + MEMORY_EMBEDDING_PROVIDER_ID, + COMPACTION_PROVIDER_ID, + ], + }); +} + +function buildKitchenCliRegistrar() { + return async ({ program } = {}) => { + program?.command?.("kitchen-sink")?.description?.("Run Kitchen Sink fixture commands."); + return { ok: true, command: "kitchen-sink" }; + }; +} + +function buildKitchenCliMetadata() { + return { + descriptors: [ + { + name: "kitchen-sink", + description: "Run Kitchen Sink fixture commands.", + hasSubcommands: true, + }, + ], + }; +} + function buildKitchenDetachedTaskRuntime() { const tasks = new Map(); @@ -574,6 +870,19 @@ function kitchenProviderError(result) { return error; } +function toBuffer(value) { + if (Buffer.isBuffer(value)) { + return value; + } + if (value instanceof Uint8Array) { + return Buffer.from(value); + } + if (typeof value === "string") { + return Buffer.from(value); + } + return Buffer.alloc(0); +} + function optionalRegister(api, method, register) { if (typeof api?.[method] !== "function") { return; diff --git a/src/scenarios.js b/src/scenarios.js index 1a07001..1eca287 100644 --- a/src/scenarios.js +++ b/src/scenarios.js @@ -7,11 +7,22 @@ export const MEDIA_PROVIDER_ID = "kitchen-sink-media"; export const TEXT_PROVIDER_ID = "kitchen-sink-llm"; export const WEB_SEARCH_PROVIDER_ID = "kitchen-sink-search"; export const WEB_FETCH_PROVIDER_ID = "kitchen-sink-fetch"; +export const SPEECH_PROVIDER_ID = "kitchen-sink-speech"; +export const REALTIME_TRANSCRIPTION_PROVIDER_ID = "kitchen-sink-realtime-transcription"; +export const REALTIME_VOICE_PROVIDER_ID = "kitchen-sink-realtime-voice"; +export const VIDEO_PROVIDER_ID = "kitchen-sink-video"; +export const MUSIC_PROVIDER_ID = "kitchen-sink-music"; +export const MEMORY_EMBEDDING_PROVIDER_ID = "kitchen-sink-memory-embedding"; +export const COMPACTION_PROVIDER_ID = "kitchen-sink-compaction"; export const CHANNEL_ID = "kitchen-sink-channel"; export const CHANNEL_ACCOUNT_ID = "local"; export const DEFAULT_IMAGE_MODEL = "kitchen-sink-image-v1"; export const DEFAULT_MEDIA_MODEL = "kitchen-sink-vision-v1"; export const DEFAULT_TEXT_MODEL = "kitchen-sink-text-v1"; +export const DEFAULT_SPEECH_MODEL = "kitchen-sink-tts-v1"; +export const DEFAULT_VIDEO_MODEL = "kitchen-sink-video-v1"; +export const DEFAULT_MUSIC_MODEL = "kitchen-sink-music-v1"; +export const DEFAULT_EMBEDDING_MODEL = "kitchen-sink-embed-v1"; export const DEFAULT_IMAGE_DELAY_MS = 10_000; const KITCHEN_SINK_OFFICE_IMAGE_FILE = "kitchen_sink_office.png"; const KITCHEN_SINK_OFFICE_IMAGE = readFileSync( @@ -411,6 +422,142 @@ export async function runKitchenFetch(url) { }; } +export function createKitchenSpeechAsset({ text, voice = "kitchen-neutral", model = DEFAULT_SPEECH_MODEL } = {}) { + const normalized = normalizePrompt(text, "Kitchen Sink speech fixture."); + const audioBuffer = createKitchenWavBuffer(normalized); + return { + audioBuffer, + buffer: audioBuffer, + mimeType: "audio/wav", + outputFormat: "wav", + fileExtension: ".wav", + voice, + voiceCompatible: true, + model, + durationMs: 480, + sampleRateHz: 16_000, + text: normalized, + metadata: fixtureMetadata("speech.synthesize", SPEECH_PROVIDER_ID, { + model, + voice, + sizeBytes: audioBuffer.byteLength, + sha256: sha256Hex(audioBuffer), + }), + }; +} + +export function createKitchenTranscription({ audio, prompt } = {}) { + const byteLength = inferByteLength(audio); + return { + provider: REALTIME_TRANSCRIPTION_PROVIDER_ID, + scenarioId: "media.audio-transcribe", + text: `Kitchen Sink transcript for ${byteLength} bytes of audio. ${normalizePrompt(prompt, "No prompt supplied.")}`, + language: "en", + segments: [ + { startMs: 0, endMs: 240, text: "Kitchen Sink transcript." }, + { startMs: 240, endMs: 480, text: "Deterministic audio fixture complete." }, + ], + confidence: 0.99, + metadata: fixtureMetadata("media.audio-transcribe", REALTIME_TRANSCRIPTION_PROVIDER_ID, { byteLength }), + }; +} + +export function createKitchenVideoResult({ prompt, model = DEFAULT_VIDEO_MODEL } = {}) { + const normalized = normalizePrompt(prompt, "kitchen sink video fixture"); + const id = `ks_video_${stableHash(normalized).slice(0, 10)}`; + return { + provider: VIDEO_PROVIDER_ID, + model, + job: mediaJob("video", id, normalized, "video.generate"), + videos: [ + { + id, + mimeType: "application/vnd.openclaw.kitchen-video+json", + fileName: `${id}.kitchen-video.json`, + durationMs: 3_000, + width: 1024, + height: 1024, + dataUrl: dataUrlForJson("application/vnd.openclaw.kitchen-video+json", { + id, + prompt: normalized, + frames: ["office-lobby-sink", "sink-closeup", "fixture-badge"], + }), + metadata: fixtureMetadata("video.generate", VIDEO_PROVIDER_ID, { model, prompt: normalized }), + }, + ], + metadata: fixtureMetadata("video.generate", VIDEO_PROVIDER_ID, { model, jobId: id }), + }; +} + +export function createKitchenMusicResult({ prompt, model = DEFAULT_MUSIC_MODEL } = {}) { + const normalized = normalizePrompt(prompt, "kitchen sink music fixture"); + const id = `ks_music_${stableHash(normalized).slice(0, 10)}`; + const audioBuffer = createKitchenWavBuffer(normalized); + return { + provider: MUSIC_PROVIDER_ID, + model, + job: mediaJob("music", id, normalized, "music.generate"), + tracks: [ + { + id, + title: "Kitchen Sink Fixture Loop", + mimeType: "audio/wav", + fileName: `${id}.wav`, + durationMs: 480, + audioBuffer, + dataUrl: `data:audio/wav;base64,${audioBuffer.toString("base64")}`, + metadata: fixtureMetadata("music.generate", MUSIC_PROVIDER_ID, { + model, + sizeBytes: audioBuffer.byteLength, + sha256: sha256Hex(audioBuffer), + }), + }, + ], + metadata: fixtureMetadata("music.generate", MUSIC_PROVIDER_ID, { model, jobId: id }), + }; +} + +export function createKitchenEmbedding(input, dimensions = 8) { + const text = Array.isArray(input) ? input.join("\n") : normalizePrompt(input, "kitchen sink memory"); + const hash = createHash("sha256").update(text).digest(); + return Array.from({ length: dimensions }, (_, index) => Number(((hash[index] / 255) * 2 - 1).toFixed(6))); +} + +export function createKitchenMemorySearch(query) { + const normalized = normalizePrompt(query, "kitchen sink memory"); + return { + provider: MEMORY_EMBEDDING_PROVIDER_ID, + scenarioId: "memory.search", + query: normalized, + results: [ + { + id: "ks-memory-runtime-surfaces", + score: 0.97, + title: "Kitchen Sink runtime surfaces", + text: "Kitchen Sink exercises providers, tools, hooks, channels, memory, compaction, and task lifecycles.", + metadata: fixtureMetadata("memory.search", MEMORY_EMBEDDING_PROVIDER_ID), + }, + ], + }; +} + +export function createKitchenCompaction(input = {}) { + const messages = Array.isArray(input.messages) ? input.messages : []; + const text = messages + .map((message) => (typeof message?.content === "string" ? message.content : "")) + .filter(Boolean) + .join(" ") + .trim(); + const summary = normalizePrompt(text, normalizePrompt(input.text, "Kitchen Sink compacted deterministic transcript.")); + return { + provider: COMPACTION_PROVIDER_ID, + scenarioId: "compaction.summary", + summary: `Kitchen Sink compacted ${messages.length || 1} turn${messages.length === 1 ? "" : "s"}: ${summary.slice(0, 180)}`, + preservedIdentifiers: [...new Set(summary.match(/\bks_[a-z]+_[a-f0-9]+\b/g) || [])], + metadata: fixtureMetadata("compaction.summary", COMPACTION_PROVIDER_ID, { messageCount: messages.length }), + }; +} + export function kitchenImageReply(result) { if (result.error) { return { @@ -797,6 +944,81 @@ function selectKitchenImageFixture(_prompt) { return KITCHEN_IMAGE_FIXTURES[0]; } +function mediaJob(kind, id, prompt, scenarioId) { + const createdAt = "2026-04-28T00:00:00.000Z"; + return { + id, + kind, + status: "completed", + prompt, + createdAt, + completedAt: createdAt, + pluginId: PLUGIN_ID, + scenarioId, + progressPercent: 100, + timeline: [ + { status: "queued", at: createdAt, summary: `Kitchen Sink ${kind} job queued.` }, + { status: "running", at: createdAt, summary: `Kitchen Sink ${kind} job running.` }, + { status: "completed", at: createdAt, summary: `Kitchen Sink ${kind} job completed.` }, + ], + }; +} + +function fixtureMetadata(scenarioId, providerId, extra = {}) { + return { + kitchenSink: true, + pluginId: PLUGIN_ID, + providerId, + scenarioId, + ...extra, + }; +} + +function createKitchenWavBuffer(seedText) { + const sampleRate = 16_000; + const durationSeconds = 0.48; + const sampleCount = Math.floor(sampleRate * durationSeconds); + const dataSize = sampleCount * 2; + const buffer = Buffer.alloc(44 + dataSize); + const frequency = 360 + (Number.parseInt(stableHash(seedText).slice(0, 2), 16) % 160); + buffer.write("RIFF", 0); + buffer.writeUInt32LE(36 + dataSize, 4); + buffer.write("WAVE", 8); + buffer.write("fmt ", 12); + buffer.writeUInt32LE(16, 16); + buffer.writeUInt16LE(1, 20); + buffer.writeUInt16LE(1, 22); + buffer.writeUInt32LE(sampleRate, 24); + buffer.writeUInt32LE(sampleRate * 2, 28); + buffer.writeUInt16LE(2, 32); + buffer.writeUInt16LE(16, 34); + buffer.write("data", 36); + buffer.writeUInt32LE(dataSize, 40); + for (let index = 0; index < sampleCount; index += 1) { + const envelope = Math.sin((Math.PI * index) / sampleCount); + const sample = Math.round(Math.sin((2 * Math.PI * frequency * index) / sampleRate) * 12000 * envelope); + buffer.writeInt16LE(sample, 44 + index * 2); + } + return buffer; +} + +function dataUrlForJson(mimeType, value) { + return `data:${mimeType};base64,${Buffer.from(JSON.stringify(value), "utf8").toString("base64")}`; +} + +function inferByteLength(value) { + if (!value) { + return 0; + } + if (typeof value.byteLength === "number") { + return value.byteLength; + } + if (typeof value.length === "number") { + return value.length; + } + return Buffer.byteLength(String(value)); +} + function sha256Hex(buffer) { return createHash("sha256").update(buffer).digest("hex"); }