From 524e0a2d2f4844dee5c18d4947c8559e2c7fdf05 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 20 May 2026 16:53:37 +0100 Subject: [PATCH] fix: make generated cli bundles deterministic --- CHANGELOG.md | 4 +- src/cli/generate/stable-json.ts | 32 ++++++++++++++++ src/cli/generate/template.ts | 18 +++++---- src/cli/generate/tools.ts | 2 +- src/generate-cli.ts | 67 +++++++++++++++++++++++++++------ tests/generate-cli.test.ts | 33 ++++++++++++++++ 6 files changed, 134 insertions(+), 22 deletions(-) create mode 100644 src/cli/generate/stable-json.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index d2f2b96..22123e8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,9 @@ ## [0.11.2] - Unreleased -- Nothing yet. +### CLI + +- Make `generate-cli --bundle` artifacts deterministic by removing bundle-only paths/timestamps from embedded metadata and sorting generated tool/schema output. (Issue #180, thanks @imroc) ## [0.11.1] - 2026-05-14 diff --git a/src/cli/generate/stable-json.ts b/src/cli/generate/stable-json.ts new file mode 100644 index 0000000..4a07e36 --- /dev/null +++ b/src/cli/generate/stable-json.ts @@ -0,0 +1,32 @@ +export function stableJsonStringify(value: unknown, space?: number): string { + const json = JSON.stringify(sortJsonValue(value), undefined, space); + if (json === undefined) { + throw new TypeError('Cannot serialize unsupported JSON root value.'); + } + return json; +} + +function sortJsonValue(value: unknown): unknown { + if (Array.isArray(value)) { + return value.map((entry) => sortJsonValue(entry)); + } + if (!isPlainObject(value)) { + return value; + } + const result: Record = {}; + for (const key of Object.keys(value).toSorted()) { + const entry = (value as Record)[key]; + if (entry !== undefined) { + result[key] = sortJsonValue(entry); + } + } + return result; +} + +function isPlainObject(value: unknown): value is Record { + if (!value || typeof value !== 'object') { + return false; + } + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} diff --git a/src/cli/generate/template.ts b/src/cli/generate/template.ts index 1cfaf5c..e0fbb0a 100644 --- a/src/cli/generate/template.ts +++ b/src/cli/generate/template.ts @@ -9,6 +9,7 @@ import { markExecutable } from './fs-helpers.js'; import { renderEmbeddedHelpSource } from './template-help.js'; import type { GeneratedOption, ToolMetadata } from './tools.js'; import { buildEmbeddedSchemaMap } from './tools.js'; +import { stableJsonStringify } from './stable-json.js'; export interface TemplateInput { outputPath?: string; @@ -75,7 +76,10 @@ export function renderTemplate({ "import { createGeneratedKeepAliveRuntime, createRuntime, createServerProxy, handleDaemonCli } from 'mcporter';", "import { createCallResult } from 'mcporter';", ].join('\n'); - const embedded = JSON.stringify(definition, (_key, value) => (value instanceof URL ? value.toString() : value), 2); + const embedded = stableJsonStringify( + JSON.parse(JSON.stringify(definition, (_key, value) => (value instanceof URL ? value.toString() : value))), + 2 + ); const relativeStdioCwd = computeRelativeStdioCwd(definition, runtimeScriptPath ?? outputPath); const generatorHeader = `Generated by ${generator.name}@${generator.version} — https://github.com/openclaw/mcporter`; const toolDocs = tools.map((tool) => ({ @@ -104,15 +108,13 @@ export function renderTemplate({ flags: entry.doc.flagUsage ?? '', })); const generatorHeaderLiteral = JSON.stringify(generatorHeader); - const toolHelpLiteral = JSON.stringify(toolHelp, undefined, 2); - const embeddedSchemas = JSON.stringify(buildEmbeddedSchemaMap(tools), undefined, 2); - const embeddedMetadata = JSON.stringify(metadata, undefined, 2); + const toolHelpLiteral = stableJsonStringify(toolHelp, 2); + const embeddedSchemas = stableJsonStringify(buildEmbeddedSchemaMap(tools), 2); + const embeddedMetadata = stableJsonStringify(metadata, 2); const toolBlocks = renderedTools.map((entry) => entry.block).join('\n\n'); const signatureMap = Object.fromEntries(renderedTools.map((entry) => [entry.commandName, entry.tsSignature])); - const signatureMapLiteral = JSON.stringify(signatureMap, undefined, 2); - const generatedHeaderComment = `// @generated by ${generator.name}@${generator.version} on ${ - metadata.generatedAt - }. DO NOT EDIT.`; + const signatureMapLiteral = stableJsonStringify(signatureMap, 2); + const generatedHeaderComment = `// @generated by ${generator.name}@${generator.version}. DO NOT EDIT.`; return `#!/usr/bin/env ${runtimeKind === 'bun' ? 'bun' : 'node'} ${generatedHeaderComment} ${imports} diff --git a/src/cli/generate/tools.ts b/src/cli/generate/tools.ts index 06f47f4..3e85092 100644 --- a/src/cli/generate/tools.ts +++ b/src/cli/generate/tools.ts @@ -52,7 +52,7 @@ export function buildToolMetadata(tool: ServerToolInfo): ToolMetadata { export function buildEmbeddedSchemaMap(tools: ToolMetadata[]): Record { const result: Record = {}; - for (const entry of tools) { + for (const entry of tools.toSorted((left, right) => left.tool.name.localeCompare(right.tool.name))) { if (entry.tool.inputSchema && typeof entry.tool.inputSchema === 'object') { result[entry.tool.name] = entry.tool.inputSchema; } diff --git a/src/generate-cli.ts b/src/generate-cli.ts index 4922b90..f1b09f9 100644 --- a/src/generate-cli.ts +++ b/src/generate-cli.ts @@ -1,4 +1,5 @@ import fs from 'node:fs/promises'; +import { createHash } from 'node:crypto'; import path from 'node:path'; import { bundleOutput, @@ -12,6 +13,8 @@ import { readPackageMetadata, writeTemplate } from './cli/generate/template.js'; import type { ToolMetadata } from './cli/generate/tools.js'; import { buildToolMetadata, toolsTestHelpers } from './cli/generate/tools.js'; import { type CliArtifactMetadata, serializeDefinition } from './cli-metadata.js'; +import { stableJsonStringify } from './cli/generate/stable-json.js'; +import type { ServerDefinition } from './config.js'; import type { ServerToolInfo } from './runtime.js'; export interface GenerateCliOptions { @@ -29,6 +32,8 @@ export interface GenerateCliOptions { readonly excludeTools?: string[]; } +const REPRODUCIBLE_GENERATED_AT = '1970-01-01T00:00:00.000Z'; + // generateCli produces a standalone CLI (and optional bundle/binary) for a given MCP server. export async function generateCli( options: GenerateCliOptions @@ -55,7 +60,11 @@ export async function generateCli( baseDefinition.description || !derivedDescription ? baseDefinition : { ...baseDefinition, description: derivedDescription }; - const toolMetadata: ToolMetadata[] = tools.map((tool) => buildToolMetadata(tool)); + const embeddedDefinition = stripBuildSources(definition); + const serializedDefinition = serializeDefinition(embeddedDefinition); + const toolMetadata: ToolMetadata[] = tools + .map((tool) => buildToolMetadata(tool)) + .toSorted((left, right) => left.tool.name.localeCompare(right.tool.name)); const generator = await readPackageMetadata(); const baseInvocation = ensureInvocationDefaults( { @@ -72,32 +81,30 @@ export async function generateCli( includeTools: options.includeTools, excludeTools: options.excludeTools, }, - definition + embeddedDefinition ); const embeddedMetadata: CliArtifactMetadata = { schemaVersion: 1, - generatedAt: new Date().toISOString(), + generatedAt: REPRODUCIBLE_GENERATED_AT, generator, server: { name, - source: definition.source, - definition: serializeDefinition(definition), + definition: serializedDefinition, }, artifact: { path: '', kind: 'template', }, - invocation: baseInvocation, + invocation: buildEmbeddedInvocation(baseInvocation, serializedDefinition), }; const shouldBundle = Boolean(options.bundle ?? options.compile); let templateTmpDir: string | undefined; let templateOutputPath = options.outputPath; if (!templateOutputPath && shouldBundle) { - const tmpPrefix = path.join(process.cwd(), 'tmp', 'mcporter-cli-'); - await fs.mkdir(path.dirname(tmpPrefix), { recursive: true }); - templateTmpDir = await fs.mkdtemp(tmpPrefix); - templateOutputPath = path.join(templateTmpDir, `${name}.ts`); + templateTmpDir = resolveImplicitTemplateDir(name, serializedDefinition); + await fs.mkdir(templateTmpDir, { recursive: true }); + templateOutputPath = path.join(templateTmpDir, `${sanitizePathSegment(name) || 'server'}.ts`); } const templateSourcePath = path.resolve(templateOutputPath ?? path.resolve(process.cwd(), `${name}.ts`)); let resolvedBundleTarget: string | undefined; @@ -121,7 +128,7 @@ export async function generateCli( runtimeScriptPath, runtimeKind, timeoutMs, - definition, + definition: embeddedDefinition, serverName: name, tools: toolMetadata, generator, @@ -156,13 +163,49 @@ export async function generateCli( } } finally { if (templateTmpDir) { - await fs.rm(templateTmpDir, { recursive: true, force: true }).catch(() => {}); + await fs.rm(outputPath, { force: true }).catch(() => {}); + await fs.rmdir(templateTmpDir).catch(() => {}); + await fs.rmdir(path.dirname(templateTmpDir)).catch(() => {}); } } return { outputPath: options.outputPath ?? outputPath, bundlePath, compilePath }; } +function stripBuildSources(definition: ServerDefinition): ServerDefinition { + const { source: _source, sources: _sources, ...runtimeDefinition } = definition; + return runtimeDefinition; +} + +function buildEmbeddedInvocation( + invocation: CliArtifactMetadata['invocation'], + definition: ReturnType +): CliArtifactMetadata['invocation'] { + return { + serverRef: stableJsonStringify(definition), + runtime: invocation.runtime, + bundler: invocation.bundler, + timeoutMs: invocation.timeoutMs, + minify: invocation.minify, + includeTools: invocation.includeTools, + excludeTools: invocation.excludeTools, + bundle: typeof invocation.bundle === 'boolean' ? invocation.bundle : undefined, + compile: invocation.compile, + }; +} + +function resolveImplicitTemplateDir(serverName: string, definition: ReturnType): string { + const hash = createHash('sha256').update(stableJsonStringify({ serverName, definition })).digest('hex').slice(0, 12); + return path.join(process.cwd(), 'tmp', 'mcporter-cli', `${sanitizePathSegment(serverName) || 'server'}-${hash}`); +} + +function sanitizePathSegment(value: string): string { + return value + .toLowerCase() + .replace(/[^a-z0-9._-]+/g, '-') + .replace(/^-+|-+$/g, ''); +} + function applyToolFilters(tools: ServerToolInfo[], includeTools?: string[], excludeTools?: string[]): ServerToolInfo[] { if (includeTools && excludeTools) { throw new Error('Internal error: both includeTools and excludeTools provided to generateCli.'); diff --git a/tests/generate-cli.test.ts b/tests/generate-cli.test.ts index 2f44467..d6ca320 100644 --- a/tests/generate-cli.test.ts +++ b/tests/generate-cli.test.ts @@ -468,6 +468,39 @@ describeGenerateCli('generateCli', () => { expect(JSON.parse(callStdout)).toEqual({ result: 42 }); }, 60_000); + it('generates byte-identical bundles for unchanged inputs', async () => { + const inline = JSON.stringify({ + name: 'deterministic-bundle', + description: 'Deterministic bundle integration server', + command: baseUrl.toString(), + }); + const firstBundlePath = path.join(tmpDir, 'deterministic-a.js'); + const secondBundlePath = path.join(tmpDir, 'deterministic-b.js'); + await fs.rm(firstBundlePath, { force: true }); + await fs.rm(secondBundlePath, { force: true }); + await ensureDistBuilt(); + + await generateCli({ + serverRef: inline, + runtime: 'node', + timeoutMs: 5_000, + bundle: firstBundlePath, + }); + await generateCli({ + serverRef: inline, + runtime: 'node', + timeoutMs: 5_000, + bundle: secondBundlePath, + }); + + const first = await fs.readFile(firstBundlePath, 'utf8'); + const second = await fs.readFile(secondBundlePath, 'utf8'); + expect(first).toBe(second); + expect(first).not.toContain(firstBundlePath); + expect(first).not.toContain(secondBundlePath); + expect(first).not.toMatch(/mcporter-cli-[A-Za-z0-9]{6}/); + }, 60_000); + it('maps CLI options to Commander camelCase properties', async () => { const inline = JSON.stringify({ name: 'case-options',