fix: make generated cli bundles deterministic
This commit is contained in:
parent
b8909e7cc0
commit
524e0a2d2f
@ -2,7 +2,9 @@
|
||||
|
||||
## [0.11.2] - Unreleased
|
||||
|
||||
- Nothing yet.
|
||||
### CLI
|
||||
|
||||
- Make `generate-cli --bundle` artifacts deterministic by removing bundle-only paths/timestamps from embedded metadata and sorting generated tool/schema output. (Issue #180, thanks @imroc)
|
||||
|
||||
## [0.11.1] - 2026-05-14
|
||||
|
||||
|
||||
32
src/cli/generate/stable-json.ts
Normal file
32
src/cli/generate/stable-json.ts
Normal file
@ -0,0 +1,32 @@
|
||||
export function stableJsonStringify(value: unknown, space?: number): string {
|
||||
const json = JSON.stringify(sortJsonValue(value), undefined, space);
|
||||
if (json === undefined) {
|
||||
throw new TypeError('Cannot serialize unsupported JSON root value.');
|
||||
}
|
||||
return json;
|
||||
}
|
||||
|
||||
function sortJsonValue(value: unknown): unknown {
|
||||
if (Array.isArray(value)) {
|
||||
return value.map((entry) => sortJsonValue(entry));
|
||||
}
|
||||
if (!isPlainObject(value)) {
|
||||
return value;
|
||||
}
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const key of Object.keys(value).toSorted()) {
|
||||
const entry = (value as Record<string, unknown>)[key];
|
||||
if (entry !== undefined) {
|
||||
result[key] = sortJsonValue(entry);
|
||||
}
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function isPlainObject(value: unknown): value is Record<string, unknown> {
|
||||
if (!value || typeof value !== 'object') {
|
||||
return false;
|
||||
}
|
||||
const prototype = Object.getPrototypeOf(value);
|
||||
return prototype === Object.prototype || prototype === null;
|
||||
}
|
||||
@ -9,6 +9,7 @@ import { markExecutable } from './fs-helpers.js';
|
||||
import { renderEmbeddedHelpSource } from './template-help.js';
|
||||
import type { GeneratedOption, ToolMetadata } from './tools.js';
|
||||
import { buildEmbeddedSchemaMap } from './tools.js';
|
||||
import { stableJsonStringify } from './stable-json.js';
|
||||
|
||||
export interface TemplateInput {
|
||||
outputPath?: string;
|
||||
@ -75,7 +76,10 @@ export function renderTemplate({
|
||||
"import { createGeneratedKeepAliveRuntime, createRuntime, createServerProxy, handleDaemonCli } from 'mcporter';",
|
||||
"import { createCallResult } from 'mcporter';",
|
||||
].join('\n');
|
||||
const embedded = JSON.stringify(definition, (_key, value) => (value instanceof URL ? value.toString() : value), 2);
|
||||
const embedded = stableJsonStringify(
|
||||
JSON.parse(JSON.stringify(definition, (_key, value) => (value instanceof URL ? value.toString() : value))),
|
||||
2
|
||||
);
|
||||
const relativeStdioCwd = computeRelativeStdioCwd(definition, runtimeScriptPath ?? outputPath);
|
||||
const generatorHeader = `Generated by ${generator.name}@${generator.version} — https://github.com/openclaw/mcporter`;
|
||||
const toolDocs = tools.map((tool) => ({
|
||||
@ -104,15 +108,13 @@ export function renderTemplate({
|
||||
flags: entry.doc.flagUsage ?? '',
|
||||
}));
|
||||
const generatorHeaderLiteral = JSON.stringify(generatorHeader);
|
||||
const toolHelpLiteral = JSON.stringify(toolHelp, undefined, 2);
|
||||
const embeddedSchemas = JSON.stringify(buildEmbeddedSchemaMap(tools), undefined, 2);
|
||||
const embeddedMetadata = JSON.stringify(metadata, undefined, 2);
|
||||
const toolHelpLiteral = stableJsonStringify(toolHelp, 2);
|
||||
const embeddedSchemas = stableJsonStringify(buildEmbeddedSchemaMap(tools), 2);
|
||||
const embeddedMetadata = stableJsonStringify(metadata, 2);
|
||||
const toolBlocks = renderedTools.map((entry) => entry.block).join('\n\n');
|
||||
const signatureMap = Object.fromEntries(renderedTools.map((entry) => [entry.commandName, entry.tsSignature]));
|
||||
const signatureMapLiteral = JSON.stringify(signatureMap, undefined, 2);
|
||||
const generatedHeaderComment = `// @generated by ${generator.name}@${generator.version} on ${
|
||||
metadata.generatedAt
|
||||
}. DO NOT EDIT.`;
|
||||
const signatureMapLiteral = stableJsonStringify(signatureMap, 2);
|
||||
const generatedHeaderComment = `// @generated by ${generator.name}@${generator.version}. DO NOT EDIT.`;
|
||||
return `#!/usr/bin/env ${runtimeKind === 'bun' ? 'bun' : 'node'}
|
||||
${generatedHeaderComment}
|
||||
${imports}
|
||||
|
||||
@ -52,7 +52,7 @@ export function buildToolMetadata(tool: ServerToolInfo): ToolMetadata {
|
||||
|
||||
export function buildEmbeddedSchemaMap(tools: ToolMetadata[]): Record<string, unknown> {
|
||||
const result: Record<string, unknown> = {};
|
||||
for (const entry of tools) {
|
||||
for (const entry of tools.toSorted((left, right) => left.tool.name.localeCompare(right.tool.name))) {
|
||||
if (entry.tool.inputSchema && typeof entry.tool.inputSchema === 'object') {
|
||||
result[entry.tool.name] = entry.tool.inputSchema;
|
||||
}
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
import fs from 'node:fs/promises';
|
||||
import { createHash } from 'node:crypto';
|
||||
import path from 'node:path';
|
||||
import {
|
||||
bundleOutput,
|
||||
@ -12,6 +13,8 @@ import { readPackageMetadata, writeTemplate } from './cli/generate/template.js';
|
||||
import type { ToolMetadata } from './cli/generate/tools.js';
|
||||
import { buildToolMetadata, toolsTestHelpers } from './cli/generate/tools.js';
|
||||
import { type CliArtifactMetadata, serializeDefinition } from './cli-metadata.js';
|
||||
import { stableJsonStringify } from './cli/generate/stable-json.js';
|
||||
import type { ServerDefinition } from './config.js';
|
||||
import type { ServerToolInfo } from './runtime.js';
|
||||
|
||||
export interface GenerateCliOptions {
|
||||
@ -29,6 +32,8 @@ export interface GenerateCliOptions {
|
||||
readonly excludeTools?: string[];
|
||||
}
|
||||
|
||||
const REPRODUCIBLE_GENERATED_AT = '1970-01-01T00:00:00.000Z';
|
||||
|
||||
// generateCli produces a standalone CLI (and optional bundle/binary) for a given MCP server.
|
||||
export async function generateCli(
|
||||
options: GenerateCliOptions
|
||||
@ -55,7 +60,11 @@ export async function generateCli(
|
||||
baseDefinition.description || !derivedDescription
|
||||
? baseDefinition
|
||||
: { ...baseDefinition, description: derivedDescription };
|
||||
const toolMetadata: ToolMetadata[] = tools.map((tool) => buildToolMetadata(tool));
|
||||
const embeddedDefinition = stripBuildSources(definition);
|
||||
const serializedDefinition = serializeDefinition(embeddedDefinition);
|
||||
const toolMetadata: ToolMetadata[] = tools
|
||||
.map((tool) => buildToolMetadata(tool))
|
||||
.toSorted((left, right) => left.tool.name.localeCompare(right.tool.name));
|
||||
const generator = await readPackageMetadata();
|
||||
const baseInvocation = ensureInvocationDefaults(
|
||||
{
|
||||
@ -72,32 +81,30 @@ export async function generateCli(
|
||||
includeTools: options.includeTools,
|
||||
excludeTools: options.excludeTools,
|
||||
},
|
||||
definition
|
||||
embeddedDefinition
|
||||
);
|
||||
const embeddedMetadata: CliArtifactMetadata = {
|
||||
schemaVersion: 1,
|
||||
generatedAt: new Date().toISOString(),
|
||||
generatedAt: REPRODUCIBLE_GENERATED_AT,
|
||||
generator,
|
||||
server: {
|
||||
name,
|
||||
source: definition.source,
|
||||
definition: serializeDefinition(definition),
|
||||
definition: serializedDefinition,
|
||||
},
|
||||
artifact: {
|
||||
path: '',
|
||||
kind: 'template',
|
||||
},
|
||||
invocation: baseInvocation,
|
||||
invocation: buildEmbeddedInvocation(baseInvocation, serializedDefinition),
|
||||
};
|
||||
|
||||
const shouldBundle = Boolean(options.bundle ?? options.compile);
|
||||
let templateTmpDir: string | undefined;
|
||||
let templateOutputPath = options.outputPath;
|
||||
if (!templateOutputPath && shouldBundle) {
|
||||
const tmpPrefix = path.join(process.cwd(), 'tmp', 'mcporter-cli-');
|
||||
await fs.mkdir(path.dirname(tmpPrefix), { recursive: true });
|
||||
templateTmpDir = await fs.mkdtemp(tmpPrefix);
|
||||
templateOutputPath = path.join(templateTmpDir, `${name}.ts`);
|
||||
templateTmpDir = resolveImplicitTemplateDir(name, serializedDefinition);
|
||||
await fs.mkdir(templateTmpDir, { recursive: true });
|
||||
templateOutputPath = path.join(templateTmpDir, `${sanitizePathSegment(name) || 'server'}.ts`);
|
||||
}
|
||||
const templateSourcePath = path.resolve(templateOutputPath ?? path.resolve(process.cwd(), `${name}.ts`));
|
||||
let resolvedBundleTarget: string | undefined;
|
||||
@ -121,7 +128,7 @@ export async function generateCli(
|
||||
runtimeScriptPath,
|
||||
runtimeKind,
|
||||
timeoutMs,
|
||||
definition,
|
||||
definition: embeddedDefinition,
|
||||
serverName: name,
|
||||
tools: toolMetadata,
|
||||
generator,
|
||||
@ -156,13 +163,49 @@ export async function generateCli(
|
||||
}
|
||||
} finally {
|
||||
if (templateTmpDir) {
|
||||
await fs.rm(templateTmpDir, { recursive: true, force: true }).catch(() => {});
|
||||
await fs.rm(outputPath, { force: true }).catch(() => {});
|
||||
await fs.rmdir(templateTmpDir).catch(() => {});
|
||||
await fs.rmdir(path.dirname(templateTmpDir)).catch(() => {});
|
||||
}
|
||||
}
|
||||
|
||||
return { outputPath: options.outputPath ?? outputPath, bundlePath, compilePath };
|
||||
}
|
||||
|
||||
function stripBuildSources(definition: ServerDefinition): ServerDefinition {
|
||||
const { source: _source, sources: _sources, ...runtimeDefinition } = definition;
|
||||
return runtimeDefinition;
|
||||
}
|
||||
|
||||
function buildEmbeddedInvocation(
|
||||
invocation: CliArtifactMetadata['invocation'],
|
||||
definition: ReturnType<typeof serializeDefinition>
|
||||
): CliArtifactMetadata['invocation'] {
|
||||
return {
|
||||
serverRef: stableJsonStringify(definition),
|
||||
runtime: invocation.runtime,
|
||||
bundler: invocation.bundler,
|
||||
timeoutMs: invocation.timeoutMs,
|
||||
minify: invocation.minify,
|
||||
includeTools: invocation.includeTools,
|
||||
excludeTools: invocation.excludeTools,
|
||||
bundle: typeof invocation.bundle === 'boolean' ? invocation.bundle : undefined,
|
||||
compile: invocation.compile,
|
||||
};
|
||||
}
|
||||
|
||||
function resolveImplicitTemplateDir(serverName: string, definition: ReturnType<typeof serializeDefinition>): string {
|
||||
const hash = createHash('sha256').update(stableJsonStringify({ serverName, definition })).digest('hex').slice(0, 12);
|
||||
return path.join(process.cwd(), 'tmp', 'mcporter-cli', `${sanitizePathSegment(serverName) || 'server'}-${hash}`);
|
||||
}
|
||||
|
||||
function sanitizePathSegment(value: string): string {
|
||||
return value
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9._-]+/g, '-')
|
||||
.replace(/^-+|-+$/g, '');
|
||||
}
|
||||
|
||||
function applyToolFilters(tools: ServerToolInfo[], includeTools?: string[], excludeTools?: string[]): ServerToolInfo[] {
|
||||
if (includeTools && excludeTools) {
|
||||
throw new Error('Internal error: both includeTools and excludeTools provided to generateCli.');
|
||||
|
||||
@ -468,6 +468,39 @@ describeGenerateCli('generateCli', () => {
|
||||
expect(JSON.parse(callStdout)).toEqual({ result: 42 });
|
||||
}, 60_000);
|
||||
|
||||
it('generates byte-identical bundles for unchanged inputs', async () => {
|
||||
const inline = JSON.stringify({
|
||||
name: 'deterministic-bundle',
|
||||
description: 'Deterministic bundle integration server',
|
||||
command: baseUrl.toString(),
|
||||
});
|
||||
const firstBundlePath = path.join(tmpDir, 'deterministic-a.js');
|
||||
const secondBundlePath = path.join(tmpDir, 'deterministic-b.js');
|
||||
await fs.rm(firstBundlePath, { force: true });
|
||||
await fs.rm(secondBundlePath, { force: true });
|
||||
await ensureDistBuilt();
|
||||
|
||||
await generateCli({
|
||||
serverRef: inline,
|
||||
runtime: 'node',
|
||||
timeoutMs: 5_000,
|
||||
bundle: firstBundlePath,
|
||||
});
|
||||
await generateCli({
|
||||
serverRef: inline,
|
||||
runtime: 'node',
|
||||
timeoutMs: 5_000,
|
||||
bundle: secondBundlePath,
|
||||
});
|
||||
|
||||
const first = await fs.readFile(firstBundlePath, 'utf8');
|
||||
const second = await fs.readFile(secondBundlePath, 'utf8');
|
||||
expect(first).toBe(second);
|
||||
expect(first).not.toContain(firstBundlePath);
|
||||
expect(first).not.toContain(secondBundlePath);
|
||||
expect(first).not.toMatch(/mcporter-cli-[A-Za-z0-9]{6}/);
|
||||
}, 60_000);
|
||||
|
||||
it('maps CLI options to Commander camelCase properties', async () => {
|
||||
const inline = JSON.stringify({
|
||||
name: 'case-options',
|
||||
|
||||
Loading…
Reference in New Issue
Block a user