Compare commits

...

3 Commits

Author SHA1 Message Date
Sarah Fortune
94a8eff5e6 Merge remote-tracking branch 'origin/main' into fix/file-store-root-writes
# Conflicts:
#	src/file-store.ts
#	test/new-primitives.test.ts
2026-05-05 18:19:02 -07:00
Sarah Fortune
942f29ab20 Stream file store writes through pinned helper 2026-05-05 17:59:08 -07:00
Sarah Fortune
14aace8597 Harden file store writes against symlink swaps 2026-05-05 17:45:14 -07:00
2 changed files with 361 additions and 113 deletions

View File

@ -1,15 +1,22 @@
import { randomUUID } from "node:crypto";
import syncFs from "node:fs";
import syncFs, { constants as fsConstants } from "node:fs";
import fs from "node:fs/promises";
import path from "node:path";
import type { Readable } from "node:stream";
import { pipeline } from "node:stream/promises";
import { FsSafeError } from "./errors.js";
import { sameFileIdentity } from "./file-identity.js";
import { createJsonStore, type JsonFileStoreOptions, type JsonStore } from "./json-document-store.js";
import { runPinnedWriteHelper } from "./pinned-write.js";
import { isPathInside, resolveSafeRelativePath } from "./path.js";
import { root, type OpenResult, type ReadResult, type Root, type RootReadOptions } from "./root.js";
import {
resolveOpenedFileRealPathForHandle,
root,
type OpenResult,
type ReadResult,
type Root,
type RootReadOptions,
} from "./root.js";
import { writeSecretFileAtomic } from "./secret-file.js";
import { writeSiblingTempFile } from "./sibling-temp.js";
export type FileStoreOptions = {
rootDir: string;
@ -105,6 +112,12 @@ function assertRelativePath(relativePath: string): string {
return raw.replaceAll("\\", "/");
}
function assertMaxBytes(size: number, maxBytes?: number): void {
if (maxBytes !== undefined && size > maxBytes) {
throw new FsSafeError("too-large", `file exceeds maximum size of ${maxBytes} bytes`);
}
}
function resolveStorePath(rootDir: string, relativePath: string): string {
return resolveSafeRelativePath(rootDir, assertRelativePath(relativePath));
}
@ -115,15 +128,144 @@ function assertStoreFilePath(rootDir: string, filePath: string): void {
}
}
async function ensureParent(filePath: string, mode: number): Promise<void> {
const dir = path.dirname(filePath);
await fs.mkdir(dir, { recursive: true, mode });
await fs.chmod(dir, mode).catch(() => undefined);
function parentRelativePath(relativePath: string): string {
const parent = path.posix.dirname(assertRelativePath(relativePath));
return parent === "." ? "" : parent;
}
function assertMaxBytes(size: number, maxBytes?: number): void {
if (maxBytes !== undefined && size > maxBytes) {
throw new FsSafeError("too-large", `file exceeds maximum size of ${maxBytes} bytes`);
async function chmodDirectoryInRootBestEffort(
scopedRoot: Root,
relativePath: string,
mode: number,
): Promise<void> {
if (!relativePath) {
return;
}
const dirPath = await scopedRoot.resolve(relativePath);
const directoryFlag = "O_DIRECTORY" in fsConstants ? (fsConstants.O_DIRECTORY as number) : 0;
const noFollowFlag =
process.platform !== "win32" && "O_NOFOLLOW" in fsConstants
? (fsConstants.O_NOFOLLOW as number)
: 0;
const handle = await fs.open(dirPath, fsConstants.O_RDONLY | directoryFlag | noFollowFlag);
try {
const stat = await handle.stat();
if (!stat.isDirectory()) {
return;
}
const realPath = await resolveOpenedFileRealPathForHandle(handle, dirPath);
if (!isPathInside(scopedRoot.rootWithSep, realPath)) {
throw new FsSafeError("outside-workspace", "directory is outside store root");
}
await handle.chmod(mode).catch(() => undefined);
} finally {
await handle.close().catch(() => undefined);
}
}
async function ensureParentInRoot(
scopedRoot: Root,
relativePath: string,
mode: number,
): Promise<void> {
const parent = parentRelativePath(relativePath);
if (!parent) {
return;
}
await scopedRoot.mkdir(parent);
await chmodDirectoryInRootBestEffort(scopedRoot, parent, mode).catch(() => undefined);
}
async function openWritableStoreRoot(params: {
rootDir: string;
dirMode: number;
maxBytes?: number;
}): Promise<Root> {
await fs.mkdir(params.rootDir, { recursive: true, mode: params.dirMode });
await fs.chmod(params.rootDir, params.dirMode).catch(() => undefined);
return await root(params.rootDir, { hardlinks: "reject", maxBytes: params.maxBytes });
}
function normalizePinnedStreamWriteError(error: unknown): Error {
if (error instanceof FsSafeError) {
return error;
}
return new FsSafeError("path-alias", "path alias escape blocked", {
cause: error instanceof Error ? error : undefined,
});
}
async function readStreamIntoBuffer(stream: Readable, maxBytes?: number): Promise<Buffer> {
const chunks: Buffer[] = [];
let total = 0;
for await (const chunk of stream) {
const buffer =
typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk as Uint8Array);
total += buffer.byteLength;
assertMaxBytes(total, maxBytes);
chunks.push(buffer);
}
return Buffer.concat(chunks);
}
async function writeStreamInRoot(params: {
scopedRoot: Root;
relativePath: string;
stream: Readable;
maxBytes?: number;
mode: number;
}): Promise<void> {
const relativePath = assertRelativePath(params.relativePath);
const relativeParentPath = parentRelativePath(relativePath);
const basename = path.posix.basename(relativePath);
if (!basename || basename === "." || basename === "/") {
throw new FsSafeError("invalid-path", "invalid target path");
}
if (process.platform === "win32") {
await params.scopedRoot.write(
relativePath,
await readStreamIntoBuffer(params.stream, params.maxBytes),
{
mkdir: false,
mode: params.mode,
},
);
return;
}
let identity;
try {
identity = await runPinnedWriteHelper({
rootPath: params.scopedRoot.rootReal,
relativeParentPath,
basename,
mkdir: false,
mode: params.mode,
overwrite: true,
maxBytes: params.maxBytes,
input: {
kind: "stream",
stream: params.stream,
},
});
} catch (error) {
throw normalizePinnedStreamWriteError(error);
}
const opened = await params.scopedRoot.open(relativePath, {
hardlinks: "reject",
nonBlockingRead: true,
});
try {
if (!sameFileIdentity(opened.stat, identity)) {
throw new FsSafeError("path-mismatch", "path changed during write");
}
if (!isPathInside(params.scopedRoot.rootWithSep, opened.realPath)) {
throw new FsSafeError("outside-workspace", "file is outside store root");
}
} finally {
await opened.handle.close().catch(() => undefined);
}
}
@ -134,7 +276,7 @@ function isNotFound(error: unknown): boolean {
(error as NodeJS.ErrnoException).code === "ENOTDIR";
}
async function copyIntoRoot(params: {
export async function copyIntoRoot(params: {
rootDir: string;
relativePath: string;
sourcePath: string;
@ -143,26 +285,26 @@ async function copyIntoRoot(params: {
mode?: number;
tempPrefix?: string;
}): Promise<string> {
const destination = resolveStorePath(params.rootDir, params.relativePath);
const relativePath = assertRelativePath(params.relativePath);
const destination = resolveStorePath(params.rootDir, relativePath);
const sourceStat = await fs.lstat(params.sourcePath);
if (sourceStat.isSymbolicLink() || !sourceStat.isFile()) {
throw new FsSafeError("not-file", "source path is not a file");
}
assertMaxBytes(sourceStat.size, params.maxBytes);
await ensureParent(destination, params.dirMode ?? 0o700);
const result = await writeSiblingTempFile({
dir: path.dirname(destination),
dirMode: params.dirMode ?? 0o700,
mode: params.mode ?? 0o600,
tempPrefix: params.tempPrefix ?? `.${path.basename(destination)}`,
writeTemp: async (tempPath) => {
await fs.copyFile(params.sourcePath, tempPath);
},
resolveFinalPath: () => destination,
syncTempFile: true,
syncParentDir: true,
const dirMode = params.dirMode ?? 0o700;
const scopedRoot = await openWritableStoreRoot({
rootDir: params.rootDir,
dirMode,
maxBytes: params.maxBytes,
});
return result.filePath;
await ensureParentInRoot(scopedRoot, relativePath, dirMode);
await scopedRoot.copyIn(relativePath, params.sourcePath, {
maxBytes: params.maxBytes,
mkdir: false,
mode: params.mode ?? 0o600,
});
return destination;
}
export function fileStore(options: FileStoreOptions): FileStore {
@ -181,7 +323,8 @@ export function fileStore(options: FileStoreOptions): FileStore {
data: string | Uint8Array,
writeOptions?: FileStoreWriteOptions,
): Promise<string> {
const destination = resolveStorePath(rootDir, relativePath);
const safeRelativePath = assertRelativePath(relativePath);
const destination = resolveStorePath(rootDir, safeRelativePath);
const content = Buffer.isBuffer(data) ? data : Buffer.from(data);
assertMaxBytes(content.byteLength, writeOptions?.maxBytes ?? maxBytes);
if (privateMode) {
@ -194,20 +337,18 @@ export function fileStore(options: FileStoreOptions): FileStore {
});
return destination;
}
await ensureParent(destination, writeOptions?.dirMode ?? dirMode);
const result = await writeSiblingTempFile({
dir: path.dirname(destination),
dirMode: writeOptions?.dirMode ?? dirMode,
mode: writeOptions?.mode ?? mode,
tempPrefix: writeOptions?.tempPrefix ?? `.${path.basename(destination)}`,
writeTemp: async (tempPath) => {
await fs.writeFile(tempPath, content);
},
resolveFinalPath: () => destination,
syncTempFile: true,
syncParentDir: true,
const writeDirMode = writeOptions?.dirMode ?? dirMode;
const scopedRoot = await openWritableStoreRoot({
rootDir,
dirMode: writeDirMode,
maxBytes: writeOptions?.maxBytes ?? maxBytes,
});
return result.filePath;
await ensureParentInRoot(scopedRoot, safeRelativePath, writeDirMode);
await scopedRoot.write(safeRelativePath, content, {
mkdir: false,
mode: writeOptions?.mode ?? mode,
});
return destination;
}
return {
@ -216,56 +357,34 @@ export function fileStore(options: FileStoreOptions): FileStore {
root: openRoot,
write,
writeStream: async (relativePath, stream, writeOptions) => {
const destination = resolveStorePath(rootDir, relativePath);
const safeRelativePath = assertRelativePath(relativePath);
const destination = resolveStorePath(rootDir, safeRelativePath);
const limit = writeOptions?.maxBytes ?? maxBytes;
if (privateMode) {
const chunks: Buffer[] = [];
let total = 0;
for await (const chunk of stream) {
const buffer =
typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk as Uint8Array);
total += buffer.byteLength;
assertMaxBytes(total, limit);
chunks.push(buffer);
}
await writeSecretFileAtomic({
rootDir,
filePath: destination,
content: Buffer.concat(chunks),
content: await readStreamIntoBuffer(stream, limit),
dirMode: writeOptions?.dirMode ?? dirMode,
mode: writeOptions?.mode ?? mode,
});
return destination;
}
await ensureParent(destination, writeOptions?.dirMode ?? dirMode);
let total = 0;
const result = await writeSiblingTempFile({
dir: path.dirname(destination),
dirMode: writeOptions?.dirMode ?? dirMode,
mode: writeOptions?.mode ?? mode,
tempPrefix: writeOptions?.tempPrefix ?? `.${path.basename(destination)}`,
writeTemp: async (tempPath) => {
const writable = await fs.open(tempPath, "w", writeOptions?.mode ?? mode);
try {
const out = writable.createWriteStream();
stream.on("data", (chunk: Buffer | string) => {
total += Buffer.byteLength(chunk);
if (limit !== undefined && total > limit) {
stream.destroy(
new FsSafeError("too-large", `file exceeds maximum size of ${limit} bytes`),
);
}
});
await pipeline(stream, out);
} finally {
await writable.close().catch(() => undefined);
}
},
resolveFinalPath: () => destination,
syncTempFile: true,
syncParentDir: true,
const writeDirMode = writeOptions?.dirMode ?? dirMode;
const scopedRoot = await openWritableStoreRoot({
rootDir,
dirMode: writeDirMode,
maxBytes: limit,
});
return result.filePath;
await ensureParentInRoot(scopedRoot, safeRelativePath, writeDirMode);
await writeStreamInRoot({
scopedRoot,
relativePath: safeRelativePath,
stream,
maxBytes: limit,
mode: writeOptions?.mode ?? mode,
});
return destination;
},
copyIn: async (relativePath, sourcePath, writeOptions) =>
privateMode
@ -405,9 +524,7 @@ export function fileStore(options: FileStoreOptions): FileStore {
};
}
function ensureParentSync(filePath: string, mode: number): void {
const dir = path.dirname(filePath);
syncFs.mkdirSync(dir, { recursive: true, mode });
function chmodDirectorySyncBestEffort(dir: string, mode: number): void {
try {
syncFs.chmodSync(dir, mode);
} catch {
@ -415,48 +532,75 @@ function ensureParentSync(filePath: string, mode: number): void {
}
}
function ensurePrivateDirectorySync(rootDir: string, targetDir: string, mode: number): void {
const root = path.resolve(rootDir);
const target = path.resolve(targetDir);
function assertDirectoryInsideRootSync(params: {
rootReal: string;
dir: string;
messagePrefix: string;
}): string {
const stat = syncFs.lstatSync(params.dir);
if (stat.isSymbolicLink() || !stat.isDirectory()) {
throw new FsSafeError("not-file", `${params.messagePrefix} must be a directory: ${params.dir}`);
}
const realPath = syncFs.realpathSync(params.dir);
if (!isPathInside(params.rootReal, realPath)) {
throw new FsSafeError("outside-workspace", `${params.messagePrefix} escapes root`);
}
return realPath;
}
function ensureStoreDirectorySync(params: {
rootDir: string;
targetDir: string;
mode: number;
privateMode: boolean;
}): string {
const root = path.resolve(params.rootDir);
const target = path.resolve(params.targetDir);
const label = params.privateMode ? "private store directory" : "store directory";
assertStoreFilePath(root, target);
let current = root;
syncFs.mkdirSync(current, { recursive: true, mode });
const rootStat = syncFs.lstatSync(current);
syncFs.mkdirSync(root, { recursive: true, mode: params.mode });
const rootStat = syncFs.lstatSync(root);
if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) {
throw new FsSafeError("not-file", `private store root must be a directory: ${current}`);
}
try {
syncFs.chmodSync(current, mode);
} catch {
// Best-effort on platforms that do not enforce POSIX modes.
throw new FsSafeError("not-file", `${label} root must be a directory: ${root}`);
}
const rootReal = syncFs.realpathSync(root);
chmodDirectorySyncBestEffort(root, params.mode);
let current = root;
for (const segment of path.relative(root, target).split(path.sep).filter(Boolean)) {
current = path.join(current, segment);
try {
const stat = syncFs.lstatSync(current);
if (stat.isSymbolicLink() || !stat.isDirectory()) {
throw new FsSafeError(
"not-file",
`private store directory component must be a directory: ${current}`,
);
throw new FsSafeError("not-file", `${label} component must be a directory: ${current}`);
}
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
throw error;
}
syncFs.mkdirSync(current, { mode });
}
const rootReal = syncFs.realpathSync(root);
const currentReal = syncFs.realpathSync(current);
if (!isPathInside(rootReal, currentReal)) {
throw new FsSafeError("outside-workspace", "private store directory escapes root");
}
try {
syncFs.chmodSync(current, mode);
} catch {
// Best-effort on platforms that do not enforce POSIX modes.
syncFs.mkdirSync(current, { mode: params.mode });
const createdStat = syncFs.lstatSync(current);
if (createdStat.isSymbolicLink() || !createdStat.isDirectory()) {
throw new FsSafeError("not-file", `${label} component must be a directory: ${current}`);
}
}
assertDirectoryInsideRootSync({
rootReal,
dir: current,
messagePrefix: label,
});
chmodDirectorySyncBestEffort(current, params.mode);
}
return assertDirectoryInsideRootSync({
rootReal,
dir: target,
messagePrefix: label,
});
}
function ensurePrivateDirectorySync(rootDir: string, targetDir: string, mode: number): void {
ensureStoreDirectorySync({ rootDir, targetDir, mode, privateMode: true });
}
function writeFileSyncAtomic(params: {
@ -469,6 +613,15 @@ function writeFileSyncAtomic(params: {
}): string {
const filePath = path.resolve(params.filePath);
assertStoreFilePath(params.rootDir, filePath);
const parentDir = path.dirname(filePath);
const parentRealPath = params.privateMode
? undefined
: ensureStoreDirectorySync({
rootDir: params.rootDir,
targetDir: parentDir,
mode: params.dirMode,
privateMode: false,
});
if (params.privateMode) {
ensurePrivateDirectorySync(params.rootDir, path.dirname(filePath), params.dirMode);
try {
@ -481,10 +634,8 @@ function writeFileSyncAtomic(params: {
throw error;
}
}
} else {
ensureParentSync(filePath, params.dirMode);
}
const tempPath = path.join(path.dirname(filePath), `.fs-safe-${process.pid}-${randomUUID()}.tmp`);
const tempPath = path.join(parentDir, `.fs-safe-${process.pid}-${randomUUID()}.tmp`);
let tempExists = false;
try {
syncFs.writeFileSync(tempPath, params.content, { flag: "wx", mode: params.mode });
@ -494,6 +645,9 @@ function writeFileSyncAtomic(params: {
} catch {
// Best-effort on platforms that do not enforce POSIX modes.
}
if (parentRealPath !== undefined && syncFs.realpathSync(parentDir) !== parentRealPath) {
throw new FsSafeError("path-mismatch", "store parent directory changed during write");
}
syncFs.renameSync(tempPath, filePath);
tempExists = false;
try {

View File

@ -1,7 +1,9 @@
import { randomUUID } from "node:crypto";
import fs from "node:fs/promises";
import syncFs from "node:fs";
import os from "node:os";
import path from "node:path";
import { Readable } from "node:stream";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
import {
appendRegularFile,
@ -26,7 +28,7 @@ import { replaceFileAtomic, replaceFileAtomicSync } from "../src/replace-file.js
import { movePathWithCopyFallback } from "../src/move-path.js";
import { writeSiblingTempFile } from "../src/sibling-temp.js";
import { acquireFileLock, createFileLockManager, withFileLock } from "../src/file-lock.js";
import { fileStore, fileStoreSync } from "../src/file-store.js";
import { copyIntoRoot, fileStore, fileStoreSync } from "../src/file-store.js";
import { jsonStore } from "../src/json-store.js";
import {
createIcaclsResetCommand,
@ -133,6 +135,9 @@ describe("file store", () => {
await store.writeJson("media/state.json", { ok: true });
await expect(store.readJson("media/state.json")).resolves.toEqual({ ok: true });
await store.writeStream("media/stream.txt", Readable.from(["stream"]));
await expect(store.readBytes("media/stream.txt")).resolves.toEqual(Buffer.from("stream"));
const source = path.join(root, "source.bin");
await fs.writeFile(source, "source", "utf8");
await store.copyIn("media/b.txt", source);
@ -150,6 +155,95 @@ describe("file store", () => {
code: "too-large",
});
});
it.runIf(process.platform !== "win32")("rejects symlink parent writes", async () => {
const storeRoot = path.join(root, "store");
const outside = path.join(root, "outside");
const source = path.join(root, "source.txt");
await fs.mkdir(storeRoot);
await fs.mkdir(outside);
await fs.writeFile(source, "pwned", "utf8");
await fs.symlink(outside, path.join(storeRoot, "link"), "dir");
const store = fileStore({ rootDir: storeRoot });
await expect(store.write("link/write.txt", "pwned")).rejects.toBeTruthy();
await expect(store.writeStream("link/stream.txt", Readable.from(["pwned"]))).rejects
.toBeTruthy();
await expect(store.copyIn("link/copy.txt", source)).rejects.toBeTruthy();
await expect(
copyIntoRoot({
rootDir: storeRoot,
relativePath: "link/advanced-copy.txt",
sourcePath: source,
}),
).rejects.toBeTruthy();
const syncStore = fileStoreSync({ rootDir: storeRoot });
expect(() => syncStore.write("link/sync-write.txt", "pwned")).toThrow();
expect(() => syncStore.writeText("link/sync-text.txt", "pwned")).toThrow();
expect(() => syncStore.writeJson("link/sync-json.txt", { pwned: true })).toThrow();
await expect(fs.readdir(outside)).resolves.toEqual([]);
});
it.runIf(process.platform !== "win32")(
"rejects symlink parent swaps during a single write operation",
async () => {
async function expectSwapBlocked(
operation: (params: {
store: ReturnType<typeof fileStore>;
storeRoot: string;
source: string;
}) => Promise<unknown>,
): Promise<void> {
const storeRoot = path.join(root, `store-${randomUUID()}`);
const outside = path.join(root, `outside-${randomUUID()}`);
const link = path.join(storeRoot, "link");
const source = path.join(root, `source-${randomUUID()}.txt`);
await fs.mkdir(link, { recursive: true });
await fs.mkdir(outside);
await fs.writeFile(source, "pwned", "utf8");
const canonicalLink = path.join(await fs.realpath(storeRoot), "link");
const store = fileStore({ rootDir: storeRoot });
const realOpen = fs.open.bind(fs);
let swapped = false;
const openSpy = vi.spyOn(fs, "open").mockImplementation(
(async (
target: Parameters<typeof fs.open>[0],
flags?: Parameters<typeof fs.open>[1],
mode?: Parameters<typeof fs.open>[2],
) => {
const targetPath = path.resolve(String(target));
if (!swapped && (targetPath === link || targetPath === canonicalLink)) {
swapped = true;
await fs.rm(link, { recursive: true, force: true });
await fs.symlink(outside, link, "dir");
}
return await realOpen(target, flags, mode);
}) as typeof fs.open,
);
try {
await expect(operation({ store, storeRoot, source })).rejects.toBeTruthy();
expect(swapped).toBe(true);
await expect(fs.readdir(outside)).resolves.toEqual([]);
} finally {
openSpy.mockRestore();
}
}
await expectSwapBlocked(({ store }) => store.write("link/write.txt", "pwned"));
await expectSwapBlocked(({ store }) =>
store.writeStream("link/stream.txt", Readable.from(["pwned"])),
);
await expectSwapBlocked(({ store, source }) => store.copyIn("link/copy.txt", source));
await expectSwapBlocked(({ storeRoot, source }) =>
copyIntoRoot({
rootDir: storeRoot,
relativePath: "link/advanced-copy.txt",
sourcePath: source,
}),
);
},
);
});
describe("json store", () => {