From 722cb41390bbfa1c28ea73dc24b467b54073f849 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 22:19:59 +0100 Subject: [PATCH 01/20] feat: make archive dependencies optional --- README.md | 2 +- docs/archive.md | 6 ++++ docs/install.md | 2 +- package.json | 2 +- pnpm-lock.yaml | 54 ++++++++++++++++++++++++------------ src/archive-zip-preflight.ts | 37 ++++++++++++++++++++++-- src/archive.ts | 29 ++++++++++++++++++- 7 files changed, 108 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 67e2094..e59da9b 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ This is a library-level guardrail, not OS-level isolation. It does not replace c pnpm add @openclaw/fs-safe ``` -Node 20.11 or newer. Core root/path/json/temp helpers avoid framework dependencies; archive helpers use `jszip` and `tar` for ZIP/TAR support. +Node 20.11 or newer. Core root/path/json/temp helpers avoid framework dependencies. Archive helpers use optional `jszip` and `tar` dependencies for ZIP/TAR support; installs that omit optional dependencies can still use every non-archive subpath. ## Quick start diff --git a/docs/archive.md b/docs/archive.md index 149bfcb..390e422 100644 --- a/docs/archive.md +++ b/docs/archive.md @@ -2,6 +2,12 @@ `@openclaw/fs-safe/archive` extracts ZIP and TAR archives behind one API, with traversal checks, blocked-link-type rejection, and entry-count and byte budgets. Extraction stages into a private directory and merges through the same safe-open boundary used by direct writes — a symlinked entry can't trick the merge into following an out-of-tree path. +Archive extraction uses optional runtime dependencies: `jszip` for ZIP and `tar` +for TAR. Installs that omit optional dependencies can still import this subpath, +inspect archive kinds, and use pure path/limit helpers, but extraction or ZIP +loading fails with a clear message until the matching optional dependency is +installed. + ```ts import { extractArchive, resolveArchiveKind } from "@openclaw/fs-safe/archive"; ``` diff --git a/docs/install.md b/docs/install.md index 5f3c0d3..4f83bc9 100644 --- a/docs/install.md +++ b/docs/install.md @@ -81,7 +81,7 @@ Use the main entry for the common surface, or the focused subpaths when you want ## Runtime dependencies -`@openclaw/fs-safe` depends on `jszip` and `tar` for [archive extraction](archive.md). Both are loaded lazily — if your code never touches the archive subpath, the runtime cost is negligible. +`@openclaw/fs-safe` lists `jszip` and `tar` as optional dependencies for [archive extraction](archive.md). They are loaded lazily and only required when ZIP/TAR helpers run. Installs that omit optional dependencies can still import and use every non-archive subpath; archive calls fail with a clear missing-optional-dependency message. There are no peer dependencies and no native build step. diff --git a/package.json b/package.json index 0f56174..364385c 100644 --- a/package.json +++ b/package.json @@ -92,7 +92,7 @@ "check": "pnpm build && pnpm test", "docs:site": "node scripts/build-docs-site.mjs" }, - "dependencies": { + "optionalDependencies": { "jszip": "^3.10.1", "tar": "7.5.13" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index d0c675b..f92f05e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -7,13 +7,6 @@ settings: importers: .: - dependencies: - jszip: - specifier: ^3.10.1 - version: 3.10.1 - tar: - specifier: 7.5.13 - version: 7.5.13 devDependencies: '@types/node': specifier: ^22.15.19 @@ -27,6 +20,13 @@ importers: vitest: specifier: ^3.1.4 version: 3.2.4(@types/node@22.19.17) + optionalDependencies: + jszip: + specifier: ^3.10.1 + version: 3.10.1 + tar: + specifier: 7.5.13 + version: 7.5.13 packages: @@ -1002,6 +1002,7 @@ snapshots: '@isaacs/fs-minipass@4.0.1': dependencies: minipass: 7.1.3 + optional: true '@istanbuljs/schema@0.1.6': {} @@ -1213,7 +1214,8 @@ snapshots: check-error@2.1.3: {} - chownr@3.0.0: {} + chownr@3.0.0: + optional: true color-convert@2.0.1: dependencies: @@ -1221,7 +1223,8 @@ snapshots: color-name@1.1.4: {} - core-util-is@1.0.3: {} + core-util-is@1.0.3: + optional: true cross-spawn@7.0.6: dependencies: @@ -1303,13 +1306,16 @@ snapshots: html-escaper@2.0.2: {} - immediate@3.0.6: {} + immediate@3.0.6: + optional: true - inherits@2.0.4: {} + inherits@2.0.4: + optional: true is-fullwidth-code-point@3.0.0: {} - isarray@1.0.0: {} + isarray@1.0.0: + optional: true isexe@2.0.0: {} @@ -1350,10 +1356,12 @@ snapshots: pako: 1.0.11 readable-stream: 2.3.8 setimmediate: 1.0.5 + optional: true lie@3.3.0: dependencies: immediate: 3.0.6 + optional: true loupe@3.2.1: {} @@ -1386,6 +1394,7 @@ snapshots: minizlib@3.1.0: dependencies: minipass: 7.1.3 + optional: true ms@2.1.3: {} @@ -1393,7 +1402,8 @@ snapshots: package-json-from-dist@1.0.1: {} - pako@1.0.11: {} + pako@1.0.11: + optional: true path-key@3.1.1: {} @@ -1416,7 +1426,8 @@ snapshots: picocolors: 1.1.1 source-map-js: 1.2.1 - process-nextick-args@2.0.1: {} + process-nextick-args@2.0.1: + optional: true readable-stream@2.3.8: dependencies: @@ -1427,6 +1438,7 @@ snapshots: safe-buffer: 5.1.2 string_decoder: 1.1.1 util-deprecate: 1.0.2 + optional: true rollup@4.60.2: dependencies: @@ -1459,11 +1471,13 @@ snapshots: '@rollup/rollup-win32-x64-msvc': 4.60.2 fsevents: 2.3.3 - safe-buffer@5.1.2: {} + safe-buffer@5.1.2: + optional: true semver@7.7.4: {} - setimmediate@1.0.5: {} + setimmediate@1.0.5: + optional: true shebang-command@2.0.0: dependencies: @@ -1496,6 +1510,7 @@ snapshots: string_decoder@1.1.1: dependencies: safe-buffer: 5.1.2 + optional: true strip-ansi@6.0.1: dependencies: @@ -1520,6 +1535,7 @@ snapshots: minipass: 7.1.3 minizlib: 3.1.0 yallist: 5.0.0 + optional: true test-exclude@7.0.2: dependencies: @@ -1546,7 +1562,8 @@ snapshots: undici-types@6.21.0: {} - util-deprecate@1.0.2: {} + util-deprecate@1.0.2: + optional: true vite-node@3.2.4(@types/node@22.19.17): dependencies: @@ -1643,4 +1660,5 @@ snapshots: string-width: 5.1.2 strip-ansi: 7.2.0 - yallist@5.0.0: {} + yallist@5.0.0: + optional: true diff --git a/src/archive-zip-preflight.ts b/src/archive-zip-preflight.ts index c2535fa..e9ccbea 100644 --- a/src/archive-zip-preflight.ts +++ b/src/archive-zip-preflight.ts @@ -1,4 +1,3 @@ -import JSZip from "jszip"; import { ARCHIVE_LIMIT_ERROR_CODE, ArchiveLimitError, @@ -7,6 +6,14 @@ import { type ArchiveExtractLimits, } from "./archive-limits.js"; +export type ZipArchiveWithFiles = { + files: Record; +}; + +type JsZipConstructor = { + loadAsync(buffer: Buffer | Uint8Array): Promise; +}; + const ZIP_EOCD_SIGNATURE = 0x06054b50; const ZIP64_EOCD_SIGNATURE = 0x06064b50; const ZIP64_EOCD_LOCATOR_SIGNATURE = 0x07064b50; @@ -193,7 +200,7 @@ export function readZipCentralDirectoryEntryCount(buffer: Buffer | Uint8Array): export async function loadZipArchiveWithPreflight( buffer: Buffer | Uint8Array, limits?: ArchiveExtractLimits, -): Promise { +): Promise { const resolvedLimits = resolveExtractLimits(limits); if (buffer.byteLength > resolvedLimits.maxArchiveBytes) { throw new ArchiveLimitError(ARCHIVE_LIMIT_ERROR_CODE.ARCHIVE_SIZE_EXCEEDS_LIMIT); @@ -202,5 +209,31 @@ export async function loadZipArchiveWithPreflight( if (entryCount !== null) { assertArchiveEntryCountWithinLimit(entryCount, resolvedLimits); } + const JSZip = await importOptionalJsZip(); return await JSZip.loadAsync(buffer); } + +async function importOptionalJsZip(): Promise { + try { + const module = await import("jszip"); + const candidate: unknown = + typeof module === "function" ? module : (module as { default?: unknown }).default; + if ( + (typeof candidate !== "object" && typeof candidate !== "function") || + candidate === null || + typeof (candidate as { loadAsync?: unknown }).loadAsync !== "function" + ) { + throw new Error('Optional archive dependency "jszip" does not expose loadAsync().'); + } + return candidate as JsZipConstructor; + } catch (err) { + throw missingOptionalArchiveDependencyError("jszip", err); + } +} + +function missingOptionalArchiveDependencyError(packageName: "jszip", cause: unknown): Error { + return new Error( + `Optional archive dependency "${packageName}" is not installed. Install it to use ZIP archive helpers from @openclaw/fs-safe/archive.`, + { cause }, + ); +} diff --git a/src/archive.ts b/src/archive.ts index 524e0b8..3bb19f7 100644 --- a/src/archive.ts +++ b/src/archive.ts @@ -5,7 +5,6 @@ import fs from "node:fs/promises"; import path from "node:path"; import { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; -import * as tar from "tar"; import { resolveArchiveOutputPath, stripArchivePath, @@ -71,6 +70,7 @@ export { createTarEntryPreflightChecker, type TarEntryInfo } from "./archive-tar export { loadZipArchiveWithPreflight, readZipCentralDirectoryEntryCount, + type ZipArchiveWithFiles, } from "./archive-zip-preflight.js"; const SUPPORTS_NOFOLLOW = process.platform !== "win32" && "O_NOFOLLOW" in fsConstants; @@ -111,6 +111,17 @@ type ZipEntry = { }; type ZipExtractBudget = ReturnType; +type TarModule = { + x(options: { + file: string; + cwd: string; + strip: number; + gzip?: boolean; + preservePaths: false; + strict: true; + onReadEntry(this: unknown, entry: unknown): void; + }): Promise; +}; const ZIP_UNIX_FILE_TYPE_MASK = 0o170000; const ZIP_UNIX_SYMLINK_TYPE = 0o120000; @@ -304,6 +315,7 @@ export async function extractArchive(params: { if (kind === "tar") { await withTimeout( (async () => { + const tar = await importOptionalTar(); const limits = resolveExtractLimits(params.limits); const stat = await fs.stat(params.archivePath); if (stat.size > limits.maxArchiveBytes) { @@ -367,3 +379,18 @@ export async function extractArchive(params: { label, ); } + +async function importOptionalTar(): Promise { + try { + return await import("tar"); + } catch (err) { + throw missingOptionalArchiveDependencyError("tar", err); + } +} + +function missingOptionalArchiveDependencyError(packageName: "tar", cause: unknown): Error { + return new Error( + `Optional archive dependency "${packageName}" is not installed. Install it to use TAR archive helpers from @openclaw/fs-safe/archive.`, + { cause }, + ); +} From 43c6d2058e1308dfc3b4284b144443ed127e2cf6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 22:23:26 +0100 Subject: [PATCH 02/20] docs: clarify fs-safe positioning --- README.md | 4 ++-- docs/index.md | 6 +++--- docs/json.md | 2 +- docs/root.md | 2 +- docs/security-model.md | 6 +++++- 5 files changed, 12 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e59da9b..bb137c1 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # @openclaw/fs-safe -Race-resistant root-bounded filesystem primitives for Node.js. +Capability-style filesystem roots for Node.js apps that handle untrusted relative paths. -Use this when trusted application code has to touch caller-controlled paths inside a directory it owns. The package gives you one `root()` boundary that survives symlink swaps, `..` traversal, hardlink aliases, and TOCTOU rename races between check and use. +Use this when trusted application code has to touch caller-controlled paths inside a directory it owns. The package gives you one `root()` handle that survives symlink swaps, `..` traversal, hardlink aliases, and TOCTOU rename races between check and use. ## Why diff --git a/docs/index.md b/docs/index.md index edee628..58d6a63 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,12 +1,12 @@ --- title: Overview permalink: / -description: "Race-resistant root-bounded filesystem primitives for Node.js. One root() boundary that survives symlink swaps, traversal, hardlink aliases, and TOCTOU rename races between check and use." +description: "Capability-style filesystem roots for Node.js apps that handle untrusted relative paths." --- # fs-safe -Trusted Node.js code that has to touch caller-controlled paths inside a directory it owns gets one boundary it can rely on. `root()` returns a handle that resolves every relative path against a real directory, refuses anything that escapes it, pins the file you opened, and verifies the write landed where you intended. +Trusted Node.js code that has to touch caller-controlled paths inside a directory it owns gets one boundary it can rely on. `root()` returns a capability-style handle that resolves every relative path against a real directory, refuses anything that escapes it, pins the file you opened, and verifies the write landed where you intended. ## Why @@ -36,7 +36,7 @@ await fs.remove("notes/archive/today.txt"); ## Pick your path - **First time?** [Install](install.md), then walk through the [Quickstart](quickstart.md). Five minutes from `pnpm add` to a working root. -- **Designing a sandboxed feature.** Read the [Security model](security-model.md) before you trust the boundary, and the [Errors](errors.md) reference so you know what to catch. +- **Designing a workspace feature.** Read the [Security model](security-model.md) before you trust the boundary, and the [Errors](errors.md) reference so you know what to catch. - **Replacing ad-hoc atomic writes.** Jump to [Atomic writes](atomic.md) or, for keyed JSON state, [JSON files](json.md). - **Extracting an upload.** Start at [Archive extraction](archive.md) — handles ZIP and TAR with traversal, link, count, and byte limits. - **Running an agent in a sandbox.** [Private temp workspaces](temp.md) plus [secret files](secret-file.md) cover the common scratch-and-credentials shape. diff --git a/docs/json.md b/docs/json.md index 654a54a..5c271fb 100644 --- a/docs/json.md +++ b/docs/json.md @@ -151,7 +151,7 @@ const state = await readJsonIfExists("./state.json"); ## See also -- [JSON store](json-store.md) — a single-file state wrapper with fallback and optional sidecar locking. +- [JSON store](json-store.md) — a single-file state wrapper with explicit per-call fallback (`readOr` / `updateOr`) and optional sidecar locking. - [Atomic writes](atomic.md) — lower-level sibling-temp replacement helpers. - [Secret files](secret-file.md) — JSON-or-text writes with mode 0600 in mode 0700 dirs. - [Private state store](private-file-store.md) — root-bounded JSON+text helpers. diff --git a/docs/root.md b/docs/root.md index 6419128..8692c21 100644 --- a/docs/root.md +++ b/docs/root.md @@ -1,6 +1,6 @@ # root() -`root()` is the primary entry point. It takes a trusted directory and returns a `Root` handle whose methods accept relative paths and refuse to escape the directory. +`root()` is the primary entry point. It takes a trusted directory and returns a capability-style `Root` handle whose methods accept relative paths and refuse to escape the directory. ```ts import { root } from "@openclaw/fs-safe"; diff --git a/docs/security-model.md b/docs/security-model.md index fd8b378..cfce736 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -1,6 +1,6 @@ # Security model -`fs-safe` is a library-level guardrail. It assumes the calling process already has whatever filesystem permissions it needs and aims to stop trivial path tricks from broadening that authority. It is not a sandbox and does not replace operating-system isolation. +`fs-safe` is a library-level guardrail: a capability-style root handle for Node.js code that handles untrusted relative paths. It assumes the calling process already has whatever filesystem permissions it needs and aims to stop trivial path tricks from broadening that authority. It is not a sandbox and does not replace operating-system isolation. ## Threat model @@ -18,6 +18,7 @@ It does **not** defend against: - a process running with permissions to write anywhere on the filesystem and choosing to ignore the library - another process with the same UID racing to mutate the same directory between two separate `fs-safe` calls — the boundary is per-call, not per-session +- traversal across filesystem boundaries, bind mounts, device files, `/proc`-style virtual filesystems, or any other path your process can normally access from inside the root - container escape, TOCTOU between fork and exec of helpers, or kernel-level vulnerabilities - semantic content checks: file types, archive payload schemas, signature verification @@ -70,6 +71,9 @@ The library does not advertise different security guarantees per platform — it ## Limitations to keep in mind +- **This is not ambient authority removal.** Code that can import `node:fs` can still bypass the handle. Keep caller-controlled path operations behind `root()` by convention, review, and tests. +- **Absolute paths are escape hatches.** APIs that accept or return absolute paths exist for audit, ingest, and advanced composition. Prefer root-relative names in normal application flow. +- **Mount and device boundaries are outside the model.** `root()` keeps path traversal inside the directory tree; it does not make device files, bind mounts, or virtual filesystems safe to expose. - **Hardlink rejection** depends on platform-supplied link counts and is best-effort; do not use it as an authorization mechanism for capability decisions. - **`fs.fchown` / mode bits** are not enforced beyond what `replaceFileAtomic` and the secret-file helpers do — if you need stronger mode enforcement, set umask and inspect mode after writes. - **Archive extraction** rejects unsafe entries by default but does not interpret payload semantics. A "malicious safe" archive (valid paths, dangerous content) is your application layer's problem. From 49f1f54cc4c650d4dd6fbc708731625dd89d6cdb Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 22:32:11 +0100 Subject: [PATCH 03/20] feat: serialize same-target writes --- docs/atomic.md | 2 + docs/security-model.md | 19 +++-- package.json | 2 +- src/private-temp-workspace.ts | 19 ++++- src/replace-file.ts | 17 +++++ src/root.ts | 140 ++++++++++++++++++++-------------- src/sibling-temp.ts | 25 ++++-- src/temp-cleanup.ts | 53 +++++++++++++ src/temp-target.ts | 8 +- src/write-queue.ts | 21 +++++ test/atomic.test.ts | 55 +++++++++++++ 11 files changed, 284 insertions(+), 77 deletions(-) create mode 100644 src/temp-cleanup.ts create mode 100644 src/write-queue.ts diff --git a/docs/atomic.md b/docs/atomic.md index 0641892..8447aba 100644 --- a/docs/atomic.md +++ b/docs/atomic.md @@ -16,6 +16,8 @@ import { Write `content` to a sibling temp file in the destination directory, optionally `fsync` the temp file, optionally `fsync` the parent directory after rename, then atomically rename over the destination. +Async replacements to the same destination are serialized inside the current process, so two overlapping `replaceFileAtomic()` calls do not interleave their temp-write/rename phases. Use a sidecar lock when multiple processes may write the same target. + ```ts import { replaceFileAtomic } from "@openclaw/fs-safe/atomic"; diff --git a/docs/security-model.md b/docs/security-model.md index cfce736..aa79abe 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -52,6 +52,8 @@ When `hardlinks: "reject"` is set, reads stat the target and refuse if `nlink > `replaceFileAtomic` writes to a sibling temp file in the destination directory, optionally `fsync`s it, optionally `fsync`s the parent directory after rename, and atomically renames over the destination. On failure mid-write, the destination is either the old contents (rename never happened) or the new contents (rename succeeded). There is no half-written intermediate state visible at the destination path. +Within one process, async writes to the same target are queued so their temp-write/rename phases do not overlap. Cross-process writers still need an external protocol such as the sidecar lock helpers. + ### Archive extraction `extractArchive` first stages into a private temp directory (mode 0700) outside the destination, validates each entry path against `..` and absolute prefixes, refuses link-type entries by default, enforces entry count and byte budgets, and only then merges the staged tree into the destination through the same boundary checks used by direct writes. @@ -71,13 +73,16 @@ The library does not advertise different security guarantees per platform — it ## Limitations to keep in mind -- **This is not ambient authority removal.** Code that can import `node:fs` can still bypass the handle. Keep caller-controlled path operations behind `root()` by convention, review, and tests. -- **Absolute paths are escape hatches.** APIs that accept or return absolute paths exist for audit, ingest, and advanced composition. Prefer root-relative names in normal application flow. -- **Mount and device boundaries are outside the model.** `root()` keeps path traversal inside the directory tree; it does not make device files, bind mounts, or virtual filesystems safe to expose. -- **Hardlink rejection** depends on platform-supplied link counts and is best-effort; do not use it as an authorization mechanism for capability decisions. -- **`fs.fchown` / mode bits** are not enforced beyond what `replaceFileAtomic` and the secret-file helpers do — if you need stronger mode enforcement, set umask and inspect mode after writes. -- **Archive extraction** rejects unsafe entries by default but does not interpret payload semantics. A "malicious safe" archive (valid paths, dangerous content) is your application layer's problem. -- **Helper spawn failures** are reported via `helper-failed` / `helper-unavailable` codes. The library falls back to Node-only paths when the Python helper is unavailable; that fallback retains atomicity guarantees but loses some fd-relative race resistance. +| Limitation | What it means | +|---|---| +| Not ambient authority removal | Code that can import `node:fs` can still bypass the handle. Keep caller-controlled path operations behind `root()` by convention, review, and tests. | +| Absolute paths are escape hatches | APIs that accept or return absolute paths exist for audit, ingest, and advanced composition. Prefer root-relative names in normal application flow. | +| Not a mount/device boundary | `root()` keeps path traversal inside the directory tree; it does not make device files, bind mounts, or virtual filesystems safe to expose. | +| Per-call, not per-session | Another process with the same privileges can still mutate the tree between two separate calls. Use one verb method for the operation you need to make race-resistant. | +| Hardlink rejection is best-effort | Link-count checks depend on platform metadata. Treat `hardlinks: "reject"` as a tripwire, not an authorization primitive. | +| Mode bits are not a full policy engine | `replaceFileAtomic` and secret-file helpers set requested modes, but you should still set umask and inspect modes when policy requires it. | +| Archive extraction is path safety, not content safety | Unsafe entry paths and links are rejected; malicious payload contents remain your application layer's problem. | +| Helper failures degrade fd-relative hardening | `helper-failed` / `helper-unavailable` mean the Node fallback is being used. Atomicity remains, but some POSIX fd-relative race resistance is unavailable. | ## Recommended deployment shape diff --git a/package.json b/package.json index 364385c..afe5417 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@openclaw/fs-safe", "version": "0.0.0", - "description": "Race-resistant root-bounded filesystem primitives for Node.js.", + "description": "Capability-style filesystem roots for Node.js apps that handle untrusted relative paths.", "license": "MIT", "repository": { "type": "git", diff --git a/src/private-temp-workspace.ts b/src/private-temp-workspace.ts index 9849e67..7b0dded 100644 --- a/src/private-temp-workspace.ts +++ b/src/private-temp-workspace.ts @@ -3,6 +3,7 @@ import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import { copyIntoRoot } from "./file-store.js"; +import { registerTempPathForExit } from "./temp-cleanup.js"; export type TempWorkspaceOptions = { rootDir: string; @@ -95,6 +96,7 @@ async function createTempWorkspace( const root = await fs.realpath(requestedRoot).catch(() => requestedRoot); await ensurePrivateDirectory(root, dirMode); const dir = await fs.mkdtemp(path.join(root, sanitizeTempPrefix(options.prefix))); + const unregisterTempDir = registerTempPathForExit(dir, { recursive: true }); await fs.chmod(dir, dirMode).catch(() => undefined); const stat = await fs.lstat(dir); if (stat.isSymbolicLink() || !stat.isDirectory()) { @@ -137,10 +139,18 @@ async function createTempWorkspace( }, read: async (fileName) => await fs.readFile(resolveWorkspaceLeaf(dir, fileName)), cleanup: async () => { - await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + try { + await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + } finally { + unregisterTempDir(); + } }, [Symbol.asyncDispose]: async () => { - await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + try { + await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); + } finally { + unregisterTempDir(); + } }, }; } @@ -180,6 +190,7 @@ export function tempWorkspaceSync( } ensurePrivateDirectorySync(root, dirMode); const dir = fsSync.mkdtempSync(path.join(root, sanitizeTempPrefix(options.prefix))); + const unregisterTempDir = registerTempPathForExit(dir, { recursive: true }); try { fsSync.chmodSync(dir, dirMode); } catch { @@ -232,6 +243,8 @@ export function tempWorkspaceSync( fsSync.rmSync(dir, { recursive: true, force: true }); } catch { // Best-effort cleanup. + } finally { + unregisterTempDir(); } }, [Symbol.dispose]: () => { @@ -239,6 +252,8 @@ export function tempWorkspaceSync( fsSync.rmSync(dir, { recursive: true, force: true }); } catch { // Best-effort cleanup. + } finally { + unregisterTempDir(); } }, }; diff --git a/src/replace-file.ts b/src/replace-file.ts index 3c8df4d..f7f4a51 100644 --- a/src/replace-file.ts +++ b/src/replace-file.ts @@ -3,6 +3,8 @@ import syncFs from "node:fs"; import type { Stats } from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; +import { registerTempPathForExit } from "./temp-cleanup.js"; +import { serializePathWrite } from "./write-queue.js"; export type ReplaceFileAtomicFileSystem = { promises: Pick< @@ -291,11 +293,21 @@ export async function replaceFileAtomic( ): Promise { const filePath = options.filePath; validateReplaceFilePath(filePath); + return await serializePathWrite(path.resolve(filePath), async () => { + return await replaceFileAtomicUnserialized(options); + }); +} + +async function replaceFileAtomicUnserialized( + options: ReplaceFileAtomicOptions, +): Promise { + const filePath = options.filePath; const fsModule = options.fileSystem?.promises ?? fs; const dir = path.dirname(filePath); const dirMode = options.dirMode ?? 0o700; const mode = await resolveMode(options); const tempPath = buildReplaceTempPath(filePath, options.tempPrefix); + const unregisterTempPath = registerTempPathForExit(tempPath); let tempExists = false; let originalError: unknown; @@ -319,6 +331,7 @@ export async function replaceFileAtomic( copyFallbackOnPermissionError: options.copyFallbackOnPermissionError === true, }); tempExists = false; + unregisterTempPath(); await fsModule.chmod(filePath, mode).catch(() => undefined); if (options.syncParentDir) { await syncDirectoryBestEffort(fsModule, dir); @@ -336,6 +349,7 @@ export async function replaceFileAtomic( throwOnCleanupError: options.throwOnCleanupError === true, }); } + unregisterTempPath(); } } @@ -349,6 +363,7 @@ export function replaceFileAtomicSync( const dirMode = options.dirMode ?? 0o700; const mode = resolveModeSync(options); const tempPath = buildReplaceTempPath(filePath, options.tempPrefix); + const unregisterTempPath = registerTempPathForExit(tempPath); let tempExists = false; let originalError: unknown; @@ -376,6 +391,7 @@ export function replaceFileAtomicSync( copyFallbackOnPermissionError: options.copyFallbackOnPermissionError === true, }); tempExists = false; + unregisterTempPath(); try { fsModule.chmodSync(filePath, mode); } catch { @@ -402,5 +418,6 @@ export function replaceFileAtomicSync( // The temp file is best-effort cleanup after write failure. } } + unregisterTempPath(); } } diff --git a/src/root.ts b/src/root.ts index ceb8a04..579b513 100644 --- a/src/root.ts +++ b/src/root.ts @@ -23,6 +23,8 @@ import { helperReaddir, helperStat, runPinnedHelper } from "./pinned-helper.js"; import { resolveRootPath } from "./root-path.js"; import { getFsSafeTestHooks } from "./test-hooks.js"; import type { DirEntry, PathStat } from "./types.js"; +import { registerTempPathForExit } from "./temp-cleanup.js"; +import { serializePathWrite } from "./write-queue.js"; export type OpenResult = { handle: FileHandle; @@ -745,6 +747,10 @@ function buildAtomicWriteTempPath(targetPath: string): string { return path.join(dir, `.${base}.${process.pid}.${randomUUID()}.tmp`); } +function rootWriteQueueKey(root: RootContext, relativePath: string): string { + return `${root.rootReal}\0${relativePath}`; +} + function createMaxBytesTransform(maxBytes: number): Transform { let bytes = 0; return new Transform({ @@ -1129,46 +1135,50 @@ async function writeFileInRoot( }, ): Promise { if (process.platform === "win32") { - await writeFileFallback(root, params); + await serializePathWrite(rootWriteQueueKey(root, params.relativePath), async () => { + await writeFileFallback(root, params); + }); return; } const pinned = await resolvePinnedWriteTargetInRoot(root, params.relativePath, params.mode); - let identity; - try { - identity = await runPinnedWriteHelper({ - rootPath: pinned.rootReal, - relativeParentPath: pinned.relativeParentPath, - basename: pinned.basename, - mkdir: params.mkdir !== false, - mode: params.mode ?? pinned.mode, - overwrite: params.overwrite, - input: { - kind: "buffer", - data: params.data, - encoding: params.encoding, - }, - }); - } catch (error) { - if (params.overwrite === false && isAlreadyExistsError(error)) { - throw new FsSafeError("already-exists", "file already exists", { - cause: error instanceof Error ? error : undefined, + await serializePathWrite(pinned.targetPath, async () => { + let identity; + try { + identity = await runPinnedWriteHelper({ + rootPath: pinned.rootReal, + relativeParentPath: pinned.relativeParentPath, + basename: pinned.basename, + mkdir: params.mkdir !== false, + mode: params.mode ?? pinned.mode, + overwrite: params.overwrite, + input: { + kind: "buffer", + data: params.data, + encoding: params.encoding, + }, }); + } catch (error) { + if (params.overwrite === false && isAlreadyExistsError(error)) { + throw new FsSafeError("already-exists", "file already exists", { + cause: error instanceof Error ? error : undefined, + }); + } + throw normalizePinnedWriteError(error); } - throw normalizePinnedWriteError(error); - } - try { - await verifyAtomicWriteResult({ - root, - targetPath: pinned.targetPath, - expectedIdentity: identity, - }); - } catch (err) { - emitWriteBoundaryWarning(`post-write verification failed: ${String(err)}`); - throw err; - } + try { + await verifyAtomicWriteResult({ + root, + targetPath: pinned.targetPath, + expectedIdentity: identity, + }); + } catch (err) { + emitWriteBoundaryWarning(`post-write verification failed: ${String(err)}`); + throw err; + } + }); } async function copyFileInRoot( @@ -1195,37 +1205,41 @@ async function copyFileInRoot( try { if (process.platform === "win32") { - await copyFileFallback(root, params, source); + await serializePathWrite(rootWriteQueueKey(root, params.relativePath), async () => { + await copyFileFallback(root, params, source); + }); return; } const pinned = await resolvePinnedWriteTargetInRoot(root, params.relativePath, params.mode); - const sourceStream = createBoundedReadStream(source, params.maxBytes); - const identity = await runPinnedWriteHelper({ - rootPath: pinned.rootReal, - relativeParentPath: pinned.relativeParentPath, - basename: pinned.basename, - mkdir: params.mkdir !== false, - mode: pinned.mode, - overwrite: true, - maxBytes: params.maxBytes, - input: { - kind: "stream", - stream: sourceStream, - }, - }).catch((error) => { - throw normalizePinnedWriteError(error); - }); - try { - await verifyAtomicWriteResult({ - root, - targetPath: pinned.targetPath, - expectedIdentity: identity, + await serializePathWrite(pinned.targetPath, async () => { + const sourceStream = createBoundedReadStream(source, params.maxBytes); + const identity = await runPinnedWriteHelper({ + rootPath: pinned.rootReal, + relativeParentPath: pinned.relativeParentPath, + basename: pinned.basename, + mkdir: params.mkdir !== false, + mode: pinned.mode, + overwrite: true, + maxBytes: params.maxBytes, + input: { + kind: "stream", + stream: sourceStream, + }, + }).catch((error) => { + throw normalizePinnedWriteError(error); }); - } catch (err) { - emitWriteBoundaryWarning(`post-copy verification failed: ${String(err)}`); - throw err; - } + try { + await verifyAtomicWriteResult({ + root, + targetPath: pinned.targetPath, + expectedIdentity: identity, + }); + } catch (err) { + emitWriteBoundaryWarning(`post-copy verification failed: ${String(err)}`); + throw err; + } + }); } finally { await source.handle.close().catch(() => {}); } @@ -1440,8 +1454,10 @@ async function writeFileFallback( const mode = params.mode ?? (target.stat.mode & 0o777); await target.handle.close().catch(() => {}); let tempPath: string | null = null; + let unregisterTempPath: (() => void) | null = null; try { tempPath = buildAtomicWriteTempPath(destinationPath); + unregisterTempPath = registerTempPathForExit(tempPath); const writtenStat = await writeTempFileForAtomicReplace({ tempPath, data: params.data, @@ -1450,6 +1466,8 @@ async function writeFileFallback( }); await fs.rename(tempPath, destinationPath); tempPath = null; + unregisterTempPath(); + unregisterTempPath = null; try { await verifyAtomicWriteResult({ root, @@ -1464,6 +1482,7 @@ async function writeFileFallback( if (tempPath) { await fs.rm(tempPath, { force: true }).catch(() => {}); } + unregisterTempPath?.(); } } @@ -1542,6 +1561,7 @@ async function copyFileFallback( let targetClosedByUs = false; let tempHandle: FileHandle | null = null; let tempPath: string | null = null; + let unregisterTempPath: (() => void) | null = null; let tempClosedByStream = false; try { target = await openWritableFileInRoot(root, { @@ -1556,6 +1576,7 @@ async function copyFileFallback( targetClosedByUs = true; tempPath = buildAtomicWriteTempPath(destinationPath); + unregisterTempPath = registerTempPathForExit(tempPath); tempHandle = await fs.open(tempPath, OPEN_WRITE_CREATE_FLAGS, mode || 0o600); const sourceStream = createBoundedReadStream(source, params.maxBytes); const targetStream = tempHandle.createWriteStream(); @@ -1574,6 +1595,8 @@ async function copyFileFallback( tempHandle = null; await fs.rename(tempPath, destinationPath); tempPath = null; + unregisterTempPath(); + unregisterTempPath = null; try { await verifyAtomicWriteResult({ root, @@ -1593,6 +1616,7 @@ async function copyFileFallback( if (tempPath) { await fs.rm(tempPath, { force: true }).catch(() => {}); } + unregisterTempPath?.(); if (!sourceClosedByStream) { await source.handle.close().catch(() => {}); } diff --git a/src/sibling-temp.ts b/src/sibling-temp.ts index 13c3e2d..4011abe 100644 --- a/src/sibling-temp.ts +++ b/src/sibling-temp.ts @@ -3,6 +3,8 @@ import fs from "node:fs/promises"; import path from "node:path"; import { sanitizeUntrustedFileName } from "./filename.js"; import { root } from "./root.js"; +import { registerTempPathForExit } from "./temp-cleanup.js"; +import { serializePathWrite } from "./write-queue.js"; export type WriteSiblingTempFileOptions = { dir: string; @@ -64,6 +66,7 @@ export async function writeSiblingTempFile( await fs.mkdir(dir, { recursive: true, mode: options.dirMode ?? 0o700 }); await fs.chmod(dir, options.dirMode ?? 0o700).catch(() => undefined); const tempPath = buildTempPath(dir, options.tempPrefix); + const unregisterTempPath = registerTempPathForExit(tempPath); let tempExists = false; try { tempExists = true; @@ -76,19 +79,23 @@ export async function writeSiblingTempFile( } const filePath = path.resolve(options.resolveFinalPath(result)); assertFinalPathIsSibling(dir, filePath); - await fs.rename(tempPath, filePath); - tempExists = false; - if (options.mode !== undefined) { - await fs.chmod(filePath, options.mode).catch(() => undefined); - } - if (options.syncParentDir) { - await syncDirectoryBestEffort(dir); - } + await serializePathWrite(filePath, async () => { + await fs.rename(tempPath, filePath); + tempExists = false; + unregisterTempPath(); + if (options.mode !== undefined) { + await fs.chmod(filePath, options.mode).catch(() => undefined); + } + if (options.syncParentDir) { + await syncDirectoryBestEffort(dir); + } + }); return { filePath, result }; } finally { if (tempExists) { await fs.rm(tempPath, { force: true }).catch(() => undefined); } + unregisterTempPath(); } } @@ -134,11 +141,13 @@ export async function writeViaSiblingTempPath(params: { fallbackFileName: params.fallbackFileName ?? "output.bin", tempPrefix: params.tempPrefix ?? ".fs-safe-output-", }); + const unregisterTempPath = registerTempPathForExit(tempPath); try { await params.writeTemp(tempPath); const targetRoot = await root(rootDir); await targetRoot.copyIn(relativeTargetPath, tempPath, { mkdir: false }); } finally { await fs.rm(tempPath, { force: true }).catch(() => {}); + unregisterTempPath(); } } diff --git a/src/temp-cleanup.ts b/src/temp-cleanup.ts new file mode 100644 index 0000000..72cea42 --- /dev/null +++ b/src/temp-cleanup.ts @@ -0,0 +1,53 @@ +import fsSync from "node:fs"; + +type TempCleanupEntry = { + path: string; + recursive: boolean; +}; + +const tempCleanupEntries = new Map(); +let cleanupRegistered = false; + +function cleanupRegisteredTempPathsSync(): void { + for (const entry of tempCleanupEntries.values()) { + try { + fsSync.rmSync(entry.path, { force: true, recursive: entry.recursive }); + } catch { + // Process-exit cleanup is best-effort. + } + } + tempCleanupEntries.clear(); +} + +export function registerTempPathForExit( + tempPath: string, + options?: { recursive?: boolean }, +): () => void { + if (!cleanupRegistered) { + cleanupRegistered = true; + process.once("exit", cleanupRegisteredTempPathsSync); + } + tempCleanupEntries.set(tempPath, { + path: tempPath, + recursive: options?.recursive === true, + }); + return () => { + tempCleanupEntries.delete(tempPath); + }; +} + +export function __cleanupRegisteredTempPathsForTest(): void { + cleanupRegisteredTempPathsSync(); +} + +export function __cleanupRegisteredTempPathForTest(tempPath: string): void { + const entry = tempCleanupEntries.get(tempPath); + if (!entry) { + return; + } + try { + fsSync.rmSync(entry.path, { force: true, recursive: entry.recursive }); + } finally { + tempCleanupEntries.delete(tempPath); + } +} diff --git a/src/temp-target.ts b/src/temp-target.ts index 12b67b3..ded6286 100644 --- a/src/temp-target.ts +++ b/src/temp-target.ts @@ -2,6 +2,7 @@ import crypto from "node:crypto"; import { mkdtemp, rm } from "node:fs/promises"; import path from "node:path"; import { resolveSecureTempRoot } from "./secure-temp-dir.js"; +import { registerTempPathForExit } from "./temp-cleanup.js"; export type TempFile = { dir: string; @@ -83,10 +84,15 @@ export async function tempFile(params: { const rootDir = resolveTempRoot(params.rootDir); const prefix = `${sanitizePrefix(params.prefix)}-`; const dir = await mkdtemp(path.join(rootDir, prefix)); + const unregisterTempDir = registerTempPathForExit(dir, { recursive: true }); const file = (fileName?: string) => path.join(dir, sanitizeTempFileName(fileName ?? params.fileName ?? "download.bin")); const cleanup = async () => { - await cleanupTempDir(dir, params.onCleanupError); + try { + await cleanupTempDir(dir, params.onCleanupError); + } finally { + unregisterTempDir(); + } }; return { dir, diff --git a/src/write-queue.ts b/src/write-queue.ts new file mode 100644 index 0000000..6022cf0 --- /dev/null +++ b/src/write-queue.ts @@ -0,0 +1,21 @@ +const writeQueues = new Map>(); + +export async function serializePathWrite(key: string, run: () => Promise): Promise { + const previous = writeQueues.get(key) ?? Promise.resolve(); + const task = (async () => { + await previous.catch(() => undefined); + return await run(); + })(); + const done = task.then( + () => undefined, + () => undefined, + ); + writeQueues.set(key, done); + try { + return await task; + } finally { + if (writeQueues.get(key) === done) { + writeQueues.delete(key); + } + } +} diff --git a/test/atomic.test.ts b/test/atomic.test.ts index c9000c2..50d99b9 100644 --- a/test/atomic.test.ts +++ b/test/atomic.test.ts @@ -8,6 +8,10 @@ import { replaceFileAtomic, replaceFileAtomicSync, } from "../src/atomic.js"; +import { + __cleanupRegisteredTempPathForTest, + registerTempPathForExit, +} from "../src/temp-cleanup.js"; const tempDirs: string[] = []; @@ -43,6 +47,57 @@ describe("atomic helpers", () => { await expect(fs.stat(observedTempPath ?? "")).rejects.toMatchObject({ code: "ENOENT" }); }); + it("serializes concurrent replacements for the same target", async () => { + const root = await tempRoot("fs-safe-atomic-queue-"); + const filePath = path.join(root, "state.txt"); + const events: string[] = []; + let releaseFirst: (() => void) | undefined; + + const first = replaceFileAtomic({ + filePath, + content: "first", + beforeRename: async () => { + events.push("first-before"); + await new Promise((resolve) => { + releaseFirst = resolve; + }); + events.push("first-release"); + }, + }); + while (!releaseFirst) { + await new Promise((resolve) => setTimeout(resolve, 1)); + } + + const second = replaceFileAtomic({ + filePath, + content: "second", + beforeRename: async () => { + events.push("second-before"); + }, + }); + + await new Promise((resolve) => setTimeout(resolve, 20)); + expect(events).toEqual(["first-before"]); + + releaseFirst(); + await Promise.all([first, second]); + + expect(events).toEqual(["first-before", "first-release", "second-before"]); + await expect(fs.readFile(filePath, "utf8")).resolves.toBe("second"); + }); + + it("registers temp paths for best-effort exit cleanup", async () => { + const root = await tempRoot("fs-safe-temp-cleanup-"); + const tempPath = path.join(root, "leftover.tmp"); + await fs.writeFile(tempPath, "temp", "utf8"); + const unregister = registerTempPathForExit(tempPath); + + __cleanupRegisteredTempPathForTest(tempPath); + + await expect(fs.access(tempPath)).rejects.toMatchObject({ code: "ENOENT" }); + unregister(); + }); + it("uses the permission-error copy fallback when requested", async () => { const root = await tempRoot("fs-safe-atomic-"); const filePath = path.join(root, "state.txt"); From bd2749649a91615cffe8f9d5f0c223c293dd3550 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 22:37:01 +0100 Subject: [PATCH 04/20] refactor: align atomic and secret helpers --- docs/archive.md | 5 +++ docs/atomic.md | 3 ++ docs/secret-file.md | 2 +- docs/temp.md | 3 ++ src/atomic.ts | 2 +- src/secret-file.ts | 16 +++++-- src/text-atomic.ts | 92 ++++++++-------------------------------- test/atomic.test.ts | 14 ++++++ test/secret-file.test.ts | 31 ++++++++++++++ 9 files changed, 88 insertions(+), 80 deletions(-) diff --git a/docs/archive.md b/docs/archive.md index 390e422..8bcc532 100644 --- a/docs/archive.md +++ b/docs/archive.md @@ -8,6 +8,11 @@ inspect archive kinds, and use pure path/limit helpers, but extraction or ZIP loading fails with a clear message until the matching optional dependency is installed. +Some package managers and CI installs skip optional dependencies +(`--no-optional`, `--omit=optional`, or equivalent). If an archive helper throws +that an optional archive dependency is not installed, install `jszip` and/or +`tar` explicitly in the consuming package. + ```ts import { extractArchive, resolveArchiveKind } from "@openclaw/fs-safe/archive"; ``` diff --git a/docs/atomic.md b/docs/atomic.md index 8447aba..6b83a72 100644 --- a/docs/atomic.md +++ b/docs/atomic.md @@ -95,6 +95,9 @@ Use it when callers must see a whole staged tree at the target path. For single- Atomic UTF-8 text write with the same secure defaults as `writeJson`: sibling temp file, temp fsync, rename, parent fsync, and final chmod best-effort. +It delegates to `replaceFileAtomic()` with a smaller call shape. Use it when +you do not need replacement hooks such as `beforeRename`, `preserveExistingMode`, +or custom copy-fallback policy. ```ts import { writeTextAtomic } from "@openclaw/fs-safe/atomic"; diff --git a/docs/secret-file.md b/docs/secret-file.md index e5aa28a..6cc0867 100644 --- a/docs/secret-file.md +++ b/docs/secret-file.md @@ -51,7 +51,7 @@ if (token) { ### `readSecretFileSync(filePath, label, options?)` -Strict reader. Throws when the file is missing, too large, empty, unreadable, or rejected by the validation checks. Use when failing loudly is the right call: +Strict reader. Throws `FsSafeError` when the file is missing, too large, empty, unreadable, or rejected by the validation checks. Use when failing loudly is the right call: ```ts const token = readSecretFileSync("/var/lib/app/auth.token"); diff --git a/docs/temp.md b/docs/temp.md index 1d8ed2f..f527aa2 100644 --- a/docs/temp.md +++ b/docs/temp.md @@ -95,6 +95,9 @@ type TempWorkspaceOptions = { When you don't need the stable workspace abstraction, the lower-level temp-file and sibling-temp helpers live behind `@openclaw/fs-safe/advanced`. They are composition primitives for stores and atomic writers, not the primary API. +`tempWorkspace()` carries the stable lifetime contract for application code; +`tempFile()` is a one-shot building block whose options may move as store and +archive internals evolve. ### `tempFile` diff --git a/src/atomic.ts b/src/atomic.ts index 8ba1882..708095d 100644 --- a/src/atomic.ts +++ b/src/atomic.ts @@ -7,6 +7,6 @@ export { type ReplaceFileAtomicSyncFileSystem, type ReplaceFileAtomicSyncOptions, } from "./replace-file.js"; -export { writeTextAtomic } from "./text-atomic.js"; +export { writeTextAtomic, type WriteTextAtomicOptions } from "./text-atomic.js"; export { replaceDirectoryAtomic, type ReplaceDirectoryAtomicOptions } from "./replace-directory.js"; export { movePathWithCopyFallback, type MovePathWithCopyFallbackOptions } from "./move-path.js"; diff --git a/src/secret-file.ts b/src/secret-file.ts index 7908dd5..1c53d2d 100644 --- a/src/secret-file.ts +++ b/src/secret-file.ts @@ -2,6 +2,7 @@ import { randomBytes } from "node:crypto"; import fs from "node:fs"; import fsp from "node:fs/promises"; import path from "node:path"; +import { FsSafeError, type FsSafeErrorCode } from "./errors.js"; import { resolveHomeRelativePath } from "./home-dir.js"; import { openPinnedFileSync } from "./pinned-open.js"; @@ -16,7 +17,7 @@ export type SecretFileReadOptions = { type SecretFileReadOutcome = | { ok: true; secret: string } - | { ok: false; message: string; error?: unknown }; + | { ok: false; code: FsSafeErrorCode; message: string; error?: unknown }; function normalizeSecretReadError(error: unknown): Error { return error instanceof Error ? error : new Error(String(error)); @@ -34,7 +35,7 @@ function readSecretFileOutcomeSync( const trimmedPath = filePath.trim(); const resolvedPath = resolveUserPath(trimmedPath); if (!resolvedPath) { - return { ok: false, message: `${label} file path is empty.` }; + return { ok: false, code: "invalid-path", message: `${label} file path is empty.` }; } const maxBytes = options.maxBytes ?? DEFAULT_SECRET_FILE_MAX_BYTES; @@ -46,6 +47,7 @@ function readSecretFileOutcomeSync( const normalized = normalizeSecretReadError(error); return { ok: false, + code: (error as NodeJS.ErrnoException).code === "ENOENT" ? "not-found" : "invalid-path", error: normalized, message: `Failed to inspect ${label} file at ${resolvedPath}: ${String(normalized)}`, }; @@ -54,18 +56,21 @@ function readSecretFileOutcomeSync( if (options.rejectSymlink && previewStat.isSymbolicLink()) { return { ok: false, + code: "symlink", message: `${label} file at ${resolvedPath} must not be a symlink.`, }; } if (!previewStat.isFile()) { return { ok: false, + code: "not-file", message: `${label} file at ${resolvedPath} must be a regular file.`, }; } if (previewStat.size > maxBytes) { return { ok: false, + code: "too-large", message: `${label} file at ${resolvedPath} exceeds ${maxBytes} bytes.`, }; } @@ -81,6 +86,7 @@ function readSecretFileOutcomeSync( ); return { ok: false, + code: opened.reason === "path" ? "not-found" : "path-mismatch", error, message: `Failed to read ${label} file at ${resolvedPath}: ${String(error)}`, }; @@ -92,6 +98,7 @@ function readSecretFileOutcomeSync( if (!secret) { return { ok: false, + code: "invalid-path", message: `${label} file at ${resolvedPath} is empty.`, }; } @@ -100,6 +107,7 @@ function readSecretFileOutcomeSync( const normalized = normalizeSecretReadError(error); return { ok: false, + code: "invalid-path", error: normalized, message: `Failed to read ${label} file at ${resolvedPath}: ${String(normalized)}`, }; @@ -117,7 +125,9 @@ export function readSecretFileSync( if (result.ok) { return result.secret; } - throw new Error(result.message, result.error ? { cause: result.error } : undefined); + throw new FsSafeError(result.code, result.message, { + cause: result.error, + }); } export function tryReadSecretFileSync( diff --git a/src/text-atomic.ts b/src/text-atomic.ts index 7416188..61b78a6 100644 --- a/src/text-atomic.ts +++ b/src/text-atomic.ts @@ -1,82 +1,24 @@ -import { randomUUID } from "node:crypto"; -import fs from "node:fs/promises"; -import path from "node:path"; +import { replaceFileAtomic } from "./replace-file.js"; -function getErrorCode(err: unknown): string | undefined { - return err instanceof Error ? (err as NodeJS.ErrnoException).code : undefined; -} - -async function replaceFileWithWindowsFallback(tempPath: string, filePath: string, mode: number) { - try { - await fs.rename(tempPath, filePath); - return; - } catch (err) { - const code = getErrorCode(err); - if (process.platform !== "win32" || (code !== "EPERM" && code !== "EEXIST")) { - throw err; - } - } - - const existing = await fs.lstat(filePath).catch(() => null); - if (existing?.isSymbolicLink()) { - await fs.rm(filePath, { force: true }); - await fs.rename(tempPath, filePath); - return; - } - - await fs.copyFile(tempPath, filePath); - try { - await fs.chmod(filePath, mode); - } catch { - // best-effort; ignore on platforms without chmod - } - await fs.rm(tempPath, { force: true }).catch(() => undefined); -} +export type WriteTextAtomicOptions = { + mode?: number; + dirMode?: number; + trailingNewline?: boolean; +}; export async function writeTextAtomic( filePath: string, content: string, - options?: { mode?: number; dirMode?: number; trailingNewline?: boolean }, -) { - const mode = options?.mode ?? 0o600; + options?: WriteTextAtomicOptions, +): Promise { const payload = options?.trailingNewline && !content.endsWith("\n") ? `${content}\n` : content; - const mkdirOptions: { recursive: true; mode?: number } = { recursive: true }; - if (typeof options?.dirMode === "number") { - mkdirOptions.mode = options.dirMode; - } - await fs.mkdir(path.dirname(filePath), mkdirOptions); - const parentDir = path.dirname(filePath); - const tmp = `${filePath}.${randomUUID()}.tmp`; - try { - const tmpHandle = await fs.open(tmp, "w", mode); - try { - await tmpHandle.writeFile(payload, { encoding: "utf8" }); - await tmpHandle.sync(); - } finally { - await tmpHandle.close().catch(() => undefined); - } - try { - await fs.chmod(tmp, mode); - } catch { - // best-effort; ignore on platforms without chmod - } - await replaceFileWithWindowsFallback(tmp, filePath, mode); - try { - const dirHandle = await fs.open(parentDir, "r"); - try { - await dirHandle.sync(); - } finally { - await dirHandle.close().catch(() => undefined); - } - } catch { - // best-effort; some platforms/filesystems do not support syncing directories. - } - try { - await fs.chmod(filePath, mode); - } catch { - // best-effort; ignore on platforms without chmod - } - } finally { - await fs.rm(tmp, { force: true }).catch(() => undefined); - } + await replaceFileAtomic({ + filePath, + content: payload, + mode: options?.mode ?? 0o600, + dirMode: options?.dirMode ?? (0o777 & ~process.umask()), + copyFallbackOnPermissionError: true, + syncTempFile: true, + syncParentDir: true, + }); } diff --git a/test/atomic.test.ts b/test/atomic.test.ts index 50d99b9..8e755ef 100644 --- a/test/atomic.test.ts +++ b/test/atomic.test.ts @@ -9,6 +9,7 @@ import { replaceFileAtomicSync, } from "../src/atomic.js"; import { + __cleanupRegisteredTempPathsForTest, __cleanupRegisteredTempPathForTest, registerTempPathForExit, } from "../src/temp-cleanup.js"; @@ -98,6 +99,19 @@ describe("atomic helpers", () => { unregister(); }); + it("cleans registered temp directories and ignores missing entries", async () => { + const root = await tempRoot("fs-safe-temp-cleanup-dir-"); + const tempDir = path.join(root, "leftover"); + await fs.mkdir(tempDir); + await fs.writeFile(path.join(tempDir, "file.txt"), "temp", "utf8"); + registerTempPathForExit(tempDir, { recursive: true }); + registerTempPathForExit(path.join(root, "missing.tmp")); + + __cleanupRegisteredTempPathsForTest(); + + await expect(fs.access(tempDir)).rejects.toMatchObject({ code: "ENOENT" }); + }); + it("uses the permission-error copy fallback when requested", async () => { const root = await tempRoot("fs-safe-atomic-"); const filePath = path.join(root, "state.txt"); diff --git a/test/secret-file.test.ts b/test/secret-file.test.ts index 208d7c4..9874aa2 100644 --- a/test/secret-file.test.ts +++ b/test/secret-file.test.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; +import { FsSafeError } from "../src/errors.js"; import { PRIVATE_SECRET_DIR_MODE, PRIVATE_SECRET_FILE_MODE, @@ -22,6 +23,16 @@ afterEach(async () => { await Promise.all(tempDirs.splice(0).map((dir) => fs.rm(dir, { force: true, recursive: true }))); }); +function expectSecretReadCode(run: () => string, code: FsSafeError["code"]): void { + try { + run(); + throw new Error("Expected readSecretFileSync to throw."); + } catch (err) { + expect(err).toBeInstanceOf(FsSafeError); + expect((err as FsSafeError).code).toBe(code); + } +} + describe("secret file helpers", () => { it("reads trimmed secrets and exposes nullable try-read semantics", async () => { const root = await tempRoot("fs-safe-secret-"); @@ -33,6 +44,26 @@ describe("secret file helpers", () => { expect(tryReadSecretFileSync(undefined, "API token")).toBeUndefined(); }); + it("throws structured errors for strict secret reads", async () => { + const root = await tempRoot("fs-safe-secret-errors-"); + const empty = path.join(root, "empty.txt"); + const big = path.join(root, "big.txt"); + await fs.writeFile(empty, "\n", "utf8"); + await fs.writeFile(big, "abcdef", "utf8"); + + expectSecretReadCode(() => readSecretFileSync("", "API token"), "invalid-path"); + expectSecretReadCode( + () => readSecretFileSync(path.join(root, "missing.txt"), "API token"), + "not-found", + ); + expectSecretReadCode(() => readSecretFileSync(root, "API token"), "not-file"); + expectSecretReadCode( + () => readSecretFileSync(big, "API token", { maxBytes: 2 }), + "too-large", + ); + expectSecretReadCode(() => readSecretFileSync(empty, "API token"), "invalid-path"); + }); + it("can reject symlinked secret paths", async () => { const root = await tempRoot("fs-safe-secret-"); const target = path.join(root, "target.txt"); From cafb181bedc2995b269bdd66d2da4f4857b05420 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 22:38:46 +0100 Subject: [PATCH 05/20] fix: remove failed copy temp files --- src/root.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/root.ts b/src/root.ts index 579b513..fce9c74 100644 --- a/src/root.ts +++ b/src/root.ts @@ -1613,16 +1613,16 @@ async function copyFileFallback( } throw err; } finally { - if (tempPath) { - await fs.rm(tempPath, { force: true }).catch(() => {}); - } - unregisterTempPath?.(); if (!sourceClosedByStream) { await source.handle.close().catch(() => {}); } if (tempHandle && !tempClosedByStream) { await tempHandle.close().catch(() => {}); } + if (tempPath) { + await fs.rm(tempPath, { force: true }).catch(() => {}); + } + unregisterTempPath?.(); if (target && !targetClosedByUs) { await target.handle.close().catch(() => {}); } From 46c1b1d22ca881df49ee653c862f031da3c75ce0 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 22:49:39 +0100 Subject: [PATCH 06/20] docs: sharpen root positioning --- README.md | 28 ++++++++++++++++++++++++---- docs/index.md | 4 +++- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index bb137c1..3f3333f 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,33 @@ Capability-style filesystem roots for Node.js apps that handle untrusted relative paths. -Use this when trusted application code has to touch caller-controlled paths inside a directory it owns. The package gives you one `root()` handle that survives symlink swaps, `..` traversal, hardlink aliases, and TOCTOU rename races between check and use. +Think Go's `os.Root` / `OpenInRoot` or Rust's [`cap-std`](https://github.com/bytecodealliance/cap-std), but for Node. Hand `root()` a trusted directory and you get back a handle whose every method resolves relative paths against it and refuses to escape — through `..`, symlink swaps, hardlink aliases, or TOCTOU rename races between check and use. -## Why +```ts +import { root } from "@openclaw/fs-safe"; -`path.resolve(root, input).startsWith(root)` validates a string. It does not pin the file you opened, defend against a symlink retarget between check and use, reject hardlinked aliases, or verify that a write landed where you intended after a rename. `fs-safe` does those things, packaged so every call site picks up the same defense without re-implementing it. +const fs = await root("/safe/workspace"); +await fs.write("notes/today.txt", "hello\n"); // ok +await fs.write("../escape.txt", "x"); // throws FsSafeError("outside-workspace") +``` -This is a library-level guardrail, not OS-level isolation. It does not replace containers, seccomp, or filesystem permissions — it is for code that already runs with the privileges of its workspace and wants to stop trivial path tricks from escaping it. +That's the whole pitch. `root()` is the product; the rest of the package — JSON stores, atomic writes, secret files, archive extraction, temp workspaces — is supporting cast for the same boundary. + +## Why this exists + +Most Node code that has to touch caller-controlled paths reaches for: + +```ts +path.resolve(root, input).startsWith(root) +``` + +That validates a *string*. It does not pin the file you opened, defend against a symlink retarget between check and use, reject hardlinked aliases of out-of-tree inodes, or verify that a write landed where you intended after a rename. The pieces to do those things exist scattered across the ecosystem — [`write-file-atomic`](https://www.npmjs.com/package/write-file-atomic) for atomic writes, `tar` / `jszip` for archive extraction, various `safefs`-style convenience wrappers — but none of them give you one root handle with traversal-resistant semantics across every operation. + +The same idea has landed in other languages. Go [added `os.Root` and `OpenInRoot`](https://go.dev/blog/osroot); Rust has had [`cap-std`](https://github.com/bytecodealliance/cap-std) for years. Node's `fs` is path-string-oriented and exposes flags like `O_NOFOLLOW` but not an ergonomic "operate inside this root" API. `fs-safe` fills that gap. + +## Not a sandbox + +This is a **library-level guardrail**, not OS-level isolation. It does not replace containers, seccomp, AppArmor, or filesystem permissions. It is for code that already runs with the privileges of its workspace and wants to stop trivial path tricks from escaping it. If your threat model is a hostile process, you need OS isolation; if your threat model is "an agent, plugin, upload handler, or CLI will eventually be tricked into writing somewhere it shouldn't," `fs-safe` catches that. ## Install diff --git a/docs/index.md b/docs/index.md index 58d6a63..21c4d12 100644 --- a/docs/index.md +++ b/docs/index.md @@ -8,11 +8,13 @@ description: "Capability-style filesystem roots for Node.js apps that handle unt Trusted Node.js code that has to touch caller-controlled paths inside a directory it owns gets one boundary it can rely on. `root()` returns a capability-style handle that resolves every relative path against a real directory, refuses anything that escapes it, pins the file you opened, and verifies the write landed where you intended. +Think Go's `os.Root` / `OpenInRoot` or Rust's [`cap-std`](https://github.com/bytecodealliance/cap-std), but for Node. `root()` is the product; everything else in this doc set — JSON stores, atomic writes, secret files, archive extraction, temp workspaces — is supporting cast for the same boundary. + ## Why `path.resolve(root, input).startsWith(root)` validates a string. It does not pin the file you opened, defend against a symlink retarget between check and use, reject hardlinked aliases, or verify that a write landed where you intended after a rename. `fs-safe` does those things, packaged so every call site picks up the same defense without re-implementing it. -This is a library-level guardrail, not OS-level isolation. It does not replace containers, seccomp, or filesystem permissions — it is for code that already runs with the privileges of its workspace and wants to stop trivial path tricks from escaping it. +This is a **library-level guardrail**, not OS-level isolation. It does not replace containers, seccomp, AppArmor, or filesystem permissions. It is for code that already runs with the privileges of its workspace and wants to stop trivial path tricks from escaping it. Typical fits: agent runtimes, plugin systems, upload extraction, local workspaces, CLIs — anywhere trusted code touches untrusted relative path names. ## Hello world From ff2e84aaeaae844bcdb08c300f8b8a9edafbfd4b Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Tue, 5 May 2026 23:25:07 +0100 Subject: [PATCH 07/20] feat: add persistent fs-safe python helper --- README.md | 25 +- docs/errors.md | 4 +- docs/index.md | 3 +- docs/install.md | 34 ++ docs/python-helper.md | 103 ++++++ docs/root.md | 20 ++ docs/security-model.md | 4 +- docs/testing.md | 37 +- package.json | 4 + scripts/build-docs-site.mjs | 2 +- src/config.ts | 6 + src/index.ts | 6 + src/pinned-helper.ts | 422 +---------------------- src/pinned-path.ts | 207 +----------- src/pinned-python-config.ts | 54 +++ src/pinned-python.ts | 654 ++++++++++++++++++++++++++++++++++++ src/pinned-write.ts | 322 ++++++------------ src/root.ts | 230 +++++++++++-- test/fs-safe.test.ts | 69 +++- test/pinned-python.test.ts | 231 +++++++++++++ 20 files changed, 1553 insertions(+), 884 deletions(-) create mode 100644 docs/python-helper.md create mode 100644 src/config.ts create mode 100644 src/pinned-python-config.ts create mode 100644 src/pinned-python.ts create mode 100644 test/pinned-python.test.ts diff --git a/README.md b/README.md index 3f3333f..e7a0a46 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,28 @@ pnpm add @openclaw/fs-safe Node 20.11 or newer. Core root/path/json/temp helpers avoid framework dependencies. Archive helpers use optional `jszip` and `tar` dependencies for ZIP/TAR support; installs that omit optional dependencies can still use every non-archive subpath. +On POSIX, `root()` uses one process-global persistent Python helper for the +fd-relative operations Node does not expose ergonomically (`renameat`, +`unlinkat`, recursive `mkdirat`-style walks, and parent-fd writes). Configure it +before first use when you need a strict environment policy: + +```ts +import { configureFsSafePython } from "@openclaw/fs-safe"; + +configureFsSafePython({ mode: "auto" }); // default: use helper, fall back if unavailable +configureFsSafePython({ mode: "off" }); // never spawn Python; use Node fallbacks +configureFsSafePython({ mode: "require" }); // fail closed if helper cannot start +``` + +Equivalent env vars: `FS_SAFE_PYTHON_MODE=auto|off|require` and +`FS_SAFE_PYTHON=/path/to/python3`. Without Python, `fs-safe` keeps lexical and +canonical root checks, no-follow opens, atomic temp+rename writes, and +post-write identity verification. What you lose is the strongest POSIX +fd-relative protection against a same-process-user racer swapping parent +directories between validation and mutation. Windows already uses the Node +fallback path. See the [Python helper policy](docs/python-helper.md) for +deployment guidance. + ## Quick start ```ts @@ -128,6 +150,7 @@ that OpenClaw needs to compose higher-level APIs are grouped under | Subpath | Contents | |---|---| | `@openclaw/fs-safe/root` | `root()`, `Root`, `RootDefaults`, related types | +| `@openclaw/fs-safe/config` | process-global Python helper configuration | | `@openclaw/fs-safe/path` | canonical path checks: `isPathInside`, `safeRealpathSync`, `isNotFoundPathError`, `isSymlinkOpenError` | | `@openclaw/fs-safe/json` | `tryReadJson`, `readJson`, `readJsonIfExists`, `writeJson`, sync variants | | `@openclaw/fs-safe/store` | `fileStore`, `jsonStore`, and `privateStateStore` | @@ -352,7 +375,7 @@ Current `FsSafeErrorCode` values are `already-exists`, `hardlink`, `helper-faile - root-bounded APIs resolve paths against a configured root and reject canonical escapes - reads open with `O_NOFOLLOW` where available, then verify fd identity matches the path identity before returning the buffer or handle - writes use pinned parent-directory helpers and atomic replacement on POSIX, with verified post-write identity -- `remove` and `mkdir` use fd-relative syscalls on POSIX through a small Python helper, with a Node fallback when the helper cannot spawn +- `remove`, `mkdir`, `move`, `stat`, `list`, and parent-fd writes use one persistent fd-relative Python helper on POSIX, with Node fallbacks when the helper is disabled or unavailable - archive extraction stages into a private directory and merges through the same boundary checks used by direct writes ## Limitations diff --git a/docs/errors.md b/docs/errors.md index b9d831e..8967bdd 100644 --- a/docs/errors.md +++ b/docs/errors.md @@ -56,8 +56,8 @@ type FsSafeErrorCode = |---|---|---| | `already-exists` | `create()`, `createJson()`, `move({ overwrite: false })`. | Target file or directory already at the destination. | | `hardlink` | Read or copy with `hardlinks: "reject"` saw `nlink > 1`. | File is hardlinked — possibly an alias of an out-of-tree inode. | -| `helper-failed` | Internal POSIX helper (Python-based fd-relative ops, sidecar lock acquire) failed. | Inspect `cause` for the underlying error. | -| `helper-unavailable` | Helper could not be spawned at all. | Python missing in PATH; restricted sandbox. Library falls back to Node-only path where possible. | +| `helper-failed` | Internal POSIX helper failed after startup. | Inspect `cause`; retrying may be unsafe if the operation may have partially completed. | +| `helper-unavailable` | Persistent Python helper was disabled or could not be spawned. | `FS_SAFE_PYTHON_MODE=off`, Python missing in PATH, restricted sandbox. `auto` falls back where possible; `require` fails closed. | | `insecure-permissions` | A secure file or path permission check found a mode/ACL that allows broader access than requested. | File or directory is group/world writable/readable; Windows ACL grants broad read. | | `invalid-path` | Input was empty, contained NUL, was an unparseable URL, or otherwise unusable. | Caller didn't validate input; input was a network path on Windows. | | `not-empty` | `remove()` on a non-empty directory. | Use `replaceDirectoryAtomic` or remove children first. | diff --git a/docs/index.md b/docs/index.md index 21c4d12..31c1f54 100644 --- a/docs/index.md +++ b/docs/index.md @@ -38,7 +38,7 @@ await fs.remove("notes/archive/today.txt"); ## Pick your path - **First time?** [Install](install.md), then walk through the [Quickstart](quickstart.md). Five minutes from `pnpm add` to a working root. -- **Designing a workspace feature.** Read the [Security model](security-model.md) before you trust the boundary, and the [Errors](errors.md) reference so you know what to catch. +- **Designing a workspace feature.** Read the [Security model](security-model.md) before you trust the boundary, the [Python helper policy](python-helper.md) before you pick deployment defaults, and the [Errors](errors.md) reference so you know what to catch. - **Replacing ad-hoc atomic writes.** Jump to [Atomic writes](atomic.md) or, for keyed JSON state, [JSON files](json.md). - **Extracting an upload.** Start at [Archive extraction](archive.md) — handles ZIP and TAR with traversal, link, count, and byte limits. - **Running an agent in a sandbox.** [Private temp workspaces](temp.md) plus [secret files](secret-file.md) cover the common scratch-and-credentials shape. @@ -49,6 +49,7 @@ await fs.remove("notes/archive/today.txt"); | Surface | Use it for | |---|---| | [`root()`](root.md) | One boundary for read/write/move/remove inside a trusted directory. | +| [Python helper policy](python-helper.md) | Choose `auto`, `off`, or `require` for POSIX fd-relative hardening. | | [`replaceFileAtomic`](atomic.md) | Sibling-temp + rename, fsync hooks, mode preservation, copy fallback. | | [`writeJson` / `readJson*`](json.md) | JSON state files with strict and lenient read variants. | | [`jsonStore`](json-store.md) | Single JSON state file with explicit fallback, atomic writes, and optional locking. | diff --git a/docs/install.md b/docs/install.md index 4f83bc9..0924efb 100644 --- a/docs/install.md +++ b/docs/install.md @@ -64,6 +64,7 @@ Use the main entry for the common surface, or the focused subpaths when you want |---|---| | `@openclaw/fs-safe` | Small common surface: `root`, root types, and errors. | | `@openclaw/fs-safe/root` | `root()`, `Root`, `RootDefaults`, related types. | +| `@openclaw/fs-safe/config` | Process-global Python helper configuration. | | `@openclaw/fs-safe/path` | `isPathInside`, `safeRealpathSync`, `isWithinDir`, error helpers. | | `@openclaw/fs-safe/json` | `tryReadJson`, `readJson`, `readJsonIfExists`, `writeJson`, sync variants. | | `@openclaw/fs-safe/store` | `fileStore()`, `jsonStore()`, and `privateStateStore()`. | @@ -85,6 +86,39 @@ Use the main entry for the common surface, or the focused subpaths when you want There are no peer dependencies and no native build step. +## Python helper policy + +On POSIX, `root()` uses one persistent Python helper process for the +fd-relative operations Node does not expose cleanly. The default is `auto`: use +the helper when it starts, fall back to Node-only behavior when it is disabled +or unavailable. + +```ts +import { configureFsSafePython } from "@openclaw/fs-safe/config"; + +configureFsSafePython({ mode: "auto" }); // default +configureFsSafePython({ mode: "off" }); // never spawn Python +configureFsSafePython({ mode: "require" }); // fail closed if unavailable +``` + +Environment variables are read at runtime: + +```bash +FS_SAFE_PYTHON_MODE=off # auto | off | require +FS_SAFE_PYTHON=/usr/bin/python3 +``` + +OpenClaw compatibility aliases are also accepted: +`OPENCLAW_FS_SAFE_PYTHON_MODE`, `OPENCLAW_FS_SAFE_PYTHON`, +`OPENCLAW_PINNED_PYTHON`, and `OPENCLAW_PINNED_WRITE_PYTHON`. + +Disabling Python keeps the public API working, but downgrades POSIX mutation +hardening from fd-relative syscalls to Node path operations guarded by lexical +and canonical checks plus identity verification. Use `require` for +security-sensitive deployments where that downgrade should be a startup/runtime +failure instead of a fallback. The full tradeoff is documented in +[Python helper policy](python-helper.md). + ## Verify the install ```ts diff --git a/docs/python-helper.md b/docs/python-helper.md new file mode 100644 index 0000000..258b0eb --- /dev/null +++ b/docs/python-helper.md @@ -0,0 +1,103 @@ +--- +title: Python helper policy +description: "How fs-safe uses its optional persistent Python helper, how to configure it, and what Node-only mode changes." +--- + +# Python helper policy + +`fs-safe` is a Node library. On POSIX systems, it can optionally keep one persistent Python helper process for filesystem operations that Node does not expose ergonomically as fd-relative APIs. + +The helper is not a sandbox and does not add new authority. It uses the same process user and the same filesystem permissions as your Node process. Its job is narrower: reduce race windows around parent-directory mutations after a root boundary has already been chosen. + +## Default + +The package default is: + +```ts +configureFsSafePython({ mode: "auto" }); +``` + +`auto` means: + +- use the helper for supported POSIX operations when it starts successfully; +- fall back to Node-only behavior when Python is missing, disabled by the host, or unavailable; +- keep the public API working in ordinary desktop, CI, Docker, and bundled-app environments. + +Applications can choose a stricter or simpler policy before the first filesystem operation: + +```ts +import { configureFsSafePython } from "@openclaw/fs-safe/config"; + +configureFsSafePython({ mode: "off" }); // never spawn Python +configureFsSafePython({ mode: "require" }); // fail closed if helper cannot start +``` + +Environment variables provide the same policy: + +```bash +FS_SAFE_PYTHON_MODE=auto # auto | off | require +FS_SAFE_PYTHON=/usr/bin/python3 +``` + +OpenClaw compatibility aliases are accepted too: `OPENCLAW_FS_SAFE_PYTHON_MODE`, `OPENCLAW_FS_SAFE_PYTHON`, `OPENCLAW_PINNED_PYTHON`, and `OPENCLAW_PINNED_WRITE_PYTHON`. + +## What the helper does + +Node's `fs` API is path-string oriented. It exposes `O_NOFOLLOW`, file handles, and some identity checks, but not a complete ergonomic `openat` / `renameat` / `unlinkat` / `mkdirat` surface for every operation `root()` needs. + +The helper fills that gap for supported POSIX operations: + +- stat/list paths relative to an already-open root directory; +- create directories while walking from a pinned parent; +- remove entries relative to a pinned parent; +- move entries with fd-relative rename semantics; +- run parent-fd write paths used by atomic replacement helpers. + +`fs-safe` sends requests to the helper over a JSON-lines protocol. It is one persistent process per Node process, not one Python spawn per filesystem call. + +## What you lose with `mode: "off"` + +Node-only mode still keeps the important application-level guardrails: + +- root-relative path validation; +- canonical root checks; +- no-follow opens where Node/platform support exists; +- file identity checks around reads and writes; +- atomic sibling-temp replacement; +- hardlink/symlink policy checks where the API requests them; +- byte limits and structured `FsSafeError` failures. + +What gets weaker is the POSIX defense against another same-UID process swapping a parent directory between validation and mutation. Without fd-relative mutation, `root().move()`, `root().remove()`, `root().mkdir()`, and some write paths rely on Node path operations plus pre/post checks instead of parent-fd syscalls. + +That is usually acceptable when the root directory is only writable by the trusted application user. It is not the right posture if untrusted local processes can race writes in the same tree and you are relying on `fs-safe` as part of the security boundary. + +## Choosing a mode + +| Mode | Use when | +|---|---| +| `auto` | You want the strongest available POSIX path when Python exists, but installs should still work without it. This is the package default. | +| `off` | You want deterministic Node-only behavior, no Python process, or a runtime that forbids spawning Python. | +| `require` | The fd-relative helper is part of your security posture and startup/runtime should fail closed if it is unavailable. | + +If you deploy with `require`, set `FS_SAFE_PYTHON` to an absolute interpreter path and test it in the same container, bundle, service manager, or sandbox that runs your app. + +## Application defaults + +Libraries should normally leave the package default alone. Applications can set a process-global policy once at startup: + +```ts +import { configureFsSafePython } from "@openclaw/fs-safe/config"; + +if (!process.env.FS_SAFE_PYTHON_MODE) { + configureFsSafePython({ mode: "off" }); +} +``` + +This is the right shape for apps that want to make Python an explicit operator choice while still letting deployment env vars opt back into `auto` or `require`. + +## Related pages + +- [Security model](security-model.md) — what `root()` does and does not promise. +- [Root API](root.md) — root-bounded read/write/move/remove methods. +- [Errors](errors.md) — `helper-unavailable` and `helper-failed` handling. +- [Testing](testing.md) — forcing helper modes in tests. diff --git a/docs/root.md b/docs/root.md index 8692c21..5048e25 100644 --- a/docs/root.md +++ b/docs/root.md @@ -92,6 +92,26 @@ fs.resolve(rel) // absolute path inside the root, after canonic These do not pin a later operation. They are safe to expose to UIs and decision points; for the actual read or write, use the verb methods so the operation pins identity at the point of use. +## Python helper mode + +On POSIX, mutation and inspection methods that need fd-relative directory +operations go through one persistent Python helper process. This avoids a +spawn-per-call cost while still using `openat`/`renameat`/`unlinkat`-style +operations that Node's `fs` API does not expose ergonomically. + +```ts +import { configureFsSafePython } from "@openclaw/fs-safe/config"; + +configureFsSafePython({ mode: "off" }); // Node-only fallback path +configureFsSafePython({ mode: "require" }); // fail if fd-relative helper unavailable +``` + +`auto` is the default. Configure the mode before creating roots. Without the +helper, root methods still run, but same-UID races that swap parent directories +between validation and mutation are harder to close completely. Use `require` +when that downgrade should be treated as a deployment failure. See +[Python helper policy](python-helper.md) for deployment guidance. + ### Properties ```ts diff --git a/docs/security-model.md b/docs/security-model.md index aa79abe..559f7b2 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -66,7 +66,7 @@ The library does not modify or constrain the global Node.js `fs` namespace, and ## Platform notes -- **POSIX (Linux, macOS):** Best-defended path. Uses `O_NOFOLLOW`, `openat`-style helpers via a small Python helper for fd-relative `unlinkat` / `mkdirat` / `renameat`, with a Node fallback when the helper cannot spawn. fd identity checks are reliable. +- **POSIX (Linux, macOS):** Best-defended path. Uses `O_NOFOLLOW`, fd identity checks, and one persistent Python helper process for fd-relative `unlinkat` / `mkdirat` / `renameat` / parent-fd write operations. Configure `FS_SAFE_PYTHON_MODE=require` when helper startup must fail closed, or `off` when you need a no-Python runtime. See [Python helper policy](python-helper.md). - **Windows:** Falls back to the safest Node-level behavior available. `O_NOFOLLOW` is not honored. Some fd-relative POSIX hardening is unavailable. The library does the path canonicalization, identity, and atomic-rename checks it can. The library does not advertise different security guarantees per platform — it advertises the same surface and relies on the strongest mechanism the platform offers. @@ -82,7 +82,7 @@ The library does not advertise different security guarantees per platform — it | Hardlink rejection is best-effort | Link-count checks depend on platform metadata. Treat `hardlinks: "reject"` as a tripwire, not an authorization primitive. | | Mode bits are not a full policy engine | `replaceFileAtomic` and secret-file helpers set requested modes, but you should still set umask and inspect modes when policy requires it. | | Archive extraction is path safety, not content safety | Unsafe entry paths and links are rejected; malicious payload contents remain your application layer's problem. | -| Helper failures degrade fd-relative hardening | `helper-failed` / `helper-unavailable` mean the Node fallback is being used. Atomicity remains, but some POSIX fd-relative race resistance is unavailable. | +| Helper failures degrade fd-relative hardening | `helper-unavailable` falls back in `auto` mode and fails closed in `require` mode. Atomicity and identity checks remain, but parent-directory swaps between validation and mutation are less tightly pinned without the helper. | ## Recommended deployment shape diff --git a/docs/testing.md b/docs/testing.md index 5e79146..bb2d6b3 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -14,7 +14,7 @@ The double-underscore prefix is a deliberate "hands off" signal: production code ## When to reach for hooks - Reproduce a TOCTOU race deterministically: simulate a symlink swap between resolve and open, or between write and rename. -- Make a "helper unavailable" branch reachable in CI without uninstalling Python from your runners. +- Force Node-only behavior without uninstalling Python from your runners. - Inject latency to test cancellation/timeout paths. If you don't need to inject a race, you don't need hooks — most tests should drive the library through normal calls and assert on observable behavior. @@ -23,10 +23,9 @@ If you don't need to inject a race, you don't need hooks — most tests should d ```ts type FsSafeTestHooks = { - beforeOpen?: (info: { absPath: string }) => void | Promise; - beforeRename?: (info: { tempPath: string; destPath: string }) => void | Promise; - afterFstat?: (info: { absPath: string; stat: import("node:fs").Stats }) => void | Promise; - helperUnavailableReason?: () => string | undefined; + afterPreOpenLstat?: (filePath: string) => Promise | void; + beforeOpen?: (filePath: string, flags: number) => Promise | void; + afterOpen?: (filePath: string, handle: import("node:fs/promises").FileHandle) => Promise | void; }; function __setFsSafeTestHooksForTest(hooks?: FsSafeTestHooks): void; @@ -35,10 +34,9 @@ function getFsSafeTestHooks(): FsSafeTestHooks | undefined; Hooks are called at well-defined points in the library's hot paths: -- **`beforeOpen`** — runs before `fs.open` or the pinned-open helper opens the file. A common use is to swap the path's target via `fs.symlink`/`fs.unlink` to drive a TOCTOU race. -- **`beforeRename`** — runs after the temp file is fully written and before the rename. Use to mutate the destination dir, simulate a parallel writer, or unlink the temp file. -- **`afterFstat`** — runs after the post-open `fstat`. Useful to validate the library called `fstat` (and not bypassed it). -- **`helperUnavailableReason`** — when defined, the library treats the POSIX helper as unavailable and surfaces the returned string as the failure reason. Use this to drive the Node-only fallback path on systems where the helper *would* spawn. +- **`afterPreOpenLstat`** — runs after the pre-open `lstat`. A common use is to swap the path's target via `fs.symlink`/`fs.unlink` to drive a TOCTOU race. +- **`beforeOpen`** — runs before `fs.open` with the exact flags the root read path will use. +- **`afterOpen`** — runs after the file handle is opened. Useful to wrap handle methods or inject a size race before a stream is consumed. `__setFsSafeTestHooksForTest(undefined)` clears all hooks. Always clean up between tests. @@ -67,7 +65,7 @@ it("rejects a swap between resolve and open", async () => { const fs = await root(dir, { symlinks: "reject" }); __setFsSafeTestHooksForTest({ - beforeOpen: async ({ absPath }) => { + afterPreOpenLstat: async (absPath) => { // swap real.txt for a symlink to decoy.txt right before the open await unlink(absPath); await symlink(path.join(dir, "decoy.txt"), absPath); @@ -83,20 +81,23 @@ it("rejects a swap between resolve and open", async () => { The `code` may be `symlink` (caught at open by `O_NOFOLLOW`) or `path-mismatch` (caught by the post-open identity check) depending on platform — both are correct refusals. -## Example: force the helper-unavailable branch +## Example: force Node-only fallback behavior ```ts -import { __setFsSafeTestHooksForTest } from "@openclaw/fs-safe/test-hooks"; +import { configureFsSafePython } from "@openclaw/fs-safe/config"; beforeEach(() => { - __setFsSafeTestHooksForTest({ - helperUnavailableReason: () => "test: pretend Python is missing", - }); + configureFsSafePython({ mode: "off" }); }); -it("falls back to the Node-only path", async () => { - // exercise an operation that would normally use the helper - // and assert the Node fallback succeeded with the same observable result +afterEach(() => { + configureFsSafePython({ mode: "auto", pythonPath: undefined }); +}); + +it("runs without the Python helper", async () => { + const fs = await root(dir); + await fs.write("file.txt", "ok"); + await expect(fs.readText("file.txt")).resolves.toBe("ok"); }); ``` diff --git a/package.json b/package.json index afe5417..9957901 100644 --- a/package.json +++ b/package.json @@ -26,6 +26,10 @@ "types": "./dist/root.d.ts", "default": "./dist/root.js" }, + "./config": { + "types": "./dist/config.d.ts", + "default": "./dist/config.js" + }, "./path": { "types": "./dist/path.d.ts", "default": "./dist/path.js" diff --git a/scripts/build-docs-site.mjs b/scripts/build-docs-site.mjs index 259c696..99d3283 100644 --- a/scripts/build-docs-site.mjs +++ b/scripts/build-docs-site.mjs @@ -26,7 +26,7 @@ const productDescription = const installCmd = "pnpm add @openclaw/fs-safe"; const sections = [ - ["Start", ["index.md", "install.md", "quickstart.md", "security-model.md"]], + ["Start", ["index.md", "install.md", "quickstart.md", "security-model.md", "python-helper.md"]], ["Root API", ["root.md", "reading.md", "writing.md", "path-scope.md"]], ["Atomic & temp", ["atomic.md", "json.md", "temp.md", "archive.md"]], ["Stores", ["json-store.md", "file-store.md", "private-file-store.md"]], diff --git a/src/config.ts b/src/config.ts new file mode 100644 index 0000000..3e42f5a --- /dev/null +++ b/src/config.ts @@ -0,0 +1,6 @@ +export { + configureFsSafePython, + getFsSafePythonConfig, + type FsSafePythonConfig, + type FsSafePythonMode, +} from "./pinned-python-config.js"; diff --git a/src/index.ts b/src/index.ts index 7afd9cf..eedadaf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,3 +26,9 @@ export { type WritableOpenMode, type WritableOpenResult, } from "./root.js"; +export { + configureFsSafePython, + getFsSafePythonConfig, + type FsSafePythonConfig, + type FsSafePythonMode, +} from "./pinned-python-config.js"; diff --git a/src/pinned-helper.ts b/src/pinned-helper.ts index a093b27..5871d96 100644 --- a/src/pinned-helper.ts +++ b/src/pinned-helper.ts @@ -1,425 +1,27 @@ -import { spawn } from "node:child_process"; -import fsSync from "node:fs"; - -import { FsSafeError } from "./errors.js"; -import { splitSafeRelativePath } from "./path.js"; import type { DirEntry, PathStat } from "./types.js"; +import { + isPinnedHelperUnavailable, + runPinnedPythonOperation, + validatePinnedOperationPayload, +} from "./pinned-python.js"; -const PINNED_HELPER_SOURCE = String.raw` -import base64 -import errno -import json -import os -import stat -import sys -import tempfile +type HelperOperation = "stat" | "readdir" | "mkdirp" | "remove" | "rename"; -operation = sys.argv[1] -root_path = sys.argv[2] -payload = json.loads(sys.stdin.read() or "{}") - -DIR_FLAGS = os.O_RDONLY -if hasattr(os, "O_DIRECTORY"): - DIR_FLAGS |= os.O_DIRECTORY -if hasattr(os, "O_NOFOLLOW"): - DIR_FLAGS |= os.O_NOFOLLOW -READ_FLAGS = os.O_RDONLY -if hasattr(os, "O_NOFOLLOW"): - READ_FLAGS |= os.O_NOFOLLOW -WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL -if hasattr(os, "O_NOFOLLOW"): - WRITE_FLAGS |= os.O_NOFOLLOW - -def fail(code, message): - print(json.dumps({"ok": False, "code": code, "message": message}), file=sys.stderr) - sys.exit(1) - -def split_relative(value): - if value in ("", "."): - return [] - if "\x00" in value or "\\" in value or value.startswith("/") or value.startswith("//"): - raise OSError(errno.EPERM, "invalid relative path") - parts = [part for part in value.split("/") if part and part != "."] - for part in parts: - if part == "..": - raise OSError(errno.EPERM, "path traversal is not allowed") - return parts - -def open_dir(path_value, dir_fd=None): - return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd) - -def walk_dir(root_fd, segments, mkdir_enabled=False): - current_fd = os.dup(root_fd) - try: - for segment in segments: - try: - next_fd = open_dir(segment, dir_fd=current_fd) - except FileNotFoundError: - if not mkdir_enabled: - raise - os.mkdir(segment, 0o777, dir_fd=current_fd) - next_fd = open_dir(segment, dir_fd=current_fd) - os.close(current_fd) - current_fd = next_fd - return current_fd - except Exception: - os.close(current_fd) - raise - -def parent_and_basename(root_fd, relative): - segments = split_relative(relative) - if not segments: - raise OSError(errno.EPERM, "operation requires a non-root path") - parent_fd = walk_dir(root_fd, segments[:-1]) - return parent_fd, segments[-1] - -def encode_stat(st): - mode = st.st_mode - return { - "dev": st.st_dev, - "gid": st.st_gid, - "ino": st.st_ino, - "isDirectory": stat.S_ISDIR(mode), - "isFile": stat.S_ISREG(mode), - "isSymbolicLink": stat.S_ISLNK(mode), - "mode": mode, - "mtimeMs": st.st_mtime * 1000, - "nlink": st.st_nlink, - "size": st.st_size, - "uid": st.st_uid, - } - -def reject_unsafe_endpoint(st): - mode = st.st_mode - if stat.S_ISLNK(mode): - raise OSError(errno.ELOOP, "symlink endpoint is not allowed") - if stat.S_ISREG(mode) and st.st_nlink > 1: - raise OSError(errno.EPERM, "hardlinked file endpoint is not allowed") - -def stat_path(root_fd, relative): - segments = split_relative(relative) - if not segments: - return encode_stat(os.fstat(root_fd)) - parent_fd, basename = parent_and_basename(root_fd, relative) - try: - st = os.lstat(basename, dir_fd=parent_fd) - if payload.get("rejectSymlink", True) and stat.S_ISLNK(st.st_mode): - raise OSError(errno.ELOOP, "symlink endpoint is not allowed") - return encode_stat(st) - finally: - os.close(parent_fd) - -def readdir_path(root_fd, relative): - dir_fd = walk_dir(root_fd, split_relative(relative)) - try: - names = sorted(os.listdir(dir_fd)) - if not payload.get("withFileTypes", False): - return names - entries = [] - for name in names: - st = os.lstat(name, dir_fd=dir_fd) - entry = encode_stat(st) - entry["name"] = name - entries.append(entry) - return entries - finally: - os.close(dir_fd) - -def mkdirp_path(root_fd, relative): - dir_fd = walk_dir(root_fd, split_relative(relative), mkdir_enabled=True) - os.close(dir_fd) - return None - -def remove_tree(parent_fd, basename): - st = os.lstat(basename, dir_fd=parent_fd) - if stat.S_ISDIR(st.st_mode) and not stat.S_ISLNK(st.st_mode): - dir_fd = open_dir(basename, dir_fd=parent_fd) - try: - for child in os.listdir(dir_fd): - remove_tree(dir_fd, child) - finally: - os.close(dir_fd) - os.rmdir(basename, dir_fd=parent_fd) - else: - os.unlink(basename, dir_fd=parent_fd) - -def remove_path(root_fd, relative): - parent_fd, basename = parent_and_basename(root_fd, relative) - try: - try: - st = os.lstat(basename, dir_fd=parent_fd) - except FileNotFoundError: - if payload.get("force", True): - return None - raise - if stat.S_ISDIR(st.st_mode) and not stat.S_ISLNK(st.st_mode): - if payload.get("recursive", False): - remove_tree(parent_fd, basename) - else: - os.rmdir(basename, dir_fd=parent_fd) - else: - os.unlink(basename, dir_fd=parent_fd) - return None - finally: - os.close(parent_fd) - -def read_path(root_fd, relative): - parent_fd, basename = parent_and_basename(root_fd, relative) - try: - fd = os.open(basename, READ_FLAGS, dir_fd=parent_fd) - try: - st = os.fstat(fd) - reject_unsafe_endpoint(st) - if not stat.S_ISREG(st.st_mode): - raise OSError(errno.EPERM, "only regular files can be read") - chunks = [] - while True: - chunk = os.read(fd, 1024 * 1024) - if not chunk: - break - chunks.append(chunk) - return {"base64": base64.b64encode(b"".join(chunks)).decode("ascii"), "stat": encode_stat(st)} - finally: - os.close(fd) - finally: - os.close(parent_fd) - -def write_path(root_fd, relative): - parent_fd, basename = parent_and_basename(root_fd, relative) - data = base64.b64decode(payload.get("base64", "")) - overwrite = payload.get("overwrite", True) - try: - if not overwrite: - try: - os.lstat(basename, dir_fd=parent_fd) - raise FileExistsError(errno.EEXIST, "destination exists", basename) - except FileNotFoundError: - pass - prefix = ".fs-safe-" + basename.replace("/", "_") + "-" - temp_name = None - fd = None - try: - for _ in range(32): - candidate = prefix + next(tempfile._get_candidate_names()) - try: - fd = os.open(candidate, WRITE_FLAGS, 0o600, dir_fd=parent_fd) - temp_name = candidate - break - except FileExistsError: - continue - if fd is None or temp_name is None: - raise FileExistsError(errno.EEXIST, "could not allocate temp file") - os.write(fd, data) - os.fsync(fd) - os.close(fd) - fd = None - os.replace(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) - os.fsync(parent_fd) - return None - finally: - if fd is not None: - os.close(fd) - if temp_name is not None: - try: - os.unlink(temp_name, dir_fd=parent_fd) - except FileNotFoundError: - pass - finally: - os.close(parent_fd) - -def rename_path(root_fd): - from_parent_fd, from_base = parent_and_basename(root_fd, payload["from"]) - to_parent_fd, to_base = parent_and_basename(root_fd, payload["to"]) - try: - from_stat = os.lstat(from_base, dir_fd=from_parent_fd) - reject_unsafe_endpoint(from_stat) - if not payload.get("overwrite", True): - try: - os.lstat(to_base, dir_fd=to_parent_fd) - raise FileExistsError(errno.EEXIST, "destination exists", to_base) - except FileNotFoundError: - pass - os.rename(from_base, to_base, src_dir_fd=from_parent_fd, dst_dir_fd=to_parent_fd) - os.fsync(from_parent_fd) - if from_parent_fd != to_parent_fd: - os.fsync(to_parent_fd) - return None - finally: - os.close(from_parent_fd) - os.close(to_parent_fd) - -try: - root_fd = open_dir(root_path) - try: - relative = payload.get("relativePath", "") - if operation == "stat": - result = stat_path(root_fd, relative) - elif operation == "readdir": - result = readdir_path(root_fd, relative) - elif operation == "mkdirp": - result = mkdirp_path(root_fd, relative) - elif operation == "remove": - result = remove_path(root_fd, relative) - elif operation == "read": - result = read_path(root_fd, relative) - elif operation == "write": - result = write_path(root_fd, relative) - elif operation == "rename": - result = rename_path(root_fd) - else: - raise RuntimeError("unknown operation: " + operation) - print(json.dumps({"ok": True, "result": result}, separators=(",", ":"))) - finally: - os.close(root_fd) -except Exception as exc: - fail(type(exc).__name__, str(exc)) -`; - -type HelperOperation = "stat" | "readdir" | "mkdirp" | "remove" | "read" | "write" | "rename"; - -const PYTHON_CANDIDATES = [ - process.env.OPENCLAW_FS_SAFE_PYTHON, - process.env.OPENCLAW_PINNED_PYTHON, - "/usr/bin/python3", - "/opt/homebrew/bin/python3", - "/usr/local/bin/python3", -].filter((value): value is string => Boolean(value)); - -let cachedPython = ""; - -function canExecute(binPath: string): boolean { - try { - fsSync.accessSync(binPath, fsSync.constants.X_OK); - return true; - } catch { - return false; - } -} - -function resolvePython(): string { - if (cachedPython) { - return cachedPython; - } - for (const candidate of PYTHON_CANDIDATES) { - if (canExecute(candidate)) { - cachedPython = candidate; - return cachedPython; - } - } - cachedPython = "python3"; - return cachedPython; -} - -function assertPinnedHelperSupported(): void { - if (process.platform === "win32") { - throw new FsSafeError( - "unsupported-platform", - "fd-relative pinned filesystem operations are not available on Windows", - ); - } -} - -function isSpawnUnavailable(error: unknown): boolean { - if (!(error instanceof Error)) { - return false; - } - const maybeErrno = error as NodeJS.ErrnoException; - return ( - typeof maybeErrno.syscall === "string" && - maybeErrno.syscall.startsWith("spawn") && - ["EACCES", "ENOENT", "ENOEXEC"].includes(maybeErrno.code ?? "") - ); -} +export { isPinnedHelperUnavailable }; export async function runPinnedHelper( operation: HelperOperation, rootDir: string, payload: Record, ): Promise { - assertPinnedHelperSupported(); - if (typeof payload.relativePath === "string") { - splitSafeRelativePath(payload.relativePath); - } - if (typeof payload.from === "string") { - splitSafeRelativePath(payload.from); - } - if (typeof payload.to === "string") { - splitSafeRelativePath(payload.to); - } - - const child = spawn(resolvePython(), ["-c", PINNED_HELPER_SOURCE, operation, rootDir], { - stdio: ["pipe", "pipe", "pipe"], + validatePinnedOperationPayload(payload); + return await runPinnedPythonOperation({ + operation, + rootPath: rootDir, + payload, }); - - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding("utf8"); - child.stderr.setEncoding("utf8"); - child.stdout.on("data", (chunk: string) => { - stdout += chunk; - }); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - - child.stdin.end(JSON.stringify(payload)); - - const [code, signal] = await new Promise<[number | null, NodeJS.Signals | null]>( - (resolve, reject) => { - child.once("error", reject); - child.once("close", (exitCode, exitSignal) => resolve([exitCode, exitSignal])); - }, - ).catch((error: unknown) => { - if (isSpawnUnavailable(error)) { - throw new FsSafeError("helper-unavailable", "Python helper is unavailable", { cause: error }); - } - throw error; - }); - - const raw = code === 0 ? stdout : stderr; - let decoded: unknown; - try { - decoded = JSON.parse(raw.trim()); - } catch { - throw new FsSafeError( - "helper-failed", - `pinned helper failed with code ${code ?? "null"} (${signal ?? "?"}): ${raw.trim()}`, - ); - } - - if ( - typeof decoded !== "object" || - decoded === null || - !("ok" in decoded) || - typeof decoded.ok !== "boolean" - ) { - throw new FsSafeError("helper-failed", "pinned helper returned an invalid response"); - } - if (!decoded.ok) { - const helperCode = "code" in decoded && typeof decoded.code === "string" ? decoded.code : ""; - const message = - "message" in decoded && typeof decoded.message === "string" - ? decoded.message - : "pinned helper failed"; - if (helperCode === "FileNotFoundError") { - throw new FsSafeError("not-found", "file not found"); - } - if (helperCode === "NotADirectoryError" || helperCode === "OSError") { - throw new FsSafeError("path-alias", message); - } - if (helperCode === "FileExistsError") { - throw new FsSafeError("already-exists", message); - } - throw new FsSafeError("helper-failed", message); - } - return (decoded as unknown as { result: T }).result; } -export type HelperReadResult = { - base64: string; - stat: PathStat; -}; - export async function helperStat(rootDir: string, relativePath: string): Promise { return await runPinnedHelper("stat", rootDir, { relativePath }); } diff --git a/src/pinned-path.ts b/src/pinned-path.ts index e155475..10d848d 100644 --- a/src/pinned-path.ts +++ b/src/pinned-path.ts @@ -1,177 +1,9 @@ -import { spawn } from "node:child_process"; -import fsSync from "node:fs"; import { FsSafeError } from "./errors.js"; - -const LOCAL_PINNED_PATH_PYTHON = [ - "import errno", - "import json", - "import os", - "import stat", - "import sys", - "", - "operation = sys.argv[1]", - "root_path = sys.argv[2]", - "relative_path = sys.argv[3]", - "", - "DIR_FLAGS = os.O_RDONLY", - "if hasattr(os, 'O_DIRECTORY'):", - " DIR_FLAGS |= os.O_DIRECTORY", - "if hasattr(os, 'O_NOFOLLOW'):", - " DIR_FLAGS |= os.O_NOFOLLOW", - "", - "def open_dir(path_value, dir_fd=None):", - " return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd)", - "", - "def split_segments(relative_path):", - " return [part for part in relative_path.split('/') if part and part != '.']", - "", - "def validate_segment(segment):", - " if segment == '..':", - " raise OSError(errno.EPERM, 'path traversal is not allowed', segment)", - "", - "def walk_existing_path(root_fd, segments):", - " current_fd = os.dup(root_fd)", - " try:", - " for segment in segments:", - " validate_segment(segment)", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " os.close(current_fd)", - " current_fd = next_fd", - " return current_fd", - " except Exception:", - " os.close(current_fd)", - " raise", - "", - "def mkdirp_within_root(root_fd, segments):", - " current_fd = os.dup(root_fd)", - " try:", - " for segment in segments:", - " validate_segment(segment)", - " try:", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " except FileNotFoundError:", - " os.mkdir(segment, 0o777, dir_fd=current_fd)", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " os.close(current_fd)", - " current_fd = next_fd", - " finally:", - " os.close(current_fd)", - "", - "def remove_within_root(root_fd, segments):", - " if not segments:", - " raise OSError(errno.EPERM, 'refusing to remove root path')", - " parent_segments = segments[:-1]", - " basename = segments[-1]", - " validate_segment(basename)", - " parent_fd = walk_existing_path(root_fd, parent_segments)", - " try:", - " target_stat = os.lstat(basename, dir_fd=parent_fd)", - " if stat.S_ISDIR(target_stat.st_mode) and not stat.S_ISLNK(target_stat.st_mode):", - " os.rmdir(basename, dir_fd=parent_fd)", - " else:", - " os.unlink(basename, dir_fd=parent_fd)", - " finally:", - " os.close(parent_fd)", - "", - "def emit_error(exc):", - " payload = {", - " 'name': exc.__class__.__name__,", - " 'errno': getattr(exc, 'errno', None),", - " 'message': str(exc),", - " }", - " print(json.dumps(payload), file=sys.stderr)", - "", - "root_fd = None", - "try:", - " root_fd = open_dir(root_path)", - " segments = split_segments(relative_path)", - " if operation == 'mkdirp':", - " mkdirp_within_root(root_fd, segments)", - " elif operation == 'remove':", - " remove_within_root(root_fd, segments)", - " else:", - " raise RuntimeError(f'unknown pinned path operation: {operation}')", - "except Exception as exc:", - " emit_error(exc)", - " sys.exit(1)", - "finally:", - " if root_fd is not None:", - " os.close(root_fd)", -].join("\n"); - -const PINNED_PATH_PYTHON_CANDIDATES = [ - process.env.OPENCLAW_PINNED_PYTHON, - // Keep the write-specific alias for backwards compatibility. - process.env.OPENCLAW_PINNED_WRITE_PYTHON, - "/usr/bin/python3", - "/opt/homebrew/bin/python3", - "/usr/local/bin/python3", -].filter((value): value is string => Boolean(value)); - -let cachedPinnedPathPython = ""; - -function canExecute(binPath: string): boolean { - try { - fsSync.accessSync(binPath, fsSync.constants.X_OK); - return true; - } catch { - return false; - } -} - -function resolvePinnedPathPython(): string { - if (cachedPinnedPathPython) { - return cachedPinnedPathPython; - } - for (const candidate of PINNED_PATH_PYTHON_CANDIDATES) { - if (canExecute(candidate)) { - cachedPinnedPathPython = candidate; - return cachedPinnedPathPython; - } - } - cachedPinnedPathPython = "python3"; - return cachedPinnedPathPython; -} - -function buildPinnedPathError(stderr: string, code: number | null, signal: NodeJS.Signals | null) { - const trimmed = stderr.trim(); - if (trimmed.startsWith("{")) { - try { - const payload = JSON.parse(trimmed) as { errno?: number; message?: string; name?: string }; - if (payload.errno === 2) { - return new FsSafeError("not-found", "file not found"); - } - if (payload.errno === 20 || payload.errno === 40) { - return new FsSafeError("path-alias", "path is not under root"); - } - if (payload.errno === 39) { - return new FsSafeError("not-empty", "directory is not empty"); - } - if (payload.errno === 1 || payload.errno === 13 || payload.errno === 21) { - return new FsSafeError("not-removable", "path is not removable under root"); - } - return new FsSafeError("helper-failed", payload.message || "pinned path helper failed"); - } catch { - // Fall through to the generic helper failure below. - } - } - return new FsSafeError( - "helper-failed", - trimmed || `Pinned path helper failed with code ${code ?? "null"} (${signal ?? "?"})`, - ); -} +import { canFallbackFromPythonError } from "./pinned-python-config.js"; +import { runPinnedHelper } from "./pinned-helper.js"; export function isPinnedPathHelperSpawnError(error: unknown): boolean { - if (!(error instanceof Error)) { - return false; - } - - const maybeErrno = error as NodeJS.ErrnoException; - if (typeof maybeErrno.syscall !== "string" || !maybeErrno.syscall.startsWith("spawn")) { - return false; - } - - return ["EACCES", "ENOENT", "ENOEXEC"].includes(maybeErrno.code ?? ""); + return canFallbackFromPythonError(error); } export async function runPinnedPathHelper(params: { @@ -179,27 +11,16 @@ export async function runPinnedPathHelper(params: { rootPath: string; relativePath: string; }): Promise { - const child = spawn( - resolvePinnedPathPython(), - ["-c", LOCAL_PINNED_PATH_PYTHON, params.operation, params.rootPath, params.relativePath], - { - stdio: ["ignore", "ignore", "pipe"], - }, - ); - - let stderr = ""; - child.stderr.setEncoding?.("utf8"); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - - const [code, signal] = await new Promise<[number | null, NodeJS.Signals | null]>( - (resolve, reject) => { - child.once("error", reject); - child.once("close", (exitCode, exitSignal) => resolve([exitCode, exitSignal])); - }, - ); - if (code !== 0) { - throw buildPinnedPathError(stderr, code, signal); + try { + await runPinnedHelper(params.operation, params.rootPath, { + relativePath: params.relativePath, + }); + } catch (error) { + if (error instanceof FsSafeError) { + throw error; + } + throw new FsSafeError("helper-failed", "pinned path helper failed", { + cause: error instanceof Error ? error : undefined, + }); } } diff --git a/src/pinned-python-config.ts b/src/pinned-python-config.ts new file mode 100644 index 0000000..45a6679 --- /dev/null +++ b/src/pinned-python-config.ts @@ -0,0 +1,54 @@ +export type FsSafePythonMode = "auto" | "off" | "require"; + +export type FsSafePythonConfig = { + mode: FsSafePythonMode; + pythonPath?: string; +}; + +let overrideConfig: Partial = {}; + +function parseMode(value: string | undefined): FsSafePythonMode | undefined { + if (!value) { + return undefined; + } + const normalized = value.trim().toLowerCase(); + if (normalized === "0" || normalized === "false" || normalized === "off" || normalized === "never") { + return "off"; + } + if (normalized === "1" || normalized === "true" || normalized === "on" || normalized === "auto") { + return "auto"; + } + if (normalized === "required" || normalized === "require") { + return "require"; + } + return undefined; +} + +export function configureFsSafePython(config: Partial): void { + overrideConfig = { ...overrideConfig, ...config }; +} + +export function getFsSafePythonConfig(): FsSafePythonConfig { + return { + mode: + overrideConfig.mode ?? + parseMode(process.env.FS_SAFE_PYTHON_MODE) ?? + parseMode(process.env.OPENCLAW_FS_SAFE_PYTHON_MODE) ?? + "auto", + pythonPath: + overrideConfig.pythonPath ?? + process.env.FS_SAFE_PYTHON ?? + process.env.OPENCLAW_FS_SAFE_PYTHON ?? + process.env.OPENCLAW_PINNED_PYTHON ?? + process.env.OPENCLAW_PINNED_WRITE_PYTHON, + }; +} + +export function canFallbackFromPythonError(error: unknown): boolean { + return ( + getFsSafePythonConfig().mode !== "require" && + error instanceof Error && + "code" in error && + (error as { code?: unknown }).code === "helper-unavailable" + ); +} diff --git a/src/pinned-python.ts b/src/pinned-python.ts new file mode 100644 index 0000000..fe9c49a --- /dev/null +++ b/src/pinned-python.ts @@ -0,0 +1,654 @@ +import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process"; +import fsSync from "node:fs"; +import { FsSafeError } from "./errors.js"; +import { getFsSafePythonConfig } from "./pinned-python-config.js"; + +const PINNED_PYTHON_WORKER_SOURCE = String.raw` +import base64 +import errno +import json +import os +import secrets +import stat +import sys + +DIR_FLAGS = os.O_RDONLY +if hasattr(os, "O_DIRECTORY"): + DIR_FLAGS |= os.O_DIRECTORY +if hasattr(os, "O_NOFOLLOW"): + DIR_FLAGS |= os.O_NOFOLLOW +READ_FLAGS = os.O_RDONLY +if hasattr(os, "O_NOFOLLOW"): + READ_FLAGS |= os.O_NOFOLLOW +WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL +if hasattr(os, "O_NOFOLLOW"): + WRITE_FLAGS |= os.O_NOFOLLOW + +def split_relative(value): + if value in ("", "."): + return [] + if "\x00" in value or value.startswith("/") or value.startswith("//"): + raise OSError(errno.EPERM, "invalid relative path") + if value.startswith("..\\"): + raise OSError(errno.EPERM, "path traversal is not allowed") + parts = [part for part in value.split("/") if part and part != "."] + for part in parts: + if part == "..": + raise OSError(errno.EPERM, "path traversal is not allowed") + return parts + +def open_dir(path_value, dir_fd=None): + return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd) + +def walk_dir(root_fd, segments, mkdir_enabled=False): + current_fd = os.dup(root_fd) + try: + for segment in segments: + try: + next_fd = open_dir(segment, dir_fd=current_fd) + except FileNotFoundError: + if not mkdir_enabled: + raise + os.mkdir(segment, 0o777, dir_fd=current_fd) + next_fd = open_dir(segment, dir_fd=current_fd) + os.close(current_fd) + current_fd = next_fd + return current_fd + except Exception: + os.close(current_fd) + raise + +def parent_and_basename(root_fd, relative): + segments = split_relative(relative) + if not segments: + raise OSError(errno.EPERM, "operation requires a non-root path") + parent_fd = walk_dir(root_fd, segments[:-1]) + return parent_fd, segments[-1] + +def encode_stat(st): + mode = st.st_mode + return { + "dev": st.st_dev, + "gid": st.st_gid, + "ino": st.st_ino, + "isDirectory": stat.S_ISDIR(mode), + "isFile": stat.S_ISREG(mode), + "isSymbolicLink": stat.S_ISLNK(mode), + "mode": mode, + "mtimeMs": st.st_mtime * 1000, + "nlink": st.st_nlink, + "size": st.st_size, + "uid": st.st_uid, + } + +def reject_unsafe_endpoint(st): + mode = st.st_mode + if stat.S_ISLNK(mode): + raise OSError(errno.ELOOP, "symlink endpoint is not allowed") + if stat.S_ISREG(mode) and st.st_nlink > 1: + raise OSError(errno.EPERM, "hardlinked file endpoint is not allowed") + +def stat_path(root_fd, payload): + relative = payload.get("relativePath", "") + segments = split_relative(relative) + if not segments: + return encode_stat(os.fstat(root_fd)) + parent_fd, basename = parent_and_basename(root_fd, relative) + try: + st = os.lstat(basename, dir_fd=parent_fd) + if payload.get("rejectSymlink", True) and stat.S_ISLNK(st.st_mode): + raise OSError(errno.ELOOP, "symlink endpoint is not allowed") + return encode_stat(st) + finally: + os.close(parent_fd) + +def readdir_path(root_fd, payload): + dir_fd = walk_dir(root_fd, split_relative(payload.get("relativePath", ""))) + try: + names = sorted(os.listdir(dir_fd)) + if not payload.get("withFileTypes", False): + return names + entries = [] + for name in names: + st = os.lstat(name, dir_fd=dir_fd) + entry = encode_stat(st) + entry["name"] = name + entries.append(entry) + return entries + finally: + os.close(dir_fd) + +def mkdirp_path(root_fd, payload): + dir_fd = walk_dir(root_fd, split_relative(payload.get("relativePath", "")), mkdir_enabled=True) + os.close(dir_fd) + return None + +def remove_tree(parent_fd, basename): + st = os.lstat(basename, dir_fd=parent_fd) + if stat.S_ISDIR(st.st_mode) and not stat.S_ISLNK(st.st_mode): + dir_fd = open_dir(basename, dir_fd=parent_fd) + try: + for child in os.listdir(dir_fd): + remove_tree(dir_fd, child) + finally: + os.close(dir_fd) + os.rmdir(basename, dir_fd=parent_fd) + else: + os.unlink(basename, dir_fd=parent_fd) + +def remove_path(root_fd, payload): + parent_fd, basename = parent_and_basename(root_fd, payload.get("relativePath", "")) + try: + try: + st = os.lstat(basename, dir_fd=parent_fd) + except FileNotFoundError: + if payload.get("force", True): + return None + raise + if stat.S_ISDIR(st.st_mode) and not stat.S_ISLNK(st.st_mode): + if payload.get("recursive", False): + remove_tree(parent_fd, basename) + else: + os.rmdir(basename, dir_fd=parent_fd) + else: + os.unlink(basename, dir_fd=parent_fd) + return None + finally: + os.close(parent_fd) + +def rename_path(root_fd, payload): + from_parent_fd, from_base = parent_and_basename(root_fd, payload["from"]) + to_parent_fd, to_base = parent_and_basename(root_fd, payload["to"]) + try: + from_stat = os.lstat(from_base, dir_fd=from_parent_fd) + reject_unsafe_endpoint(from_stat) + if not payload.get("overwrite", True): + try: + os.lstat(to_base, dir_fd=to_parent_fd) + raise FileExistsError(errno.EEXIST, "destination exists", to_base) + except FileNotFoundError: + pass + os.rename(from_base, to_base, src_dir_fd=from_parent_fd, dst_dir_fd=to_parent_fd) + os.fsync(from_parent_fd) + if from_parent_fd != to_parent_fd: + os.fsync(to_parent_fd) + return None + finally: + os.close(from_parent_fd) + os.close(to_parent_fd) + +def create_temp_file(parent_fd, basename, mode): + prefix = "." + basename + "." + for _ in range(128): + candidate = prefix + secrets.token_hex(6) + ".tmp" + try: + fd = os.open(candidate, WRITE_FLAGS, mode, dir_fd=parent_fd) + return candidate, fd + except FileExistsError: + continue + raise RuntimeError("failed to allocate pinned temp file") + +def write_path(root_fd, payload): + parent_fd = walk_dir(root_fd, split_relative(payload.get("relativeParentPath", "")), bool(payload.get("mkdir", True))) + temp_fd = None + temp_name = None + basename = payload["basename"] + mode = int(payload.get("mode", 0o600)) + overwrite = bool(payload.get("overwrite", True)) + max_bytes = int(payload.get("maxBytes", -1)) + data = base64.b64decode(payload.get("base64", "")) + try: + if max_bytes >= 0 and len(data) > max_bytes: + raise RuntimeError("fs-safe-too-large:%d:%d" % (max_bytes, len(data))) + if not overwrite: + try: + os.lstat(basename, dir_fd=parent_fd) + raise FileExistsError(errno.EEXIST, "destination exists", basename) + except FileNotFoundError: + pass + temp_name, temp_fd = create_temp_file(parent_fd, basename, mode) + view = memoryview(data) + while view: + written = os.write(temp_fd, view) + if written <= 0: + raise OSError(errno.EIO, "short write") + view = view[written:] + os.fsync(temp_fd) + os.close(temp_fd) + temp_fd = None + os.replace(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) + temp_name = None + os.fsync(parent_fd) + result_stat = os.stat(basename, dir_fd=parent_fd, follow_symlinks=False) + return {"dev": result_stat.st_dev, "ino": result_stat.st_ino} + finally: + if temp_fd is not None: + os.close(temp_fd) + if temp_name is not None: + try: + os.unlink(temp_name, dir_fd=parent_fd) + except FileNotFoundError: + pass + os.close(parent_fd) + +def copy_path(root_fd, payload): + source_fd = os.open(payload["sourcePath"], READ_FLAGS) + parent_fd = None + temp_fd = None + temp_name = None + try: + source_stat = os.fstat(source_fd) + if not stat.S_ISREG(source_stat.st_mode): + raise RuntimeError("fs-safe-not-file") + if source_stat.st_dev != int(payload["sourceDev"]) or source_stat.st_ino != int(payload["sourceIno"]): + raise RuntimeError("fs-safe-source-mismatch") + basename = payload["basename"] + mode = int(payload.get("mode", 0o600)) + overwrite = bool(payload.get("overwrite", True)) + max_bytes = int(payload.get("maxBytes", -1)) + if max_bytes >= 0 and source_stat.st_size > max_bytes: + raise RuntimeError("fs-safe-too-large:%d:%d" % (max_bytes, source_stat.st_size)) + parent_fd = walk_dir(root_fd, split_relative(payload.get("relativeParentPath", "")), bool(payload.get("mkdir", True))) + if not overwrite: + try: + os.lstat(basename, dir_fd=parent_fd) + raise FileExistsError(errno.EEXIST, "destination exists", basename) + except FileNotFoundError: + pass + temp_name, temp_fd = create_temp_file(parent_fd, basename, mode) + written_bytes = 0 + while True: + chunk = os.read(source_fd, 65536) + if not chunk: + break + written_bytes += len(chunk) + if max_bytes >= 0 and written_bytes > max_bytes: + raise RuntimeError("fs-safe-too-large:%d:%d" % (max_bytes, written_bytes)) + view = memoryview(chunk) + while view: + written = os.write(temp_fd, view) + if written <= 0: + raise OSError(errno.EIO, "short write") + view = view[written:] + os.fsync(temp_fd) + os.close(temp_fd) + temp_fd = None + os.replace(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd) + temp_name = None + os.fsync(parent_fd) + result_stat = os.stat(basename, dir_fd=parent_fd, follow_symlinks=False) + return {"dev": result_stat.st_dev, "ino": result_stat.st_ino} + finally: + os.close(source_fd) + if temp_fd is not None: + os.close(temp_fd) + if temp_name is not None and parent_fd is not None: + try: + os.unlink(temp_name, dir_fd=parent_fd) + except FileNotFoundError: + pass + if parent_fd is not None: + os.close(parent_fd) + +def run_operation(operation, root_path, payload): + root_fd = open_dir(root_path) + try: + if operation == "stat": + return stat_path(root_fd, payload) + if operation == "readdir": + return readdir_path(root_fd, payload) + if operation == "mkdirp": + return mkdirp_path(root_fd, payload) + if operation == "remove": + return remove_path(root_fd, payload) + if operation == "rename": + return rename_path(root_fd, payload) + if operation == "write": + return write_path(root_fd, payload) + if operation == "copy": + return copy_path(root_fd, payload) + raise RuntimeError("unknown operation: " + operation) + finally: + os.close(root_fd) + +for line in sys.stdin: + try: + request = json.loads(line) + result = run_operation(request["operation"], request["rootPath"], request.get("payload") or {}) + response = {"id": request["id"], "ok": True, "result": result} + except Exception as exc: + response = { + "id": request.get("id") if isinstance(locals().get("request"), dict) else None, + "ok": False, + "code": exc.__class__.__name__, + "errno": getattr(exc, "errno", None), + "message": str(exc), + } + print(json.dumps(response, separators=(",", ":")), flush=True) +`; + +type PinnedPythonOperation = + | "copy" + | "stat" + | "readdir" + | "mkdirp" + | "remove" + | "rename" + | "write"; + +type PendingRequest = { + reject(error: unknown): void; + resolve(value: unknown): void; +}; + +type PinnedPythonWorker = { + child: ChildProcessWithoutNullStreams; + pending: Map; + stderr: string; + stdoutBuffer: string; +}; + +let nextRequestId = 1; +let worker: PinnedPythonWorker | null = null; + +export function __resetPinnedPythonWorkerForTest(): void { + const currentWorker = worker; + worker = null; + if (!currentWorker) { + return; + } + currentWorker.pending.clear(); + currentWorker.child.kill("SIGTERM"); +} + +const PYTHON_CANDIDATE_DEFAULTS = [ + "/usr/bin/python3", + "/opt/homebrew/bin/python3", + "/usr/local/bin/python3", +]; + +function canExecute(binPath: string): boolean { + try { + fsSync.accessSync(binPath, fsSync.constants.X_OK); + return true; + } catch { + return false; + } +} + +function resolvePython(): string { + const configured = getFsSafePythonConfig().pythonPath; + if (configured) { + return configured; + } + for (const candidate of PYTHON_CANDIDATE_DEFAULTS) { + if (canExecute(candidate)) { + return candidate; + } + } + return "python3"; +} + +function assertPinnedHelperSupported(): void { + if (process.platform === "win32") { + throw new FsSafeError( + "unsupported-platform", + "fd-relative pinned filesystem operations are not available on Windows", + ); + } + if (getFsSafePythonConfig().mode === "off") { + throw new FsSafeError("helper-unavailable", "Python helper is disabled"); + } +} + +function isSpawnUnavailable(error: unknown): boolean { + if (!(error instanceof Error)) { + return false; + } + const maybeErrno = error as NodeJS.ErrnoException; + return ( + typeof maybeErrno.syscall === "string" && + maybeErrno.syscall.startsWith("spawn") && + ["EACCES", "ENOENT", "ENOEXEC"].includes(maybeErrno.code ?? "") + ); +} + +function mapWorkerError(response: Record): Error { + const code = typeof response.code === "string" ? response.code : ""; + const errno = typeof response.errno === "number" ? response.errno : undefined; + const message = + typeof response.message === "string" && response.message + ? response.message + : "pinned helper failed"; + const tooLarge = message.match(/fs-safe-too-large:(\d+):(\d+)/); + if (tooLarge) { + const [, limit, got] = tooLarge; + return new FsSafeError("too-large", `file exceeds limit of ${limit} bytes (got at least ${got})`); + } + if (message.includes("fs-safe-not-file")) { + return new FsSafeError("not-file", "not a file"); + } + if (message.includes("fs-safe-source-mismatch")) { + return new FsSafeError("path-mismatch", "source path changed during copy"); + } + if (code === "FileNotFoundError" || errno === 2) { + return new FsSafeError("not-found", "file not found"); + } + if (code === "FileExistsError" || errno === 17) { + return new FsSafeError("already-exists", message); + } + if (errno === 39) { + return new FsSafeError("not-empty", "directory is not empty"); + } + if (errno === 1 || errno === 13 || errno === 21) { + return new FsSafeError("not-removable", "path is not removable under root"); + } + if (code === "NotADirectoryError" || code === "OSError" || errno === 20 || errno === 40) { + return new FsSafeError("path-alias", message); + } + return new FsSafeError("helper-failed", message); +} + +function rejectPending(error: Error): void { + if (!worker) { + return; + } + setWorkerRef(worker, false); + for (const pending of worker.pending.values()) { + pending.reject(error); + } + worker.pending.clear(); + worker = null; +} + +function handleWorkerLine(line: string): void { + if (!worker || !line.trim()) { + return; + } + let decoded: unknown; + try { + decoded = JSON.parse(line) as unknown; + } catch { + rejectPending(new FsSafeError("helper-failed", `pinned helper returned invalid JSON: ${line}`)); + return; + } + if (typeof decoded !== "object" || decoded === null || !("id" in decoded)) { + rejectPending(new FsSafeError("helper-failed", "pinned helper returned invalid response")); + return; + } + const response = decoded as { id?: unknown; ok?: unknown; result?: unknown }; + const id = typeof response.id === "number" ? response.id : undefined; + if (id === undefined) { + return; + } + const pending = worker.pending.get(id); + if (!pending) { + return; + } + worker.pending.delete(id); + if (worker.pending.size === 0) { + setWorkerRef(worker, false); + } + if (response.ok === true) { + pending.resolve(response.result); + return; + } + pending.reject(mapWorkerError(decoded as Record)); +} + +function getWorker() { + assertPinnedHelperSupported(); + if (worker) { + return worker; + } + const child = spawn(resolvePython(), ["-u", "-c", PINNED_PYTHON_WORKER_SOURCE], { + stdio: ["pipe", "pipe", "pipe"], + }); + worker = { child, pending: new Map(), stderr: "", stdoutBuffer: "" }; + child.stdout.setEncoding("utf8"); + child.stderr.setEncoding("utf8"); + child.stdout.on("data", (chunk: string) => { + const current = worker; + if (!current) { + return; + } + current.stdoutBuffer += chunk; + for (;;) { + const newline = current.stdoutBuffer.indexOf("\n"); + if (newline < 0) { + break; + } + const line = current.stdoutBuffer.slice(0, newline); + current.stdoutBuffer = current.stdoutBuffer.slice(newline + 1); + handleWorkerLine(line); + } + }); + child.stderr.on("data", (chunk: string) => { + if (worker) { + worker.stderr = `${worker.stderr}${chunk}`.slice(-4096); + } + }); + child.once("error", (error) => { + const mapped = isSpawnUnavailable(error) + ? new FsSafeError("helper-unavailable", "Python helper is unavailable", { cause: error }) + : error instanceof Error + ? error + : new Error(String(error)); + rejectPending(mapped); + }); + child.once("close", (code, signal) => { + const stderr = worker?.stderr.trim(); + rejectPending( + new FsSafeError( + "helper-failed", + stderr || `pinned helper exited with code ${code ?? "null"} (${signal ?? "?"})`, + ), + ); + }); + process.once("exit", () => { + child.kill("SIGTERM"); + }); + setWorkerRef(worker, false); + return worker; +} + +function setRefable(value: unknown, ref: boolean): void { + if (!value) { + return; + } + const method = ref ? "ref" : "unref"; + const refable = value as { ref?: () => void; unref?: () => void }; + refable[method]?.(); +} + +function setWorkerRef(currentWorker: PinnedPythonWorker, ref: boolean): void { + setRefable(currentWorker.child, ref); + setRefable(currentWorker.child.stdin, ref); + setRefable(currentWorker.child.stdout, ref); + setRefable(currentWorker.child.stderr, ref); +} + +export async function runPinnedPythonOperation(params: { + operation: PinnedPythonOperation; + rootPath: string; + payload: Record; +}): Promise { + const requestId = nextRequestId++; + const currentWorker = getWorker(); + if (typeof currentWorker.child.stdin?.write !== "function") { + throw new FsSafeError("helper-unavailable", "Python helper stdin is unavailable"); + } + setWorkerRef(currentWorker, true); + return await new Promise((resolve, reject) => { + currentWorker.pending.set(requestId, { + reject, + resolve: (value) => resolve(value as T), + }); + const request = JSON.stringify({ + id: requestId, + operation: params.operation, + rootPath: params.rootPath, + payload: params.payload, + }); + currentWorker.child.stdin.write(`${request}\n`, (error) => { + if (error) { + currentWorker.pending.delete(requestId); + if (currentWorker.pending.size === 0) { + setWorkerRef(currentWorker, false); + } + reject(error); + } + }); + }); +} + +export function assertPinnedPythonOperationAvailable(): void { + const currentWorker = getWorker(); + if (typeof currentWorker.child.stdin?.write !== "function") { + throw new FsSafeError("helper-unavailable", "Python helper stdin is unavailable"); + } +} + +export function validatePinnedOperationPayload(payload: Record): void { + if (typeof payload.relativePath === "string") { + validatePinnedRelativePath(payload.relativePath); + } + if (typeof payload.relativeParentPath === "string") { + validatePinnedRelativePath(payload.relativeParentPath); + } + if (typeof payload.from === "string") { + validatePinnedRelativePath(payload.from); + } + if (typeof payload.to === "string") { + validatePinnedRelativePath(payload.to); + } +} + +export function isPinnedHelperUnavailable(error: unknown): boolean { + return error instanceof Error && + "code" in error && + (error as { code?: unknown }).code === "helper-unavailable"; +} + +function validatePinnedRelativePath(relativePath: string): void { + if (relativePath.length === 0 || relativePath === ".") { + return; + } + if (relativePath.includes("\0")) { + throw new FsSafeError("invalid-path", "relative path contains a NUL byte"); + } + if ( + relativePath.startsWith("/") || + relativePath.startsWith("//") || + relativePath === ".." || + relativePath.startsWith("../") || + relativePath.startsWith("..\\") + ) { + throw new FsSafeError("invalid-path", "relative path must not escape root"); + } + for (const segment of relativePath.split("/")) { + if (segment === "..") { + throw new FsSafeError("invalid-path", "relative path must not contain '..'"); + } + } +} diff --git a/src/pinned-write.ts b/src/pinned-write.ts index ba4b592..7a9afb2 100644 --- a/src/pinned-write.ts +++ b/src/pinned-write.ts @@ -1,5 +1,3 @@ -import { spawn } from "node:child_process"; -import { once } from "node:events"; import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; @@ -7,6 +5,12 @@ import { Transform, type Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { FsSafeError } from "./errors.js"; import type { FileIdentityStat } from "./file-identity.js"; +import { canFallbackFromPythonError, getFsSafePythonConfig } from "./pinned-python-config.js"; +import { + assertPinnedPythonOperationAvailable, + runPinnedPythonOperation, + validatePinnedOperationPayload, +} from "./pinned-python.js"; type PinnedWriteInput = | { kind: "buffer"; data: string | Buffer; encoding?: BufferEncoding } @@ -18,6 +22,18 @@ function byteLength(input: string | Buffer, encoding: BufferEncoding | undefined : input.byteLength; } +function assertSafeBasename(basename: string): void { + if ( + !basename || + basename === "." || + basename === ".." || + basename.includes("/") || + basename.includes("\0") + ) { + throw new FsSafeError("invalid-path", "invalid target path"); + } +} + function assertWithinMaxBytes(bytes: number, maxBytes: number | undefined): void { if (maxBytes !== undefined && bytes > maxBytes) { throw new FsSafeError( @@ -62,161 +78,27 @@ async function pipelineWithMaxBytes( await pipeline(stream, destination); } -const LOCAL_PINNED_WRITE_PYTHON = [ - "import errno", - "import os", - "import secrets", - "import stat", - "import sys", - "", - "root_path = sys.argv[1]", - "relative_parent = sys.argv[2]", - "basename = sys.argv[3]", - 'mkdir_enabled = sys.argv[4] == "1"', - "file_mode = int(sys.argv[5], 8)", - 'overwrite_enabled = sys.argv[6] == "1"', - "max_bytes = int(sys.argv[7])", - "", - "DIR_FLAGS = os.O_RDONLY", - "if hasattr(os, 'O_DIRECTORY'):", - " DIR_FLAGS |= os.O_DIRECTORY", - "if hasattr(os, 'O_NOFOLLOW'):", - " DIR_FLAGS |= os.O_NOFOLLOW", - "", - "WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL", - "if hasattr(os, 'O_NOFOLLOW'):", - " WRITE_FLAGS |= os.O_NOFOLLOW", - "", - "def open_dir(path_value, dir_fd=None):", - " return os.open(path_value, DIR_FLAGS, dir_fd=dir_fd)", - "", - "def walk_parent(root_fd, rel_parent, mkdir_enabled):", - " current_fd = os.dup(root_fd)", - " try:", - " for segment in [part for part in rel_parent.split('/') if part and part != '.']:", - " if segment == '..':", - " raise OSError(errno.EPERM, 'path traversal is not allowed', segment)", - " try:", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " except FileNotFoundError:", - " if not mkdir_enabled:", - " raise", - " os.mkdir(segment, 0o777, dir_fd=current_fd)", - " next_fd = open_dir(segment, dir_fd=current_fd)", - " os.close(current_fd)", - " current_fd = next_fd", - " return current_fd", - " except Exception:", - " os.close(current_fd)", - " raise", - "", - "def create_temp_file(parent_fd, basename, mode):", - " prefix = '.' + basename + '.'", - " for _ in range(128):", - " candidate = prefix + secrets.token_hex(6) + '.tmp'", - " try:", - " fd = os.open(candidate, WRITE_FLAGS, mode, dir_fd=parent_fd)", - " return candidate, fd", - " except FileExistsError:", - " continue", - " raise RuntimeError('failed to allocate pinned temp file')", - "", - "root_fd = open_dir(root_path)", - "parent_fd = None", - "temp_fd = None", - "temp_name = None", - "try:", - " parent_fd = walk_parent(root_fd, relative_parent, mkdir_enabled)", - " temp_name, temp_fd = create_temp_file(parent_fd, basename, file_mode)", - " written_bytes = 0", - " while True:", - " chunk = sys.stdin.buffer.read(65536)", - " if not chunk:", - " break", - " next_size = written_bytes + len(chunk)", - " if max_bytes >= 0 and next_size > max_bytes:", - " raise RuntimeError(f'fs-safe-too-large:{max_bytes}:{next_size}')", - " view = memoryview(chunk)", - " while view:", - " written = os.write(temp_fd, view)", - " if written <= 0:", - " raise OSError(errno.EIO, 'short write')", - " view = view[written:]", - " written_bytes = next_size", - " os.fsync(temp_fd)", - " os.close(temp_fd)", - " temp_fd = None", - " if overwrite_enabled:", - " os.replace(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd)", - " temp_name = None", - " else:", - " os.link(temp_name, basename, src_dir_fd=parent_fd, dst_dir_fd=parent_fd, follow_symlinks=False)", - " os.unlink(temp_name, dir_fd=parent_fd)", - " temp_name = None", - " os.fsync(parent_fd)", - " result_stat = os.stat(basename, dir_fd=parent_fd, follow_symlinks=False)", - " print(f'{result_stat.st_dev}|{result_stat.st_ino}')", - "finally:", - " if temp_fd is not None:", - " os.close(temp_fd)", - " if temp_name is not None and parent_fd is not None:", - " try:", - " os.unlink(temp_name, dir_fd=parent_fd)", - " except FileNotFoundError:", - " pass", - " if parent_fd is not None:", - " os.close(parent_fd)", - " os.close(root_fd)", -].join("\n"); - -const PINNED_WRITE_PYTHON_CANDIDATES = [ - process.env.OPENCLAW_PINNED_WRITE_PYTHON, - "/usr/bin/python3", - "/opt/homebrew/bin/python3", - "/usr/local/bin/python3", -].filter((value): value is string => Boolean(value)); - -let cachedPinnedWritePython = ""; - -function canExecute(binPath: string): boolean { - try { - fsSync.accessSync(binPath, fsSync.constants.X_OK); - return true; - } catch { - return false; +async function inputToBase64( + input: PinnedWriteInput, + maxBytes: number | undefined, +): Promise { + if (input.kind === "buffer") { + assertWithinMaxBytes(byteLength(input.data, input.encoding), maxBytes); + return ( + typeof input.data === "string" + ? Buffer.from(input.data, input.encoding ?? "utf8") + : input.data + ).toString("base64"); } -} - -function resolvePinnedWritePython(): string { - if (cachedPinnedWritePython) { - return cachedPinnedWritePython; + const chunks: Buffer[] = []; + let bytes = 0; + for await (const chunk of input.stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk as Uint8Array); + bytes += buffer.byteLength; + assertWithinMaxBytes(bytes, maxBytes); + chunks.push(buffer); } - for (const candidate of PINNED_WRITE_PYTHON_CANDIDATES) { - if (canExecute(candidate)) { - cachedPinnedWritePython = candidate; - return cachedPinnedWritePython; - } - } - cachedPinnedWritePython = "python3"; - return cachedPinnedWritePython; -} - -function parsePinnedIdentity(stdout: string): FileIdentityStat { - const line = stdout - .trim() - .split(/\r?\n/) - .map((value) => value.trim()) - .findLast(Boolean); - if (!line) { - throw new Error("Pinned write helper returned no identity"); - } - const [devRaw, inoRaw] = line.split("|"); - const dev = Number.parseInt(devRaw ?? "", 10); - const ino = Number.parseInt(inoRaw ?? "", 10); - if (!Number.isFinite(dev) || !Number.isFinite(ino)) { - throw new Error(`Pinned write helper returned invalid identity: ${line}`); - } - return { dev, ino }; + return Buffer.concat(chunks, bytes).toString("base64"); } export async function runPinnedWriteHelper(params: { @@ -229,80 +111,78 @@ export async function runPinnedWriteHelper(params: { maxBytes?: number; input: PinnedWriteInput; }): Promise { - const child = spawn( - resolvePinnedWritePython(), - [ - "-c", - LOCAL_PINNED_WRITE_PYTHON, - params.rootPath, - params.relativeParentPath, - params.basename, - params.mkdir ? "1" : "0", - (params.mode || 0o600).toString(8), - params.overwrite === false ? "0" : "1", - params.maxBytes === undefined ? "-1" : String(params.maxBytes), - ], - { - stdio: ["pipe", "pipe", "pipe"], - }, - ); - - let stdout = ""; - let stderr = ""; - child.stdout.setEncoding?.("utf8"); - child.stderr.setEncoding?.("utf8"); - child.stdout.on("data", (chunk: string) => { - stdout += chunk; + if (getFsSafePythonConfig().mode === "off") { + return await runPinnedWriteFallback(params); + } + assertSafeBasename(params.basename); + validatePinnedOperationPayload({ + relativeParentPath: params.relativeParentPath, }); - child.stderr.on("data", (chunk: string) => { - stderr += chunk; - }); - - const exitPromise = once(child, "close") as Promise<[number | null, NodeJS.Signals | null]>; - try { - if (!child.stdin) { - const identity = await runPinnedWriteFallback(params); - await exitPromise.catch(() => {}); - return identity; - } - - if (params.input.kind === "buffer") { - const input = params.input; - await new Promise((resolve, reject) => { - child.stdin.once("error", reject); - if (typeof input.data === "string") { - child.stdin.end(input.data, input.encoding ?? "utf8", () => resolve()); - return; - } - child.stdin.end(input.data, () => resolve()); - }); - } else { - await pipeline(params.input.stream, child.stdin); - } - - const [code, signal] = await exitPromise; - if (code !== 0) { - const tooLarge = stderr.match(/fs-safe-too-large:(\d+):(\d+)/); - if (tooLarge) { - const [, limit, got] = tooLarge; - throw new FsSafeError( - "too-large", - `file exceeds limit of ${limit} bytes (got at least ${got})`, - ); + if (params.input.kind === "stream") { + try { + assertPinnedPythonOperationAvailable(); + } catch (error) { + if (canFallbackFromPythonError(error)) { + return await runPinnedWriteFallback(params); } - throw new Error( - stderr.trim() || - `Pinned write helper failed with code ${code ?? "null"} (${signal ?? "?"})`, - ); + throw error; } - return parsePinnedIdentity(stdout); + } + const payload = { + base64: await inputToBase64(params.input, params.maxBytes), + basename: params.basename, + maxBytes: params.maxBytes ?? -1, + mkdir: params.mkdir, + mode: params.mode || 0o600, + overwrite: params.overwrite !== false, + relativeParentPath: params.relativeParentPath, + }; + try { + return await runPinnedPythonOperation({ + operation: "write", + rootPath: params.rootPath, + payload, + }); } catch (error) { - child.kill("SIGKILL"); - await exitPromise.catch(() => {}); + if (canFallbackFromPythonError(error)) { + return await runPinnedWriteFallback(params); + } throw error; } } +export async function runPinnedCopyHelper(params: { + rootPath: string; + relativeParentPath: string; + basename: string; + mkdir: boolean; + mode: number; + overwrite?: boolean; + maxBytes?: number; + sourcePath: string; + sourceIdentity: FileIdentityStat; +}): Promise { + assertSafeBasename(params.basename); + validatePinnedOperationPayload({ + relativeParentPath: params.relativeParentPath, + }); + return await runPinnedPythonOperation({ + operation: "copy", + rootPath: params.rootPath, + payload: { + basename: params.basename, + maxBytes: params.maxBytes ?? -1, + mkdir: params.mkdir, + mode: params.mode || 0o600, + overwrite: params.overwrite !== false, + relativeParentPath: params.relativeParentPath, + sourceDev: params.sourceIdentity.dev, + sourceIno: params.sourceIdentity.ino, + sourcePath: params.sourcePath, + }, + }); +} + async function runPinnedWriteFallback(params: { rootPath: string; relativeParentPath: string; diff --git a/src/root.ts b/src/root.ts index fce9c74..b961bce 100644 --- a/src/root.ts +++ b/src/root.ts @@ -10,7 +10,8 @@ import { pipeline } from "node:stream/promises"; import { FsSafeError } from "./errors.js"; import { sameFileIdentity } from "./file-identity.js"; import { isPinnedPathHelperSpawnError, runPinnedPathHelper } from "./pinned-path.js"; -import { runPinnedWriteHelper } from "./pinned-write.js"; +import { runPinnedCopyHelper, runPinnedWriteHelper } from "./pinned-write.js"; +import { canFallbackFromPythonError, getFsSafePythonConfig } from "./pinned-python-config.js"; import { expandHomePrefix } from "./home-dir.js"; import { assertNoPathAliasEscape, PATH_ALIAS_POLICIES } from "./path-policy.js"; import { @@ -19,7 +20,11 @@ import { isPathInside, isSymlinkOpenError, } from "./path.js"; -import { helperReaddir, helperStat, runPinnedHelper } from "./pinned-helper.js"; +import { + helperReaddir, + helperStat, + runPinnedHelper, +} from "./pinned-helper.js"; import { resolveRootPath } from "./root-path.js"; import { getFsSafeTestHooks } from "./test-hooks.js"; import type { DirEntry, PathStat } from "./types.js"; @@ -567,7 +572,14 @@ class RootHandle implements Root { } async stat(relativePath: string): Promise { - return await helperStat(this.rootReal, relativePath); + try { + return await helperStat(this.rootReal, relativePath); + } catch (error) { + if (canFallbackFromPythonError(error)) { + return await statPathFallback(this.context, relativePath); + } + throw error; + } } async list(relativePath: string, options?: { withFileTypes?: false }): Promise; @@ -576,9 +588,16 @@ class RootHandle implements Root { relativePath: string, options: { withFileTypes?: boolean } = {}, ): Promise { - return options.withFileTypes === true - ? await helperReaddir(this.rootReal, relativePath, true) - : await helperReaddir(this.rootReal, relativePath, false); + try { + return options.withFileTypes === true + ? await helperReaddir(this.rootReal, relativePath, true) + : await helperReaddir(this.rootReal, relativePath, false); + } catch (error) { + if (canFallbackFromPythonError(error)) { + return await listPathFallback(this.context, relativePath, options.withFileTypes === true); + } + throw error; + } } async move( @@ -586,11 +605,23 @@ class RootHandle implements Root { toRelative: string, options: { overwrite?: boolean } = {}, ): Promise { - await runPinnedHelper("rename", this.rootReal, { - from: fromRelative, - overwrite: options.overwrite ?? false, - to: toRelative, - }); + try { + await runPinnedHelper("rename", this.rootReal, { + from: fromRelative, + overwrite: options.overwrite ?? false, + to: toRelative, + }); + } catch (error) { + if (canFallbackFromPythonError(error)) { + await movePathFallback(this.context, { + fromRelative, + overwrite: options.overwrite ?? false, + toRelative, + }); + return; + } + throw error; + } } } @@ -1213,22 +1244,30 @@ async function copyFileInRoot( const pinned = await resolvePinnedWriteTargetInRoot(root, params.relativePath, params.mode); await serializePathWrite(pinned.targetPath, async () => { - const sourceStream = createBoundedReadStream(source, params.maxBytes); - const identity = await runPinnedWriteHelper({ - rootPath: pinned.rootReal, - relativeParentPath: pinned.relativeParentPath, - basename: pinned.basename, - mkdir: params.mkdir !== false, - mode: pinned.mode, - overwrite: true, - maxBytes: params.maxBytes, - input: { - kind: "stream", - stream: sourceStream, - }, - }).catch((error) => { + let identity; + try { + if (getFsSafePythonConfig().mode === "off") { + await copyFileFallback(root, params, source); + return; + } + identity = await runPinnedCopyHelper({ + rootPath: pinned.rootReal, + relativeParentPath: pinned.relativeParentPath, + basename: pinned.basename, + mkdir: params.mkdir !== false, + mode: pinned.mode, + overwrite: true, + maxBytes: params.maxBytes, + sourcePath: source.realPath, + sourceIdentity: { dev: source.stat.dev, ino: source.stat.ino }, + }); + } catch (error) { + if (canFallbackFromPythonError(error)) { + await copyFileFallback(root, params, source); + return; + } throw normalizePinnedWriteError(error); - }); + } try { await verifyAtomicWriteResult({ root, @@ -1428,6 +1467,145 @@ async function mkdirPathFallback(resolved: { resolved: string }): Promise await fs.mkdir(resolved.resolved, { recursive: true }); } +function pathStatFromStats(stat: Stats): PathStat { + return { + dev: Number(stat.dev), + gid: Number(stat.gid), + ino: Number(stat.ino), + isDirectory: stat.isDirectory(), + isFile: stat.isFile(), + isSymbolicLink: stat.isSymbolicLink(), + mode: stat.mode, + mtimeMs: stat.mtimeMs, + nlink: stat.nlink, + size: stat.size, + uid: stat.uid, + }; +} + +async function statPathFallback(root: RootContext, relativePath: string): Promise { + const resolved = await resolvePinnedPathInRoot(root, { relativePath, allowRoot: true }); + try { + return pathStatFromStats(await fs.lstat(resolved.resolved)); + } catch (error) { + if (isNotFoundPathError(error)) { + throw new FsSafeError("not-found", "file not found", { + cause: error instanceof Error ? error : undefined, + }); + } + throw error; + } +} + +async function listPathFallback( + root: RootContext, + relativePath: string, + withFileTypes: boolean, +): Promise { + const resolved = await resolvePinnedPathInRoot(root, { relativePath, allowRoot: true }); + try { + const names = await fs.readdir(resolved.resolved); + const sortedNames = names.toSorted(); + if (!withFileTypes) { + return sortedNames; + } + const entries: DirEntry[] = []; + for (const name of sortedNames) { + entries.push({ + name, + ...pathStatFromStats(await fs.lstat(path.join(resolved.resolved, name))), + }); + } + return entries; + } catch (error) { + if (isNotFoundPathError(error)) { + throw new FsSafeError("not-found", "directory not found", { + cause: error instanceof Error ? error : undefined, + }); + } + throw error; + } +} + +async function movePathFallback( + root: RootContext, + params: { + fromRelative: string; + toRelative: string; + overwrite: boolean; + }, +): Promise { + const source = await resolvePathInRoot(root, params.fromRelative); + await resolvePinnedRootPathInRoot(root, { + relativePath: params.fromRelative, + policy: PATH_ALIAS_POLICIES.strict, + }); + const target = await resolvePathInRoot(root, params.toRelative); + await resolvePinnedRootPathInRoot(root, { + relativePath: params.toRelative, + policy: PATH_ALIAS_POLICIES.unlinkTarget, + }); + try { + await assertNoPathAliasEscape({ + absolutePath: target.resolved, + rootPath: target.rootReal, + boundaryLabel: "root", + }); + } catch (error) { + throw new FsSafeError("path-alias", "path alias escape blocked", { + cause: error instanceof Error ? error : undefined, + }); + } + + let sourceStat: Stats; + try { + sourceStat = await fs.lstat(source.resolved); + } catch (error) { + if (isNotFoundPathError(error)) { + throw new FsSafeError("not-found", "file not found", { + cause: error instanceof Error ? error : undefined, + }); + } + throw error; + } + if (sourceStat.isSymbolicLink()) { + throw new FsSafeError("symlink", "symlink not allowed"); + } + if (sourceStat.isFile() && sourceStat.nlink > 1) { + throw new FsSafeError("hardlink", "hardlinked path not allowed"); + } + + if (!params.overwrite) { + try { + await fs.lstat(target.resolved); + throw new FsSafeError("already-exists", "destination exists"); + } catch (error) { + if (error instanceof FsSafeError) { + throw error; + } + if (!isNotFoundPathError(error)) { + throw error; + } + } + } + + try { + await fs.rename(source.resolved, target.resolved); + } catch (error) { + if (isNotFoundPathError(error)) { + throw new FsSafeError("not-found", "file not found", { + cause: error instanceof Error ? error : undefined, + }); + } + if (hasNodeErrorCode(error, "EEXIST")) { + throw new FsSafeError("already-exists", "destination exists", { + cause: error instanceof Error ? error : undefined, + }); + } + throw error; + } +} + async function writeFileFallback( root: RootContext, params: { diff --git a/test/fs-safe.test.ts b/test/fs-safe.test.ts index 06c0eae..5ffb5fd 100644 --- a/test/fs-safe.test.ts +++ b/test/fs-safe.test.ts @@ -1,9 +1,9 @@ import { appendFileSync } from "node:fs"; -import { mkdtemp, readdir, readFile, stat, symlink, writeFile } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; -import { FsSafeError, root as openRoot } from "../src/index.js"; +import { configureFsSafePython, FsSafeError, root as openRoot } from "../src/index.js"; import { __setFsSafeTestHooksForTest } from "../src/test-hooks.js"; const tempDirs: string[] = []; @@ -15,6 +15,7 @@ async function tempRoot(prefix: string): Promise { } afterEach(async () => { + configureFsSafePython({ mode: "auto", pythonPath: undefined }); __setFsSafeTestHooksForTest(undefined); const { rm } = await import("node:fs/promises"); await Promise.all(tempDirs.splice(0).map((dir) => rm(dir, { force: true, recursive: true }))); @@ -49,6 +50,25 @@ describe("@openclaw/fs-safe", () => { }); }); + it("can disable the Python helper and keep root operations available", async () => { + configureFsSafePython({ mode: "off" }); + const rootPath = await tempRoot("fs-safe-python-off-"); + const sourceRoot = await tempRoot("fs-safe-python-off-source-"); + const sourcePath = path.join(sourceRoot, "source.txt"); + const root = await openRoot(rootPath); + await writeFile(sourcePath, "copied"); + + await root.mkdir("nested"); + await root.write("nested/file.txt", "hello"); + await root.copyIn("nested/copied.txt", sourcePath, { maxBytes: 16 }); + await expect(root.stat("nested/file.txt")).resolves.toMatchObject({ isFile: true }); + await expect(root.list("nested")).resolves.toEqual(["copied.txt", "file.txt"]); + await root.move("nested/file.txt", "nested/moved.txt"); + await expect(root.readText("nested/moved.txt")).resolves.toBe("hello"); + await root.remove("nested/copied.txt"); + await expect(root.exists("nested/copied.txt")).resolves.toBe(false); + }); + it("applies per-root defaults", async () => { const rootPath = await tempRoot("fs-safe-defaults-"); const root = await openRoot(rootPath, { @@ -204,13 +224,7 @@ describe("@openclaw/fs-safe", () => { if (filePath !== sourcePath) { return; } - const createReadStream = handle.createReadStream.bind(handle); - Object.defineProperty(handle, "createReadStream", { - value: (...args: Parameters) => { - appendFileSync(sourcePath, "567890"); - return createReadStream(...args); - }, - }); + appendFileSync(sourcePath, "567890"); }, }); @@ -221,6 +235,43 @@ describe("@openclaw/fs-safe", () => { await expect(readdir(rootPath)).resolves.toEqual([]); }); + it("rejects pinned copy when the source path is swapped after identity capture", async () => { + if (process.platform === "win32") { + return; + } + const { runPinnedCopyHelper } = await import("../src/pinned-write.js"); + const rootPath = await tempRoot("fs-safe-copy-source-swap-root-"); + const sourceRoot = await tempRoot("fs-safe-copy-source-swap-source-"); + const sourcePath = path.join(sourceRoot, "source.txt"); + await writeFile(sourcePath, "original"); + const sourceIdentity = await stat(sourcePath); + await rm(sourcePath); + await writeFile(sourcePath, "replacement"); + + try { + await runPinnedCopyHelper({ + rootPath, + relativeParentPath: "", + basename: "copied.txt", + mkdir: true, + mode: 0o600, + overwrite: true, + maxBytes: 1024, + sourcePath, + sourceIdentity: { dev: sourceIdentity.dev, ino: sourceIdentity.ino }, + }); + throw new Error("expected pinned copy source swap to fail"); + } catch (error) { + if (error instanceof FsSafeError && error.code === "helper-unavailable") { + return; + } + expect(error).toMatchObject({ code: "path-mismatch" }); + } + await expect(stat(path.join(rootPath, "copied.txt"))).rejects.toMatchObject({ + code: "ENOENT", + }); + }); + it("removes symlink leaves without following them", async () => { const rootPath = await tempRoot("fs-safe-remove-"); const root = await openRoot(rootPath); diff --git a/test/pinned-python.test.ts b/test/pinned-python.test.ts new file mode 100644 index 0000000..9d0dd27 --- /dev/null +++ b/test/pinned-python.test.ts @@ -0,0 +1,231 @@ +import { EventEmitter } from "node:events"; +import fs from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { configureFsSafePython, root } from "../src/index.js"; +import { + canFallbackFromPythonError, + getFsSafePythonConfig, +} from "../src/pinned-python-config.js"; +import { + __resetPinnedPythonWorkerForTest, + runPinnedPythonOperation, +} from "../src/pinned-python.js"; +import * as configSubpath from "../src/config.js"; + +const spawnMock = vi.hoisted(() => vi.fn()); + +vi.mock("node:child_process", () => ({ + spawn: spawnMock, +})); + +type FakeChild = EventEmitter & { + kill: ReturnType; + ref: ReturnType; + stderr: EventEmitter & { + ref: ReturnType; + setEncoding: ReturnType; + unref: ReturnType; + }; + stdin: EventEmitter & { + ref: ReturnType; + unref: ReturnType; + write: ReturnType; + }; + stdout: EventEmitter & { + ref: ReturnType; + setEncoding: ReturnType; + unref: ReturnType; + }; + unref: ReturnType; +}; + +const tempDirs = new Set(); +const originalEnv = { + FS_SAFE_PYTHON: process.env.FS_SAFE_PYTHON, + FS_SAFE_PYTHON_MODE: process.env.FS_SAFE_PYTHON_MODE, + OPENCLAW_FS_SAFE_PYTHON: process.env.OPENCLAW_FS_SAFE_PYTHON, + OPENCLAW_FS_SAFE_PYTHON_MODE: process.env.OPENCLAW_FS_SAFE_PYTHON_MODE, + OPENCLAW_PINNED_PYTHON: process.env.OPENCLAW_PINNED_PYTHON, + OPENCLAW_PINNED_WRITE_PYTHON: process.env.OPENCLAW_PINNED_WRITE_PYTHON, +}; + +function restoreEnv(): void { + for (const [key, value] of Object.entries(originalEnv)) { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } + } +} + +async function tempRoot(prefix: string): Promise { + const dir = await fs.mkdtemp(path.join(os.tmpdir(), prefix)); + tempDirs.add(dir); + return dir; +} + +function makeChild( + write?: (line: string, callback?: (error?: Error | null) => void) => void, +): FakeChild { + const child = new EventEmitter() as FakeChild; + child.ref = vi.fn(); + child.unref = vi.fn(); + child.kill = vi.fn(); + child.stdout = Object.assign(new EventEmitter(), { + ref: vi.fn(), + setEncoding: vi.fn(), + unref: vi.fn(), + }); + child.stderr = Object.assign(new EventEmitter(), { + ref: vi.fn(), + setEncoding: vi.fn(), + unref: vi.fn(), + }); + child.stdin = Object.assign(new EventEmitter(), { + ref: vi.fn(), + unref: vi.fn(), + write: vi.fn((line: string, callback?: (error?: Error | null) => void) => { + write?.(line, callback); + callback?.(); + return true; + }), + }); + return child; +} + +function makeRespondingChild(): FakeChild { + const child = makeChild((line) => { + const request = JSON.parse(line) as { id: number }; + queueMicrotask(() => { + child.stdout.emit( + "data", + `${JSON.stringify({ id: request.id, ok: true, result: { ok: true } })}\n`, + ); + }); + }); + return child; +} + +function makeFailingChild(): FakeChild { + const child = makeChild(); + queueMicrotask(() => { + const error = Object.assign(new Error("spawn ENOENT"), { + code: "ENOENT", + syscall: "spawn python3", + }); + child.emit("error", error); + }); + return child; +} + +afterEach(async () => { + configureFsSafePython({ mode: "auto", pythonPath: undefined }); + __resetPinnedPythonWorkerForTest(); + restoreEnv(); + spawnMock.mockReset(); + for (const dir of tempDirs) { + await fs.rm(dir, { recursive: true, force: true }); + } + tempDirs.clear(); +}); + +describe("Python helper configuration", () => { + it("reads environment mode and python path aliases", () => { + delete process.env.FS_SAFE_PYTHON_MODE; + delete process.env.OPENCLAW_FS_SAFE_PYTHON_MODE; + delete process.env.FS_SAFE_PYTHON; + delete process.env.OPENCLAW_FS_SAFE_PYTHON; + delete process.env.OPENCLAW_PINNED_PYTHON; + delete process.env.OPENCLAW_PINNED_WRITE_PYTHON; + + expect(getFsSafePythonConfig()).toEqual({ mode: "auto", pythonPath: undefined }); + + process.env.FS_SAFE_PYTHON_MODE = "off"; + process.env.FS_SAFE_PYTHON = "/tmp/python-a"; + expect(getFsSafePythonConfig()).toEqual({ mode: "off", pythonPath: "/tmp/python-a" }); + + delete process.env.FS_SAFE_PYTHON_MODE; + delete process.env.FS_SAFE_PYTHON; + process.env.OPENCLAW_FS_SAFE_PYTHON_MODE = "required"; + process.env.OPENCLAW_PINNED_WRITE_PYTHON = "/tmp/python-b"; + expect(getFsSafePythonConfig()).toEqual({ + mode: "require", + pythonPath: "/tmp/python-b", + }); + + configureFsSafePython({ mode: "auto", pythonPath: "/tmp/python-c" }); + expect(getFsSafePythonConfig()).toEqual({ mode: "auto", pythonPath: "/tmp/python-c" }); + expect(configSubpath.getFsSafePythonConfig()).toEqual({ + mode: "auto", + pythonPath: "/tmp/python-c", + }); + }); + + it("only allows helper-unavailable fallback outside require mode", () => { + const error = Object.assign(new Error("missing"), { code: "helper-unavailable" }); + + configureFsSafePython({ mode: "auto" }); + expect(canFallbackFromPythonError(error)).toBe(true); + + configureFsSafePython({ mode: "off" }); + expect(canFallbackFromPythonError(error)).toBe(true); + + configureFsSafePython({ mode: "require" }); + expect(canFallbackFromPythonError(error)).toBe(false); + }); +}); + +describe("persistent Python helper worker", () => { + it("reuses one worker and unreferences it while idle", async () => { + const child = makeRespondingChild(); + spawnMock.mockReturnValue(child); + configureFsSafePython({ mode: "auto", pythonPath: "/tmp/fake-python" }); + + await expect( + runPinnedPythonOperation<{ ok: boolean }>({ + operation: "stat", + rootPath: "/tmp/root", + payload: { relativePath: "a.txt" }, + }), + ).resolves.toEqual({ ok: true }); + await expect( + runPinnedPythonOperation<{ ok: boolean }>({ + operation: "stat", + rootPath: "/tmp/root", + payload: { relativePath: "b.txt" }, + }), + ).resolves.toEqual({ ok: true }); + + expect(spawnMock).toHaveBeenCalledTimes(1); + expect(child.ref).toHaveBeenCalled(); + expect(child.unref).toHaveBeenCalled(); + expect(child.stdin.write).toHaveBeenCalledTimes(2); + }); + + it("falls back in auto mode but fails closed in require mode", async () => { + const rootDir = await tempRoot("fs-safe-python-policy-"); + await fs.writeFile(path.join(rootDir, "file.txt"), "ok"); + + spawnMock.mockImplementation(makeFailingChild); + configureFsSafePython({ mode: "auto", pythonPath: "/tmp/missing-python" }); + const autoRoot = await root(rootDir); + await expect(autoRoot.stat("file.txt")).resolves.toMatchObject({ + isFile: true, + }); + await expect(autoRoot.list("")).resolves.toEqual(["file.txt"]); + await fs.writeFile(path.join(rootDir, "move.txt"), "move"); + await autoRoot.move("move.txt", "moved.txt"); + await expect(fs.readFile(path.join(rootDir, "moved.txt"), "utf8")).resolves.toBe("move"); + + __resetPinnedPythonWorkerForTest(); + spawnMock.mockClear(); + spawnMock.mockImplementation(makeFailingChild); + configureFsSafePython({ mode: "require", pythonPath: "/tmp/missing-python" }); + await expect((await root(rootDir)).stat("file.txt")).rejects.toMatchObject({ + code: "helper-unavailable", + }); + }); +}); From e210a26af2c010250545eb0c562dccd8dab5699f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:07:28 +0100 Subject: [PATCH 08/20] feat: unify store helpers --- README.md | 5 ++- docs/file-store.md | 11 ++++- docs/private-file-store.md | 19 ++++---- docs/quickstart.md | 2 + docs/security-model.md | 2 + package.json | 2 +- scripts/build-docs-site.mjs | 84 +++++++++++++++++++++++++++++++++- scripts/docs-site-assets.mjs | 12 +++++ scripts/prepack-build.mjs | 16 +++++++ src/file-store.ts | 87 ++++++++++++++++++++++++++++-------- src/private-file-store.ts | 61 ++++++++++++++----------- test/api-coverage.test.ts | 13 +++++- test/new-primitives.test.ts | 3 ++ 13 files changed, 254 insertions(+), 63 deletions(-) create mode 100644 scripts/prepack-build.mjs diff --git a/README.md b/README.md index e7a0a46..56750a9 100644 --- a/README.md +++ b/README.md @@ -219,8 +219,8 @@ and policy knobs. Use `privateStateStore()` when the state is a directory of private text or JSON files rather than one known JSON file: credentials, auth profiles, tokens, and per-agent private state. It always writes files at `0o600` under directories at -`0o700`, returns `null` for missing reads, and intentionally keeps the method -set small. +`0o700`, returns `null` for missing text/JSON reads, and otherwise shares the +same managed-store shape as `fileStore`. Use `fileStore()` for cache/blob/media-style directories where callers need safe relative paths, size limits, atomic replacement, stream writes, and @@ -236,6 +236,7 @@ const media = fileStore({ }); await media.write("inbound/photo.jpg", bytes); +await media.writeJson("state/photo.json", { id: "photo" }); const opened = await media.open("inbound/photo.jpg"); await media.pruneExpired({ ttlMs: 10 * 60 * 1000, recursive: true }); ``` diff --git a/docs/file-store.md b/docs/file-store.md index f1861da..9acdd66 100644 --- a/docs/file-store.md +++ b/docs/file-store.md @@ -27,6 +27,7 @@ const cache = fileStore({ mode: 0o600, // file mode for writes (default 0o600) dirMode: 0o700, // mode for parent directories created on demand (default 0o700) maxBytes: 64 * 1024 * 1024, // optional: refuse writes/reads larger than this + private: true, // optional: document intent for private 0600/0700 stores }); ``` @@ -43,6 +44,10 @@ type FileStore = { open(rel, options?): Promise; read(rel, options?): Promise; readBytes(rel, options?): Promise; + readText(rel, options?): Promise; + readJson(rel, options?): Promise; + writeText(rel, data: string, options?): Promise; + writeJson(rel, data: unknown, options?): Promise; remove(rel): Promise; exists(rel): Promise; pruneExpired(options: FileStorePruneOptions): Promise; @@ -65,6 +70,10 @@ const path = await cache.write("entries/2026/05/05.json", JSON.stringify(entry)) Buffer or string. Returns the final absolute path. Throws `too-large` if `data.byteLength` exceeds `maxBytes`. +### `writeText(rel, data, options?)` / `writeJson(rel, data, options?)` + +Convenience wrappers over `write`. `writeJson` pretty-prints with a trailing newline by default and accepts `{ trailingNewline: false }` when the exact bytes matter. + ### `writeStream(rel, stream, options?)` ```ts @@ -97,7 +106,7 @@ type FileStoreWriteOptions = { ## Reads -`open`, `read`, `readBytes` delegate to a fresh `Root` with `hardlinks: "reject"` and the store's `maxBytes`. Same return shapes as `Root`. +`open`, `read`, `readBytes`, `readText`, and `readJson` delegate to a fresh `Root` with `hardlinks: "reject"` and the store's `maxBytes`. Same return shapes as `Root`. ## `remove(rel)` / `exists(rel)` diff --git a/docs/private-file-store.md b/docs/private-file-store.md index 03418c3..917b6e4 100644 --- a/docs/private-file-store.md +++ b/docs/private-file-store.md @@ -1,6 +1,6 @@ # Private state store -`privateStateStore({ rootDir })` returns a small handle for reading and writing **JSON or text state** inside a trusted root directory. Every write atomically creates the parent directory tree at mode `0o700` and the file at mode `0o600`. +`privateStateStore({ rootDir })` returns a private-mode `fileStore` handle for **JSON or text state** inside a trusted root directory. Every write atomically creates the parent directory tree at mode `0o700` and the file at mode `0o600`. ```ts import { privateStateStore } from "@openclaw/fs-safe/store"; @@ -15,9 +15,9 @@ const loaded = await store.readJson("state.json"); - You have a single trusted directory holding small JSON or text state. - You want every write to land at mode `0o600` in dirs at `0o700` without thinking about it. -- You don't need `move`, `remove`, `list`, `copyIn`, or streaming — only read/write. +- You want lenient missing-file reads for text/JSON state, while keeping `remove`, `exists`, `open`, `copyIn`, stream writes, and pruning available from the same handle. -For richer file-store needs (remove, exists, open, copy-in, pruning, streams), use [`fileStore`](file-store.md). For general root operations, use [`root()`](root.md). For one-off credential reads, use the [secret-file helpers](secret-file.md). +For non-private modes or cache/media-style stores, use [`fileStore`](file-store.md). For general root operations, use [`root()`](root.md). For one-off credential reads, use the [secret-file helpers](secret-file.md). ## API @@ -26,15 +26,12 @@ type PrivateStateStoreOptions = { rootDir: string; }; -type PrivateStateStore = { - rootDir: string; - path(relativePath: string): string; - +type PrivateStateStore = Omit & { readText(relativePath: string, options?: { maxBytes?: number }): Promise; readJson(relativePath: string, options?: { maxBytes?: number }): Promise; - writeText(relativePath: string, content: string | Uint8Array): Promise; - writeJson(relativePath: string, value: unknown, options?: { trailingNewline?: boolean }): Promise; + writeText(relativePath: string, content: string | Uint8Array): Promise; + writeJson(relativePath: string, value: unknown, options?: { trailingNewline?: boolean }): Promise; }; function privateStateStore(options: PrivateStateStoreOptions): PrivateStateStore; @@ -42,7 +39,7 @@ function privateStateStore(options: PrivateStateStoreOptions): PrivateStateStore `store.path(rel)` returns the absolute path the store would use, useful for logging or for handing to other libraries that take absolute paths. -`readText` and `readJson` return `null` when the file is missing — lenient by design. Callers that want strict failure on missing should check the result and throw. +`readText` and `readJson` return `null` when the file is missing — lenient by design. Other inherited store methods keep the stricter `fileStore` behavior. Callers that want strict failure on missing should check the result and throw. ## Advanced standalone helpers @@ -122,7 +119,7 @@ if (!config) throw new Error("config missing"); | Writes always set 0600 on the file and 0700 on the parents. | Writes use the umask unless you override. | | No streaming. | `open()` returns a `FileHandle`. | -If you find yourself asking "does the store have an X?" — reach for `fileStore()` or `root()`. +If you find yourself asking for a root-level operation the store does not expose, call `store.root()` or use `root()` directly. ## See also diff --git a/docs/quickstart.md b/docs/quickstart.md index b226ac6..157b655 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -2,6 +2,8 @@ Five minutes. By the end you will have a working `root()` and know how to read, write, atomically replace, and unpack an archive — without your code being able to escape the workspace. +If you have used Go's `os.Root` / `OpenInRoot` or Rust's [`cap-std`](https://github.com/bytecodealliance/cap-std), this is the same shape: a capability-style handle that carries the boundary across every operation. The first thing to internalize is that you stop reasoning about *paths* and start reasoning about *the handle*. + ## 1. Build a root ```ts diff --git a/docs/security-model.md b/docs/security-model.md index 559f7b2..ea2e4b9 100644 --- a/docs/security-model.md +++ b/docs/security-model.md @@ -2,6 +2,8 @@ `fs-safe` is a library-level guardrail: a capability-style root handle for Node.js code that handles untrusted relative paths. It assumes the calling process already has whatever filesystem permissions it needs and aims to stop trivial path tricks from broadening that authority. It is not a sandbox and does not replace operating-system isolation. +The same shape exists in other languages: Go's [`os.Root` / `OpenInRoot`](https://go.dev/blog/osroot) and Rust's [`cap-std`](https://github.com/bytecodealliance/cap-std) both expose a root handle whose operations refuse to escape it. `fs-safe` is the Node-side equivalent: a single `root()` capability that carries the boundary across every read, write, move, and remove, instead of leaving each call site to redo `path.resolve(...).startsWith(...)` and hope. + ## Threat model You hand a `root()` boundary to a piece of code that takes caller-controlled relative paths. The library defends against a caller that: diff --git a/package.json b/package.json index 9957901..2efc8e8 100644 --- a/package.json +++ b/package.json @@ -90,7 +90,7 @@ "scripts": { "benchmark": "node scripts/benchmark.mjs", "build": "tsc -p tsconfig.json", - "prepack": "pnpm build", + "prepack": "node scripts/prepack-build.mjs", "test": "vitest run", "test:coverage": "vitest run --coverage", "check": "pnpm build && pnpm test", diff --git a/scripts/build-docs-site.mjs b/scripts/build-docs-site.mjs index 99d3283..97c5d56 100644 --- a/scripts/build-docs-site.mjs +++ b/scripts/build-docs-site.mjs @@ -222,7 +222,7 @@ function markdownToHtml(markdown, currentRel) { closeList(); flushBlockquote(); if (fence) { - html.push(`
${escapeHtml(fence.lines.join("\n"))}
`); + html.push(`
${highlightCode(fence.lines.join("\n"), fence.lang)}
`); fence = null; } else { fence = { lang: fenceMatch[1] || "text", lines: [] }; @@ -541,6 +541,88 @@ function escapeHtml(value) { return String(value ?? "").replace(/[&<>"']/g, (char) => ({ "&": "&", "<": "<", ">": ">", '"': """, "'": "'" })[char]); } +function highlightLangAliases() { + return { + ts: "ts", + typescript: "ts", + js: "ts", + javascript: "ts", + tsx: "ts", + jsx: "ts", + bash: "bash", + sh: "bash", + shell: "bash", + zsh: "bash", + jsonc: "jsonc", + json: "jsonc", + }; +} + +function highlightRules() { + return { + ts: [ + { type: "comment", pattern: /\/\/[^\n]*|\/\*[\s\S]*?\*\// }, + { type: "string", pattern: /"(?:[^"\\\n]|\\[\s\S])*"|'(?:[^'\\\n]|\\[\s\S])*'|`(?:[^`\\$]|\\[\s\S]|\$(?!\{)|\$\{[^}]*\})*`/ }, + { type: "keyword", pattern: /\b(?:abstract|any|as|async|await|boolean|break|case|catch|class|const|continue|declare|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|infer|instanceof|interface|is|keyof|let|namespace|never|new|number|of|out|override|package|private|protected|public|readonly|return|satisfies|set|static|string|super|switch|symbol|this|throw|try|type|typeof|undefined|unique|unknown|var|void|while|yield)\b/ }, + { type: "boolean", pattern: /\b(?:true|false|null|undefined)\b/ }, + { type: "number", pattern: /\b(?:0[xX][\da-fA-F_]+|0[bB][01_]+|0[oO][0-7_]+|\d[\d_]*(?:\.\d+)?(?:[eE][+-]?\d+)?)n?\b/ }, + { type: "type", pattern: /\b[A-Z][A-Za-z0-9_]*\b/ }, + { type: "function", pattern: /\b[a-zA-Z_$][\w$]*(?=\s*\()/ }, + { type: null, pattern: /\b[a-zA-Z_$][\w$]*\b/ }, + { type: "operator", pattern: /=>|\.\.\.|[!=<>]==?|&&|\|\||\?\?|\+\+|--|\*\*|[+\-*/%&|^!~?]=?|=/ }, + { type: "punctuation", pattern: /[{}\[\](),.;:@]/ }, + ], + bash: [ + { type: "comment", pattern: /#[^\n]*/ }, + { type: "string", pattern: /"(?:[^"\\]|\\[\s\S])*"|'[^']*'/ }, + { type: "variable", pattern: /\$\{[^}]+\}|\$[A-Za-z_]\w*|\$[0-9*@#?!$-]/ }, + { type: "keyword", pattern: /\b(?:if|then|else|elif|fi|for|while|until|do|done|case|esac|in|function|return|exit|break|continue|export|local|readonly|set|unset|declare)\b/ }, + { type: "function", pattern: /(?<=^|[\s|;&(])(?:pnpm|npm|yarn|bun|node|git|cd|ls|mkdir|rm|cp|mv|cat|echo|grep|sed|awk|find|curl|wget|tar|zip|unzip)\b/ }, + { type: "number", pattern: /\b\d+\b/ }, + { type: null, pattern: /\b[a-zA-Z_][\w-]*\b/ }, + { type: "operator", pattern: /\|\||&&|>>|<<|[|&;<>!=]/ }, + { type: "punctuation", pattern: /[{}\[\](),.:]/ }, + ], + jsonc: [ + { type: "comment", pattern: /\/\/[^\n]*|\/\*[\s\S]*?\*\// }, + { type: "property", pattern: /"(?:[^"\\\n]|\\[\s\S])*"(?=\s*:)/ }, + { type: "string", pattern: /"(?:[^"\\\n]|\\[\s\S])*"/ }, + { type: "boolean", pattern: /\b(?:true|false|null)\b/ }, + { type: "number", pattern: /-?\b\d+(?:\.\d+)?(?:[eE][+-]?\d+)?\b/ }, + { type: "punctuation", pattern: /[{}\[\]:,]/ }, + ], + }; +} + +function highlightCode(code, langInput) { + const aliases = highlightLangAliases(); + const lang = aliases[langInput] || null; + const rules = lang ? highlightRules()[lang] : null; + if (!rules) return escapeHtml(code); + let out = ""; + let pos = 0; + while (pos < code.length) { + let matched = false; + for (const { type, pattern } of rules) { + const re = new RegExp(pattern.source, "y"); + re.lastIndex = pos; + const m = re.exec(code); + if (m && m.index === pos && m[0].length > 0) { + const text = escapeHtml(m[0]); + out += type ? `${text}` : text; + pos += m[0].length; + matched = true; + break; + } + } + if (!matched) { + out += escapeHtml(code[pos]); + pos += 1; + } + } + return out; +} + function escapeAttr(value) { return escapeHtml(value); } diff --git a/scripts/docs-site-assets.mjs b/scripts/docs-site-assets.mjs index b424b2c..ee1b3fc 100644 --- a/scripts/docs-site-assets.mjs +++ b/scripts/docs-site-assets.mjs @@ -89,6 +89,8 @@ main{min-width:0;padding:32px clamp(20px,4.5vw,56px) 80px;max-width:1180px;margi .home-cta .btn{display:inline-flex;align-items:center;gap:7px;border-radius:8px;padding:10px 16px;font-weight:600;font-size:.92rem;text-decoration:none;transition:background .15s,border-color .15s,color .15s,transform .12s} .home-cta .btn-primary{background:var(--accent);color:#fff;border:1px solid var(--accent)} .home-cta .btn-primary:hover{background:var(--accent-strong);border-color:var(--accent-strong);text-decoration:none} +:root[data-theme="dark"] .home-cta .btn-primary{color:#0a0e16} +:root[data-theme="dark"] .home-cta .btn-primary:hover{color:#04080f} .home-cta .btn-ghost{padding:10px 16px} .home-install{display:flex;align-items:center;gap:12px;background:var(--code-bg);color:var(--code-fg);border-radius:8px;padding:10px 10px 10px 16px;font:500 .9rem/1.2 "JetBrains Mono","SF Mono",ui-monospace,monospace;max-width:32em;border:1px solid #1f2937} .home-install .prompt{color:#64748b;user-select:none;flex:0 0 auto} @@ -123,6 +125,16 @@ body:not(.home) .doc>h1:first-child{display:none} .doc pre::-webkit-scrollbar{height:8px;width:8px} .doc pre::-webkit-scrollbar-thumb{background:#334155;border-radius:8px} .doc pre code{display:block;background:transparent;border:0;color:inherit;padding:0;font-size:1em;white-space:pre} +.doc pre code .token.comment{color:#7c8597;font-style:italic} +.doc pre code .token.string{color:#a8e0a3} +.doc pre code .token.number,.doc pre code .token.boolean{color:#f6c177} +.doc pre code .token.keyword{color:#e387cb} +.doc pre code .token.type{color:#7dd3fc} +.doc pre code .token.function{color:#82caff} +.doc pre code .token.property{color:#7dd3fc} +.doc pre code .token.variable{color:#fcd28a} +.doc pre code .token.operator{color:#e387cb} +.doc pre code .token.punctuation{color:#a0a8b6} .doc pre .copy{position:absolute;top:8px;right:8px;background:rgba(255,255,255,.06);color:var(--code-fg);border:1px solid rgba(255,255,255,.16);border-radius:6px;padding:3px 9px;font:500 .7rem/1 "Inter",sans-serif;cursor:pointer;opacity:0;transition:opacity .15s,background .15s,border-color .15s} .doc pre:hover .copy,.doc pre .copy:focus{opacity:1} .doc pre .copy:hover{background:rgba(255,255,255,.12)} diff --git a/scripts/prepack-build.mjs b/scripts/prepack-build.mjs new file mode 100644 index 0000000..19bb681 --- /dev/null +++ b/scripts/prepack-build.mjs @@ -0,0 +1,16 @@ +#!/usr/bin/env node +import { existsSync } from "node:fs"; +import { spawnSync } from "node:child_process"; + +function run(command, args) { + const result = spawnSync(command, args, { stdio: "inherit", shell: process.platform === "win32" }); + if (result.status !== 0) { + process.exit(result.status ?? 1); + } +} + +if (!existsSync("node_modules/typescript/bin/tsc")) { + run("pnpm", ["install", "--prod=false", "--ignore-scripts", "--frozen-lockfile=false"]); +} + +run("pnpm", ["exec", "tsc", "-p", "tsconfig.json"]); diff --git a/src/file-store.ts b/src/file-store.ts index 0eb6d89..277f5d2 100644 --- a/src/file-store.ts +++ b/src/file-store.ts @@ -9,6 +9,7 @@ import { resolveSafeRelativePath } from "./path.js"; export type FileStoreOptions = { rootDir: string; + private?: boolean; dirMode?: number; mode?: number; maxBytes?: number; @@ -50,8 +51,26 @@ export type FileStore = { open(relativePath: string, options?: RootReadOptions): Promise; read(relativePath: string, options?: RootReadOptions): Promise; readBytes(relativePath: string, options?: RootReadOptions): Promise; + readText( + relativePath: string, + options?: RootReadOptions & { encoding?: BufferEncoding }, + ): Promise; + readJson( + relativePath: string, + options?: RootReadOptions & { encoding?: BufferEncoding }, + ): Promise; remove(relativePath: string): Promise; exists(relativePath: string): Promise; + writeText( + relativePath: string, + data: string, + options?: FileStoreWriteOptions, + ): Promise; + writeJson( + relativePath: string, + data: unknown, + options?: FileStoreWriteOptions & { trailingNewline?: boolean }, + ): Promise; pruneExpired(options: FileStorePruneOptions): Promise; }; @@ -120,29 +139,35 @@ export function fileStore(options: FileStoreOptions): FileStore { return await root(rootDir, { hardlinks: "reject", maxBytes }); } + async function write( + relativePath: string, + data: string | Buffer, + writeOptions?: FileStoreWriteOptions, + ): Promise { + const destination = resolveStorePath(rootDir, relativePath); + const content = Buffer.isBuffer(data) ? data : Buffer.from(data); + assertMaxBytes(content.byteLength, writeOptions?.maxBytes ?? maxBytes); + await ensureParent(destination, writeOptions?.dirMode ?? dirMode); + const result = await writeSiblingTempFile({ + dir: path.dirname(destination), + dirMode: writeOptions?.dirMode ?? dirMode, + mode: writeOptions?.mode ?? mode, + tempPrefix: writeOptions?.tempPrefix ?? `.${path.basename(destination)}`, + writeTemp: async (tempPath) => { + await fs.writeFile(tempPath, content); + }, + resolveFinalPath: () => destination, + syncTempFile: true, + syncParentDir: true, + }); + return result.filePath; + } + return { rootDir, path: (relativePath) => resolveStorePath(rootDir, relativePath), root: openRoot, - write: async (relativePath, data, writeOptions) => { - const destination = resolveStorePath(rootDir, relativePath); - const content = Buffer.isBuffer(data) ? data : Buffer.from(data); - assertMaxBytes(content.byteLength, writeOptions?.maxBytes ?? maxBytes); - await ensureParent(destination, writeOptions?.dirMode ?? dirMode); - const result = await writeSiblingTempFile({ - dir: path.dirname(destination), - dirMode: writeOptions?.dirMode ?? dirMode, - mode: writeOptions?.mode ?? mode, - tempPrefix: writeOptions?.tempPrefix ?? `.${path.basename(destination)}`, - writeTemp: async (tempPath) => { - await fs.writeFile(tempPath, content); - }, - resolveFinalPath: () => destination, - syncTempFile: true, - syncParentDir: true, - }); - return result.filePath; - }, + write, writeStream: async (relativePath, stream, writeOptions) => { const destination = resolveStorePath(rootDir, relativePath); const limit = writeOptions?.maxBytes ?? maxBytes; @@ -192,10 +217,34 @@ export function fileStore(options: FileStoreOptions): FileStore { await (await openRoot()).read(assertRelativePath(relativePath), readOptions), readBytes: async (relativePath, readOptions) => await (await openRoot()).readBytes(assertRelativePath(relativePath), readOptions), + readText: async (relativePath, readOptions) => { + const { encoding = "utf8", ...options } = readOptions ?? {}; + return (await (await openRoot()).read(assertRelativePath(relativePath), options)).buffer + .toString(encoding); + }, + readJson: async ( + relativePath: string, + readOptions?: RootReadOptions & { encoding?: BufferEncoding }, + ) => { + const { encoding = "utf8", ...options } = readOptions ?? {}; + return JSON.parse( + (await (await openRoot()).read(assertRelativePath(relativePath), options)).buffer + .toString(encoding), + ) as T; + }, remove: async (relativePath) => { await (await openRoot()).remove(assertRelativePath(relativePath)); }, exists: async (relativePath) => await (await openRoot()).exists(assertRelativePath(relativePath)), + writeText: async (relativePath, data, writeOptions) => await write(relativePath, data, writeOptions), + writeJson: async (relativePath, data, writeOptions) => { + const json = JSON.stringify(data, null, 2); + return await write( + relativePath, + writeOptions?.trailingNewline === false ? json : `${json}\n`, + writeOptions, + ); + }, pruneExpired: async (pruneOptions) => { const now = Date.now(); const recursive = pruneOptions.recursive ?? false; diff --git a/src/private-file-store.ts b/src/private-file-store.ts index 53ba57f..54d7222 100644 --- a/src/private-file-store.ts +++ b/src/private-file-store.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto"; import path from "node:path"; import fs from "node:fs"; import { FsSafeError } from "./errors.js"; +import { fileStore, type FileStore } from "./file-store.js"; import { isPathInside } from "./path.js"; import { readRegularFileSync } from "./regular-file.js"; import { root } from "./root.js"; @@ -11,13 +12,11 @@ export type PrivateStateStoreOptions = { rootDir: string; }; -export type PrivateStateStore = { - rootDir: string; - path(relativePath: string): string; +export type PrivateStateStore = Omit & { readText(relativePath: string, options?: { maxBytes?: number }): Promise; readJson(relativePath: string, options?: { maxBytes?: number }): Promise; - writeText(relativePath: string, content: string | Uint8Array): Promise; - writeJson(relativePath: string, value: unknown, options?: { trailingNewline?: boolean }): Promise; + writeText(relativePath: string, content: string | Uint8Array): Promise; + writeJson(relativePath: string, value: unknown, options?: { trailingNewline?: boolean }): Promise; }; function resolvePrivateStorePath(rootDir: string, relativePath: string): string { @@ -237,36 +236,46 @@ export function writePrivateJsonAtomicSync(params: { } export function privateStateStore(options: PrivateStateStoreOptions): PrivateStateStore { - const root = path.resolve(options.rootDir); + const rootDir = path.resolve(options.rootDir); + const store = fileStore({ rootDir, private: true }); return { - rootDir: root, - path: (relativePath) => resolvePrivateStorePath(root, relativePath), - readText: async (relativePath, options) => - await readPrivateText({ - rootDir: root, - filePath: resolvePrivateStorePath(root, relativePath), + ...store, + rootDir, + path: (relativePath) => resolvePrivateStorePath(rootDir, relativePath), + readText: async (relativePath, options) => { + const safePath = resolvePrivateStorePath(rootDir, relativePath); + return await readPrivateText({ + rootDir, + filePath: safePath, maxBytes: options?.maxBytes, - }), - readJson: async (relativePath, options) => - await readPrivateJson({ - rootDir: root, - filePath: resolvePrivateStorePath(root, relativePath), - maxBytes: options?.maxBytes, - }), - writeText: async (relativePath, content) => { - await writePrivateTextAtomic({ - rootDir: root, - filePath: resolvePrivateStorePath(root, relativePath), - content, }); }, + readJson: async (relativePath: string, options?: { maxBytes?: number }) => { + const safePath = resolvePrivateStorePath(rootDir, relativePath); + return await readPrivateJson({ + rootDir, + filePath: safePath, + maxBytes: options?.maxBytes, + }); + }, + writeText: async (relativePath, content) => { + const safePath = resolvePrivateStorePath(rootDir, relativePath); + await writePrivateTextAtomic({ + rootDir, + filePath: safePath, + content, + }); + return safePath; + }, writeJson: async (relativePath, value, options) => { + const safePath = resolvePrivateStorePath(rootDir, relativePath); await writePrivateJsonAtomic({ - rootDir: root, - filePath: resolvePrivateStorePath(root, relativePath), + rootDir, + filePath: safePath, value, trailingNewline: options?.trailingNewline, }); + return safePath; }, }; } diff --git a/test/api-coverage.test.ts b/test/api-coverage.test.ts index 02f7f20..eb0c361 100644 --- a/test/api-coverage.test.ts +++ b/test/api-coverage.test.ts @@ -741,12 +741,17 @@ describe("file stores and private stores", () => { const sourceRoot = await tempRoot("fs-safe-store-source-"); const source = path.join(sourceRoot, "source.txt"); await fs.writeFile(source, "copy", "utf8"); - const store = fileStore({ rootDir: root, maxBytes: 16 }); + const store = fileStore({ rootDir: root, maxBytes: 64 }); expect(store.path("a/b.txt")).toBe(path.join(root, "a", "b.txt")); await expect(store.write("a/b.txt", "data")).resolves.toBe(path.join(root, "a", "b.txt")); await expect(store.readBytes("a/b.txt")).resolves.toEqual(Buffer.from("data")); - await expect(store.write("too-large.txt", Buffer.alloc(17))).rejects.toMatchObject({ + await expect(store.readText("a/b.txt")).resolves.toBe("data"); + await expect(store.writeJson("state.json", { ok: true })).resolves.toBe( + path.join(root, "state.json"), + ); + await expect(store.readJson("state.json")).resolves.toEqual({ ok: true }); + await expect(store.write("too-large.txt", Buffer.alloc(65))).rejects.toMatchObject({ code: "too-large", }); await store.writeStream("stream.txt", Readable.from(["hello"])); @@ -779,9 +784,13 @@ describe("file stores and private stores", () => { await expect(store.readText("nested/value.txt")).resolves.toBe("secret"); await store.writeJson("nested/value.json", { ok: true }, { trailingNewline: true }); await expect(store.readJson("nested/value.json")).resolves.toEqual({ ok: true }); + await expect(store.exists("nested/value.json")).resolves.toBe(true); + await expect(store.readBytes("nested/value.txt")).resolves.toEqual(Buffer.from("secret")); expect(store.path("nested/value.txt")).toBe(path.join(root, "nested", "value.txt")); expect(() => store.path("../escape.txt")).toThrow("stay under"); await expect(store.readText("missing.txt")).resolves.toBeNull(); + await store.remove("nested/value.json"); + await expect(store.exists("nested/value.json")).resolves.toBe(false); const syncText = path.join(root, "sync", "value.txt"); writePrivateTextAtomicSync({ rootDir: root, filePath: syncText, content: "sync" }); diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index ee605e9..b1097ff 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -127,6 +127,9 @@ describe("file store", () => { const store = fileStore({ rootDir: root, maxBytes: 1024 }); await store.write("media/a.txt", "hello"); await expect(store.readBytes("media/a.txt")).resolves.toEqual(Buffer.from("hello")); + await expect(store.readText("media/a.txt")).resolves.toBe("hello"); + await store.writeJson("media/state.json", { ok: true }); + await expect(store.readJson("media/state.json")).resolves.toEqual({ ok: true }); const source = path.join(root, "source.bin"); await fs.writeFile(source, "source", "utf8"); From b2e6db91c55135d89abc7b8359a05c6dd4613cf6 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:08:28 +0100 Subject: [PATCH 09/20] fix: make git source prepack self-contained --- scripts/prepack-build.mjs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/prepack-build.mjs b/scripts/prepack-build.mjs index 19bb681..5c7da80 100644 --- a/scripts/prepack-build.mjs +++ b/scripts/prepack-build.mjs @@ -2,6 +2,8 @@ import { existsSync } from "node:fs"; import { spawnSync } from "node:child_process"; +const tscBin = process.platform === "win32" ? "node_modules/.bin/tsc.cmd" : "node_modules/.bin/tsc"; + function run(command, args) { const result = spawnSync(command, args, { stdio: "inherit", shell: process.platform === "win32" }); if (result.status !== 0) { @@ -9,8 +11,15 @@ function run(command, args) { } } -if (!existsSync("node_modules/typescript/bin/tsc")) { - run("pnpm", ["install", "--prod=false", "--ignore-scripts", "--frozen-lockfile=false"]); +if (!existsSync(tscBin)) { + run("pnpm", [ + "add", + "--save-dev", + "typescript@^5.8.3", + "@types/node@^22.15.19", + "--ignore-scripts", + "--lockfile=false", + ]); } -run("pnpm", ["exec", "tsc", "-p", "tsconfig.json"]); +run(tscBin, ["-p", "tsconfig.json"]); From 2f4c4262f05581cd54311fcf91949ad293901e9e Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:09:25 +0100 Subject: [PATCH 10/20] fix: clear lockfile-only during source prepack --- scripts/prepack-build.mjs | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/scripts/prepack-build.mjs b/scripts/prepack-build.mjs index 5c7da80..1bda229 100644 --- a/scripts/prepack-build.mjs +++ b/scripts/prepack-build.mjs @@ -2,24 +2,36 @@ import { existsSync } from "node:fs"; import { spawnSync } from "node:child_process"; -const tscBin = process.platform === "win32" ? "node_modules/.bin/tsc.cmd" : "node_modules/.bin/tsc"; +const tscBin = + process.platform === "win32" ? "node_modules/.bin/tsc.cmd" : "node_modules/.bin/tsc"; -function run(command, args) { - const result = spawnSync(command, args, { stdio: "inherit", shell: process.platform === "win32" }); +function run(command, args, env = {}) { + const result = spawnSync(command, args, { + stdio: "inherit", + shell: process.platform === "win32", + env: { ...process.env, ...env }, + }); if (result.status !== 0) { process.exit(result.status ?? 1); } } if (!existsSync(tscBin)) { - run("pnpm", [ - "add", - "--save-dev", - "typescript@^5.8.3", - "@types/node@^22.15.19", - "--ignore-scripts", - "--lockfile=false", - ]); + run( + "pnpm", + [ + "add", + "--save-dev", + "typescript@^5.8.3", + "@types/node@^22.15.19", + "--ignore-scripts", + "--lockfile=false", + ], + { + npm_config_lockfile_only: "false", + PNPM_CONFIG_LOCKFILE_ONLY: "false", + }, + ); } run(tscBin, ["-p", "tsconfig.json"]); From be7de52292a68a1ad3881c33bd7d6de2b9d552c8 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:21:03 +0100 Subject: [PATCH 11/20] docs: clarify store helper roles --- docs/file-store.md | 9 ++++++--- docs/private-file-store.md | 22 ++++++++++++---------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/docs/file-store.md b/docs/file-store.md index 9acdd66..bf2ff64 100644 --- a/docs/file-store.md +++ b/docs/file-store.md @@ -27,10 +27,12 @@ const cache = fileStore({ mode: 0o600, // file mode for writes (default 0o600) dirMode: 0o700, // mode for parent directories created on demand (default 0o700) maxBytes: 64 * 1024 * 1024, // optional: refuse writes/reads larger than this - private: true, // optional: document intent for private 0600/0700 stores + private: true, // optional intent marker for private 0600/0700 stores; defaults already match }); ``` +The `private` flag is currently a self-documenting marker — `mode` and `dirMode` already default to `0o600` / `0o700`, so a plain `fileStore({ rootDir })` is private by default. Pass `private: true` when callers want the intent visible at the call site. + Returns a `FileStore`: ```ts @@ -163,11 +165,12 @@ Returns the final absolute path. Throws `not-file` if the source is a symlink or |---|---| | Object-style with mode+dirMode baked in. | Method-style boundary; mode is per-call or per-default. | | `writeStream` with built-in byte budget. | Manual via `openWritable()`. | -| `copyIn` returns the final path. | `copyIn` returns void. | +| `writeText` / `writeJson` return the final absolute path. | `Root.write` / `writeJson` return void. | +| `copyIn` returns the final absolute path. | `Root.copyIn` returns void. | | `pruneExpired` walks by `mtime`. | No prune helper. | | Reads delegate via `Root` internally. | The boundary itself. | -If you need richer ops (move, list, append, JSON), call `store.root()` to get a `Root` and use that. +If you need richer ops (move, list, append, mkdir), call `store.root()` to get a `Root` and use that. ## Common patterns diff --git a/docs/private-file-store.md b/docs/private-file-store.md index 917b6e4..c47a3cf 100644 --- a/docs/private-file-store.md +++ b/docs/private-file-store.md @@ -13,9 +13,9 @@ const loaded = await store.readJson("state.json"); ## When to reach for it -- You have a single trusted directory holding small JSON or text state. -- You want every write to land at mode `0o600` in dirs at `0o700` without thinking about it. -- You want lenient missing-file reads for text/JSON state, while keeping `remove`, `exists`, `open`, `copyIn`, stream writes, and pruning available from the same handle. +- You want a `fileStore` whose JSON and text writes go through the secret-file atomic path (parent dirs created at `0o700`, file mode re-asserted to `0o600` after rename, symlink/hardlink refusal on the parent chain). +- You want `readText` / `readJson` to return `null` for missing files instead of throwing `not-found` — convenient for state that may not exist yet. +- You still need the rest of the `fileStore` shape (`writeStream`, `copyIn`, `remove`, `exists`, `open`, `read`, `pruneExpired`). For non-private modes or cache/media-style stores, use [`fileStore`](file-store.md). For general root operations, use [`root()`](root.md). For one-off credential reads, use the [secret-file helpers](secret-file.md). @@ -110,14 +110,16 @@ if (!config) throw new Error("config missing"); - **Symlinks.** Refused everywhere along the resolved path. - **Sync writes.** The standalone `*Sync` writers are appropriate for boot paths or test fixtures. They use the same atomic-rename mechanism as the async variant. -## Difference from `Root` +## Difference from `fileStore` and `Root` -| `privateStateStore` | `Root` | -|---|---| -| Reads return `null` on miss. | Reads throw with code `not-found`. | -| Four verbs (text in/out, JSON in/out). | Full surface (move, remove, list, …). | -| Writes always set 0600 on the file and 0700 on the parents. | Writes use the umask unless you override. | -| No streaming. | `open()` returns a `FileHandle`. | +`privateStateStore` extends [`fileStore`](file-store.md): you keep `read`, `readBytes`, `open`, `writeStream`, `copyIn`, `remove`, `exists`, `pruneExpired`, and `path()`. The four JSON/text methods (`readText`, `readJson`, `writeText`, `writeJson`) are overridden to be lenient on missing reads and to route writes through the secret-file atomic helpers so modes are re-asserted after rename. + +| `privateStateStore` | `fileStore` | `Root` | +|---|---|---| +| `readText` / `readJson` return `null` on miss. | Throw `not-found` like `Root.read*`. | Throws with code `not-found`. | +| `writeText` / `writeJson` go through the secret-file atomic path (mode re-asserted post-rename). | Sibling-temp + rename with `mode` (default `0o600`). | Pinned-write helper plus identity verification. | +| File mode `0o600`, dir mode `0o700` are baked in; per-call overrides not exposed for the JSON/text path. | `mode` / `dirMode` configurable per store and per call. | `mode` / `dirMode` configurable per call or via defaults. | +| All other `FileStore` methods (`writeStream`, `copyIn`, `pruneExpired`, …) work unchanged. | Same. | Method-style boundary; the store delegates to a `Root` for these. | If you find yourself asking for a root-level operation the store does not expose, call `store.root()` or use `root()` directly. From 32acf97225e76906cc7e2c4c83c1edef198e4bf2 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:37:14 +0100 Subject: [PATCH 12/20] refactor: unify private file store --- README.md | 13 +- docs/file-store.md | 11 +- docs/index.md | 4 +- docs/install.md | 2 +- docs/json.md | 2 +- docs/private-file-store.md | 138 +++------------- docs/secret-file.md | 2 +- src/advanced.ts | 15 +- src/file-store.ts | 307 +++++++++++++++++++++++++++++++++--- src/private-file-store.ts | 281 --------------------------------- src/store.ts | 8 +- test/api-coverage.test.ts | 41 ++--- test/new-primitives.test.ts | 56 ++----- 13 files changed, 363 insertions(+), 517 deletions(-) delete mode 100644 src/private-file-store.ts diff --git a/README.md b/README.md index 56750a9..fe7c285 100644 --- a/README.md +++ b/README.md @@ -153,7 +153,7 @@ that OpenClaw needs to compose higher-level APIs are grouped under | `@openclaw/fs-safe/config` | process-global Python helper configuration | | `@openclaw/fs-safe/path` | canonical path checks: `isPathInside`, `safeRealpathSync`, `isNotFoundPathError`, `isSymlinkOpenError` | | `@openclaw/fs-safe/json` | `tryReadJson`, `readJson`, `readJsonIfExists`, `writeJson`, sync variants | -| `@openclaw/fs-safe/store` | `fileStore`, `jsonStore`, and `privateStateStore` | +| `@openclaw/fs-safe/store` | `fileStore`, `fileStoreSync`, and `jsonStore` | | `@openclaw/fs-safe/secret` | strict and try-style secret file read/write helpers | | `@openclaw/fs-safe/atomic` | `replaceFileAtomic`, `replaceFileAtomicSync`, `replaceDirectoryAtomic`, `movePathWithCopyFallback` | | `@openclaw/fs-safe/temp` | `tempWorkspace`, `tempWorkspaceSync`, `withTempWorkspace`, `resolveSecureTempRoot` | @@ -216,15 +216,11 @@ the common merge-into-defaults case. Standalone helpers use options bags because they do not carry a bound root and often need multiple authority, path, and policy knobs. -Use `privateStateStore()` when the state is a directory of private text or JSON -files rather than one known JSON file: credentials, auth profiles, tokens, and -per-agent private state. It always writes files at `0o600` under directories at -`0o700`, returns `null` for missing text/JSON reads, and otherwise shares the -same managed-store shape as `fileStore`. - Use `fileStore()` for cache/blob/media-style directories where callers need safe relative paths, size limits, atomic replacement, stream writes, and -TTL cleanup behind one root: +TTL cleanup behind one root. Pass `private: true` for credentials, auth +profiles, tokens, and per-agent private state; private mode keeps the same +store shape while routing writes through the secret-file atomic path. ```ts import { fileStore } from "@openclaw/fs-safe/store"; @@ -237,6 +233,7 @@ const media = fileStore({ await media.write("inbound/photo.jpg", bytes); await media.writeJson("state/photo.json", { id: "photo" }); +const cached = await media.readJsonIfExists("state/photo.json"); const opened = await media.open("inbound/photo.jpg"); await media.pruneExpired({ ttlMs: 10 * 60 * 1000, recursive: true }); ``` diff --git a/docs/file-store.md b/docs/file-store.md index bf2ff64..a524cce 100644 --- a/docs/file-store.md +++ b/docs/file-store.md @@ -27,11 +27,14 @@ const cache = fileStore({ mode: 0o600, // file mode for writes (default 0o600) dirMode: 0o700, // mode for parent directories created on demand (default 0o700) maxBytes: 64 * 1024 * 1024, // optional: refuse writes/reads larger than this - private: true, // optional intent marker for private 0600/0700 stores; defaults already match + private: true, // use secret-file atomic writes for private state }); ``` -The `private` flag is currently a self-documenting marker — `mode` and `dirMode` already default to `0o600` / `0o700`, so a plain `fileStore({ rootDir })` is private by default. Pass `private: true` when callers want the intent visible at the call site. +Use `private: true` for credentials, auth profiles, tokens, and other private +state. Private mode keeps the same `FileStore` shape but routes writes through +the secret-file atomic path, refusing symlink parent components and re-asserting +mode after rename. Returns a `FileStore`: @@ -47,8 +50,10 @@ type FileStore = { read(rel, options?): Promise; readBytes(rel, options?): Promise; readText(rel, options?): Promise; + readTextIfExists(rel, options?): Promise; readJson(rel, options?): Promise; - writeText(rel, data: string, options?): Promise; + readJsonIfExists(rel, options?): Promise; + writeText(rel, data: string | Uint8Array, options?): Promise; writeJson(rel, data: unknown, options?): Promise; remove(rel): Promise; exists(rel): Promise; diff --git a/docs/index.md b/docs/index.md index 31c1f54..955b8e3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -53,8 +53,8 @@ await fs.remove("notes/archive/today.txt"); | [`replaceFileAtomic`](atomic.md) | Sibling-temp + rename, fsync hooks, mode preservation, copy fallback. | | [`writeJson` / `readJson*`](json.md) | JSON state files with strict and lenient read variants. | | [`jsonStore`](json-store.md) | Single JSON state file with explicit fallback, atomic writes, and optional locking. | -| [`fileStore`](file-store.md) | Managed multi-file/blob store with modes, stream writes, copy-in, and pruning. | -| [`privateStateStore`](private-file-store.md) | Multi-file private text/JSON state at 0600 under 0700 dirs. | +| [`fileStore`](file-store.md) | Managed multi-file/blob store with modes, stream writes, copy-in, pruning, and private mode. | +| [Private file-store mode](private-file-store.md) | `fileStore({ private: true })` for private JSON/text state at 0600 under 0700 dirs. | | [`tempWorkspace`](temp.md) | 0700 scratch dir with auto-cleanup. | | [`readSecureFile`](secure-file.md) | Absolute file reads with fd pinning, permissions, owner, size, and timeout checks. | | [`walkDirectory`](walk.md) | Budget-bounded recursive directory scan with symlink policy and filters. | diff --git a/docs/install.md b/docs/install.md index 0924efb..91e7074 100644 --- a/docs/install.md +++ b/docs/install.md @@ -67,7 +67,7 @@ Use the main entry for the common surface, or the focused subpaths when you want | `@openclaw/fs-safe/config` | Process-global Python helper configuration. | | `@openclaw/fs-safe/path` | `isPathInside`, `safeRealpathSync`, `isWithinDir`, error helpers. | | `@openclaw/fs-safe/json` | `tryReadJson`, `readJson`, `readJsonIfExists`, `writeJson`, sync variants. | -| `@openclaw/fs-safe/store` | `fileStore()`, `jsonStore()`, and `privateStateStore()`. | +| `@openclaw/fs-safe/store` | `fileStore()`, `fileStoreSync()`, and `jsonStore()`. | | `@openclaw/fs-safe/secret` | Secret file read/write helpers. | | `@openclaw/fs-safe/atomic` | `replaceFileAtomic`, `writeTextAtomic`, `replaceDirectoryAtomic`, `movePathWithCopyFallback`. | | `@openclaw/fs-safe/temp` | `tempWorkspace`, `withTempWorkspace`, sync variants, `resolveSecureTempRoot`. | diff --git a/docs/json.md b/docs/json.md index 5c271fb..77c2b7e 100644 --- a/docs/json.md +++ b/docs/json.md @@ -154,5 +154,5 @@ const state = await readJsonIfExists("./state.json"); - [JSON store](json-store.md) — a single-file state wrapper with explicit per-call fallback (`readOr` / `updateOr`) and optional sidecar locking. - [Atomic writes](atomic.md) — lower-level sibling-temp replacement helpers. - [Secret files](secret-file.md) — JSON-or-text writes with mode 0600 in mode 0700 dirs. -- [Private state store](private-file-store.md) — root-bounded JSON+text helpers. +- [Private file-store mode](private-file-store.md) — root-bounded JSON+text state stores. - [Sidecar lock](sidecar-lock.md) — cross-process coordination. diff --git a/docs/private-file-store.md b/docs/private-file-store.md index c47a3cf..d9b74b3 100644 --- a/docs/private-file-store.md +++ b/docs/private-file-store.md @@ -1,131 +1,45 @@ -# Private state store +# Private file-store mode -`privateStateStore({ rootDir })` returns a private-mode `fileStore` handle for **JSON or text state** inside a trusted root directory. Every write atomically creates the parent directory tree at mode `0o700` and the file at mode `0o600`. +Private state is not a separate store family. Use `fileStore({ private: true })` +when a directory holds credentials, tokens, auth profiles, or other private +JSON/text state. ```ts -import { privateStateStore } from "@openclaw/fs-safe/store"; +import { fileStore } from "@openclaw/fs-safe/store"; -const store = privateStateStore({ rootDir: "/var/lib/app" }); +const store = fileStore({ rootDir: "/var/lib/app", private: true }); await store.writeJson("state.json", state); -const loaded = await store.readJson("state.json"); +const loaded = await store.readJsonIfExists("state.json"); ``` -## When to reach for it +## Behavior -- You want a `fileStore` whose JSON and text writes go through the secret-file atomic path (parent dirs created at `0o700`, file mode re-asserted to `0o600` after rename, symlink/hardlink refusal on the parent chain). -- You want `readText` / `readJson` to return `null` for missing files instead of throwing `not-found` — convenient for state that may not exist yet. -- You still need the rest of the `fileStore` shape (`writeStream`, `copyIn`, `remove`, `exists`, `open`, `read`, `pruneExpired`). +- Writes create parent directories at `0o700` and files at `0o600` unless you + pass stricter `dirMode` / `mode` options. +- Private-mode writes route through the secret-file atomic path, which refuses + symlink parent components and re-asserts mode after rename. +- `readText()` and `readJson()` are strict and throw on missing files. +- `readTextIfExists()` and `readJsonIfExists()` return `null` on missing files. +- `write()`, `writeText()`, `writeJson()`, `writeStream()`, and `copyIn()` all + keep the same root-relative `FileStore` shape. -For non-private modes or cache/media-style stores, use [`fileStore`](file-store.md). For general root operations, use [`root()`](root.md). For one-off credential reads, use the [secret-file helpers](secret-file.md). +## Sync writes -## API +Use `fileStoreSync({ private: true })` for boot paths or sync-only integration +points: ```ts -type PrivateStateStoreOptions = { - rootDir: string; -}; +import { fileStoreSync } from "@openclaw/fs-safe/store"; -type PrivateStateStore = Omit & { - readText(relativePath: string, options?: { maxBytes?: number }): Promise; - readJson(relativePath: string, options?: { maxBytes?: number }): Promise; - - writeText(relativePath: string, content: string | Uint8Array): Promise; - writeJson(relativePath: string, value: unknown, options?: { trailingNewline?: boolean }): Promise; -}; - -function privateStateStore(options: PrivateStateStoreOptions): PrivateStateStore; +fileStoreSync({ rootDir: "/var/lib/app", private: true }).writeJson("config.json", config); ``` -`store.path(rel)` returns the absolute path the store would use, useful for logging or for handing to other libraries that take absolute paths. - -`readText` and `readJson` return `null` when the file is missing — lenient by design. Other inherited store methods keep the stricter `fileStore` behavior. Callers that want strict failure on missing should check the result and throw. - -## Advanced standalone helpers - -The standalone function form lives in `@openclaw/fs-safe/advanced`. Use it when you don't want to pin a single root: - -```ts -import { - writePrivateTextAtomic, // async - writePrivateTextAtomicSync, // sync - writePrivateJsonAtomic, // async - writePrivateJsonAtomicSync, // sync - readPrivateText, // async - readPrivateTextSync, // sync - readPrivateJson, // async - readPrivateJsonSync, // sync -} from "@openclaw/fs-safe/advanced"; -``` - -Each standalone takes `{ rootDir, filePath, ... }` directly: - -```ts -await writePrivateJsonAtomic({ - rootDir: "/var/lib/app", - filePath: "/var/lib/app/state.json", - value: state, - trailingNewline: true, -}); -``` - -`filePath` is an absolute path. The helper asserts it stays inside `rootDir` and refuses anything that would escape. - -## Examples - -### Read-modify-write - -```ts -const store = privateStateStore({ rootDir: "/var/lib/app" }); - -const state = (await store.readJson("state.json")) ?? initialState(); -state.count += 1; -await store.writeJson("state.json", state, { trailingNewline: true }); -``` - -### Sync at boot - -```ts -import { readPrivateJsonSync } from "@openclaw/fs-safe/advanced"; - -const config = - readPrivateJsonSync({ rootDir: "/etc/app", filePath: "/etc/app/config.json" }) ?? - defaultConfig(); -applyConfig(config); -``` - -### Bounded reads - -```ts -const config = await store.readJson("config.json", { maxBytes: 64 * 1024 }); -if (!config) throw new Error("config missing"); -``` - -`maxBytes` is forwarded into the read; oversized files throw `too-large` from the underlying [`Root`](root.md). - -## Behavior notes - -- **Mode bits.** Writes always end at file mode `0o600` and create parent directories at `0o700`. The store does not narrow modes on existing wider parents — it sets the mode at creation only. Audit existing trees yourself. -- **Hardlinks.** Reads refuse files with `nlink > 1` (defense-in-depth, since the file might alias an out-of-tree inode). -- **Symlinks.** Refused everywhere along the resolved path. -- **Sync writes.** The standalone `*Sync` writers are appropriate for boot paths or test fixtures. They use the same atomic-rename mechanism as the async variant. - -## Difference from `fileStore` and `Root` - -`privateStateStore` extends [`fileStore`](file-store.md): you keep `read`, `readBytes`, `open`, `writeStream`, `copyIn`, `remove`, `exists`, `pruneExpired`, and `path()`. The four JSON/text methods (`readText`, `readJson`, `writeText`, `writeJson`) are overridden to be lenient on missing reads and to route writes through the secret-file atomic helpers so modes are re-asserted after rename. - -| `privateStateStore` | `fileStore` | `Root` | -|---|---|---| -| `readText` / `readJson` return `null` on miss. | Throw `not-found` like `Root.read*`. | Throws with code `not-found`. | -| `writeText` / `writeJson` go through the secret-file atomic path (mode re-asserted post-rename). | Sibling-temp + rename with `mode` (default `0o600`). | Pinned-write helper plus identity verification. | -| File mode `0o600`, dir mode `0o700` are baked in; per-call overrides not exposed for the JSON/text path. | `mode` / `dirMode` configurable per store and per call. | `mode` / `dirMode` configurable per call or via defaults. | -| All other `FileStore` methods (`writeStream`, `copyIn`, `pruneExpired`, …) work unchanged. | Same. | Method-style boundary; the store delegates to a `Root` for these. | - -If you find yourself asking for a root-level operation the store does not expose, call `store.root()` or use `root()` directly. +The sync store intentionally exposes a smaller surface: path resolution, +lenient reads, and atomic text/JSON writes. ## See also -- [`root()`](root.md) — full method-style boundary. -- [Secret files](secret-file.md) — standalone read/write of mode-0600 credential files. -- [JSON files](json.md) — strict/lenient JSON helpers without per-store fanout. -- [Atomic writes](atomic.md) — what these writes use under the hood. +- [`fileStore`](file-store.md) — full store API. +- [Secret files](secret-file.md) — standalone credential file reads and writes. +- [JSON files](json.md) — strict/lenient JSON helpers without a bound store. diff --git a/docs/secret-file.md b/docs/secret-file.md index 6cc0867..b66120a 100644 --- a/docs/secret-file.md +++ b/docs/secret-file.md @@ -151,4 +151,4 @@ await withTimeout( - [JSON files](json.md) — `writeJson` accepts `mode: 0o600` for non-secret JSON state. - [Atomic writes](atomic.md) — the lower-level `replaceFileAtomic` used by these helpers. -- [Private state store](private-file-store.md) — root-bounded JSON+text helpers without secret-file mode policy. +- [Private file-store mode](private-file-store.md) — root-bounded JSON+text stores using secret-file write policy. diff --git a/src/advanced.ts b/src/advanced.ts index 83b6095..aa1eab3 100644 --- a/src/advanced.ts +++ b/src/advanced.ts @@ -15,7 +15,7 @@ export { export { sameFileIdentity, type FileIdentityStat } from "./file-identity.js"; export { sanitizeUntrustedFileName } from "./filename.js"; export { pathExists, pathExistsSync } from "./fs.js"; -export { copyIntoRoot } from "./file-store.js"; +export { copyIntoRoot, fileStoreSync, type FileStoreSync } from "./file-store.js"; export { resolveLocalPathFromRootsSync, readLocalFileFromRoots, @@ -131,16 +131,3 @@ export { type WindowsAclEntry, type WindowsAclSummary, } from "./permissions.js"; -export { - privateStateStore, - readPrivateJson, - readPrivateJsonSync, - readPrivateText, - readPrivateTextSync, - writePrivateJsonAtomic, - writePrivateJsonAtomicSync, - writePrivateTextAtomic, - writePrivateTextAtomicSync, - type PrivateStateStore, - type PrivateStateStoreOptions, -} from "./private-file-store.js"; diff --git a/src/file-store.ts b/src/file-store.ts index 277f5d2..75d7ab8 100644 --- a/src/file-store.ts +++ b/src/file-store.ts @@ -1,11 +1,14 @@ +import { randomUUID } from "node:crypto"; +import syncFs from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; import type { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { FsSafeError } from "./errors.js"; +import { isPathInside, resolveSafeRelativePath } from "./path.js"; import { root, type OpenResult, type ReadResult, type Root, type RootReadOptions } from "./root.js"; +import { writeSecretFileAtomic } from "./secret-file.js"; import { writeSiblingTempFile } from "./sibling-temp.js"; -import { resolveSafeRelativePath } from "./path.js"; export type FileStoreOptions = { rootDir: string; @@ -22,6 +25,8 @@ export type FileStoreWriteOptions = { tempPrefix?: string; }; +export type FileStoreReadOptions = RootReadOptions & { encoding?: BufferEncoding }; + export type FileStorePruneOptions = { ttlMs: number; recursive?: boolean; @@ -35,7 +40,7 @@ export type FileStore = { root(): Promise; write( relativePath: string, - data: string | Buffer, + data: string | Uint8Array, options?: FileStoreWriteOptions, ): Promise; writeStream( @@ -53,17 +58,19 @@ export type FileStore = { readBytes(relativePath: string, options?: RootReadOptions): Promise; readText( relativePath: string, - options?: RootReadOptions & { encoding?: BufferEncoding }, + options?: FileStoreReadOptions, ): Promise; - readJson( + readTextIfExists(relativePath: string, options?: FileStoreReadOptions): Promise; + readJson(relativePath: string, options?: FileStoreReadOptions): Promise; + readJsonIfExists( relativePath: string, - options?: RootReadOptions & { encoding?: BufferEncoding }, - ): Promise; + options?: FileStoreReadOptions, + ): Promise; remove(relativePath: string): Promise; exists(relativePath: string): Promise; writeText( relativePath: string, - data: string, + data: string | Uint8Array, options?: FileStoreWriteOptions, ): Promise; writeJson( @@ -74,6 +81,20 @@ export type FileStore = { pruneExpired(options: FileStorePruneOptions): Promise; }; +export type FileStoreSync = { + readonly rootDir: string; + path(relativePath: string): string; + readTextIfExists(relativePath: string, options?: { maxBytes?: number }): string | null; + readJsonIfExists(relativePath: string, options?: { maxBytes?: number }): T | null; + write(relativePath: string, data: string | Uint8Array, options?: FileStoreWriteOptions): string; + writeText(relativePath: string, data: string | Uint8Array, options?: FileStoreWriteOptions): string; + writeJson( + relativePath: string, + data: unknown, + options?: FileStoreWriteOptions & { trailingNewline?: boolean }, + ): string; +}; + function assertRelativePath(relativePath: string): string { const raw = relativePath.trim(); if (!raw) { @@ -86,6 +107,12 @@ function resolveStorePath(rootDir: string, relativePath: string): string { return resolveSafeRelativePath(rootDir, assertRelativePath(relativePath)); } +function assertStoreFilePath(rootDir: string, filePath: string): void { + if (!isPathInside(rootDir, filePath)) { + throw new FsSafeError("outside-workspace", "file path escapes store root"); + } +} + async function ensureParent(filePath: string, mode: number): Promise { const dir = path.dirname(filePath); await fs.mkdir(dir, { recursive: true, mode }); @@ -98,6 +125,13 @@ function assertMaxBytes(size: number, maxBytes?: number): void { } } +function isNotFound(error: unknown): boolean { + return error instanceof FsSafeError + ? error.code === "not-found" + : (error as NodeJS.ErrnoException).code === "ENOENT" || + (error as NodeJS.ErrnoException).code === "ENOTDIR"; +} + export async function copyIntoRoot(params: { rootDir: string; relativePath: string; @@ -131,6 +165,7 @@ export async function copyIntoRoot(params: { export function fileStore(options: FileStoreOptions): FileStore { const rootDir = path.resolve(options.rootDir); + const privateMode = options.private ?? false; const dirMode = options.dirMode ?? 0o700; const mode = options.mode ?? 0o600; const maxBytes = options.maxBytes; @@ -141,12 +176,22 @@ export function fileStore(options: FileStoreOptions): FileStore { async function write( relativePath: string, - data: string | Buffer, + data: string | Uint8Array, writeOptions?: FileStoreWriteOptions, ): Promise { const destination = resolveStorePath(rootDir, relativePath); const content = Buffer.isBuffer(data) ? data : Buffer.from(data); assertMaxBytes(content.byteLength, writeOptions?.maxBytes ?? maxBytes); + if (privateMode) { + await writeSecretFileAtomic({ + rootDir, + filePath: destination, + content, + dirMode: writeOptions?.dirMode ?? dirMode, + mode: writeOptions?.mode ?? mode, + }); + return destination; + } await ensureParent(destination, writeOptions?.dirMode ?? dirMode); const result = await writeSiblingTempFile({ dir: path.dirname(destination), @@ -171,6 +216,24 @@ export function fileStore(options: FileStoreOptions): FileStore { writeStream: async (relativePath, stream, writeOptions) => { const destination = resolveStorePath(rootDir, relativePath); const limit = writeOptions?.maxBytes ?? maxBytes; + if (privateMode) { + const chunks: Buffer[] = []; + let total = 0; + for await (const chunk of stream) { + const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)); + total += buffer.byteLength; + assertMaxBytes(total, limit); + chunks.push(buffer); + } + await writeSecretFileAtomic({ + rootDir, + filePath: destination, + content: Buffer.concat(chunks), + dirMode: writeOptions?.dirMode ?? dirMode, + mode: writeOptions?.mode ?? mode, + }); + return destination; + } await ensureParent(destination, writeOptions?.dirMode ?? dirMode); let total = 0; const result = await writeSiblingTempFile({ @@ -202,15 +265,24 @@ export function fileStore(options: FileStoreOptions): FileStore { return result.filePath; }, copyIn: async (relativePath, sourcePath, writeOptions) => - await copyIntoRoot({ - rootDir, - relativePath, - sourcePath, - dirMode: writeOptions?.dirMode ?? dirMode, - maxBytes: writeOptions?.maxBytes ?? maxBytes, - mode: writeOptions?.mode ?? mode, - tempPrefix: writeOptions?.tempPrefix, - }), + privateMode + ? await (async () => { + const sourceStat = await fs.lstat(sourcePath); + if (sourceStat.isSymbolicLink() || !sourceStat.isFile()) { + throw new FsSafeError("not-file", "source path is not a file"); + } + assertMaxBytes(sourceStat.size, writeOptions?.maxBytes ?? maxBytes); + return await write(relativePath, await fs.readFile(sourcePath), writeOptions); + })() + : await copyIntoRoot({ + rootDir, + relativePath, + sourcePath, + dirMode: writeOptions?.dirMode ?? dirMode, + maxBytes: writeOptions?.maxBytes ?? maxBytes, + mode: writeOptions?.mode ?? mode, + tempPrefix: writeOptions?.tempPrefix, + }), open: async (relativePath, readOptions) => await (await openRoot()).open(assertRelativePath(relativePath), readOptions), read: async (relativePath, readOptions) => @@ -222,16 +294,36 @@ export function fileStore(options: FileStoreOptions): FileStore { return (await (await openRoot()).read(assertRelativePath(relativePath), options)).buffer .toString(encoding); }, - readJson: async ( - relativePath: string, - readOptions?: RootReadOptions & { encoding?: BufferEncoding }, - ) => { + readTextIfExists: async (relativePath, readOptions) => { + try { + return await (await openRoot()).readText(assertRelativePath(relativePath), readOptions); + } catch (error) { + if (isNotFound(error)) { + return null; + } + throw error; + } + }, + readJson: async (relativePath: string, readOptions?: FileStoreReadOptions) => { const { encoding = "utf8", ...options } = readOptions ?? {}; return JSON.parse( (await (await openRoot()).read(assertRelativePath(relativePath), options)).buffer .toString(encoding), ) as T; }, + readJsonIfExists: async ( + relativePath: string, + readOptions?: FileStoreReadOptions, + ) => { + try { + return await (await openRoot()).readJson(assertRelativePath(relativePath), readOptions); + } catch (error) { + if (isNotFound(error)) { + return null; + } + throw error; + } + }, remove: async (relativePath) => { await (await openRoot()).remove(assertRelativePath(relativePath)); }, @@ -281,3 +373,176 @@ export function fileStore(options: FileStoreOptions): FileStore { }, }; } + +function ensureParentSync(filePath: string, mode: number): void { + const dir = path.dirname(filePath); + syncFs.mkdirSync(dir, { recursive: true, mode }); + try { + syncFs.chmodSync(dir, mode); + } catch { + // Best-effort on platforms that do not enforce POSIX modes. + } +} + +function ensurePrivateDirectorySync(rootDir: string, targetDir: string, mode: number): void { + const root = path.resolve(rootDir); + const target = path.resolve(targetDir); + assertStoreFilePath(root, target); + let current = root; + syncFs.mkdirSync(current, { recursive: true, mode }); + const rootStat = syncFs.lstatSync(current); + if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { + throw new FsSafeError("not-file", `private store root must be a directory: ${current}`); + } + try { + syncFs.chmodSync(current, mode); + } catch { + // Best-effort on platforms that do not enforce POSIX modes. + } + for (const segment of path.relative(root, target).split(path.sep).filter(Boolean)) { + current = path.join(current, segment); + try { + const stat = syncFs.lstatSync(current); + if (stat.isSymbolicLink() || !stat.isDirectory()) { + throw new FsSafeError( + "not-file", + `private store directory component must be a directory: ${current}`, + ); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + syncFs.mkdirSync(current, { mode }); + } + const rootReal = syncFs.realpathSync(root); + const currentReal = syncFs.realpathSync(current); + if (!isPathInside(rootReal, currentReal)) { + throw new FsSafeError("outside-workspace", "private store directory escapes root"); + } + try { + syncFs.chmodSync(current, mode); + } catch { + // Best-effort on platforms that do not enforce POSIX modes. + } + } +} + +function writeFileSyncAtomic(params: { + rootDir: string; + filePath: string; + content: string | Uint8Array; + privateMode: boolean; + dirMode: number; + mode: number; +}): string { + const filePath = path.resolve(params.filePath); + assertStoreFilePath(params.rootDir, filePath); + if (params.privateMode) { + ensurePrivateDirectorySync(params.rootDir, path.dirname(filePath), params.dirMode); + try { + const stat = syncFs.lstatSync(filePath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new FsSafeError("not-file", `private store target must be a regular file: ${filePath}`); + } + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") { + throw error; + } + } + } else { + ensureParentSync(filePath, params.dirMode); + } + const tempPath = path.join(path.dirname(filePath), `.fs-safe-${process.pid}-${randomUUID()}.tmp`); + let tempExists = false; + try { + syncFs.writeFileSync(tempPath, params.content, { flag: "wx", mode: params.mode }); + tempExists = true; + try { + syncFs.chmodSync(tempPath, params.mode); + } catch { + // Best-effort on platforms that do not enforce POSIX modes. + } + syncFs.renameSync(tempPath, filePath); + tempExists = false; + try { + syncFs.chmodSync(filePath, params.mode); + } catch { + // Best-effort on platforms that do not enforce POSIX modes. + } + return filePath; + } finally { + if (tempExists) { + try { + syncFs.unlinkSync(tempPath); + } catch { + // Best-effort cleanup after write failure. + } + } + } +} + +export function fileStoreSync(options: FileStoreOptions): FileStoreSync { + const rootDir = path.resolve(options.rootDir); + const privateMode = options.private ?? false; + const dirMode = options.dirMode ?? 0o700; + const mode = options.mode ?? 0o600; + const maxBytes = options.maxBytes; + + function write( + relativePath: string, + data: string | Uint8Array, + writeOptions?: FileStoreWriteOptions, + ): string { + const destination = resolveStorePath(rootDir, relativePath); + const content = Buffer.isBuffer(data) ? data : Buffer.from(data); + assertMaxBytes(content.byteLength, writeOptions?.maxBytes ?? maxBytes); + return writeFileSyncAtomic({ + rootDir, + filePath: destination, + content, + privateMode, + dirMode: writeOptions?.dirMode ?? dirMode, + mode: writeOptions?.mode ?? mode, + }); + } + + return { + rootDir, + path: (relativePath) => resolveStorePath(rootDir, relativePath), + readTextIfExists: (relativePath, readOptions) => { + const targetPath = resolveStorePath(rootDir, relativePath); + try { + const stat = syncFs.lstatSync(targetPath); + if (stat.isSymbolicLink() || !stat.isFile()) { + throw new FsSafeError("not-file", "store target is not a file"); + } + assertMaxBytes(stat.size, readOptions?.maxBytes ?? maxBytes); + if (privateMode && stat.nlink > 1) { + throw new FsSafeError("hardlink", "private store target must not be hardlinked"); + } + return syncFs.readFileSync(targetPath, "utf8"); + } catch (error) { + if (isNotFound(error)) { + return null; + } + throw error; + } + }, + readJsonIfExists: (relativePath: string, readOptions?: { maxBytes?: number }) => { + const raw = fileStoreSync({ rootDir, private: privateMode, dirMode, mode, maxBytes }) + .readTextIfExists(relativePath, readOptions); + return raw === null ? null : (JSON.parse(raw) as T); + }, + write, + writeText: (relativePath, data, writeOptions) => write(relativePath, data, writeOptions), + writeJson: (relativePath, data, writeOptions) => { + const json = JSON.stringify(data, null, 2); + return write( + relativePath, + writeOptions?.trailingNewline === false ? json : `${json}\n`, + writeOptions, + ); + }, + }; +} diff --git a/src/private-file-store.ts b/src/private-file-store.ts deleted file mode 100644 index 54d7222..0000000 --- a/src/private-file-store.ts +++ /dev/null @@ -1,281 +0,0 @@ -import { randomUUID } from "node:crypto"; -import path from "node:path"; -import fs from "node:fs"; -import { FsSafeError } from "./errors.js"; -import { fileStore, type FileStore } from "./file-store.js"; -import { isPathInside } from "./path.js"; -import { readRegularFileSync } from "./regular-file.js"; -import { root } from "./root.js"; -import { writeSecretFileAtomic } from "./secret-file.js"; - -export type PrivateStateStoreOptions = { - rootDir: string; -}; - -export type PrivateStateStore = Omit & { - readText(relativePath: string, options?: { maxBytes?: number }): Promise; - readJson(relativePath: string, options?: { maxBytes?: number }): Promise; - writeText(relativePath: string, content: string | Uint8Array): Promise; - writeJson(relativePath: string, value: unknown, options?: { trailingNewline?: boolean }): Promise; -}; - -function resolvePrivateStorePath(rootDir: string, relativePath: string): string { - const root = path.resolve(rootDir); - const raw = relativePath.trim(); - if (!raw || path.isAbsolute(raw) || raw.includes("\0")) { - throw new Error("Private file path must be a relative path."); - } - const resolved = path.resolve(root, raw); - const rel = path.relative(root, resolved); - if (!rel || rel.startsWith("..") || path.isAbsolute(rel)) { - throw new Error("Private file path must stay under the private store root."); - } - return resolved; -} - -export async function writePrivateTextAtomic(params: { - rootDir: string; - filePath: string; - content: string | Uint8Array; -}): Promise { - await writeSecretFileAtomic(params); -} - -export async function readPrivateText(params: { - rootDir: string; - filePath: string; - maxBytes?: number; -}): Promise { - const rootDir = path.resolve(params.rootDir); - const filePath = path.resolve(params.filePath); - const relative = path.relative(rootDir, filePath); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Private file path must stay under the private store root."); - } - try { - const storeRoot = await root(rootDir, { hardlinks: "reject", maxBytes: params.maxBytes }); - const result = await storeRoot.read(relative); - return result.buffer.toString("utf8"); - } catch (err) { - if (err instanceof FsSafeError && err.code === "not-found") { - return null; - } - throw err; - } -} - -function assertPrivateReadPathSync(params: { rootDir: string; filePath: string }): void { - const rootDir = path.resolve(params.rootDir); - const filePath = path.resolve(params.filePath); - const relative = path.relative(rootDir, filePath); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Private file path must stay under the private store root."); - } - const rootStat = fs.lstatSync(rootDir); - if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { - throw new Error(`Private file root must be a directory: ${rootDir}`); - } - const rootReal = fs.realpathSync(rootDir); - const fileReal = fs.realpathSync(filePath); - if (!isPathInside(rootReal, fileReal)) { - throw new Error("Private file path must stay under the private store root."); - } -} - -export function readPrivateTextSync(params: { - rootDir: string; - filePath: string; - maxBytes?: number; -}): string | null { - try { - assertPrivateReadPathSync(params); - const result = readRegularFileSync({ - filePath: path.resolve(params.filePath), - maxBytes: params.maxBytes, - }); - if (result.stat.nlink > 1) { - throw new Error(`Private file target must not be hardlinked: ${params.filePath}`); - } - return result.buffer.toString("utf8"); - } catch (err) { - if ((err as NodeJS.ErrnoException).code === "ENOENT") { - return null; - } - throw err; - } -} - -export async function readPrivateJson(params: { - rootDir: string; - filePath: string; - maxBytes?: number; -}): Promise { - const raw = await readPrivateText(params); - return raw === null ? null : (JSON.parse(raw) as T); -} - -export function readPrivateJsonSync(params: { - rootDir: string; - filePath: string; - maxBytes?: number; -}): T | null { - const raw = readPrivateTextSync(params); - return raw === null ? null : (JSON.parse(raw) as T); -} - -function ensurePrivateDirectorySync(rootDir: string, targetDir: string): void { - const root = path.resolve(rootDir); - const target = path.resolve(targetDir); - const relative = path.relative(root, target); - if (relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Private file directory must stay under the private store root."); - } - let current = root; - fs.mkdirSync(current, { recursive: true, mode: 0o700 }); - const rootStat = fs.lstatSync(current); - if (rootStat.isSymbolicLink() || !rootStat.isDirectory()) { - throw new Error(`Private file root must be a directory: ${current}`); - } - for (const segment of relative.split(path.sep).filter(Boolean)) { - current = path.join(current, segment); - try { - const stat = fs.lstatSync(current); - if (stat.isSymbolicLink() || !stat.isDirectory()) { - throw new Error(`Private file directory component must be a directory: ${current}`); - } - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - throw err; - } - fs.mkdirSync(current, { mode: 0o700 }); - } - try { - fs.chmodSync(current, 0o700); - } catch { - // Best-effort on platforms that do not enforce POSIX modes. - } - } -} - -export function writePrivateTextAtomicSync(params: { - rootDir: string; - filePath: string; - content: string | Uint8Array; -}): void { - const root = path.resolve(params.rootDir); - const filePath = path.resolve(params.filePath); - const relative = path.relative(root, filePath); - if (!relative || relative.startsWith("..") || path.isAbsolute(relative)) { - throw new Error("Private file path must stay under the private store root."); - } - ensurePrivateDirectorySync(root, path.dirname(filePath)); - try { - const stat = fs.lstatSync(filePath); - if (stat.isSymbolicLink() || !stat.isFile()) { - throw new Error(`Private file target must be a regular file: ${filePath}`); - } - } catch (err) { - if ((err as NodeJS.ErrnoException).code !== "ENOENT") { - throw err; - } - } - const tempPath = path.join(path.dirname(filePath), `.tmp-${process.pid}-${randomUUID()}`); - let created = false; - try { - fs.writeFileSync(tempPath, params.content, { mode: 0o600, flag: "wx" }); - created = true; - try { - fs.chmodSync(tempPath, 0o600); - } catch { - // Best-effort on platforms that do not enforce POSIX modes. - } - fs.renameSync(tempPath, filePath); - created = false; - try { - fs.chmodSync(filePath, 0o600); - } catch { - // Best-effort on platforms that do not enforce POSIX modes. - } - } finally { - if (created) { - try { - fs.unlinkSync(tempPath); - } catch { - // The temp file is best-effort cleanup after write failure. - } - } - } -} - -export async function writePrivateJsonAtomic(params: { - rootDir: string; - filePath: string; - value: unknown; - trailingNewline?: boolean; -}): Promise { - const json = JSON.stringify(params.value, null, 2); - await writeSecretFileAtomic({ - rootDir: params.rootDir, - filePath: params.filePath, - content: params.trailingNewline && !json.endsWith("\n") ? `${json}\n` : json, - }); -} - -export function writePrivateJsonAtomicSync(params: { - rootDir: string; - filePath: string; - value: unknown; - trailingNewline?: boolean; -}): void { - const json = JSON.stringify(params.value, null, 2); - writePrivateTextAtomicSync({ - rootDir: params.rootDir, - filePath: params.filePath, - content: params.trailingNewline && !json.endsWith("\n") ? `${json}\n` : json, - }); -} - -export function privateStateStore(options: PrivateStateStoreOptions): PrivateStateStore { - const rootDir = path.resolve(options.rootDir); - const store = fileStore({ rootDir, private: true }); - return { - ...store, - rootDir, - path: (relativePath) => resolvePrivateStorePath(rootDir, relativePath), - readText: async (relativePath, options) => { - const safePath = resolvePrivateStorePath(rootDir, relativePath); - return await readPrivateText({ - rootDir, - filePath: safePath, - maxBytes: options?.maxBytes, - }); - }, - readJson: async (relativePath: string, options?: { maxBytes?: number }) => { - const safePath = resolvePrivateStorePath(rootDir, relativePath); - return await readPrivateJson({ - rootDir, - filePath: safePath, - maxBytes: options?.maxBytes, - }); - }, - writeText: async (relativePath, content) => { - const safePath = resolvePrivateStorePath(rootDir, relativePath); - await writePrivateTextAtomic({ - rootDir, - filePath: safePath, - content, - }); - return safePath; - }, - writeJson: async (relativePath, value, options) => { - const safePath = resolvePrivateStorePath(rootDir, relativePath); - await writePrivateJsonAtomic({ - rootDir, - filePath: safePath, - value, - trailingNewline: options?.trailingNewline, - }); - return safePath; - }, - }; -} diff --git a/src/store.ts b/src/store.ts index ba6e0d1..586bdc8 100644 --- a/src/store.ts +++ b/src/store.ts @@ -1,8 +1,11 @@ export { fileStore, + fileStoreSync, type FileStore, type FileStoreOptions, type FileStorePruneOptions, + type FileStoreReadOptions, + type FileStoreSync, type FileStoreWriteOptions, } from "./file-store.js"; export { @@ -11,8 +14,3 @@ export { type JsonStoreLockOptions, type JsonStoreOptions, } from "./json-store.js"; -export { - privateStateStore, - type PrivateStateStore, - type PrivateStateStoreOptions, -} from "./private-file-store.js"; diff --git a/test/api-coverage.test.ts b/test/api-coverage.test.ts index eb0c361..349d94d 100644 --- a/test/api-coverage.test.ts +++ b/test/api-coverage.test.ts @@ -9,7 +9,7 @@ import { extractArchive } from "../src/archive.js"; import { loadZipArchiveWithPreflight, readZipCentralDirectoryEntryCount } from "../src/archive-zip-preflight.js"; import { createAsyncLock } from "../src/async-lock.js"; import { writeTextAtomic } from "../src/atomic.js"; -import { copyIntoRoot, fileStore } from "../src/file-store.js"; +import { copyIntoRoot, fileStore, fileStoreSync } from "../src/file-store.js"; import { assertCanonicalPathWithinBase, resolveSafeInstallDir, @@ -49,13 +49,6 @@ import { splitSafeRelativePath, } from "../src/path.js"; import { assertNoHardlinkedFinalPath, assertNoPathAliasEscape } from "../src/path-policy.js"; -import { - privateStateStore, - readPrivateJsonSync, - readPrivateTextSync, - writePrivateJsonAtomicSync, - writePrivateTextAtomicSync, -} from "../src/private-file-store.js"; import { ROOT_PATH_ALIAS_POLICIES, resolveRootPath, resolveRootPathSync } from "../src/root-path.js"; import { ensureDirectoryWithinRoot, @@ -776,9 +769,9 @@ describe("file stores and private stores", () => { await expect(fs.stat(old)).rejects.toMatchObject({ code: "ENOENT" }); }); - it("covers private store sync and async helpers", async () => { + it("covers private file store mode", async () => { const root = await tempRoot("fs-safe-private-store-"); - const store = privateStateStore({ rootDir: root }); + const store = fileStore({ rootDir: root, private: true }); await store.writeText("nested/value.txt", "secret"); await expect(store.readText("nested/value.txt")).resolves.toBe("secret"); @@ -787,28 +780,18 @@ describe("file stores and private stores", () => { await expect(store.exists("nested/value.json")).resolves.toBe(true); await expect(store.readBytes("nested/value.txt")).resolves.toEqual(Buffer.from("secret")); expect(store.path("nested/value.txt")).toBe(path.join(root, "nested", "value.txt")); - expect(() => store.path("../escape.txt")).toThrow("stay under"); - await expect(store.readText("missing.txt")).resolves.toBeNull(); + expect(() => store.path("../escape.txt")).toThrow("relative path"); + await expect(store.readTextIfExists("missing.txt")).resolves.toBeNull(); + await expect(store.readJsonIfExists("missing.json")).resolves.toBeNull(); await store.remove("nested/value.json"); await expect(store.exists("nested/value.json")).resolves.toBe(false); - const syncText = path.join(root, "sync", "value.txt"); - writePrivateTextAtomicSync({ rootDir: root, filePath: syncText, content: "sync" }); - expect(readPrivateTextSync({ rootDir: root, filePath: syncText })).toBe("sync"); - const syncJson = path.join(root, "sync", "value.json"); - writePrivateJsonAtomicSync({ - rootDir: root, - filePath: syncJson, - value: { ok: true }, - trailingNewline: true, - }); - expect(readPrivateJsonSync({ rootDir: root, filePath: syncJson })).toEqual({ ok: true }); - expect(readPrivateTextSync({ rootDir: root, filePath: path.join(root, "missing.txt") })).toBe( - null, - ); - expect(() => readPrivateTextSync({ rootDir: root, filePath: path.dirname(root) })).toThrow( - "stay under", - ); + const syncStore = fileStoreSync({ rootDir: root, private: true }); + const syncText = syncStore.writeText("sync/value.txt", "sync"); + expect(await fs.readFile(syncText, "utf8")).toBe("sync"); + const syncJson = syncStore.writeJson("sync/value.json", { ok: true }, { trailingNewline: true }); + expect(JSON.parse(await fs.readFile(syncJson, "utf8"))).toEqual({ ok: true }); + expect(() => syncStore.writeText("../escape.txt", "nope")).toThrow("relative path"); }); }); diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index b1097ff..a3d72f6 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -3,14 +3,6 @@ import syncFs from "node:fs"; import os from "node:os"; import path from "node:path"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { - privateStateStore, - readPrivateJson, - readPrivateJsonSync, - readPrivateText, - readPrivateTextSync, - writePrivateJsonAtomicSync, -} from "../src/private-file-store.js"; import { appendRegularFile, appendRegularFileSync, @@ -34,7 +26,7 @@ import { replaceFileAtomic, replaceFileAtomicSync } from "../src/replace-file.js import { movePathWithCopyFallback } from "../src/move-path.js"; import { writeSiblingTempFile } from "../src/sibling-temp.js"; import { createSidecarLockManager } from "../src/sidecar-lock.js"; -import { fileStore } from "../src/file-store.js"; +import { fileStore, fileStoreSync } from "../src/file-store.js"; import { jsonStore } from "../src/json-store.js"; import { createIcaclsResetCommand, @@ -308,9 +300,9 @@ describe("directory walking", () => { }); }); -describe("private state store", () => { +describe("private file store mode", () => { it("writes JSON under the store root", async () => { - const store = privateStateStore({ rootDir: root }); + const store = fileStore({ rootDir: root, private: true }); await store.writeJson("nested/state.json", { ok: true }, { trailingNewline: true }); expect(await fs.readFile(path.join(root, "nested", "state.json"), "utf8")).toBe( '{\n "ok": true\n}\n', @@ -319,41 +311,27 @@ describe("private state store", () => { }); it("rejects paths outside the store root", async () => { - const store = privateStateStore({ rootDir: root }); - await expect(store.writeText("../escape.txt", "nope")).rejects.toThrow(/stay under/); - await expect(store.readText("../escape.txt")).rejects.toThrow(/stay under/); + const store = fileStore({ rootDir: root, private: true }); + await expect(store.writeText("../escape.txt", "nope")).rejects.toThrow(/relative path/); + await expect(store.readTextIfExists("../escape.txt")).rejects.toThrow(/outside workspace root/); }); it("supports sync JSON writes", async () => { - const filePath = path.join(root, "sync.json"); - writePrivateJsonAtomicSync({ rootDir: root, filePath, value: { ok: true } }); + const filePath = fileStoreSync({ rootDir: root, private: true }).writeJson("sync.json", { + ok: true, + }); expect(JSON.parse(await fs.readFile(filePath, "utf8"))).toEqual({ ok: true }); }); - it("reads private text and JSON by absolute path", async () => { - const textPath = path.join(root, "state.txt"); - const jsonPath = path.join(root, "state.json"); - await fs.writeFile(textPath, "hello", "utf8"); - await fs.writeFile(jsonPath, '{"ok":true}', "utf8"); + it("has explicit lenient read helpers", async () => { + const store = fileStore({ rootDir: root, private: true }); + await store.writeText("state.txt", "hello"); + await store.writeJson("state.json", { ok: true }); - await expect(readPrivateText({ rootDir: root, filePath: textPath })).resolves.toBe("hello"); - await expect(readPrivateJson({ rootDir: root, filePath: jsonPath })).resolves.toEqual({ - ok: true, - }); - await expect(readPrivateText({ rootDir: root, filePath: path.join(root, "missing") })) - .resolves - .toBeNull(); - }); - - it("reads private text and JSON synchronously", async () => { - const textPath = path.join(root, "sync-state.txt"); - const jsonPath = path.join(root, "sync-state.json"); - await fs.writeFile(textPath, "hello", "utf8"); - await fs.writeFile(jsonPath, '{"ok":true}', "utf8"); - - expect(readPrivateTextSync({ rootDir: root, filePath: textPath })).toBe("hello"); - expect(readPrivateJsonSync({ rootDir: root, filePath: jsonPath })).toEqual({ ok: true }); - expect(readPrivateTextSync({ rootDir: root, filePath: path.join(root, "missing") })).toBeNull(); + await expect(store.readTextIfExists("state.txt")).resolves.toBe("hello"); + await expect(store.readJsonIfExists("state.json")).resolves.toEqual({ ok: true }); + await expect(store.readTextIfExists("missing.txt")).resolves.toBeNull(); + await expect(store.readJsonIfExists("missing.json")).resolves.toBeNull(); }); }); From 56f18594da80a2738e87fdbb1b8f0957beb569dc Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:42:23 +0100 Subject: [PATCH 13/20] fix: handle binary private store streams --- src/file-store.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/file-store.ts b/src/file-store.ts index 75d7ab8..a38215e 100644 --- a/src/file-store.ts +++ b/src/file-store.ts @@ -220,7 +220,8 @@ export function fileStore(options: FileStoreOptions): FileStore { const chunks: Buffer[] = []; let total = 0; for await (const chunk of stream) { - const buffer = Buffer.isBuffer(chunk) ? chunk : Buffer.from(String(chunk)); + const buffer = + typeof chunk === "string" ? Buffer.from(chunk) : Buffer.from(chunk as Uint8Array); total += buffer.byteLength; assertMaxBytes(total, limit); chunks.push(buffer); From b9434cd3637653ee367fe37485cde13a418ef478 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 00:53:09 +0100 Subject: [PATCH 14/20] refactor: bind json state to file stores --- README.md | 16 ++--- docs/file-store.md | 15 +++++ docs/json-store.md | 23 ++++++- src/file-store.ts | 30 +++++++++ src/json-document-store.ts | 116 ++++++++++++++++++++++++++++++++++ src/json-store.ts | 120 +++++------------------------------- src/store.ts | 1 + test/api-coverage.test.ts | 3 +- test/new-primitives.test.ts | 7 ++- 9 files changed, 212 insertions(+), 119 deletions(-) create mode 100644 src/json-document-store.ts diff --git a/README.md b/README.md index fe7c285..be57d17 100644 --- a/README.md +++ b/README.md @@ -197,20 +197,22 @@ await replaceFileAtomic({ ## Stores -Use `jsonStore()` for small state files that need explicit fallback reads, atomic writes, -and optional sidecar locking around read-modify-write updates: +Use `fileStore().json()` for small state files that need explicit fallback +reads, atomic writes, and optional sidecar locking around read-modify-write +updates: ```ts -import { jsonStore } from "@openclaw/fs-safe/store"; +import { fileStore } from "@openclaw/fs-safe/store"; -const store = jsonStore({ - filePath: "/safe/workspace/state/settings.json", - lock: true, -}); +const files = fileStore({ rootDir: "/safe/workspace/state", private: true }); +const store = files.json("settings.json", { lock: true }); await store.updateOr({ enabled: false }, (current) => ({ ...current, enabled: true })); ``` +`jsonStore({ filePath })` is the absolute-path convenience wrapper for the same +primitive. + Use `update()` when missing state is part of your model; use `updateOr()` for the common merge-into-defaults case. Standalone helpers use options bags because they do not carry a bound root and often need multiple authority, path, diff --git a/docs/file-store.md b/docs/file-store.md index a524cce..7fa69c0 100644 --- a/docs/file-store.md +++ b/docs/file-store.md @@ -55,6 +55,7 @@ type FileStore = { readJsonIfExists(rel, options?): Promise; writeText(rel, data: string | Uint8Array, options?): Promise; writeJson(rel, data: unknown, options?): Promise; + json(rel, options?): JsonStore; remove(rel): Promise; exists(rel): Promise; pruneExpired(options: FileStorePruneOptions): Promise; @@ -81,6 +82,20 @@ Buffer or string. Returns the final absolute path. Throws `too-large` if `data.b Convenience wrappers over `write`. `writeJson` pretty-prints with a trailing newline by default and accepts `{ trailingNewline: false }` when the exact bytes matter. +### `json(rel, options?)` + +Returns a typed single-file JSON state helper for a file under this store. It +inherits the store's root, mode, max-size, and private-write policy, then adds +`readOr`, `readRequired`, `update`, `updateOr`, and optional sidecar locking: + +```ts +const state = cache.json("state/settings.json", { lock: true }); +await state.updateOr(defaultState, (current) => ({ ...current, enabled: true })); +``` + +Use this when one JSON file owns one piece of state. `jsonStore({ filePath })` +is the absolute-path convenience wrapper for the same primitive. + ### `writeStream(rel, stream, options?)` ```ts diff --git a/docs/json-store.md b/docs/json-store.md index f949366..94f9dcc 100644 --- a/docs/json-store.md +++ b/docs/json-store.md @@ -1,6 +1,10 @@ # JSON store -`jsonStore` is exported from `@openclaw/fs-safe/store`. It is a small read-modify-write wrapper around a single JSON file. It bakes in atomic writes, explicit fallback reads, and optional cross-process locking via [`createSidecarLockManager`](sidecar-lock.md). +`jsonStore` is exported from `@openclaw/fs-safe/store`. It is the absolute-path +convenience wrapper for `fileStore(...).json(...)`: a small read-modify-write +handle around a single JSON file. It bakes in atomic writes, explicit fallback +reads, and optional cross-process locking via +[`createSidecarLockManager`](sidecar-lock.md). ```ts import { jsonStore } from "@openclaw/fs-safe/store"; @@ -14,13 +18,25 @@ await settings.write({ ...current, volume: 1 }); await settings.updateOr({ theme: "dark", volume: 0.7 }, (prev) => ({ ...prev, theme: "light" })); ``` +If you already have a store/root context, prefer binding the JSON file from that +store: + +```ts +import { fileStore } from "@openclaw/fs-safe/store"; + +const files = fileStore({ rootDir: "/var/lib/app", private: true }); +const settings = files.json("settings.json", { lock: true }); +``` + ## When to reach for it - You have a single JSON state file and want `read / readOr / readRequired / write / update` semantics. - You want every write atomic at file mode `0o600` and parents at `0o700` by default. - You want optional cross-process locking with one boolean. -For ad-hoc read/write of multiple JSON files, use the standalone helpers in [`json`](json.md). For object-style storage of many files at known modes, use [`fileStore`](file-store.md). +For ad-hoc read/write of multiple JSON files, use the standalone helpers in +[`json`](json.md). For object-style storage of many files at known modes, use +[`fileStore`](file-store.md) and bind JSON files with `store.json(rel)`. ## Factory: `jsonStore(options)` @@ -51,6 +67,9 @@ type JsonStore = { }; ``` +`jsonStore({ filePath })` resolves `rootDir = dirname(filePath)` and calls +`fileStore({ rootDir, private: true }).json(basename(filePath), options)`. + The store does **not** validate the parsed value against `T` at runtime — the cast is unchecked. Wrap with a schema (zod/valibot) if the file might be hand-edited or written by another process you don't control. ## `read()` diff --git a/src/file-store.ts b/src/file-store.ts index a38215e..544c2d1 100644 --- a/src/file-store.ts +++ b/src/file-store.ts @@ -5,6 +5,7 @@ import path from "node:path"; import type { Readable } from "node:stream"; import { pipeline } from "node:stream/promises"; import { FsSafeError } from "./errors.js"; +import { createJsonStore, type JsonFileStoreOptions, type JsonStore } from "./json-document-store.js"; import { isPathInside, resolveSafeRelativePath } from "./path.js"; import { root, type OpenResult, type ReadResult, type Root, type RootReadOptions } from "./root.js"; import { writeSecretFileAtomic } from "./secret-file.js"; @@ -78,6 +79,7 @@ export type FileStore = { data: unknown, options?: FileStoreWriteOptions & { trailingNewline?: boolean }, ): Promise; + json(relativePath: string, options?: JsonFileStoreOptions): JsonStore; pruneExpired(options: FileStorePruneOptions): Promise; }; @@ -338,6 +340,34 @@ export function fileStore(options: FileStoreOptions): FileStore { writeOptions, ); }, + json: (relativePath: string, jsonOptions?: JsonFileStoreOptions) => { + const filePath = resolveStorePath(rootDir, relativePath); + return createJsonStore( + { + filePath, + readIfExists: async () => { + try { + return await (await openRoot()).readJson(assertRelativePath(relativePath)); + } catch (error) { + if (isNotFound(error)) { + return null; + } + throw error; + } + }, + readRequired: async () => + await (await openRoot()).readJson(assertRelativePath(relativePath)), + write: async (value, options) => { + const json = JSON.stringify(value, null, 2); + await write( + relativePath, + options?.trailingNewline === false ? json : `${json}\n`, + ); + }, + }, + jsonOptions, + ); + }, pruneExpired: async (pruneOptions) => { const now = Date.now(); const recursive = pruneOptions.recursive ?? false; diff --git a/src/json-document-store.ts b/src/json-document-store.ts new file mode 100644 index 0000000..4f3a9e0 --- /dev/null +++ b/src/json-document-store.ts @@ -0,0 +1,116 @@ +import { createSidecarLockManager, type SidecarLockRetryOptions } from "./sidecar-lock.js"; + +export type JsonStoreLockOptions = { + staleMs?: number; + timeoutMs?: number; + retry?: SidecarLockRetryOptions; + managerKey?: string; +}; + +export type JsonFileStoreOptions = { + trailingNewline?: boolean; + lock?: boolean | JsonStoreLockOptions; +}; + +export type JsonStore = { + readonly filePath: string; + read(): Promise; + readOr(fallback: T): Promise; + readRequired(): Promise; + write(value: T): Promise; + update(run: (current: T | undefined) => T | Promise): Promise; + updateOr(fallback: T, run: (current: T) => T | Promise): Promise; +}; + +export type JsonStoreAdapter = { + filePath: string; + readIfExists(): Promise; + readRequired(): Promise; + write(value: T, options?: { trailingNewline?: boolean }): Promise; +}; + +function cloneFallback(value: T): T { + if (value && typeof value === "object") { + return structuredClone(value); + } + return value; +} + +function resolveLockOptions( + filePath: string, + options: JsonFileStoreOptions, +): Required | null { + if (!options.lock) { + return null; + } + const lockOptions = options.lock === true ? {} : options.lock; + return { + managerKey: lockOptions.managerKey ?? `fs-safe.json-store:${filePath}`, + retry: lockOptions.retry ?? {}, + staleMs: lockOptions.staleMs ?? 30_000, + timeoutMs: lockOptions.timeoutMs ?? 30_000, + }; +} + +export function createJsonStore( + adapter: JsonStoreAdapter, + options: JsonFileStoreOptions = {}, +): JsonStore { + const lockOptions = resolveLockOptions(adapter.filePath, options); + const locks = lockOptions ? createSidecarLockManager(lockOptions.managerKey) : null; + + async function read(): Promise { + return (await adapter.readIfExists()) ?? undefined; + } + + async function readOr(fallback: T): Promise { + return (await read()) ?? cloneFallback(fallback); + } + + async function write(value: T): Promise { + await adapter.write(value, { + trailingNewline: options.trailingNewline ?? true, + }); + } + + async function withOptionalLock(run: () => Promise): Promise { + if (!locks || !lockOptions) { + return await run(); + } + return await locks.withLock( + { + targetPath: adapter.filePath, + staleMs: lockOptions.staleMs, + timeoutMs: lockOptions.timeoutMs, + retry: lockOptions.retry, + allowReentrant: true, + payload: () => ({ pid: process.pid, createdAt: new Date().toISOString() }), + }, + run, + ); + } + + return { + filePath: adapter.filePath, + read, + readOr, + readRequired: adapter.readRequired, + write: async (value) => { + await withOptionalLock(async () => { + await write(value); + }); + }, + update: async (run) => + await withOptionalLock(async () => { + const next = await run(await read()); + await write(next); + return next; + }), + updateOr: async (fallback, run) => + await withOptionalLock(async () => { + const next = await run((await read()) ?? cloneFallback(fallback)); + await write(next); + return next; + }), + }; +} diff --git a/src/json-store.ts b/src/json-store.ts index 96a0a3e..b98188c 100644 --- a/src/json-store.ts +++ b/src/json-store.ts @@ -1,116 +1,26 @@ import path from "node:path"; -import { createSidecarLockManager, type SidecarLockRetryOptions } from "./sidecar-lock.js"; -import { readJson, readJsonIfExists, writeJson } from "./json.js"; +import { fileStore } from "./file-store.js"; +import { + createJsonStore, + type JsonFileStoreOptions, + type JsonStore, + type JsonStoreLockOptions, +} from "./json-document-store.js"; -export type JsonStoreLockOptions = { - staleMs?: number; - timeoutMs?: number; - retry?: SidecarLockRetryOptions; - managerKey?: string; -}; - -export type JsonStoreOptions = { +export type JsonStoreOptions = JsonFileStoreOptions & { filePath: string; dirMode?: number; mode?: number; - trailingNewline?: boolean; - lock?: boolean | JsonStoreLockOptions; }; -export type JsonStore = { - readonly filePath: string; - read(): Promise; - readOr(fallback: T): Promise; - readRequired(): Promise; - write(value: T): Promise; - update(run: (current: T | undefined) => T | Promise): Promise; - updateOr(fallback: T, run: (current: T) => T | Promise): Promise; -}; - -function cloneFallback(value: T): T { - if (value && typeof value === "object") { - return structuredClone(value); - } - return value; -} - -function resolveLockOptions(options: JsonStoreOptions): Required | null { - if (!options.lock) { - return null; - } - const lockOptions = options.lock === true ? {} : options.lock; - return { - managerKey: lockOptions.managerKey ?? `fs-safe.json-store:${path.resolve(options.filePath)}`, - retry: lockOptions.retry ?? {}, - staleMs: lockOptions.staleMs ?? 30_000, - timeoutMs: lockOptions.timeoutMs ?? 30_000, - }; -} +export type { JsonFileStoreOptions, JsonStore, JsonStoreLockOptions }; export function jsonStore(options: JsonStoreOptions): JsonStore { const filePath = path.resolve(options.filePath); - const lockOptions = resolveLockOptions({ ...options, filePath }); - const locks = lockOptions ? createSidecarLockManager(lockOptions.managerKey) : null; - - async function read(): Promise { - const value = await readJsonIfExists(filePath); - return value === null ? undefined : value; - } - - async function readOr(fallback: T): Promise { - return (await read()) ?? cloneFallback(fallback); - } - - async function requireValue(): Promise { - return await readJson(filePath); - } - - async function write(value: T): Promise { - await writeJson(filePath, value, { - mode: options.mode ?? 0o600, - dirMode: options.dirMode ?? 0o700, - trailingNewline: options.trailingNewline ?? true, - }); - } - - async function withOptionalLock(run: () => Promise): Promise { - if (!locks || !lockOptions) { - return await run(); - } - return await locks.withLock( - { - targetPath: filePath, - staleMs: lockOptions.staleMs, - timeoutMs: lockOptions.timeoutMs, - retry: lockOptions.retry, - allowReentrant: true, - payload: () => ({ pid: process.pid, createdAt: new Date().toISOString() }), - }, - run, - ); - } - - return { - filePath, - read, - readOr, - readRequired: requireValue, - write: async (value) => { - await withOptionalLock(async () => { - await write(value); - }); - }, - update: async (run) => - await withOptionalLock(async () => { - const next = await run(await read()); - await write(next); - return next; - }), - updateOr: async (fallback, run) => - await withOptionalLock(async () => { - const next = await run((await read()) ?? cloneFallback(fallback)); - await write(next); - return next; - }), - }; + return fileStore({ + rootDir: path.dirname(filePath), + private: true, + mode: options.mode, + dirMode: options.dirMode, + }).json(path.basename(filePath), options); } diff --git a/src/store.ts b/src/store.ts index 586bdc8..29f2901 100644 --- a/src/store.ts +++ b/src/store.ts @@ -11,6 +11,7 @@ export { export { jsonStore, type JsonStore, + type JsonFileStoreOptions, type JsonStoreLockOptions, type JsonStoreOptions, } from "./json-store.js"; diff --git a/test/api-coverage.test.ts b/test/api-coverage.test.ts index 349d94d..1f79f74 100644 --- a/test/api-coverage.test.ts +++ b/test/api-coverage.test.ts @@ -567,8 +567,7 @@ describe("JSON and regular-file helpers", () => { it("covers json store fallback, unlocked writes, locked writes, and updates", async () => { const root = await tempRoot("fs-safe-json-store-extra-"); const fallback = { count: 1 }; - const store = jsonStore({ - filePath: path.join(root, "state.json"), + const store = fileStore({ rootDir: root, private: true }).json<{ count: number }>("state.json", { lock: { managerKey: `coverage-json-store-${Date.now()}-${Math.random()}`, staleMs: 60_000, diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index a3d72f6..b5a46a1 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -145,14 +145,15 @@ describe("file store", () => { describe("json store", () => { it("reads fallback, writes atomically, and updates under a lock", async () => { const filePath = path.join(root, "state", "store.json"); - const store = jsonStore({ - filePath, + const store = fileStore({ rootDir: path.dirname(filePath), private: true }).json<{ + count: number; + }>(path.basename(filePath), { lock: true, }); await expect(store.read()).resolves.toBeUndefined(); await expect(store.readOr({ count: 10 })).resolves.toEqual({ count: 10 }); - await expect(store.readRequired()).rejects.toMatchObject({ name: "JsonFileReadError" }); + await expect(store.readRequired()).rejects.toMatchObject({ code: "not-found" }); await store.updateOr({ count: 0 }, (current) => ({ count: current.count + 1 })); await expect(store.read()).resolves.toEqual({ count: 1 }); await expect(store.readRequired()).resolves.toEqual({ count: 1 }); From 542657b9d2fc43b8833b4fd6f29724bdcc1d5538 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 01:17:03 +0100 Subject: [PATCH 15/20] refactor: back temp workspaces with file stores --- docs/temp.md | 17 +++++- src/private-temp-workspace.ts | 103 +++++++++++++--------------------- test/api-coverage.test.ts | 4 ++ test/new-primitives.test.ts | 2 + 4 files changed, 62 insertions(+), 64 deletions(-) diff --git a/docs/temp.md b/docs/temp.md index f527aa2..1395925 100644 --- a/docs/temp.md +++ b/docs/temp.md @@ -23,6 +23,7 @@ The compact factory. Returns: ```ts type TempWorkspace = { dir: string; + store: FileStore; file(fileName: string): string; path(fileName: string): string; writePrivate(fileName: string, data: string | Uint8Array): Promise; @@ -45,7 +46,21 @@ await runBuild(workspace.dir, inputPath); `writePrivate` writes at `mode` (default `0o600`); `writeText` and `writeJson` are convenience wrappers for the common scratch-file shapes; `copyIn` ingests an absolute source path through the same atomic-rename machinery as `Root.copyIn`. `read` is a small accessor that reads back any file you wrote into the workspace. -The sync variant `tempWorkspaceSync` exposes the same surface with sync return types. +`store` is a `fileStore({ rootDir: workspace.dir, private: true })` handle. Use +it when you want the richer store surface, including `writeStream`, `exists`, +`remove`, `readJsonIfExists`, or `store.json(rel)`: + +```ts +await using workspace = await tempWorkspace({ rootDir: "/tmp/my-app", prefix: "build-" }); +const state = workspace.store.json("state.json"); +await state.write({ ready: true }); +``` + +The workspace owns cleanup; the store is only a view over the workspace +directory. + +The sync variant `tempWorkspaceSync` exposes the same surface with sync return +types and a `FileStoreSync` at `workspace.store`. ### `withTempWorkspace` diff --git a/src/private-temp-workspace.ts b/src/private-temp-workspace.ts index 7b0dded..ccca060 100644 --- a/src/private-temp-workspace.ts +++ b/src/private-temp-workspace.ts @@ -2,7 +2,12 @@ import { randomUUID } from "node:crypto"; import fsSync from "node:fs"; import fs from "node:fs/promises"; import path from "node:path"; -import { copyIntoRoot } from "./file-store.js"; +import { + fileStore, + fileStoreSync, + type FileStore, + type FileStoreSync, +} from "./file-store.js"; import { registerTempPathForExit } from "./temp-cleanup.js"; export type TempWorkspaceOptions = { @@ -14,6 +19,7 @@ export type TempWorkspaceOptions = { export type TempWorkspace = { dir: string; + store: FileStore; file(fileName: string): string; path(fileName: string): string; writePrivate(fileName: string, data: string | Uint8Array): Promise; @@ -31,6 +37,7 @@ export type TempWorkspace = { export type TempWorkspaceSync = { dir: string; + store: FileStoreSync; file(fileName: string): string; path(fileName: string): string; writePrivate(fileName: string, data: string | Uint8Array): string; @@ -65,6 +72,11 @@ function resolveWorkspaceLeaf(dir: string, fileName: string): string { return path.join(dir, raw); } +function assertWorkspaceFileName(fileName: string): string { + resolveWorkspaceLeaf(".", fileName); + return fileName.trim(); +} + async function ensurePrivateDirectory(dir: string, mode: number): Promise { await fs.mkdir(dir, { recursive: true, mode }); const stat = await fs.stat(dir); @@ -102,42 +114,25 @@ async function createTempWorkspace( if (stat.isSymbolicLink() || !stat.isDirectory()) { throw new Error(`Temp workspace must be a directory: ${dir}`); } + const store = fileStore({ rootDir: dir, private: true, dirMode, mode }); return { dir, + store, file: (fileName) => resolveWorkspaceLeaf(dir, fileName), path: (fileName) => resolveWorkspaceLeaf(dir, fileName), - writePrivate: async (fileName, data) => { - const filePath = resolveWorkspaceLeaf(dir, fileName); - await fs.writeFile(filePath, data, { mode, flag: "wx" }); - await fs.chmod(filePath, mode).catch(() => undefined); - return filePath; - }, - writeText: async (fileName, data) => { - const filePath = resolveWorkspaceLeaf(dir, fileName); - await fs.writeFile(filePath, data, { encoding: "utf8", mode, flag: "wx" }); - await fs.chmod(filePath, mode).catch(() => undefined); - return filePath; - }, - writeJson: async (fileName, data, writeOptions) => { - const json = JSON.stringify(data, null, 2); - const payload = writeOptions?.trailingNewline === false ? json : `${json}\n`; - const filePath = resolveWorkspaceLeaf(dir, fileName); - await fs.writeFile(filePath, payload, { encoding: "utf8", mode, flag: "wx" }); - await fs.chmod(filePath, mode).catch(() => undefined); - return filePath; - }, - copyIn: async (fileName, sourcePath) => { - const filePath = resolveWorkspaceLeaf(dir, fileName); - await copyIntoRoot({ - rootDir: dir, - relativePath: fileName, - sourcePath, + writePrivate: async (fileName, data) => + await store.write(assertWorkspaceFileName(fileName), data, { mode }), + writeText: async (fileName, data) => + await store.writeText(assertWorkspaceFileName(fileName), data, { mode }), + writeJson: async (fileName, data, writeOptions) => + await store.writeJson(assertWorkspaceFileName(fileName), data, { mode, - }); - return filePath; - }, - read: async (fileName) => await fs.readFile(resolveWorkspaceLeaf(dir, fileName)), + trailingNewline: writeOptions?.trailingNewline, + }), + copyIn: async (fileName, sourcePath) => + await store.copyIn(assertWorkspaceFileName(fileName), sourcePath, { mode }), + read: async (fileName) => await store.readBytes(assertWorkspaceFileName(fileName)), cleanup: async () => { try { await fs.rm(dir, { recursive: true, force: true }).catch(() => undefined); @@ -200,44 +195,26 @@ export function tempWorkspaceSync( if (stat.isSymbolicLink() || !stat.isDirectory()) { throw new Error(`Temp workspace must be a directory: ${dir}`); } + const store = fileStoreSync({ rootDir: dir, private: true, dirMode, mode }); return { dir, + store, file: (fileName) => resolveWorkspaceLeaf(dir, fileName), path: (fileName) => resolveWorkspaceLeaf(dir, fileName), - writePrivate: (fileName, data) => { - const filePath = resolveWorkspaceLeaf(dir, fileName); - fsSync.writeFileSync(filePath, data, { mode, flag: "wx" }); - try { - fsSync.chmodSync(filePath, mode); - } catch { - // Best-effort on platforms that do not enforce POSIX modes. - } - return filePath; + writePrivate: (fileName, data) => + store.write(assertWorkspaceFileName(fileName), data, { mode }), + writeText: (fileName, data) => + store.writeText(assertWorkspaceFileName(fileName), data, { mode }), + writeJson: (fileName, data, writeOptions) => + store.writeJson(assertWorkspaceFileName(fileName), data, { + mode, + trailingNewline: writeOptions?.trailingNewline, + }), + read: (fileName) => { + const filePath = store.path(assertWorkspaceFileName(fileName)); + return fsSync.readFileSync(filePath); }, - writeText: (fileName, data) => { - const filePath = resolveWorkspaceLeaf(dir, fileName); - fsSync.writeFileSync(filePath, data, { encoding: "utf8", mode, flag: "wx" }); - try { - fsSync.chmodSync(filePath, mode); - } catch { - // Best-effort on platforms that do not enforce POSIX modes. - } - return filePath; - }, - writeJson: (fileName, data, writeOptions) => { - const json = JSON.stringify(data, null, 2); - const payload = writeOptions?.trailingNewline === false ? json : `${json}\n`; - const filePath = resolveWorkspaceLeaf(dir, fileName); - fsSync.writeFileSync(filePath, payload, { encoding: "utf8", mode, flag: "wx" }); - try { - fsSync.chmodSync(filePath, mode); - } catch { - // Best-effort on platforms that do not enforce POSIX modes. - } - return filePath; - }, - read: (fileName) => fsSync.readFileSync(resolveWorkspaceLeaf(dir, fileName)), cleanup: () => { try { fsSync.rmSync(dir, { recursive: true, force: true }); diff --git a/test/api-coverage.test.ts b/test/api-coverage.test.ts index 1f79f74..da58e9a 100644 --- a/test/api-coverage.test.ts +++ b/test/api-coverage.test.ts @@ -626,12 +626,14 @@ describe("temporary workspace and symlink parent helpers", () => { const workspace = await tempWorkspace({ rootDir: root, prefix: "bad prefix!" }); expect(() => workspace.file("../bad")).toThrow("Invalid temp workspace"); const privateFile = await workspace.writePrivate("private.bin", Buffer.from("private")); + await workspace.store.writeText("store.txt", "stored"); const textFile = await workspace.writeText("text.txt", "text"); const jsonFile = await workspace.writeJson("data.json", { ok: true }, { trailingNewline: false, }); await expect(workspace.copyIn("copy.txt", source)).resolves.toBe(workspace.path("copy.txt")); await expect(workspace.read("text.txt")).resolves.toEqual(Buffer.from("text")); + await expect(workspace.store.readText("store.txt")).resolves.toBe("stored"); expect(path.basename(privateFile)).toBe("private.bin"); expect(path.basename(textFile)).toBe("text.txt"); await expect(fs.readFile(jsonFile, "utf8")).resolves.toBe('{\n "ok": true\n}'); @@ -650,6 +652,8 @@ describe("temporary workspace and symlink parent helpers", () => { expect(syncWorkspace.writePrivate("private.bin", Buffer.from("private"))).toContain( "private.bin", ); + expect(syncWorkspace.store.writeText("store.txt", "stored")).toContain("store.txt"); + expect(syncWorkspace.store.readTextIfExists("store.txt")).toBe("stored"); expect(syncWorkspace.writeText("text.txt", "text")).toContain("text.txt"); expect(syncWorkspace.writeJson("data.json", { ok: true }, { trailingNewline: false })) .toContain("data.json"); diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index b5a46a1..8bd308e 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -97,6 +97,8 @@ describe("private temp workspaces", () => { const filePath = await tmp.writePrivate("input.txt", "hello"); expect(filePath).toBe(tmp.file("input.txt")); expect(tmp.path("input.txt")).toBe(filePath); + await tmp.store.json<{ ok: boolean }>("state.json").write({ ok: true }); + await expect(tmp.store.readJson("state.json")).resolves.toEqual({ ok: true }); } await expect(fs.stat(workspaceDir)).rejects.toMatchObject({ code: "ENOENT" }); From 02b9a9d2ae80f6f93a2d78a9e630c831ce5beb6c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 01:26:06 +0100 Subject: [PATCH 16/20] refactor: trim temp workspace surface --- docs/file-store.md | 22 +--------------------- docs/temp.md | 11 +++++------ src/advanced.ts | 1 - src/file-store.ts | 2 +- src/private-temp-workspace.ts | 12 ++++-------- test/api-coverage.test.ts | 12 ++++++------ test/new-primitives.test.ts | 12 ++++++------ 7 files changed, 23 insertions(+), 49 deletions(-) diff --git a/docs/file-store.md b/docs/file-store.md index 7fa69c0..319d333 100644 --- a/docs/file-store.md +++ b/docs/file-store.md @@ -159,26 +159,6 @@ type FileStorePruneOptions = { Symlinks are skipped. The walk is best-effort — failures on individual entries don't abort the whole prune. Compares against `mtimeMs`. -## Standalone: `copyIntoRoot` - -The same one-shot copy primitive used by `FileStore.copyIn`, available from the advanced surface for callers that don't want to instantiate a store: - -```ts -import { copyIntoRoot } from "@openclaw/fs-safe/advanced"; - -await copyIntoRoot({ - rootDir: "/var/cache/app", - relativePath: "ingest/upload.bin", - sourcePath: "/tmp/upload.bin", - mode: 0o600, - dirMode: 0o700, - maxBytes: 32 * 1024 * 1024, - tempPrefix: ".upload.bin", // optional -}); -``` - -Returns the final absolute path. Throws `not-file` if the source is a symlink or non-regular file; throws `too-large` if it exceeds `maxBytes`. - ## Difference from `Root` | `FileStore` | `Root` | @@ -230,4 +210,4 @@ await root.move(`pending/${id}`, `done/${id}`); - [`root()`](root.md) — the boundary `FileStore` is built on; reach for it when you need move/list/append. - [JSON store](json-store.md) — the JSON-state-file equivalent of this surface. - [Atomic writes](atomic.md) — `writeSiblingTempFile` is what every write goes through. -- [Temp workspaces](temp.md) — `TempWorkspace.copyIn` uses `copyIntoRoot`. +- [Temp workspaces](temp.md) — private scratch directories backed by `FileStore`. diff --git a/docs/temp.md b/docs/temp.md index 1395925..6b61fc6 100644 --- a/docs/temp.md +++ b/docs/temp.md @@ -24,9 +24,8 @@ The compact factory. Returns: type TempWorkspace = { dir: string; store: FileStore; - file(fileName: string): string; path(fileName: string): string; - writePrivate(fileName: string, data: string | Uint8Array): Promise; + write(fileName: string, data: string | Uint8Array): Promise; writeText(fileName: string, data: string): Promise; writeJson(fileName: string, data: unknown, options?: { trailingNewline?: boolean }): Promise; copyIn(fileName: string, sourcePath: string): Promise; @@ -40,11 +39,11 @@ type TempWorkspace = { import { tempWorkspace } from "@openclaw/fs-safe/temp"; await using workspace = await tempWorkspace({ rootDir: "/tmp/my-app", prefix: "build-" }); -const inputPath = await workspace.writePrivate("input.txt", "data"); +const inputPath = await workspace.write("input.txt", "data"); await runBuild(workspace.dir, inputPath); ``` -`writePrivate` writes at `mode` (default `0o600`); `writeText` and `writeJson` are convenience wrappers for the common scratch-file shapes; `copyIn` ingests an absolute source path through the same atomic-rename machinery as `Root.copyIn`. `read` is a small accessor that reads back any file you wrote into the workspace. +`write` writes at `mode` (default `0o600`); `writeText` and `writeJson` are convenience wrappers for the common scratch-file shapes; `copyIn` ingests an absolute source path through the same atomic-rename machinery as `Root.copyIn`. `read` is a small accessor that reads back any file you wrote into the workspace. `store` is a `fileStore({ rootDir: workspace.dir, private: true })` handle. Use it when you want the richer store surface, including `writeStream`, `exists`, @@ -70,7 +69,7 @@ The recommended shape. Auto-cleanup on every exit path: import { withTempWorkspace } from "@openclaw/fs-safe/temp"; const result = await withTempWorkspace({ rootDir: "/tmp/my-app", prefix: "build-" }, async (workspace) => { - await workspace.writePrivate("input.txt", "data"); + await workspace.write("input.txt", "data"); return await runBuild(workspace.dir); }); ``` @@ -101,7 +100,7 @@ type TempWorkspaceOptions = { rootDir: string; // parent directory for workspaces prefix: string; // dir prefix (sanitized) dirMode?: number; // dir mode; default 0o700 - mode?: number; // writePrivate file mode; default 0o600 + mode?: number; // file write mode; default 0o600 }; ``` diff --git a/src/advanced.ts b/src/advanced.ts index aa1eab3..62d607d 100644 --- a/src/advanced.ts +++ b/src/advanced.ts @@ -15,7 +15,6 @@ export { export { sameFileIdentity, type FileIdentityStat } from "./file-identity.js"; export { sanitizeUntrustedFileName } from "./filename.js"; export { pathExists, pathExistsSync } from "./fs.js"; -export { copyIntoRoot, fileStoreSync, type FileStoreSync } from "./file-store.js"; export { resolveLocalPathFromRootsSync, readLocalFileFromRoots, diff --git a/src/file-store.ts b/src/file-store.ts index 544c2d1..60e5b54 100644 --- a/src/file-store.ts +++ b/src/file-store.ts @@ -134,7 +134,7 @@ function isNotFound(error: unknown): boolean { (error as NodeJS.ErrnoException).code === "ENOTDIR"; } -export async function copyIntoRoot(params: { +async function copyIntoRoot(params: { rootDir: string; relativePath: string; sourcePath: string; diff --git a/src/private-temp-workspace.ts b/src/private-temp-workspace.ts index ccca060..9c00476 100644 --- a/src/private-temp-workspace.ts +++ b/src/private-temp-workspace.ts @@ -20,9 +20,8 @@ export type TempWorkspaceOptions = { export type TempWorkspace = { dir: string; store: FileStore; - file(fileName: string): string; path(fileName: string): string; - writePrivate(fileName: string, data: string | Uint8Array): Promise; + write(fileName: string, data: string | Uint8Array): Promise; writeText(fileName: string, data: string): Promise; writeJson( fileName: string, @@ -38,9 +37,8 @@ export type TempWorkspace = { export type TempWorkspaceSync = { dir: string; store: FileStoreSync; - file(fileName: string): string; path(fileName: string): string; - writePrivate(fileName: string, data: string | Uint8Array): string; + write(fileName: string, data: string | Uint8Array): string; writeText(fileName: string, data: string): string; writeJson(fileName: string, data: unknown, options?: { trailingNewline?: boolean }): string; read(fileName: string): Buffer; @@ -119,9 +117,8 @@ async function createTempWorkspace( return { dir, store, - file: (fileName) => resolveWorkspaceLeaf(dir, fileName), path: (fileName) => resolveWorkspaceLeaf(dir, fileName), - writePrivate: async (fileName, data) => + write: async (fileName, data) => await store.write(assertWorkspaceFileName(fileName), data, { mode }), writeText: async (fileName, data) => await store.writeText(assertWorkspaceFileName(fileName), data, { mode }), @@ -200,9 +197,8 @@ export function tempWorkspaceSync( return { dir, store, - file: (fileName) => resolveWorkspaceLeaf(dir, fileName), path: (fileName) => resolveWorkspaceLeaf(dir, fileName), - writePrivate: (fileName, data) => + write: (fileName, data) => store.write(assertWorkspaceFileName(fileName), data, { mode }), writeText: (fileName, data) => store.writeText(assertWorkspaceFileName(fileName), data, { mode }), diff --git a/test/api-coverage.test.ts b/test/api-coverage.test.ts index da58e9a..a048a13 100644 --- a/test/api-coverage.test.ts +++ b/test/api-coverage.test.ts @@ -9,7 +9,7 @@ import { extractArchive } from "../src/archive.js"; import { loadZipArchiveWithPreflight, readZipCentralDirectoryEntryCount } from "../src/archive-zip-preflight.js"; import { createAsyncLock } from "../src/async-lock.js"; import { writeTextAtomic } from "../src/atomic.js"; -import { copyIntoRoot, fileStore, fileStoreSync } from "../src/file-store.js"; +import { fileStore, fileStoreSync } from "../src/file-store.js"; import { assertCanonicalPathWithinBase, resolveSafeInstallDir, @@ -624,8 +624,8 @@ describe("temporary workspace and symlink parent helpers", () => { await fs.writeFile(source, "copy", "utf8"); const workspace = await tempWorkspace({ rootDir: root, prefix: "bad prefix!" }); - expect(() => workspace.file("../bad")).toThrow("Invalid temp workspace"); - const privateFile = await workspace.writePrivate("private.bin", Buffer.from("private")); + expect(() => workspace.path("../bad")).toThrow("Invalid temp workspace"); + const privateFile = await workspace.write("private.bin", Buffer.from("private")); await workspace.store.writeText("store.txt", "stored"); const textFile = await workspace.writeText("text.txt", "text"); const jsonFile = await workspace.writeJson("data.json", { ok: true }, { @@ -648,8 +648,8 @@ describe("temporary workspace and symlink parent helpers", () => { const syncWorkspace = tempWorkspaceSync({ rootDir: root, prefix: ".." }); try { - expect(() => syncWorkspace.file("bad/name")).toThrow("Invalid temp workspace"); - expect(syncWorkspace.writePrivate("private.bin", Buffer.from("private"))).toContain( + expect(() => syncWorkspace.path("bad/name")).toThrow("Invalid temp workspace"); + expect(syncWorkspace.write("private.bin", Buffer.from("private"))).toContain( "private.bin", ); expect(syncWorkspace.store.writeText("store.txt", "stored")).toContain("store.txt"); @@ -756,7 +756,7 @@ describe("file stores and private stores", () => { maxBytes: 4, })).rejects.toMatchObject({ code: "too-large" }); await expect(store.copyIn("copied.txt", source)).resolves.toBe(path.join(root, "copied.txt")); - await expect(copyIntoRoot({ rootDir: root, relativePath: "bad.txt", sourcePath: sourceRoot })) + await expect(store.copyIn("bad.txt", sourceRoot)) .rejects .toMatchObject({ code: "not-file" }); await expect(store.exists("copied.txt")).resolves.toBe(true); diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index 8bd308e..45735d8 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -52,7 +52,7 @@ describe("private temp workspaces", () => { let workspaceDir = ""; const content = await withTempWorkspace({ rootDir: root, prefix: "work-" }, async (tmp) => { workspaceDir = tmp.dir; - const filePath = await tmp.writePrivate("input.txt", "hello"); + const filePath = await tmp.write("input.txt", "hello"); expect(await fs.readFile(filePath, "utf8")).toBe("hello"); return await tmp.read("input.txt"); }); @@ -64,7 +64,7 @@ describe("private temp workspaces", () => { it("rejects path-like file names", async () => { const tmp = await tempWorkspace({ rootDir: root, prefix: "work-" }); try { - await expect(tmp.writePrivate("../escape.txt", "nope")).rejects.toThrow(/Invalid/); + await expect(tmp.write("../escape.txt", "nope")).rejects.toThrow(/Invalid/); } finally { await tmp.cleanup(); } @@ -74,7 +74,7 @@ describe("private temp workspaces", () => { let workspaceDir = ""; const result = withTempWorkspaceSync({ rootDir: root, prefix: "sync-" }, (tmp) => { workspaceDir = tmp.dir; - const filePath = tmp.writePrivate("input.txt", "hello"); + const filePath = tmp.write("input.txt", "hello"); expect(tmp.read("input.txt").toString("utf8")).toBe("hello"); return filePath; }); @@ -83,7 +83,7 @@ describe("private temp workspaces", () => { const tmp = tempWorkspaceSync({ rootDir: root, prefix: "sync-" }); try { - expect(tmp.writePrivate("again.txt", "ok")).toContain("again.txt"); + expect(tmp.write("again.txt", "ok")).toContain("again.txt"); } finally { tmp.cleanup(); } @@ -94,8 +94,8 @@ describe("private temp workspaces", () => { { await using tmp = await tempWorkspace({ rootDir: root, prefix: "compact-" }); workspaceDir = tmp.dir; - const filePath = await tmp.writePrivate("input.txt", "hello"); - expect(filePath).toBe(tmp.file("input.txt")); + const filePath = await tmp.write("input.txt", "hello"); + expect(filePath).toBe(tmp.path("input.txt")); expect(tmp.path("input.txt")).toBe(filePath); await tmp.store.json<{ ok: boolean }>("state.json").write({ ok: true }); await expect(tmp.store.readJson("state.json")).resolves.toEqual({ ok: true }); From c70178e7e170e466473d064c15ff542ffd05099b Mon Sep 17 00:00:00 2001 From: Jesse Merhi <79823012+jesse-merhi@users.noreply.github.com> Date: Wed, 6 May 2026 10:33:25 +1000 Subject: [PATCH 17/20] test: add additional bypass parity coverage (#3) --- test/additional-bypass-parity.test.ts | 172 ++++++++++++++++++++++++++ test/fs-safe.test.ts | 7 +- 2 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 test/additional-bypass-parity.test.ts diff --git a/test/additional-bypass-parity.test.ts b/test/additional-bypass-parity.test.ts new file mode 100644 index 0000000..4955c6e --- /dev/null +++ b/test/additional-bypass-parity.test.ts @@ -0,0 +1,172 @@ +import fs from "node:fs"; +import fsp from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { pathToFileURL } from "node:url"; +import { afterEach, describe, expect, it, vi } from "vitest"; +import { safeFileURLToPath } from "../src/local-file-access.js"; +import { assertCanonicalPathWithinBase, resolveSafeInstallDir } from "../src/install-path.js"; +import { createJsonStore } from "../src/json-document-store.js"; +import { movePathToTrash } from "../src/trash.js"; +import { resolveArchiveOutputPath, validateArchiveEntryPath } from "../src/archive-entry.js"; +import { prepareArchiveOutputPath } from "../src/archive-staging.js"; +import { sanitizeTempFileName, tempFile } from "../src/temp-target.js"; +import { walkDirectory, walkDirectorySync } from "../src/walk.js"; + +type TempLayout = { + base: string; + outside: string; + outsideFile: string; +}; + +const tempDirs: string[] = []; + +const ARCHIVE_ESCAPE_PAYLOADS = [ + "../evil.txt", + "../../evil.txt", + "nested/../../evil.txt", + "/absolute/evil.txt", + "//server/share/evil.txt", + "C:/Windows/win.ini", + "C:\\Windows\\win.ini", + "..\\evil.txt", + "nested\\..\\..\\evil.txt", +] as const; + +async function makeTempLayout(prefix: string): Promise { + const base = await fsp.mkdtemp(path.join(os.tmpdir(), `${prefix}-base-`)); + const outside = await fsp.mkdtemp(path.join(os.tmpdir(), `${prefix}-outside-`)); + tempDirs.push(base, outside); + const outsideFile = path.join(outside, "secret.txt"); + await fsp.writeFile(outsideFile, "outside secret"); + return { base, outside, outsideFile }; +} + +afterEach(async () => { + vi.restoreAllMocks(); + await Promise.all(tempDirs.splice(0).map((dir) => fsp.rm(dir, { force: true, recursive: true }))); +}); + +describe("additional bypass parity", () => { + it("rejects archive traversal payloads before resolving output paths", async () => { + const layout = await makeTempLayout("fs-safe-archive-payloads"); + + for (const payload of ARCHIVE_ESCAPE_PAYLOADS) { + expect(() => validateArchiveEntryPath(payload), `validate ${payload}`).toThrow(); + await expect( + prepareArchiveOutputPath({ destDir: layout.base, relativePath: payload, originalPath: payload }), + ).rejects.toThrow(); + } + }); + + it("keeps archive output resolution inside the destination for benign weird names", async () => { + const layout = await makeTempLayout("fs-safe-archive-literals"); + const payloads = ["%2e%2e%2fevil.txt", "..%2fevil.txt", "safe/..hidden/file.txt"]; + + for (const payload of payloads) { + validateArchiveEntryPath(payload); + const output = resolveArchiveOutputPath({ rootDir: layout.base, relPath: payload, originalPath: payload }); + expect(output.startsWith(`${layout.base}${path.sep}`)).toBe(true); + } + }); + + it("sanitizes temp file names and keeps temp file helpers inside their created directory", async () => { + const layout = await makeTempLayout("fs-safe-temp"); + expect(sanitizeTempFileName("../../evil.txt")).toBe("evil.txt"); + expect(sanitizeTempFileName("..\\evil.txt")).toBe("..-evil.txt"); + expect(sanitizeTempFileName("\u0000../evil.txt")).toBe("evil.txt"); + + const target = await tempFile({ rootDir: layout.base, prefix: "../../prefix", fileName: "../../evil.txt" }); + tempDirs.push(target.dir); + expect(target.dir.startsWith(`${layout.base}${path.sep}`)).toBe(true); + expect(target.path).toBe(path.join(target.dir, "evil.txt")); + expect(target.file("../../other.txt")).toBe(path.join(target.dir, "other.txt")); + await target.cleanup(); + await expect(fsp.stat(target.dir)).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("rejects remote or encoded-separator file URLs while accepting local file URLs", () => { + const local = pathToFileURL(path.join(os.tmpdir(), "safe.txt")).toString(); + expect(safeFileURLToPath(local)).toBe(path.join(os.tmpdir(), "safe.txt")); + expect(() => safeFileURLToPath("https://example.com/secret.txt")).toThrow(); + expect(() => safeFileURLToPath("file://evil.example/secret.txt")).toThrow(); + expect(() => safeFileURLToPath("file:///tmp/%2Fetc/passwd")).toThrow(); + expect(() => safeFileURLToPath("file:///tmp/%5Cevil")).toThrow(); + }); + + it("keeps install directories and canonical base checks inside their base", async () => { + const layout = await makeTempLayout("fs-safe-install"); + const safe = resolveSafeInstallDir({ baseDir: layout.base, id: "../../evil/pkg", invalidNameMessage: "bad package" }); + expect(safe).toMatchObject({ ok: true }); + if (!safe.ok) throw new Error("expected safe install dir"); + expect(safe.path.startsWith(`${layout.base}${path.sep}`)).toBe(true); + + await expect( + assertCanonicalPathWithinBase({ baseDir: layout.base, candidatePath: layout.outsideFile, boundaryLabel: "install base" }), + ).rejects.toThrow(); + const insideDir = path.join(layout.base, "inside"); + await fsp.mkdir(insideDir); + await expect( + assertCanonicalPathWithinBase({ + baseDir: layout.base, + boundaryLabel: "install base", + candidatePath: path.join(insideDir, "future-file.txt"), + }), + ).resolves.toBeUndefined(); + }); + + it("walks do not follow symlinks by default and do not loop when following cycles", async () => { + const layout = await makeTempLayout("fs-safe-walk"); + await fsp.mkdir(path.join(layout.base, "dir")); + await fsp.writeFile(path.join(layout.base, "dir", "inside.txt"), "inside"); + await fsp.symlink(layout.outside, path.join(layout.base, "outside-link"), "dir"); + await fsp.symlink(layout.base, path.join(layout.base, "dir", "cycle"), "dir"); + + const skipped = await walkDirectory(layout.base); + expect(skipped.entries.some((entry) => entry.path.startsWith(layout.outside))).toBe(false); + expect(skipped.entries.some((entry) => entry.relativePath.includes("outside-link"))).toBe(false); + + const followed = await walkDirectory(layout.base, { symlinks: "follow", maxEntries: 20 }); + expect(followed.entries.length).toBeLessThanOrEqual(20); + expect(followed.entries.some((entry) => entry.path.startsWith(layout.outside))).toBe(false); + + const syncFollowed = walkDirectorySync(layout.base, { symlinks: "follow", maxEntries: 20 }); + expect(syncFollowed.entries.length).toBeLessThanOrEqual(20); + }); + + it("refuses to trash targets outside explicit allowed roots and does not move them", async () => { + const layout = await makeTempLayout("fs-safe-trash"); + await expect(movePathToTrash(layout.outsideFile, { allowedRoots: [layout.base] })).rejects.toThrow(); + await expect(fsp.readFile(layout.outsideFile, "utf8")).resolves.toBe("outside secret"); + }); + + it("json stores cannot bypass adapter-enforced root checks through lock/update flow", async () => { + const layout = await makeTempLayout("fs-safe-json-store"); + const filePath = path.join(layout.base, "state.json"); + const adapter = { + filePath, + async readIfExists(): Promise<{ ok: boolean } | null> { + try { + return JSON.parse(await fsp.readFile(filePath, "utf8")) as { ok: boolean }; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } + }, + async readRequired(): Promise<{ ok: boolean }> { + return JSON.parse(await fsp.readFile(filePath, "utf8")) as { ok: boolean }; + }, + async write(value: { ok: boolean }): Promise { + const resolved = path.resolve(filePath); + if (!resolved.startsWith(`${layout.base}${path.sep}`)) { + throw new Error("adapter escaped root"); + } + await fsp.writeFile(filePath, JSON.stringify(value)); + }, + }; + const store = createJsonStore(adapter, { lock: true }); + await expect(store.updateOr({ ok: false }, () => ({ ok: true }))).resolves.toEqual({ ok: true }); + await expect(fsp.readFile(filePath, "utf8")).resolves.toBe('{"ok":true}'); + await expect(fsp.readFile(layout.outsideFile, "utf8")).resolves.toBe("outside secret"); + }); +}); diff --git a/test/fs-safe.test.ts b/test/fs-safe.test.ts index 5ffb5fd..662c6be 100644 --- a/test/fs-safe.test.ts +++ b/test/fs-safe.test.ts @@ -1,5 +1,5 @@ import { appendFileSync } from "node:fs"; -import { mkdtemp, readdir, readFile, rm, stat, symlink, writeFile } from "node:fs/promises"; +import { mkdtemp, readdir, readFile, rename, rm, stat, symlink, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { afterEach, describe, expect, it } from "vitest"; @@ -243,11 +243,14 @@ describe("@openclaw/fs-safe", () => { const rootPath = await tempRoot("fs-safe-copy-source-swap-root-"); const sourceRoot = await tempRoot("fs-safe-copy-source-swap-source-"); const sourcePath = path.join(sourceRoot, "source.txt"); + const replacementPath = path.join(sourceRoot, "replacement.txt"); await writeFile(sourcePath, "original"); + await writeFile(replacementPath, "replacement"); const sourceIdentity = await stat(sourcePath); await rm(sourcePath); - await writeFile(sourcePath, "replacement"); + await rename(replacementPath, sourcePath); + configureFsSafePython({ mode: "require" }); try { await runPinnedCopyHelper({ rootPath, From 2e83f7d9b96eeb0a07c9ac1431d66e81b049e69f Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 01:43:39 +0100 Subject: [PATCH 18/20] refactor: narrow low-level lock and pinned-open surface --- README.md | 3 +- docs/index.md | 2 +- docs/install.md | 3 +- docs/json-store.md | 8 +-- docs/json.md | 2 +- docs/pinned-open.md | 125 ---------------------------------- docs/reading.md | 2 +- docs/sidecar-lock.md | 130 ++++++++++++++++-------------------- docs/temp.md | 2 +- docs/timing.md | 2 +- package.json | 4 ++ src/advanced.ts | 8 --- src/file-lock.ts | 83 +++++++++++++++++++++++ src/json-document-store.ts | 5 +- test/new-primitives.test.ts | 9 ++- 15 files changed, 166 insertions(+), 222 deletions(-) delete mode 100644 docs/pinned-open.md create mode 100644 src/file-lock.ts diff --git a/README.md b/README.md index be57d17..7ab6764 100644 --- a/README.md +++ b/README.md @@ -158,10 +158,11 @@ that OpenClaw needs to compose higher-level APIs are grouped under | `@openclaw/fs-safe/atomic` | `replaceFileAtomic`, `replaceFileAtomicSync`, `replaceDirectoryAtomic`, `movePathWithCopyFallback` | | `@openclaw/fs-safe/temp` | `tempWorkspace`, `tempWorkspaceSync`, `withTempWorkspace`, `resolveSecureTempRoot` | | `@openclaw/fs-safe/secure-file` | fd-pinned absolute file reads with owner, mode, ACL, trusted-dir, size, and timeout checks | +| `@openclaw/fs-safe/file-lock` | `acquireFileLock`, `withFileLock`, `createFileLockManager`, and related lock types | | `@openclaw/fs-safe/permissions` | POSIX mode and Windows ACL inspection plus remediation formatting helpers | | `@openclaw/fs-safe/walk` | budget-bounded directory walking with symlink policy, filters, and truncation accounting; not root-bounded | | `@openclaw/fs-safe/archive` | `extractArchive`, `resolveArchiveKind`, `ArchiveLimitError`, preflight helpers | -| `@openclaw/fs-safe/advanced` | lower-level composition helpers such as path scopes, pinned open, sidecar locks, install paths, filename sanitizing, temp-file targets, sibling-temp writes, local-root readers, regular-file helpers, `pathExists`, and `withTimeout`; less stable than focused public subpaths | +| `@openclaw/fs-safe/advanced` | lower-level composition helpers such as path scopes, root-file open, install paths, filename sanitizing, temp-file targets, sibling-temp writes, local-root readers, regular-file helpers, `pathExists`, and `withTimeout`; less stable than focused public subpaths | | `@openclaw/fs-safe/errors` | `FsSafeError`, `FsSafeErrorCode` | | `@openclaw/fs-safe/types` | shared types: `DirEntry`, `PathStat`, … | | `@openclaw/fs-safe/test-hooks` | hooks the test suite uses to inject races; only active under `NODE_ENV=test` | diff --git a/docs/index.md b/docs/index.md index 955b8e3..f55ee10 100644 --- a/docs/index.md +++ b/docs/index.md @@ -61,7 +61,7 @@ await fs.remove("notes/archive/today.txt"); | [`extractArchive`](archive.md) | ZIP/TAR extraction with size, count, link, and traversal limits. | | [Secret files](secret-file.md) | Mode-0600 credentials with size and TOCTOU defense. | | [Permissions](permissions.md) | POSIX mode and Windows ACL inspection/remediation helpers. | -| [`createSidecarLockManager`](sidecar-lock.md) | Cross-process file lock with retry and stale-lock recovery. | +| [`acquireFileLock`](sidecar-lock.md) | Cross-process file lock with retry and stale-lock recovery. | | [`FsSafeError`](errors.md) | Closed code union (with `policy` / `operational` category) you can branch on. | | [`pathScope()`](path-scope.md) | Lower-level absolute-path boundary helper; lives behind `@openclaw/fs-safe/advanced`. | diff --git a/docs/install.md b/docs/install.md index 91e7074..a814115 100644 --- a/docs/install.md +++ b/docs/install.md @@ -72,10 +72,11 @@ Use the main entry for the common surface, or the focused subpaths when you want | `@openclaw/fs-safe/atomic` | `replaceFileAtomic`, `writeTextAtomic`, `replaceDirectoryAtomic`, `movePathWithCopyFallback`. | | `@openclaw/fs-safe/temp` | `tempWorkspace`, `withTempWorkspace`, sync variants, `resolveSecureTempRoot`. | | `@openclaw/fs-safe/secure-file` | `readSecureFile` for pinned absolute file reads with permissions checks. | +| `@openclaw/fs-safe/file-lock` | `acquireFileLock`, `withFileLock`, `createFileLockManager`, and related lock types. | | `@openclaw/fs-safe/permissions` | POSIX mode and Windows ACL inspection/remediation helpers. | | `@openclaw/fs-safe/walk` | `walkDirectory`, `walkDirectorySync`, related types. Budget-bounded, not root-bounded. | | `@openclaw/fs-safe/archive` | `extractArchive`, `resolveArchiveKind`, limits, preflight helpers. | -| `@openclaw/fs-safe/advanced` | Lower-level composition helpers: path scopes, pinned open, root-file open, install paths, local-root readers, temp-file targets, sibling-temp writes, sidecar locks, regular-file helpers, `pathExists`, `withTimeout`, and related advanced types. This surface is less stable than the focused public subpaths. | +| `@openclaw/fs-safe/advanced` | Lower-level composition helpers: path scopes, root-file open, install paths, local-root readers, temp-file targets, sibling-temp writes, regular-file helpers, `pathExists`, `withTimeout`, and related advanced types. This surface is less stable than the focused public subpaths. | | `@openclaw/fs-safe/errors` | `FsSafeError`, `FsSafeErrorCode`. | | `@openclaw/fs-safe/types` | Shared types: `DirEntry`, `PathStat`, `BasePathOptions`, … | | `@openclaw/fs-safe/test-hooks` | Test-only hooks for injecting races. Active under `NODE_ENV=test`. | diff --git a/docs/json-store.md b/docs/json-store.md index 94f9dcc..9dacda7 100644 --- a/docs/json-store.md +++ b/docs/json-store.md @@ -4,7 +4,7 @@ convenience wrapper for `fileStore(...).json(...)`: a small read-modify-write handle around a single JSON file. It bakes in atomic writes, explicit fallback reads, and optional cross-process locking via -[`createSidecarLockManager`](sidecar-lock.md). +[`acquireFileLock`](sidecar-lock.md). ```ts import { jsonStore } from "@openclaw/fs-safe/store"; @@ -52,7 +52,7 @@ type JsonStoreOptions = { type JsonStoreLockOptions = { staleMs?: number; // default 30_000 timeoutMs?: number; // default 30_000 - retry?: SidecarLockRetryOptions; + retry?: FileLockRetryOptions; managerKey?: string; // default `fs-safe.json-store:` }; @@ -187,7 +187,7 @@ if (current.version !== CURRENT_VERSION) { | `jsonStore` | Raw helpers | |---|---| | Read-modify-write in one call (`update`). | Compose `readJsonIfExists` + `writeJson` yourself. | -| Optional cross-process lock with one flag. | Manage `withSidecarLock` yourself. | +| Optional cross-process lock with one flag. | Manage `withFileLock` yourself. | | Explicit `readOr` / `updateOr` fallbacks. | Caller handles `null` and clones. | | Mode/dirMode locked per store. | Per-call. | @@ -196,5 +196,5 @@ if (current.version !== CURRENT_VERSION) { ## See also - [JSON files](json.md) — the standalone helpers `jsonStore` is built on. -- [Sidecar lock](sidecar-lock.md) — the cross-process lock used when `lock: true`. +- [File lock](sidecar-lock.md) — the cross-process lock used when `lock: true`. - [File store](file-store.md) — the multi-file equivalent of this surface. diff --git a/docs/json.md b/docs/json.md index 77c2b7e..82932cc 100644 --- a/docs/json.md +++ b/docs/json.md @@ -155,4 +155,4 @@ const state = await readJsonIfExists("./state.json"); - [Atomic writes](atomic.md) — lower-level sibling-temp replacement helpers. - [Secret files](secret-file.md) — JSON-or-text writes with mode 0600 in mode 0700 dirs. - [Private file-store mode](private-file-store.md) — root-bounded JSON+text state stores. -- [Sidecar lock](sidecar-lock.md) — cross-process coordination. +- [File lock](sidecar-lock.md) — cross-process coordination. diff --git a/docs/pinned-open.md b/docs/pinned-open.md deleted file mode 100644 index 1f9467d..0000000 --- a/docs/pinned-open.md +++ /dev/null @@ -1,125 +0,0 @@ -# Pinned open - -`openPinnedFileSync()` is a low-level synchronous file open that re-pins identity after the open: it `open`s with `O_NOFOLLOW`, then `fstat`s and verifies the result matches what the path resolves to. It is the building block under `Root.open()` and a few of the other primitives — exposed for callers that want the same defense without going through `root()`. - -```ts -import { openPinnedFileSync } from "@openclaw/fs-safe/advanced"; -``` - -## Why a separate primitive - -`fs.openSync` plus `fs.statSync` plus `fs.realpathSync` is three syscalls and a swap window. `openPinnedFileSync` performs them in one helper that: - -1. Opens with `O_NOFOLLOW` (where the platform supports it). -2. `fstat`s the open fd. -3. Compares the fd's identity to the resolved path's identity. -4. Returns a typed result: `ok` plus the `fd`, or `not-ok` with a `reason`. - -If you're already using `root().open()`, you don't need this — but if you're building a new helper that needs a synchronous pinned open, this is the foundation. - -## Signature - -```ts -type PinnedOpenSyncFailureReason = "path" | "validation" | "io"; -type PinnedOpenSyncAllowedType = "file" | "directory"; - -type PinnedOpenSyncResult = - | { ok: true; fd: number; stat: Stats } - | { ok: false; reason: PinnedOpenSyncFailureReason; cause?: unknown }; - -function openPinnedFileSync(params: { - rootDir: string; // canonical root absolute path - filePath: string; // absolute target path - fs?: PinnedOpenSyncFs; // injectable for tests - allowedType?: PinnedOpenSyncAllowedType; // default "file" - flags?: number; // default O_RDONLY | O_NOFOLLOW -}): PinnedOpenSyncResult; -``` - -`PinnedOpenSyncFs` is a small subset of `node:fs`: - -```ts -type PinnedOpenSyncFs = Pick< - typeof fs, - "openSync" | "fstatSync" | "closeSync" | "realpathSync" | "lstatSync" ->; -``` - -## Reasons for failure - -- `"path"` — the resolved path does not stay inside `rootDir`, contains a symlink, or otherwise fails the lexical/canonical check. -- `"validation"` — the open succeeded but `fstat` does not match the path's identity (TOCTOU race), the file is not the allowed type, or it has unexpected `nlink`. -- `"io"` — the underlying `openSync` threw. Inspect `cause` for the original `NodeJS.ErrnoException`. - -The caller is responsible for closing the `fd` on success: - -```ts -import fs from "node:fs"; -import { openPinnedFileSync } from "@openclaw/fs-safe/advanced"; - -const r = openPinnedFileSync({ - rootDir: "/srv/workspace", - filePath: "/srv/workspace/state.json", -}); - -if (!r.ok) { - if (r.reason === "io") throw new Error("io error", { cause: r.cause }); - throw new Error(`pinned open refused: ${r.reason}`); -} - -try { - const buf = Buffer.alloc(r.stat.size); - fs.readSync(r.fd, buf, 0, buf.length, 0); - // ... -} finally { - fs.closeSync(r.fd); -} -``` - -## Allowed type - -By default the helper requires the result to be a regular file. Pass `allowedType: "directory"` when you want to pin-open a directory (for `fdopendir`-style use). Any other type triggers `"validation"`. - -## Test injection - -The optional `fs` field accepts a partial `node:fs` interface. Use it in unit tests to simulate a TOCTOU swap or a denied open: - -```ts -import { openPinnedFileSync } from "@openclaw/fs-safe/advanced"; - -const fakeFs = { - ...fs, - openSync: () => fakeFd, - fstatSync: () => ({ ...realStat, ino: differentIno } as Stats), - closeSync: () => {}, - realpathSync: () => filePath, - lstatSync: () => realStat, -}; -const r = openPinnedFileSync({ rootDir, filePath, fs: fakeFs }); -expect(r.ok).toBe(false); -expect(r.reason).toBe("validation"); -``` - -## Companions - -The same module exports a higher-level helper used by the rest of the library: - -```ts -import { - canUseRootFileOpen, - matchRootFileOpenFailure, - openRootFile, - openRootFileSync, -} from "@openclaw/fs-safe/advanced"; -``` - -- `openRootFileSync(params)` — `openPinnedFileSync` plus a richer result that distinguishes `"validation"` failures by sub-reason. -- `openRootFile(params)` — async variant, accepts a `signal: AbortSignal` for cancellation. -- `canUseRootFileOpen(io)` — checks whether the platform supports `O_NOFOLLOW`. Useful for falling back when the real defense is unavailable. -- `matchRootFileOpenFailure(failure, handlers)` — exhaustive switch helper for branching on the failure reason. - -## See also - -- [`root()`](root.md) — the high-level wrapper most callers want. -- [Reading](reading.md) — read pipeline that uses `openPinnedFileSync` internally. -- [Path helpers](path.md) — the lexical building blocks (`isPathInside`, `safeRealpathSync`). diff --git a/docs/reading.md b/docs/reading.md index 6e9676f..ca157bd 100644 --- a/docs/reading.md +++ b/docs/reading.md @@ -171,4 +171,4 @@ See [Errors](errors.md) for the full list. - [Writing](writing.md) — companion verbs for produce-side I/O. - [JSON files](json.md) — standalone strict/lenient JSON helpers. -- [Pinned open](pinned-open.md) — low-level synchronous pinned file open. +- [Secure file reads](secure-file.md) — pinned absolute file reads with permission checks. diff --git a/docs/sidecar-lock.md b/docs/sidecar-lock.md index 044ffaa..3d1c8e9 100644 --- a/docs/sidecar-lock.md +++ b/docs/sidecar-lock.md @@ -1,14 +1,12 @@ -# Sidecar lock +# File lock -`createSidecarLockManager(key)` provides a cross-process file lock with retry, stale-lock reclaim, and process-exit cleanup. The lock is implemented as a sidecar file (e.g. `state.json` ↔ `state.json.lock`) — only one acquirer can create the sidecar with `O_CREAT | O_EXCL` at a time. +`acquireFileLock()` and `withFileLock()` provide a cross-process file lock with retry, stale-lock reclaim, and process-exit cleanup. The lock is implemented as a sidecar file (e.g. `state.json` ↔ `state.json.lock`) — only one acquirer can create the sidecar with `O_CREAT | O_EXCL` at a time. ```ts -import { createSidecarLockManager } from "@openclaw/fs-safe/advanced"; +import { acquireFileLock } from "@openclaw/fs-safe/file-lock"; -const locks = createSidecarLockManager("snapshot"); - -const handle = await locks.acquire({ - targetPath: "/var/lib/app/state.json", +const handle = await acquireFileLock("/var/lib/app/state.json", { + managerKey: "snapshot", staleMs: 5 * 60_000, payload: async () => ({ pid: process.pid, host: os.hostname() }), }); @@ -25,29 +23,34 @@ The lock file sits next to the protected resource. If a process crashes mid-lock The library installs a `process.on("exit")` handler that releases all currently-held locks synchronously, so well-behaved exits leave no stale sidecars. Crashes still need the reclaim path. -## Manager API +## API ```ts -function createSidecarLockManager(key: string): { - acquire(options: SidecarLockAcquireOptions): Promise; - withLock(options: SidecarLockAcquireOptions, fn: () => Promise): Promise; - drain(): Promise; - reset(): void; - heldEntries(): SidecarLockHeldEntry[]; -}; +function acquireFileLock( + targetPath: string, + options: FileLockAcquireOptions, +): Promise; + +function withFileLock( + targetPath: string, + options: FileLockAcquireOptions, + fn: () => Promise, +): Promise; + +function createFileLockManager(key: string): FileLockManager; ``` -The `key` is a per-manager identifier used to keep state isolated across multiple managers in the same process. Use distinct keys for distinct lock domains (`"snapshot"`, `"compact"`, `"build"`). +`managerKey` is an optional identifier used to keep state isolated across multiple lock domains in the same process. Use distinct keys for distinct domains (`"snapshot"`, `"compact"`, `"build"`). If omitted, fs-safe derives one from the target path. ## Acquire options ```ts -type SidecarLockAcquireOptions> = { - targetPath: string; // the resource you want to protect +type FileLockAcquireOptions> = { + managerKey?: string; // optional in-process manager namespace lockPath?: string; // override; defaults to `${targetPath}.lock` staleMs: number; // how long until a held lock is considered stale timeoutMs?: number; // overall acquire deadline; default unbounded - retry?: SidecarLockRetryOptions; + retry?: FileLockRetryOptions; allowReentrant?: boolean; // if this process already holds it, increment a count instead of failing payload: () => TPayload | Promise; shouldReclaim?: (params: { @@ -61,7 +64,7 @@ type SidecarLockAcquireOptions> = { metadata?: Record; // attached to heldEntries() output for diagnostics }; -type SidecarLockRetryOptions = { +type FileLockRetryOptions = { retries?: number; // number of retry attempts after the first failure factor?: number; // exponential backoff factor (default 2) minTimeout?: number; // initial delay (ms) @@ -75,7 +78,7 @@ type SidecarLockRetryOptions = { ## Release handle ```ts -type SidecarLockHandle = { +type FileLockHandle = { lockPath: string; normalizedTargetPath: string; release: () => Promise; @@ -86,7 +89,10 @@ type SidecarLockHandle = { Always release in a `finally`: ```ts -const handle = await locks.acquire({ targetPath, staleMs: 60_000, payload: () => ({ pid: process.pid }) }); +const handle = await acquireFileLock(targetPath, { + staleMs: 60_000, + payload: () => ({ pid: process.pid }), +}); try { await doExclusiveWork(); } finally { @@ -96,29 +102,13 @@ try { If your process dies before `release()` runs and skips the exit handler, the next acquirer reclaims the lock once `staleMs` elapses (or your `shouldReclaim` returns true). -## `withLock` — common shape made one-liner +## `withFileLock` — common shape made one-liner ```ts -const result = await locks.withLock( - { targetPath: "/var/lib/app/state.json", staleMs: 30_000, payload: () => ({ pid: process.pid }) }, - async () => { - return await runCompaction(); - }, -); -``` - -Acquires, runs `fn`, releases regardless of success/failure. Returns the result of `fn`. - -## Top-level `withSidecarLock` - -When you don't need a long-lived manager, the standalone `withSidecarLock` creates one on the fly and runs your work under it: - -```ts -import { withSidecarLock } from "@openclaw/fs-safe/advanced"; - -const result = await withSidecarLock( +const result = await withFileLock( "/var/lib/app/state.json", { + managerKey: "compact", staleMs: 30_000, payload: () => ({ pid: process.pid, what: "compact" }), }, @@ -128,7 +118,26 @@ const result = await withSidecarLock( ); ``` -`WithSidecarLockOptions` is `SidecarLockAcquireOptions` minus `targetPath` (the first positional argument), plus an optional `managerKey` to share an existing manager namespace. +Acquires, runs `fn`, releases regardless of success/failure. Returns the result of `fn`. + +## Long-lived managers + +Most callers should use `acquireFileLock()` or `withFileLock()`. Use `createFileLockManager(key)` only when a long-lived service needs diagnostics or lifecycle control over locks it currently holds: + +```ts +const locks = createFileLockManager("session-writes"); +const handle = await locks.acquire(sessionPath, { + staleMs: 60_000, + payload: () => ({ pid: process.pid }), +}); + +for (const held of locks.heldEntries()) { + console.log(held.lockPath, held.acquiredAt); +} + +await handle.release(); +await locks.drain(); +``` ## Reclaim policy: `shouldReclaim` @@ -137,8 +146,7 @@ The default policy reclaims locks whose `acquiredAt` is older than `staleMs`. Pa ```ts import { kill } from "node:process"; -const handle = await locks.acquire({ - targetPath, +const handle = await acquireFileLock(targetPath, { staleMs: 60_000, payload: () => ({ pid: process.pid }), shouldReclaim: ({ payload, nowMs, staleMs }) => { @@ -157,26 +165,6 @@ const handle = await locks.acquire({ `heldByThisProcess` is true when this manager already holds the lock (relevant for the reentrant case). -## Diagnostics: `heldEntries` - -```ts -for (const entry of locks.heldEntries()) { - console.log(entry.normalizedTargetPath, "held since", new Date(entry.acquiredAt).toISOString()); - console.log(" metadata:", entry.metadata); -} -``` - -Each held entry exposes `forceRelease()` for admin tooling — use only when you're sure no real holder is still doing work. - -## `drain` and `reset` - -```ts -await locks.drain(); // async: release every currently-held lock with force -locks.reset(); // sync: same, but synchronous (use in tests / shutdown) -``` - -`drain()` is the right call during graceful shutdown when you want to clean up locks held by long-running tasks. `reset()` is the same operation in synchronous form, used by the built-in exit cleanup and useful in tests. - ## What sidecar locks defend against - **Two processes writing the same file at once.** `acquire` serializes the critical section. @@ -194,9 +182,9 @@ locks.reset(); // sync: same, but synchronous (use in tests / shutdo ### Compact under lock ```ts -await locks.withLock( +await withFileLock( + "/var/lib/app/db.sqlite", { - targetPath: "/var/lib/app/db.sqlite", staleMs: 30_000, payload: () => ({ pid: process.pid, what: "compact" }), }, @@ -210,8 +198,9 @@ await locks.withLock( ```ts try { - await locks.withLock( - { targetPath, staleMs: 30_000, retry: { retries: 0 }, payload: () => ({ pid: process.pid }) }, + await withFileLock( + targetPath, + { staleMs: 30_000, retry: { retries: 0 }, payload: () => ({ pid: process.pid }) }, async () => await work(), ); } catch (err) { @@ -222,9 +211,9 @@ try { ### Wait politely with backoff ```ts -await locks.withLock( +await withFileLock( + targetPath, { - targetPath, staleMs: 60_000, timeoutMs: 30_000, retry: { retries: 30, minTimeout: 100, maxTimeout: 5_000, factor: 1.7, randomize: true }, @@ -238,4 +227,3 @@ await locks.withLock( - [Atomic writes](atomic.md) — single-writer atomicity that often replaces the need for a lock entirely. - `createAsyncLock` from `@openclaw/fs-safe/advanced` — in-process serialization for a single Node process. -- [`createSidecarLockManager` source](https://github.com/openclaw/fs-safe/blob/main/src/sidecar-lock.ts). diff --git a/docs/temp.md b/docs/temp.md index 6b61fc6..bdcf873 100644 --- a/docs/temp.md +++ b/docs/temp.md @@ -273,4 +273,4 @@ it("processes a fixture", async () => { - [Atomic writes](atomic.md) — `replaceDirectoryAtomic` for whole-directory swaps. - [`root()`](root.md) — `fs.copyIn(rel, sourceAbs)` for moving files from a temp into a `Root`. -- [Sidecar lock](sidecar-lock.md) — when many processes share a temp tree. +- [File lock](sidecar-lock.md) — when many processes share a temp tree. diff --git a/docs/timing.md b/docs/timing.md index 9ae764c..7deeaaa 100644 --- a/docs/timing.md +++ b/docs/timing.md @@ -117,5 +117,5 @@ Better, gate it from the caller — `withTimeout(p, 0, ...)` returns the promise ## See also - [Archive extraction](archive.md) — `extractArchive` already takes `timeoutMs`. -- [Sidecar lock](sidecar-lock.md) — `retry.maxAttempts` × `retry.maxDelayMs` is a different form of bounded waiting. +- [File lock](sidecar-lock.md) — retry policy is a different form of bounded waiting. - [`AbortSignal.timeout`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal/timeout_static) — standard-library cancellation when you need to *abort*, not just *give up*. diff --git a/package.json b/package.json index 2efc8e8..5d35851 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,10 @@ "types": "./dist/secure-file.d.ts", "default": "./dist/secure-file.js" }, + "./file-lock": { + "types": "./dist/file-lock.d.ts", + "default": "./dist/file-lock.js" + }, "./walk": { "types": "./dist/walk.d.ts", "default": "./dist/walk.js" diff --git a/src/advanced.ts b/src/advanced.ts index 62d607d..627db9b 100644 --- a/src/advanced.ts +++ b/src/advanced.ts @@ -40,13 +40,6 @@ export { PATH_ALIAS_POLICIES, type PathAliasPolicy, } from "./path-policy.js"; -export { - openPinnedFileSync, - type PinnedOpenSyncAllowedType, - type PinnedOpenSyncFailureReason, - type PinnedOpenSyncFs, - type PinnedOpenSyncResult, -} from "./pinned-open.js"; export { openRootFile, openRootFileSync, @@ -89,7 +82,6 @@ export { assertNoSymlinkParentsSync, type AssertNoSymlinkParentsOptions, } from "./symlink-parents.js"; -export { createSidecarLockManager, withSidecarLock } from "./sidecar-lock.js"; export { movePathToTrash, type MovePathToTrashOptions } from "./trash.js"; export { withTimeout } from "./timing.js"; export { resolveHomeRelativePath } from "./home-dir.js"; diff --git a/src/file-lock.ts b/src/file-lock.ts new file mode 100644 index 0000000..65d6cdf --- /dev/null +++ b/src/file-lock.ts @@ -0,0 +1,83 @@ +import { + createSidecarLockManager, + type SidecarLockAcquireOptions, + type SidecarLockHandle, + type SidecarLockHeldEntry, + type SidecarLockRetryOptions, +} from "./sidecar-lock.js"; + +export type FileLockRetryOptions = SidecarLockRetryOptions; + +export type FileLockAcquireOptions> = Omit< + SidecarLockAcquireOptions, + "targetPath" +> & { + managerKey?: string; +}; + +export type FileLockHandle = SidecarLockHandle; +export type FileLockHeldEntry = SidecarLockHeldEntry; + +export type FileLockManager = { + acquire>( + targetPath: string, + options: FileLockAcquireOptions, + ): Promise; + withLock>( + targetPath: string, + options: FileLockAcquireOptions, + fn: () => Promise, + ): Promise; + drain(): Promise; + reset(): void; + heldEntries(): FileLockHeldEntry[]; +}; + +function resolveFileLockManagerKey(targetPath: string, managerKey?: string): string { + return managerKey ?? `fs-safe.file-lock:${targetPath}`; +} + +export async function acquireFileLock>( + targetPath: string, + options: FileLockAcquireOptions, +): Promise { + return await createFileLockManager(resolveFileLockManagerKey(targetPath, options.managerKey)) + .acquire(targetPath, options); +} + +export async function withFileLock>( + targetPath: string, + options: FileLockAcquireOptions, + fn: () => Promise, +): Promise { + return await createFileLockManager(resolveFileLockManagerKey(targetPath, options.managerKey)) + .withLock(targetPath, options, fn); +} + +export function createFileLockManager(key: string): FileLockManager { + const manager = createSidecarLockManager(key); + return { + acquire: async (targetPath, options) => { + const { managerKey: _managerKey, ...acquireOptions } = options; + return await manager.acquire({ ...acquireOptions, targetPath }); + }, + withLock: async (targetPath, options, fn) => { + const { managerKey: _managerKey, ...acquireOptions } = options; + return await manager.withLock({ ...acquireOptions, targetPath }, fn); + }, + drain: manager.drain, + reset: manager.reset, + heldEntries: manager.heldEntries, + }; +} + +export async function drainFileLockManagerForTest( + targetPath: string, + managerKey?: string, +): Promise { + await createFileLockManager(resolveFileLockManagerKey(targetPath, managerKey)).drain(); +} + +export function resetFileLockManagerForTest(targetPath: string, managerKey?: string): void { + createFileLockManager(resolveFileLockManagerKey(targetPath, managerKey)).reset(); +} diff --git a/src/json-document-store.ts b/src/json-document-store.ts index 4f3a9e0..1218de1 100644 --- a/src/json-document-store.ts +++ b/src/json-document-store.ts @@ -1,9 +1,10 @@ -import { createSidecarLockManager, type SidecarLockRetryOptions } from "./sidecar-lock.js"; +import type { FileLockRetryOptions } from "./file-lock.js"; +import { createSidecarLockManager } from "./sidecar-lock.js"; export type JsonStoreLockOptions = { staleMs?: number; timeoutMs?: number; - retry?: SidecarLockRetryOptions; + retry?: FileLockRetryOptions; managerKey?: string; }; diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index 45735d8..13963c6 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -25,7 +25,7 @@ import { pathScope } from "../src/root-paths.js"; import { replaceFileAtomic, replaceFileAtomicSync } from "../src/replace-file.js"; import { movePathWithCopyFallback } from "../src/move-path.js"; import { writeSiblingTempFile } from "../src/sibling-temp.js"; -import { createSidecarLockManager } from "../src/sidecar-lock.js"; +import { acquireFileLock } from "../src/file-lock.js"; import { fileStore, fileStoreSync } from "../src/file-store.js"; import { jsonStore } from "../src/json-store.js"; import { @@ -338,15 +338,14 @@ describe("private file store mode", () => { }); }); -describe("sidecar locks", () => { +describe("file locks", () => { it("supports await using cleanup", async () => { - const manager = createSidecarLockManager(`test-${Date.now()}-${Math.random()}`); const targetPath = path.join(root, "locked.txt"); let lockPath = ""; { - await using lock = await manager.acquire({ - targetPath, + await using lock = await acquireFileLock(targetPath, { + managerKey: `test-${Date.now()}-${Math.random()}`, staleMs: 60_000, payload: () => ({ owner: "test" }), }); From 837e37259f1dde21a753600d09e1386e55809a67 Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 01:52:28 +0100 Subject: [PATCH 19/20] chore: release fs-safe 0.1.0 --- CHANGELOG.md | 15 +++ package.json | 3 +- test/api-coverage.test.ts | 56 +++++++++- test/new-primitives.test.ts | 200 +++++++++++++++++++++++++++++++++++- vitest.config.ts | 4 +- 5 files changed, 273 insertions(+), 5 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..9cb8cfe --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,15 @@ +# Changelog + +## 0.1.0 - 2026-05-06 + +### Features + +- Added `root()` capability-style filesystem handles for root-bounded reads, writes, appends, moves, copies, directory listing, stat, mkdir, remove, JSON, streams, and existence checks. +- Added traversal, symlink, hardlink, alias, and post-open/post-write identity checks for untrusted relative paths. +- Added process-global Python helper configuration for stronger POSIX fd-relative mutation paths, with `auto`, `off`, and `require` modes. +- Added atomic file and directory replacement helpers with mode control, fsync options, retry handling, and copy-fallback behavior. +- Added JSON helpers, `fileStore()`, `jsonStore()`, private store mode, and file-backed temporary workspaces. +- Added secure absolute file reads, secret-file helpers, permissions inspection, Windows ACL helpers, and local-root readers. +- Added archive extraction and preflight helpers for ZIP/TAR with optional `jszip` and `tar` dependencies, size/count/path/link limits, and staged destination writes. +- Added file locks, async locks, bounded directory walking, install-path sanitizers, filename sanitization, regular-file helpers, trash moves, and advanced composition helpers. +- Added OpenClaw bypass-parity coverage, API coverage, a benchmark workflow, docs site generation, security docs, and coverage CI. diff --git a/package.json b/package.json index 5d35851..1b6d8c8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@openclaw/fs-safe", - "version": "0.0.0", + "version": "0.1.0", "description": "Capability-style filesystem roots for Node.js apps that handle untrusted relative paths.", "license": "MIT", "repository": { @@ -12,6 +12,7 @@ "dist/**/*.d.ts", "dist/**/*.d.ts.map", "README.md", + "CHANGELOG.md", "SECURITY.md", "LICENSE" ], diff --git a/test/api-coverage.test.ts b/test/api-coverage.test.ts index a048a13..460cfdc 100644 --- a/test/api-coverage.test.ts +++ b/test/api-coverage.test.ts @@ -4,7 +4,7 @@ import path from "node:path"; import { Readable } from "node:stream"; import JSZip from "jszip"; import * as tar from "tar"; -import { afterEach, describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { extractArchive } from "../src/archive.js"; import { loadZipArchiveWithPreflight, readZipCentralDirectoryEntryCount } from "../src/archive-zip-preflight.js"; import { createAsyncLock } from "../src/async-lock.js"; @@ -35,6 +35,7 @@ import { trySafeFileURLToPath, } from "../src/local-file-access.js"; import { resolveLocalPathFromRootsSync } from "../src/local-roots.js"; +import { movePathWithCopyFallback } from "../src/move-path.js"; import { hasNodeErrorCode, isNotFoundPathError, @@ -483,6 +484,59 @@ describe("ZIP preflight", () => { JSZip, ); }); + + it("handles non-Buffer zip views and malformed central directory metadata", async () => { + const emptyZip = new JSZip(); + const emptyBuffer = await emptyZip.generateAsync({ type: "nodebuffer" }); + expect(readZipCentralDirectoryEntryCount(new Uint8Array(emptyBuffer))).toBe(0); + + const zip = new JSZip(); + zip.file("commented.txt", "ok"); + zip.comment = "hello"; + const commented = await zip.generateAsync({ type: "nodebuffer" }); + expect(readZipCentralDirectoryEntryCount(commented)).toBe(1); + + const malformed = Buffer.from(commented); + const eocdOffset = malformed.lastIndexOf(Buffer.from([0x50, 0x4b, 0x05, 0x06])); + expect(eocdOffset).toBeGreaterThanOrEqual(0); + malformed.writeUInt32LE(0xffffffff, eocdOffset + 12); + malformed.writeUInt32LE(0xffffffff, eocdOffset + 16); + expect(readZipCentralDirectoryEntryCount(malformed)).toBe(1); + }); +}); + +describe("move fallback helper", () => { + it("renames on the same filesystem and falls back to copy/remove on EXDEV", async () => { + const root = await tempRoot("fs-safe-move-extra-"); + const from = path.join(root, "from.txt"); + const renamed = path.join(root, "renamed.txt"); + await fs.writeFile(from, "rename", "utf8"); + await movePathWithCopyFallback({ from, to: renamed }); + await expect(fs.readFile(renamed, "utf8")).resolves.toBe("rename"); + await expect(fs.stat(from)).rejects.toMatchObject({ code: "ENOENT" }); + + const crossDeviceFrom = path.join(root, "cross-device.txt"); + const crossDeviceTo = path.join(root, "copied.txt"); + await fs.writeFile(crossDeviceFrom, "copy", "utf8"); + const originalRename = fs.rename.bind(fs); + const renameSpy = vi.spyOn(fs, "rename").mockImplementation(async (source, dest) => { + if (source === crossDeviceFrom && dest === crossDeviceTo) { + const error = new Error("cross device") as NodeJS.ErrnoException; + error.code = "EXDEV"; + throw error; + } + return await originalRename(source, dest); + }); + + try { + await movePathWithCopyFallback({ from: crossDeviceFrom, to: crossDeviceTo }); + } finally { + renameSpy.mockRestore(); + } + + await expect(fs.readFile(crossDeviceTo, "utf8")).resolves.toBe("copy"); + await expect(fs.stat(crossDeviceFrom)).rejects.toMatchObject({ code: "ENOENT" }); + }); }); describe("archive extraction", () => { diff --git a/test/new-primitives.test.ts b/test/new-primitives.test.ts index 13963c6..8ea2407 100644 --- a/test/new-primitives.test.ts +++ b/test/new-primitives.test.ts @@ -25,13 +25,21 @@ import { pathScope } from "../src/root-paths.js"; import { replaceFileAtomic, replaceFileAtomicSync } from "../src/replace-file.js"; import { movePathWithCopyFallback } from "../src/move-path.js"; import { writeSiblingTempFile } from "../src/sibling-temp.js"; -import { acquireFileLock } from "../src/file-lock.js"; +import { acquireFileLock, createFileLockManager, withFileLock } from "../src/file-lock.js"; import { fileStore, fileStoreSync } from "../src/file-store.js"; import { jsonStore } from "../src/json-store.js"; import { createIcaclsResetCommand, + formatIcaclsResetCommand, + formatPermissionDetail, + formatPermissionRemediation, + formatWindowsAclSummary, + inspectPathPermissions, inspectWindowsAcl, + modeBits, parseIcaclsOutput, + resolveWindowsUserPrincipal, + summarizeWindowsAcl, } from "../src/permissions.js"; import { readSecureFile } from "../src/secure-file.js"; import { walkDirectory, walkDirectorySync } from "../src/walk.js"; @@ -216,6 +224,81 @@ describe("secure file reads", () => { }); }); + it("covers symlink, directory, size, and trusted-dir secure read branches", async () => { + const target = path.join(root, "target.txt"); + const link = path.join(root, "link.txt"); + const trusted = path.join(root, "trusted"); + const outsideTrusted = path.join(root, "outside-trusted"); + await fs.writeFile(target, "secret", { mode: 0o600 }); + await fs.symlink(target, link); + await fs.mkdir(trusted); + await fs.mkdir(outsideTrusted); + + await expect(readSecureFile({ filePath: "relative.txt" })).rejects.toMatchObject({ + code: "invalid-path", + }); + await expect(readSecureFile({ filePath: root })).rejects.toMatchObject({ code: "not-file" }); + await expect( + readSecureFile({ + filePath: link, + trust: { allowSymlink: true, trustedDirs: [outsideTrusted] }, + permissions: { allowInsecure: true }, + }), + ).rejects.toMatchObject({ code: "outside-workspace" }); + + const result = await readSecureFile({ + filePath: link, + trust: { allowSymlink: true, trustedDirs: [root] }, + permissions: { allowInsecure: true }, + io: { maxBytes: 100, timeoutMs: 1000 }, + }); + expect(result.buffer.toString("utf8")).toBe("secret"); + + await expect( + readSecureFile({ + filePath: target, + permissions: { allowInsecure: true }, + io: { maxBytes: 2 }, + }), + ).rejects.toMatchObject({ code: "too-large" }); + }); + + it("uses Windows ACL permission checks for secure reads when requested", async () => { + const filePath = path.join(root, "windows-secret.txt"); + await fs.writeFile(filePath, "secret", { mode: 0o600 }); + const exec = vi.fn().mockResolvedValue({ + stdout: "*S-1-5-18:(F)\n", + stderr: "", + }); + + const result = await readSecureFile({ + filePath, + inject: { platform: "win32", exec }, + permissions: { allowReadableByOthers: true }, + }); + expect(result.buffer.toString("utf8")).toBe("secret"); + expect(result.permissions?.source).toBe("windows-acl"); + + const unsafeExec = vi.fn().mockResolvedValue({ + stdout: "Everyone:(R)\n", + stderr: "", + }); + await expect( + readSecureFile({ + filePath, + inject: { platform: "win32", exec: unsafeExec }, + }), + ).rejects.toMatchObject({ code: "insecure-permissions" }); + + const failedExec = vi.fn().mockRejectedValue(new Error("icacls failed")); + await expect( + readSecureFile({ + filePath, + inject: { platform: "win32", exec: failedExec }, + }), + ).rejects.toMatchObject({ code: "permission-unverified" }); + }); + it("parses icacls output into ACL entries", () => { const entries = parseIcaclsOutput( String.raw`C:\Users\me\secret.txt *S-1-5-18:(F) @@ -263,6 +346,81 @@ describe("secure file reads", () => { }); expect(command?.command).toBe("C:\\Windows\\System32\\icacls.exe"); }); + + it("covers permission formatting and ACL classification helpers", async () => { + const missing = await inspectPathPermissions(path.join(root, "missing.txt")); + expect(missing.ok).toBe(false); + + const target = path.join(root, "acl-target.txt"); + const link = path.join(root, "acl-link.txt"); + await fs.writeFile(target, "ok", { mode: 0o640 }); + await fs.symlink(target, link); + const posix = await inspectPathPermissions(link, { platform: "linux" }); + expect(posix.isSymlink).toBe(true); + expect(formatPermissionDetail(target, posix)).toContain("mode="); + expect( + formatPermissionRemediation({ + targetPath: target, + perms: posix, + isDir: false, + posixMode: 0o600, + }), + ).toBe(`chmod 600 ${target}`); + + const entries = parseIcaclsOutput( + [ + `"C:\\Secrets\\token.txt" DOMAIN\\me:(F)`, + "Everyone:(R)", + "BUILTIN\\Users:(M)", + "*S-1-5-21-123:(R)", + "Denied:(DENY)(F)", + "Successfully processed 1 files; Failed processing 0 files", + ].join("\n"), + String.raw`C:\Secrets\token.txt`, + ); + const summary = summarizeWindowsAcl(entries, { + USERDOMAIN: "DOMAIN", + USERNAME: "me", + USERSID: "S-1-5-21-999", + }); + expect(summary.trusted.map((entry) => entry.principal)).toContain("DOMAIN\\me"); + expect(summary.untrustedWorld.some((entry) => entry.principal === "Everyone")).toBe(true); + expect(summary.untrustedGroup.some((entry) => entry.principal === "*S-1-5-21-123")).toBe(true); + expect(formatWindowsAclSummary({ ok: true, entries, ...summary })).toContain("Everyone"); + expect(formatWindowsAclSummary({ ok: false, entries: [], trusted: [], untrustedWorld: [], untrustedGroup: [] })) + .toBe("unknown"); + expect(resolveWindowsUserPrincipal({ USERDOMAIN: "DOMAIN", USERNAME: "me" })).toBe( + "DOMAIN\\me", + ); + expect(resolveWindowsUserPrincipal({}, () => ({ username: "fallback" }))).toBe("fallback"); + expect(createIcaclsResetCommand(target, { isDir: true, userInfo: () => ({}) })).toBeNull(); + expect( + formatIcaclsResetCommand(String.raw`C:\Secrets\token.txt`, { + isDir: true, + env: { SystemRoot: "D:\\Windows", USERNAME: "me" }, + }), + ).toContain('"me:(OI)(CI)F"'); + expect(modeBits(0o100777)).toBe(0o777); + }); + + it("resolves the current user SID when ACL output only contains an unknown SID", async () => { + const target = String.raw`C:\Secrets\token.txt`; + const exec = vi + .fn() + .mockResolvedValueOnce({ + stdout: `${target} *S-1-5-21-42:(F)\nEveryone:(R)\n`, + stderr: "", + }) + .mockResolvedValueOnce({ + stdout: '"USER","SID"\n"DOMAIN\\me","S-1-5-21-42"\n', + stderr: "", + }); + + const result = await inspectWindowsAcl(target, { exec, env: { SystemRoot: "C:\\Windows" } }); + expect(result.ok).toBe(true); + expect(result.trusted.some((entry) => entry.principal === "*S-1-5-21-42")).toBe(true); + expect(exec).toHaveBeenCalledTimes(2); + }); }); describe("directory walking", () => { @@ -355,6 +513,46 @@ describe("file locks", () => { await expect(fs.stat(lockPath)).rejects.toMatchObject({ code: "ENOENT" }); }); + + it("supports manager lifecycle and top-level withFileLock", async () => { + const targetPath = path.join(root, "managed-lock.txt"); + const manager = createFileLockManager(`manager-${Date.now()}-${Math.random()}`); + + const lock = await manager.acquire(targetPath, { + staleMs: 60_000, + allowReentrant: true, + metadata: { suite: "new-primitives" }, + payload: () => ({ owner: "manager" }), + }); + const reentrant = await manager.acquire(targetPath, { + staleMs: 60_000, + allowReentrant: true, + payload: () => ({ owner: "manager" }), + }); + expect(manager.heldEntries()).toHaveLength(1); + expect(manager.heldEntries()[0]?.metadata).toEqual({ suite: "new-primitives" }); + await reentrant.release(); + await lock.release(); + expect(manager.heldEntries()).toEqual([]); + + await expect( + manager.withLock( + targetPath, + { staleMs: 60_000, payload: () => ({ owner: "manager" }) }, + async () => "ok", + ), + ).resolves.toBe("ok"); + + await expect( + withFileLock( + path.join(root, "top-level-lock.txt"), + { staleMs: 60_000, payload: () => ({ owner: "top-level" }) }, + async () => "locked", + ), + ).resolves.toBe("locked"); + manager.reset(); + await manager.drain(); + }); }); describe("regular file append", () => { diff --git a/vitest.config.ts b/vitest.config.ts index 56b111d..8c70aeb 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -22,9 +22,9 @@ export default defineConfig({ "src/test-hooks.ts", ], thresholds: { - lines: 82, + lines: 85, functions: 94, - statements: 82, + statements: 85, branches: 76, }, }, From 621d643d379533791ab5108e1cf639fc7d8e2c8c Mon Sep 17 00:00:00 2001 From: Peter Steinberger Date: Wed, 6 May 2026 01:53:46 +0100 Subject: [PATCH 20/20] chore: normalize npm package metadata --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1b6d8c8..8bd26f8 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "license": "MIT", "repository": { "type": "git", - "url": "https://github.com/openclaw/fs-safe.git" + "url": "git+https://github.com/openclaw/fs-safe.git" }, "files": [ "dist/**/*.js",