#!/usr/bin/env node import fs from "node:fs"; import path from "node:path"; import { execFile, execFileSync } from "node:child_process"; import { promisify } from "node:util"; const execFileAsync = promisify(execFile); import matter from "gray-matter"; import { ignoredDocDirs, ignoredDocFiles, localeLabels, mintlifyLocaleToDir, rtlLocales } from "./config.mjs"; import { siteCss, siteJs } from "./assets.mjs"; import { createMarkdownRenderer, renderMdxish } from "./mdx-ish.mjs"; import { renderPageOgSvg } from "./og-card-template.mjs"; const root = process.cwd(); const docsDir = path.join(root, "docs"); const siteAssetsDir = path.join(root, "scripts", "docs-site"); const outDir = path.join(root, "dist", "docs-site"); const config = JSON.parse(fs.readFileSync(path.join(docsDir, "docs.json"), "utf8")); const md = createMarkdownRenderer(); const basePath = normalizeBasePath(process.env.DOCS_SITE_BASE_PATH ?? ""); const legacyBasePath = normalizeBasePath(process.env.DOCS_SITE_LEGACY_BASE_PATH ?? "/docs"); const canonicalOrigin = (process.env.DOCS_SITE_CANONICAL_ORIGIN ?? (process.env.DOCS_SITE_CNAME ? `https://${process.env.DOCS_SITE_CNAME}` : "")).replace(/\/$/, ""); const ogImagePath = "/og-card.png"; const renderedPageOgCards = new Set(); const rsvgAvailable = checkRsvg(); const chatApiUrl = process.env.DOCS_SITE_CHAT_API_URL ?? "/ask-molty/api/chat"; const assetVersion = buildAssetVersion(); fs.rmSync(outDir, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); fs.mkdirSync(outDir, { recursive: true }); const locales = buildLocales(config); const pages = collectPages(locales); const pageByKey = new Map(pages.map((page) => [pageKey(page.locale, page.slug), page])); const navByLocale = new Map(locales.map((locale) => [locale.code, buildNav(locale)])); const localeFlags = { en: "๐Ÿ‡บ๐Ÿ‡ธ", "zh-CN": "๐Ÿ‡จ๐Ÿ‡ณ", "zh-TW": "๐Ÿ‡จ๐Ÿ‡ณ", "ja-JP": "๐Ÿ‡ฏ๐Ÿ‡ต", es: "๐Ÿ‡ช๐Ÿ‡ธ", "pt-BR": "๐Ÿ‡ง๐Ÿ‡ท", ko: "๐Ÿ‡ฐ๐Ÿ‡ท", de: "๐Ÿ‡ฉ๐Ÿ‡ช", fr: "๐Ÿ‡ซ๐Ÿ‡ท", ar: "๐Ÿ‡ธ๐Ÿ‡ฆ", it: "๐Ÿ‡ฎ๐Ÿ‡น", vi: "๐Ÿ‡ป๐Ÿ‡ณ", nl: "๐Ÿ‡ณ๐Ÿ‡ฑ", tr: "๐Ÿ‡น๐Ÿ‡ท", uk: "๐Ÿ‡บ๐Ÿ‡ฆ", id: "๐Ÿ‡ฎ๐Ÿ‡ฉ", pl: "๐Ÿ‡ต๐Ÿ‡ฑ", fa: "๐Ÿ‡ฎ๐Ÿ‡ท", th: "๐Ÿ‡น๐Ÿ‡ญ" }; const localePickerLabels = { "pt-BR": "Portuguรชs (BR)" }; copyPublicFiles(); await renderPageOgCards(); for (const page of pages) writePage(page); writeLlmsIndex(); writeRobotsTxt(); writeSitemap(); writeRedirects(); writeStaticAssets(); console.log(`built ${pages.length} pages in ${path.relative(root, outDir)}`); function buildLocales(docsConfig) { const ordered = []; for (const entry of docsConfig.navigation?.languages ?? []) { const code = mintlifyLocaleToDir[entry.language] ?? entry.language; ordered.push({ code, source: entry, root: code === "en" }); } for (const dirent of fs.readdirSync(docsDir, { withFileTypes: true })) { if (!dirent.isDirectory() || ignoredDocDirs.has(dirent.name)) continue; if (localeLabels[dirent.name] && !ordered.some((locale) => locale.code === dirent.name)) { ordered.push({ code: dirent.name, source: ordered[0]?.source, root: false }); } } return ordered.filter((locale) => locale.root || fs.existsSync(path.join(docsDir, locale.code))); } function collectPages(localeList) { const result = []; for (const locale of localeList) { const base = locale.root ? docsDir : path.join(docsDir, locale.code); for (const file of walkDocs(base)) { const rel = path.relative(base, file).replaceAll(path.sep, "/"); if (ignoredDocFiles.has(rel) || rel.endsWith("/AGENTS.md")) continue; const raw = fs.readFileSync(file, "utf8"); const parsed = matter(raw); const slug = fileSlug(rel); const title = parsed.data.title || firstHeading(parsed.content) || titleize(path.basename(slug)); result.push({ locale: locale.code, dir: locale.root ? "" : locale.code, slug, file, rel, raw, title, summary: parsed.data.summary ?? "", readWhen: parsed.data.read_when ?? [], body: parsed.content }); } } return result; } function walkDocs(dir) { if (!fs.existsSync(dir)) return []; return fs.readdirSync(dir, { withFileTypes: true }).flatMap((entry) => { if (entry.name.startsWith(".")) return []; const full = path.join(dir, entry.name); if (entry.isDirectory()) return ignoredDocDirs.has(entry.name) ? [] : walkDocs(full); return /\.(md|mdx)$/.test(entry.name) ? [full] : []; }); } function buildNav(locale) { const source = locale.source ?? locales[0]?.source; const tabs = (source?.tabs ?? []).map((tab) => ({ title: tab.tab, groups: (tab.groups ?? []).map((group) => navGroup(locale.code, group)).filter(Boolean) })); return tabs.filter((tab) => tab.groups.length); } function navGroup(locale, group) { const pages = flattenPages(locale, group.pages ?? []); return pages.length ? { title: group.group ?? "Docs", pages } : null; } function flattenPages(locale, entries) { const output = []; for (const entry of entries) { if (typeof entry === "string") { const page = pageByKey.get(pageKey(locale, navEntrySlug(locale, entry))); if (page) output.push(page); } else if (entry?.pages) { const nested = flattenPages(locale, entry.pages); if (nested.length) output.push({ group: entry.group ?? "More", pages: nested }); } } return output; } function navEntrySlug(locale, entry) { const slug = normalizeSlug(entry); return slug.startsWith(`${locale}/`) ? normalizeSlug(slug.slice(locale.length + 1)) : slug; } function writePage(page) { const nav = navByLocale.get(page.locale) ?? []; const flat = flattenNav(nav); const activeIndex = flat.findIndex((item) => item.slug === page.slug); const activeTab = activeTabTitle(nav, page.slug); const prev = activeIndex > 0 ? flat[activeIndex - 1] : null; const next = activeIndex >= 0 && activeIndex < flat.length - 1 ? flat[activeIndex + 1] : null; const html = rewriteInternalUrls(renderMdxish(page.body, md), page.locale); const toc = tableOfContents(html); const outPath = path.join(outDir, pageRoute(page).replace(/^\//, ""), "index.html"); fs.mkdirSync(path.dirname(outPath), { recursive: true }); fs.writeFileSync(outPath, layout({ page, nav, activeTab, html, toc, prev, next }), "utf8"); const mdPath = path.join(outDir, pageMarkdownRoute(page).replace(/^\//, "")); fs.mkdirSync(path.dirname(mdPath), { recursive: true }); fs.writeFileSync(mdPath, page.raw, "utf8"); } function layout({ page, nav, activeTab, html, toc, prev, next }) { const lang = htmlLang(page.locale); const dir = rtlLocales.has(page.locale) ? "rtl" : "ltr"; const title = `${page.title} - ${config.name}`; const description = page.summary || config.description || ""; const ogTitle = page.slug === "index" ? config.name : `${page.title} ยท ${config.name}`; const canonicalUrl = canonicalOrigin ? `${canonicalOrigin}${pageRoute(page)}` : ""; const pageOgPath = page.locale === "en" && renderedPageOgCards.has(page.slug) ? `/og/${page.slug}.png` : ogImagePath; const ogImageUrl = canonicalOrigin ? `${canonicalOrigin}${pageOgPath}` : publicPath(pageOgPath); return ` ${escapeHtml(title)} ${canonicalUrl ? `` : ""} ${canonicalUrl ? `` : ""} ${siteHeader(page, nav, activeTab)}
${sidebar(page, nav, activeTab)}

${escapeHtml(groupForPage(nav, page.slug) ?? activeTab)}

${escapeHtml(page.title)}

${html}
${pager(prev, next)}
${tocHtml(toc)}
${searchModal()} ${chatWidget()} `; } function assetUrl(file) { return `${publicPath(file)}?v=${encodeURIComponent(assetVersion)}`; } function siteHeader(page, nav, activeTab) { const tabs = nav.map((tab) => { const href = pageUrl(firstPage(tab)); const active = tab.title === activeTab ? " active" : ""; return `${escapeHtml(tab.title)}`; }).join(""); return `
${languagePicker(page)}
`; } function sidebar(page, nav, activeTab) { const groups = (nav.find((tab) => tab.title === activeTab) ?? nav[0])?.groups ?? []; return ``; } function languagePicker(page) { const current = locales.find((locale) => locale.code === page.locale) ?? locales[0]; const currentLabel = localeDisplayName(current.code); const currentFlag = localeFlag(current.code); const options = locales.map((locale) => { const active = locale.code === page.locale; return `${escapeHtml(localeDisplayName(locale.code))}`; }).join(""); return `
${options}
`; } function localeFlag(code) { return localeFlags[code] ?? "๐ŸŒ"; } function localeDisplayName(code) { return localePickerLabels[code] ?? localeLabels[code] ?? code; } function topLink(label, href, iconName) { return `${icon(iconName)}${escapeHtml(label)}`; } function icon(name) { const attrs = `class="icon icon-${escapeAttr(name)}" width="18" height="18" viewBox="0 0 24 24" aria-hidden="true" focusable="false"`; if (name === "github") return ``; if (name === "discord") return ``; const paths = { "search": '', "package": '', "moon": '', "chevron-down": '', }; return `${paths[name] ?? ""}`; } function navGroupHtml(activePage, group) { return `

${escapeHtml(group.title)}

${group.pages.map((entry) => { if (entry.group) return `

${escapeHtml(entry.group)}

${entry.pages.map((page) => navLink(activePage, page)).join("")}
`; return navLink(activePage, entry); }).join("")}
`; } function navLink(activePage, page) { const active = activePage.locale === page.locale && activePage.slug === page.slug ? " active" : ""; return `${escapeHtml(page.title)}`; } function tableOfContents(html) { return [...html.matchAll(/]*\bid="([^"]+)"[^>]*>([\s\S]*?)<\/h\1>/g)] .map((m) => ({ level: Number(m[1]), id: m[2], title: decodeHtmlEntities(stripTags(m[3]).replace(/^#\s*/, "")) })) .slice(0, 24); } function tocHtml(items) { if (!items.length) return ""; return ``; } function pager(prev, next) { if (!prev && !next) return ""; return ``; } function searchModal() { return `
`; } function writeLlmsIndex() { const origin = docsOrigin(); const lines = [ `# ${config.name}`, "", config.description ?? "OpenClaw documentation.", "", "> Use this file as a lightweight map of the OpenClaw documentation. Fetch individual pages as Markdown with `.md` URLs or `Accept: text/markdown`; OpenClaw does not publish a full-site LLM corpus.", "", "## Agent Resources", "", `- [Markdown page export](${origin}/start/getting-started.md): Append \`.md\` to any docs page URL for clean Markdown.`, `- [Sitemap](${origin}/sitemap.xml): Search crawler URL index.`, `- [Robots policy](${origin}/robots.txt): Bot and crawler policy.`, "", "## Documentation Index", "", ]; for (const page of englishDocsPages()) { const summary = page.summary ? `: ${stripMdxForLlms(page.summary).replace(/\s+/g, " ").trim()}` : ""; lines.push(`- [${page.title}](${origin}${pageRoute(page)})${summary}`); } const content = `${lines.join("\n")}\n`; fs.writeFileSync(path.join(outDir, "llms.txt"), content, "utf8"); fs.writeFileSync(path.join(outDir, "llm.txt"), content, "utf8"); const wellKnownDir = path.join(outDir, ".well-known"); fs.mkdirSync(wellKnownDir, { recursive: true }); fs.writeFileSync(path.join(wellKnownDir, "llms.txt"), content, "utf8"); } function writeRobotsTxt() { const origin = docsOrigin(); const botAgents = [ "GPTBot", "OAI-SearchBot", "ChatGPT-User", "ClaudeBot", "Claude-User", "PerplexityBot", "Perplexity-User", "Google-Extended", ]; const lines = [ "# OpenClaw documentation crawler policy", "# Human docs are HTML. Agent-optimized docs are available as Markdown via .md URLs or Accept: text/markdown.", "# No full-site LLM corpus is published; use /llms.txt as the index and fetch only the pages you need.", "", "User-agent: *", "Allow: /", "Disallow: /ask-molty/api/", "Disallow: /llms-full.txt", "Disallow: /.well-known/llms-full.txt", "", ]; for (const agent of botAgents) { lines.push(`User-agent: ${agent}`); lines.push("Allow: /"); lines.push("Disallow: /ask-molty/api/"); lines.push("Disallow: /llms-full.txt"); lines.push("Disallow: /.well-known/llms-full.txt"); lines.push(""); } lines.push(`Sitemap: ${origin}/sitemap.xml`); lines.push(`LLMS: ${origin}/llms.txt`); lines.push(""); fs.writeFileSync(path.join(outDir, "robots.txt"), lines.join("\n"), "utf8"); } function writeSitemap() { const origin = docsOrigin(); const urls = [...new Set(pages.map((page) => `${origin}${pageRoute(page)}`))] .sort((a, b) => a.localeCompare(b)); const xml = [ '', '', ...urls.map((url) => ` ${escapeXml(url)}`), "", "", ].join("\n"); fs.writeFileSync(path.join(outDir, "sitemap.xml"), xml, "utf8"); } function englishDocsPages() { return pages .filter((page) => page.locale === "en" && !localeLabels[page.rel.split("/")[0]]) .sort((a, b) => a.slug.localeCompare(b.slug)); } function docsOrigin() { return (canonicalOrigin || "https://documentation.openclaw.ai").replace(/\/$/, ""); } function chatWidget() { if (!chatApiUrl) return ""; return `
`; } function writeRedirects() { for (const redirect of config.redirects ?? []) { const source = cleanPath(redirect.source); const dest = cleanPath(redirect.destination); writeRedirectFile(source, publicPath(dest)); for (const prefix of new Set([basePath, legacyBasePath].filter(Boolean))) { writeRedirectFile(`${prefix}${source}`, publicPath(dest)); } } } function writeRedirectFile(source, dest) { const target = path.join(outDir, source.replace(/^\//, ""), "index.html"); if (fs.existsSync(target)) return; fs.mkdirSync(path.dirname(target), { recursive: true }); fs.writeFileSync(target, redirectHtml(dest), "utf8"); } function redirectHtml(dest) { return `Redirecting - ${escapeHtml(config.name)}Redirecting`; } function stripMdxForLlms(input) { return input .replace(/^import\s+.+?;?\s*$/gm, "") .replace(/<([A-Z][A-Za-z0-9_.-]*)([^>]*)\/>/g, (_, name, attrs) => componentLabel(name, attrs)) .replace(/<([A-Z][A-Za-z0-9_.-]*)([^>]*)>/g, (_, name, attrs) => componentLabel(name, attrs)) .replace(/<\/[A-Z][A-Za-z0-9_.-]*>/g, "") .replace(/\n{3,}/g, "\n\n"); } function componentLabel(name, attrs) { const parsed = Object.fromEntries([...String(attrs).matchAll(/([A-Za-z0-9_-]+)=(?:"([^"]*)"|'([^']*)')/g)].map((match) => [match[1], match[2] ?? match[3] ?? ""])); const label = parsed.title ?? parsed.name ?? parsed.href ?? ""; return label ? `\n${label}\n` : `\n${name}\n`; } async function renderPageOgCards() { if (!rsvgAvailable) { console.warn("rsvg-convert unavailable; skipping per-page OG cards (using base og-card.png)"); return; } const enNav = navByLocale.get("en") ?? []; const navSlugs = collectNavSlugs(enNav); const ogDir = path.join(outDir, "og"); const targets = pages.filter((page) => page.locale === "en" && page.slug !== "index" && navSlugs.has(page.slug) ); const start = Date.now(); const concurrency = Math.max(2, Math.min(8, Number(process.env.DOCS_SITE_OG_CONCURRENCY) || 6)); let cursor = 0; let count = 0; await Promise.all(Array.from({ length: concurrency }, async () => { while (cursor < targets.length) { const page = targets[cursor++]; const kicker = groupForPage(enNav, page.slug) ?? activeTabTitle(enNav, page.slug) ?? config.name; const svg = renderPageOgSvg({ title: page.title, kicker, summary: page.summary }); const outFile = path.join(ogDir, `${page.slug}.png`); fs.mkdirSync(path.dirname(outFile), { recursive: true }); try { const child = execFile("rsvg-convert", ["-w", "1200", "-h", "630", "-o", outFile]); child.stdin.end(svg); await new Promise((resolve, reject) => { child.on("error", reject); child.on("close", (code) => code === 0 ? resolve() : reject(new Error(`rsvg-convert exit ${code}`))); }); renderedPageOgCards.add(page.slug); count++; } catch (err) { console.warn(`og card render failed for ${page.slug}: ${err.message}`); } } })); console.log(`rendered ${count}/${targets.length} per-page og cards in ${Date.now() - start}ms`); } function collectNavSlugs(nav) { const slugs = new Set(); for (const tab of nav) { for (const group of tab.groups ?? []) { for (const entry of group.pages ?? []) { if (entry.group) for (const sub of entry.pages ?? []) slugs.add(sub.slug); else if (entry.slug) slugs.add(entry.slug); } } } return slugs; } function checkRsvg() { try { execFileSync("rsvg-convert", ["--version"], { stdio: "ignore" }); return true; } catch { return false; } } function buildAssetVersion() { const fromEnv = process.env.GITHUB_SHA || process.env.DOCS_SITE_ASSET_VERSION; if (fromEnv) return fromEnv.slice(0, 12); try { return execFileSync("git", ["rev-parse", "--short=12", "HEAD"], { encoding: "utf8" }).trim(); } catch { return "dev"; } } function writeStaticAssets() { const assetsDir = path.join(outDir, "assets"); fs.mkdirSync(assetsDir, { recursive: true }); fs.writeFileSync(path.join(assetsDir, "docs-site.css"), siteCss(), "utf8"); fs.writeFileSync(path.join(assetsDir, "docs-site.js"), siteJs(), "utf8"); fs.writeFileSync(path.join(outDir, ".nojekyll"), "", "utf8"); for (const file of ["og-card.png", "og-card.svg"]) { const source = path.join(siteAssetsDir, file); if (fs.existsSync(source)) fs.copyFileSync(source, path.join(outDir, file)); } if (process.env.DOCS_SITE_CNAME) { fs.writeFileSync(path.join(outDir, "CNAME"), `${process.env.DOCS_SITE_CNAME}\n`, "utf8"); } } function copyPublicFiles() { copyDir(path.join(docsDir, "assets"), path.join(outDir, "assets")); for (const entry of fs.readdirSync(docsDir, { withFileTypes: true })) { if (entry.isFile() && !ignoredDocFiles.has(entry.name) && !/\.(md|mdx|json)$/.test(entry.name)) { fs.copyFileSync(path.join(docsDir, entry.name), path.join(outDir, entry.name)); } } } function copyDir(source, dest) { if (!fs.existsSync(source)) return; fs.cpSync(source, dest, { recursive: true }); } function activeTabTitle(nav, slug) { return nav.find((tab) => flattenNav([tab]).some((page) => page.slug === slug))?.title ?? nav[0]?.title ?? ""; } function groupForPage(nav, slug) { for (const tab of nav) { for (const group of tab.groups) { if (group.pages.some((entry) => entry.group ? entry.pages.some((page) => page.slug === slug) : entry.slug === slug)) { return group.title; } } } } function flattenNav(nav) { return nav.flatMap((tab) => tab.groups.flatMap((group) => group.pages.flatMap((entry) => entry.group ? entry.pages : [entry]))); } function firstPage(tab) { for (const group of tab.groups) { for (const entry of group.pages) return entry.group ? entry.pages[0] : entry; } return pages[0]; } function localeUrlForSlug(locale, slug) { return pageByKey.has(pageKey(locale, slug)) ? pageUrl(pageByKey.get(pageKey(locale, slug))) : publicPath(locale === "en" ? "/" : `/${locale}/`); } function pageUrl(page) { return publicPath(pageRoute(page)); } function pageRoute(page) { const prefix = page.locale === "en" ? "" : `/${page.locale}`; return page.slug === "index" ? (prefix || "/") : `${prefix}/${page.slug}`; } function pageMarkdownRoute(page) { const prefix = page.locale === "en" ? "" : `/${page.locale}`; return page.slug === "index" ? `${prefix || ""}/index.md` : `${prefix}/${page.slug}.md`; } function rewriteInternalUrls(html, locale) { return html.replace(/\b(href|src)="\/([^"#?]*)([#?][^"]*)?"/g, (match, attr, target, suffix = "") => { if (attr === "src") return `${attr}="${publicPath(`/${target}`)}${suffix}"`; if (!target || target.startsWith("assets/") || target.startsWith("pagefind/")) { return `${attr}="${publicPath(`/${target}`)}${suffix}"`; } const segments = target.replace(/\/$/, "").split("/"); const maybeLocale = segments[0]; if (pageByKey.has(pageKey(maybeLocale, normalizeSlug(segments.slice(1).join("/") || "index")))) { return `${attr}="${pageUrl(pageByKey.get(pageKey(maybeLocale, normalizeSlug(segments.slice(1).join("/") || "index"))))}${suffix}"`; } const slug = normalizeSlug(target.replace(/\/$/, "")); const page = pageByKey.get(pageKey(locale, slug)) ?? pageByKey.get(pageKey("en", slug)); return page ? `${attr}="${pageUrl(page)}${suffix}"` : `${attr}="${publicPath(`/${target}`)}${suffix}"`; }); } function pageKey(locale, slug) { return `${locale}:${slug}`; } function fileSlug(rel) { return normalizeSlug(rel.replace(/\.(md|mdx)$/, "")); } function normalizeSlug(value) { return value.replace(/\/index$/, "") || "index"; } function cleanPath(value) { const [pathname, hash = ""] = String(value).split("#"); const cleaned = pathname.replace(/\/$/, "") || "/"; return hash ? `${cleaned}#${hash}` : cleaned; } function publicPath(value) { if (!basePath) return value; if (value === "/") return `${basePath}/`; return `${basePath}${value.startsWith("/") ? value : `/${value}`}`; } function normalizeBasePath(value) { if (!value || value === "/") return ""; return `/${value.replace(/^\/+|\/+$/g, "")}`; } function htmlLang(locale) { return locale === "zh-CN" ? "zh-CN" : locale === "zh-TW" ? "zh-TW" : locale; } function firstHeading(markdown) { return markdown.match(/^#\s+(.+)$/m)?.[1]?.replace(/<[^>]+>/g, "").trim(); } function titleize(value) { return value.replaceAll("-", " ").replace(/\b\w/g, (m) => m.toUpperCase()); } function stripTags(value) { return value.replace(/<[^>]*>/g, "").replace(/\s+/g, " ").trim(); } function decodeHtmlEntities(value) { return String(value).replace(/&(#x[0-9a-f]+|#\d+|amp|lt|gt|quot|apos);/gi, (match, entity) => { const lower = entity.toLowerCase(); if (lower === "amp") return "&"; if (lower === "lt") return "<"; if (lower === "gt") return ">"; if (lower === "quot") return "\""; if (lower === "apos") return "'"; const code = lower.startsWith("#x") ? Number.parseInt(lower.slice(2), 16) : Number.parseInt(lower.slice(1), 10); return Number.isFinite(code) ? String.fromCodePoint(code) : match; }); } function escapeHtml(value) { return String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """); } function escapeAttr(value) { return escapeHtml(value).replaceAll("'", "'"); } function escapeXml(value) { return String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'"); }