diff --git a/scripts/docs-site/build.mjs b/scripts/docs-site/build.mjs index 94bee2919..5f50f1d00 100644 --- a/scripts/docs-site/build.mjs +++ b/scripts/docs-site/build.mjs @@ -62,7 +62,9 @@ const localePickerLabels = { copyPublicFiles(); await renderPageOgCards(); for (const page of pages) writePage(page); +writeLlmsIndex(); writeLlmsFull(); +writeSitemap(); writeRedirects(); writeStaticAssets(); console.log(`built ${pages.length} pages in ${path.relative(root, outDir)}`); @@ -330,9 +332,7 @@ function searchModal() { function writeLlmsFull() { const llmsOrigin = (process.env.DOCS_SITE_CANONICAL_ORIGIN ?? (process.env.DOCS_SITE_CNAME ? `https://${process.env.DOCS_SITE_CNAME}` : "")).replace(/\/$/, ""); - const englishPages = pages - .filter((page) => page.locale === "en" && !localeLabels[page.rel.split("/")[0]]) - .sort((a, b) => a.slug.localeCompare(b.slug)); + const englishPages = englishDocsPages(); const content = englishPages.map((page) => { const source = llmsOrigin ? `${llmsOrigin}${pageRoute(page)}` : pageRoute(page); return `# ${page.title}\nSource: ${source}\n\n${stripMdxForLlms(page.body).trim()}\n`; @@ -340,6 +340,53 @@ function writeLlmsFull() { fs.writeFileSync(path.join(outDir, "llms-full.txt"), `${content}\n`, "utf8"); } +function writeLlmsIndex() { + const origin = docsOrigin(); + const lines = [ + `# ${config.name}`, + "", + config.description ?? "OpenClaw documentation.", + "", + "## Full Documentation", + "", + `- [llms-full.txt](${origin}/llms-full.txt): Full plain-text documentation bundle for LLM context.`, + "", + "## Documentation Index", + "", + ]; + for (const page of englishDocsPages()) { + const summary = page.summary ? `: ${stripMdxForLlms(page.summary).replace(/\s+/g, " ").trim()}` : ""; + lines.push(`- [${page.title}](${origin}${pageRoute(page)})${summary}`); + } + const content = `${lines.join("\n")}\n`; + fs.writeFileSync(path.join(outDir, "llms.txt"), content, "utf8"); + fs.writeFileSync(path.join(outDir, "llm.txt"), content, "utf8"); +} + +function writeSitemap() { + const origin = docsOrigin(); + const urls = [...new Set(pages.map((page) => `${origin}${pageRoute(page)}`))] + .sort((a, b) => a.localeCompare(b)); + const xml = [ + '', + '', + ...urls.map((url) => ` ${escapeXml(url)}`), + "", + "", + ].join("\n"); + fs.writeFileSync(path.join(outDir, "sitemap.xml"), xml, "utf8"); +} + +function englishDocsPages() { + return pages + .filter((page) => page.locale === "en" && !localeLabels[page.rel.split("/")[0]]) + .sort((a, b) => a.slug.localeCompare(b.slug)); +} + +function docsOrigin() { + return (canonicalOrigin || "https://documentation.openclaw.ai").replace(/\/$/, ""); +} + function chatWidget() { if (!chatApiUrl) return ""; return `
@@ -617,3 +664,7 @@ function escapeHtml(value) { function escapeAttr(value) { return escapeHtml(value).replaceAll("'", "'"); } + +function escapeXml(value) { + return String(value).replaceAll("&", "&").replaceAll("<", "<").replaceAll(">", ">").replaceAll('"', """).replaceAll("'", "'"); +} diff --git a/scripts/docs-site/smoke.mjs b/scripts/docs-site/smoke.mjs index a62a61c1f..66c76083b 100644 --- a/scripts/docs-site/smoke.mjs +++ b/scripts/docs-site/smoke.mjs @@ -10,6 +10,10 @@ const required = [ "it/channels/index.html", "zh-CN/tools/reactions/index.html", "concepts/models.md", + "llm.txt", + "llms.txt", + "llms-full.txt", + "sitemap.xml", "de/tools/reactions/index.html", "de/gateway/heartbeat/index.html", "pagefind/pagefind.js",