feat: serve agent-facing markdown via content negotiation

astro.config.mjs

@@ -3,7 +3,6 @@ import { defineConfig } from "astro/config";
 import starlight from "@astrojs/starlight";
 import ecTwoSlash from "expressive-code-twoslash";
 import topics from "starlight-sidebar-topics";
-import starlightMarkdown from "starlight-markdown";
 import mermaid from "astro-mermaid";
 import { fileURLToPath } from "node:url";
 
@@ -104,7 +103,6 @@ export default defineConfig({
 				{ icon: 'github', label: 'GitHub', href: 'https://bomb.sh/on/github' },
 			],
 			plugins: [
-				starlightMarkdown(),
 				topics([
 					{
 						label: "Clack",

package.json

@@ -10,6 +10,7 @@
     "build": "astro build && cp public/_headers dist/_headers",
     "preview": "astro preview",
     "astro": "astro",
+    "test": "node --test \"src/**/*.test.ts\"",
     "snapshot": "node --experimental-strip-types ./scripts/snapshot.ts",
     "generate:docs-index": "node --experimental-strip-types ./scripts/generate-docs-index.ts"
   },
@@ -31,11 +32,13 @@
     "expressive-code-twoslash": "^0.5.3",
     "mermaid": "^11.16.0",
     "sharp": "^0.33.5",
-    "starlight-markdown": "^0.1.5",
     "starlight-sidebar-topics": "^0.6.2"
   },
   "devDependencies": {
     "astro-vtbot": "^3.1.0",
+    "remark": "^15.0.1",
+    "remark-gfm": "^4.0.1",
+    "remark-mdx": "^3.1.1",
     "tinyexec": "^1.0.2",
     "wrangler": "^4.97.0"
   },

router/package.json

@@ -5,7 +5,8 @@
 	"type": "module",
 	"scripts": {
 		"dev": "wrangler dev",
-		"deploy": "wrangler deploy"
+		"deploy": "wrangler deploy",
+		"test": "node --test \"src/**/*.test.ts\""
 	},
 	"devDependencies": {
 		"@cloudflare/workers-types": "^4.20250327.0",

router/src/index.ts

@@ -1,5 +1,17 @@
+import { prefersMarkdown, toMarkdownPath } from "./negotiate";
+
 export interface Env { }
 
+// Public origin used in Link headers (canonical/alternate) so agents always
+// discover the proxied bomb.sh URLs, never the internal workers.dev ones.
+const SITE = "https://bomb.sh";
+
+const MARKDOWN_404 = `# 404: Not Found
+
+This page does not exist. An index of all Bombshell documentation is
+available at ${SITE}/docs/index.md
+`;
+
 // Where to proxy docs requests. In production this is the live site. On
 // Cloudflare branch previews both Workers share the same branch slug
 // (e.g. `fix-404-bombsh-docs-router` ↔ `fix-404-bombshell-docs`), so we point
@@ -40,6 +52,39 @@ export default {
 
 		if (url.pathname.startsWith("/docs")) {
 			const origin = docsOrigin(url.host);
+
+			// Agent-facing markdown: explicit `.md` paths always serve markdown;
+			// extensionless page routes negotiate on `Accept: text/markdown`.
+			// Negotiation rewrites to a distinct origin URL, so HTML and markdown
+			// variants get distinct cache keys — Cloudflare's cache ignores `Vary`.
+			const markdownPath = toMarkdownPath(url.pathname);
+			const wantsMarkdown =
+				markdownPath !== null &&
+				(url.pathname.endsWith(".md") ||
+					prefersMarkdown(request.headers.get("Accept")));
+
+			if (wantsMarkdown && markdownPath) {
+				const response = await fetch(new URL(markdownPath, origin));
+				if (response.status === 404) {
+					return new Response(MARKDOWN_404, {
+						status: 404,
+						headers: {
+							"Content-Type": "text/markdown; charset=utf-8",
+							"Vary": "Accept",
+						},
+					});
+				}
+				const headers = new Headers(response.headers);
+				headers.set("Content-Type", "text/markdown; charset=utf-8");
+				headers.set("Vary", "Accept");
+				const htmlPath = markdownPath.slice(0, -"index.md".length);
+				headers.set("Link", `<${SITE}${htmlPath}>; rel="canonical"`);
+				return new Response(response.body, {
+					status: response.status,
+					headers,
+				});
+			}
+
 			let response = await fetch(new URL(url.pathname, docsOrigin(url.host)));
 			console.log({ from: url, to: new URL(url.pathname, docsOrigin(url.host)) });
 
@@ -56,7 +101,15 @@ export default {
 			headers.set("Cross-Origin-Resource-Policy", "cross-origin");
 			headers.set("Referrer-Policy", "strict-origin-when-cross-origin");
 
-			// If we got 404, return the HTML, but set status to 404 manually, 
+			// Advertise the markdown twin to agents crawling the HTML variant.
+			if (markdownPath && status === 200) {
+				headers.set(
+					"Link",
+					`<${SITE}${markdownPath}>; rel="alternate"; type="text/markdown"`,
+				);
+			}
+
+			// If we got 404, return the HTML, but set status to 404 manually,
 			// because the response status would be 200
 			return new Response(response.body, {
 				status: status,

router/src/negotiate.test.ts

@@ -0,0 +1,74 @@
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { prefersMarkdown, toMarkdownPath } from "./negotiate.ts";
+
+test("prefersMarkdown: no Accept header", () => {
+	assert.equal(prefersMarkdown(null), false);
+});
+
+test("prefersMarkdown: typical browser Accept header", () => {
+	assert.equal(
+		prefersMarkdown(
+			"text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+		),
+		false,
+	);
+});
+
+test("prefersMarkdown: bare text/markdown", () => {
+	assert.equal(prefersMarkdown("text/markdown"), true);
+});
+
+test("prefersMarkdown: markdown preferred over html", () => {
+	assert.equal(prefersMarkdown("text/markdown,text/html;q=0.8"), true);
+});
+
+test("prefersMarkdown: markdown listed but html preferred", () => {
+	assert.equal(prefersMarkdown("text/html,text/markdown;q=0.5"), false);
+});
+
+test("prefersMarkdown: markdown explicitly refused", () => {
+	assert.equal(prefersMarkdown("text/markdown;q=0"), false);
+});
+
+test("prefersMarkdown: wildcard only is not markdown", () => {
+	assert.equal(prefersMarkdown("*/*"), false);
+});
+
+test("toMarkdownPath: docs root", () => {
+	assert.equal(toMarkdownPath("/docs"), "/docs/index.md");
+	assert.equal(toMarkdownPath("/docs/"), "/docs/index.md");
+});
+
+test("toMarkdownPath: page route with trailing slash", () => {
+	assert.equal(
+		toMarkdownPath("/docs/clack/basics/getting-started/"),
+		"/docs/clack/basics/getting-started/index.md",
+	);
+});
+
+test("toMarkdownPath: page route without trailing slash", () => {
+	assert.equal(
+		toMarkdownPath("/docs/clack/basics/getting-started"),
+		"/docs/clack/basics/getting-started/index.md",
+	);
+});
+
+test("toMarkdownPath: explicit .md request maps to index.md twin", () => {
+	assert.equal(toMarkdownPath("/docs/args/api.md"), "/docs/args/api/index.md");
+});
+
+test("toMarkdownPath: already an index.md path is unchanged", () => {
+	assert.equal(
+		toMarkdownPath("/docs/tty/api/index.md"),
+		"/docs/tty/api/index.md",
+	);
+});
+
+test("toMarkdownPath: assets and files with extensions are ignored", () => {
+	assert.equal(toMarkdownPath("/docs/_astro/hoisted.BQ1yu2o0.js"), null);
+	assert.equal(toMarkdownPath("/docs/favicon.svg"), null);
+	assert.equal(toMarkdownPath("/docs/docs-index.json"), null);
+	assert.equal(toMarkdownPath("/docs/og-docs.png"), null);
+	assert.equal(toMarkdownPath("/docs/404.html"), null);
+});

router/src/negotiate.ts

@@ -0,0 +1,38 @@
+// Content negotiation for agent-facing markdown, per
+// https://cra.mr/optimizing-content-for-agents: agents signal themselves with
+// `Accept: text/markdown`; humans never do.
+
+/** Parse the quality value for a media type out of an Accept header. */
+function quality(accept: string, type: string): number {
+	for (const part of accept.split(",")) {
+		const [media, ...params] = part.trim().split(";");
+		if (media.trim().toLowerCase() !== type) continue;
+		for (const param of params) {
+			const [key, value] = param.trim().split("=");
+			if (key === "q") return Number.parseFloat(value) || 0;
+		}
+		return 1;
+	}
+	return 0;
+}
+
+export function prefersMarkdown(accept: string | null): boolean {
+	if (!accept) return false;
+	const markdown = quality(accept, "text/markdown");
+	if (markdown === 0) return false;
+	return markdown >= quality(accept, "text/html");
+}
+
+/**
+ * Map a request path to its static markdown twin, or null if the path is not
+ * a documentation page (assets, feeds, and other files keep their extension).
+ */
+export function toMarkdownPath(pathname: string): string | null {
+	if (pathname.endsWith("/index.md")) return pathname;
+	if (pathname.endsWith(".md")) {
+		return `${pathname.slice(0, -".md".length)}/index.md`;
+	}
+	const lastSegment = pathname.slice(pathname.lastIndexOf("/") + 1);
+	if (lastSegment.includes(".")) return null;
+	return pathname.endsWith("/") ? `${pathname}index.md` : `${pathname}/index.md`;
+}

scripts/generate-docs-index.ts

@@ -1,6 +1,6 @@
 /**
- * Walks `src/content/docs` and emits `public/llms.txt` plus
- * `public/docs-index.json` for agent discoverability and offline search.
+ * Walks `src/content/docs` and emits `public/docs-index.json`, a
+ * machine-readable page index for offline search and local agent tooling.
  */
 import fs from 'node:fs/promises';
 import path from 'node:path';
@@ -98,42 +98,6 @@ function isIndexed(page: DocPage): boolean {
 	return true;
 }
 
-function generateLlmsTxt(pages: DocPage[]): string {
-	const indexed = pages.filter(isIndexed);
-	const lines = [
-		'# Bombshell Documentation',
-		'',
-		'> Effortlessly build beautiful command-line apps. Docs for Clack, Args, Tab, and TTY.',
-		'',
-		`Canonical docs: ${BASE_URL}/`,
-		'',
-	];
-
-	const homepage = indexed.find((page) => page.slug === '');
-	if (homepage) {
-		lines.push(`- [${homepage.title}](${homepage.url}): ${homepage.description}`, '');
-	}
-
-	const sections = new Map<string, DocPage[]>();
-	for (const page of indexed) {
-		if (page.slug === '') continue;
-		const section = page.slug.split('/')[0];
-		if (!sections.has(section)) sections.set(section, []);
-		sections.get(section)!.push(page);
-	}
-
-	for (const [section, sectionPages] of [...sections.entries()].sort()) {
-		const label = section.charAt(0).toUpperCase() + section.slice(1);
-		lines.push(`## ${label}`, '');
-		for (const page of sectionPages.sort((a, b) => a.slug.localeCompare(b.slug))) {
-			lines.push(`- [${page.title}](${page.url}): ${page.description}`);
-		}
-		lines.push('');
-	}
-
-	return `${lines.join('\n').trimEnd()}\n`;
-}
-
 async function main() {
 	const pages = await walkDocs(docsDir);
 	const indexed = pages.filter(isIndexed);
@@ -150,8 +114,6 @@ async function main() {
 			2,
 		)}\n`,
 	);
-	await fs.writeFile(path.join(rootDir, 'public/llms.txt'), generateLlmsTxt(pages));
-
 	console.log(`Generated docs index with ${indexed.length} pages`);
 }