serve robots.txt and sitemap.xml

Two exact paths a crawler asks for by name, so they are mux entries rather than
resolver cases — no bundle can collide, since a key always sits under a section.

robots.txt at the site root is served verbatim, because a site that ships one has
said something deliberate; otherwise the engine emits the minimum that is true and
points at the sitemap. The sitemap lists every bundle in every language it exists
in, since each variant is separately reachable, with lastmod only where a bundle
has a date. Every URL comes from content.URL like every other path the engine
emits, so a sitemap cannot disagree with what is actually served.

Both need a declared base. Without one the sitemap answers 404 rather than listing
paths no crawler can resolve, and robots omits the Sitemap line rather than
writing a relative one.

write() was setting text/html for every caller, and headers only go out with the
first byte — so a handler setting its own type would have had it silently replaced,
which is how a sitemap gets served as a web page. It now splits into write and
writeAs, and the tests assert the content types rather than only the bodies.

internal/web/discover.go

@@ -0,0 +1,71 @@
+package web
+
+import (
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"strings"
+
+	"khosra/internal/content"
+)
+
+// robots and sitemap are the two files a crawler looks for by exact name.
+const (
+	robotsPath  = "/robots.txt"
+	sitemapPath = "/sitemap.xml"
+)
+
+// serveRobots answers /robots.txt, preferring the site's own file.
+//
+// A site that ships robots.txt has said something deliberate, so it is served verbatim; otherwise the engine
+// emits the minimum that is true — everything is public, and here is the sitemap. The Sitemap line only
+// appears with a declared base, because a relative sitemap reference is not something a crawler accepts.
+func serveRobots(w http.ResponseWriter, req *http.Request, siteFS fs.FS, base string) {
+	if siteFS != nil {
+		if data, err := fs.ReadFile(siteFS, "robots.txt"); err == nil {
+			writeAs(w, "text/plain; charset=utf-8", data, "robots.txt")
+			return
+		}
+	}
+	var out strings.Builder
+	out.WriteString("User-agent: *\nDisallow:\n")
+	if base != "" {
+		fmt.Fprintf(&out, "Sitemap: %s\n", content.Absolute(base, sitemapPath))
+	}
+	writeAs(w, "text/plain; charset=utf-8", []byte(out.String()), "robots.txt")
+}
+
+// serveSitemap answers /sitemap.xml with every bundle in every language it exists in.
+//
+// It needs a declared base: the sitemap format has no room for a relative URL, so without one the honest
+// answer is that this file does not exist rather than a file full of paths no crawler can use (ADR-0039).
+// Every URL comes from content.URL, like every other path the engine emits, so a sitemap can never disagree
+// with what is actually served.
+func serveSitemap(w http.ResponseWriter, req *http.Request, site *content.Site, base string) {
+	if base == "" {
+		slog.Warn("no sitemap: the site declares no base URL", "file", content.SettingsFile)
+		http.NotFound(w, req)
+		return
+	}
+	var out strings.Builder
+	out.WriteString(`<?xml version="1.0" encoding="utf-8"?>` + "\n")
+	out.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` + "\n")
+	for _, entry := range site.Everything() {
+		fmt.Fprintf(&out, "<url><loc>%s</loc>", xmlEscape(content.Absolute(base, content.URL(entry.Key, entry.Lang))))
+		if !entry.Date.IsZero() {
+			fmt.Fprintf(&out, "<lastmod>%s</lastmod>", entry.Date.Format("2006-01-02"))
+		}
+		out.WriteString("</url>\n")
+	}
+	out.WriteString("</urlset>\n")
+	writeAs(w, "application/xml; charset=utf-8", []byte(out.String()), "sitemap.xml")
+}
+
+// xmlEscape escapes the five characters XML reserves. A URL should contain none of them, and a sitemap that
+// silently breaks on the one that does is worse than a slightly paranoid replacement.
+func xmlEscape(s string) string {
+	return strings.NewReplacer(
+		"&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&quot;", "'", "&apos;",
+	).Replace(s)
+}