serve robots.txt and sitemap.xml

Two exact paths a crawler asks for by name, so they are mux entries rather than
resolver cases — no bundle can collide, since a key always sits under a section.

robots.txt at the site root is served verbatim, because a site that ships one has
said something deliberate; otherwise the engine emits the minimum that is true and
points at the sitemap. The sitemap lists every bundle in every language it exists
in, since each variant is separately reachable, with lastmod only where a bundle
has a date. Every URL comes from content.URL like every other path the engine
emits, so a sitemap cannot disagree with what is actually served.

Both need a declared base. Without one the sitemap answers 404 rather than listing
paths no crawler can resolve, and robots omits the Sitemap line rather than
writing a relative one.

write() was setting text/html for every caller, and headers only go out with the
first byte — so a handler setting its own type would have had it silently replaced,
which is how a sitemap gets served as a web page. It now splits into write and
writeAs, and the tests assert the content types rather than only the bodies.

internal/content/settings.go

@@ -0,0 +1,59 @@
+package content
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// SettingsFile is where a site declares itself, at the site root.
+const SettingsFile = "site.yaml"
+
+// Settings are the site's own declarations (ADR-0039).
+//
+// Declared keys only: this is not a bag for arbitrary engine internals, which is how a settings file turns
+// into unbounded configuration. Section-level and bundle-level settings are not here — a bundle's own
+// frontmatter covers it, and the rest of the cascade is still parked.
+type Settings struct {
+	// Base is the site's canonical origin, without a trailing slash: "https://khosra.example". Empty when
+	// the site declares none, in which case the engine emits paths and nothing absolute.
+	Base string `yaml:"base"`
+	// Title names the site. A theme may put it in a document title or a feed; the engine only carries it.
+	Title string `yaml:"title"`
+}
+
+// LoadSettings reads site.yaml from the site root.
+//
+// A missing file is not an error, because a bare site root must still serve (ADR-0026). A malformed one is,
+// and the caller treats it as fatal: unlike a single bad bundle, which is skipped loudly (ADR-0029), a
+// broken declaration misconfigures every page on the site.
+func LoadSettings(fsys fs.FS) (Settings, error) {
+	data, err := fs.ReadFile(fsys, SettingsFile)
+	if errors.Is(err, fs.ErrNotExist) {
+		return Settings{}, nil
+	}
+	if err != nil {
+		return Settings{}, fmt.Errorf("read %s: %w", SettingsFile, err)
+	}
+	var s Settings
+	if err := yaml.Unmarshal(data, &s); err != nil {
+		return Settings{}, fmt.Errorf("parse %s: %w", SettingsFile, err)
+	}
+	s.Base = strings.TrimSuffix(strings.TrimSpace(s.Base), "/")
+	s.Title = strings.TrimSpace(s.Title)
+	return s, nil
+}
+
+// Absolute turns a path the engine emitted into a full URL.
+//
+// With no declared base it returns the path unchanged, so a site that has not said where it lives still
+// links correctly to itself — every internal path is root-relative already.
+func Absolute(base, path string) string {
+	if base == "" {
+		return path
+	}
+	return base + path
+}

internal/content/settings_test.go

@@ -0,0 +1,45 @@
+package content
+
+import (
+	"testing"
+	"testing/fstest"
+)
+
+func TestSettingsAreOptionalAndNormalised(t *testing.T) {
+	// A bare site root must still serve, so an absent file is not an error (ADR-0026).
+	got, err := LoadSettings(fstest.MapFS{})
+	if err != nil || got.Base != "" || got.Title != "" {
+		t.Fatalf("absent site.yaml gave %+v, %v — want the zero value and no error", got, err)
+	}
+	got, err = LoadSettings(fstest.MapFS{
+		SettingsFile: {Data: []byte("base: https://khosra.example/\ntitle: খসড়া\n")},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Base != "https://khosra.example" {
+		t.Errorf("base = %q — a trailing slash must be trimmed, or every URL doubles it", got.Base)
+	}
+	if got.Title != "খসড়া" {
+		t.Errorf("title = %q", got.Title)
+	}
+}
+
+func TestMalformedSettingsAreAnError(t *testing.T) {
+	// Unlike one bad bundle, which is skipped loudly (ADR-0029), this misconfigures every page, so the
+	// caller treats it as fatal.
+	if _, err := LoadSettings(fstest.MapFS{SettingsFile: {Data: []byte("base: [unclosed\n")}}); err == nil {
+		t.Error("a malformed site.yaml must be an error")
+	}
+}
+
+func TestAbsoluteOnlyPrefixesWhenABaseExists(t *testing.T) {
+	for _, c := range []struct{ base, path, want string }{
+		{"https://khosra.example", "/posts/hello/", "https://khosra.example/posts/hello/"},
+		{"", "/posts/hello/", "/posts/hello/"},
+	} {
+		if got := Absolute(c.base, c.path); got != c.want {
+			t.Errorf("Absolute(%q, %q) = %q, want %q", c.base, c.path, got, c.want)
+		}
+	}
+}

internal/content/site.go

@@ -265,6 +265,20 @@ func (b Bundle) Section() string {
 	return sec
 }
 
+// Everything lists every variant of every bundle, sorted by key then language.
+//
+// Every *variant*, unlike a Query, which answers with one variant per key: a sitemap lists each language as
+// its own URL, because each is separately reachable.
+func (s *Site) Everything() []Bundle {
+	out := make([]Bundle, 0, len(s.byKeyLang))
+	for _, key := range s.keys() {
+		for _, lang := range s.Variants(key) {
+			out = append(out, s.byKeyLang[key+"\x00"+lang])
+		}
+	}
+	return out
+}
+
 // Sections lists every section that holds at least one bundle, sorted.
 func (s *Site) Sections() []string {
 	seen := map[string]bool{}

internal/ext/shortcodes/shortcodes_test.go

@@ -43,7 +43,7 @@ func wired(t *testing.T, siteFS fstest.MapFS) *render.Renderer {
 	if siteFS != nil {
 		fsys = siteFS
 	}
-	r, err := render.New(fsys, func(p render.Partial) []goldmark.Extender {
+	r, err := render.New(fsys, content.Settings{}, func(p render.Partial) []goldmark.Extender {
 		return []goldmark.Extender{New(p)}
 	})
 	if err != nil {

internal/render/chrome_test.go

@@ -61,7 +61,7 @@ func TestDatesReadInTheirOwnScript(t *testing.T) {
 }
 
 func TestTypographerSmoothsProseAndLeavesCodeAlone(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,7 +86,7 @@ func TestTypographerSmoothsProseAndLeavesCodeAlone(t *testing.T) {
 }
 
 func TestMachineReadableOutputStaysASCII(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}

internal/render/render.go

@@ -38,6 +38,8 @@ type head struct {
 	Alternates []Alternate
 	// Style is the reference theme's stylesheet, inlined so a bare site root needs no asset route.
 	Style template.CSS
+	// Site is what the site declared about itself in site.yaml (ADR-0039). Zero when it declared nothing.
+	Site content.Settings
 }
 
 // Page is one bundle rendered.
@@ -117,6 +119,8 @@ type Renderer struct {
 	style    template.CSS
 	// files is the site root, handed to features through Origin. Nil when there is none.
 	files fs.FS
+	// settings are the site's declarations, constant for the life of the process.
+	settings content.Settings
 }
 
 // Partial renders a named fragment. A feature under internal/ext is handed one of these at wiring time,
@@ -169,7 +173,7 @@ func WithOrigin(pc parser.Context, origin Origin) {
 // extensions to enable. A callback rather than a parameter of feature types, because internal/render must
 // not import internal/ext — only cmd knows which features a build includes (conventions.md, ADR-0036). It
 // may be nil.
-func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, error) {
+func New(siteFS fs.FS, settings content.Settings, extend func(Partial) []goldmark.Extender) (*Renderer, error) {
 	page, err := parseSet(siteFS, "templates/base.html", "templates/page.html")
 	if err != nil {
 		return nil, fmt.Errorf("bundle templates: %w", err)
@@ -186,7 +190,7 @@ func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, err
 	if err != nil {
 		return nil, err
 	}
-	r := &Renderer{page: page, list: list, partials: partials, style: css, files: siteFS}
+	r := &Renderer{page: page, list: list, partials: partials, style: css, files: siteFS, settings: settings}
 	// The typographer smooths quotes, dashes and ellipses in authored prose and leaves code spans alone,
 	// because it works on the parsed tree rather than the text. That is the only change the engine makes to
 	// an author's words (ADR-0034), and it is a parser option rather than a render transform, so it does
@@ -202,6 +206,25 @@ func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, err
 	return r, nil
 }
 
+// head builds the document shell every kind of page shares.
+//
+// canonical arrives as a path and leaves absolute when the site declared a base: a canonical link and an
+// hreflang are read by machines that resolve neither against the page (ADR-0039).
+func (r *Renderer) head(title, lang, canonical string) head {
+	return head{
+		Title:     title,
+		Lang:      lang,
+		Canonical: r.absolute(canonical),
+		Style:     r.style,
+		Site:      r.settings,
+	}
+}
+
+// absolute is the site's own URL for a path the engine emitted, or the path itself when no base is declared.
+func (r *Renderer) absolute(path string) string {
+	return content.Absolute(r.settings.Base, path)
+}
+
 // Partial renders one named fragment. A missing template is an error the caller degrades on, never a
 // failed request (extensions.md rule 5).
 func (r *Renderer) Partial(name string, data Fragment) ([]byte, error) {
@@ -276,14 +299,14 @@ func (r *Renderer) Bundle(b content.Bundle, served string, variants []string, se
 		title = b.Key
 	}
 	p := Page{
-		head:     head{Title: title, Lang: served, Canonical: content.URL(b.Key, served), Style: r.style},
+		head:     r.head(title, served, content.URL(b.Key, served)),
 		Key:      b.Key,
 		HTML:     template.HTML(body.String()),
 		Extra:    b.Extra,
 		Sequence: r.sequence(seq, served),
 	}
 	for _, l := range variants {
-		p.Alternates = append(p.Alternates, Alternate{Lang: l, URL: content.URL(b.Key, l)})
+		p.Alternates = append(p.Alternates, Alternate{Lang: l, URL: r.absolute(content.URL(b.Key, l))})
 	}
 	return r.execute(r.page, p, b.Key)
 }
@@ -363,7 +386,7 @@ func (r *Renderer) paginate(title, lang, canonical string, all []content.Bundle,
 	start := (page - 1) * content.PerPage
 	end := min(start+content.PerPage, len(all))
 	l := List{
-		head:  head{Title: title, Lang: lang, Canonical: canonical, Style: r.style},
+		head:  r.head(title, lang, canonical),
 		Page:  page,
 		Pages: pages,
 	}

internal/render/render_test.go

@@ -9,7 +9,7 @@ import (
 )
 
 func TestBundleRendersMarkdownIntoTheTheme(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -33,7 +33,7 @@ func TestBundleRendersMarkdownIntoTheTheme(t *testing.T) {
 }
 
 func TestBundleWithoutTitleFallsBackToKey(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -54,7 +54,7 @@ func TestSiteOverridesOneBlockAndInheritsTheRest(t *testing.T) {
 	siteFS := fstest.MapFS{
 		"templates/page.html": {Data: []byte(`{{define "main"}}<section class="mine">{{.Title}}</section>{{end}}`)},
 	}
-	r, err := New(siteFS, nil)
+	r, err := New(siteFS, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,7 +82,7 @@ func TestAListingOverrideDoesNotLeakIntoBundlePages(t *testing.T) {
 	siteFS := fstest.MapFS{
 		"templates/list.html": {Data: []byte(`{{define "main"}}LISTING ONLY{{end}}`)},
 	}
-	r, err := New(siteFS, nil)
+	r, err := New(siteFS, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +101,7 @@ func TestAListingOverrideDoesNotLeakIntoBundlePages(t *testing.T) {
 
 func TestSiteStylesheetReplacesTheReferenceOne(t *testing.T) {
 	siteFS := fstest.MapFS{"templates/theme.css": {Data: []byte("body{color:rebeccapurple}")}}
-	r, err := New(siteFS, nil)
+	r, err := New(siteFS, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -114,3 +114,50 @@ func TestSiteStylesheetReplacesTheReferenceOne(t *testing.T) {
 		t.Error("the site stylesheet should replace the reference one")
 	}
 }
+
+func TestADeclaredBaseMakesMachineReadableURLsAbsolute(t *testing.T) {
+	// A canonical link and an hreflang are read by machines that resolve neither against the page, so both
+	// go absolute as soon as the site says where it lives (ADR-0039).
+	r, err := New(nil, content.Settings{Base: "https://khosra.example", Title: "Khosra"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := content.Parse("posts/hello.md", []byte("---\ntitle: Hello\n---\nhi\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := r.Bundle(b, "en", []string{"en", "bn"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := string(out)
+	for _, want := range []string{
+		`rel="canonical" href="https://khosra.example/posts/hello/"`,
+		`hreflang="bn" href="https://khosra.example/bn/posts/hello/"`,
+		`property="og:url" content="https://khosra.example/posts/hello/"`,
+		`<title>Hello · Khosra</title>`,
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("missing %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestWithoutABaseEverythingStaysRelative(t *testing.T) {
+	r, err := New(nil, content.Settings{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := content.Parse("posts/hello.md", []byte("---\ntitle: Hello\n---\nhi\n"))
+	out, err := r.Bundle(b, "en", []string{"en"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := string(out)
+	if !strings.Contains(got, `rel="canonical" href="/posts/hello/"`) {
+		t.Errorf("a site that declared no base still links to itself:\n%s", got)
+	}
+	if strings.Contains(got, "og:site_name") {
+		t.Error("no declared title means no site_name tag, rather than an empty one")
+	}
+}

internal/render/templates/base.html

@@ -4,11 +4,18 @@
 <head>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
-<title>{{.Title}}</title>
+<title>{{.Title}}{{if .Site.Title}} · {{.Site.Title}}{{end}}</title>
 <link rel="canonical" href="{{.Canonical}}">
 {{- range .Alternates}}
 <link rel="alternate" hreflang="{{.Lang}}" href="{{.URL}}">
 {{- end}}
+<meta property="og:title" content="{{.Title}}">
+<meta property="og:url" content="{{.Canonical}}">
+<meta property="og:type" content="article">
+<meta property="og:locale" content="{{.Lang}}">
+{{- if .Site.Title}}
+<meta property="og:site_name" content="{{.Site.Title}}">
+{{- end}}
 <style>{{.Style}}</style>
 </head>
 <body>

internal/web/discover.go

@@ -0,0 +1,71 @@
+package web
+
+import (
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"strings"
+
+	"khosra/internal/content"
+)
+
+// robots and sitemap are the two files a crawler looks for by exact name.
+const (
+	robotsPath  = "/robots.txt"
+	sitemapPath = "/sitemap.xml"
+)
+
+// serveRobots answers /robots.txt, preferring the site's own file.
+//
+// A site that ships robots.txt has said something deliberate, so it is served verbatim; otherwise the engine
+// emits the minimum that is true — everything is public, and here is the sitemap. The Sitemap line only
+// appears with a declared base, because a relative sitemap reference is not something a crawler accepts.
+func serveRobots(w http.ResponseWriter, req *http.Request, siteFS fs.FS, base string) {
+	if siteFS != nil {
+		if data, err := fs.ReadFile(siteFS, "robots.txt"); err == nil {
+			writeAs(w, "text/plain; charset=utf-8", data, "robots.txt")
+			return
+		}
+	}
+	var out strings.Builder
+	out.WriteString("User-agent: *\nDisallow:\n")
+	if base != "" {
+		fmt.Fprintf(&out, "Sitemap: %s\n", content.Absolute(base, sitemapPath))
+	}
+	writeAs(w, "text/plain; charset=utf-8", []byte(out.String()), "robots.txt")
+}
+
+// serveSitemap answers /sitemap.xml with every bundle in every language it exists in.
+//
+// It needs a declared base: the sitemap format has no room for a relative URL, so without one the honest
+// answer is that this file does not exist rather than a file full of paths no crawler can use (ADR-0039).
+// Every URL comes from content.URL, like every other path the engine emits, so a sitemap can never disagree
+// with what is actually served.
+func serveSitemap(w http.ResponseWriter, req *http.Request, site *content.Site, base string) {
+	if base == "" {
+		slog.Warn("no sitemap: the site declares no base URL", "file", content.SettingsFile)
+		http.NotFound(w, req)
+		return
+	}
+	var out strings.Builder
+	out.WriteString(`<?xml version="1.0" encoding="utf-8"?>` + "\n")
+	out.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` + "\n")
+	for _, entry := range site.Everything() {
+		fmt.Fprintf(&out, "<url><loc>%s</loc>", xmlEscape(content.Absolute(base, content.URL(entry.Key, entry.Lang))))
+		if !entry.Date.IsZero() {
+			fmt.Fprintf(&out, "<lastmod>%s</lastmod>", entry.Date.Format("2006-01-02"))
+		}
+		out.WriteString("</url>\n")
+	}
+	out.WriteString("</urlset>\n")
+	writeAs(w, "application/xml; charset=utf-8", []byte(out.String()), "sitemap.xml")
+}
+
+// xmlEscape escapes the five characters XML reserves. A URL should contain none of them, and a sitemap that
+// silently breaks on the one that does is worse than a slightly paranoid replacement.
+func xmlEscape(s string) string {
+	return strings.NewReplacer(
+		"&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&quot;", "'", "&apos;",
+	).Replace(s)
+}

internal/web/discover_test.go

@@ -0,0 +1,101 @@
+package web
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	"khosra/internal/content"
+	"khosra/internal/render"
+)
+
+func crawlerHandler(t *testing.T, settings content.Settings, extra fstest.MapFS) http.Handler {
+	t.Helper()
+	fsys := fstest.MapFS{
+		"content/posts/hello.md":    {Data: []byte("---\ntitle: Hello\ndate: 2026-03-08\n---\nx\n")},
+		"content/posts/hello.bn.md": {Data: []byte("---\ntitle: হ্যালো\ndate: 2026-03-08\n---\nx\n")},
+		"content/pages/about.md":    {Data: []byte("---\ntitle: About\n---\nx\n")},
+	}
+	for name, file := range extra {
+		fsys[name] = file
+	}
+	bundles, err := content.Scan(fsys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := render.New(nil, settings, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return Handler(content.NewSite(bundles), r, fsys, settings)
+}
+
+func TestSitemapListsEveryVariantAbsolutely(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("got %d, want 200", rec.Code)
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") {
+		t.Errorf("content-type = %q — a sitemap served as HTML is a sitemap nothing reads", ct)
+	}
+	body := rec.Body.String()
+	for _, want := range []string{
+		"<loc>https://khosra.example/posts/hello/</loc>",
+		"<loc>https://khosra.example/bn/posts/hello/</loc>", // each language is its own URL
+		"<loc>https://khosra.example/pages/about/</loc>",
+		"<lastmod>2026-03-08</lastmod>",
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("missing %q:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, "<lastmod></lastmod>") {
+		t.Error("an undated bundle should carry no lastmod at all")
+	}
+}
+
+func TestNoBaseMeansNoSitemap(t *testing.T) {
+	// The format has no room for a relative URL, so the honest answer is that the file does not exist
+	// (ADR-0039) rather than one full of paths no crawler can resolve.
+	h := crawlerHandler(t, content.Settings{}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	if rec.Code != http.StatusNotFound {
+		t.Errorf("got %d, want 404", rec.Code)
+	}
+}
+
+func TestRobotsIsGeneratedOrTheSitesOwn(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	body := rec.Body.String()
+	if !strings.Contains(body, "User-agent: *") || !strings.Contains(body, "Sitemap: https://khosra.example/sitemap.xml") {
+		t.Errorf("generated robots should point at the sitemap:\n%s", body)
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
+		t.Errorf("content-type = %q", ct)
+	}
+
+	// A site that ships its own has said something deliberate.
+	h = crawlerHandler(t, content.Settings{Base: "https://khosra.example"},
+		fstest.MapFS{"robots.txt": {Data: []byte("User-agent: *\nDisallow: /drafts/\n")}})
+	rec = httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	if got := rec.Body.String(); !strings.Contains(got, "Disallow: /drafts/") || strings.Contains(got, "Sitemap:") {
+		t.Errorf("the site's own robots.txt should be served verbatim:\n%s", got)
+	}
+}
+
+func TestRobotsWithoutABaseOmitsTheSitemapLine(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	if got := rec.Body.String(); strings.Contains(got, "Sitemap:") {
+		t.Errorf("a relative sitemap reference is not something a crawler accepts:\n%s", got)
+	}
+}

internal/web/web.go

@@ -15,11 +15,19 @@ import (
 // Handler serves a site.
 //
 // One mux entry, because URL shape is the resolver's business rather than the mux's: see resolve.
-func Handler(site *content.Site, r *render.Renderer, siteFS fs.FS) http.Handler {
+func Handler(site *content.Site, r *render.Renderer, siteFS fs.FS, settings content.Settings) http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /", func(w http.ResponseWriter, req *http.Request) {
 		serve(w, req, site, r)
 	})
+	// Two exact paths a crawler asks for by name, so they are mux entries rather than resolver cases: no
+	// bundle can own them, since a key always sits under a section.
+	mux.HandleFunc("GET "+robotsPath, func(w http.ResponseWriter, req *http.Request) {
+		serveRobots(w, req, siteFS, settings.Base)
+	})
+	mux.HandleFunc("GET "+sitemapPath, func(w http.ResponseWriter, req *http.Request) {
+		serveSitemap(w, req, site, settings.Base)
+	})
 	if siteFS != nil {
 		if sub, err := fs.Sub(siteFS, "static"); err == nil {
 			mux.Handle("GET /static/", http.StripPrefix("/static/", serveStatic(sub)))
@@ -98,7 +106,16 @@ func serveTags(w http.ResponseWriter, req *http.Request, site *content.Site, r *
 
 // write sends a rendered page, logging a failed write rather than pretending it succeeded.
 func write(w http.ResponseWriter, out []byte, what string) {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	writeAs(w, "text/html; charset=utf-8", out, what)
+}
+
+// writeAs sends bytes with the type they actually are.
+//
+// Separate from write because headers are only sent with the first byte, so a handler that set its own type
+// before calling write would have had it silently replaced by HTML — which is how a sitemap ends up served
+// as a web page.
+func writeAs(w http.ResponseWriter, contentType string, out []byte, what string) {
+	w.Header().Set("Content-Type", contentType)
 	if _, err := w.Write(out); err != nil {
 		slog.Warn("write failed", "what", what, "err", err)
 	}

internal/web/web_test.go

@@ -24,11 +24,11 @@ func testHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }
 
 func TestServeBundleAtItsPermalink(t *testing.T) {
@@ -72,11 +72,11 @@ func multilingualHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }
 
 func TestPrefixedLanguageServesThatVariant(t *testing.T) {
@@ -124,11 +124,11 @@ func aliasHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }
 
 func TestAliasRedirectsToCanonical(t *testing.T) {
@@ -171,11 +171,11 @@ func listingHandler(t *testing.T, n int) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }
 
 func TestSectionIndexListsNewestFirst(t *testing.T) {
@@ -239,11 +239,11 @@ func TestStaticFilesAreServedAndDirectoriesAreNot(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(fsys, nil)
+	r, err := render.New(fsys, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	h := Handler(content.NewSite(bundles), r, fsys)
+	h := Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 	for path, want := range map[string]int{
 		"/static/style.css":    http.StatusOK,
 		"/static/img/logo.svg": http.StatusOK,
@@ -280,11 +280,11 @@ func TestAStaticPathThatEscapesTheRootIs404(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(fsys, nil)
+	r, err := render.New(fsys, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	h := Handler(content.NewSite(nil), r, fsys)
+	h := Handler(content.NewSite(nil), r, fsys, content.Settings{})
 	for path, want := range map[string]int{
 		"/static/ok.css":     http.StatusOK,
 		"/static/escape.txt": http.StatusNotFound,
@@ -313,11 +313,11 @@ func seriesHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, nil)
+	return Handler(content.NewSite(bundles), r, nil, content.Settings{})
 }
 
 func TestSequenceNavigationLinksNeighbours(t *testing.T) {
@@ -403,11 +403,11 @@ func tagHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, nil)
+	return Handler(content.NewSite(bundles), r, nil, content.Settings{})
 }
 
 func TestGlobalTagListingSpansSectionsGroupedByOne(t *testing.T) {