serve robots.txt and sitemap.xml

Two exact paths a crawler asks for by name, so they are mux entries rather than resolver cases — no bundle can collide, since a key always sits under a section. robots.txt at the site root is served verbatim, because a site that ships one has said something deliberate; otherwise the engine emits the minimum that is true and points at the sitemap. The sitemap lists every bundle in every language it exists in, since each variant is separately reachable, with lastmod only where a bundle has a date. Every URL comes from content.URL like every other path the engine emits, so a sitemap cannot disagree with what is actually served. Both need a declared base. Without one the sitemap answers 404 rather than listing paths no crawler can resolve, and robots omits the Sitemap line rather than writing a relative one. write() was setting text/html for every caller, and headers only go out with the first byte — so a handler setting its own type would have had it silently replaced, which is how a sitemap gets served as a web page. It now splits into write and writeAs, and the tests assert the content types rather than only the bodies.
2026-08-01 02:23:36 +06:00
parent 470e7f18b9
commit bfe980e028
16 changed files with 476 additions and 51 deletions
@@ -8,6 +8,7 @@ import (
 	"log/slog"
 	"net/http"
 	"os"
+	"strings"

 	"khosra/internal/content"
 	"khosra/internal/render"
@@ -17,6 +18,7 @@ import (
 func main() {
 	site := flag.String("site", os.Getenv("KHOSRA_SITE"), "path to the site root (or KHOSRA_SITE)")
 	addr := flag.String("addr", "localhost:8080", "address to listen on")
+	base := flag.String("base", "", "canonical site origin, overriding site.yaml (e.g. https://khosra.example)")
 	flag.Parse()

 	if *site == "" {
@@ -30,13 +32,20 @@ func main() {
 	if err != nil {
 		fatal("cannot read content", err)
 	}
-	renderer, err := render.New(fsys, extenders)
+	settings, err := content.LoadSettings(fsys)
+	if err != nil {
+		fatal("cannot read site settings", err)
+	}
+	if *base != "" {
+		settings.Base = strings.TrimSuffix(*base, "/")
+	}
+	renderer, err := render.New(fsys, settings, extenders)
 	if err != nil {
 		fatal("cannot prepare the theme", err)
 	}

 	slog.Info("serving", "site", *site, "bundles", len(bundles), "addr", *addr)
-	if err := http.ListenAndServe(*addr, web.Handler(content.NewSite(bundles), renderer, fsys)); err != nil {
+	if err := http.ListenAndServe(*addr, web.Handler(content.NewSite(bundles), renderer, fsys, settings)); err != nil {
 		fatal("server stopped", err)
 	}
 }
@@ -18,6 +18,7 @@ overrides the defaults the binary embeds, so a bare root still renders.

 ```
 <site>/
+  site.yaml                  # what the site declares about itself, optional
  content/                   # bundles — the disk contract below
  static/                    # verbatim, served as-is
  templates/                 # html/template overrides, optional
@@ -207,10 +208,27 @@ test: a tag is free-form and cross-cutting, a declared taxonomy has known terms
 Feeds follow the same shape: `/feed.xml` carries every type declared `primary`, `/{section}/feed.xml`
 carries a section, and `/tags/{tag}/feed.xml` comes free from the same Query.

-## The settings cascade `[spec]`
+## Site settings

-Recorded intent (`ideas/deferred-decisions.md`), not built. Until it exists, settings come from a bundle's own frontmatter.
-The shape: site → section → bundle, nearest explicit value winning.
+`site.yaml` at the site root declares the site (ADR-0039). Declared keys only — absent is fine, since a bare
+site root still serves; malformed is a fatal startup error, because unlike one bad bundle it misconfigures
+every page.
+
+| Key | Meaning |
+|---|---|
+| `base` | The canonical origin, `https://khosra.example`. A trailing slash is trimmed. Absent, the engine emits root-relative paths and nothing absolute |
+| `title` | Names the site. A theme may use it in a document title or a feed |
+
+`-base` overrides the file, so a staging host needs no edit to content.
+
+A canonical link, an `hreflang` and an OpenGraph URL become absolute as soon as `base` is declared, because
+each is read by something that resolves neither against the page. Links between pages stay root-relative.
+
+### Below the site level `[spec]`
+
+Section-level and bundle-level settings are **not** built: a bundle's own frontmatter covers it, and nothing
+reads a section override yet. The parked shape is site → section → bundle, nearest explicit value winning
+(`ideas/deferred-decisions.md`).

 | Level | Where it lives |
 |---|---|
@@ -324,8 +342,17 @@ cached (ADR-0005), such a Stage also declares how long its output stays true —
 publication is the same shape seen from the other side: the page becomes reachable at a moment nobody is
 requesting it.

-## Metadata output `[spec]`
+## Metadata and discovery

-OpenGraph, Twitter cards, JSON-LD, microformats2, canonical links, and `hreflang` are Stages reading
-only fields that already exist on the page. SEO adds no new disk fields; if it seems to need one, the
-field belongs in the model for its own sake.
+Canonical links, `hreflang` and OpenGraph are emitted from fields that already exist on the page — the theme
+composes them, so none of it is a transform and none of it adds a disk field. SEO adds no frontmatter; if it
+seems to need some, the field belongs in the model for its own sake.
+
+`/robots.txt` and `/sitemap.xml` are served at those exact paths. No bundle can collide with them, since a
+key always sits under a section. A `robots.txt` at the site root is served verbatim, because a site that
+ships one has said something deliberate; otherwise the engine emits the minimum that is true. The sitemap
+lists every bundle in every language it exists in — each variant is separately reachable, so each is its own
+entry — with `lastmod` only where a bundle carries a date. Both need `base`; without it the sitemap answers
+404 rather than listing paths no crawler can resolve.
+
+`[spec]` Twitter cards, JSON-LD and microformats2 have no consumer yet.
@@ -1,6 +1,6 @@
 # State

-**Verified against:** `ff62729` on 2026-07-30 — update this line every change.
+**Verified against:** `2b0387e` on 2026-07-30 — update this line every change.
 If this file disagrees with the code, the code is right and this file is a bug.

 ## Inventory
@@ -10,23 +10,27 @@ If this file disagrees with the code, the code is right and this file is a bug.
 | `go.mod` | module `khosra`; `goldmark`, `x/text`, `yaml.v3` direct | 10 |
 | `internal/content/doc.go` | package comment | 5 |
 | `internal/content/content.go` | bundles: `os.Root` open, walk, frontmatter split, key/lang derivation, NFC, tag slugs, partial files, permalink building | 368 |
-| `internal/content/site.go` | the indexed site: lookup with language fallback, aliases, `Query` and `Run`, sections, `Sequence` | 286 |
-| `internal/render/render.go` | goldmark with the typographer, per-kind template sets with site override, the `Partial`/`Origin` seams features render and resolve through, `Page`/`List`/`Sequence`/`head` | 386 |
+| `internal/content/settings.go` | `site.yaml`: the site's own declarations (`base`, `title`) and absolute-URL building (ADR-0039) | 59 |
+| `internal/content/site.go` | the indexed site: lookup with language fallback, aliases, `Query` and `Run`, sections, `Sequence`, `Everything` | 300 |
+| `internal/render/render.go` | goldmark with the typographer, per-kind template sets with site override, the `Partial`/`Origin` seams features render and resolve through, `Page`/`List`/`Sequence`/`head` | 409 |
 | `internal/render/chrome.go` | the engine's own words: phrase table, month names, digits, and the `t`/`num`/`day` template funcs (ADR-0034) | 105 |
 | `internal/render/templates/` | reference theme: `base.html`, `page.html`, `list.html`, `shortcodes.html`, `theme.css` (ADR-0026) | — |
 | `internal/ext/shortcodes/` | first feature: `{{< name key="value" >}}` block parser and node renderer, rendering through a theme fragment (ADR-0036). `figure`, `gallery`, `include` | 315 |
 | `internal/ext/widows/` | second feature: joins the last two words of a paragraph or heading with a non-breaking space, over the tree so code spans are safe | 108 |
 | `cmd/khosra/wire.go` | the only list of enabled features (`extensions.md`) | 20 |
 | `internal/web/resolve.go` | URL → (key, lang, page, tag) or a canonical redirect: language prefix, `/en/…` fork guard, pagination, tags, trailing slash | 112 |
+| `internal/web/discover.go` | `/robots.txt` and `/sitemap.xml`, absolute and only with a declared base (ADR-0039) | 74 |
 | `internal/web/web.go` | handler: resolve, look up with fallback, section and tag listings, sequence, `/static/` (misses and refusals alike answer 404), degrade on failure | 152 |
-| `cmd/khosra/main.go` | flags, wiring, startup — the only place things are assembled | 53 |
-| `*_test.go` | table-driven, one file per source file; symlink escape (content and static), canonical paths, language fallback, aliases, pagination, tags, sequences, chrome, typography, shortcode escaping, galleries, includes, partials, widows, 404 | 1562 |
+| `cmd/khosra/main.go` | flags (`-site`, `-addr`, `-base`), wiring, startup — the only place things are assembled | 60 |
+| `*_test.go` | table-driven, one file per source file; symlink escape (content and static), canonical paths, language fallback, aliases, pagination, tags, sequences, chrome, typography, shortcode escaping, galleries, includes, partials, widows, site settings, absolute URLs, robots, sitemap, 404 | 1755 |

 Serves a bundle at `/{section}/{slug}/`, a paginated listing per section, tag listings global and
-section-narrowed, sequence navigation and a series archive on any nested bundle, and `static/` verbatim.
+section-narrowed, sequence navigation and a series archive on any nested bundle, `static/` verbatim, plus `/robots.txt` and
+`/sitemap.xml`.
 Chrome text, dates and digits render in English or Bengali; authored text is untouched but for typographic
 smoothing and widow prevention (ADR-0034). This repo holds engine source only — the site root is external and passed with
-`-site` (ADR-0011).
+`-site` (ADR-0011). `site.yaml` declares `base` and `title`; with a base, canonical, hreflang and OpenGraph
+URLs go absolute (ADR-0039).

 Frontmatter the parser lifts today: `title`, `date`, `tags`, `aliases`, `order`. Every other key in
 `content-model.md`'s table — including `slug`, `draft` and `type` — lands in `Extra` unread, so that table
@@ -42,7 +46,7 @@ this change*.
 | Counter | Now | Extraction due at | What it buys |
 |---|---|---|---|
 | Render transforms — **page-level only** | 0 | **3** | Stage pipeline (ordered `func(ctx,*Page) error`). Parse-phase work does *not* count and must not: goldmark's extender list is already an ordered pipeline for it, so typography, shortcodes and widows compose there (`cmd/khosra/wire.go`) and a second pipeline beside it would be pure duplication. This counts transforms over the assembled page, which nothing hosts yet — OpenGraph and JSON-LD (queue 15) are the first candidates |
-| Routing cases | 5 | **2** — done | Resolver at `internal/web/resolve.go`: bundle, language prefix, pagination, tag, section-narrowed tag |
+| Routing cases | 7 | **2** — done | Resolver at `internal/web/resolve.go`: bundle, language prefix, pagination, tag, section-narrowed tag |
 | Collection pages | 4 | **1** — done | Query primitive: `content.Query{Section, Tag, Lang}` + `Site.Run`. The fourth — a series archive — resolves through `Site.Sequence` instead: membership is structural and the sort ascends, so it shares the index but not the Query |
 | Views / output formats | 2 | **2** — due | Two template sets exist (bundle, listing); the View layer is Arc 2's third item |
 | Effects | 0 | **2** | Effect runner + trigger wiring (change / schedule / demand) |
@@ -19,8 +19,9 @@ A bundle page receives:
 | `.HTML` | the rendered body, already escaped |
 | `.Extra` | every frontmatter key the parser does not name (ADR-0002) |
 | `.Style` | the reference theme's stylesheet, inlined so a bare site root needs no asset route |
-| `.Canonical` | the permalink of the variant actually served — not the URL requested, which differs when the fallback chain supplied another language |
-| `.Alternates` | every language this key exists in, as `.Lang` and `.URL`, for `hreflang` |
+| `.Canonical` | the permalink of the variant actually served — not the URL requested, which differs when the fallback chain supplied another language. **Absolute** when the site declares `base`, since a canonical link is resolved by machines rather than by the page (ADR-0039) |
+| `.Alternates` | every language this key exists in, as `.Lang` and `.URL`, for `hreflang`. Absolute on the same terms |
+| `.Site` | what the site declared about itself: `.Site.Base` and `.Site.Title`, either possibly empty |
 | `.Sequence` | the series this page sits in, absent when it sits in none (ADR-0033) |

 `.Sequence` carries the reading order and this page's place in it:
@@ -0,0 +1,59 @@
+package content
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// SettingsFile is where a site declares itself, at the site root.
+const SettingsFile = "site.yaml"
+
+// Settings are the site's own declarations (ADR-0039).
+//
+// Declared keys only: this is not a bag for arbitrary engine internals, which is how a settings file turns
+// into unbounded configuration. Section-level and bundle-level settings are not here — a bundle's own
+// frontmatter covers it, and the rest of the cascade is still parked.
+type Settings struct {
+	// Base is the site's canonical origin, without a trailing slash: "https://khosra.example". Empty when
+	// the site declares none, in which case the engine emits paths and nothing absolute.
+	Base string `yaml:"base"`
+	// Title names the site. A theme may put it in a document title or a feed; the engine only carries it.
+	Title string `yaml:"title"`
+}
+
+// LoadSettings reads site.yaml from the site root.
+//
+// A missing file is not an error, because a bare site root must still serve (ADR-0026). A malformed one is,
+// and the caller treats it as fatal: unlike a single bad bundle, which is skipped loudly (ADR-0029), a
+// broken declaration misconfigures every page on the site.
+func LoadSettings(fsys fs.FS) (Settings, error) {
+	data, err := fs.ReadFile(fsys, SettingsFile)
+	if errors.Is(err, fs.ErrNotExist) {
+		return Settings{}, nil
+	}
+	if err != nil {
+		return Settings{}, fmt.Errorf("read %s: %w", SettingsFile, err)
+	}
+	var s Settings
+	if err := yaml.Unmarshal(data, &s); err != nil {
+		return Settings{}, fmt.Errorf("parse %s: %w", SettingsFile, err)
+	}
+	s.Base = strings.TrimSuffix(strings.TrimSpace(s.Base), "/")
+	s.Title = strings.TrimSpace(s.Title)
+	return s, nil
+}
+
+// Absolute turns a path the engine emitted into a full URL.
+//
+// With no declared base it returns the path unchanged, so a site that has not said where it lives still
+// links correctly to itself — every internal path is root-relative already.
+func Absolute(base, path string) string {
+	if base == "" {
+		return path
+	}
+	return base + path
+}
@@ -0,0 +1,45 @@
+package content
+
+import (
+	"testing"
+	"testing/fstest"
+)
+
+func TestSettingsAreOptionalAndNormalised(t *testing.T) {
+	// A bare site root must still serve, so an absent file is not an error (ADR-0026).
+	got, err := LoadSettings(fstest.MapFS{})
+	if err != nil || got.Base != "" || got.Title != "" {
+		t.Fatalf("absent site.yaml gave %+v, %v — want the zero value and no error", got, err)
+	}
+	got, err = LoadSettings(fstest.MapFS{
+		SettingsFile: {Data: []byte("base: https://khosra.example/\ntitle: খসড়া\n")},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Base != "https://khosra.example" {
+		t.Errorf("base = %q — a trailing slash must be trimmed, or every URL doubles it", got.Base)
+	}
+	if got.Title != "খসড়া" {
+		t.Errorf("title = %q", got.Title)
+	}
+}
+
+func TestMalformedSettingsAreAnError(t *testing.T) {
+	// Unlike one bad bundle, which is skipped loudly (ADR-0029), this misconfigures every page, so the
+	// caller treats it as fatal.
+	if _, err := LoadSettings(fstest.MapFS{SettingsFile: {Data: []byte("base: [unclosed\n")}}); err == nil {
+		t.Error("a malformed site.yaml must be an error")
+	}
+}
+
+func TestAbsoluteOnlyPrefixesWhenABaseExists(t *testing.T) {
+	for _, c := range []struct{ base, path, want string }{
+		{"https://khosra.example", "/posts/hello/", "https://khosra.example/posts/hello/"},
+		{"", "/posts/hello/", "/posts/hello/"},
+	} {
+		if got := Absolute(c.base, c.path); got != c.want {
+			t.Errorf("Absolute(%q, %q) = %q, want %q", c.base, c.path, got, c.want)
+		}
+	}
+}
@@ -265,6 +265,20 @@ func (b Bundle) Section() string {
 	return sec
 }

+// Everything lists every variant of every bundle, sorted by key then language.
+//
+// Every *variant*, unlike a Query, which answers with one variant per key: a sitemap lists each language as
+// its own URL, because each is separately reachable.
+func (s *Site) Everything() []Bundle {
+	out := make([]Bundle, 0, len(s.byKeyLang))
+	for _, key := range s.keys() {
+		for _, lang := range s.Variants(key) {
+			out = append(out, s.byKeyLang[key+"\x00"+lang])
+		}
+	}
+	return out
+}
+
 // Sections lists every section that holds at least one bundle, sorted.
 func (s *Site) Sections() []string {
 	seen := map[string]bool{}
@@ -43,7 +43,7 @@ func wired(t *testing.T, siteFS fstest.MapFS) *render.Renderer {
 	if siteFS != nil {
 		fsys = siteFS
 	}
-	r, err := render.New(fsys, func(p render.Partial) []goldmark.Extender {
+	r, err := render.New(fsys, content.Settings{}, func(p render.Partial) []goldmark.Extender {
 		return []goldmark.Extender{New(p)}
 	})
 	if err != nil {
@@ -61,7 +61,7 @@ func TestDatesReadInTheirOwnScript(t *testing.T) {
 }

 func TestTypographerSmoothsProseAndLeavesCodeAlone(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -86,7 +86,7 @@ func TestTypographerSmoothsProseAndLeavesCodeAlone(t *testing.T) {
 }

 func TestMachineReadableOutputStaysASCII(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -38,6 +38,8 @@ type head struct {
 	Alternates []Alternate
 	// Style is the reference theme's stylesheet, inlined so a bare site root needs no asset route.
 	Style template.CSS
+	// Site is what the site declared about itself in site.yaml (ADR-0039). Zero when it declared nothing.
+	Site content.Settings
 }

 // Page is one bundle rendered.
@@ -117,6 +119,8 @@ type Renderer struct {
 	style    template.CSS
 	// files is the site root, handed to features through Origin. Nil when there is none.
 	files fs.FS
+	// settings are the site's declarations, constant for the life of the process.
+	settings content.Settings
 }

 // Partial renders a named fragment. A feature under internal/ext is handed one of these at wiring time,
@@ -169,7 +173,7 @@ func WithOrigin(pc parser.Context, origin Origin) {
 // extensions to enable. A callback rather than a parameter of feature types, because internal/render must
 // not import internal/ext — only cmd knows which features a build includes (conventions.md, ADR-0036). It
 // may be nil.
-func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, error) {
+func New(siteFS fs.FS, settings content.Settings, extend func(Partial) []goldmark.Extender) (*Renderer, error) {
 	page, err := parseSet(siteFS, "templates/base.html", "templates/page.html")
 	if err != nil {
 		return nil, fmt.Errorf("bundle templates: %w", err)
@@ -186,7 +190,7 @@ func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, err
 	if err != nil {
 		return nil, err
 	}
-	r := &Renderer{page: page, list: list, partials: partials, style: css, files: siteFS}
+	r := &Renderer{page: page, list: list, partials: partials, style: css, files: siteFS, settings: settings}
 	// The typographer smooths quotes, dashes and ellipses in authored prose and leaves code spans alone,
 	// because it works on the parsed tree rather than the text. That is the only change the engine makes to
 	// an author's words (ADR-0034), and it is a parser option rather than a render transform, so it does
@@ -202,6 +206,25 @@ func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, err
 	return r, nil
 }

+// head builds the document shell every kind of page shares.
+//
+// canonical arrives as a path and leaves absolute when the site declared a base: a canonical link and an
+// hreflang are read by machines that resolve neither against the page (ADR-0039).
+func (r *Renderer) head(title, lang, canonical string) head {
+	return head{
+		Title:     title,
+		Lang:      lang,
+		Canonical: r.absolute(canonical),
+		Style:     r.style,
+		Site:      r.settings,
+	}
+}
+
+// absolute is the site's own URL for a path the engine emitted, or the path itself when no base is declared.
+func (r *Renderer) absolute(path string) string {
+	return content.Absolute(r.settings.Base, path)
+}
+
 // Partial renders one named fragment. A missing template is an error the caller degrades on, never a
 // failed request (extensions.md rule 5).
 func (r *Renderer) Partial(name string, data Fragment) ([]byte, error) {
@@ -276,14 +299,14 @@ func (r *Renderer) Bundle(b content.Bundle, served string, variants []string, se
 		title = b.Key
 	}
 	p := Page{
-		head:     head{Title: title, Lang: served, Canonical: content.URL(b.Key, served), Style: r.style},
+		head:     r.head(title, served, content.URL(b.Key, served)),
 		Key:      b.Key,
 		HTML:     template.HTML(body.String()),
 		Extra:    b.Extra,
 		Sequence: r.sequence(seq, served),
 	}
 	for _, l := range variants {
-		p.Alternates = append(p.Alternates, Alternate{Lang: l, URL: content.URL(b.Key, l)})
+		p.Alternates = append(p.Alternates, Alternate{Lang: l, URL: r.absolute(content.URL(b.Key, l))})
 	}
 	return r.execute(r.page, p, b.Key)
 }
@@ -363,7 +386,7 @@ func (r *Renderer) paginate(title, lang, canonical string, all []content.Bundle,
 	start := (page - 1) * content.PerPage
 	end := min(start+content.PerPage, len(all))
 	l := List{
-		head:  head{Title: title, Lang: lang, Canonical: canonical, Style: r.style},
+		head:  r.head(title, lang, canonical),
 		Page:  page,
 		Pages: pages,
 	}
@@ -9,7 +9,7 @@ import (
 )

 func TestBundleRendersMarkdownIntoTheTheme(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -33,7 +33,7 @@ func TestBundleRendersMarkdownIntoTheTheme(t *testing.T) {
 }

 func TestBundleWithoutTitleFallsBackToKey(t *testing.T) {
-	r, err := New(nil, nil)
+	r, err := New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -54,7 +54,7 @@ func TestSiteOverridesOneBlockAndInheritsTheRest(t *testing.T) {
 	siteFS := fstest.MapFS{
 		"templates/page.html": {Data: []byte(`{{define "main"}}<section class="mine">{{.Title}}</section>{{end}}`)},
 	}
-	r, err := New(siteFS, nil)
+	r, err := New(siteFS, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -82,7 +82,7 @@ func TestAListingOverrideDoesNotLeakIntoBundlePages(t *testing.T) {
 	siteFS := fstest.MapFS{
 		"templates/list.html": {Data: []byte(`{{define "main"}}LISTING ONLY{{end}}`)},
 	}
-	r, err := New(siteFS, nil)
+	r, err := New(siteFS, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -101,7 +101,7 @@ func TestAListingOverrideDoesNotLeakIntoBundlePages(t *testing.T) {

 func TestSiteStylesheetReplacesTheReferenceOne(t *testing.T) {
 	siteFS := fstest.MapFS{"templates/theme.css": {Data: []byte("body{color:rebeccapurple}")}}
-	r, err := New(siteFS, nil)
+	r, err := New(siteFS, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -114,3 +114,50 @@ func TestSiteStylesheetReplacesTheReferenceOne(t *testing.T) {
 		t.Error("the site stylesheet should replace the reference one")
 	}
 }
+
+func TestADeclaredBaseMakesMachineReadableURLsAbsolute(t *testing.T) {
+	// A canonical link and an hreflang are read by machines that resolve neither against the page, so both
+	// go absolute as soon as the site says where it lives (ADR-0039).
+	r, err := New(nil, content.Settings{Base: "https://khosra.example", Title: "Khosra"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, err := content.Parse("posts/hello.md", []byte("---\ntitle: Hello\n---\nhi\n"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	out, err := r.Bundle(b, "en", []string{"en", "bn"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := string(out)
+	for _, want := range []string{
+		`rel="canonical" href="https://khosra.example/posts/hello/"`,
+		`hreflang="bn" href="https://khosra.example/bn/posts/hello/"`,
+		`property="og:url" content="https://khosra.example/posts/hello/"`,
+		`<title>Hello · Khosra</title>`,
+	} {
+		if !strings.Contains(got, want) {
+			t.Errorf("missing %q:\n%s", want, got)
+		}
+	}
+}
+
+func TestWithoutABaseEverythingStaysRelative(t *testing.T) {
+	r, err := New(nil, content.Settings{}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b, _ := content.Parse("posts/hello.md", []byte("---\ntitle: Hello\n---\nhi\n"))
+	out, err := r.Bundle(b, "en", []string{"en"}, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	got := string(out)
+	if !strings.Contains(got, `rel="canonical" href="/posts/hello/"`) {
+		t.Errorf("a site that declared no base still links to itself:\n%s", got)
+	}
+	if strings.Contains(got, "og:site_name") {
+		t.Error("no declared title means no site_name tag, rather than an empty one")
+	}
+}
@@ -4,11 +4,18 @@
 <head>
 <meta charset="utf-8">
 <meta name="viewport" content="width=device-width, initial-scale=1">
-<title>{{.Title}}</title>
+<title>{{.Title}}{{if .Site.Title}} · {{.Site.Title}}{{end}}</title>
 <link rel="canonical" href="{{.Canonical}}">
 {{- range .Alternates}}
 <link rel="alternate" hreflang="{{.Lang}}" href="{{.URL}}">
 {{- end}}
+<meta property="og:title" content="{{.Title}}">
+<meta property="og:url" content="{{.Canonical}}">
+<meta property="og:type" content="article">
+<meta property="og:locale" content="{{.Lang}}">
+{{- if .Site.Title}}
+<meta property="og:site_name" content="{{.Site.Title}}">
+{{- end}}
 <style>{{.Style}}</style>
 </head>
 <body>
@@ -0,0 +1,71 @@
+package web
+
+import (
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"net/http"
+	"strings"
+
+	"khosra/internal/content"
+)
+
+// robots and sitemap are the two files a crawler looks for by exact name.
+const (
+	robotsPath  = "/robots.txt"
+	sitemapPath = "/sitemap.xml"
+)
+
+// serveRobots answers /robots.txt, preferring the site's own file.
+//
+// A site that ships robots.txt has said something deliberate, so it is served verbatim; otherwise the engine
+// emits the minimum that is true — everything is public, and here is the sitemap. The Sitemap line only
+// appears with a declared base, because a relative sitemap reference is not something a crawler accepts.
+func serveRobots(w http.ResponseWriter, req *http.Request, siteFS fs.FS, base string) {
+	if siteFS != nil {
+		if data, err := fs.ReadFile(siteFS, "robots.txt"); err == nil {
+			writeAs(w, "text/plain; charset=utf-8", data, "robots.txt")
+			return
+		}
+	}
+	var out strings.Builder
+	out.WriteString("User-agent: *\nDisallow:\n")
+	if base != "" {
+		fmt.Fprintf(&out, "Sitemap: %s\n", content.Absolute(base, sitemapPath))
+	}
+	writeAs(w, "text/plain; charset=utf-8", []byte(out.String()), "robots.txt")
+}
+
+// serveSitemap answers /sitemap.xml with every bundle in every language it exists in.
+//
+// It needs a declared base: the sitemap format has no room for a relative URL, so without one the honest
+// answer is that this file does not exist rather than a file full of paths no crawler can use (ADR-0039).
+// Every URL comes from content.URL, like every other path the engine emits, so a sitemap can never disagree
+// with what is actually served.
+func serveSitemap(w http.ResponseWriter, req *http.Request, site *content.Site, base string) {
+	if base == "" {
+		slog.Warn("no sitemap: the site declares no base URL", "file", content.SettingsFile)
+		http.NotFound(w, req)
+		return
+	}
+	var out strings.Builder
+	out.WriteString(`<?xml version="1.0" encoding="utf-8"?>` + "\n")
+	out.WriteString(`<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">` + "\n")
+	for _, entry := range site.Everything() {
+		fmt.Fprintf(&out, "<url><loc>%s</loc>", xmlEscape(content.Absolute(base, content.URL(entry.Key, entry.Lang))))
+		if !entry.Date.IsZero() {
+			fmt.Fprintf(&out, "<lastmod>%s</lastmod>", entry.Date.Format("2006-01-02"))
+		}
+		out.WriteString("</url>\n")
+	}
+	out.WriteString("</urlset>\n")
+	writeAs(w, "application/xml; charset=utf-8", []byte(out.String()), "sitemap.xml")
+}
+
+// xmlEscape escapes the five characters XML reserves. A URL should contain none of them, and a sitemap that
+// silently breaks on the one that does is worse than a slightly paranoid replacement.
+func xmlEscape(s string) string {
+	return strings.NewReplacer(
+		"&", "&amp;", "<", "&lt;", ">", "&gt;", `"`, "&quot;", "'", "&apos;",
+	).Replace(s)
+}
@@ -0,0 +1,101 @@
+package web
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	"khosra/internal/content"
+	"khosra/internal/render"
+)
+
+func crawlerHandler(t *testing.T, settings content.Settings, extra fstest.MapFS) http.Handler {
+	t.Helper()
+	fsys := fstest.MapFS{
+		"content/posts/hello.md":    {Data: []byte("---\ntitle: Hello\ndate: 2026-03-08\n---\nx\n")},
+		"content/posts/hello.bn.md": {Data: []byte("---\ntitle: হ্যালো\ndate: 2026-03-08\n---\nx\n")},
+		"content/pages/about.md":    {Data: []byte("---\ntitle: About\n---\nx\n")},
+	}
+	for name, file := range extra {
+		fsys[name] = file
+	}
+	bundles, err := content.Scan(fsys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, err := render.New(nil, settings, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return Handler(content.NewSite(bundles), r, fsys, settings)
+}
+
+func TestSitemapListsEveryVariantAbsolutely(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("got %d, want 200", rec.Code)
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") {
+		t.Errorf("content-type = %q — a sitemap served as HTML is a sitemap nothing reads", ct)
+	}
+	body := rec.Body.String()
+	for _, want := range []string{
+		"<loc>https://khosra.example/posts/hello/</loc>",
+		"<loc>https://khosra.example/bn/posts/hello/</loc>", // each language is its own URL
+		"<loc>https://khosra.example/pages/about/</loc>",
+		"<lastmod>2026-03-08</lastmod>",
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("missing %q:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, "<lastmod></lastmod>") {
+		t.Error("an undated bundle should carry no lastmod at all")
+	}
+}
+
+func TestNoBaseMeansNoSitemap(t *testing.T) {
+	// The format has no room for a relative URL, so the honest answer is that the file does not exist
+	// (ADR-0039) rather than one full of paths no crawler can resolve.
+	h := crawlerHandler(t, content.Settings{}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	if rec.Code != http.StatusNotFound {
+		t.Errorf("got %d, want 404", rec.Code)
+	}
+}
+
+func TestRobotsIsGeneratedOrTheSitesOwn(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	body := rec.Body.String()
+	if !strings.Contains(body, "User-agent: *") || !strings.Contains(body, "Sitemap: https://khosra.example/sitemap.xml") {
+		t.Errorf("generated robots should point at the sitemap:\n%s", body)
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
+		t.Errorf("content-type = %q", ct)
+	}
+
+	// A site that ships its own has said something deliberate.
+	h = crawlerHandler(t, content.Settings{Base: "https://khosra.example"},
+		fstest.MapFS{"robots.txt": {Data: []byte("User-agent: *\nDisallow: /drafts/\n")}})
+	rec = httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	if got := rec.Body.String(); !strings.Contains(got, "Disallow: /drafts/") || strings.Contains(got, "Sitemap:") {
+		t.Errorf("the site's own robots.txt should be served verbatim:\n%s", got)
+	}
+}
+
+func TestRobotsWithoutABaseOmitsTheSitemapLine(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	if got := rec.Body.String(); strings.Contains(got, "Sitemap:") {
+		t.Errorf("a relative sitemap reference is not something a crawler accepts:\n%s", got)
+	}
+}
@@ -15,11 +15,19 @@ import (
 // Handler serves a site.
 //
 // One mux entry, because URL shape is the resolver's business rather than the mux's: see resolve.
-func Handler(site *content.Site, r *render.Renderer, siteFS fs.FS) http.Handler {
+func Handler(site *content.Site, r *render.Renderer, siteFS fs.FS, settings content.Settings) http.Handler {
 	mux := http.NewServeMux()
 	mux.HandleFunc("GET /", func(w http.ResponseWriter, req *http.Request) {
 		serve(w, req, site, r)
 	})
+	// Two exact paths a crawler asks for by name, so they are mux entries rather than resolver cases: no
+	// bundle can own them, since a key always sits under a section.
+	mux.HandleFunc("GET "+robotsPath, func(w http.ResponseWriter, req *http.Request) {
+		serveRobots(w, req, siteFS, settings.Base)
+	})
+	mux.HandleFunc("GET "+sitemapPath, func(w http.ResponseWriter, req *http.Request) {
+		serveSitemap(w, req, site, settings.Base)
+	})
 	if siteFS != nil {
 		if sub, err := fs.Sub(siteFS, "static"); err == nil {
 			mux.Handle("GET /static/", http.StripPrefix("/static/", serveStatic(sub)))
@@ -98,7 +106,16 @@ func serveTags(w http.ResponseWriter, req *http.Request, site *content.Site, r *

 // write sends a rendered page, logging a failed write rather than pretending it succeeded.
 func write(w http.ResponseWriter, out []byte, what string) {
-	w.Header().Set("Content-Type", "text/html; charset=utf-8")
+	writeAs(w, "text/html; charset=utf-8", out, what)
+}
+
+// writeAs sends bytes with the type they actually are.
+//
+// Separate from write because headers are only sent with the first byte, so a handler that set its own type
+// before calling write would have had it silently replaced by HTML — which is how a sitemap ends up served
+// as a web page.
+func writeAs(w http.ResponseWriter, contentType string, out []byte, what string) {
+	w.Header().Set("Content-Type", contentType)
 	if _, err := w.Write(out); err != nil {
 		slog.Warn("write failed", "what", what, "err", err)
 	}
@@ -24,11 +24,11 @@ func testHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }

 func TestServeBundleAtItsPermalink(t *testing.T) {
@@ -72,11 +72,11 @@ func multilingualHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }

 func TestPrefixedLanguageServesThatVariant(t *testing.T) {
@@ -124,11 +124,11 @@ func aliasHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }

 func TestAliasRedirectsToCanonical(t *testing.T) {
@@ -171,11 +171,11 @@ func listingHandler(t *testing.T, n int) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, fsys)
+	return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 }

 func TestSectionIndexListsNewestFirst(t *testing.T) {
@@ -239,11 +239,11 @@ func TestStaticFilesAreServedAndDirectoriesAreNot(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(fsys, nil)
+	r, err := render.New(fsys, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	h := Handler(content.NewSite(bundles), r, fsys)
+	h := Handler(content.NewSite(bundles), r, fsys, content.Settings{})
 	for path, want := range map[string]int{
 		"/static/style.css":    http.StatusOK,
 		"/static/img/logo.svg": http.StatusOK,
@@ -280,11 +280,11 @@ func TestAStaticPathThatEscapesTheRootIs404(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(fsys, nil)
+	r, err := render.New(fsys, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	h := Handler(content.NewSite(nil), r, fsys)
+	h := Handler(content.NewSite(nil), r, fsys, content.Settings{})
 	for path, want := range map[string]int{
 		"/static/ok.css":     http.StatusOK,
 		"/static/escape.txt": http.StatusNotFound,
@@ -313,11 +313,11 @@ func seriesHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, nil)
+	return Handler(content.NewSite(bundles), r, nil, content.Settings{})
 }

 func TestSequenceNavigationLinksNeighbours(t *testing.T) {
@@ -403,11 +403,11 @@ func tagHandler(t *testing.T) http.Handler {
 	if err != nil {
 		t.Fatal(err)
 	}
-	r, err := render.New(nil, nil)
+	r, err := render.New(nil, content.Settings{}, nil)
 	if err != nil {
 		t.Fatal(err)
 	}
-	return Handler(content.NewSite(bundles), r, nil)
+	return Handler(content.NewSite(bundles), r, nil, content.Settings{})
 }

 func TestGlobalTagListingSpansSectionsGroupedByOne(t *testing.T) {