serve robots.txt and sitemap.xml

Two exact paths a crawler asks for by name, so they are mux entries rather than
resolver cases — no bundle can collide, since a key always sits under a section.

robots.txt at the site root is served verbatim, because a site that ships one has
said something deliberate; otherwise the engine emits the minimum that is true and
points at the sitemap. The sitemap lists every bundle in every language it exists
in, since each variant is separately reachable, with lastmod only where a bundle
has a date. Every URL comes from content.URL like every other path the engine
emits, so a sitemap cannot disagree with what is actually served.

Both need a declared base. Without one the sitemap answers 404 rather than listing
paths no crawler can resolve, and robots omits the Sitemap line rather than
writing a relative one.

write() was setting text/html for every caller, and headers only go out with the
first byte — so a handler setting its own type would have had it silently replaced,
which is how a sitemap gets served as a web page. It now splits into write and
writeAs, and the tests assert the content types rather than only the bodies.

internal/content/settings.go

@@ -0,0 +1,59 @@
+package content
+
+import (
+	"errors"
+	"fmt"
+	"io/fs"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+)
+
+// SettingsFile is where a site declares itself, at the site root.
+const SettingsFile = "site.yaml"
+
+// Settings are the site's own declarations (ADR-0039).
+//
+// Declared keys only: this is not a bag for arbitrary engine internals, which is how a settings file turns
+// into unbounded configuration. Section-level and bundle-level settings are not here — a bundle's own
+// frontmatter covers it, and the rest of the cascade is still parked.
+type Settings struct {
+	// Base is the site's canonical origin, without a trailing slash: "https://khosra.example". Empty when
+	// the site declares none, in which case the engine emits paths and nothing absolute.
+	Base string `yaml:"base"`
+	// Title names the site. A theme may put it in a document title or a feed; the engine only carries it.
+	Title string `yaml:"title"`
+}
+
+// LoadSettings reads site.yaml from the site root.
+//
+// A missing file is not an error, because a bare site root must still serve (ADR-0026). A malformed one is,
+// and the caller treats it as fatal: unlike a single bad bundle, which is skipped loudly (ADR-0029), a
+// broken declaration misconfigures every page on the site.
+func LoadSettings(fsys fs.FS) (Settings, error) {
+	data, err := fs.ReadFile(fsys, SettingsFile)
+	if errors.Is(err, fs.ErrNotExist) {
+		return Settings{}, nil
+	}
+	if err != nil {
+		return Settings{}, fmt.Errorf("read %s: %w", SettingsFile, err)
+	}
+	var s Settings
+	if err := yaml.Unmarshal(data, &s); err != nil {
+		return Settings{}, fmt.Errorf("parse %s: %w", SettingsFile, err)
+	}
+	s.Base = strings.TrimSuffix(strings.TrimSpace(s.Base), "/")
+	s.Title = strings.TrimSpace(s.Title)
+	return s, nil
+}
+
+// Absolute turns a path the engine emitted into a full URL.
+//
+// With no declared base it returns the path unchanged, so a site that has not said where it lives still
+// links correctly to itself — every internal path is root-relative already.
+func Absolute(base, path string) string {
+	if base == "" {
+		return path
+	}
+	return base + path
+}

internal/content/settings_test.go

@@ -0,0 +1,45 @@
+package content
+
+import (
+	"testing"
+	"testing/fstest"
+)
+
+func TestSettingsAreOptionalAndNormalised(t *testing.T) {
+	// A bare site root must still serve, so an absent file is not an error (ADR-0026).
+	got, err := LoadSettings(fstest.MapFS{})
+	if err != nil || got.Base != "" || got.Title != "" {
+		t.Fatalf("absent site.yaml gave %+v, %v — want the zero value and no error", got, err)
+	}
+	got, err = LoadSettings(fstest.MapFS{
+		SettingsFile: {Data: []byte("base: https://khosra.example/\ntitle: খসড়া\n")},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if got.Base != "https://khosra.example" {
+		t.Errorf("base = %q — a trailing slash must be trimmed, or every URL doubles it", got.Base)
+	}
+	if got.Title != "খসড়া" {
+		t.Errorf("title = %q", got.Title)
+	}
+}
+
+func TestMalformedSettingsAreAnError(t *testing.T) {
+	// Unlike one bad bundle, which is skipped loudly (ADR-0029), this misconfigures every page, so the
+	// caller treats it as fatal.
+	if _, err := LoadSettings(fstest.MapFS{SettingsFile: {Data: []byte("base: [unclosed\n")}}); err == nil {
+		t.Error("a malformed site.yaml must be an error")
+	}
+}
+
+func TestAbsoluteOnlyPrefixesWhenABaseExists(t *testing.T) {
+	for _, c := range []struct{ base, path, want string }{
+		{"https://khosra.example", "/posts/hello/", "https://khosra.example/posts/hello/"},
+		{"", "/posts/hello/", "/posts/hello/"},
+	} {
+		if got := Absolute(c.base, c.path); got != c.want {
+			t.Errorf("Absolute(%q, %q) = %q, want %q", c.base, c.path, got, c.want)
+		}
+	}
+}

internal/content/site.go

@@ -265,6 +265,20 @@ func (b Bundle) Section() string {
 	return sec
 }
 
+// Everything lists every variant of every bundle, sorted by key then language.
+//
+// Every *variant*, unlike a Query, which answers with one variant per key: a sitemap lists each language as
+// its own URL, because each is separately reachable.
+func (s *Site) Everything() []Bundle {
+	out := make([]Bundle, 0, len(s.byKeyLang))
+	for _, key := range s.keys() {
+		for _, lang := range s.Variants(key) {
+			out = append(out, s.byKeyLang[key+"\x00"+lang])
+		}
+	}
+	return out
+}
+
 // Sections lists every section that holds at least one bundle, sorted.
 func (s *Site) Sections() []string {
 	seen := map[string]bool{}