move robots and sitemap out of core, and raise the ceiling on purpose

Item 0 of the roadmap's order of work, and it blocked everything after it: core
sat at 2965 of 3000 while the review scheduled four core-bound items, the first
of which — logging — wanted the whole remainder.

/robots.txt and /sitemap.xml are exact paths somebody else's software asks for by
name. They own no core concept and pass every test the architecture applies to a
feature; they lived in internal/web only because a feature could not own a route
until ADR-0081. internal/ext/discover/ now holds them. Core 2965 → 2913.

The seam gained one parameter to make it possible: a func() *content.Site, since a
sitemap must list what is served now and the index is swapped whole on every
rebuild (ADR-0077). A captured pointer would have frozen the site at startup —
which is the kind of bug that only shows up after a rebuild, in production.

The ceiling rises to 3400 as well as the move, because the move alone could not buy
the room. feed.go and web/extras.go cannot follow discover out: a feed lives at
/{section}/feed.xml and extras under a bundle's own URL, so both are resolver cases
while the seam mounts exact paths only. Raising by the minimum that unblocks one
item produces a ceiling nobody believes, so 3400 fits the View cluster with
headroom. HARNESS.md asks that a raise be read as evidence something belongs in
ext before evidence the number was small; both readings were true, so both actions
were taken.

web no longer reserves those two paths, so a clash between features is wire.go's:
it merges route maps in declaration order, keeps the earlier claim, logs the loser.
Verified — a site shipping root/robots.txt starts, serves the engine's robots.txt,
and logs the passthrough claim, where an unguarded mux.Handle would have panicked.

Evidence: robots.txt and sitemap.xml are byte-identical before and after the move
against the demo site (67 and 2701 bytes, cmp clean), and the sitemap keeps its
application/xml type.

One real cost, recorded in both places rather than hidden. internal/web's
visibility test asserted that a listing, a feed *and* a sitemap all hide
unpublished bundles — one property, one test, because all three share a Query. The
sitemap half moved to the feature instead of a web test importing ext, which would
invert the one-way layering the architecture gate enforces. That property is now
asserted twice, once per package owning a surface.

Three gates caught real mistakes on the way: the staged-tree check found a partial
stage where git rm had staged a deletion while the caller edits were unstaged, the
coupling gates demanded state.md and HARNESS.md, and the nesting advisory rejected
a closure that put the merge loop one level too deep — fixed by making it a plain
function rather than tolerated.

Extensions 6 → 7. Routing cases unmoved: exact paths are mux entries, never
resolver cases, which is what that counter's exclusion column already said.

13 files. Core 2913/3400, ext 2495/3500.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

internal/ext/discover/discover_test.go

@@ -0,0 +1,139 @@
+package discover
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"testing/fstest"
+
+	"khosra/internal/content"
+)
+
+// crawlerHandler mounts just this feature's routes, which is all these two paths need — no renderer, no
+// resolver, no mux full of bundle handling.
+func crawlerHandler(t *testing.T, settings content.Settings, extra fstest.MapFS) http.Handler {
+	t.Helper()
+	fsys := fstest.MapFS{
+		"content/posts/hello.md":    {Data: []byte("---\ntitle: Hello\ndate: 2026-03-08\n---\nx\n")},
+		"content/posts/hello.bn.md": {Data: []byte("---\ntitle: হ্যালো\ndate: 2026-03-08\n---\nx\n")},
+		"content/pages/about.md":    {Data: []byte("---\ntitle: About\n---\nx\n")},
+	}
+	for name, file := range extra {
+		fsys[name] = file
+	}
+	bundles, err := content.Scan(fsys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	site := content.NewSite(bundles)
+	mux := http.NewServeMux()
+	for pattern, handler := range Routes(fsys, settings, func() *content.Site { return site }) {
+		mux.Handle("GET "+pattern, handler)
+	}
+	return mux
+}
+
+func TestSitemapListsEveryVariantAbsolutely(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	if rec.Code != http.StatusOK {
+		t.Fatalf("got %d, want 200", rec.Code)
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") {
+		t.Errorf("content-type = %q — a sitemap served as HTML is a sitemap nothing reads", ct)
+	}
+	body := rec.Body.String()
+	for _, want := range []string{
+		"<loc>https://khosra.example/posts/hello/</loc>",
+		"<loc>https://khosra.example/bn/posts/hello/</loc>", // each language is its own URL
+		"<loc>https://khosra.example/pages/about/</loc>",
+		"<lastmod>2026-03-08</lastmod>",
+	} {
+		if !strings.Contains(body, want) {
+			t.Errorf("missing %q:\n%s", want, body)
+		}
+	}
+	if strings.Contains(body, "<lastmod></lastmod>") {
+		t.Error("an undated bundle should carry no lastmod at all")
+	}
+}
+
+func TestNoBaseMeansNoSitemap(t *testing.T) {
+	// The format has no room for a relative URL, so the honest answer is that the file does not exist
+	// (ADR-0039) rather than one full of paths no crawler can resolve.
+	h := crawlerHandler(t, content.Settings{}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	if rec.Code != http.StatusNotFound {
+		t.Errorf("got %d, want 404", rec.Code)
+	}
+}
+
+func TestRobotsIsGeneratedOrTheSitesOwn(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	body := rec.Body.String()
+	if !strings.Contains(body, "User-agent: *") || !strings.Contains(body, "Sitemap: https://khosra.example/sitemap.xml") {
+		t.Errorf("generated robots should point at the sitemap:\n%s", body)
+	}
+	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
+		t.Errorf("content-type = %q", ct)
+	}
+
+	// A site that ships its own has said something deliberate.
+	h = crawlerHandler(t, content.Settings{Base: "https://khosra.example"},
+		fstest.MapFS{"robots.txt": {Data: []byte("User-agent: *\nDisallow: /drafts/\n")}})
+	rec = httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	if got := rec.Body.String(); !strings.Contains(got, "Disallow: /drafts/") || strings.Contains(got, "Sitemap:") {
+		t.Errorf("the site's own robots.txt should be served verbatim:\n%s", got)
+	}
+}
+
+func TestRobotsWithoutABaseOmitsTheSitemapLine(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{}, nil)
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
+	if got := rec.Body.String(); strings.Contains(got, "Sitemap:") {
+		t.Errorf("a relative sitemap reference is not something a crawler accepts:\n%s", got)
+	}
+}
+
+// A slug moves a bundle's address without moving its key, so a sitemap built from keys would advertise URLs
+// that 404. Moved here from internal/web with the sitemap itself (ADR-0085).
+func TestTheSitemapUsesTheSluggedAddress(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{
+		"content/posts/hello-world.md": {Data: []byte("---\ntitle: Hello\ndate: 2026-03-01\nslug: ekti-post\n---\nx\n")},
+	})
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	body := rec.Body.String()
+	if !strings.Contains(body, "/posts/ekti-post/") || strings.Contains(body, "hello-world") {
+		t.Errorf("a sitemap that disagrees with what is served is worse than none:\n%s", body)
+	}
+}
+
+// The sitemap goes through the same Query every listing does, so a draft or a future-dated bundle is absent
+// from it for the same reason it is absent from a section page. The other half of this lives in
+// internal/web's visibility test, which owns the surfaces that stayed there (ADR-0085).
+func TestTheSitemapOmitsUnpublishedBundles(t *testing.T) {
+	h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{
+		"content/art/draft/index.md":  {Data: []byte("---\ntitle: Draft\ndraft: true\n---\nx\n")},
+		"content/art/future/index.md": {Data: []byte("---\ntitle: Future\ndate: 2099-01-01\n---\nx\n")},
+		"content/art/live/index.md":   {Data: []byte("---\ntitle: Live\ndate: 2020-01-01\n---\nx\n")},
+	})
+	rec := httptest.NewRecorder()
+	h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
+	body := rec.Body.String()
+	if !strings.Contains(body, "/art/live/") {
+		t.Errorf("the sitemap lost a published bundle:\n%s", body)
+	}
+	for _, hidden := range []string{"draft", "future"} {
+		if strings.Contains(body, hidden) {
+			t.Errorf("the sitemap leaked the %s bundle:\n%s", hidden, body)
+		}
+	}
+}