package discover import ( "net/http" "net/http/httptest" "strings" "testing" "testing/fstest" "khosra/internal/content" ) // crawlerHandler mounts just this feature's routes, which is all these two paths need — no renderer, no // resolver, no mux full of bundle handling. func crawlerHandler(t *testing.T, settings content.Settings, extra fstest.MapFS) http.Handler { t.Helper() fsys := fstest.MapFS{ "content/posts/hello.md": {Data: []byte("---\ntitle: Hello\ndate: 2026-03-08\n---\nx\n")}, "content/posts/hello.bn.md": {Data: []byte("---\ntitle: হ্যালো\ndate: 2026-03-08\n---\nx\n")}, "content/pages/about.md": {Data: []byte("---\ntitle: About\n---\nx\n")}, } for name, file := range extra { fsys[name] = file } bundles, err := content.Scan(fsys) if err != nil { t.Fatal(err) } site := content.NewSite(bundles) mux := http.NewServeMux() for pattern, handler := range Routes(fsys, settings, func() *content.Site { return site }) { mux.Handle("GET "+pattern, handler) } return mux } func TestSitemapListsEveryVariantAbsolutely(t *testing.T) { h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)) if rec.Code != http.StatusOK { t.Fatalf("got %d, want 200", rec.Code) } if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") { t.Errorf("content-type = %q — a sitemap served as HTML is a sitemap nothing reads", ct) } body := rec.Body.String() for _, want := range []string{ "https://khosra.example/posts/hello/", "https://khosra.example/bn/posts/hello/", // each language is its own URL "https://khosra.example/pages/about/", "2026-03-08", } { if !strings.Contains(body, want) { t.Errorf("missing %q:\n%s", want, body) } } if strings.Contains(body, "") { t.Error("an undated bundle should carry no lastmod at all") } } func TestNoBaseMeansNoSitemap(t *testing.T) { // The format has no room for a relative URL, so the honest answer is that the file does not exist // (ADR-0039) rather than one full of paths no crawler can resolve. h := crawlerHandler(t, content.Settings{}, nil) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)) if rec.Code != http.StatusNotFound { t.Errorf("got %d, want 404", rec.Code) } } func TestRobotsIsGeneratedOrTheSitesOwn(t *testing.T) { h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil)) body := rec.Body.String() if !strings.Contains(body, "User-agent: *") || !strings.Contains(body, "Sitemap: https://khosra.example/sitemap.xml") { t.Errorf("generated robots should point at the sitemap:\n%s", body) } if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { t.Errorf("content-type = %q", ct) } // A site that ships its own has said something deliberate. h = crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{"robots.txt": {Data: []byte("User-agent: *\nDisallow: /drafts/\n")}}) rec = httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil)) if got := rec.Body.String(); !strings.Contains(got, "Disallow: /drafts/") || strings.Contains(got, "Sitemap:") { t.Errorf("the site's own robots.txt should be served verbatim:\n%s", got) } } func TestRobotsWithoutABaseOmitsTheSitemapLine(t *testing.T) { h := crawlerHandler(t, content.Settings{}, nil) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil)) if got := rec.Body.String(); strings.Contains(got, "Sitemap:") { t.Errorf("a relative sitemap reference is not something a crawler accepts:\n%s", got) } } // A slug moves a bundle's address without moving its key, so a sitemap built from keys would advertise URLs // that 404. Moved here from internal/web with the sitemap itself (ADR-0085). func TestTheSitemapUsesTheSluggedAddress(t *testing.T) { h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{ "content/posts/hello-world.md": {Data: []byte("---\ntitle: Hello\ndate: 2026-03-01\nslug: ekti-post\n---\nx\n")}, }) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)) body := rec.Body.String() if !strings.Contains(body, "/posts/ekti-post/") || strings.Contains(body, "hello-world") { t.Errorf("a sitemap that disagrees with what is served is worse than none:\n%s", body) } } // The sitemap goes through the same Query every listing does, so a draft or a future-dated bundle is absent // from it for the same reason it is absent from a section page. The other half of this lives in // internal/web's visibility test, which owns the surfaces that stayed there (ADR-0085). func TestTheSitemapOmitsUnpublishedBundles(t *testing.T) { h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{ "content/art/draft/index.md": {Data: []byte("---\ntitle: Draft\ndraft: true\n---\nx\n")}, "content/art/future/index.md": {Data: []byte("---\ntitle: Future\ndate: 2099-01-01\n---\nx\n")}, "content/art/live/index.md": {Data: []byte("---\ntitle: Live\ndate: 2020-01-01\n---\nx\n")}, }) rec := httptest.NewRecorder() h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)) body := rec.Body.String() if !strings.Contains(body, "/art/live/") { t.Errorf("the sitemap lost a published bundle:\n%s", body) } for _, hidden := range []string{"draft", "future"} { if strings.Contains(body, hidden) { t.Errorf("the sitemap leaked the %s bundle:\n%s", hidden, body) } } }