package discover
import (
"net/http"
"net/http/httptest"
"strings"
"testing"
"testing/fstest"
"khosra/internal/content"
)
// crawlerHandler mounts just this feature's routes, which is all these two paths need — no renderer, no
// resolver, no mux full of bundle handling.
func crawlerHandler(t *testing.T, settings content.Settings, extra fstest.MapFS) http.Handler {
t.Helper()
fsys := fstest.MapFS{
"content/posts/hello.md": {Data: []byte("---\ntitle: Hello\ndate: 2026-03-08\n---\nx\n")},
"content/posts/hello.bn.md": {Data: []byte("---\ntitle: হ্যালো\ndate: 2026-03-08\n---\nx\n")},
"content/pages/about.md": {Data: []byte("---\ntitle: About\n---\nx\n")},
}
for name, file := range extra {
fsys[name] = file
}
bundles, err := content.Scan(fsys)
if err != nil {
t.Fatal(err)
}
site := content.NewSite(bundles)
mux := http.NewServeMux()
for pattern, handler := range Routes(fsys, settings, func() *content.Site { return site }) {
mux.Handle("GET "+pattern, handler)
}
return mux
}
func TestSitemapListsEveryVariantAbsolutely(t *testing.T) {
h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
if rec.Code != http.StatusOK {
t.Fatalf("got %d, want 200", rec.Code)
}
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") {
t.Errorf("content-type = %q — a sitemap served as HTML is a sitemap nothing reads", ct)
}
body := rec.Body.String()
for _, want := range []string{
"https://khosra.example/posts/hello/",
"https://khosra.example/bn/posts/hello/", // each language is its own URL
"https://khosra.example/pages/about/",
"2026-03-08",
} {
if !strings.Contains(body, want) {
t.Errorf("missing %q:\n%s", want, body)
}
}
if strings.Contains(body, "") {
t.Error("an undated bundle should carry no lastmod at all")
}
}
func TestNoBaseMeansNoSitemap(t *testing.T) {
// The format has no room for a relative URL, so the honest answer is that the file does not exist
// (ADR-0039) rather than one full of paths no crawler can resolve.
h := crawlerHandler(t, content.Settings{}, nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
if rec.Code != http.StatusNotFound {
t.Errorf("got %d, want 404", rec.Code)
}
}
func TestRobotsIsGeneratedOrTheSitesOwn(t *testing.T) {
h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
body := rec.Body.String()
if !strings.Contains(body, "User-agent: *") || !strings.Contains(body, "Sitemap: https://khosra.example/sitemap.xml") {
t.Errorf("generated robots should point at the sitemap:\n%s", body)
}
if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") {
t.Errorf("content-type = %q", ct)
}
// A site that ships its own has said something deliberate.
h = crawlerHandler(t, content.Settings{Base: "https://khosra.example"},
fstest.MapFS{"robots.txt": {Data: []byte("User-agent: *\nDisallow: /drafts/\n")}})
rec = httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
if got := rec.Body.String(); !strings.Contains(got, "Disallow: /drafts/") || strings.Contains(got, "Sitemap:") {
t.Errorf("the site's own robots.txt should be served verbatim:\n%s", got)
}
}
func TestRobotsWithoutABaseOmitsTheSitemapLine(t *testing.T) {
h := crawlerHandler(t, content.Settings{}, nil)
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil))
if got := rec.Body.String(); strings.Contains(got, "Sitemap:") {
t.Errorf("a relative sitemap reference is not something a crawler accepts:\n%s", got)
}
}
// A slug moves a bundle's address without moving its key, so a sitemap built from keys would advertise URLs
// that 404. Moved here from internal/web with the sitemap itself (ADR-0085).
func TestTheSitemapUsesTheSluggedAddress(t *testing.T) {
h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{
"content/posts/hello-world.md": {Data: []byte("---\ntitle: Hello\ndate: 2026-03-01\nslug: ekti-post\n---\nx\n")},
})
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
body := rec.Body.String()
if !strings.Contains(body, "/posts/ekti-post/") || strings.Contains(body, "hello-world") {
t.Errorf("a sitemap that disagrees with what is served is worse than none:\n%s", body)
}
}
// The sitemap goes through the same Query every listing does, so a draft or a future-dated bundle is absent
// from it for the same reason it is absent from a section page. The other half of this lives in
// internal/web's visibility test, which owns the surfaces that stayed there (ADR-0085).
func TestTheSitemapOmitsUnpublishedBundles(t *testing.T) {
h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, fstest.MapFS{
"content/art/draft/index.md": {Data: []byte("---\ntitle: Draft\ndraft: true\n---\nx\n")},
"content/art/future/index.md": {Data: []byte("---\ntitle: Future\ndate: 2099-01-01\n---\nx\n")},
"content/art/live/index.md": {Data: []byte("---\ntitle: Live\ndate: 2020-01-01\n---\nx\n")},
})
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil))
body := rec.Body.String()
if !strings.Contains(body, "/art/live/") {
t.Errorf("the sitemap lost a published bundle:\n%s", body)
}
for _, hidden := range []string{"draft", "future"} {
if strings.Contains(body, hidden) {
t.Errorf("the sitemap leaked the %s bundle:\n%s", hidden, body)
}
}
}