bdeshi/khosra
0
0
Fork 0
Code Issues Pull Requests Releases Activity

serve robots.txt and sitemap.xml

Browse Source 
Two exact paths a crawler asks for by name, so they are mux entries rather than
resolver cases — no bundle can collide, since a key always sits under a section.

robots.txt at the site root is served verbatim, because a site that ships one has
said something deliberate; otherwise the engine emits the minimum that is true and
points at the sitemap. The sitemap lists every bundle in every language it exists
in, since each variant is separately reachable, with lastmod only where a bundle
has a date. Every URL comes from content.URL like every other path the engine
emits, so a sitemap cannot disagree with what is actually served.

Both need a declared base. Without one the sitemap answers 404 rather than listing
paths no crawler can resolve, and robots omits the Sitemap line rather than
writing a relative one.

write() was setting text/html for every caller, and headers only go out with the
first byte — so a handler setting its own type would have had it silently replaced,
which is how a sitemap gets served as a web page. It now splits into write and
writeAs, and the tests assert the content types rather than only the bodies.
This commit is contained in:
bdeshi
2026-08-01 02:23:36 +06:00
parent 470e7f18b9
commit bfe980e028
16 changed files with 476 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/web/web_test.go
+16 -16
View File
@@ -24,11 +24,11 @@ func testHandler(t *testing.T) http.Handler {
if err != nil {
t.Fatal(err)
}
r, err := render.New(nil, nil)
r, err := render.New(nil, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
return Handler(content.NewSite(bundles), r, fsys)
return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
}
func TestServeBundleAtItsPermalink(t *testing.T) {
@@ -72,11 +72,11 @@ func multilingualHandler(t *testing.T) http.Handler {
if err != nil {
t.Fatal(err)
}
r, err := render.New(nil, nil)
r, err := render.New(nil, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
return Handler(content.NewSite(bundles), r, fsys)
return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
}
func TestPrefixedLanguageServesThatVariant(t *testing.T) {
@@ -124,11 +124,11 @@ func aliasHandler(t *testing.T) http.Handler {
if err != nil {
t.Fatal(err)
}
r, err := render.New(nil, nil)
r, err := render.New(nil, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
return Handler(content.NewSite(bundles), r, fsys)
return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
}
func TestAliasRedirectsToCanonical(t *testing.T) {
@@ -171,11 +171,11 @@ func listingHandler(t *testing.T, n int) http.Handler {
if err != nil {
t.Fatal(err)
}
r, err := render.New(nil, nil)
r, err := render.New(nil, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
return Handler(content.NewSite(bundles), r, fsys)
return Handler(content.NewSite(bundles), r, fsys, content.Settings{})
}
func TestSectionIndexListsNewestFirst(t *testing.T) {
@@ -239,11 +239,11 @@ func TestStaticFilesAreServedAndDirectoriesAreNot(t *testing.T) {
if err != nil {
t.Fatal(err)
}
r, err := render.New(fsys, nil)
r, err := render.New(fsys, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
h := Handler(content.NewSite(bundles), r, fsys)
h := Handler(content.NewSite(bundles), r, fsys, content.Settings{})
for path, want := range map[string]int{
"/static/style.css": http.StatusOK,
"/static/img/logo.svg": http.StatusOK,
@@ -280,11 +280,11 @@ func TestAStaticPathThatEscapesTheRootIs404(t *testing.T) {
if err != nil {
t.Fatal(err)
}
r, err := render.New(fsys, nil)
r, err := render.New(fsys, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
h := Handler(content.NewSite(nil), r, fsys)
h := Handler(content.NewSite(nil), r, fsys, content.Settings{})
for path, want := range map[string]int{
"/static/ok.css": http.StatusOK,
"/static/escape.txt": http.StatusNotFound,
@@ -313,11 +313,11 @@ func seriesHandler(t *testing.T) http.Handler {
if err != nil {
t.Fatal(err)
}
r, err := render.New(nil, nil)
r, err := render.New(nil, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
return Handler(content.NewSite(bundles), r, nil)
return Handler(content.NewSite(bundles), r, nil, content.Settings{})
}
func TestSequenceNavigationLinksNeighbours(t *testing.T) {
@@ -403,11 +403,11 @@ func tagHandler(t *testing.T) http.Handler {
if err != nil {
t.Fatal(err)
}
r, err := render.New(nil, nil)
r, err := render.New(nil, content.Settings{}, nil)
if err != nil {
t.Fatal(err)
}
return Handler(content.NewSite(bundles), r, nil)
return Handler(content.NewSite(bundles), r, nil, content.Settings{})
}
func TestGlobalTagListingSpansSectionsGroupedByOne(t *testing.T) {