package web import ( "fmt" "io/fs" "log/slog" "net/http" "strings" "khosra/internal/content" ) // robots and sitemap are the two files a crawler looks for by exact name. const ( robotsPath = "/robots.txt" sitemapPath = "/sitemap.xml" ) // serveRobots answers /robots.txt, preferring the site's own file. // // A site that ships robots.txt has said something deliberate, so it is served verbatim; otherwise the engine // emits the minimum that is true — everything is public, and here is the sitemap. The Sitemap line only // appears with a declared base, because a relative sitemap reference is not something a crawler accepts. func serveRobots(w http.ResponseWriter, req *http.Request, siteFS fs.FS, base string) { if siteFS != nil { if data, err := fs.ReadFile(siteFS, "robots.txt"); err == nil { writeAs(w, "text/plain; charset=utf-8", data, "robots.txt") return } } var out strings.Builder out.WriteString("User-agent: *\nDisallow:\n") if base != "" { fmt.Fprintf(&out, "Sitemap: %s\n", content.Absolute(base, sitemapPath)) } writeAs(w, "text/plain; charset=utf-8", []byte(out.String()), "robots.txt") } // serveSitemap answers /sitemap.xml with every bundle in every language it exists in. // // It needs a declared base: the sitemap format has no room for a relative URL, so without one the honest // answer is that this file does not exist rather than a file full of paths no crawler can use (ADR-0039). // Every URL comes from content.URL, like every other path the engine emits, so a sitemap can never disagree // with what is actually served. func serveSitemap(w http.ResponseWriter, req *http.Request, site *content.Site, base string) { if base == "" { slog.Warn("no sitemap: the site declares no base URL", "file", content.SettingsFile) http.NotFound(w, req) return } var out strings.Builder out.WriteString(`` + "\n") out.WriteString(`` + "\n") for _, entry := range site.Everything() { fmt.Fprintf(&out, "%s", xmlEscape(content.Absolute(base, content.URL(entry.Route, entry.Lang)))) if !entry.Date.IsZero() { fmt.Fprintf(&out, "%s", entry.Date.Format("2006-01-02")) } out.WriteString("\n") } out.WriteString("\n") writeAs(w, "application/xml; charset=utf-8", []byte(out.String()), "sitemap.xml") } // xmlEscape escapes the five characters XML reserves. A URL should contain none of them, and a sitemap that // silently breaks on the one that does is worse than a slightly paranoid replacement. func xmlEscape(s string) string { return strings.NewReplacer( "&", "&", "<", "<", ">", ">", `"`, """, "'", "'", ).Replace(s) }