diff --git a/cmd/khosra/main.go b/cmd/khosra/main.go index 824cbcf..645893c 100644 --- a/cmd/khosra/main.go +++ b/cmd/khosra/main.go @@ -8,6 +8,7 @@ import ( "log/slog" "net/http" "os" + "strings" "khosra/internal/content" "khosra/internal/render" @@ -17,6 +18,7 @@ import ( func main() { site := flag.String("site", os.Getenv("KHOSRA_SITE"), "path to the site root (or KHOSRA_SITE)") addr := flag.String("addr", "localhost:8080", "address to listen on") + base := flag.String("base", "", "canonical site origin, overriding site.yaml (e.g. https://khosra.example)") flag.Parse() if *site == "" { @@ -30,13 +32,20 @@ func main() { if err != nil { fatal("cannot read content", err) } - renderer, err := render.New(fsys, extenders) + settings, err := content.LoadSettings(fsys) + if err != nil { + fatal("cannot read site settings", err) + } + if *base != "" { + settings.Base = strings.TrimSuffix(*base, "/") + } + renderer, err := render.New(fsys, settings, extenders) if err != nil { fatal("cannot prepare the theme", err) } slog.Info("serving", "site", *site, "bundles", len(bundles), "addr", *addr) - if err := http.ListenAndServe(*addr, web.Handler(content.NewSite(bundles), renderer, fsys)); err != nil { + if err := http.ListenAndServe(*addr, web.Handler(content.NewSite(bundles), renderer, fsys, settings)); err != nil { fatal("server stopped", err) } } diff --git a/docs/content-model.md b/docs/content-model.md index 28e6ac7..77cb6c3 100644 --- a/docs/content-model.md +++ b/docs/content-model.md @@ -18,6 +18,7 @@ overrides the defaults the binary embeds, so a bare root still renders. ``` / + site.yaml # what the site declares about itself, optional content/ # bundles — the disk contract below static/ # verbatim, served as-is templates/ # html/template overrides, optional @@ -207,10 +208,27 @@ test: a tag is free-form and cross-cutting, a declared taxonomy has known terms Feeds follow the same shape: `/feed.xml` carries every type declared `primary`, `/{section}/feed.xml` carries a section, and `/tags/{tag}/feed.xml` comes free from the same Query. -## The settings cascade `[spec]` +## Site settings -Recorded intent (`ideas/deferred-decisions.md`), not built. Until it exists, settings come from a bundle's own frontmatter. -The shape: site → section → bundle, nearest explicit value winning. +`site.yaml` at the site root declares the site (ADR-0039). Declared keys only — absent is fine, since a bare +site root still serves; malformed is a fatal startup error, because unlike one bad bundle it misconfigures +every page. + +| Key | Meaning | +|---|---| +| `base` | The canonical origin, `https://khosra.example`. A trailing slash is trimmed. Absent, the engine emits root-relative paths and nothing absolute | +| `title` | Names the site. A theme may use it in a document title or a feed | + +`-base` overrides the file, so a staging host needs no edit to content. + +A canonical link, an `hreflang` and an OpenGraph URL become absolute as soon as `base` is declared, because +each is read by something that resolves neither against the page. Links between pages stay root-relative. + +### Below the site level `[spec]` + +Section-level and bundle-level settings are **not** built: a bundle's own frontmatter covers it, and nothing +reads a section override yet. The parked shape is site → section → bundle, nearest explicit value winning +(`ideas/deferred-decisions.md`). | Level | Where it lives | |---|---| @@ -324,8 +342,17 @@ cached (ADR-0005), such a Stage also declares how long its output stays true — publication is the same shape seen from the other side: the page becomes reachable at a moment nobody is requesting it. -## Metadata output `[spec]` +## Metadata and discovery -OpenGraph, Twitter cards, JSON-LD, microformats2, canonical links, and `hreflang` are Stages reading -only fields that already exist on the page. SEO adds no new disk fields; if it seems to need one, the -field belongs in the model for its own sake. +Canonical links, `hreflang` and OpenGraph are emitted from fields that already exist on the page — the theme +composes them, so none of it is a transform and none of it adds a disk field. SEO adds no frontmatter; if it +seems to need some, the field belongs in the model for its own sake. + +`/robots.txt` and `/sitemap.xml` are served at those exact paths. No bundle can collide with them, since a +key always sits under a section. A `robots.txt` at the site root is served verbatim, because a site that +ships one has said something deliberate; otherwise the engine emits the minimum that is true. The sitemap +lists every bundle in every language it exists in — each variant is separately reachable, so each is its own +entry — with `lastmod` only where a bundle carries a date. Both need `base`; without it the sitemap answers +404 rather than listing paths no crawler can resolve. + +`[spec]` Twitter cards, JSON-LD and microformats2 have no consumer yet. diff --git a/docs/state.md b/docs/state.md index 6696703..5aea51c 100644 --- a/docs/state.md +++ b/docs/state.md @@ -1,6 +1,6 @@ # State -**Verified against:** `ff62729` on 2026-07-30 — update this line every change. +**Verified against:** `2b0387e` on 2026-07-30 — update this line every change. If this file disagrees with the code, the code is right and this file is a bug. ## Inventory @@ -10,23 +10,27 @@ If this file disagrees with the code, the code is right and this file is a bug. | `go.mod` | module `khosra`; `goldmark`, `x/text`, `yaml.v3` direct | 10 | | `internal/content/doc.go` | package comment | 5 | | `internal/content/content.go` | bundles: `os.Root` open, walk, frontmatter split, key/lang derivation, NFC, tag slugs, partial files, permalink building | 368 | -| `internal/content/site.go` | the indexed site: lookup with language fallback, aliases, `Query` and `Run`, sections, `Sequence` | 286 | -| `internal/render/render.go` | goldmark with the typographer, per-kind template sets with site override, the `Partial`/`Origin` seams features render and resolve through, `Page`/`List`/`Sequence`/`head` | 386 | +| `internal/content/settings.go` | `site.yaml`: the site's own declarations (`base`, `title`) and absolute-URL building (ADR-0039) | 59 | +| `internal/content/site.go` | the indexed site: lookup with language fallback, aliases, `Query` and `Run`, sections, `Sequence`, `Everything` | 300 | +| `internal/render/render.go` | goldmark with the typographer, per-kind template sets with site override, the `Partial`/`Origin` seams features render and resolve through, `Page`/`List`/`Sequence`/`head` | 409 | | `internal/render/chrome.go` | the engine's own words: phrase table, month names, digits, and the `t`/`num`/`day` template funcs (ADR-0034) | 105 | | `internal/render/templates/` | reference theme: `base.html`, `page.html`, `list.html`, `shortcodes.html`, `theme.css` (ADR-0026) | — | | `internal/ext/shortcodes/` | first feature: `{{< name key="value" >}}` block parser and node renderer, rendering through a theme fragment (ADR-0036). `figure`, `gallery`, `include` | 315 | | `internal/ext/widows/` | second feature: joins the last two words of a paragraph or heading with a non-breaking space, over the tree so code spans are safe | 108 | | `cmd/khosra/wire.go` | the only list of enabled features (`extensions.md`) | 20 | | `internal/web/resolve.go` | URL → (key, lang, page, tag) or a canonical redirect: language prefix, `/en/…` fork guard, pagination, tags, trailing slash | 112 | +| `internal/web/discover.go` | `/robots.txt` and `/sitemap.xml`, absolute and only with a declared base (ADR-0039) | 74 | | `internal/web/web.go` | handler: resolve, look up with fallback, section and tag listings, sequence, `/static/` (misses and refusals alike answer 404), degrade on failure | 152 | -| `cmd/khosra/main.go` | flags, wiring, startup — the only place things are assembled | 53 | -| `*_test.go` | table-driven, one file per source file; symlink escape (content and static), canonical paths, language fallback, aliases, pagination, tags, sequences, chrome, typography, shortcode escaping, galleries, includes, partials, widows, 404 | 1562 | +| `cmd/khosra/main.go` | flags (`-site`, `-addr`, `-base`), wiring, startup — the only place things are assembled | 60 | +| `*_test.go` | table-driven, one file per source file; symlink escape (content and static), canonical paths, language fallback, aliases, pagination, tags, sequences, chrome, typography, shortcode escaping, galleries, includes, partials, widows, site settings, absolute URLs, robots, sitemap, 404 | 1755 | Serves a bundle at `/{section}/{slug}/`, a paginated listing per section, tag listings global and -section-narrowed, sequence navigation and a series archive on any nested bundle, and `static/` verbatim. +section-narrowed, sequence navigation and a series archive on any nested bundle, `static/` verbatim, plus `/robots.txt` and +`/sitemap.xml`. Chrome text, dates and digits render in English or Bengali; authored text is untouched but for typographic smoothing and widow prevention (ADR-0034). This repo holds engine source only — the site root is external and passed with -`-site` (ADR-0011). +`-site` (ADR-0011). `site.yaml` declares `base` and `title`; with a base, canonical, hreflang and OpenGraph +URLs go absolute (ADR-0039). Frontmatter the parser lifts today: `title`, `date`, `tags`, `aliases`, `order`. Every other key in `content-model.md`'s table — including `slug`, `draft` and `type` — lands in `Extra` unread, so that table @@ -42,7 +46,7 @@ this change*. | Counter | Now | Extraction due at | What it buys | |---|---|---|---| | Render transforms — **page-level only** | 0 | **3** | Stage pipeline (ordered `func(ctx,*Page) error`). Parse-phase work does *not* count and must not: goldmark's extender list is already an ordered pipeline for it, so typography, shortcodes and widows compose there (`cmd/khosra/wire.go`) and a second pipeline beside it would be pure duplication. This counts transforms over the assembled page, which nothing hosts yet — OpenGraph and JSON-LD (queue 15) are the first candidates | -| Routing cases | 5 | **2** — done | Resolver at `internal/web/resolve.go`: bundle, language prefix, pagination, tag, section-narrowed tag | +| Routing cases | 7 | **2** — done | Resolver at `internal/web/resolve.go`: bundle, language prefix, pagination, tag, section-narrowed tag | | Collection pages | 4 | **1** — done | Query primitive: `content.Query{Section, Tag, Lang}` + `Site.Run`. The fourth — a series archive — resolves through `Site.Sequence` instead: membership is structural and the sort ascends, so it shares the index but not the Query | | Views / output formats | 2 | **2** — due | Two template sets exist (bundle, listing); the View layer is Arc 2's third item | | Effects | 0 | **2** | Effect runner + trigger wiring (change / schedule / demand) | diff --git a/docs/theme-contract.md b/docs/theme-contract.md index 64413c7..ef65672 100644 --- a/docs/theme-contract.md +++ b/docs/theme-contract.md @@ -19,8 +19,9 @@ A bundle page receives: | `.HTML` | the rendered body, already escaped | | `.Extra` | every frontmatter key the parser does not name (ADR-0002) | | `.Style` | the reference theme's stylesheet, inlined so a bare site root needs no asset route | -| `.Canonical` | the permalink of the variant actually served — not the URL requested, which differs when the fallback chain supplied another language | -| `.Alternates` | every language this key exists in, as `.Lang` and `.URL`, for `hreflang` | +| `.Canonical` | the permalink of the variant actually served — not the URL requested, which differs when the fallback chain supplied another language. **Absolute** when the site declares `base`, since a canonical link is resolved by machines rather than by the page (ADR-0039) | +| `.Alternates` | every language this key exists in, as `.Lang` and `.URL`, for `hreflang`. Absolute on the same terms | +| `.Site` | what the site declared about itself: `.Site.Base` and `.Site.Title`, either possibly empty | | `.Sequence` | the series this page sits in, absent when it sits in none (ADR-0033) | `.Sequence` carries the reading order and this page's place in it: diff --git a/internal/content/settings.go b/internal/content/settings.go new file mode 100644 index 0000000..8cbd8ea --- /dev/null +++ b/internal/content/settings.go @@ -0,0 +1,59 @@ +package content + +import ( + "errors" + "fmt" + "io/fs" + "strings" + + "gopkg.in/yaml.v3" +) + +// SettingsFile is where a site declares itself, at the site root. +const SettingsFile = "site.yaml" + +// Settings are the site's own declarations (ADR-0039). +// +// Declared keys only: this is not a bag for arbitrary engine internals, which is how a settings file turns +// into unbounded configuration. Section-level and bundle-level settings are not here — a bundle's own +// frontmatter covers it, and the rest of the cascade is still parked. +type Settings struct { + // Base is the site's canonical origin, without a trailing slash: "https://khosra.example". Empty when + // the site declares none, in which case the engine emits paths and nothing absolute. + Base string `yaml:"base"` + // Title names the site. A theme may put it in a document title or a feed; the engine only carries it. + Title string `yaml:"title"` +} + +// LoadSettings reads site.yaml from the site root. +// +// A missing file is not an error, because a bare site root must still serve (ADR-0026). A malformed one is, +// and the caller treats it as fatal: unlike a single bad bundle, which is skipped loudly (ADR-0029), a +// broken declaration misconfigures every page on the site. +func LoadSettings(fsys fs.FS) (Settings, error) { + data, err := fs.ReadFile(fsys, SettingsFile) + if errors.Is(err, fs.ErrNotExist) { + return Settings{}, nil + } + if err != nil { + return Settings{}, fmt.Errorf("read %s: %w", SettingsFile, err) + } + var s Settings + if err := yaml.Unmarshal(data, &s); err != nil { + return Settings{}, fmt.Errorf("parse %s: %w", SettingsFile, err) + } + s.Base = strings.TrimSuffix(strings.TrimSpace(s.Base), "/") + s.Title = strings.TrimSpace(s.Title) + return s, nil +} + +// Absolute turns a path the engine emitted into a full URL. +// +// With no declared base it returns the path unchanged, so a site that has not said where it lives still +// links correctly to itself — every internal path is root-relative already. +func Absolute(base, path string) string { + if base == "" { + return path + } + return base + path +} diff --git a/internal/content/settings_test.go b/internal/content/settings_test.go new file mode 100644 index 0000000..19cea49 --- /dev/null +++ b/internal/content/settings_test.go @@ -0,0 +1,45 @@ +package content + +import ( + "testing" + "testing/fstest" +) + +func TestSettingsAreOptionalAndNormalised(t *testing.T) { + // A bare site root must still serve, so an absent file is not an error (ADR-0026). + got, err := LoadSettings(fstest.MapFS{}) + if err != nil || got.Base != "" || got.Title != "" { + t.Fatalf("absent site.yaml gave %+v, %v — want the zero value and no error", got, err) + } + got, err = LoadSettings(fstest.MapFS{ + SettingsFile: {Data: []byte("base: https://khosra.example/\ntitle: খসড়া\n")}, + }) + if err != nil { + t.Fatal(err) + } + if got.Base != "https://khosra.example" { + t.Errorf("base = %q — a trailing slash must be trimmed, or every URL doubles it", got.Base) + } + if got.Title != "খসড়া" { + t.Errorf("title = %q", got.Title) + } +} + +func TestMalformedSettingsAreAnError(t *testing.T) { + // Unlike one bad bundle, which is skipped loudly (ADR-0029), this misconfigures every page, so the + // caller treats it as fatal. + if _, err := LoadSettings(fstest.MapFS{SettingsFile: {Data: []byte("base: [unclosed\n")}}); err == nil { + t.Error("a malformed site.yaml must be an error") + } +} + +func TestAbsoluteOnlyPrefixesWhenABaseExists(t *testing.T) { + for _, c := range []struct{ base, path, want string }{ + {"https://khosra.example", "/posts/hello/", "https://khosra.example/posts/hello/"}, + {"", "/posts/hello/", "/posts/hello/"}, + } { + if got := Absolute(c.base, c.path); got != c.want { + t.Errorf("Absolute(%q, %q) = %q, want %q", c.base, c.path, got, c.want) + } + } +} diff --git a/internal/content/site.go b/internal/content/site.go index 1ffc484..6ec6616 100644 --- a/internal/content/site.go +++ b/internal/content/site.go @@ -265,6 +265,20 @@ func (b Bundle) Section() string { return sec } +// Everything lists every variant of every bundle, sorted by key then language. +// +// Every *variant*, unlike a Query, which answers with one variant per key: a sitemap lists each language as +// its own URL, because each is separately reachable. +func (s *Site) Everything() []Bundle { + out := make([]Bundle, 0, len(s.byKeyLang)) + for _, key := range s.keys() { + for _, lang := range s.Variants(key) { + out = append(out, s.byKeyLang[key+"\x00"+lang]) + } + } + return out +} + // Sections lists every section that holds at least one bundle, sorted. func (s *Site) Sections() []string { seen := map[string]bool{} diff --git a/internal/ext/shortcodes/shortcodes_test.go b/internal/ext/shortcodes/shortcodes_test.go index 1744ff2..f267cd9 100644 --- a/internal/ext/shortcodes/shortcodes_test.go +++ b/internal/ext/shortcodes/shortcodes_test.go @@ -43,7 +43,7 @@ func wired(t *testing.T, siteFS fstest.MapFS) *render.Renderer { if siteFS != nil { fsys = siteFS } - r, err := render.New(fsys, func(p render.Partial) []goldmark.Extender { + r, err := render.New(fsys, content.Settings{}, func(p render.Partial) []goldmark.Extender { return []goldmark.Extender{New(p)} }) if err != nil { diff --git a/internal/render/chrome_test.go b/internal/render/chrome_test.go index a1456f4..8b834e3 100644 --- a/internal/render/chrome_test.go +++ b/internal/render/chrome_test.go @@ -61,7 +61,7 @@ func TestDatesReadInTheirOwnScript(t *testing.T) { } func TestTypographerSmoothsProseAndLeavesCodeAlone(t *testing.T) { - r, err := New(nil, nil) + r, err := New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } @@ -86,7 +86,7 @@ func TestTypographerSmoothsProseAndLeavesCodeAlone(t *testing.T) { } func TestMachineReadableOutputStaysASCII(t *testing.T) { - r, err := New(nil, nil) + r, err := New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } diff --git a/internal/render/render.go b/internal/render/render.go index 77511e8..b298f67 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -38,6 +38,8 @@ type head struct { Alternates []Alternate // Style is the reference theme's stylesheet, inlined so a bare site root needs no asset route. Style template.CSS + // Site is what the site declared about itself in site.yaml (ADR-0039). Zero when it declared nothing. + Site content.Settings } // Page is one bundle rendered. @@ -117,6 +119,8 @@ type Renderer struct { style template.CSS // files is the site root, handed to features through Origin. Nil when there is none. files fs.FS + // settings are the site's declarations, constant for the life of the process. + settings content.Settings } // Partial renders a named fragment. A feature under internal/ext is handed one of these at wiring time, @@ -169,7 +173,7 @@ func WithOrigin(pc parser.Context, origin Origin) { // extensions to enable. A callback rather than a parameter of feature types, because internal/render must // not import internal/ext — only cmd knows which features a build includes (conventions.md, ADR-0036). It // may be nil. -func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, error) { +func New(siteFS fs.FS, settings content.Settings, extend func(Partial) []goldmark.Extender) (*Renderer, error) { page, err := parseSet(siteFS, "templates/base.html", "templates/page.html") if err != nil { return nil, fmt.Errorf("bundle templates: %w", err) @@ -186,7 +190,7 @@ func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, err if err != nil { return nil, err } - r := &Renderer{page: page, list: list, partials: partials, style: css, files: siteFS} + r := &Renderer{page: page, list: list, partials: partials, style: css, files: siteFS, settings: settings} // The typographer smooths quotes, dashes and ellipses in authored prose and leaves code spans alone, // because it works on the parsed tree rather than the text. That is the only change the engine makes to // an author's words (ADR-0034), and it is a parser option rather than a render transform, so it does @@ -202,6 +206,25 @@ func New(siteFS fs.FS, extend func(Partial) []goldmark.Extender) (*Renderer, err return r, nil } +// head builds the document shell every kind of page shares. +// +// canonical arrives as a path and leaves absolute when the site declared a base: a canonical link and an +// hreflang are read by machines that resolve neither against the page (ADR-0039). +func (r *Renderer) head(title, lang, canonical string) head { + return head{ + Title: title, + Lang: lang, + Canonical: r.absolute(canonical), + Style: r.style, + Site: r.settings, + } +} + +// absolute is the site's own URL for a path the engine emitted, or the path itself when no base is declared. +func (r *Renderer) absolute(path string) string { + return content.Absolute(r.settings.Base, path) +} + // Partial renders one named fragment. A missing template is an error the caller degrades on, never a // failed request (extensions.md rule 5). func (r *Renderer) Partial(name string, data Fragment) ([]byte, error) { @@ -276,14 +299,14 @@ func (r *Renderer) Bundle(b content.Bundle, served string, variants []string, se title = b.Key } p := Page{ - head: head{Title: title, Lang: served, Canonical: content.URL(b.Key, served), Style: r.style}, + head: r.head(title, served, content.URL(b.Key, served)), Key: b.Key, HTML: template.HTML(body.String()), Extra: b.Extra, Sequence: r.sequence(seq, served), } for _, l := range variants { - p.Alternates = append(p.Alternates, Alternate{Lang: l, URL: content.URL(b.Key, l)}) + p.Alternates = append(p.Alternates, Alternate{Lang: l, URL: r.absolute(content.URL(b.Key, l))}) } return r.execute(r.page, p, b.Key) } @@ -363,7 +386,7 @@ func (r *Renderer) paginate(title, lang, canonical string, all []content.Bundle, start := (page - 1) * content.PerPage end := min(start+content.PerPage, len(all)) l := List{ - head: head{Title: title, Lang: lang, Canonical: canonical, Style: r.style}, + head: r.head(title, lang, canonical), Page: page, Pages: pages, } diff --git a/internal/render/render_test.go b/internal/render/render_test.go index 593f3fc..b5f1848 100644 --- a/internal/render/render_test.go +++ b/internal/render/render_test.go @@ -9,7 +9,7 @@ import ( ) func TestBundleRendersMarkdownIntoTheTheme(t *testing.T) { - r, err := New(nil, nil) + r, err := New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } @@ -33,7 +33,7 @@ func TestBundleRendersMarkdownIntoTheTheme(t *testing.T) { } func TestBundleWithoutTitleFallsBackToKey(t *testing.T) { - r, err := New(nil, nil) + r, err := New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } @@ -54,7 +54,7 @@ func TestSiteOverridesOneBlockAndInheritsTheRest(t *testing.T) { siteFS := fstest.MapFS{ "templates/page.html": {Data: []byte(`{{define "main"}}
{{.Title}}
{{end}}`)}, } - r, err := New(siteFS, nil) + r, err := New(siteFS, content.Settings{}, nil) if err != nil { t.Fatal(err) } @@ -82,7 +82,7 @@ func TestAListingOverrideDoesNotLeakIntoBundlePages(t *testing.T) { siteFS := fstest.MapFS{ "templates/list.html": {Data: []byte(`{{define "main"}}LISTING ONLY{{end}}`)}, } - r, err := New(siteFS, nil) + r, err := New(siteFS, content.Settings{}, nil) if err != nil { t.Fatal(err) } @@ -101,7 +101,7 @@ func TestAListingOverrideDoesNotLeakIntoBundlePages(t *testing.T) { func TestSiteStylesheetReplacesTheReferenceOne(t *testing.T) { siteFS := fstest.MapFS{"templates/theme.css": {Data: []byte("body{color:rebeccapurple}")}} - r, err := New(siteFS, nil) + r, err := New(siteFS, content.Settings{}, nil) if err != nil { t.Fatal(err) } @@ -114,3 +114,50 @@ func TestSiteStylesheetReplacesTheReferenceOne(t *testing.T) { t.Error("the site stylesheet should replace the reference one") } } + +func TestADeclaredBaseMakesMachineReadableURLsAbsolute(t *testing.T) { + // A canonical link and an hreflang are read by machines that resolve neither against the page, so both + // go absolute as soon as the site says where it lives (ADR-0039). + r, err := New(nil, content.Settings{Base: "https://khosra.example", Title: "Khosra"}, nil) + if err != nil { + t.Fatal(err) + } + b, err := content.Parse("posts/hello.md", []byte("---\ntitle: Hello\n---\nhi\n")) + if err != nil { + t.Fatal(err) + } + out, err := r.Bundle(b, "en", []string{"en", "bn"}, nil) + if err != nil { + t.Fatal(err) + } + got := string(out) + for _, want := range []string{ + `rel="canonical" href="https://khosra.example/posts/hello/"`, + `hreflang="bn" href="https://khosra.example/bn/posts/hello/"`, + `property="og:url" content="https://khosra.example/posts/hello/"`, + `Hello · Khosra`, + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q:\n%s", want, got) + } + } +} + +func TestWithoutABaseEverythingStaysRelative(t *testing.T) { + r, err := New(nil, content.Settings{}, nil) + if err != nil { + t.Fatal(err) + } + b, _ := content.Parse("posts/hello.md", []byte("---\ntitle: Hello\n---\nhi\n")) + out, err := r.Bundle(b, "en", []string{"en"}, nil) + if err != nil { + t.Fatal(err) + } + got := string(out) + if !strings.Contains(got, `rel="canonical" href="/posts/hello/"`) { + t.Errorf("a site that declared no base still links to itself:\n%s", got) + } + if strings.Contains(got, "og:site_name") { + t.Error("no declared title means no site_name tag, rather than an empty one") + } +} diff --git a/internal/render/templates/base.html b/internal/render/templates/base.html index 7f6a9b9..b79bdb4 100644 --- a/internal/render/templates/base.html +++ b/internal/render/templates/base.html @@ -4,11 +4,18 @@ -{{.Title}} +{{.Title}}{{if .Site.Title}} · {{.Site.Title}}{{end}} {{- range .Alternates}} {{- end}} + + + + +{{- if .Site.Title}} + +{{- end}} diff --git a/internal/web/discover.go b/internal/web/discover.go new file mode 100644 index 0000000..ed6ca02 --- /dev/null +++ b/internal/web/discover.go @@ -0,0 +1,71 @@ +package web + +import ( + "fmt" + "io/fs" + "log/slog" + "net/http" + "strings" + + "khosra/internal/content" +) + +// robots and sitemap are the two files a crawler looks for by exact name. +const ( + robotsPath = "/robots.txt" + sitemapPath = "/sitemap.xml" +) + +// serveRobots answers /robots.txt, preferring the site's own file. +// +// A site that ships robots.txt has said something deliberate, so it is served verbatim; otherwise the engine +// emits the minimum that is true — everything is public, and here is the sitemap. The Sitemap line only +// appears with a declared base, because a relative sitemap reference is not something a crawler accepts. +func serveRobots(w http.ResponseWriter, req *http.Request, siteFS fs.FS, base string) { + if siteFS != nil { + if data, err := fs.ReadFile(siteFS, "robots.txt"); err == nil { + writeAs(w, "text/plain; charset=utf-8", data, "robots.txt") + return + } + } + var out strings.Builder + out.WriteString("User-agent: *\nDisallow:\n") + if base != "" { + fmt.Fprintf(&out, "Sitemap: %s\n", content.Absolute(base, sitemapPath)) + } + writeAs(w, "text/plain; charset=utf-8", []byte(out.String()), "robots.txt") +} + +// serveSitemap answers /sitemap.xml with every bundle in every language it exists in. +// +// It needs a declared base: the sitemap format has no room for a relative URL, so without one the honest +// answer is that this file does not exist rather than a file full of paths no crawler can use (ADR-0039). +// Every URL comes from content.URL, like every other path the engine emits, so a sitemap can never disagree +// with what is actually served. +func serveSitemap(w http.ResponseWriter, req *http.Request, site *content.Site, base string) { + if base == "" { + slog.Warn("no sitemap: the site declares no base URL", "file", content.SettingsFile) + http.NotFound(w, req) + return + } + var out strings.Builder + out.WriteString(`` + "\n") + out.WriteString(`` + "\n") + for _, entry := range site.Everything() { + fmt.Fprintf(&out, "%s", xmlEscape(content.Absolute(base, content.URL(entry.Key, entry.Lang)))) + if !entry.Date.IsZero() { + fmt.Fprintf(&out, "%s", entry.Date.Format("2006-01-02")) + } + out.WriteString("\n") + } + out.WriteString("\n") + writeAs(w, "application/xml; charset=utf-8", []byte(out.String()), "sitemap.xml") +} + +// xmlEscape escapes the five characters XML reserves. A URL should contain none of them, and a sitemap that +// silently breaks on the one that does is worse than a slightly paranoid replacement. +func xmlEscape(s string) string { + return strings.NewReplacer( + "&", "&", "<", "<", ">", ">", `"`, """, "'", "'", + ).Replace(s) +} diff --git a/internal/web/discover_test.go b/internal/web/discover_test.go new file mode 100644 index 0000000..426725a --- /dev/null +++ b/internal/web/discover_test.go @@ -0,0 +1,101 @@ +package web + +import ( + "net/http" + "net/http/httptest" + "strings" + "testing" + "testing/fstest" + + "khosra/internal/content" + "khosra/internal/render" +) + +func crawlerHandler(t *testing.T, settings content.Settings, extra fstest.MapFS) http.Handler { + t.Helper() + fsys := fstest.MapFS{ + "content/posts/hello.md": {Data: []byte("---\ntitle: Hello\ndate: 2026-03-08\n---\nx\n")}, + "content/posts/hello.bn.md": {Data: []byte("---\ntitle: হ্যালো\ndate: 2026-03-08\n---\nx\n")}, + "content/pages/about.md": {Data: []byte("---\ntitle: About\n---\nx\n")}, + } + for name, file := range extra { + fsys[name] = file + } + bundles, err := content.Scan(fsys) + if err != nil { + t.Fatal(err) + } + r, err := render.New(nil, settings, nil) + if err != nil { + t.Fatal(err) + } + return Handler(content.NewSite(bundles), r, fsys, settings) +} + +func TestSitemapListsEveryVariantAbsolutely(t *testing.T) { + h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)) + if rec.Code != http.StatusOK { + t.Fatalf("got %d, want 200", rec.Code) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "application/xml") { + t.Errorf("content-type = %q — a sitemap served as HTML is a sitemap nothing reads", ct) + } + body := rec.Body.String() + for _, want := range []string{ + "https://khosra.example/posts/hello/", + "https://khosra.example/bn/posts/hello/", // each language is its own URL + "https://khosra.example/pages/about/", + "2026-03-08", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q:\n%s", want, body) + } + } + if strings.Contains(body, "") { + t.Error("an undated bundle should carry no lastmod at all") + } +} + +func TestNoBaseMeansNoSitemap(t *testing.T) { + // The format has no room for a relative URL, so the honest answer is that the file does not exist + // (ADR-0039) rather than one full of paths no crawler can resolve. + h := crawlerHandler(t, content.Settings{}, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/sitemap.xml", nil)) + if rec.Code != http.StatusNotFound { + t.Errorf("got %d, want 404", rec.Code) + } +} + +func TestRobotsIsGeneratedOrTheSitesOwn(t *testing.T) { + h := crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil)) + body := rec.Body.String() + if !strings.Contains(body, "User-agent: *") || !strings.Contains(body, "Sitemap: https://khosra.example/sitemap.xml") { + t.Errorf("generated robots should point at the sitemap:\n%s", body) + } + if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, "text/plain") { + t.Errorf("content-type = %q", ct) + } + + // A site that ships its own has said something deliberate. + h = crawlerHandler(t, content.Settings{Base: "https://khosra.example"}, + fstest.MapFS{"robots.txt": {Data: []byte("User-agent: *\nDisallow: /drafts/\n")}}) + rec = httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil)) + if got := rec.Body.String(); !strings.Contains(got, "Disallow: /drafts/") || strings.Contains(got, "Sitemap:") { + t.Errorf("the site's own robots.txt should be served verbatim:\n%s", got) + } +} + +func TestRobotsWithoutABaseOmitsTheSitemapLine(t *testing.T) { + h := crawlerHandler(t, content.Settings{}, nil) + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/robots.txt", nil)) + if got := rec.Body.String(); strings.Contains(got, "Sitemap:") { + t.Errorf("a relative sitemap reference is not something a crawler accepts:\n%s", got) + } +} diff --git a/internal/web/web.go b/internal/web/web.go index 7e5a9eb..c85b3fa 100644 --- a/internal/web/web.go +++ b/internal/web/web.go @@ -15,11 +15,19 @@ import ( // Handler serves a site. // // One mux entry, because URL shape is the resolver's business rather than the mux's: see resolve. -func Handler(site *content.Site, r *render.Renderer, siteFS fs.FS) http.Handler { +func Handler(site *content.Site, r *render.Renderer, siteFS fs.FS, settings content.Settings) http.Handler { mux := http.NewServeMux() mux.HandleFunc("GET /", func(w http.ResponseWriter, req *http.Request) { serve(w, req, site, r) }) + // Two exact paths a crawler asks for by name, so they are mux entries rather than resolver cases: no + // bundle can own them, since a key always sits under a section. + mux.HandleFunc("GET "+robotsPath, func(w http.ResponseWriter, req *http.Request) { + serveRobots(w, req, siteFS, settings.Base) + }) + mux.HandleFunc("GET "+sitemapPath, func(w http.ResponseWriter, req *http.Request) { + serveSitemap(w, req, site, settings.Base) + }) if siteFS != nil { if sub, err := fs.Sub(siteFS, "static"); err == nil { mux.Handle("GET /static/", http.StripPrefix("/static/", serveStatic(sub))) @@ -98,7 +106,16 @@ func serveTags(w http.ResponseWriter, req *http.Request, site *content.Site, r * // write sends a rendered page, logging a failed write rather than pretending it succeeded. func write(w http.ResponseWriter, out []byte, what string) { - w.Header().Set("Content-Type", "text/html; charset=utf-8") + writeAs(w, "text/html; charset=utf-8", out, what) +} + +// writeAs sends bytes with the type they actually are. +// +// Separate from write because headers are only sent with the first byte, so a handler that set its own type +// before calling write would have had it silently replaced by HTML — which is how a sitemap ends up served +// as a web page. +func writeAs(w http.ResponseWriter, contentType string, out []byte, what string) { + w.Header().Set("Content-Type", contentType) if _, err := w.Write(out); err != nil { slog.Warn("write failed", "what", what, "err", err) } diff --git a/internal/web/web_test.go b/internal/web/web_test.go index a789a8f..e63f90d 100644 --- a/internal/web/web_test.go +++ b/internal/web/web_test.go @@ -24,11 +24,11 @@ func testHandler(t *testing.T) http.Handler { if err != nil { t.Fatal(err) } - r, err := render.New(nil, nil) + r, err := render.New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } - return Handler(content.NewSite(bundles), r, fsys) + return Handler(content.NewSite(bundles), r, fsys, content.Settings{}) } func TestServeBundleAtItsPermalink(t *testing.T) { @@ -72,11 +72,11 @@ func multilingualHandler(t *testing.T) http.Handler { if err != nil { t.Fatal(err) } - r, err := render.New(nil, nil) + r, err := render.New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } - return Handler(content.NewSite(bundles), r, fsys) + return Handler(content.NewSite(bundles), r, fsys, content.Settings{}) } func TestPrefixedLanguageServesThatVariant(t *testing.T) { @@ -124,11 +124,11 @@ func aliasHandler(t *testing.T) http.Handler { if err != nil { t.Fatal(err) } - r, err := render.New(nil, nil) + r, err := render.New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } - return Handler(content.NewSite(bundles), r, fsys) + return Handler(content.NewSite(bundles), r, fsys, content.Settings{}) } func TestAliasRedirectsToCanonical(t *testing.T) { @@ -171,11 +171,11 @@ func listingHandler(t *testing.T, n int) http.Handler { if err != nil { t.Fatal(err) } - r, err := render.New(nil, nil) + r, err := render.New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } - return Handler(content.NewSite(bundles), r, fsys) + return Handler(content.NewSite(bundles), r, fsys, content.Settings{}) } func TestSectionIndexListsNewestFirst(t *testing.T) { @@ -239,11 +239,11 @@ func TestStaticFilesAreServedAndDirectoriesAreNot(t *testing.T) { if err != nil { t.Fatal(err) } - r, err := render.New(fsys, nil) + r, err := render.New(fsys, content.Settings{}, nil) if err != nil { t.Fatal(err) } - h := Handler(content.NewSite(bundles), r, fsys) + h := Handler(content.NewSite(bundles), r, fsys, content.Settings{}) for path, want := range map[string]int{ "/static/style.css": http.StatusOK, "/static/img/logo.svg": http.StatusOK, @@ -280,11 +280,11 @@ func TestAStaticPathThatEscapesTheRootIs404(t *testing.T) { if err != nil { t.Fatal(err) } - r, err := render.New(fsys, nil) + r, err := render.New(fsys, content.Settings{}, nil) if err != nil { t.Fatal(err) } - h := Handler(content.NewSite(nil), r, fsys) + h := Handler(content.NewSite(nil), r, fsys, content.Settings{}) for path, want := range map[string]int{ "/static/ok.css": http.StatusOK, "/static/escape.txt": http.StatusNotFound, @@ -313,11 +313,11 @@ func seriesHandler(t *testing.T) http.Handler { if err != nil { t.Fatal(err) } - r, err := render.New(nil, nil) + r, err := render.New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } - return Handler(content.NewSite(bundles), r, nil) + return Handler(content.NewSite(bundles), r, nil, content.Settings{}) } func TestSequenceNavigationLinksNeighbours(t *testing.T) { @@ -403,11 +403,11 @@ func tagHandler(t *testing.T) http.Handler { if err != nil { t.Fatal(err) } - r, err := render.New(nil, nil) + r, err := render.New(nil, content.Settings{}, nil) if err != nil { t.Fatal(err) } - return Handler(content.NewSite(bundles), r, nil) + return Handler(content.NewSite(bundles), r, nil, content.Settings{}) } func TestGlobalTagListingSpansSectionsGroupedByOne(t *testing.T) {