// Package content reads a site root into bundles. It knows the disk and nothing about HTTP. // // Every read goes through an [os.Root] (ADR-0031), so no path — from a filename or later from a // request — can escape the site root, even through a symlink. package content import ( "bytes" "fmt" "io/fs" "log/slog" "os" "path" "sort" "strconv" "strings" "time" "golang.org/x/text/unicode/norm" "gopkg.in/yaml.v3" ) // DefaultLang is the locale served at the root of the URL space; every other language is served under a // prefix (ADR-0009). A filename with no language suffix means this locale (ADR-0021). const DefaultLang = "en" // Bundle is one addressable piece of content in one language. // // Only Title is lifted out of frontmatter; every other key lands in Extra, so a template or a later // feature can read a field the parser has never heard of (ADR-0002). Absence is always the zero value, // never an error. type Bundle struct { // Key identifies the bundle across languages: its path under content/, without language suffix or // extension, NFC-normalised. This is the identity in ADR-0004 and never contains a language. Key string // Lang is always set; a file with no suffix reports DefaultLang. Lang string // Path is the file this bundle was read from, relative to the site root. Path string // Title is empty when frontmatter omits it. Whether that is legal depends on the type, which // nothing decides yet, so the parser accepts it. Title string // Date is publication time, zero when frontmatter omits it. Undated bundles sort after dated ones. Date time.Time // Aliases are paths that must keep resolving to this bundle, each redirecting to its canonical URL // (ADR-0008). Additive only: an alias is a promise never withdrawn. Aliases []string // Body is everything after the frontmatter, unrendered. Body []byte // Extra holds every frontmatter key other than title, exactly as YAML parsed it. Extra map[string]any } // OpenSite opens a site root for reading. // // The returned fs.FS is backed by [os.Root], which refuses any name that would resolve outside dir, // including through a symlink — unlike os.DirFS, which does not (ADR-0031). The root is held for the // life of the process. func OpenSite(dir string) (fs.FS, error) { root, err := os.OpenRoot(dir) if err != nil { return nil, fmt.Errorf("open site root %s: %w", dir, err) } return root.FS(), nil } // Scan reads every bundle under content/ in fsys. // // A bundle that cannot be parsed, or that collides with another on the same key and language, is logged // at error level and left out; neither is fatal, because one mistyped colon must not take down a site // (ADR-0029). An error is returned only when the walk itself fails. func Scan(fsys fs.FS) ([]Bundle, error) { var found []Bundle err := fs.WalkDir(fsys, "content", func(p string, d fs.DirEntry, err error) error { switch { case err != nil: return err case d.IsDir(): if skipDir(path.Base(p)) { return fs.SkipDir } return nil case !strings.HasSuffix(p, ".md"): return nil } data, err := fs.ReadFile(fsys, p) if err != nil { slog.Error("skipping unreadable bundle", "path", p, "err", err) return nil } b, err := Parse(strings.TrimPrefix(p, "content/"), data) if err != nil { slog.Error("skipping unparseable bundle", "path", p, "err", err) return nil } b.Path = p found = append(found, b) return nil }) if err != nil { return nil, fmt.Errorf("scan content: %w", err) } return dropCollisions(found), nil } // Parse reads one bundle from the bytes of a file, named relative to content/. func Parse(name string, data []byte) (Bundle, error) { key, lang, ok := splitName(name) if !ok { return Bundle{}, fmt.Errorf("not a bundle filename: %s", name) } front, body := splitFrontmatter(data) b := Bundle{Key: key, Lang: lang, Body: body, Extra: map[string]any{}} if len(front) > 0 { if err := yaml.Unmarshal(front, &b.Extra); err != nil { return Bundle{}, fmt.Errorf("frontmatter: %w", err) } } if t, isStr := b.Extra["title"].(string); isStr { b.Title = t } delete(b.Extra, "title") b.Aliases = stringList(b.Extra["aliases"]) delete(b.Extra, "aliases") b.Date = asTime(b.Extra["date"]) return b, nil } // stringList reads a YAML scalar or sequence of strings as keys: normalised, without surrounding // slashes. Anything that is not a string is ignored rather than failing the bundle. func stringList(v any) []string { var out []string add := func(x any) { if str, ok := x.(string); ok { if k := Normalise(strings.Trim(str, "/")); k != "" { out = append(out, k) } } } switch t := v.(type) { case string: add(t) case []any: for _, x := range t { add(x) } } return out } // asTime reads a frontmatter date. yaml.v3 hands back a time.Time for an unquoted timestamp and a string // for a quoted one, so both spellings work and anything else is simply absent. func asTime(v any) time.Time { switch t := v.(type) { case time.Time: return t case string: for _, layout := range []string{time.RFC3339, "2006-01-02"} { if parsed, err := time.Parse(layout, t); err == nil { return parsed } } } return time.Time{} } // Normalise puts s into NFC. // // Every identifier goes through this: Bengali conjuncts have several byte encodings for text that looks // identical, and macOS hands back NFD, so without it two indistinguishable files take different keys // (ADR-0015). func Normalise(s string) string { return norm.NFC.String(s) } // splitName derives a bundle key and language from a path under content/. // // index.md and _index.md name the directory they sit in; anything else names itself. A trailing // two- or three-letter lowercase segment is a language suffix. func splitName(name string) (key, lang string, ok bool) { if !strings.HasSuffix(name, ".md") || name == "" { return "", "", false } dir, base := path.Split(strings.TrimSuffix(name, ".md")) dir = strings.TrimSuffix(dir, "/") if base == "" { return "", "", false } lang = DefaultLang if i := strings.LastIndex(base, "."); i > 0 && isLangTag(base[i+1:]) { lang, base = base[i+1:], base[:i] } if base == "index" || base == "_index" { key = dir } else { key = path.Join(dir, base) } return Normalise(key), lang, true } // isLangTag reports whether s looks like a language suffix: two or three lowercase letters. func isLangTag(s string) bool { if len(s) < 2 || len(s) > 3 { return false } for _, r := range s { if r < 'a' || r > 'z' { return false } } return true } // skipDir reports whether a directory is not content: hidden, or underscore-prefixed. func skipDir(base string) bool { return strings.HasPrefix(base, ".") && base != "." || strings.HasPrefix(base, "_") } // splitFrontmatter separates a leading --- delimited YAML block from the body. A file without one is // all body. func splitFrontmatter(data []byte) (front, body []byte) { const fence = "---" rest, hasFence := trimLeadingFence(data, fence) if !hasFence { return nil, data } if i := bytes.Index(rest, []byte("\n"+fence)); i >= 0 { front = rest[:i+1] body = rest[i+1+len(fence):] return front, bytes.TrimLeft(body, "\r\n") } return nil, data } // trimLeadingFence removes an opening fence line, reporting whether one was there. func trimLeadingFence(data []byte, fence string) ([]byte, bool) { s := bytes.TrimLeft(data, "\ufeff") if !bytes.HasPrefix(s, []byte(fence)) { return data, false } s = s[len(fence):] s = bytes.TrimLeft(s, "\r") if !bytes.HasPrefix(s, []byte("\n")) { return data, false } return s[1:], true } // dropCollisions removes every bundle sharing a key and language with another. // // Two spellings of the same variant — about.md and about.en.md, or about.md and about/index.md — are // ambiguous rather than harmless, so none of them is served (ADR-0021). func dropCollisions(all []Bundle) []Bundle { seen := map[string]int{} for _, b := range all { seen[b.Key+"\x00"+b.Lang]++ } kept := make([]Bundle, 0, len(all)) for _, b := range all { if seen[b.Key+"\x00"+b.Lang] > 1 { slog.Error("skipping ambiguous bundle: two files claim one key and language", "key", b.Key, "lang", b.Lang, "path", b.Path) continue } kept = append(kept, b) } return kept } // Site is a set of bundles indexed for lookup by permalink key. type Site struct { byKeyLang map[string]Bundle aliases map[string]string } // NewSite indexes bundles for lookup. Later variants of a key and language cannot occur, because Scan // drops ambiguity before this sees it. func NewSite(bundles []Bundle) *Site { s := &Site{ byKeyLang: make(map[string]Bundle, len(bundles)), aliases: map[string]string{}, } for _, b := range bundles { s.byKeyLang[b.Key+"\x00"+b.Lang] = b } s.indexAliases(bundles) return s } // indexAliases maps each alias to the key it redirects to. // // An alias that names a real bundle, or that two bundles both claim, is ambiguous: it is logged and // dropped rather than picking a winner, and the real bundle keeps its URL (ADR-0029). func (s *Site) indexAliases(bundles []Bundle) { claimed := map[string][]string{} for _, b := range bundles { for _, a := range b.Aliases { claimed[a] = append(claimed[a], b.Key) } } for alias, keys := range claimed { if _, isReal := s.byKeyLang[alias+"\x00"+DefaultLang]; isReal { slog.Error("ignoring alias that names a real bundle", "alias", alias, "claimed_by", keys) continue } if len(keys) > 1 { slog.Error("ignoring alias claimed by more than one bundle", "alias", alias, "claimed_by", keys) continue } s.aliases[alias] = keys[0] } } // Alias returns the key an alias redirects to. func (s *Site) Alias(alias string) (string, bool) { key, ok := s.aliases[alias] return key, ok } // Lookup returns the best variant of a key for a requested language, and the language actually served. // // The fallback chain is requested → default → any (ADR-0009); "any" is resolved in sorted order so the // same request always answers the same way. A key with no variants at all reports false. func (s *Site) Lookup(key, lang string) (b Bundle, served string, ok bool) { for _, try := range []string{lang, DefaultLang} { if try == "" { continue } if b, ok = s.byKeyLang[key+"\x00"+try]; ok { return b, try, true } } for _, l := range s.Variants(key) { b = s.byKeyLang[key+"\x00"+l] return b, l, true } return Bundle{}, "", false } // Variants lists the languages a key exists in, sorted. func (s *Site) Variants(key string) []string { var langs []string for kl := range s.byKeyLang { k, l, found := strings.Cut(kl, "\x00") if found && k == key { langs = append(langs, l) } } sort.Strings(langs) return langs } // HasLang reports whether any bundle is written in lang. The resolver needs this to tell a language // prefix from a section that happens to share its name. func (s *Site) HasLang(lang string) bool { for kl := range s.byKeyLang { if _, l, found := strings.Cut(kl, "\x00"); found && l == lang { return true } } return false } // Len reports how many bundles the site holds. func (s *Site) Len() int { return len(s.byKeyLang) } // PerPage is how many entries a listing shows. // // Changing it renumbers page URLs, which ADR-0028 calls a URL event; it becomes a setting when the // cascade exists rather than being one knob early. const PerPage = 10 // Query selects bundles into an ordered list. // // Every grouping in the engine is a Query — sections now, tags and series later. It carries no cache // signature yet: nothing caches, and a signature with no consumer is speculation (architecture.md). type Query struct { // Section is the first path segment of a key. Empty matches every section. Section string // Lang is the language to serve, with the usual fallback per key (ADR-0009). Lang string } // Run applies q, newest first, with undated bundles after dated ones and ties broken by key so the same // query always answers in the same order. func (s *Site) Run(q Query) []Bundle { seen := map[string]bool{} var out []Bundle for kl := range s.byKeyLang { key, _, found := strings.Cut(kl, "\x00") if !found || seen[key] { continue } if q.Section != "" && !strings.HasPrefix(key, q.Section+"/") { continue } seen[key] = true if b, _, ok := s.Lookup(key, q.Lang); ok { out = append(out, b) } } sort.Slice(out, func(i, j int) bool { a, b := out[i], out[j] switch { case !a.Date.Equal(b.Date): return a.Date.After(b.Date) default: return a.Key < b.Key } }) return out } // Sections lists every section that holds at least one bundle, sorted. func (s *Site) Sections() []string { seen := map[string]bool{} for kl := range s.byKeyLang { key, _, found := strings.Cut(kl, "\x00") if !found { continue } if sec, _, nested := strings.Cut(key, "/"); nested { seen[sec] = true } } out := make([]string, 0, len(seen)) for sec := range seen { out = append(out, sec) } sort.Strings(out) return out } // URL is the permalink of a variant: /{section}/{slug}/, with a language prefix for anything but the // default locale (ADR-0008, ADR-0009). Templates never build a path by hand. func URL(key, lang string) string { if lang == "" || lang == DefaultLang { return "/" + key + "/" } return "/" + lang + "/" + key + "/" } // PageURL is the permalink of a listing page. Page one is the bare listing URL, never /page/1/ // (ADR-0028). func PageURL(key, lang string, page int) string { base := URL(key, lang) if page <= 1 { return base } return base + "page/" + strconv.Itoa(page) + "/" }