// Package content reads a site root into bundles. It knows the disk and nothing about HTTP. // // Every read goes through an [os.Root] (ADR-0031), so no path — from a filename or later from a // request — can escape the site root, even through a symlink. package content import ( "bytes" "fmt" "io/fs" "log/slog" "os" "path" "sort" "strings" "golang.org/x/text/unicode/norm" "gopkg.in/yaml.v3" ) // DefaultLang is the locale served at the root of the URL space; every other language is served under a // prefix (ADR-0009). A filename with no language suffix means this locale (ADR-0021). const DefaultLang = "en" // Bundle is one addressable piece of content in one language. // // Only Title is lifted out of frontmatter; every other key lands in Extra, so a template or a later // feature can read a field the parser has never heard of (ADR-0002). Absence is always the zero value, // never an error. type Bundle struct { // Key identifies the bundle across languages: its path under content/, without language suffix or // extension, NFC-normalised. This is the identity in ADR-0004 and never contains a language. Key string // Lang is always set; a file with no suffix reports DefaultLang. Lang string // Path is the file this bundle was read from, relative to the site root. Path string // Title is empty when frontmatter omits it. Whether that is legal depends on the type, which // nothing decides yet, so the parser accepts it. Title string // Aliases are paths that must keep resolving to this bundle, each redirecting to its canonical URL // (ADR-0008). Additive only: an alias is a promise never withdrawn. Aliases []string // Body is everything after the frontmatter, unrendered. Body []byte // Extra holds every frontmatter key other than title, exactly as YAML parsed it. Extra map[string]any } // OpenSite opens a site root for reading. // // The returned fs.FS is backed by [os.Root], which refuses any name that would resolve outside dir, // including through a symlink — unlike os.DirFS, which does not (ADR-0031). The root is held for the // life of the process. func OpenSite(dir string) (fs.FS, error) { root, err := os.OpenRoot(dir) if err != nil { return nil, fmt.Errorf("open site root %s: %w", dir, err) } return root.FS(), nil } // Scan reads every bundle under content/ in fsys. // // A bundle that cannot be parsed, or that collides with another on the same key and language, is logged // at error level and left out; neither is fatal, because one mistyped colon must not take down a site // (ADR-0029). An error is returned only when the walk itself fails. func Scan(fsys fs.FS) ([]Bundle, error) { var found []Bundle err := fs.WalkDir(fsys, "content", func(p string, d fs.DirEntry, err error) error { switch { case err != nil: return err case d.IsDir(): if skipDir(path.Base(p)) { return fs.SkipDir } return nil case !strings.HasSuffix(p, ".md"): return nil } data, err := fs.ReadFile(fsys, p) if err != nil { slog.Error("skipping unreadable bundle", "path", p, "err", err) return nil } b, err := Parse(strings.TrimPrefix(p, "content/"), data) if err != nil { slog.Error("skipping unparseable bundle", "path", p, "err", err) return nil } b.Path = p found = append(found, b) return nil }) if err != nil { return nil, fmt.Errorf("scan content: %w", err) } return dropCollisions(found), nil } // Parse reads one bundle from the bytes of a file, named relative to content/. func Parse(name string, data []byte) (Bundle, error) { key, lang, ok := splitName(name) if !ok { return Bundle{}, fmt.Errorf("not a bundle filename: %s", name) } front, body := splitFrontmatter(data) b := Bundle{Key: key, Lang: lang, Body: body, Extra: map[string]any{}} if len(front) > 0 { if err := yaml.Unmarshal(front, &b.Extra); err != nil { return Bundle{}, fmt.Errorf("frontmatter: %w", err) } } if t, isStr := b.Extra["title"].(string); isStr { b.Title = t } delete(b.Extra, "title") b.Aliases = stringList(b.Extra["aliases"]) delete(b.Extra, "aliases") return b, nil } // stringList reads a YAML scalar or sequence of strings as keys: normalised, without surrounding // slashes. Anything that is not a string is ignored rather than failing the bundle. func stringList(v any) []string { var out []string add := func(x any) { if str, ok := x.(string); ok { if k := Normalise(strings.Trim(str, "/")); k != "" { out = append(out, k) } } } switch t := v.(type) { case string: add(t) case []any: for _, x := range t { add(x) } } return out } // Normalise puts s into NFC. // // Every identifier goes through this: Bengali conjuncts have several byte encodings for text that looks // identical, and macOS hands back NFD, so without it two indistinguishable files take different keys // (ADR-0015). func Normalise(s string) string { return norm.NFC.String(s) } // splitName derives a bundle key and language from a path under content/. // // index.md and _index.md name the directory they sit in; anything else names itself. A trailing // two- or three-letter lowercase segment is a language suffix. func splitName(name string) (key, lang string, ok bool) { if !strings.HasSuffix(name, ".md") || name == "" { return "", "", false } dir, base := path.Split(strings.TrimSuffix(name, ".md")) dir = strings.TrimSuffix(dir, "/") if base == "" { return "", "", false } lang = DefaultLang if i := strings.LastIndex(base, "."); i > 0 && isLangTag(base[i+1:]) { lang, base = base[i+1:], base[:i] } if base == "index" || base == "_index" { key = dir } else { key = path.Join(dir, base) } return Normalise(key), lang, true } // isLangTag reports whether s looks like a language suffix: two or three lowercase letters. func isLangTag(s string) bool { if len(s) < 2 || len(s) > 3 { return false } for _, r := range s { if r < 'a' || r > 'z' { return false } } return true } // skipDir reports whether a directory is not content: hidden, or underscore-prefixed. func skipDir(base string) bool { return strings.HasPrefix(base, ".") && base != "." || strings.HasPrefix(base, "_") } // splitFrontmatter separates a leading --- delimited YAML block from the body. A file without one is // all body. func splitFrontmatter(data []byte) (front, body []byte) { const fence = "---" rest, hasFence := trimLeadingFence(data, fence) if !hasFence { return nil, data } if i := bytes.Index(rest, []byte("\n"+fence)); i >= 0 { front = rest[:i+1] body = rest[i+1+len(fence):] return front, bytes.TrimLeft(body, "\r\n") } return nil, data } // trimLeadingFence removes an opening fence line, reporting whether one was there. func trimLeadingFence(data []byte, fence string) ([]byte, bool) { s := bytes.TrimLeft(data, "\ufeff") if !bytes.HasPrefix(s, []byte(fence)) { return data, false } s = s[len(fence):] s = bytes.TrimLeft(s, "\r") if !bytes.HasPrefix(s, []byte("\n")) { return data, false } return s[1:], true } // dropCollisions removes every bundle sharing a key and language with another. // // Two spellings of the same variant — about.md and about.en.md, or about.md and about/index.md — are // ambiguous rather than harmless, so none of them is served (ADR-0021). func dropCollisions(all []Bundle) []Bundle { seen := map[string]int{} for _, b := range all { seen[b.Key+"\x00"+b.Lang]++ } kept := make([]Bundle, 0, len(all)) for _, b := range all { if seen[b.Key+"\x00"+b.Lang] > 1 { slog.Error("skipping ambiguous bundle: two files claim one key and language", "key", b.Key, "lang", b.Lang, "path", b.Path) continue } kept = append(kept, b) } return kept } // Site is a set of bundles indexed for lookup by permalink key. type Site struct { byKeyLang map[string]Bundle aliases map[string]string } // NewSite indexes bundles for lookup. Later variants of a key and language cannot occur, because Scan // drops ambiguity before this sees it. func NewSite(bundles []Bundle) *Site { s := &Site{ byKeyLang: make(map[string]Bundle, len(bundles)), aliases: map[string]string{}, } for _, b := range bundles { s.byKeyLang[b.Key+"\x00"+b.Lang] = b } s.indexAliases(bundles) return s } // indexAliases maps each alias to the key it redirects to. // // An alias that names a real bundle, or that two bundles both claim, is ambiguous: it is logged and // dropped rather than picking a winner, and the real bundle keeps its URL (ADR-0029). func (s *Site) indexAliases(bundles []Bundle) { claimed := map[string][]string{} for _, b := range bundles { for _, a := range b.Aliases { claimed[a] = append(claimed[a], b.Key) } } for alias, keys := range claimed { if _, isReal := s.byKeyLang[alias+"\x00"+DefaultLang]; isReal { slog.Error("ignoring alias that names a real bundle", "alias", alias, "claimed_by", keys) continue } if len(keys) > 1 { slog.Error("ignoring alias claimed by more than one bundle", "alias", alias, "claimed_by", keys) continue } s.aliases[alias] = keys[0] } } // Alias returns the key an alias redirects to. func (s *Site) Alias(alias string) (string, bool) { key, ok := s.aliases[alias] return key, ok } // Lookup returns the best variant of a key for a requested language, and the language actually served. // // The fallback chain is requested → default → any (ADR-0009); "any" is resolved in sorted order so the // same request always answers the same way. A key with no variants at all reports false. func (s *Site) Lookup(key, lang string) (b Bundle, served string, ok bool) { for _, try := range []string{lang, DefaultLang} { if try == "" { continue } if b, ok = s.byKeyLang[key+"\x00"+try]; ok { return b, try, true } } for _, l := range s.Variants(key) { b = s.byKeyLang[key+"\x00"+l] return b, l, true } return Bundle{}, "", false } // Variants lists the languages a key exists in, sorted. func (s *Site) Variants(key string) []string { var langs []string for kl := range s.byKeyLang { k, l, found := strings.Cut(kl, "\x00") if found && k == key { langs = append(langs, l) } } sort.Strings(langs) return langs } // HasLang reports whether any bundle is written in lang. The resolver needs this to tell a language // prefix from a section that happens to share its name. func (s *Site) HasLang(lang string) bool { for kl := range s.byKeyLang { if _, l, found := strings.Cut(kl, "\x00"); found && l == lang { return true } } return false } // Len reports how many bundles the site holds. func (s *Site) Len() int { return len(s.byKeyLang) } // URL is the permalink of a variant: /{section}/{slug}/, with a language prefix for anything but the // default locale (ADR-0008, ADR-0009). Templates never build a path by hand. func URL(key, lang string) string { if lang == "" || lang == DefaultLang { return "/" + key + "/" } return "/" + lang + "/" + key + "/" }