content: read a site root into bundles
Bundle loading with no HTTP: walk content/, split YAML frontmatter, derive an NFC-normalised key and a language from the filename, and lift only title out of frontmatter so every other key stays readable through Extra (ADR-0002). Path safety is os.Root rather than a hand-rolled cleaner (ADR-0031). os.DirFS documents that it does not prevent symlink escape; os.Root refuses any name resolving outside the root, so the guard is a property of the type instead of a check to remember at each call site. Test: a symlink to a file above the root cannot be read. This clears the traversal item off the latent list. A bundle that will not parse is logged and skipped, never fatal (ADR-0029), as is a key claimed by two spellings of one variant (ADR-0021). Bundle carries only Key, Lang, Path, Title, Body and Extra; Date, Slug, Draft and Aliases arrive with the features that read them.
This commit is contained in:
@@ -0,0 +1,217 @@
|
||||
// Package content reads a site root into bundles. It knows the disk and nothing about HTTP.
|
||||
//
|
||||
// Every read goes through an [os.Root] (ADR-0031), so no path — from a filename or later from a
|
||||
// request — can escape the site root, even through a symlink.
|
||||
package content
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"log/slog"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
// DefaultLang is the locale served at the root of the URL space; every other language is served under a
|
||||
// prefix (ADR-0009). A filename with no language suffix means this locale (ADR-0021).
|
||||
const DefaultLang = "en"
|
||||
|
||||
// Bundle is one addressable piece of content in one language.
|
||||
//
|
||||
// Only Title is lifted out of frontmatter; every other key lands in Extra, so a template or a later
|
||||
// feature can read a field the parser has never heard of (ADR-0002). Absence is always the zero value,
|
||||
// never an error.
|
||||
type Bundle struct {
|
||||
// Key identifies the bundle across languages: its path under content/, without language suffix or
|
||||
// extension, NFC-normalised. This is the identity in ADR-0004 and never contains a language.
|
||||
Key string
|
||||
// Lang is always set; a file with no suffix reports DefaultLang.
|
||||
Lang string
|
||||
// Path is the file this bundle was read from, relative to the site root.
|
||||
Path string
|
||||
// Title is empty when frontmatter omits it. Whether that is legal depends on the type, which
|
||||
// nothing decides yet, so the parser accepts it.
|
||||
Title string
|
||||
// Body is everything after the frontmatter, unrendered.
|
||||
Body []byte
|
||||
// Extra holds every frontmatter key other than title, exactly as YAML parsed it.
|
||||
Extra map[string]any
|
||||
}
|
||||
|
||||
// OpenSite opens a site root for reading.
|
||||
//
|
||||
// The returned fs.FS is backed by [os.Root], which refuses any name that would resolve outside dir,
|
||||
// including through a symlink — unlike os.DirFS, which does not (ADR-0031). The root is held for the
|
||||
// life of the process.
|
||||
func OpenSite(dir string) (fs.FS, error) {
|
||||
root, err := os.OpenRoot(dir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open site root %s: %w", dir, err)
|
||||
}
|
||||
return root.FS(), nil
|
||||
}
|
||||
|
||||
// Scan reads every bundle under content/ in fsys.
|
||||
//
|
||||
// A bundle that cannot be parsed, or that collides with another on the same key and language, is logged
|
||||
// at error level and left out; neither is fatal, because one mistyped colon must not take down a site
|
||||
// (ADR-0029). An error is returned only when the walk itself fails.
|
||||
func Scan(fsys fs.FS) ([]Bundle, error) {
|
||||
var found []Bundle
|
||||
err := fs.WalkDir(fsys, "content", func(p string, d fs.DirEntry, err error) error {
|
||||
switch {
|
||||
case err != nil:
|
||||
return err
|
||||
case d.IsDir():
|
||||
if skipDir(path.Base(p)) {
|
||||
return fs.SkipDir
|
||||
}
|
||||
return nil
|
||||
case !strings.HasSuffix(p, ".md"):
|
||||
return nil
|
||||
}
|
||||
data, err := fs.ReadFile(fsys, p)
|
||||
if err != nil {
|
||||
slog.Error("skipping unreadable bundle", "path", p, "err", err)
|
||||
return nil
|
||||
}
|
||||
b, err := Parse(strings.TrimPrefix(p, "content/"), data)
|
||||
if err != nil {
|
||||
slog.Error("skipping unparseable bundle", "path", p, "err", err)
|
||||
return nil
|
||||
}
|
||||
b.Path = p
|
||||
found = append(found, b)
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("scan content: %w", err)
|
||||
}
|
||||
return dropCollisions(found), nil
|
||||
}
|
||||
|
||||
// Parse reads one bundle from the bytes of a file, named relative to content/.
|
||||
func Parse(name string, data []byte) (Bundle, error) {
|
||||
key, lang, ok := splitName(name)
|
||||
if !ok {
|
||||
return Bundle{}, fmt.Errorf("not a bundle filename: %s", name)
|
||||
}
|
||||
front, body := splitFrontmatter(data)
|
||||
b := Bundle{Key: key, Lang: lang, Body: body, Extra: map[string]any{}}
|
||||
if len(front) > 0 {
|
||||
if err := yaml.Unmarshal(front, &b.Extra); err != nil {
|
||||
return Bundle{}, fmt.Errorf("frontmatter: %w", err)
|
||||
}
|
||||
}
|
||||
if t, isStr := b.Extra["title"].(string); isStr {
|
||||
b.Title = t
|
||||
}
|
||||
delete(b.Extra, "title")
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// Normalise puts s into NFC.
|
||||
//
|
||||
// Every identifier goes through this: Bengali conjuncts have several byte encodings for text that looks
|
||||
// identical, and macOS hands back NFD, so without it two indistinguishable files take different keys
|
||||
// (ADR-0015).
|
||||
func Normalise(s string) string { return norm.NFC.String(s) }
|
||||
|
||||
// splitName derives a bundle key and language from a path under content/.
|
||||
//
|
||||
// index.md and _index.md name the directory they sit in; anything else names itself. A trailing
|
||||
// two- or three-letter lowercase segment is a language suffix.
|
||||
func splitName(name string) (key, lang string, ok bool) {
|
||||
if !strings.HasSuffix(name, ".md") || name == "" {
|
||||
return "", "", false
|
||||
}
|
||||
dir, base := path.Split(strings.TrimSuffix(name, ".md"))
|
||||
dir = strings.TrimSuffix(dir, "/")
|
||||
if base == "" {
|
||||
return "", "", false
|
||||
}
|
||||
lang = DefaultLang
|
||||
if i := strings.LastIndex(base, "."); i > 0 && isLangTag(base[i+1:]) {
|
||||
lang, base = base[i+1:], base[:i]
|
||||
}
|
||||
if base == "index" || base == "_index" {
|
||||
key = dir
|
||||
} else {
|
||||
key = path.Join(dir, base)
|
||||
}
|
||||
return Normalise(key), lang, true
|
||||
}
|
||||
|
||||
// isLangTag reports whether s looks like a language suffix: two or three lowercase letters.
|
||||
func isLangTag(s string) bool {
|
||||
if len(s) < 2 || len(s) > 3 {
|
||||
return false
|
||||
}
|
||||
for _, r := range s {
|
||||
if r < 'a' || r > 'z' {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// skipDir reports whether a directory is not content: hidden, or underscore-prefixed.
|
||||
func skipDir(base string) bool {
|
||||
return strings.HasPrefix(base, ".") && base != "." || strings.HasPrefix(base, "_")
|
||||
}
|
||||
|
||||
// splitFrontmatter separates a leading --- delimited YAML block from the body. A file without one is
|
||||
// all body.
|
||||
func splitFrontmatter(data []byte) (front, body []byte) {
|
||||
const fence = "---"
|
||||
rest, hasFence := trimLeadingFence(data, fence)
|
||||
if !hasFence {
|
||||
return nil, data
|
||||
}
|
||||
if i := bytes.Index(rest, []byte("\n"+fence)); i >= 0 {
|
||||
front = rest[:i+1]
|
||||
body = rest[i+1+len(fence):]
|
||||
return front, bytes.TrimLeft(body, "\r\n")
|
||||
}
|
||||
return nil, data
|
||||
}
|
||||
|
||||
// trimLeadingFence removes an opening fence line, reporting whether one was there.
|
||||
func trimLeadingFence(data []byte, fence string) ([]byte, bool) {
|
||||
s := bytes.TrimLeft(data, "\ufeff")
|
||||
if !bytes.HasPrefix(s, []byte(fence)) {
|
||||
return data, false
|
||||
}
|
||||
s = s[len(fence):]
|
||||
s = bytes.TrimLeft(s, "\r")
|
||||
if !bytes.HasPrefix(s, []byte("\n")) {
|
||||
return data, false
|
||||
}
|
||||
return s[1:], true
|
||||
}
|
||||
|
||||
// dropCollisions removes every bundle sharing a key and language with another.
|
||||
//
|
||||
// Two spellings of the same variant — about.md and about.en.md, or about.md and about/index.md — are
|
||||
// ambiguous rather than harmless, so none of them is served (ADR-0021).
|
||||
func dropCollisions(all []Bundle) []Bundle {
|
||||
seen := map[string]int{}
|
||||
for _, b := range all {
|
||||
seen[b.Key+"\x00"+b.Lang]++
|
||||
}
|
||||
kept := make([]Bundle, 0, len(all))
|
||||
for _, b := range all {
|
||||
if seen[b.Key+"\x00"+b.Lang] > 1 {
|
||||
slog.Error("skipping ambiguous bundle: two files claim one key and language",
|
||||
"key", b.Key, "lang", b.Lang, "path", b.Path)
|
||||
continue
|
||||
}
|
||||
kept = append(kept, b)
|
||||
}
|
||||
return kept
|
||||
}
|
||||
Reference in New Issue
Block a user