khosra/internal/content/content.go

package content

import (
	"bytes"
	"fmt"
	"io/fs"
	"log/slog"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/unicode/norm"
	"gopkg.in/yaml.v3"
)

// DefaultLang is the locale served at the root of the URL space; every other language is served under a
// prefix (ADR-0009). A filename with no language suffix means this locale (ADR-0021).
const DefaultLang = "en"

// Bundle is one addressable piece of content in one language.
//
// Only Title is lifted out of frontmatter; every other key lands in Extra, so a template or a later
// feature can read a field the parser has never heard of (ADR-0002). Absence is always the zero value,
// never an error.
type Bundle struct {
	// Key identifies the bundle across languages: its path under content/, without language suffix or
	// extension, NFC-normalised. This is the identity in ADR-0004 and never contains a language.
	Key string
	// Lang is always set; a file with no suffix reports DefaultLang.
	Lang string
	// Path is the file this bundle was read from, relative to the site root.
	Path string
	// Title is empty when frontmatter omits it. Whether that is legal depends on the type, which
	// nothing decides yet, so the parser accepts it.
	Title string
	// Date is publication time, zero when frontmatter omits it. Undated bundles sort after dated ones.
	Date time.Time
	// Tags are free-form terms, case and script preserved as the author wrote them. The URL form is
	// TagSlug of each (ADR-0018).
	Tags []string
	// Aliases are paths that must keep resolving to this bundle, each redirecting to its canonical URL
	// (ADR-0008). Additive only: an alias is a promise never withdrawn.
	Aliases []string
	// Slug is a hand-chosen final path segment, empty unless frontmatter declares one. It renames the
	// bundle's address in every language (ADR-0035) and never its Key, which stays the identity.
	Slug string
	// Route is the path this bundle is served at: its Key, unless a slug renamed the last segment. Set by
	// NewSite, which is the only place that can see whether every variant agrees.
	Route string
	// Draft is true when frontmatter says so. A draft is not served at all until `-dev` reveals it, and
	// neither are the files inside its bundle (ADR-0024).
	Draft bool
	// Styles and Scripts are this bundle's own CSS and JS files, named in frontmatter and living beside the
	// body. Bundle-relative and nothing else: a name that climbs out is dropped at parse (ADR-0079). Empty
	// for the overwhelming majority of bundles, which is the point — an ordinary page carries no script.
	Styles, Scripts []string
	// Use names theme assets this bundle wants without calling the shortcode that would pull them in — the
	// frontmatter half of the same mechanism. These are names the theme resolves, never file paths.
	Use []string
	// Order is this bundle's position in the series it is nested under, zero when frontmatter omits it.
	// The convention is sparse (10, 20, 30), so zero is not a position: an unordered member sorts by name
	// after every ordered one (ADR-0033).
	Order int
	// Body is everything after the frontmatter, unrendered.
	Body []byte
	// Extra holds every frontmatter key other than title, exactly as YAML parsed it.
	Extra map[string]any
}

// OpenSite opens a site root for reading.
//
// The returned fs.FS is backed by [os.Root], which refuses any name that would resolve outside dir,
// including through a symlink — unlike os.DirFS, which does not (ADR-0031). The root is held for the
// life of the process.
func OpenSite(dir string) (fs.FS, error) {
	root, err := os.OpenRoot(dir)
	if err != nil {
		return nil, fmt.Errorf("open site root %s: %w", dir, err)
	}
	return root.FS(), nil
}

// Problem is something wrong with the content that the engine worked around.
//
// Every one of these is also a log line when the server starts, but `check` needs them as data rather than as
// text, so they are collected here and reported by whoever asked (ADR-0029).
type Problem struct {
	// Path names the file, or the key when the trouble is about identity rather than one file.
	Path string
	// Detail says what was wrong, in the terms an author can act on.
	Detail string
}

// Scan reads every bundle under content/ in fsys, logging anything it worked around.
//
// A bundle that cannot be parsed, or that collides with another on the same key and language, is logged
// at **warn** and left out; neither is fatal, because one mistyped colon must not take down a site
// (ADR-0029). Warn rather than error is the whole point of the distinction: the engine coped and the site
// still serves, so error stays reserved for a request or a build step that actually failed
// (`conventions.md`). An error is returned only when the walk itself fails.
func Scan(fsys fs.FS) ([]Bundle, error) {
	found, problems, err := ScanReport(fsys)
	for _, p := range problems {
		slog.Warn("content problem", "path", p.Path, "detail", p.Detail)
	}
	return found, err
}

// ScanReport is Scan with the problems returned instead of only logged.
func ScanReport(fsys fs.FS) ([]Bundle, []Problem, error) {
	var problems []Problem
	var found []Bundle
	err := fs.WalkDir(fsys, "content", func(p string, d fs.DirEntry, err error) error {
		switch {
		case err != nil:
			return err
		case d.IsDir():
			if skipDir(path.Base(p)) {
				return fs.SkipDir
			}
			return nil
		case !strings.HasSuffix(p, ".md"):
			return nil
		case isPartial(path.Base(p)):
			return nil
		}
		data, err := fs.ReadFile(fsys, p)
		if err != nil {
			problems = append(problems, Problem{p, "unreadable, so it is not served: " + err.Error()})
			return nil
		}
		b, err := Parse(strings.TrimPrefix(p, "content/"), data)
		if err != nil {
			problems = append(problems, Problem{p, "not served, cannot be parsed: " + err.Error()})
			return nil
		}
		b.Path = p
		found = append(found, b)
		return nil
	})
	if err != nil {
		return nil, problems, fmt.Errorf("scan content: %w", err)
	}
	kept, collisions := dropCollisions(found)
	return kept, append(problems, collisions...), nil
}

// Parse reads one bundle from the bytes of a file, named relative to content/.
func Parse(name string, data []byte) (Bundle, error) {
	key, lang, ok := splitName(name)
	if !ok {
		return Bundle{}, fmt.Errorf("not a bundle filename: %s", name)
	}
	front, body := splitFrontmatter(data)
	b := Bundle{Key: key, Lang: lang, Body: body, Extra: map[string]any{}}
	if len(front) > 0 {
		if err := yaml.Unmarshal(front, &b.Extra); err != nil {
			return Bundle{}, fmt.Errorf("frontmatter: %w", err)
		}
	}
	if t, isStr := b.Extra["title"].(string); isStr {
		b.Title = t
	}
	delete(b.Extra, "title")
	b.Aliases = stringList(b.Extra["aliases"])
	delete(b.Extra, "aliases")
	b.Date = asTime(b.Extra["date"])
	delete(b.Extra, "date")
	b.Tags = terms(b.Extra["tags"])
	delete(b.Extra, "tags")
	b.Order = asInt(b.Extra["order"])
	delete(b.Extra, "order")
	b.Draft, _ = b.Extra["draft"].(bool)
	delete(b.Extra, "draft")
	b.Styles = bundleFiles(b.Extra["styles"], b.Key)
	delete(b.Extra, "styles")
	b.Scripts = bundleFiles(b.Extra["scripts"], b.Key)
	delete(b.Extra, "scripts")
	b.Use = terms(b.Extra["use"])
	delete(b.Extra, "use")
	if slug, isStr := b.Extra["slug"].(string); isStr {
		// One segment, normalised like every other identifier (ADR-0015). Slashes would let a slug move the
		// bundle to another section, which is a move, not a rename.
		b.Slug = Normalise(strings.Trim(strings.TrimSpace(slug), "/"))
	}
	delete(b.Extra, "slug")
	b.Route = b.Key
	return b, nil
}

// Published reports whether a bundle is visible to a reader at the given moment.
//
// Two ways not to be: marked a draft, or dated in the future. The second is the same rule seen from the other
// side — a bundle becomes public exactly when its own date arrives, with nothing to run and nothing to
// invalidate (content-model.md).
func (b Bundle) Published(at time.Time) bool {
	return !b.Draft && !b.Date.After(at)
}

// Assets is the directory holding a bundle's own local files, and false for a bundle that has none.
//
// Only a directory bundle has one. A single-file bundle's neighbours belong to its section rather than to it,
// and its URL ends in a slash that no file beside it sits under — so an author with assets writes a directory
// bundle (content-model.md).
func (b Bundle) Assets() (string, bool) {
	base := path.Base(b.Path)
	if strings.HasPrefix(base, "index.") || strings.HasPrefix(base, "_index.") {
		return path.Dir(b.Path), true
	}
	return "", false
}

// stringList reads a YAML scalar or sequence of strings as keys: normalised, without surrounding
// slashes. Anything that is not a string is ignored rather than failing the bundle.
func stringList(v any) []string {
	var out []string
	add := func(x any) {
		if str, ok := x.(string); ok {
			if k := Normalise(strings.Trim(str, "/")); k != "" {
				out = append(out, k)
			}
		}
	}
	switch t := v.(type) {
	case string:
		add(t)
	case []any:
		for _, x := range t {
			add(x)
		}
	}
	return out
}

// asTime reads a frontmatter date. yaml.v3 hands back a time.Time for an unquoted timestamp and a string
// for a quoted one, so both spellings work and anything else is simply absent.
func asTime(v any) time.Time {
	switch t := v.(type) {
	case time.Time:
		return t
	case string:
		for _, layout := range []string{time.RFC3339, "2006-01-02"} {
			if parsed, err := time.Parse(layout, t); err == nil {
				return parsed
			}
		}
	}
	return time.Time{}
}

// asInt reads a frontmatter integer. yaml.v3 hands back an int for an unquoted number and a string for a
// quoted one, so both spellings work and anything else is simply absent.
func asInt(v any) int {
	switch t := v.(type) {
	case int:
		return t
	case string:
		if n, err := strconv.Atoi(t); err == nil {
			return n
		}
	}
	return 0
}

// terms reads a scalar or sequence of tag names, preserving case and script.
// bundleFiles reads a scalar-or-list of filenames that must stay inside the bundle.
//
// A name that climbs out is dropped and logged rather than fatal (ADR-0029), the same refusal an include
// and a code block's `file=` already make (ADR-0038): a page may ship its own stylesheet, never reach a
// template, a dotfile, or another bundle's files with one.
func bundleFiles(v any, where string) []string {
	var out []string
	for _, name := range terms(v) {
		if strings.Contains(name, "..") || strings.HasPrefix(name, "/") {
			slog.Warn("asset name leaves its bundle and is ignored", "name", name, "bundle", where)
			continue
		}
		out = append(out, name)
	}
	return out
}

func terms(v any) []string {
	var out []string
	add := func(x any) {
		if str, ok := x.(string); ok {
			if t := strings.TrimSpace(Normalise(str)); t != "" {
				out = append(out, t)
			}
		}
	}
	switch t := v.(type) {
	case string:
		add(t)
	case []any:
		for _, x := range t {
			add(x)
		}
	}
	return out
}

// TagSlug is the URL form of a tag: normalised, lowercased, spaces joined by hyphens.
//
// Lowercasing is a no-op for scripts without case, so Bengali terms pass through unchanged. A hand-chosen
// slug per term waits for the type declaration that owns term overrides (ADR-0015).
func TagSlug(tag string) string {
	return strings.Join(strings.Fields(strings.ToLower(Normalise(tag))), "-")
}

// Normalise puts s into NFC.
//
// Every identifier goes through this: Bengali conjuncts have several byte encodings for text that looks
// identical, and macOS hands back NFD, so without it two indistinguishable files take different keys
// (ADR-0015).
func Normalise(s string) string { return norm.NFC.String(s) }

// splitName derives a bundle key and language from a path under content/.
//
// index.md and _index.md name the directory they sit in; anything else names itself. A trailing
// two- or three-letter lowercase segment is a language suffix.
func splitName(name string) (key, lang string, ok bool) {
	if !strings.HasSuffix(name, ".md") || name == "" {
		return "", "", false
	}
	dir, base := path.Split(strings.TrimSuffix(name, ".md"))
	dir = strings.TrimSuffix(dir, "/")
	if base == "" {
		return "", "", false
	}
	lang = DefaultLang
	if i := strings.LastIndex(base, "."); i > 0 && isLangTag(base[i+1:]) {
		lang, base = base[i+1:], base[:i]
	}
	if base == "index" || base == "_index" {
		key = dir
	} else {
		key = path.Join(dir, base)
	}
	return Normalise(key), lang, true
}

// isLangTag reports whether s looks like a language suffix: two or three lowercase letters.
func isLangTag(s string) bool {
	if len(s) < 2 || len(s) > 3 {
		return false
	}
	for _, r := range s {
		if r < 'a' || r > 'z' {
			return false
		}
	}
	return true
}

// isPartial reports whether a filename is a fragment rather than a bundle of its own.
//
// An underscore prefix, the same mark a directory already uses, with `_index` excepted because that names
// the directory it sits in. Without this a file meant only to be included would also be a bundle: it would
// take a URL, appear in its section's listing, and turn its bundle into a one-member series
// (content-model.md).
func isPartial(base string) bool {
	name := strings.TrimSuffix(base, ".md")
	if i := strings.LastIndex(name, "."); i > 0 && isLangTag(name[i+1:]) {
		name = name[:i]
	}
	return strings.HasPrefix(name, "_") && name != "_index"
}

// skipDir reports whether a directory is not content: hidden, underscore-prefixed, or a bundle's extras.
//
// Extras are skipped entirely, so a `.md` in there is an asset and never a bundle — no frontmatter, no identity,
// no language variants, and no URL of its own beyond the extras route (content-model.md).
func skipDir(base string) bool {
	return strings.HasPrefix(base, ".") && base != "." ||
		strings.HasPrefix(base, "_") ||
		base == ExtrasDir
}

// splitFrontmatter separates a leading --- delimited YAML block from the body. A file without one is
// all body.
func splitFrontmatter(data []byte) (front, body []byte) {
	const fence = "---"
	rest, hasFence := trimLeadingFence(data, fence)
	if !hasFence {
		return nil, data
	}
	if i := bytes.Index(rest, []byte("\n"+fence)); i >= 0 {
		front = rest[:i+1]
		body = rest[i+1+len(fence):]
		return front, bytes.TrimLeft(body, "\r\n")
	}
	return nil, data
}

// trimLeadingFence removes an opening fence line, reporting whether one was there.
func trimLeadingFence(data []byte, fence string) ([]byte, bool) {
	s := bytes.TrimLeft(data, "\ufeff")
	if !bytes.HasPrefix(s, []byte(fence)) {
		return data, false
	}
	s = s[len(fence):]
	s = bytes.TrimLeft(s, "\r")
	if !bytes.HasPrefix(s, []byte("\n")) {
		return data, false
	}
	return s[1:], true
}

// dropCollisions removes every bundle sharing a key and language with another.
//
// Two spellings of the same variant — about.md and about.en.md, or about.md and about/index.md — are
// ambiguous rather than harmless, so none of them is served (ADR-0021).
func dropCollisions(all []Bundle) ([]Bundle, []Problem) {
	seen := map[string]int{}
	for _, b := range all {
		seen[b.Key+"\x00"+b.Lang]++
	}
	var problems []Problem
	kept := make([]Bundle, 0, len(all))
	for _, b := range all {
		if seen[b.Key+"\x00"+b.Lang] > 1 {
			problems = append(problems, Problem{b.Path,
				"not served: another file claims the same key (" + b.Key + ") and language (" + b.Lang + ")"})
			continue
		}
		kept = append(kept, b)
	}
	return kept, problems
}

// PerPage is how many entries a listing shows.
//
// Changing it renumbers page URLs, which ADR-0028 calls a URL event; it becomes a setting when the
// cascade exists rather than being one knob early.
const PerPage = 10

// URL is the permalink of a variant: /{section}/{slug}/, with a language prefix for anything but the
// default locale (ADR-0008, ADR-0009). Templates never build a path by hand.
//
// An empty key is the site root, which is "/" and not "//" — the case a feed for the whole site asks for.
func URL(key, lang string) string {
	prefix := ""
	if lang != "" && lang != DefaultLang {
		prefix = lang
	}
	segments := path.Join(prefix, key)
	if segments == "" {
		return "/"
	}
	return "/" + segments + "/"
}

// TagURL is the permalink of a tag listing, optionally narrowed to a section (ADR-0018).
func TagURL(section, slug, lang string, page int) string {
	key := TagsSegment + "/" + slug
	if section != "" {
		key = section + "/" + key
	}
	return PageURL(key, lang, page)
}

// DerivedPrefix is where generated files are served from. Reserved like any other engine-owned path, and
// deliberately not under content: nothing an author writes is addressed there (ADR-0042).
const DerivedPrefix = "/derived/"

// DerivedURL is the address of one generated file.
func DerivedURL(name string) string { return DerivedPrefix + name }

// TagsSegment is reserved at the top level and inside every section, so no bundle may be slugged with it.
const TagsSegment = "tags"

// PageURL is the permalink of a listing page. Page one is the bare listing URL, never /page/1/
// (ADR-0028).
func PageURL(key, lang string, page int) string {
	base := URL(key, lang)
	if page <= 1 {
		return base
	}
	return base + "page/" + strconv.Itoa(page) + "/"
}