diff --git a/docs/content-model.md b/docs/content-model.md index 90101b1..8470faa 100644 --- a/docs/content-model.md +++ b/docs/content-model.md @@ -286,9 +286,21 @@ bundle, in filename order, which is why the sparse numeric-prefix convention ord numbers in URLs (ADR-0016). A subdirectory is not part of the gallery, and neither is a file the browser cannot show. -`include` is `[spec]`: file inclusion will resolve relative to the including bundle, may not escape the site -root, and needs a depth limit on the first attempt, since a file that includes itself is otherwise a crash. -Transclusion of another bundle's body is Arc 4 and needs a cycle guard too. +`{{< include file="notes.md" >}}` renders another file from the bundle as Markdown, in place. Three rules, +all of them consequences of ADR-0038: + +- The name is relative to the bundle and **stays inside it**. A name containing `..` is refused, so an + include cannot reach a template, a dotfile, or anything else in the site root that is not this bundle's. +- **One level.** An include inside an included file renders nothing and is logged. A file that includes + itself is therefore a log line, not a crash. +- A missing file, an unreadable one, or a call with no `file` argument renders nothing and logs. The page + still serves (ADR-0029). + +A `gallery` inside an included file still resolves against the same bundle. + +Sharing one fragment between bundles is deliberately not possible yet: it needs somewhere to keep shared +parts, which is a decision about the disk contract rather than a missing feature. Transclusion of another +bundle's *body* is Arc 4 and needs a cycle guard of its own. ## Images `[spec]` diff --git a/docs/extensions.md b/docs/extensions.md index 599a5e1..f9133fd 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -63,7 +63,7 @@ wearing a disguise. | Phase | Operates on | Examples | |---|---|---| -| `PhaseLoad` | raw bytes + frontmatter | includes, translation fallback | +| `PhaseLoad` | raw bytes + frontmatter | translation fallback. *Not* includes: they turned out to be parse-phase, because splicing another file's parsed nodes into a page is invalid rather than merely awkward (ADR-0038) | | `PhaseParse` | the parsed Markdown tree | shortcodes, transclusion, image derivatives | | `PhaseMarkup` | rendered HTML fragments, code spans skipped | widows. Smart quotes and dashes turned out to be a Markdown parser option, and chrome localisation a template function (ADR-0034) — neither needed a phase | | `PhasePage` | the assembled page object | OpenGraph, JSON-LD, related posts, series nav | diff --git a/docs/state.md b/docs/state.md index 4c8c57e..64847bd 100644 --- a/docs/state.md +++ b/docs/state.md @@ -1,6 +1,6 @@ # State -**Verified against:** `61ae5c9` on 2026-07-30 — update this line every change. +**Verified against:** `d67cfd1` on 2026-07-30 — update this line every change. If this file disagrees with the code, the code is right and this file is a bug. ## Inventory @@ -11,15 +11,15 @@ If this file disagrees with the code, the code is right and this file is a bug. | `internal/content/doc.go` | package comment | 5 | | `internal/content/content.go` | bundles: `os.Root` open, walk, frontmatter split, key/lang derivation, NFC, tag slugs, permalink building | 352 | | `internal/content/site.go` | the indexed site: lookup with language fallback, aliases, `Query` and `Run`, sections, `Sequence` | 286 | -| `internal/render/render.go` | goldmark with the typographer, per-kind template sets with site override, the `Partial`/`Origin` seams features render and resolve through, `Page`/`List`/`Sequence`/`head` | 380 | +| `internal/render/render.go` | goldmark with the typographer, per-kind template sets with site override, the `Partial`/`Origin` seams features render and resolve through, `Page`/`List`/`Sequence`/`head` | 386 | | `internal/render/chrome.go` | the engine's own words: phrase table, month names, digits, and the `t`/`num`/`day` template funcs (ADR-0034) | 105 | | `internal/render/templates/` | reference theme: `base.html`, `page.html`, `list.html`, `shortcodes.html`, `theme.css` (ADR-0026) | — | -| `internal/ext/shortcodes/` | first feature: `{{< name key="value" >}}` block parser and node renderer, rendering through a theme fragment (ADR-0036). `figure`, `gallery` | 212 | +| `internal/ext/shortcodes/` | first feature: `{{< name key="value" >}}` block parser and node renderer, rendering through a theme fragment (ADR-0036). `figure`, `gallery`, `include` | 315 | | `cmd/khosra/wire.go` | the only list of enabled features (`extensions.md`) | 19 | | `internal/web/resolve.go` | URL → (key, lang, page, tag) or a canonical redirect: language prefix, `/en/…` fork guard, pagination, tags, trailing slash | 112 | | `internal/web/web.go` | handler: resolve, look up with fallback, section and tag listings, sequence, `/static/` (misses and refusals alike answer 404), degrade on failure | 152 | | `cmd/khosra/main.go` | flags, wiring, startup — the only place things are assembled | 53 | -| `*_test.go` | table-driven, one file per source file; symlink escape (content and static), canonical paths, language fallback, aliases, pagination, tags, sequences, chrome, typography, shortcode escaping, galleries, 404 | 1363 | +| `*_test.go` | table-driven, one file per source file; symlink escape (content and static), canonical paths, language fallback, aliases, pagination, tags, sequences, chrome, typography, shortcode escaping, galleries, includes, 404 | 1437 | Serves a bundle at `/{section}/{slug}/`, a paginated listing per section, tag listings global and section-narrowed, sequence navigation and a series archive on any nested bundle, and `static/` verbatim. diff --git a/docs/theme-contract.md b/docs/theme-contract.md index bd27561..64413c7 100644 --- a/docs/theme-contract.md +++ b/docs/theme-contract.md @@ -90,6 +90,9 @@ Every fragment receives the same two fields (ADR-0037): | `{{< figure src="…" alt="…" caption="…" >}}` | `figure` | `.Args.src`, `.Args.alt`, `.Args.caption` | | `{{< gallery >}}` | `gallery` | `.Items` — the picture filenames beside the bundle, in filename order | +`{{< include file="…" >}}` has **no fragment**: an included file is content, so it renders as Markdown in +place and a theme has nothing to style about it (ADR-0038). + Authored and engine-supplied data are kept apart so a `src` argument can never be confused with a `src` the engine found. Arguments are escaped by `html/template` like any other data, in whichever context the template puts them — which is what keeps an author's text out of the markup. A call whose template is diff --git a/internal/ext/shortcodes/doc.go b/internal/ext/shortcodes/doc.go index 83bc5a9..6989e96 100644 --- a/internal/ext/shortcodes/doc.go +++ b/internal/ext/shortcodes/doc.go @@ -3,5 +3,5 @@ // Contributes: a Markdown block parser and node renderer (PhaseParse). // Cascade keys: none. // Contract fields: a template per shortcode name in templates/shortcodes.html, receiving its arguments. -// Not doing: inline shortcodes, file inclusion, galleries — each waits for a second real use. +// Not doing: inline shortcodes; includes that nest (ADR-0038); sharing a fragment between bundles. package shortcodes diff --git a/internal/ext/shortcodes/shortcodes.go b/internal/ext/shortcodes/shortcodes.go index 70dfbdf..470e370 100644 --- a/internal/ext/shortcodes/shortcodes.go +++ b/internal/ext/shortcodes/shortcodes.go @@ -1,6 +1,8 @@ package shortcodes import ( + "bytes" + "fmt" "io/fs" "log/slog" "path" @@ -37,15 +39,100 @@ type extension struct { partial render.Partial } -// Extend registers the block parser and the node renderer. Priorities sit above goldmark's paragraph -// parser so a line that is only a call never becomes a paragraph. +// Extend registers the block parser, the include expander, and the node renderer. Priorities sit above +// goldmark's paragraph parser so a line that is only a call never becomes a paragraph. +// +// The expander is handed md itself, because an included file is converted by the same configuration as the +// page including it — not by a second pipeline that could drift from this one (ADR-0038). func (e extension) Extend(md goldmark.Markdown) { - md.Parser().AddOptions(parser.WithBlockParsers( - util.Prioritized(blocks{}, 100))) + md.Parser().AddOptions( + parser.WithBlockParsers(util.Prioritized(blocks{}, 100)), + parser.WithASTTransformers(util.Prioritized(includes{md: md}, 100)), + ) md.Renderer().AddOptions(renderer.WithNodeRenderers( util.Prioritized(fragments{partial: e.partial}, 100))) } +// nested marks a parse that is already inside an included file, so one level is all there is (ADR-0038). +var nested = parser.NewContextKey() + +// includes fills in each include call with the converted content of the file it names. +// +// A transformer, running after the parse, rather than a node renderer: converting needs the parse context to +// know which bundle this is, and a node renderer never receives one. The included file is converted on its +// own bytes and its output stored on the node — never by splicing its nodes into this tree, which cannot +// work, since a goldmark node holds offsets into the source it came from (ADR-0038). +type includes struct { + md goldmark.Markdown +} + +func (in includes) Transform(doc *ast.Document, reader text.Reader, pc parser.Context) { + insideInclude := pc.Get(nested) != nil + for _, call := range pending(doc) { + if insideInclude { + slog.Error("ignoring an include inside an included file", "file", call.args["file"]) + continue + } + content, err := in.convert(call.args["file"], pc) + if err != nil { + slog.Error("skipping include", "file", call.args["file"], "err", err) + continue + } + call.content = content + } +} + +// convert reads one included file and renders it, relative to the bundle being rendered. +// +// The nested parse carries the same Origin, so a gallery inside an included file still resolves against the +// bundle, and it is marked nested, so an include there renders nothing. +func (in includes) convert(name string, pc parser.Context) ([]byte, error) { + if name == "" { + return nil, fmt.Errorf("include needs a file argument") + } + // A name is relative to the bundle and stays inside it. os.Root already refuses a path leaving the site + // root, but path.Join collapses ".." long before it gets there, so without this an include could read + // anything else in the site root — a template, a stray dotfile — and publish it. Sharing one fragment + // between bundles is a fair wish and not this: it needs somewhere to put shared parts, chosen on purpose. + if strings.Contains(name, "..") { + return nil, fmt.Errorf("include stays inside its bundle: %s", name) + } + origin, ok := render.OriginFrom(pc) + if !ok || origin.Files == nil { + return nil, fmt.Errorf("no site root to include from") + } + data, err := fs.ReadFile(origin.Files, path.Join(origin.Dir, name)) + if err != nil { + return nil, err + } + inner := parser.NewContext() + render.WithOrigin(inner, origin) + inner.Set(nested, true) + var out bytes.Buffer + if err := in.md.Convert(data, &out, parser.WithContext(inner)); err != nil { + return nil, err + } + return out.Bytes(), nil +} + +// pending lists the include calls in a tree. +func pending(doc *ast.Document) []*node { + var found []*node + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + if call, is := n.(*node); is && call.name == "include" { + found = append(found, call) + } + return ast.WalkContinue, nil + }) + if err != nil { + slog.Error("walking for includes", "err", err) + } + return found +} + // kind identifies a parsed call in the tree. var kind = ast.NewNodeKind("Shortcode") @@ -56,6 +143,12 @@ type node struct { args map[string]string // items are what the feature gathered at parse time, when it still knew which bundle this is. items []string + // content is output the feature produced itself, written instead of a theme fragment. An included file + // is content, not decoration, so it has no template (ADR-0038). + content []byte + // isContent marks a call whose output is content, so a failure renders nothing rather than falling + // through to a fragment lookup and reporting a missing template that was never expected to exist. + isContent bool } func (n *node) Kind() ast.NodeKind { return kind } @@ -75,10 +168,14 @@ func (blocks) Open(parent ast.Node, reader text.Reader, pc parser.Context) (ast. } reader.Advance(seg.Len() - 1) n := &node{name: name, args: args} - // A call that reads the filesystem does it here, where the parse context says which bundle this is. - // The renderer has no context, so anything gathered has to be gathered now. - if name == "gallery" { + switch name { + case "gallery": + // Reading the filesystem happens here, where the parse context says which bundle this is; the + // renderer never gets one, so anything gathered has to be gathered now. n.items = images(pc) + case "include": + // Filled in by the transformer, which runs once this parse is complete. + n.isContent = true } return n, parser.NoChildren } @@ -144,6 +241,12 @@ func (f fragments) render(w util.BufWriter, source []byte, n ast.Node, entering return ast.WalkContinue, nil } call := n.(*node) + if call.isContent { + if _, err := w.Write(call.content); err != nil { + return ast.WalkStop, err + } + return ast.WalkContinue, nil + } out, err := f.partial(call.name, render.Fragment{Args: call.args, Items: call.items}) if err != nil { slog.Error("skipping shortcode", "name", call.name, "err", err) diff --git a/internal/ext/shortcodes/shortcodes_test.go b/internal/ext/shortcodes/shortcodes_test.go index 6c07208..1744ff2 100644 --- a/internal/ext/shortcodes/shortcodes_test.go +++ b/internal/ext/shortcodes/shortcodes_test.go @@ -177,6 +177,80 @@ func TestGalleryWithoutASiteRootRendersNothing(t *testing.T) { } } +func TestIncludeRendersTheFileBesideTheBundle(t *testing.T) { + fsys := fstest.MapFS{ + "content/pages/about/index.md": {Data: []byte("---\ntitle: About\n---\nFirst.\n\n{{< include file=\"more.md\" >}}\n\nLast.\n")}, + "content/pages/about/more.md": {Data: []byte("## Included\n\nWith *emphasis* and a [link](/posts/).\n")}, + } + got := bundle(t, fsys, "pages/about") + // Converted as Markdown, not pasted as text: the heading, emphasis and link prove it. + for _, want := range []string{"

Included

", "emphasis", `href="/posts/"`} { + if !strings.Contains(got, want) { + t.Errorf("missing %q — an include is Markdown, not a string:\n%s", want, got) + } + } + first, included, last := strings.Index(got, "First."), strings.Index(got, "Included"), strings.Index(got, "Last.") + if first < 0 || included < first || last < included { + t.Errorf("included content belongs where the call was:\n%s", got) + } +} + +func TestAnIncludedFileCannotItselfInclude(t *testing.T) { + // One level, by design (ADR-0038). A file including itself is the case that would otherwise recurse + // until the stack gave out — a crash caused by content, which ADR-0029 forbids. + fsys := fstest.MapFS{ + "content/pages/loop/index.md": {Data: []byte("---\ntitle: Loop\n---\nBefore.\n\n{{< include file=\"self.md\" >}}\n\nAfter.\n")}, + "content/pages/loop/self.md": {Data: []byte("Round.\n\n{{< include file=\"self.md\" >}}\n")}, + } + got := bundle(t, fsys, "pages/loop") + for _, want := range []string{"Before.", "Round.", "After."} { + if !strings.Contains(got, want) { + t.Errorf("missing %q — one level must still render:\n%s", want, got) + } + } + if n := strings.Count(got, "Round."); n != 1 { + t.Errorf("expanded %d times, want exactly one level", n) + } + if strings.Contains(got, "{{<") { + t.Errorf("the ignored nested call renders nothing, it is not printed:\n%s", got) + } +} + +func TestAGalleryInsideAnIncludedFileStillResolves(t *testing.T) { + // The nested parse carries the same Origin, which is what makes this work. + fsys := fstest.MapFS{ + "content/art/set/index.md": {Data: []byte("---\ntitle: Set\n---\n{{< include file=\"body.md\" >}}\n")}, + "content/art/set/body.md": {Data: []byte("Studies:\n\n{{< gallery >}}\n")}, + "content/art/set/one.jpg": {Data: []byte("x")}, + "content/art/set/two.png": {Data: []byte("x")}, + } + got := bundle(t, fsys, "art/set") + if !strings.Contains(got, "one.jpg") || !strings.Contains(got, "two.png") { + t.Errorf("a gallery inside an include should resolve against the same bundle:\n%s", got) + } +} + +func TestIncludeCannotEscapeTheSiteRootAndDegradesOnMisses(t *testing.T) { + fsys := fstest.MapFS{ + // `..` is refused outright: path.Join would collapse it to a real path inside the site root, which + // would let an include publish a template or a dotfile that is not content (ADR-0038). + "content/pages/a/index.md": {Data: []byte("---\ntitle: A\n---\n{{< include file=\"../../../etc/passwd\" >}}\n\nSurvived.\n")}, + "content/pages/d/index.md": {Data: []byte("---\ntitle: D\n---\n{{< include file=\"../../../secret.md\" >}}\n\nSurvived.\n")}, + "secret.md": {Data: []byte("NOT CONTENT\n")}, + "content/pages/b/index.md": {Data: []byte("---\ntitle: B\n---\n{{< include file=\"nothing.md\" >}}\n\nSurvived.\n")}, + "content/pages/c/index.md": {Data: []byte("---\ntitle: C\n---\n{{< include >}}\n\nSurvived.\n")}, + } + for _, key := range []string{"pages/a", "pages/b", "pages/c", "pages/d"} { + got := bundle(t, fsys, key) + if !strings.Contains(got, "Survived.") { + t.Errorf("%s: the page must survive a bad include:\n%s", key, got) + } + if strings.Contains(got, "root:") || strings.Contains(got, "passwd") || strings.Contains(got, "NOT CONTENT") { + t.Fatalf("%s: an include read something it must not:\n%s", key, got) + } + } +} + func TestASiteRedefinesOneFragment(t *testing.T) { site := fstest.MapFS{ "templates/shortcodes.html": {Data: []byte(`{{define "figure"}}
{{.Args.src}}
{{end}}`)}, diff --git a/internal/render/render.go b/internal/render/render.go index 2259850..77511e8 100644 --- a/internal/render/render.go +++ b/internal/render/render.go @@ -154,6 +154,12 @@ func OriginFrom(pc parser.Context) (Origin, bool) { return origin, ok } +// WithOrigin records the bundle on a parse context. A feature that starts a parse of its own — an included +// file — carries the same Origin into it, so a path there resolves against the same bundle (ADR-0038). +func WithOrigin(pc parser.Context, origin Origin) { + pc.Set(originKey, origin) +} + // New parses the theme and prepares the Markdown converter. // // siteFS may be nil, in which case only the embedded reference theme is used. A malformed template is a @@ -260,7 +266,7 @@ func (r *Renderer) Bundle(b content.Bundle, served string, variants []string, se // The parse carries which bundle it is, so a feature can resolve a path in a call against the bundle's // own directory (ADR-0031: through the rooted filesystem, never a joined path). pc := parser.NewContext() - pc.Set(originKey, Origin{Dir: path.Dir(b.Path), Files: r.files}) + WithOrigin(pc, Origin{Dir: path.Dir(b.Path), Files: r.files}) var body bytes.Buffer if err := r.md.Convert(b.Body, &body, parser.WithContext(pc)); err != nil { return nil, fmt.Errorf("markdown %s: %w", b.Path, err)