diff --git a/cmd/khosra/wire.go b/cmd/khosra/wire.go index 51351fd..67cdb73 100644 --- a/cmd/khosra/wire.go +++ b/cmd/khosra/wire.go @@ -4,6 +4,7 @@ import ( "github.com/yuin/goldmark" "khosra/internal/ext/shortcodes" + "khosra/internal/ext/widows" "khosra/internal/render" ) @@ -15,5 +16,6 @@ import ( func extenders(partial render.Partial) []goldmark.Extender { return []goldmark.Extender{ shortcodes.New(partial), + widows.New(), } } diff --git a/docs/content-model.md b/docs/content-model.md index 08bbf87..28e6ac7 100644 --- a/docs/content-model.md +++ b/docs/content-model.md @@ -266,9 +266,10 @@ functions in `theme-contract.md`. A Bengali page reads `পৃষ্ঠা ২ / **Machine-readable output never localises.** A `datetime` attribute, a URL, or anything a parser reads stays ASCII in every locale. -Widow prevention is not implemented: doing it safely needs a transform over the parsed tree rather than a -pass over rendered HTML, which cannot tell prose from an escaped code span. It waits for the Stage -pipeline. +**Widows are prevented**: the last two words of a paragraph or heading are joined by a non-breaking space, +so a single word never falls alone onto its own line. It works over the parsed tree, which is what keeps it +out of code spans — a pass over rendered HTML could not tell prose from an escaped one. A block ending in a +code span, link or emphasis is left alone, because the last "word" is then a construct rather than a word. ## Shortcodes diff --git a/docs/extensions.md b/docs/extensions.md index f9133fd..7e28bcb 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -65,7 +65,7 @@ wearing a disguise. |---|---|---| | `PhaseLoad` | raw bytes + frontmatter | translation fallback. *Not* includes: they turned out to be parse-phase, because splicing another file's parsed nodes into a page is invalid rather than merely awkward (ADR-0038) | | `PhaseParse` | the parsed Markdown tree | shortcodes, transclusion, image derivatives | -| `PhaseMarkup` | rendered HTML fragments, code spans skipped | widows. Smart quotes and dashes turned out to be a Markdown parser option, and chrome localisation a template function (ADR-0034) — neither needed a phase | +| `PhaseMarkup` | rendered HTML fragments, code spans skipped | nothing yet. Everything expected here turned out to belong earlier or later: smart quotes and dashes are a Markdown parser option, chrome localisation is a template function (ADR-0034), and widows are a tree transform — over rendered HTML none of them could tell prose from an escaped code span | | `PhasePage` | the assembled page object | OpenGraph, JSON-LD, related posts, series nav | | `PhaseOutput` | the final byte stream | minification, dithering, gemtext conversion | diff --git a/docs/state.md b/docs/state.md index 7a9b8a6..6696703 100644 --- a/docs/state.md +++ b/docs/state.md @@ -1,6 +1,6 @@ # State -**Verified against:** `d67cfd1` on 2026-07-30 — update this line every change. +**Verified against:** `ff62729` on 2026-07-30 — update this line every change. If this file disagrees with the code, the code is right and this file is a bug. ## Inventory @@ -15,7 +15,8 @@ If this file disagrees with the code, the code is right and this file is a bug. | `internal/render/chrome.go` | the engine's own words: phrase table, month names, digits, and the `t`/`num`/`day` template funcs (ADR-0034) | 105 | | `internal/render/templates/` | reference theme: `base.html`, `page.html`, `list.html`, `shortcodes.html`, `theme.css` (ADR-0026) | — | | `internal/ext/shortcodes/` | first feature: `{{< name key="value" >}}` block parser and node renderer, rendering through a theme fragment (ADR-0036). `figure`, `gallery`, `include` | 315 | -| `cmd/khosra/wire.go` | the only list of enabled features (`extensions.md`) | 19 | +| `internal/ext/widows/` | second feature: joins the last two words of a paragraph or heading with a non-breaking space, over the tree so code spans are safe | 108 | +| `cmd/khosra/wire.go` | the only list of enabled features (`extensions.md`) | 20 | | `internal/web/resolve.go` | URL → (key, lang, page, tag) or a canonical redirect: language prefix, `/en/…` fork guard, pagination, tags, trailing slash | 112 | | `internal/web/web.go` | handler: resolve, look up with fallback, section and tag listings, sequence, `/static/` (misses and refusals alike answer 404), degrade on failure | 152 | | `cmd/khosra/main.go` | flags, wiring, startup — the only place things are assembled | 53 | @@ -24,7 +25,7 @@ If this file disagrees with the code, the code is right and this file is a bug. Serves a bundle at `/{section}/{slug}/`, a paginated listing per section, tag listings global and section-narrowed, sequence navigation and a series archive on any nested bundle, and `static/` verbatim. Chrome text, dates and digits render in English or Bengali; authored text is untouched but for typographic -smoothing (ADR-0034). This repo holds engine source only — the site root is external and passed with +smoothing and widow prevention (ADR-0034). This repo holds engine source only — the site root is external and passed with `-site` (ADR-0011). Frontmatter the parser lifts today: `title`, `date`, `tags`, `aliases`, `order`. Every other key in @@ -40,12 +41,12 @@ this change*. | Counter | Now | Extraction due at | What it buys | |---|---|---|---| -| Render transforms — **page-level only** | 0 | **3** | Stage pipeline (ordered `func(ctx,*Page) error`). Parse-phase work does *not* count and must not: goldmark's extender list is already an ordered pipeline for it, so typography and shortcodes compose there (`cmd/khosra/wire.go`) and a second pipeline beside it would be pure duplication. This counts transforms over the assembled page, which nothing hosts yet — OpenGraph and JSON-LD (queue 15) are the first candidates | +| Render transforms — **page-level only** | 0 | **3** | Stage pipeline (ordered `func(ctx,*Page) error`). Parse-phase work does *not* count and must not: goldmark's extender list is already an ordered pipeline for it, so typography, shortcodes and widows compose there (`cmd/khosra/wire.go`) and a second pipeline beside it would be pure duplication. This counts transforms over the assembled page, which nothing hosts yet — OpenGraph and JSON-LD (queue 15) are the first candidates | | Routing cases | 5 | **2** — done | Resolver at `internal/web/resolve.go`: bundle, language prefix, pagination, tag, section-narrowed tag | | Collection pages | 4 | **1** — done | Query primitive: `content.Query{Section, Tag, Lang}` + `Site.Run`. The fourth — a series archive — resolves through `Site.Sequence` instead: membership is structural and the sort ascends, so it shares the index but not the Query | | Views / output formats | 2 | **2** — due | Two template sets exist (bundle, listing); the View layer is Arc 2's third item | | Effects | 0 | **2** | Effect runner + trigger wiring (change / schedule / demand) | -| Extensions | 1 | **3** | Extension registry (`extensions.md`). The wire file arrived with the first feature rather than the registry — `cmd/khosra/wire.go`, one line, no struct | +| Extensions | 2 | **3** | Extension registry (`extensions.md`). The wire file arrived with the first feature rather than the registry — `cmd/khosra/wire.go`, one line, no struct | | Interface implementations | — | **2** | The interface itself | | Non-stdlib dependencies | 3 direct | budget in `scripts/budgets.env` | — | diff --git a/internal/ext/widows/doc.go b/internal/ext/widows/doc.go new file mode 100644 index 0000000..b8a64b8 --- /dev/null +++ b/internal/ext/widows/doc.go @@ -0,0 +1,7 @@ +// Package widows keeps the last word of a paragraph or heading from falling alone onto its own line. +// +// Contributes: a Markdown AST transformer (PhaseParse). +// Cascade keys: none. +// Contract fields: none — it changes one space in the text, and nothing a theme reads. +// Not doing: hyphenation, orphans, balancing headings across lines — those are the theme's typography. +package widows diff --git a/internal/ext/widows/widows.go b/internal/ext/widows/widows.go new file mode 100644 index 0000000..bd0fb17 --- /dev/null +++ b/internal/ext/widows/widows.go @@ -0,0 +1,101 @@ +package widows + +import ( + "bytes" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +// nbsp is the space that will not break. Written as bytes because it replaces bytes in the text. +var nbsp = []byte(" ") + +// New returns the Markdown extension. +func New() goldmark.Extender { return extension{} } + +type extension struct{} + +func (extension) Extend(md goldmark.Markdown) { + md.Parser().AddOptions(parser.WithASTTransformers(util.Prioritized(transformer{}, 200))) +} + +// transformer joins the last two words of every paragraph and heading. +// +// It works on the parsed tree rather than on rendered HTML, which is the whole reason this waited for the +// shortcode machinery: a pass over HTML cannot tell prose from an escaped code span, and would happily edit +// the inside of one (content-model.md). +type transformer struct{} + +func (transformer) Transform(doc *ast.Document, reader text.Reader, pc parser.Context) { + source := reader.Source() + err := ast.Walk(doc, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch n.Kind() { + case ast.KindParagraph, ast.KindHeading: + join(n, source) + } + return ast.WalkContinue, nil + }) + if err != nil { + // Walk only fails if this function does, and it does not. + return + } +} + +// join replaces the last space of a block's final line with a non-breaking one. +// +// Only plain text qualifies. A block ending in a code span, a link or emphasis is left alone: the last +// "word" is then a construct rather than a word, and joining it to the one before would mean reaching inside +// markup for a typographic nicety. +func join(block ast.Node, source []byte) { + run := finalLine(block) + if len(run) == 0 { + return + } + var line []byte + for _, t := range run { + line = append(line, t.Segment.Value(source)...) + } + i := bytes.LastIndexByte(line, ' ') + if i < 0 || i == len(line)-1 { + // One word, or a trailing space with nothing after it: nothing to keep together. + return + } + // A String node carries its own bytes, so the text can differ from the source. Editing the segment is + // impossible — a segment is an offset into bytes every other node shares. + joined := make([]byte, 0, len(line)+len(nbsp)) + joined = append(joined, line[:i]...) + joined = append(joined, nbsp...) + joined = append(joined, line[i+1:]...) + block.ReplaceChild(block, run[0], ast.NewString(joined)) + for _, t := range run[1:] { + block.RemoveChild(block, t) + } +} + +// finalLine is the block's last line as the consecutive text nodes that make it up. +// +// Several nodes, not one: another extension may have split the run. The typographer splits it wherever it +// looks for a substitution, so a paragraph ending "hand." arrives as two text nodes and the last of them +// holds no space at all — which is exactly how the first version of this passed its own tests and did +// nothing in the assembled engine. +func finalLine(block ast.Node) []*ast.Text { + var run []*ast.Text + last := block.LastChild() + for n := last; n != nil; n = n.PreviousSibling() { + t, isText := n.(*ast.Text) + if !isText { + break + } + if n != last && (t.SoftLineBreak() || t.HardLineBreak()) { + break // the final line starts after this node + } + run = append([]*ast.Text{t}, run...) + } + return run +} diff --git a/internal/ext/widows/widows_test.go b/internal/ext/widows/widows_test.go new file mode 100644 index 0000000..5afd430 --- /dev/null +++ b/internal/ext/widows/widows_test.go @@ -0,0 +1,95 @@ +package widows + +import ( + "strings" + "testing" + + "github.com/yuin/goldmark" + gmext "github.com/yuin/goldmark/extension" +) + +const nb = "\u00a0" + +func convert(t *testing.T, markdown string) string { + t.Helper() + md := goldmark.New(goldmark.WithExtensions(New())) + var out strings.Builder + if err := md.Convert([]byte(markdown), &out); err != nil { + t.Fatal(err) + } + return out.String() +} + +func TestTheLastTwoWordsStayTogether(t *testing.T) { + got := convert(t, "The rain did not stop for nine days.\n") + if !strings.Contains(got, "nine"+nb+"days.") { + t.Errorf("expected a non-breaking space before the last word:\n%q", got) + } + if strings.Count(got, nb) != 1 { + t.Errorf("exactly one space should change, got %d:\n%q", strings.Count(got, nb), got) + } +} + +func TestHeadingsGetItToo(t *testing.T) { + got := convert(t, "## The Long Monsoon\n") + if !strings.Contains(got, "Long"+nb+"Monsoon") { + t.Errorf("a heading widow is the ugliest one:\n%q", got) + } +} + +func TestNothingToJoinIsLeftAlone(t *testing.T) { + for _, markdown := range []string{"Word\n", "\n"} { + got := convert(t, markdown) + if strings.Contains(got, nb) { + t.Errorf("%q should be untouched, got %q", markdown, got) + } + } +} + +func TestABlockEndingInMarkupIsLeftAlone(t *testing.T) { + // The last "word" is a construct, not a word. Reaching inside markup for a typographic nicety is how a + // transform starts corrupting content. + for _, markdown := range []string{ + "Run it with `khosra -site`\n", + "Read more [in the archive](/posts/)\n", + "It was *raining*\n", + } { + got := convert(t, markdown) + if strings.Contains(got, nb) { + t.Errorf("%q ends in markup and should be untouched, got %q", markdown, got) + } + } +} + +func TestCodeSpansAreNeverEdited(t *testing.T) { + got := convert(t, "Run `go test ./...` and then read the output.\n") + if strings.Contains(got, "go"+nb+"test") || strings.Contains(got, "test"+nb) { + t.Errorf("a code span must survive byte for byte:\n%q", got) + } + if !strings.Contains(got, "the"+nb+"output.") { + t.Errorf("the prose after it should still be joined:\n%q", got) + } +} + +func TestTextIsStillEscaped(t *testing.T) { + // The joined text becomes a String node, which is a different render path — it must escape like any + // other text, or this transform would be an injection route. + got := convert(t, "Compare a < b and c > d.\n") + if strings.Contains(got, "a < b") || !strings.Contains(got, "<") { + t.Errorf("a rewritten run must stay escaped:\n%q", got) + } +} + +func TestItStillWorksBesideTheTypographer(t *testing.T) { + // The bug this exists to catch: the typographer splits a text run wherever it looks for a substitution, + // so a paragraph ending "hand." arrives as two text nodes and the last holds no space. A version that + // only inspected the last child passed every test above and did nothing in the real engine. + md := goldmark.New(goldmark.WithExtensions(gmext.Typographer, New())) + var out strings.Builder + if err := md.Convert([]byte("Built by hand.\n"), &out); err != nil { + t.Fatal(err) + } + if !strings.Contains(out.String(), "by"+nb+"hand.") { + t.Errorf("widows must survive being combined with other extensions:\n%q", out.String()) + } +}