Snapshot content folders and decide which edits become revisions Patchwork keeps the version history of pages that keep changing and publishes the revisions an author announces with a frontmatter note. This first commit is the core, with no feeds or HTTP yet. - slug: the moonflowersociety.org rule, checked against a parity table generated by the site's own JavaScript - doc: normalization, frontmatter, ignore regions, block splitting, plain text per block, word tokens, content hashes - diff: Myers on blocks and words, moves, modify pairing, excerpts - store: SQLite schema and migrations (pure Go, no cgo) - config: patchwork.yaml with defaults and env overrides - engine: baseline, silent, published, amended and folded revisions, two-scan removal - source/folder: scans with an upload settle time, watch with file events, debounce and polling - cli: init, scan, watch, log, diff, doctor, version Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
- .forgejo/workflows/ci.yml
- .gitattributes
- .gitignore
- LICENSE
- README.md
- cmd/patchwork/doctor.go
- cmd/patchwork/history.go
- cmd/patchwork/main.go
- cmd/patchwork/scan.go
- go.mod
- go.sum
- internal/config/config.go
- internal/config/config_test.go
- internal/config/example.yaml
- internal/diff/blocks.go
- internal/diff/diff_test.go
- internal/diff/excerpt.go
- internal/diff/meta.go
- internal/diff/myers.go
- internal/diff/testdata/fuzz/FuzzMyers/ea961d192aa59047
- internal/diff/words.go
- internal/doc/blocks.go
- internal/doc/doc.go
- internal/doc/doc_test.go
- internal/doc/frontmatter.go
- internal/doc/ignore.go
- internal/doc/normalize.go
- internal/doc/plaintext.go
- internal/doc/tokens.go
- internal/engine/engine.go
- internal/engine/engine_test.go
- internal/engine/golden_test.go
- internal/engine/missing.go
- internal/engine/testdata/cases/code-fence-edit/expected.json
- internal/engine/testdata/cases/code-fence-edit/v1.md
- internal/engine/testdata/cases/code-fence-edit/v2.md
- internal/engine/testdata/cases/crlf-bom-nfd-only/expected.json
- internal/engine/testdata/cases/crlf-bom-nfd-only/v1.md
- internal/engine/testdata/cases/crlf-bom-nfd-only/v2.md
- internal/engine/testdata/cases/empty-file/expected.json
- internal/engine/testdata/cases/empty-file/v1.md
- internal/engine/testdata/cases/empty-file/v2.md
- internal/engine/testdata/cases/fold-window/expected.json
- internal/engine/testdata/cases/fold-window/steps.txt
- internal/engine/testdata/cases/fold-window/v1.md
- internal/engine/testdata/cases/fold-window/v2.md
- internal/engine/testdata/cases/fold-window/v3.md
- internal/engine/testdata/cases/fold-window/v4.md
- internal/engine/testdata/cases/heading-rename/expected.json
- internal/engine/testdata/cases/heading-rename/v1.md
- internal/engine/testdata/cases/heading-rename/v2.md
- internal/engine/testdata/cases/ignore-region-only/expected.json
- internal/engine/testdata/cases/ignore-region-only/v1.md
- internal/engine/testdata/cases/ignore-region-only/v2.md
- internal/engine/testdata/cases/ignored-key-only/expected.json
- internal/engine/testdata/cases/ignored-key-only/v1.md
- internal/engine/testdata/cases/ignored-key-only/v2.md
- internal/engine/testdata/cases/invalid-utf8/expected.json
- internal/engine/testdata/cases/invalid-utf8/v1.md
- internal/engine/testdata/cases/list-item-insert/expected.json
- internal/engine/testdata/cases/list-item-insert/v1.md
- internal/engine/testdata/cases/list-item-insert/v2.md
- internal/engine/testdata/cases/long-essay-truncated/expected.json
- internal/engine/testdata/cases/long-essay-truncated/v1.md
- internal/engine/testdata/cases/long-essay-truncated/v2.md
- internal/engine/testdata/cases/malformed-yaml/expected.json
- internal/engine/testdata/cases/malformed-yaml/v1.md
- internal/engine/testdata/cases/malformed-yaml/v2.md
- internal/engine/testdata/cases/malformed-yaml/v3.md
- internal/engine/testdata/cases/maturity-only/expected.json
- internal/engine/testdata/cases/maturity-only/v1.md
- internal/engine/testdata/cases/maturity-only/v2.md
- internal/engine/testdata/cases/maturity-only/v3.md
- internal/engine/testdata/cases/moved-paragraph/expected.json
- internal/engine/testdata/cases/moved-paragraph/v1.md
- internal/engine/testdata/cases/moved-paragraph/v2.md
- internal/engine/testdata/cases/new-file-with-note/expected.json
- internal/engine/testdata/cases/new-file-with-note/v1.md
- internal/engine/testdata/cases/new-file-with-note/v2.md
- internal/engine/testdata/cases/no-frontmatter/expected.json
- internal/engine/testdata/cases/no-frontmatter/v1.md
- internal/engine/testdata/cases/no-frontmatter/v2.md
- internal/engine/testdata/cases/table-edit/expected.json
- internal/engine/testdata/cases/table-edit/v1.md
- internal/engine/testdata/cases/table-edit/v2.md
- internal/slug/slug.go
- internal/slug/slug_test.go
- internal/slug/testdata/parity.json
- internal/source/folder/folder.go
- internal/source/folder/folder_test.go
- internal/source/folder/watch.go
- internal/store/meta.go
- internal/store/pages.go
- internal/store/revisions.go
- internal/store/schema.go
- internal/store/snapshots.go
- internal/store/store.go
- internal/store/store_test.go
@@ -0,0 +1,18 @@+name: ci + +on: + push: + branches: [main] + pull_request: + +# Needs a Forgejo runner registered with the "docker" label. +jobs: + test: + runs-on: docker + container: + image: golang:1.26-bookworm + steps: + - uses: https://code.forgejo.org/actions/checkout@v4 + - run: go vet ./... + - run: go test -race ./... + - run: go run honnef.co/go/tools/cmd/staticcheck@latest ./...
@@ -0,0 +1,3 @@+# Fixtures test line endings, BOMs and invalid bytes: keep them byte for byte. +internal/engine/testdata/** -text +*.go text eol=lf
@@ -0,0 +1,18 @@+# build +/patchwork +/patchwork.exe +/dist/ + +# local databases and backups +*.db +*.db-wal +*.db-shm + +# local config (the committed example is patchwork.example.yaml) +/patchwork.yaml + +# os / editor +.DS_Store +Thumbs.db +.vscode/ +.idea/
A
LICENSE
@@ -0,0 +1,21 @@+MIT License + +Copyright (c) 2026 Pablo Murad + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE.
@@ -0,0 +1,39 @@+# Patchwork + +Your RSS feed tells readers what you published. Patchwork tells them what you changed. + +Patchwork keeps the version history of the pages you revise and publishes each revision you choose to announce as a feed entry. An entry carries your note, the kind of change, how grown the page is, and a readable diff of the passages that changed. It's meant for digital gardens, living notes and documentation: pages that keep growing after their first publication. + +> Status: early development. The core (versions, diffs, publish rules) is being built; feeds and the HTTP API come next. Nothing here is stable yet. + +## How it works + +Patchwork watches a folder of Markdown files. When a file changes, it stores a snapshot in SQLite. A snapshot becomes a public revision only when you say so, by adding a note to the frontmatter: + +```yaml +--- +title: Remote work +revision: "Two years in, I changed my mind about the hybrid model." +revision_kind: position # correction | expansion | position | reorganization +maturity: sprout # seed | sprout | bloom +--- +``` + +Edits without a new note are kept as history and never reach the feed, so fixing a typo makes no noise. Several quiet edits followed by one with a note show up as a single revision, diffed against the last one you published. + +Patchwork then serves (or exports as files): + +- an Atom feed and a JSON Feed of revisions, for the whole site and for each page; +- a JSON API with each page's history, every published version, and structured diffs your site can render in its own style. + +The first site using it is [moonflowersociety.org](https://moonflowersociety.org), a nocturnal garden of ideas. + +## Install + +```sh +go install git.pablomurad.com/pablo/patchwork/cmd/patchwork@latest +``` + +## License + +MIT. See [LICENSE](LICENSE).
@@ -0,0 +1,117 @@+package main + +import ( + "context" + "fmt" + "os" + "sort" + "strings" + "unicode/utf8" + + "git.pablomurad.com/pablo/patchwork/internal/config" + "git.pablomurad.com/pablo/patchwork/internal/doc" + "git.pablomurad.com/pablo/patchwork/internal/slug" + "git.pablomurad.com/pablo/patchwork/internal/source/folder" + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +// cmdDoctor checks everything it can without changing anything. It keeps +// going after a problem so one run shows them all. +func cmdDoctor(ctx context.Context, args []string) error { + fs := newFlags("doctor", "") + cfgPath := configFlag(fs) + if err := fs.Parse(args); err != nil { + return err + } + problems := 0 + ok := func(format string, a ...any) { fmt.Fprintf(stdout, "ok "+format+"\n", a...) } + warn := func(format string, a ...any) { fmt.Fprintf(stdout, "warn "+format+"\n", a...) } + bad := func(format string, a ...any) { problems++; fmt.Fprintf(stdout, "FAIL "+format+"\n", a...) } + + cfg, err := config.Load(*cfgPath) + if err != nil { + bad("config: %v", err) + return errProblems + } + ok("config %s", *cfgPath) + if cfg.Site.BaseURL == "" { + warn("site.base_url is empty; feeds need the public address of the site") + } + if cfg.Site.TagAuthority == "" { + warn("site.tag_authority is empty; feed entry ids need it, and it must never change once feeds are public") + } + + if strings.HasPrefix(cfg.DB, `\\`) || strings.HasPrefix(cfg.DB, "//") { + warn("db %s is on a network share; SQLite locking is not safe there", cfg.DB) + } + st, err := store.Open(cfg.DB) + if err != nil { + bad("database: %v", err) + } else { + defer st.Close() + pages, err := st.Pages(ctx, "", false) + if err != nil { + bad("database: %v", err) + } else { + ok("database %s (%d active pages)", cfg.DB, len(pages)) + } + } + + m := folder.Matcher{Include: cfg.Source.Include, Exclude: cfg.Source.Exclude} + files, err := folder.List(cfg.Source.Path, m) + if err != nil { + bad("source: %v", err) + return errProblems + } + if len(files) == 0 { + warn("source %s has no files matching %v", cfg.Source.Path, cfg.Source.Include) + } else { + ok("source %s (%d files)", cfg.Source.Path, len(files)) + } + + rule, _ := slug.Lookup(cfg.Source.Slug) + opt := cfg.DocOptions() + bySlug := map[string][]string{} + for _, f := range files { + raw, err := os.ReadFile(f.Abs) + if err != nil { + bad("%s: %v", f.Rel, err) + continue + } + s := rule(f.Rel) + bySlug[s] = append(bySlug[s], f.Rel) + if s == "" { + bad("%s: the file name produces an empty slug, so the page has no URL; rename it", f.Rel) + } + if !utf8.Valid(raw) { + bad("%s: not valid UTF-8", f.Rel) + } + d := doc.Parse(raw, opt) + if d.FrontmatterErr != nil { + bad("%s: frontmatter is not valid YAML: %v", f.Rel, d.FrontmatterErr) + } + if !doc.Valid(d.Kind, doc.Kinds) { + bad("%s: %s %q should be one of %s", f.Rel, opt.KindKey, d.Kind, strings.Join(doc.Kinds, ", ")) + } + if !doc.Valid(d.Maturity, doc.Maturities) { + bad("%s: %s %q should be one of %s", f.Rel, opt.MaturityKey, d.Maturity, strings.Join(doc.Maturities, ", ")) + } + } + var slugs []string + for s, paths := range bySlug { + if s != "" && len(paths) > 1 { + slugs = append(slugs, s) + } + } + sort.Strings(slugs) + for _, s := range slugs { + bad("slug %q is shared by %s; only the first one seen gets the URL", s, strings.Join(bySlug[s], ", ")) + } + + if problems > 0 { + fmt.Fprintf(stdout, "\n%d problem(s)\n", problems) + return errProblems + } + fmt.Fprintln(stdout, "\nno problems found") + return nil +}
@@ -0,0 +1,256 @@+package main + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "os" + "strconv" + "strings" + + "git.pablomurad.com/pablo/patchwork/internal/diff" + "git.pablomurad.com/pablo/patchwork/internal/doc" + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +// findPage accepts a slug or a path relative to the source folder. +func (a *app) findPage(ctx context.Context, ref string) (*store.Page, error) { + p, err := a.store.PageBySlug(ctx, ref) + if errors.Is(err, store.ErrNotFound) { + p, err = a.store.PageByPath(ctx, a.engine.Source, strings.TrimPrefix(ref, "./")) + } + if errors.Is(err, store.ErrNotFound) { + return nil, fmt.Errorf("no page with slug or path %q", ref) + } + return p, err +} + +func cmdLog(ctx context.Context, args []string) error { + fs := newFlags("log", "<slug|path>") + cfgPath := configFlag(fs) + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() != 1 { + fs.Usage() + return errProblems + } + a, err := openApp(*cfgPath) + if err != nil { + return err + } + defer a.Close() + + p, err := a.findPage(ctx, fs.Arg(0)) + if err != nil { + return err + } + revs, err := a.store.Revisions(ctx, p.ID) + if err != nil { + return err + } + snaps, err := a.store.SnapshotCount(ctx, p.ID) + if err != nil { + return err + } + last, err := a.store.LastSnapshot(ctx, p.ID) + if err != nil { + return err + } + + fmt.Fprintf(stdout, "%s (%s)\n", orDash(last.Title), p.Path) + fmt.Fprintf(stdout, "slug %s · first published %s · maturity %s\n", orDash(p.Slug), p.OriginalDate, orDash(last.Maturity)) + fmt.Fprintf(stdout, "%d public versions, %d snapshots", len(revs), snaps) + if p.RemovedAt != nil { + fmt.Fprintf(stdout, " · removed %s", shortTime(*p.RemovedAt)) + } + fmt.Fprintln(stdout) + fmt.Fprintln(stdout) + for i := len(revs) - 1; i >= 0; i-- { + r := revs[i] + if r.Baseline() { + fmt.Fprintf(stdout, "#1 %s baseline\n", shortTime(r.PublishedAt)) + continue + } + line := fmt.Sprintf("#%d %s %-14s +%d -%d %s", r.Number, shortTime(r.PublishedAt), orDash(r.Kind), r.WordsAdded, r.WordsRemoved, r.Note) + if r.RetractedAt != nil { + line += " (retracted)" + } else if !r.UpdatedAt.Equal(r.PublishedAt) { + line += fmt.Sprintf(" (updated %s)", shortTime(r.UpdatedAt)) + } + fmt.Fprintln(stdout, line) + } + return nil +} + +func cmdDiff(ctx context.Context, args []string) error { + fs := newFlags("diff", "<slug|path> [n | a b]") + cfgPath := configFlag(fs) + plain := fs.Bool("plain", false, "no colors") + if err := fs.Parse(args); err != nil { + return err + } + if fs.NArg() < 1 || fs.NArg() > 3 { + fs.Usage() + return errProblems + } + a, err := openApp(*cfgPath) + if err != nil { + return err + } + defer a.Close() + + p, err := a.findPage(ctx, fs.Arg(0)) + if err != nil { + return err + } + nums := make([]int, 0, 2) + for _, s := range fs.Args()[1:] { + n, err := strconv.Atoi(strings.TrimPrefix(s, "#")) + if err != nil { + return fmt.Errorf("version %q is not a number", s) + } + nums = append(nums, n) + } + + var to *store.Revision + if len(nums) == 0 { + to, err = a.store.LastRevision(ctx, p.ID) + } else { + to, err = a.store.RevisionByNumber(ctx, p.ID, nums[len(nums)-1]) + } + if err != nil { + return fmt.Errorf("version: %w", err) + } + var from *store.Revision + if len(nums) == 2 { + from, err = a.store.RevisionByNumber(ctx, p.ID, nums[0]) + } else { + from, err = a.store.RevisionBefore(ctx, p.ID, to.Number) + } + if errors.Is(err, store.ErrNotFound) { + fmt.Fprintf(stdout, "#%d is the first version; nothing to compare it with\n", to.Number) + return nil + } + if err != nil { + return err + } + + var full diff.Full + cached := len(nums) < 2 && to.DiffJSON != "" && to.DiffVersion == diff.Version + if cached { + err = json.Unmarshal([]byte(to.DiffJSON), &full) + } else { + full, err = a.compare(ctx, from, to) + } + if err != nil { + return err + } + + color := !*plain && os.Getenv("NO_COLOR") == "" && isTerminal(os.Stdout) + fmt.Fprintf(stdout, "#%d → #%d", from.Number, to.Number) + if to.Note != "" && !to.Baseline() { + fmt.Fprintf(stdout, " %q", to.Note) + } + fmt.Fprintf(stdout, " (+%d -%d words)\n\n", full.Stats.WordsAdded, full.Stats.WordsRemoved) + printDiff(full, color) + return nil +} + +func (a *app) compare(ctx context.Context, from, to *store.Revision) (diff.Full, error) { + fs, err := a.store.Snapshot(ctx, from.SnapshotID) + if err != nil { + return diff.Full{}, err + } + ts, err := a.store.Snapshot(ctx, to.SnapshotID) + if err != nil { + return diff.Full{}, err + } + opt := a.cfg.DocOptions() + return diff.Compare(doc.Parse([]byte(fs.Raw), opt), doc.Parse([]byte(ts.Raw), opt), opt.DiffKeys), nil +} + +const ( + ansiRed = "\x1b[31m" + ansiGreen = "\x1b[32m" + ansiDim = "\x1b[2m" + ansiReset = "\x1b[0m" +) + +func printDiff(full diff.Full, color bool) { + paint := func(code, s string) string { + if !color || s == "" { + return s + } + return code + s + ansiReset + } + for _, m := range full.Meta { + fmt.Fprintf(stdout, "%s: %s → %s\n", m.Key, paint(ansiRed, metaValue(m.Old)), paint(ansiGreen, metaValue(m.New))) + } + if len(full.Meta) > 0 { + fmt.Fprintln(stdout) + } + ops, _ := diff.Excerpt(full.Ops, 1, len(full.Ops)) + for _, op := range ops { + switch op.Op { + case diff.OpGap: + fmt.Fprintln(stdout, paint(ansiDim, fmt.Sprintf(" … %d unchanged", op.Count))) + case diff.OpEq: + fmt.Fprintln(stdout, paint(ansiDim, marked(" ", op.Text))) + case diff.OpInsert: + fmt.Fprintln(stdout, paint(ansiGreen, marked("+ ", op.Text))) + case diff.OpDelete: + fmt.Fprintln(stdout, paint(ansiRed, marked("- ", op.Text))) + case diff.OpMove: + fmt.Fprintf(stdout, "↕ (moved from block %d)\n%s\n", *op.From+1, marked(" ", op.Text)) + case diff.OpModify: + var b strings.Builder + b.WriteString("~ ") + for _, s := range op.Segments { + switch s.T { + case diff.SegDel: + if color { + b.WriteString(paint(ansiRed, s.S)) + } else { + b.WriteString("[-" + s.S + "-]") + } + case diff.SegIns: + if color { + b.WriteString(paint(ansiGreen, s.S)) + } else { + b.WriteString("{+" + s.S + "+}") + } + default: + b.WriteString(s.S) + } + } + fmt.Fprintln(stdout, strings.ReplaceAll(b.String(), "\n", "\n ")) + } + fmt.Fprintln(stdout) + } +} + +// marked prefixes a block's first line and indents the rest under it. +func marked(prefix, text string) string { + return prefix + strings.ReplaceAll(text, "\n", "\n"+strings.Repeat(" ", len(prefix))) +} + +func metaValue(v any) string { + if v == nil { + return "(none)" + } + return fmt.Sprint(v) +} + +func isTerminal(f *os.File) bool { + fi, err := f.Stat() + return err == nil && fi.Mode()&os.ModeCharDevice != 0 +} + +func orDash(s string) string { + if s == "" { + return "-" + } + return s +}
@@ -0,0 +1,154 @@+// Command patchwork publishes revision feeds for living pages. +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "log/slog" + "os" + "os/signal" + + "git.pablomurad.com/pablo/patchwork/internal/config" + "git.pablomurad.com/pablo/patchwork/internal/engine" + "git.pablomurad.com/pablo/patchwork/internal/source/folder" + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +// version is set at build time with -ldflags "-X main.version=...". +var version = "dev" + +const usage = `patchwork publishes revision feeds for living pages. + +Usage: + patchwork <command> [flags] + +Commands: + init write a commented patchwork.yaml + scan scan the content folder once and apply the publish rules + watch keep scanning as files change + log show a page's public history + diff show what changed in a revision + doctor check the config, the content and the database + version print the version + +Every command except init and version reads patchwork.yaml from the +current directory, or the file named by --config or PATCHWORK_CONFIG. +Run "patchwork <command> -h" for a command's flags. +` + +func main() { + if len(os.Args) < 2 { + fmt.Fprint(os.Stderr, usage) + os.Exit(2) + } + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt) + defer stop() + + cmd, args := os.Args[1], os.Args[2:] + var err error + switch cmd { + case "init": + err = cmdInit(args) + case "scan": + err = cmdScan(ctx, args) + case "watch": + err = cmdWatch(ctx, args) + case "log": + err = cmdLog(ctx, args) + case "diff": + err = cmdDiff(ctx, args) + case "doctor": + err = cmdDoctor(ctx, args) + case "version", "-v", "--version": + fmt.Println("patchwork", version) + case "help", "-h", "--help": + fmt.Print(usage) + default: + fmt.Fprintf(os.Stderr, "patchwork: unknown command %q\n\n%s", cmd, usage) + os.Exit(2) + } + if err != nil { + if errors.Is(err, flag.ErrHelp) { + os.Exit(2) + } + if errors.Is(err, errProblems) { + os.Exit(1) + } + fmt.Fprintln(os.Stderr, "patchwork:", err) + os.Exit(1) + } +} + +// errProblems makes a command exit 1 after it has printed its own report. +var errProblems = errors.New("problems found") + +func newFlags(name, args string) *flag.FlagSet { + fs := flag.NewFlagSet(name, flag.ContinueOnError) + fs.Usage = func() { + fmt.Fprintf(fs.Output(), "Usage: patchwork %s %s\n", name, args) + fs.PrintDefaults() + } + return fs +} + +func configFlag(fs *flag.FlagSet) *string { + def := os.Getenv("PATCHWORK_CONFIG") + if def == "" { + def = "patchwork.yaml" + } + return fs.String("config", def, "path to patchwork.yaml") +} + +// app is what most commands need: config, store and engine. +type app struct { + cfg *config.Config + store *store.Store + engine *engine.Engine + log *slog.Logger +} + +func openApp(configPath string) (*app, error) { + cfg, err := config.Load(configPath) + if err != nil { + return nil, err + } + st, err := store.Open(cfg.DB) + if err != nil { + return nil, err + } + e, err := engine.New(st, cfg) + if err != nil { + st.Close() + return nil, err + } + return &app{cfg: cfg, store: st, engine: e, log: slog.New(slog.NewTextHandler(os.Stderr, nil))}, nil +} + +func (a *app) Close() error { return a.store.Close() } + +func (a *app) runner() *folder.Runner { + m := folder.Matcher{Include: a.cfg.Source.Include, Exclude: a.cfg.Source.Exclude} + return folder.NewRunner(a.cfg.Source.Path, m, a.engine, a.log) +} + +func cmdInit(args []string) error { + fs := newFlags("init", "[-o patchwork.yaml] [-force]") + out := fs.String("o", "patchwork.yaml", "file to write") + force := fs.Bool("force", false, "overwrite an existing file") + if err := fs.Parse(args); err != nil { + return err + } + if _, err := os.Stat(*out); err == nil && !*force { + return fmt.Errorf("%s already exists (use -force to overwrite)", *out) + } + if err := os.WriteFile(*out, config.Example, 0o644); err != nil { + return err + } + fmt.Printf("wrote %s; set site.base_url and source.path, then run: patchwork doctor\n", *out) + return nil +} + +var stdout io.Writer = os.Stdout
@@ -0,0 +1,106 @@+package main + +import ( + "context" + "errors" + "fmt" + "os" + "time" + + "git.pablomurad.com/pablo/patchwork/internal/engine" + "git.pablomurad.com/pablo/patchwork/internal/source/folder" +) + +func cmdScan(ctx context.Context, args []string) error { + fs := newFlags("scan", "[-dry-run] [-all]") + cfgPath := configFlag(fs) + dry := fs.Bool("dry-run", false, "show what would happen without writing anything") + all := fs.Bool("all", false, "also list unchanged files") + if err := fs.Parse(args); err != nil { + return err + } + a, err := openApp(*cfgPath) + if err != nil { + return err + } + defer a.Close() + + r := a.runner() + // A one-off scan reads files right away; waiting for an upload to + // settle is the watcher's job. + r.Settle = 0 + rep, err := r.Scan(ctx, *dry) + if err != nil { + return err + } + printReport(rep, *all) + if *dry { + fmt.Fprintln(stdout, "(dry run: nothing was written)") + } + if len(rep.Errors) > 0 { + return errProblems + } + return nil +} + +func cmdWatch(ctx context.Context, args []string) error { + fs := newFlags("watch", "[-poll-only]") + cfgPath := configFlag(fs) + pollOnly := fs.Bool("poll-only", false, "don't use file events (for network or bind mounts where they never arrive)") + if err := fs.Parse(args); err != nil { + return err + } + a, err := openApp(*cfgPath) + if err != nil { + return err + } + defer a.Close() + + r := a.runner() + r.PollOnly = *pollOnly + a.log.Info("watching", "source", a.cfg.Source.Path, "db", a.cfg.DB) + err = r.Watch(ctx, a.cfg.Watch.Debounce.Duration, a.cfg.Watch.Poll.Duration, func(rep folder.Report, err error) { + if err != nil { + a.log.Error("scan failed", "err", err) + return + } + printReport(rep, false) + }) + if errors.Is(err, context.Canceled) { + return nil + } + return err +} + +func printReport(rep folder.Report, all bool) { + for _, res := range rep.Results { + if res.Action == engine.Unchanged && !res.Restored && !all { + continue + } + fmt.Fprintln(stdout, formatResult(res)) + for _, w := range res.Warnings { + fmt.Fprintf(os.Stderr, " warning: %s: %s\n", res.Path, w) + } + } + for _, p := range rep.Pending { + fmt.Fprintf(stdout, "%-10s %s (still being written; will retry)\n", "pending", p) + } + for _, err := range rep.Errors { + fmt.Fprintf(os.Stderr, "error: %v\n", err) + } +} + +func formatResult(res engine.Result) string { + action := string(res.Action) + if res.Restored { + action = "restored" + } + s := fmt.Sprintf("%-10s %s", action, res.Path) + switch res.Action { + case engine.Published, engine.Amended, engine.Folded: + s += fmt.Sprintf(" #%d %q", res.Number, res.Note) + } + return s +} + +func shortTime(t time.Time) string { return t.Local().Format("2006-01-02 15:04") }
A
go.mod
@@ -0,0 +1,20 @@+module git.pablomurad.com/pablo/patchwork + +go 1.26.4 + +require ( + github.com/dustin/go-humanize v1.1.0 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.24 // indirect + github.com/ncruces/go-strftime v1.1.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/yuin/goldmark v1.8.6 // indirect + golang.org/x/sys v0.49.0 // indirect + golang.org/x/text v0.43.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + modernc.org/libc v1.79.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.12.1 // indirect + modernc.org/sqlite v1.61.0 // indirect +)
A
go.sum
@@ -0,0 +1,29 @@+github.com/dustin/go-humanize v1.1.0 h1:dbKTrvD0klcbBV/h4AWJdMuZogJACoMlvWIWZ5b2xWg= +github.com/dustin/go-humanize v1.1.0/go.mod h1:hc1CvRkJMsgxqjmjMQF3QNRAZBwY8AXBAzKYoSX9sFI= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= +github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= +github.com/ncruces/go-strftime v1.1.0 h1:vtPmPWMkb539RGV9WpWT9EwJGvrOCKvMNI4obWL47Hk= +github.com/ncruces/go-strftime v1.1.0/go.mod h1:yVaq7iwCN09vHRrKBrrflXT0M9tTMG2KmZ9Fj7xeG/g= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/yuin/goldmark v1.8.6 h1:d0VcaP1sx9GkFVkoW+KtggpGi2KZ965i14b0+bDQST4= +github.com/yuin/goldmark v1.8.6/go.mod h1:ip/1k0VRfGynBgxOz0yCqHrbZXhcjxyuS66Brc7iBKg= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= +golang.org/x/text v0.43.0 h1:1QivrlhwAsnMOcqgOdl9my7s9OyZl8O5/UGUTJVXoWc= +golang.org/x/text v0.43.0/go.mod h1:hbSIYA/amXcRXTFSZqh8tWHEoqRvvbKPJ2uz/KU+a+s= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/libc v1.79.0 h1:9Dvvv++bAMSkSuBzuOFAatbTV5OIoq6Y6FohmD3e7q8= +modernc.org/libc v1.79.0/go.mod h1:uT3kR61NsbeKJSLqq83GJwSnZMVLlSjyAz4DfTaBy1M= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g= +modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.61.0 h1:sQi63oLQ9uquAUA4CzOisH35Q3pEszkPBuHM0tFL8CY= +modernc.org/sqlite v1.61.0/go.mod h1:f4gr5tbpx2umklaT/QVtbi/50hNdM+4ng+2zMo8KXJ0=
@@ -0,0 +1,257 @@+// Package config loads patchwork.yaml. +package config + +import ( + _ "embed" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + // Distroless images and some Windows machines ship no zoneinfo. + _ "time/tzdata" + + "gopkg.in/yaml.v3" + + "git.pablomurad.com/pablo/patchwork/internal/doc" + "git.pablomurad.com/pablo/patchwork/internal/slug" +) + +// Example is the commented config written by `patchwork init`. +// +//go:embed example.yaml +var Example []byte + +// Config is a parsed patchwork.yaml. +type Config struct { + Site Site `yaml:"site"` + Source Source `yaml:"source"` + Frontmatter Frontmatter `yaml:"frontmatter"` + Publish Publish `yaml:"publish"` + Watch Watch `yaml:"watch"` + DB string `yaml:"db"` + HTTP HTTP `yaml:"http"` + Feeds Feeds `yaml:"feeds"` + + // Dir is the directory relative paths in the file resolve against. + Dir string `yaml:"-"` +} + +type Site struct { + Title string `yaml:"title"` + BaseURL string `yaml:"base_url"` + Timezone string `yaml:"timezone"` + Author string `yaml:"author"` + // TagAuthority is the "domain,year" part of tag: URIs used as feed + // entry ids. It must never change once feeds are public. + TagAuthority string `yaml:"tag_authority"` + PageURL string `yaml:"page_url"` + VersionURL string `yaml:"version_url"` + HistoryURL string `yaml:"history_url"` + Feeds SiteFeeds `yaml:"feeds"` +} + +// SiteFeeds are the public paths where the host serves Patchwork's feeds. +// Feeds must point at themselves with their public URL, not at Patchwork's +// internal address. +type SiteFeeds struct { + Atom string `yaml:"atom"` + JSON string `yaml:"json"` + PageAtom string `yaml:"page_atom"` + PageJSON string `yaml:"page_json"` +} + +type Source struct { + Mode string `yaml:"mode"` + Path string `yaml:"path"` + Include []string `yaml:"include"` + Exclude []string `yaml:"exclude"` + Slug string `yaml:"slug"` +} + +type Frontmatter struct { + NoteKey string `yaml:"note_key"` + KindKey string `yaml:"kind_key"` + MaturityKey string `yaml:"maturity_key"` + DateKey string `yaml:"date_key"` + DiffKeys []string `yaml:"diff_keys"` +} + +type Publish struct { + FoldWindow Duration `yaml:"fold_window"` +} + +type Watch struct { + Debounce Duration `yaml:"debounce"` + Poll Duration `yaml:"poll"` +} + +type HTTP struct { + Addr string `yaml:"addr"` +} + +type Feeds struct { + Limit int `yaml:"limit"` + ExcerptBlocks int `yaml:"excerpt_blocks"` +} + +// Duration reads Go duration strings such as "20m" or "3s". +type Duration struct{ time.Duration } + +func (d *Duration) UnmarshalYAML(n *yaml.Node) error { + var s string + if err := n.Decode(&s); err != nil { + return err + } + v, err := time.ParseDuration(s) + if err != nil { + return fmt.Errorf("line %d: %w", n.Line, err) + } + d.Duration = v + return nil +} + +func (d Duration) MarshalYAML() (any, error) { return d.String(), nil } + +// Load reads a config file, fills defaults, applies PATCHWORK_* environment +// overrides and validates the result. +func Load(path string) (*Config, error) { + raw, err := os.ReadFile(path) + if err != nil { + return nil, err + } + c := Default() + dec := yaml.NewDecoder(strings.NewReader(string(raw))) + dec.KnownFields(true) + // io.EOF means an empty file: all defaults. + if err := dec.Decode(c); err != nil && !errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s: %w", path, err) + } + abs, err := filepath.Abs(path) + if err != nil { + return nil, err + } + c.Dir = filepath.Dir(abs) + c.applyEnv() + c.resolvePaths() + if err := c.Validate(); err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + return c, nil +} + +// Default returns the built-in configuration. +func Default() *Config { + o := doc.DefaultOptions() + return &Config{ + Site: Site{ + Timezone: "UTC", + PageURL: "/{slug}", + VersionURL: "/{slug}/v/{n}", + HistoryURL: "/{slug}/history", + Feeds: SiteFeeds{ + Atom: "/revisions.atom", + JSON: "/revisions.json", + PageAtom: "/{slug}/revisions.atom", + PageJSON: "/{slug}/revisions.json", + }, + }, + Source: Source{ + Mode: "folder", + Path: ".", + Include: []string{"**/*.md", "**/*.markdown"}, + Exclude: []string{".*", "*~", "*.tmp", "*.filepart", "*.part", "*.swp"}, + Slug: "path", + }, + Frontmatter: Frontmatter{ + NoteKey: o.NoteKey, KindKey: o.KindKey, MaturityKey: o.MaturityKey, + DateKey: "date", DiffKeys: o.DiffKeys, + }, + Publish: Publish{FoldWindow: Duration{20 * time.Minute}}, + Watch: Watch{Debounce: Duration{3 * time.Second}, Poll: Duration{60 * time.Second}}, + DB: defaultDB(), + HTTP: HTTP{Addr: "127.0.0.1:8080"}, + Feeds: Feeds{Limit: 50, ExcerptBlocks: 6}, + } +} + +// defaultDB keeps the database on a local disk: SQLite's locking is not +// safe on network shares. +func defaultDB() string { + if dir, err := os.UserCacheDir(); err == nil { + return filepath.Join(dir, "patchwork", "patchwork.db") + } + return "patchwork.db" +} + +func (c *Config) applyEnv() { + set := func(dst *string, key string) { + if v, ok := os.LookupEnv(key); ok { + *dst = v + } + } + set(&c.DB, "PATCHWORK_DB") + set(&c.Source.Path, "PATCHWORK_SOURCE_PATH") + set(&c.HTTP.Addr, "PATCHWORK_HTTP_ADDR") + set(&c.Site.BaseURL, "PATCHWORK_BASE_URL") +} + +func (c *Config) resolvePaths() { + if !filepath.IsAbs(c.Source.Path) { + c.Source.Path = filepath.Join(c.Dir, c.Source.Path) + } + if !filepath.IsAbs(c.DB) { + c.DB = filepath.Join(c.Dir, c.DB) + } +} + +// Validate checks values that would otherwise fail later or silently. +func (c *Config) Validate() error { + var errs []error + if c.Source.Mode != "folder" { + errs = append(errs, fmt.Errorf("source.mode %q is not supported yet (want folder)", c.Source.Mode)) + } + if _, err := slug.Lookup(c.Source.Slug); err != nil { + errs = append(errs, fmt.Errorf("source.slug: %w", err)) + } + if _, err := time.LoadLocation(c.Site.Timezone); err != nil { + errs = append(errs, fmt.Errorf("site.timezone: %w", err)) + } + if c.Site.BaseURL != "" && !strings.HasPrefix(c.Site.BaseURL, "http://") && !strings.HasPrefix(c.Site.BaseURL, "https://") { + errs = append(errs, fmt.Errorf("site.base_url must start with http:// or https://")) + } + if c.Frontmatter.NoteKey == "" { + errs = append(errs, errors.New("frontmatter.note_key is empty")) + } + for _, p := range append(append([]string{}, c.Source.Include...), c.Source.Exclude...) { + if _, err := filepath.Match(strings.TrimPrefix(p, "**/"), "x"); err != nil { + errs = append(errs, fmt.Errorf("bad pattern %q: %w", p, err)) + } + } + if c.Publish.FoldWindow.Duration < 0 || c.Watch.Debounce.Duration < 0 || c.Watch.Poll.Duration < 0 { + errs = append(errs, errors.New("durations must not be negative")) + } + if c.Feeds.Limit <= 0 || c.Feeds.ExcerptBlocks <= 0 { + errs = append(errs, errors.New("feeds.limit and feeds.excerpt_blocks must be positive")) + } + return errors.Join(errs...) +} + +// DocOptions are the frontmatter keys in the form the doc package takes. +func (c *Config) DocOptions() doc.Options { + return doc.Options{ + NoteKey: c.Frontmatter.NoteKey, KindKey: c.Frontmatter.KindKey, + MaturityKey: c.Frontmatter.MaturityKey, DiffKeys: c.Frontmatter.DiffKeys, + } +} + +// Location is the site's time zone. Validate has already checked it. +func (c *Config) Location() *time.Location { + loc, err := time.LoadLocation(c.Site.Timezone) + if err != nil { + return time.UTC + } + return loc +}
@@ -0,0 +1,82 @@+package config + +import ( + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func write(t *testing.T, body string) string { + t.Helper() + p := filepath.Join(t.TempDir(), "patchwork.yaml") + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func TestExampleLoads(t *testing.T) { + c, err := Load(write(t, string(Example))) + if err != nil { + t.Fatal(err) + } + if c.Publish.FoldWindow.Duration != 20*time.Minute || c.Source.Slug != "path" || c.Feeds.ExcerptBlocks != 6 { + t.Fatalf("got %+v", c) + } + if !filepath.IsAbs(c.Source.Path) || filepath.Base(c.Source.Path) != "content" { + t.Fatalf("source path not resolved: %s", c.Source.Path) + } +} + +func TestEmptyFileIsDefaults(t *testing.T) { + c, err := Load(write(t, "")) + if err != nil { + t.Fatal(err) + } + if c.Frontmatter.NoteKey != "revision" || c.Watch.Poll.Duration != time.Minute { + t.Fatalf("got %+v", c) + } +} + +func TestEnvOverrides(t *testing.T) { + t.Setenv("PATCHWORK_DB", "/tmp/override.db") + t.Setenv("PATCHWORK_BASE_URL", "https://moonflowersociety.org") + c, err := Load(write(t, "site:\n base_url: https://example.com\n")) + if err != nil { + t.Fatal(err) + } + if filepath.ToSlash(c.DB) != "/tmp/override.db" && !strings.HasSuffix(filepath.ToSlash(c.DB), "/tmp/override.db") { + t.Fatalf("db %s", c.DB) + } + if c.Site.BaseURL != "https://moonflowersociety.org" { + t.Fatalf("base url %s", c.Site.BaseURL) + } +} + +func TestValidation(t *testing.T) { + cases := map[string]string{ + "unknown key": "sorce:\n path: x\n", + "bad slug": "source:\n slug: weird\n", + "bad timezone": "site:\n timezone: Mars/Olympus\n", + "bad duration": "publish:\n fold_window: soon\n", + "git mode": "source:\n mode: git\n", + "bad base url": "site:\n base_url: example.com\n", + } + for name, body := range cases { + if _, err := Load(write(t, body)); err == nil { + t.Errorf("%s: want error", name) + } + } +} + +func TestTimezoneWithoutSystemZoneinfo(t *testing.T) { + c, err := Load(write(t, "site:\n timezone: America/Sao_Paulo\n")) + if err != nil { + t.Fatal(err) + } + if c.Location().String() != "America/Sao_Paulo" { + t.Fatal(c.Location()) + } +}
@@ -0,0 +1,62 @@+# patchwork.yaml +# Relative paths resolve against this file's directory. + +site: + title: My Garden + # Public address of the site. Feed links and ids are built from it. + base_url: https://example.com + timezone: UTC # calendar dates are computed in this zone + author: Your Name + # The "domain,year" part of feed entry ids (tag: URIs). Pick it once: + # changing it later makes every feed reader see all entries as new. + tag_authority: "example.com,2026" + # Where the host site shows a page, one version, and the history. + page_url: "/{slug}" + version_url: "/{slug}/v/{n}" + history_url: "/{slug}/history" + # Public paths where the host serves Patchwork's feeds. + feeds: + atom: /revisions.atom + json: /revisions.json + page_atom: "/{slug}/revisions.atom" + page_json: "/{slug}/revisions.json" + +source: + mode: folder + path: ./content + # "*.md" matches the top level only; "**/*.md" matches at any depth. + include: ["**/*.md", "**/*.markdown"] + # Matched against the file name: hidden files, editor and upload leftovers. + exclude: [".*", "*~", "*.tmp", "*.filepart", "*.part", "*.swp"] + # path: slugify every directory and the file name ("notes/my-note"). + # datestrip-slugify: file name only, minus a leading YYYY-MM-DD date. + slug: path + +frontmatter: + note_key: revision # a new note here publishes a revision + kind_key: revision_kind # correction | expansion | position | reorganization + maturity_key: maturity # seed | sprout | bloom + date_key: date # original publication date, if present + # Frontmatter keys readers see. Changing them counts as a content change; + # every other key is ignored. + diff_keys: [title, summary, tags] + +publish: + # An upload with no new note this soon after a publish is folded into + # that revision (the typo you spot right after uploading). 0 disables it. + fold_window: 20m + +watch: + debounce: 3s # wait for a burst of file events to settle + poll: 60s # full rescan, in case file events are missed + +# Keep the database on a local disk. Network shares break SQLite locking. +# Omit to use the user cache directory. +# db: /var/lib/patchwork/patchwork.db + +http: + addr: 127.0.0.1:8080 + +feeds: + limit: 50 # entries per feed + excerpt_blocks: 6 # changed blocks shown per entry
@@ -0,0 +1,283 @@+package diff + +import ( + "strings" + + "git.pablomurad.com/pablo/patchwork/internal/doc" +) + +// Op kinds of a block diff. +const ( + OpEq = "eq" + OpInsert = "insert" + OpDelete = "delete" + OpModify = "modify" + OpMove = "move" + OpGap = "gap" +) + +// Op is one entry of a block diff, in reading order of the new version. +// Deleted blocks sit where they used to be; a moved block sits at its new +// place and records the old one. +type Op struct { + Op string `json:"op"` + Kind string `json:"kind,omitempty"` + Text string `json:"text,omitempty"` + Segments []Segment `json:"segments,omitempty"` + // Replaced marks a modify so changed that word highlights would be + // noise; its segments are the whole old text and the whole new text. + Replaced bool `json:"replaced,omitempty"` + From *int `json:"from,omitempty"` + To *int `json:"to,omitempty"` + Count int `json:"count,omitempty"` +} + +// Stats summarize a diff. +type Stats struct { + WordsAdded int `json:"words_added"` + WordsRemoved int `json:"words_removed"` + BlocksChanged int `json:"blocks_changed"` +} + +// Result is a full block diff: every block of the new version appears, with +// unchanged ones as eq ops. +type Result struct { + Ops []Op `json:"ops"` + Stats Stats `json:"stats"` +} + +// Version identifies the diff algorithm and encoding. Bump it whenever the +// output for the same input changes; rebuild-diffs then refreshes the cache. +const Version = 1 + +// Full is what Patchwork caches for a revision: the block diff against the +// previous public version plus changes to visible frontmatter. +type Full struct { + Ops []Op `json:"ops"` + Stats Stats `json:"stats"` + Meta []MetaChange `json:"meta_changes,omitempty"` +} + +// Compare diffs two parsed versions of a document. +func Compare(old, new *doc.Doc, metaKeys []string) Full { + r := Blocks(old.Blocks, new.Blocks) + return Full{Ops: r.Ops, Stats: r.Stats, Meta: Meta(old, new, metaKeys)} +} + +// pairThreshold is the word-set similarity (Dice) above which a deleted and +// an inserted block are read as one edited block. +const pairThreshold = 0.5 + +type item struct { + kind EditKind + a, b int // block indexes; -1 when absent +} + +// Blocks diffs two block lists. +func Blocks(old, new []doc.Block) Result { + ha := make([]string, len(old)) + for i, b := range old { + ha[i] = b.Hash + } + hb := make([]string, len(new)) + for i, b := range new { + hb[i] = b.Hash + } + + var items []item + for _, e := range Myers(ha, hb) { + switch e.Kind { + case Eq: + for k := 0; k < e.A1-e.A0; k++ { + items = append(items, item{Eq, e.A0 + k, e.B0 + k}) + } + case Del: + for i := e.A0; i < e.A1; i++ { + items = append(items, item{Del, i, -1}) + } + case Ins: + for j := e.B0; j < e.B1; j++ { + items = append(items, item{Ins, -1, j}) + } + } + } + + // Moves first: an inserted block identical to a deleted one anywhere. + movedFrom := map[int]int{} // new index -> old index + moved := map[int]bool{} // old indexes consumed by a move + byHash := map[string][]int{} + for _, it := range items { + if it.kind == Del { + byHash[old[it.a].Hash] = append(byHash[old[it.a].Hash], it.a) + } + } + for _, it := range items { + if it.kind != Ins { + continue + } + if cands := byHash[new[it.b].Hash]; len(cands) > 0 { + movedFrom[it.b] = cands[0] + moved[cands[0]] = true + byHash[new[it.b].Hash] = cands[1:] + } + } + + var res Result + for i := 0; i < len(items); { + if items[i].kind == Eq { + res.Ops = append(res.Ops, Op{Op: OpEq, Kind: new[items[i].b].Kind, Text: new[items[i].b].Text}) + i++ + continue + } + j := i + for j < len(items) && items[j].kind != Eq { + j++ + } + res.Ops = append(res.Ops, hunk(old, new, items[i:j], movedFrom, moved, &res.Stats)...) + i = j + } + return res +} + +// hunk orders one run of deletions and insertions, pairing similar blocks +// into modify ops. +func hunk(old, new []doc.Block, items []item, movedFrom map[int]int, moved map[int]bool, st *Stats) []Op { + var dels, ins []int + for _, it := range items { + if it.kind == Del && !moved[it.a] { + dels = append(dels, it.a) + } + if it.kind == Ins { + ins = append(ins, it.b) + } + } + + var plainIns []int + for _, j := range ins { + if _, isMove := movedFrom[j]; !isMove { + plainIns = append(plainIns, j) + } + } + + pairOf := map[int]int{} // new index -> old index + if len(dels) == 1 && len(plainIns) == 1 { + // One block replaced by one block in the same place reads as an + // edit however much it changed; Words decides between highlights + // and a whole replacement. + pairOf[plainIns[0]] = dels[0] + } else { + // Greedy pairing that keeps order: each insertion takes the most + // similar later deletion. + next := 0 + for _, j := range plainIns { + best, bestScore := -1, pairThreshold + for k := next; k < len(dels); k++ { + if s := dice(old[dels[k]].Text, new[j].Text); s >= bestScore { + best, bestScore = k, s + } + } + if best >= 0 { + pairOf[j] = dels[best] + next = best + 1 + } + } + } + // nextPaired[j]: the old index of the first pairing at or after + // insertion j, so deletions print before the insertions that follow + // them, the way diffs are read. + nextPaired := map[int]int{} + upcoming := len(old) + for k := len(ins) - 1; k >= 0; k-- { + if d, ok := pairOf[ins[k]]; ok { + upcoming = d + } + nextPaired[ins[k]] = upcoming + } + + var out []Op + emitted := map[int]bool{} + flushDels := func(upTo int) { + for _, d := range dels { + if d >= upTo { + break + } + if paired(pairOf, d) || emitted[d] { + continue + } + emitted[d] = true + out = append(out, Op{Op: OpDelete, Kind: old[d].Kind, Text: old[d].Text}) + st.WordsRemoved += countWords(old[d].Text) + st.BlocksChanged++ + } + } + for _, j := range ins { + if from, isMove := movedFrom[j]; isMove { + f, t := from, j + out = append(out, Op{Op: OpMove, Kind: new[j].Kind, Text: new[j].Text, From: &f, To: &t}) + st.BlocksChanged++ + continue + } + if d, ok := pairOf[j]; ok { + flushDels(d) + segs, replaced, added, removed := Words(old[d].Text, new[j].Text) + out = append(out, Op{Op: OpModify, Kind: new[j].Kind, Segments: segs, Replaced: replaced}) + st.WordsAdded += added + st.WordsRemoved += removed + st.BlocksChanged++ + continue + } + flushDels(nextPaired[j]) + out = append(out, Op{Op: OpInsert, Kind: new[j].Kind, Text: new[j].Text}) + st.WordsAdded += countWords(new[j].Text) + st.BlocksChanged++ + } + flushDels(len(old)) + return out +} + +func paired(pairOf map[int]int, d int) bool { + for _, v := range pairOf { + if v == d { + return true + } + } + return false +} + +// dice is 2|A∩B| / (|A|+|B|) over the lowercase word sets of two texts. +func dice(a, b string) float64 { + sa, sb := wordSet(a), wordSet(b) + if len(sa)+len(sb) == 0 { + if a == b { + return 1 + } + return 0 + } + common := 0 + for w := range sa { + if sb[w] { + common++ + } + } + return 2 * float64(common) / float64(len(sa)+len(sb)) +} + +func wordSet(s string) map[string]bool { + set := map[string]bool{} + for _, t := range doc.Tokenize(s) { + if doc.IsWord(t) { + set[strings.ToLower(t)] = true + } + } + return set +} + +func countWords(s string) int { + n := 0 + for _, t := range doc.Tokenize(s) { + if doc.IsWord(t) { + n++ + } + } + return n +}
@@ -0,0 +1,204 @@+package diff + +import ( + "strings" + "testing" + + "git.pablomurad.com/pablo/patchwork/internal/doc" +) + +// apply rebuilds b from a and an edit script. +func apply[T comparable](a, b []T, edits []Edit) (gotA, gotB []T) { + for _, e := range edits { + switch e.Kind { + case Eq: + gotA = append(gotA, a[e.A0:e.A1]...) + gotB = append(gotB, a[e.A0:e.A1]...) + case Del: + gotA = append(gotA, a[e.A0:e.A1]...) + case Ins: + gotB = append(gotB, b[e.B0:e.B1]...) + } + } + return +} + +func TestMyersMinimal(t *testing.T) { + a := strings.Split("ABCABBA", "") + b := strings.Split("CBABAC", "") + edits := Myers(a, b) + ga, gb := apply(a, b, edits) + if strings.Join(ga, "") != "ABCABBA" || strings.Join(gb, "") != "CBABAC" { + t.Fatalf("bad script %+v", edits) + } + d := 0 + for _, e := range edits { + if e.Kind != Eq { + d += (e.A1 - e.A0) + (e.B1 - e.B0) + } + } + if d != 5 { + t.Fatalf("edit distance %d, want 5 (the classic Myers example)", d) + } +} + +func FuzzMyers(f *testing.F) { + f.Add("the quick brown fox", "the slow brown dog") + f.Add("", "abc") + f.Add("abc", "") + f.Fuzz(func(t *testing.T, x, y string) { + // Compare runes, not the input strings: []rune turns invalid UTF-8 + // into U+FFFD, which is a property of the conversion, not of Myers. + a, b := []rune(x), []rune(y) + ga, gb := apply(a, b, Myers(a, b)) + if string(ga) != string(a) || string(gb) != string(b) { + t.Fatalf("script does not rebuild inputs: %q %q", string(ga), string(gb)) + } + }) +} + +func rebuild(segs []Segment) (old, new string) { + var a, b strings.Builder + for _, s := range segs { + if s.T != SegIns { + a.WriteString(s.S) + } + if s.T != SegDel { + b.WriteString(s.S) + } + } + return a.String(), b.String() +} + +func FuzzWords(f *testing.F) { + f.Add("E se cada protótipo fosse cuidado", "E se cada rascunho fosse regado") + f.Add("one two three", "") + f.Fuzz(func(t *testing.T, x, y string) { + segs, _, _, _ := Words(x, y) + a, b := rebuild(segs) + if a != x || b != y { + t.Fatalf("segments do not rebuild: %q / %q", a, b) + } + }) +} + +func TestWordsHighlightsChangedWords(t *testing.T) { + segs, replaced, added, removed := Words( + "O modelo híbrido é o pior dos dois mundos para equipes pequenas.", + "O modelo híbrido funciona para equipes pequenas.") + if replaced { + t.Fatal("should not be a whole replacement") + } + want := []Segment{ + {SegEq, "O modelo híbrido "}, + {SegDel, "é o pior dos dois mundos"}, + {SegIns, "funciona"}, + {SegEq, " para equipes pequenas."}, + } + if len(segs) != len(want) { + t.Fatalf("got %q", segs) + } + for i := range want { + if segs[i] != want[i] { + t.Fatalf("segment %d = %q, want %q", i, segs[i], want[i]) + } + } + if added != 1 || removed != 6 { + t.Fatalf("added %d removed %d", added, removed) + } +} + +func TestWordsCleanupAbsorbsLonelySpaces(t *testing.T) { + segs, _, _, _ := Words("the cat sat on the mat today", "a dog sat on the mat today") + if segs[0].T != SegDel || segs[0].S != "the cat" || segs[1].S != "a dog" { + t.Fatalf("got %q", segs) + } +} + +func TestWordsReplacedWhenRewritten(t *testing.T) { + segs, replaced, _, _ := Words("Completely different sentence here.", "Nothing in common at all, really.") + if !replaced || len(segs) != 2 { + t.Fatalf("want replacement, got %v %q", replaced, segs) + } +} + +func blocks(src string) []doc.Block { return doc.SplitBlocks(src) } + +func opsString(ops []Op) string { + var out []string + for _, op := range ops { + out = append(out, op.Op) + } + return strings.Join(out, ",") +} + +func TestBlocks(t *testing.T) { + cases := []struct{ name, old, new, want string }{ + {"no change", "a\n\nb", "a\n\nb", "eq,eq"}, + {"modify", "Intro.\n\nThe hybrid model is the worst of both worlds.\n\nEnd.", + "Intro.\n\nThe hybrid model is the best of both worlds.\n\nEnd.", "eq,modify,eq"}, + {"insert list item", "- one\n- two", "- one\n- one and a half\n- two", "eq,insert,eq"}, + {"delete", "keep\n\ndrop this paragraph\n\nkeep too", "keep\n\nkeep too", "eq,delete,eq"}, + {"move", "First paragraph here.\n\nSecond paragraph here.\n\nThird one.", + "Third one.\n\nFirst paragraph here.\n\nSecond paragraph here.", "move,eq,eq"}, + {"one for one replace is an edit", "Alpha beta gamma.", "Totally new words now.", "modify"}, + {"unrelated two for two", "Alpha beta gamma.\n\nDelta epsilon.", "Totally new words.\n\nNothing shared.", "delete,delete,insert,insert"}, + {"insert and unrelated delete", "keep\n\nold unrelated text\n\nend", "keep\n\nbrand new paragraph\n\nanother new one\n\nend", "eq,delete,insert,insert,eq"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + res := Blocks(blocks(c.old), blocks(c.new)) + if got := opsString(res.Ops); got != c.want { + t.Fatalf("got %s, want %s", got, c.want) + } + }) + } +} + +func TestBlocksMoveRecordsPositions(t *testing.T) { + res := Blocks(blocks("A one.\n\nB two.\n\nC three."), blocks("C three.\n\nA one.\n\nB two.")) + mv := res.Ops[0] + if mv.Op != OpMove || *mv.From != 2 || *mv.To != 0 { + t.Fatalf("got %+v", mv) + } + if res.Stats.BlocksChanged != 1 || res.Stats.WordsAdded != 0 { + t.Fatalf("stats %+v", res.Stats) + } +} + +func TestBlocksPairsSimilarInOrder(t *testing.T) { + old := "First idea about gardens at night.\n\nSecond idea about wind and grass." + new := "First idea about gardens at dusk.\n\nSecond idea about wind and tall grass." + res := Blocks(blocks(old), blocks(new)) + if opsString(res.Ops) != "modify,modify" { + t.Fatalf("got %s", opsString(res.Ops)) + } + if res.Stats.WordsAdded != 2 || res.Stats.WordsRemoved != 1 { + t.Fatalf("stats %+v", res.Stats) + } +} + +func TestExcerpt(t *testing.T) { + ops := []Op{{Op: OpEq}, {Op: OpEq}, {Op: OpEq}, {Op: OpModify}, {Op: OpEq}, {Op: OpEq}, {Op: OpEq}, {Op: OpInsert}, {Op: OpEq}} + got, cut := Excerpt(ops, 1, 6) + if cut || opsString(got) != "gap,eq,modify,eq,gap,eq,insert,eq" { + t.Fatalf("got %s cut=%v", opsString(got), cut) + } + if got[0].Count != 2 || got[4].Count != 1 { + t.Fatalf("gap counts %d %d", got[0].Count, got[4].Count) + } + got, cut = Excerpt(ops, 0, 1) + if !cut || opsString(got) != "gap,modify" { + t.Fatalf("got %s cut=%v", opsString(got), cut) + } +} + +func TestMeta(t *testing.T) { + opt := doc.DefaultOptions() + a := doc.Parse([]byte("---\ntitle: Old\ntags: [a, b]\nmaturity: seed\n---\nx"), opt) + b := doc.Parse([]byte("---\ntitle: New\ntags: [a, b]\nmaturity: sprout\n---\nx"), opt) + ch := Meta(a, b, opt.DiffKeys) + if len(ch) != 2 || ch[0].Key != "title" || ch[1].Key != "maturity" || ch[1].New != "sprout" { + t.Fatalf("got %+v", ch) + } +}
@@ -0,0 +1,46 @@+package diff + +// Excerpt keeps the changed ops of a full diff, with up to context unchanged +// blocks around each, and folds the other unchanged runs into gap ops. It +// stops after maxChanged changed ops and reports whether it cut anything. +func Excerpt(ops []Op, context, maxChanged int) ([]Op, bool) { + keep := make([]bool, len(ops)) + for i, op := range ops { + if op.Op == OpEq { + continue + } + keep[i] = true + for k := 1; k <= context; k++ { + if i-k >= 0 && ops[i-k].Op == OpEq { + keep[i-k] = true + } + if i+k < len(ops) && ops[i+k].Op == OpEq { + keep[i+k] = true + } + } + } + + var out []Op + gap, changed := 0, 0 + for i, op := range ops { + if !keep[i] { + gap++ + continue + } + if op.Op != OpEq { + if changed == maxChanged { + return out, true + } + changed++ + } + if gap > 0 { + out = append(out, Op{Op: OpGap, Count: gap}) + gap = 0 + } + out = append(out, op) + } + if gap > 0 && len(out) > 0 { + out = append(out, Op{Op: OpGap, Count: gap}) + } + return out, false +}
@@ -0,0 +1,42 @@+package diff + +import ( + "encoding/json" + + "git.pablomurad.com/pablo/patchwork/internal/doc" +) + +// MetaChange is a change to a frontmatter value readers see, like the title. +type MetaChange struct { + Key string `json:"key"` + Old any `json:"old"` + New any `json:"new"` +} + +// Meta lists changes to the given frontmatter keys and to the maturity. +func Meta(old, new *doc.Doc, keys []string) []MetaChange { + var out []MetaChange + for _, k := range keys { + a, b := old.MetaValue(k), new.MetaValue(k) + if !sameValue(a, b) { + out = append(out, MetaChange{Key: k, Old: a, New: b}) + } + } + if old.Maturity != new.Maturity { + out = append(out, MetaChange{Key: "maturity", Old: nullable(old.Maturity), New: nullable(new.Maturity)}) + } + return out +} + +func sameValue(a, b any) bool { + ja, _ := json.Marshal(a) + jb, _ := json.Marshal(b) + return string(ja) == string(jb) +} + +func nullable(s string) any { + if s == "" { + return nil + } + return s +}
@@ -0,0 +1,148 @@+// Package diff compares two versions of a document at two levels: blocks +// (paragraphs, list items, code) and, inside a changed block, words. +package diff + +// EditKind is the type of one run in an edit script. +type EditKind int + +const ( + Eq EditKind = iota + Del + Ins +) + +// Edit is a run of the edit script: a[A0:A1] deleted, b[B0:B1] inserted, or +// a[A0:A1] == b[B0:B1] kept. +type Edit struct { + Kind EditKind + A0, A1, B0, B1 int +} + +// maxD bounds the search. Two versions that differ in more than maxD +// elements get one big replacement instead of an expensive minimal script; +// nobody reads a minimal diff of a rewritten page anyway. +const maxD = 4000 + +// Myers returns a shortest edit script from a to b (Myers 1986, the greedy +// O(ND) forward search with trace back), as runs in order. +func Myers[T comparable](a, b []T) []Edit { + pre := 0 + for pre < len(a) && pre < len(b) && a[pre] == b[pre] { + pre++ + } + suf := 0 + for suf < len(a)-pre && suf < len(b)-pre && a[len(a)-1-suf] == b[len(b)-1-suf] { + suf++ + } + mid := middle(a[pre:len(a)-suf], b[pre:len(b)-suf]) + + var out []Edit + add := func(e Edit) { + if e.A0 == e.A1 && e.B0 == e.B1 { + return + } + if n := len(out); n > 0 && out[n-1].Kind == e.Kind && out[n-1].A1 == e.A0 && out[n-1].B1 == e.B0 { + out[n-1].A1, out[n-1].B1 = e.A1, e.B1 + return + } + out = append(out, e) + } + add(Edit{Eq, 0, pre, 0, pre}) + for _, e := range mid { + add(Edit{e.Kind, e.A0 + pre, e.A1 + pre, e.B0 + pre, e.B1 + pre}) + } + add(Edit{Eq, len(a) - suf, len(a), len(b) - suf, len(b)}) + return out +} + +func middle[T comparable](a, b []T) []Edit { + n, m := len(a), len(b) + if n == 0 && m == 0 { + return nil + } + if n == 0 || m == 0 { + return replaceAll(n, m) + } + max := n + m + if max > maxD { + max = maxD + } + offset := max + 1 + v := make([]int, 2*max+3) + var trace [][]int + for d := 0; d <= max; d++ { + trace = append(trace, append([]int(nil), v...)) + for k := -d; k <= d; k += 2 { + var x int + if k == -d || (k != d && v[offset+k-1] < v[offset+k+1]) { + x = v[offset+k+1] + } else { + x = v[offset+k-1] + 1 + } + y := x - k + for x < n && y < m && a[x] == b[y] { + x++ + y++ + } + v[offset+k] = x + if x >= n && y >= m { + return backtrack(trace, offset, n, m) + } + } + } + return replaceAll(n, m) +} + +func backtrack(trace [][]int, offset, n, m int) []Edit { + var rev []Edit + x, y := n, m + for d := len(trace) - 1; d > 0; d-- { + v := trace[d] + k := x - y + var prevK int + if k == -d || (k != d && v[offset+k-1] < v[offset+k+1]) { + prevK = k + 1 + } else { + prevK = k - 1 + } + prevX := v[offset+prevK] + prevY := prevX - prevK + for x > prevX && y > prevY { + rev = append(rev, Edit{Eq, x - 1, x, y - 1, y}) + x-- + y-- + } + if x == prevX { + rev = append(rev, Edit{Ins, x, x, y - 1, y}) + } else { + rev = append(rev, Edit{Del, x - 1, x, y, y}) + } + x, y = prevX, prevY + } + for x > 0 && y > 0 { + rev = append(rev, Edit{Eq, x - 1, x, y - 1, y}) + x-- + y-- + } + out := make([]Edit, 0, len(rev)) + for i := len(rev) - 1; i >= 0; i-- { + e := rev[i] + if n := len(out); n > 0 && out[n-1].Kind == e.Kind && out[n-1].A1 == e.A0 && out[n-1].B1 == e.B0 { + out[n-1].A1, out[n-1].B1 = e.A1, e.B1 + continue + } + out = append(out, e) + } + return out +} + +func replaceAll(n, m int) []Edit { + var out []Edit + if n > 0 { + out = append(out, Edit{Del, 0, n, 0, 0}) + } + if m > 0 { + out = append(out, Edit{Ins, n, n, 0, m}) + } + return out +}
@@ -0,0 +1,3 @@+go test fuzz v1 +string("\xa4") +string("0")
@@ -0,0 +1,152 @@+package diff + +import ( + "strings" + "unicode/utf8" + + "git.pablomurad.com/pablo/patchwork/internal/doc" +) + +// Segment kinds inside a modified block. +const ( + SegEq = "eq" + SegDel = "del" + SegIns = "ins" +) + +// Segment is a run of text in a word diff. +type Segment struct { + T string `json:"t"` + S string `json:"s"` +} + +// replaceRatio: above this share of changed text, highlights stop helping and +// the block is shown as old text, then new text. +const replaceRatio = 0.6 + +// Words diffs two texts word by word. Concatenating the eq and del segments +// gives back old; eq and ins give back new. +func Words(old, new string) (segs []Segment, replaced bool, added, removed int) { + ta, tb := doc.Tokenize(old), doc.Tokenize(new) + for _, e := range Myers(ta, tb) { + switch e.Kind { + case Eq: + segs = appendSeg(segs, SegEq, strings.Join(ta[e.A0:e.A1], "")) + case Del: + segs = appendSeg(segs, SegDel, strings.Join(ta[e.A0:e.A1], "")) + case Ins: + segs = appendSeg(segs, SegIns, strings.Join(tb[e.B0:e.B1], "")) + } + } + segs = cleanup(segs) + + changed := 0 + for _, s := range segs { + switch s.T { + case SegDel: + changed += utf8.RuneCountInString(s.S) + removed += countWords(s.S) + case SegIns: + changed += utf8.RuneCountInString(s.S) + added += countWords(s.S) + } + } + total := utf8.RuneCountInString(old) + utf8.RuneCountInString(new) + if total > 0 && float64(changed)/float64(total) > replaceRatio { + segs = nil + if old != "" { + segs = append(segs, Segment{SegDel, old}) + } + if new != "" { + segs = append(segs, Segment{SegIns, new}) + } + return segs, true, countWords(new), countWords(old) + } + return segs, false, added, removed +} + +// cleanup absorbs short equalities trapped between edits ("the cat sat" → +// "a dog sat" should not keep " " as a lonely match), the semantic cleanup of +// diff-match-patch: an equality no longer than the edits on both sides of it +// becomes part of them. Each edit group is then written as one del followed +// by one ins. +func cleanup(segs []Segment) []Segment { + for changed := true; changed; { + changed = false + groups := group(segs) + for i := 1; i+1 < len(groups); i++ { + g := groups[i] + if g.eq == "" || groups[i-1].eq != "" || groups[i+1].eq != "" { + continue + } + n := utf8.RuneCountInString(g.eq) + if n <= groups[i-1].size() && n <= groups[i+1].size() { + groups[i] = edit{del: g.eq, ins: g.eq} + segs = ungroup(groups) + changed = true + break + } + } + } + return ungroup(group(segs)) +} + +type edit struct{ eq, del, ins string } + +func (e edit) size() int { return utf8.RuneCountInString(e.del) + utf8.RuneCountInString(e.ins) } + +// group merges consecutive segments: every equality is its own group and +// every run of del/ins between equalities is one edit group. +func group(segs []Segment) []edit { + var out []edit + for _, s := range segs { + if s.T == SegEq { + out = append(out, edit{eq: s.S}) + continue + } + if len(out) == 0 || out[len(out)-1].eq != "" { + out = append(out, edit{}) + } + last := &out[len(out)-1] + if s.T == SegDel { + last.del += s.S + } else { + last.ins += s.S + } + } + // Adjacent edit groups (left after absorbing an equality) merge. + merged := out[:0] + for _, g := range out { + if n := len(merged); n > 0 && g.eq == "" && merged[n-1].eq == "" { + merged[n-1].del += g.del + merged[n-1].ins += g.ins + continue + } + merged = append(merged, g) + } + return merged +} + +func ungroup(groups []edit) []Segment { + var out []Segment + for _, g := range groups { + if g.eq != "" { + out = appendSeg(out, SegEq, g.eq) + continue + } + out = appendSeg(out, SegDel, g.del) + out = appendSeg(out, SegIns, g.ins) + } + return out +} + +func appendSeg(segs []Segment, t, s string) []Segment { + if s == "" { + return segs + } + if n := len(segs); n > 0 && segs[n-1].T == t { + segs[n-1].S += s + return segs + } + return append(segs, Segment{t, s}) +}
@@ -0,0 +1,329 @@+package doc + +import ( + "crypto/sha256" + "encoding/hex" + "strings" +) + +// Block kinds. +const ( + KindParagraph = "paragraph" + KindHeading = "heading" + KindCode = "code" + KindListItem = "list_item" + KindQuote = "quote" + KindHTML = "html" + KindTable = "table" + KindBreak = "break" +) + +// Block is the unit of the block-level diff: a paragraph, a heading, one +// list item with whatever is nested under it, a whole code block, quote or +// table. +type Block struct { + Kind string `json:"kind"` + Source string `json:"source"` + Text string `json:"text"` + Hash string `json:"-"` +} + +// SplitBlocks cuts Markdown into blocks with a line scanner instead of a full +// CommonMark parse. A diff needs stable block boundaries, not a render tree, +// and a parser's tree loses the exact source span of fences and thematic +// breaks. The rules follow CommonMark where it matters for boundaries. +func SplitBlocks(body string) []Block { + lines := strings.Split(body, "\n") + n := len(lines) + var blocks []Block + emit := func(kind string, start, end int) { + for end > start && isBlank(lines[end-1]) { + end-- + } + src := strings.Join(lines[start:end], "\n") + blocks = append(blocks, Block{Kind: kind, Source: src, Text: plainText(kind, src), Hash: blockHash(src)}) + } + + for i := 0; i < n; { + l := lines[i] + if isBlank(l) { + i++ + continue + } + start := i + switch { + case fenceMarker(l) != "": + f := fenceMarker(l) + for i++; i < n && !closesFence(lines[i], f); i++ { + } + if i < n { + i++ + } + emit(KindCode, start, i) + case isATXHeading(l): + i++ + emit(KindHeading, start, i) + case isThematicBreak(l): + i++ + emit(KindBreak, start, i) + case isQuote(l): + for i++; i < n && !isBlank(lines[i]) && (isQuote(lines[i]) || !interrupts(lines[i])); i++ { + } + emit(KindQuote, start, i) + case isListItem(l): + markerIndent, contentIndent, _ := listMarker(l) + i++ + for i < n { + if isBlank(lines[i]) { + j := i + for j < n && isBlank(lines[j]) { + j++ + } + if j < n && indentOf(lines[j]) >= contentIndent { + i = j + continue + } + break + } + if mi, _, ok := listMarker(lines[i]); ok && mi <= markerIndent && !isThematicBreak(lines[i]) { + break + } + if indentOf(lines[i]) < contentIndent && interruptsList(lines[i]) { + break + } + i++ + } + emit(KindListItem, start, i) + case isHTMLStart(l): + for i++; i < n && !isBlank(lines[i]); i++ { + } + emit(KindHTML, start, i) + case i+1 < n && strings.Contains(l, "|") && isDelimiterRow(lines[i+1]): + for i += 2; i < n && !isBlank(lines[i]) && strings.Contains(lines[i], "|"); i++ { + } + emit(KindTable, start, i) + case indentOf(l) >= 4: + for i++; i < n; { + if isBlank(lines[i]) { + j := i + for j < n && isBlank(lines[j]) { + j++ + } + if j < n && indentOf(lines[j]) >= 4 { + i = j + continue + } + break + } + if indentOf(lines[i]) < 4 { + break + } + i++ + } + emit(KindCode, start, i) + default: + kind := KindParagraph + for i++; i < n && !isBlank(lines[i]); i++ { + if isSetextUnderline(lines[i]) { + i++ + kind = KindHeading + break + } + if interrupts(lines[i]) { + break + } + } + emit(kind, start, i) + } + } + return blocks +} + +func blockHash(src string) string { + sum := sha256.Sum256([]byte(strings.Join(strings.Fields(src), " "))) + return hex.EncodeToString(sum[:8]) +} + +func isBlank(l string) bool { return strings.TrimSpace(l) == "" } + +// indentOf counts leading columns, with tabs advancing to the next stop of 4. +func indentOf(l string) int { + col := 0 + for _, c := range l { + switch c { + case ' ': + col++ + case '\t': + col += 4 - col%4 + default: + return col + } + } + return col +} + +func fenceMarker(l string) string { + if indentOf(l) > 3 { + return "" + } + t := strings.TrimLeft(l, " \t") + if len(t) < 3 || (t[0] != '`' && t[0] != '~') { + return "" + } + n := 0 + for n < len(t) && t[n] == t[0] { + n++ + } + if n < 3 || (t[0] == '`' && strings.Contains(t[n:], "`")) { + return "" + } + return t[:n] +} + +func closesFence(l, open string) bool { + if indentOf(l) > 3 { + return false + } + t := strings.TrimLeft(l, " \t") + n := 0 + for n < len(t) && t[n] == open[0] { + n++ + } + return n >= len(open) && strings.TrimSpace(t[n:]) == "" +} + +func isATXHeading(l string) bool { + if indentOf(l) > 3 { + return false + } + t := strings.TrimLeft(l, " \t") + n := 0 + for n < len(t) && t[n] == '#' { + n++ + } + return n >= 1 && n <= 6 && (n == len(t) || t[n] == ' ' || t[n] == '\t') +} + +func isThematicBreak(l string) bool { + if indentOf(l) > 3 { + return false + } + t := strings.TrimSpace(l) + if t == "" || !strings.ContainsRune("-*_", rune(t[0])) { + return false + } + count := 0 + for _, c := range t { + switch { + case c == rune(t[0]): + count++ + case c == ' ' || c == '\t': + default: + return false + } + } + return count >= 3 +} + +func isSetextUnderline(l string) bool { + if indentOf(l) > 3 { + return false + } + t := strings.TrimSpace(l) + return t != "" && (strings.Trim(t, "=") == "" || strings.Trim(t, "-") == "") +} + +func isQuote(l string) bool { + return indentOf(l) <= 3 && strings.HasPrefix(strings.TrimLeft(l, " \t"), ">") +} + +func isHTMLStart(l string) bool { + if indentOf(l) > 3 { + return false + } + t := strings.TrimLeft(l, " \t") + if len(t) < 2 || t[0] != '<' { + return false + } + c := t[1] + return c == '/' || c == '!' || c == '?' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') +} + +func isDelimiterRow(l string) bool { + t := strings.TrimSpace(l) + t = strings.TrimPrefix(t, "|") + t = strings.TrimSuffix(t, "|") + if t == "" { + return false + } + for _, cell := range strings.Split(t, "|") { + c := strings.TrimSpace(cell) + c = strings.TrimPrefix(c, ":") + c = strings.TrimSuffix(c, ":") + if c == "" || strings.Trim(c, "-") != "" { + return false + } + } + return true +} + +// listMarker reports a list item start: "- ", "* ", "+ ", "1. " or "1) ". +// contentIndent is the column where the item's text begins, which nested +// content must reach to stay inside the item. +func listMarker(l string) (markerIndent, contentIndent int, ok bool) { + markerIndent = indentOf(l) + if markerIndent > 3 { + return 0, 0, false + } + t := strings.TrimLeft(l, " \t") + m := 0 + switch { + case t != "" && strings.ContainsRune("-*+", rune(t[0])): + m = 1 + default: + for m < len(t) && m < 9 && t[m] >= '0' && t[m] <= '9' { + m++ + } + if m == 0 || m >= len(t) || (t[m] != '.' && t[m] != ')') { + return 0, 0, false + } + m++ + } + if m < len(t) && t[m] != ' ' && t[m] != '\t' { + return 0, 0, false + } + spaces := 0 + for m+spaces < len(t) && t[m+spaces] == ' ' { + spaces++ + } + if spaces == 0 || spaces > 4 || m+spaces == len(t) { + spaces = 1 + } + return markerIndent, markerIndent + m + spaces, true +} + +func isListItem(l string) bool { + _, _, ok := listMarker(l) + return ok && !isThematicBreak(l) +} + +// interrupts reports whether a line ends a paragraph without a blank line. +// An ordered item interrupts only when it starts at 1, as in CommonMark, so +// "2026. was a long year" stays inside its paragraph. +func interrupts(l string) bool { + if isATXHeading(l) || fenceMarker(l) != "" || isThematicBreak(l) || isQuote(l) || isHTMLStart(l) { + return true + } + if !isListItem(l) { + return false + } + t := strings.TrimLeft(l, " \t") + if strings.ContainsRune("-*+", rune(t[0])) { + return true + } + return strings.HasPrefix(t, "1.") || strings.HasPrefix(t, "1)") +} + +func interruptsList(l string) bool { + return isATXHeading(l) || fenceMarker(l) != "" || isThematicBreak(l) || isQuote(l) || isHTMLStart(l) +}
@@ -0,0 +1,113 @@+// Package doc turns a content file into what Patchwork compares between +// versions: normalized text, frontmatter, the author's editorial fields, the +// diffable blocks, and the hashes that decide whether anything changed. +package doc + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" +) + +// Options name the frontmatter keys Patchwork reads. +type Options struct { + NoteKey string + KindKey string + MaturityKey string + // DiffKeys are frontmatter keys whose changes count as content changes + // (title, summary, tags). Every other key is ignored. + DiffKeys []string +} + +// DefaultOptions match the field names in the moonflower demo. +func DefaultOptions() Options { + return Options{NoteKey: "revision", KindKey: "revision_kind", MaturityKey: "maturity", DiffKeys: []string{"title", "summary", "tags"}} +} + +// Doc is one parsed version of a file. +type Doc struct { + Text string + Frontmatter map[string]any + FrontmatterErr error + Body string + Blocks []Block + + Note string + Kind string + Maturity string + Title string + + // RawHash changes when anything in the file changes beyond line endings, + // BOM, Unicode form and trailing spaces. + RawHash string + // ContentHash covers what readers see: the body without ignored regions + // plus the DiffKeys values. Editing only the note leaves it unchanged. + ContentHash string +} + +// Parse reads one version of a file. +func Parse(raw []byte, opt Options) *Doc { + d := &Doc{Text: Normalize(raw)} + d.RawHash = hashText(d.Text) + + yamlSrc, body, ok := splitFrontmatter(d.Text) + d.Body = body + d.Frontmatter = map[string]any{} + if ok { + d.Frontmatter, d.FrontmatterErr = parseFrontmatter(yamlSrc) + } + d.Note = stringField(d.Frontmatter, opt.NoteKey) + d.Kind = stringField(d.Frontmatter, opt.KindKey) + d.Maturity = stringField(d.Frontmatter, opt.MaturityKey) + + visible := stripIgnored(body) + d.Blocks = SplitBlocks(visible) + d.Title = stringField(d.Frontmatter, "title") + if d.Title == "" { + for _, b := range d.Blocks { + if b.Kind == KindHeading { + d.Title = b.Text + break + } + } + } + + meta := map[string]any{} + for _, k := range opt.DiffKeys { + if v, ok := d.Frontmatter[k]; ok { + meta[k] = v + } + } + metaJSON, _ := json.Marshal(meta) // map keys marshal sorted + sum := sha256.Sum256([]byte(hashText(visible) + "\x00" + string(metaJSON))) + d.ContentHash = hex.EncodeToString(sum[:]) + return d +} + +// MetaValue returns a DiffKeys value in a stable, comparable form. +func (d *Doc) MetaValue(key string) any { + v, ok := d.Frontmatter[key] + if !ok { + return nil + } + return v +} + +// Change kinds an author may declare. +var Kinds = []string{"correction", "expansion", "position", "reorganization"} + +// Maturities an author may declare, from least to most grown. +var Maturities = []string{"seed", "sprout", "bloom"} + +// Valid reports whether v is empty or one of allowed. +func Valid(v string, allowed []string) bool { + if v == "" { + return true + } + for _, a := range allowed { + if v == a { + return true + } + } + return false +}
@@ -0,0 +1,168 @@+package doc + +import ( + "strings" + "testing" +) + +func kinds(bs []Block) string { + var out []string + for _, b := range bs { + out = append(out, b.Kind) + } + return strings.Join(out, ",") +} + +func TestSplitBlocks(t *testing.T) { + cases := []struct { + name, in, want string + }{ + {"paragraphs", "one\ntwo\n\nthree", "paragraph,paragraph"}, + {"atx interrupts", "text\n# Head\nmore", "paragraph,heading,paragraph"}, + {"setext", "Title\n=====\n\nbody", "heading,paragraph"}, + {"thematic vs setext", "para\n\n---\n\nnext", "paragraph,break,paragraph"}, + {"fence with blank lines", "```go\na\n\nb\n```\nafter", "code,paragraph"}, + {"unclosed fence", "```\nnever closed\n\nstill code", "code"}, + {"list items", "- a\n- b\n continued\n- c", "list_item,list_item,list_item"}, + {"nested list stays", "- a\n - nested\n - nested2\n- b", "list_item,list_item"}, + {"loose item paragraphs", "1. first\n\n second para\n\n2. next", "list_item,list_item"}, + {"list ends at outdent", "- a\n\nparagraph", "list_item,paragraph"}, + {"quote", "> a\n> b\nlazy\n\npara", "quote,paragraph"}, + {"table", "| a | b |\n|---|:-:|\n| 1 | 2 |\n\nafter", "table,paragraph"}, + {"html", "<div>\nx\n</div>\n\np", "html,paragraph"}, + {"indented code", "para\n\n code\n\n more\n\nend", "paragraph,code,paragraph"}, + {"ordered 2 does not interrupt", "In\n2026. was long", "paragraph"}, + {"star break not list", "* * *", "break"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := kinds(SplitBlocks(c.in)); got != c.want { + t.Fatalf("got %s, want %s", got, c.want) + } + }) + } +} + +func TestBlockSourcesCoverFences(t *testing.T) { + bs := SplitBlocks("```\ncode\n```") + if bs[0].Source != "```\ncode\n```" || bs[0].Text != "code" { + t.Fatalf("got %+v", bs[0]) + } +} + +func TestPlainText(t *testing.T) { + cases := map[string]string{ + "A [link](https://x.y) and **bold** _em_ `code`.": "A link and bold em code.", + "# Heading *one*": "Heading one", + "- item with <https://auto.link>": "item with https://auto.link", + "| a | **b** |\n|---|---|\n| 1 | 2 |": "a | b\n1 | 2", + "line one\nline two": "line one line two", + " after": "alt text after", + "> quoted *text*": "quoted text", + "Ação & reação": "Ação & reação", + "hard \nbreak": "hard break", + `escaped \*stars\* and é`: "escaped *stars* and é", + "`a & b` code": "a & b code", + } + for in, want := range cases { + bs := SplitBlocks(in) + if len(bs) != 1 { + t.Fatalf("%q split into %d blocks", in, len(bs)) + } + if bs[0].Text != want { + t.Errorf("plainText(%q) = %q, want %q", in, bs[0].Text, want) + } + } +} + +func TestTokenizeRoundTrip(t *testing.T) { + for _, s := range []string{"Don't stop, d'água! Ação\tok", "", "🌙 moon", "a-b_c 12.5%"} { + toks := Tokenize(s) + if strings.Join(toks, "") != s { + t.Fatalf("round trip of %q: %q", s, toks) + } + } + got := Tokenize("don't d'água x'") + want := []string{"don't", " ", "d'água", " ", "x", "'"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("got %q", got) + } +} + +func TestFrontmatterAndHashes(t *testing.T) { + opt := DefaultOptions() + a := Parse([]byte("---\ntitle: Moon\nrevision: First note\nmaturity: seed\nflowerColor: '#fff'\n---\nBody text.\n"), opt) + if a.FrontmatterErr != nil || a.Note != "First note" || a.Maturity != "seed" || a.Title != "Moon" { + t.Fatalf("parsed %+v", a) + } + + // Only the note and an ignored key change: content is the same. + b := Parse([]byte("---\ntitle: Moon\nrevision: Second note\nmaturity: seed\nflowerColor: '#000'\n---\nBody text.\n"), opt) + if a.ContentHash != b.ContentHash || a.RawHash == b.RawHash { + t.Fatal("note-only edit should keep ContentHash and change RawHash") + } + + // CRLF, BOM and trailing spaces are not edits. + c := Parse([]byte("\xef\xbb\xbf---\r\ntitle: Moon\r\nrevision: First note\r\nmaturity: seed\r\nflowerColor: '#fff'\r\n---\r\nBody text. \r\n\r\n"), opt) + if c.RawHash != a.RawHash { + t.Fatal("CRLF/BOM/trailing space should not change RawHash") + } + + // NFD and NFC spellings are the same text. + nfc := Parse([]byte("Ação"), opt) + nfd := Parse([]byte("Ação"), opt) + if nfc.RawHash != nfd.RawHash { + t.Fatal("NFD input should normalize to NFC") + } + + // A title change is a content change. + d := Parse([]byte("---\ntitle: Sun\nrevision: First note\nmaturity: seed\n---\nBody text.\n"), opt) + if d.ContentHash == a.ContentHash { + t.Fatal("title change should change ContentHash") + } +} + +func TestMalformedFrontmatter(t *testing.T) { + d := Parse([]byte("---\ntitle: [unclosed\n---\nBody"), DefaultOptions()) + if d.FrontmatterErr == nil { + t.Fatal("want YAML error") + } + if d.Body != "Body" || len(d.Blocks) != 1 { + t.Fatalf("body should survive: %+v", d) + } +} + +func TestNoFrontmatterUsesHeadingTitle(t *testing.T) { + d := Parse([]byte("# Jardim de agentes autônomos\n\ntext"), DefaultOptions()) + if d.Title != "Jardim de agentes autônomos" { + t.Fatalf("title %q", d.Title) + } + unclosed := Parse([]byte("---\nnot frontmatter"), DefaultOptions()) + if len(unclosed.Frontmatter) != 0 || unclosed.Body != "---\nnot frontmatter" { + t.Fatalf("unclosed delimiter is body: %+v", unclosed) + } +} + +func TestIgnoreRegions(t *testing.T) { + opt := DefaultOptions() + a := Parse([]byte("Keep.\n\n<!-- patchwork:ignore -->\nUpdated 2026-10-01\n<!-- /patchwork:ignore -->\n\nEnd."), opt) + b := Parse([]byte("Keep.\n\n<!-- patchwork:ignore -->\nUpdated 2026-10-23\n<!-- /patchwork:ignore -->\n\nEnd."), opt) + if a.ContentHash != b.ContentHash { + t.Fatal("ignored region edits must not change ContentHash") + } + if kinds(a.Blocks) != "paragraph,paragraph" { + t.Fatalf("blocks %s", kinds(a.Blocks)) + } + fenced := Parse([]byte("```\n<!-- patchwork:ignore -->\nshown\n```"), opt) + if !strings.Contains(fenced.Blocks[0].Text, "shown") { + t.Fatal("markers inside code fences must be literal") + } +} + +func TestYAMLScalarsAsText(t *testing.T) { + d := Parse([]byte("---\nrevision: 2\nmaturity: yes\n---\n"), DefaultOptions()) + // yaml.v3 follows YAML 1.2, where "yes" is a plain string. + if d.Note != "2" || d.Maturity != "yes" { + t.Fatalf("note %q maturity %q", d.Note, d.Maturity) + } +}
@@ -0,0 +1,63 @@+package doc + +import ( + "strings" + + "gopkg.in/yaml.v3" +) + +// splitFrontmatter separates a leading YAML block delimited by "---" lines, +// the convention gray-matter, Hugo, Eleventy and Astro share. A file whose +// first line is "---" but never closes it has no frontmatter. +func splitFrontmatter(text string) (yamlSrc, body string, ok bool) { + first, rest, found := strings.Cut(text, "\n") + if !found || strings.TrimRight(first, " \t") != "---" { + return "", text, false + } + offset := 0 + for { + line, after, more := strings.Cut(rest[offset:], "\n") + if t := strings.TrimRight(line, " \t"); t == "---" || t == "..." { + return rest[:offset], after, true + } + if !more { + return "", text, false + } + offset += len(line) + 1 + } +} + +func parseFrontmatter(src string) (map[string]any, error) { + fm := map[string]any{} + if strings.TrimSpace(src) == "" { + return fm, nil + } + if err := yaml.Unmarshal([]byte(src), &fm); err != nil { + return map[string]any{}, err + } + if fm == nil { + fm = map[string]any{} + } + return fm, nil +} + +// stringField reads a scalar frontmatter value as trimmed text. YAML turns +// `revision: 2` into an int and `maturity: yes` into a bool; the author still +// meant the words. +func stringField(fm map[string]any, key string) string { + v, ok := fm[key] + if !ok || v == nil { + return "" + } + switch t := v.(type) { + case string: + return strings.TrimSpace(t) + case []any, map[string]any: + return "" + } + b, err := yaml.Marshal(v) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +}
@@ -0,0 +1,48 @@+package doc + +import "strings" + +const ( + ignoreOpen = "<!-- patchwork:ignore -->" + ignoreClose = "<!-- /patchwork:ignore -->" +) + +// stripIgnored removes regions between ignore markers. Markers count only on +// their own line and outside fenced code, so a post that documents the +// markers can show them in a code block. An unclosed region runs to the end. +func stripIgnored(body string) string { + if !strings.Contains(body, ignoreOpen) { + return body + } + lines := strings.Split(body, "\n") + out := lines[:0] + var fence string + ignoring := false + for _, l := range lines { + t := strings.TrimSpace(l) + if !ignoring { + if f := fenceMarker(l); f != "" { + switch { + case fence == "": + fence = f + case closesFence(l, fence): + fence = "" + } + } + } + if fence == "" { + if !ignoring && t == ignoreOpen { + ignoring = true + continue + } + if ignoring && t == ignoreClose { + ignoring = false + continue + } + } + if !ignoring { + out = append(out, l) + } + } + return strings.Join(out, "\n") +}
@@ -0,0 +1,31 @@+package doc + +import ( + "crypto/sha256" + "encoding/hex" + "strings" + + "golang.org/x/text/unicode/norm" +) + +// Normalize returns the text Patchwork stores: no BOM, LF line endings, NFC. +// Editors and upload tools change these freely, and none of them is a +// revision. +func Normalize(raw []byte) string { + s := string(raw) + s = strings.TrimPrefix(s, "\xef\xbb\xbf") + s = strings.ReplaceAll(s, "\r\n", "\n") + s = strings.ReplaceAll(s, "\r", "\n") + return norm.NFC.String(s) +} + +// hashText hashes text with trailing whitespace removed from every line and +// trailing blank lines dropped, so re-saving a file never looks like an edit. +func hashText(s string) string { + lines := strings.Split(s, "\n") + for i, l := range lines { + lines[i] = strings.TrimRight(l, " \t") + } + sum := sha256.Sum256([]byte(strings.TrimRight(strings.Join(lines, "\n"), "\n"))) + return hex.EncodeToString(sum[:]) +}
@@ -0,0 +1,100 @@+package doc + +import ( + "strings" + + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" +) + +var md = goldmark.New(goldmark.WithExtensions(extension.GFM)) + +// plainText is what a reader sees of a block: link text without the URL, +// emphasis without asterisks. Word diffs run on it so a revision reads as +// prose, not as Markdown syntax. +func plainText(kind, src string) string { + switch kind { + case KindCode: + return codeText(src) + case KindBreak: + return "" + case KindHTML: + return collapse(src) + case KindTable: + return tableText(src) + } + source := []byte(src) + root := md.Parser().Parse(text.NewReader(source)) + var b strings.Builder + _ = ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + if n.Type() == ast.TypeBlock { + b.WriteByte(' ') + } + return ast.WalkContinue, nil + } + switch t := n.(type) { + case *ast.Text: + v := t.Segment.Value(source) + if _, inCode := t.Parent().(*ast.CodeSpan); !inCode { + v = util.ResolveEntityNames(util.ResolveNumericReferences(util.UnescapePunctuations(v))) + } + b.Write(v) + if t.SoftLineBreak() || t.HardLineBreak() { + b.WriteByte(' ') + } + case *ast.String: + b.Write(t.Value) + case *ast.AutoLink: + b.Write(t.Label(source)) + return ast.WalkSkipChildren, nil + case *ast.FencedCodeBlock, *ast.CodeBlock, *ast.HTMLBlock: + lines := n.Lines() + for i := 0; i < lines.Len(); i++ { + seg := lines.At(i) + b.Write(seg.Value(source)) + } + return ast.WalkSkipChildren, nil + case *ast.RawHTML: + return ast.WalkSkipChildren, nil + } + return ast.WalkContinue, nil + }) + return collapse(b.String()) +} + +// tableText keeps one line per row and " | " between cells, so a changed +// cell is still read in its row. The delimiter row is dropped. +func tableText(src string) string { + var rows []string + for i, line := range strings.Split(src, "\n") { + if i == 1 && isDelimiterRow(line) { + continue + } + t := strings.TrimSpace(line) + t = strings.TrimSuffix(strings.TrimPrefix(t, "|"), "|") + cells := strings.Split(t, "|") + for j, c := range cells { + cells[j] = plainText(KindParagraph, strings.TrimSpace(c)) + } + rows = append(rows, strings.Join(cells, " | ")) + } + return strings.Join(rows, "\n") +} + +// codeText keeps a code block's lines and drops its fences. +func codeText(src string) string { + lines := strings.Split(src, "\n") + if f := fenceMarker(lines[0]); f != "" { + lines = lines[1:] + if len(lines) > 0 && closesFence(lines[len(lines)-1], f) { + lines = lines[:len(lines)-1] + } + } + return strings.Join(lines, "\n") +} + +func collapse(s string) string { return strings.Join(strings.Fields(s), " ") }
@@ -0,0 +1,55 @@+package doc + +import ( + "unicode" + "unicode/utf8" +) + +// Tokenize splits text into words, whitespace runs and single punctuation +// marks. Concatenating the tokens gives back the input exactly. An apostrophe +// between letters stays inside the word ("don't", "d'água"). +func Tokenize(s string) []string { + var out []string + for i := 0; i < len(s); { + r, size := utf8.DecodeRuneInString(s[i:]) + j := i + size + switch { + case isWordRune(r): + for j < len(s) { + r2, s2 := utf8.DecodeRuneInString(s[j:]) + if isWordRune(r2) { + j += s2 + continue + } + if r2 == '\'' || r2 == '’' { + if r3, _ := utf8.DecodeRuneInString(s[j+s2:]); j+s2 < len(s) && unicode.IsLetter(r3) { + j += s2 + continue + } + } + break + } + case unicode.IsSpace(r): + for j < len(s) { + r2, s2 := utf8.DecodeRuneInString(s[j:]) + if !unicode.IsSpace(r2) { + break + } + j += s2 + } + } + out = append(out, s[i:j]) + i = j + } + return out +} + +// IsWord reports whether a token from Tokenize is a word. +func IsWord(tok string) bool { + r, _ := utf8.DecodeRuneInString(tok) + return isWordRune(r) +} + +func isWordRune(r rune) bool { + return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) || r == '_' +}
@@ -0,0 +1,311 @@+// Package engine applies Patchwork's publish rules: it turns observed file +// states into snapshots, and decides which snapshots become public +// revisions. +package engine + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "time" + "unicode/utf8" + + "git.pablomurad.com/pablo/patchwork/internal/config" + "git.pablomurad.com/pablo/patchwork/internal/diff" + "git.pablomurad.com/pablo/patchwork/internal/doc" + "git.pablomurad.com/pablo/patchwork/internal/slug" + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +// Action is what ingesting one observation did. +type Action string + +const ( + Unchanged Action = "unchanged" // same text as the last snapshot + Baseline Action = "baseline" // first sight of the file: version 1 + Silent Action = "silent" // stored as history only + Published Action = "published" // a new public revision + Amended Action = "amended" // the last revision's note was rewritten + Folded Action = "folded" // a quick follow-up joined the last revision + Removed Action = "removed" // the file is gone +) + +// Observation is the content of one file at one moment. +type Observation struct { + Path string // relative to the source root, '/'-separated + Raw []byte + Mtime time.Time + // At overrides the observation time (snapshot --at); zero means now. + At time.Time +} + +// Result reports what happened to one file. +type Result struct { + Path string + Slug string + Action Action + Number int // the revision created or changed; 1 for a baseline + Note string + Restored bool // the page had been missing or removed + Warnings []string +} + +// Engine ingests observations into a store. +type Engine struct { + Store *store.Store + Cfg *config.Config + Source string + Now func() time.Time + + slugRule slug.Rule + opts doc.Options + loc *time.Location +} + +// New builds an engine for one source. +func New(st *store.Store, cfg *config.Config) (*Engine, error) { + rule, err := slug.Lookup(cfg.Source.Slug) + if err != nil { + return nil, err + } + return &Engine{ + Store: st, Cfg: cfg, Source: cfg.Source.Mode, Now: time.Now, + slugRule: rule, opts: cfg.DocOptions(), loc: cfg.Location(), + }, nil +} + +// Ingest stores an observation and applies the publish rules. With dryRun +// nothing is written, but the result says what would have happened. +func (e *Engine) Ingest(ctx context.Context, obs Observation, dryRun bool) (Result, error) { + var res Result + err := e.Store.Tx(ctx, dryRun, func(q *store.Queries) error { + var err error + res, err = e.ingest(ctx, q, obs) + return err + }) + return res, err +} + +func (e *Engine) ingest(ctx context.Context, q *store.Queries, obs Observation) (Result, error) { + at := obs.At + if at.IsZero() { + at = e.Now() + } + d := doc.Parse(obs.Raw, e.opts) + res := Result{Path: obs.Path, Note: d.Note} + res.Warnings = e.check(d, obs) + kind, maturity := e.enum(d, &res) + + page, err := q.PageByPath(ctx, e.Source, obs.Path) + if errors.Is(err, store.ErrNotFound) { + return e.baseline(ctx, q, obs, d, at, kind, maturity, res) + } + if err != nil { + return res, err + } + res.Slug = page.Slug + + if page.RemovedAt != nil || page.MissingScans > 0 { + res.Restored = page.RemovedAt != nil + page.RemovedAt, page.MissingScans = nil, 0 + } + page.Slug = e.slugRule(obs.Path) + page.OriginalDate = e.originalDate(d, obs.Path, page.FirstSeenAt) + res.Slug = page.Slug + if err := q.UpdatePage(ctx, page); err != nil { + return res, err + } + + prev, err := q.LastSnapshot(ctx, page.ID) + if err != nil { + return res, err + } + if prev.RawHash == d.RawHash { + res.Action = Unchanged + return res, nil + } + snap := e.snapshot(page.ID, obs, d, at, kind, maturity) + if err := q.InsertSnapshot(ctx, snap); err != nil { + return res, err + } + + last, err := q.LastRevision(ctx, page.ID) + if err != nil { + return res, fmt.Errorf("%s has no revision: %w", obs.Path, err) + } + lastSnap, err := q.Snapshot(ctx, last.SnapshotID) + if err != nil { + return res, err + } + + noteNew := d.Note != "" && d.Note != last.Note + sameContent := d.ContentHash == lastSnap.ContentHash && maturity == last.Maturity + fold := e.Cfg.Publish.FoldWindow.Duration + + switch { + case noteNew && sameContent && last.Baseline(): + res.Action = Silent + res.Warnings = append(res.Warnings, "new revision note but no visible change since the first version; nothing published") + + case noteNew && sameContent: + // Only the note changed: the author is fixing the note. + last.Note, last.Kind, last.SnapshotID, last.UpdatedAt = d.Note, kind, snap.ID, at + if err := q.UpdateRevision(ctx, last); err != nil { + return res, err + } + res.Action, res.Number = Amended, last.Number + + case noteNew: + n, err := q.MaxRevisionNumber(ctx, page.ID) + if err != nil { + return res, err + } + rev := &store.Revision{ + PageID: page.ID, Number: n + 1, SnapshotID: snap.ID, + Note: d.Note, Kind: kind, Maturity: maturity, PublishedAt: at, UpdatedAt: at, + } + e.setDiff(rev, lastSnap, d) + if err := q.InsertRevision(ctx, rev); err != nil { + return res, err + } + res.Action, res.Number = Published, rev.Number + + case d.Note != "" && d.Note == last.Note && !last.Baseline() && fold > 0 && at.Sub(last.PublishedAt) <= fold: + // Same note, shortly after publishing: the typo spotted right + // after the upload belongs to that revision. + base, err := q.RevisionBefore(ctx, page.ID, last.Number) + if err != nil { + return res, err + } + baseSnap, err := q.Snapshot(ctx, base.SnapshotID) + if err != nil { + return res, err + } + last.SnapshotID, last.Kind, last.Maturity, last.UpdatedAt = snap.ID, kind, maturity, at + e.setDiff(last, baseSnap, d) + if err := q.UpdateRevision(ctx, last); err != nil { + return res, err + } + res.Action, res.Number = Folded, last.Number + + default: + res.Action = Silent + } + return res, nil +} + +func (e *Engine) baseline(ctx context.Context, q *store.Queries, obs Observation, d *doc.Doc, at time.Time, kind, maturity string, res Result) (Result, error) { + page := &store.Page{ + UID: store.NewUID(), Source: e.Source, Path: obs.Path, + Slug: e.slugRule(obs.Path), FirstSeenAt: at, + } + page.OriginalDate = e.originalDate(d, obs.Path, at) + if page.Slug == "" { + res.Warnings = append(res.Warnings, "the file name produces an empty slug; the page has no URL") + } + if err := q.InsertPage(ctx, page); err != nil { + return res, err + } + snap := e.snapshot(page.ID, obs, d, at, kind, maturity) + if err := q.InsertSnapshot(ctx, snap); err != nil { + return res, err + } + // The baseline keeps the note already in the file, unpublished, so a + // later upload that still carries it doesn't announce an old note. + rev := &store.Revision{ + PageID: page.ID, Number: 1, SnapshotID: snap.ID, + Note: d.Note, Kind: kind, Maturity: maturity, PublishedAt: at, UpdatedAt: at, + } + if err := q.InsertRevision(ctx, rev); err != nil { + return res, err + } + res.Slug, res.Action, res.Number = page.Slug, Baseline, 1 + return res, nil +} + +func (e *Engine) snapshot(pageID int64, obs Observation, d *doc.Doc, at time.Time, kind, maturity string) *store.Snapshot { + fm, err := json.Marshal(d.Frontmatter) + if err != nil { + fm = []byte("{}") + } + s := &store.Snapshot{ + PageID: pageID, ObservedAt: at, RawHash: d.RawHash, ContentHash: d.ContentHash, + Raw: d.Text, FrontmatterJSON: string(fm), Title: d.Title, Note: d.Note, Kind: kind, Maturity: maturity, + } + if !obs.Mtime.IsZero() { + m := obs.Mtime + s.SourceMtime = &m + } + return s +} + +func (e *Engine) setDiff(rev *store.Revision, base *store.Snapshot, cur *doc.Doc) { + full := diff.Compare(doc.Parse([]byte(base.Raw), e.opts), cur, e.opts.DiffKeys) + b, err := json.Marshal(full) + if err != nil { + panic(err) // the diff types always marshal + } + rev.DiffJSON, rev.DiffVersion = string(b), diff.Version + rev.WordsAdded, rev.WordsRemoved = full.Stats.WordsAdded, full.Stats.WordsRemoved +} + +func (e *Engine) check(d *doc.Doc, obs Observation) []string { + var w []string + if !utf8.Valid(obs.Raw) { + w = append(w, "the file is not valid UTF-8") + } + if d.FrontmatterErr != nil { + w = append(w, "frontmatter is not valid YAML, so no revision note is read: "+firstLine(d.FrontmatterErr.Error())) + } + return w +} + +// enum keeps only declared values of revision_kind and maturity; anything +// else is dropped with a warning rather than published. +func (e *Engine) enum(d *doc.Doc, res *Result) (kind, maturity string) { + kind, maturity = d.Kind, d.Maturity + if !doc.Valid(kind, doc.Kinds) { + res.Warnings = append(res.Warnings, fmt.Sprintf("%s %q is not one of %s; ignored", e.opts.KindKey, kind, strings.Join(doc.Kinds, ", "))) + kind = "" + } + if !doc.Valid(maturity, doc.Maturities) { + res.Warnings = append(res.Warnings, fmt.Sprintf("%s %q is not one of %s; ignored", e.opts.MaturityKey, maturity, strings.Join(doc.Maturities, ", "))) + maturity = "" + } + return kind, maturity +} + +// originalDate is the page's first publication date: frontmatter first, +// then a date at the start of the file name, then the day Patchwork first +// saw the file. +func (e *Engine) originalDate(d *doc.Doc, relPath string, firstSeen time.Time) string { + switch v := d.Frontmatter[e.Cfg.Frontmatter.DateKey].(type) { + case time.Time: + // YAML reads `date: 2026-06-21` as midnight UTC; that is a calendar + // date, not an instant, so it must not shift into the site's zone. + if v.Hour() == 0 && v.Minute() == 0 && v.Second() == 0 && v.Location() == time.UTC { + return v.Format("2006-01-02") + } + return v.In(e.loc).Format("2006-01-02") + case string: + s := strings.TrimSpace(v) + if t, err := time.Parse("2006-01-02", s); err == nil { + return t.Format("2006-01-02") + } + if t, err := time.Parse(time.RFC3339, s); err == nil { + return t.In(e.loc).Format("2006-01-02") + } + } + if date, ok := slug.DatePrefix(path.Base(relPath)); ok { + return date + } + return firstSeen.In(e.loc).Format("2006-01-02") +} + +func firstLine(s string) string { + line, _, _ := strings.Cut(s, "\n") + return line +}
@@ -0,0 +1,309 @@+package engine + +import ( + "context" + "encoding/json" + "path/filepath" + "strings" + "testing" + "time" + + "git.pablomurad.com/pablo/patchwork/internal/config" + "git.pablomurad.com/pablo/patchwork/internal/diff" + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +type harness struct { + t *testing.T + e *Engine + now time.Time +} + +func newHarness(t *testing.T) *harness { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "pw.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + cfg := config.Default() + cfg.Source.Slug = "datestrip-slugify" + cfg.Site.Timezone = "America/Sao_Paulo" + e, err := New(st, cfg) + if err != nil { + t.Fatal(err) + } + h := &harness{t: t, e: e, now: time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)} + e.Now = func() time.Time { return h.now } + return h +} + +func (h *harness) after(d time.Duration) *harness { h.now = h.now.Add(d); return h } + +func (h *harness) put(path, body string) Result { + h.t.Helper() + res, err := h.e.Ingest(context.Background(), Observation{Path: path, Raw: []byte(body)}, false) + if err != nil { + h.t.Fatal(err) + } + return res +} + +func (h *harness) expect(res Result, action Action, number int) { + h.t.Helper() + if res.Action != action || res.Number != number { + h.t.Fatalf("got %s #%d (warnings %q), want %s #%d", res.Action, res.Number, res.Warnings, action, number) + } +} + +func (h *harness) revisions(path string) []store.Revision { + h.t.Helper() + ctx := context.Background() + p, err := h.e.Store.PageByPath(ctx, "folder", path) + if err != nil { + h.t.Fatal(err) + } + revs, err := h.e.Store.Revisions(ctx, p.ID) + if err != nil { + h.t.Fatal(err) + } + return revs +} + +func (h *harness) diffOf(r store.Revision) diff.Full { + h.t.Helper() + var f diff.Full + if err := json.Unmarshal([]byte(r.DiffJSON), &f); err != nil { + h.t.Fatal(err) + } + return f +} + +func page(note, body string) string { + fm := "---\ntitle: Remote work\n" + if note != "" { + fm += "revision: " + note + "\n" + } + return fm + "---\n" + body +} + +const ( + p1 = "The hybrid model is the worst of both worlds." + p2 = "Small teams should pick one place and stay there." +) + +func TestBaselineThenSilentThenPublish(t *testing.T) { + h := newHarness(t) + h.expect(h.put("remote.md", page("", p1+"\n\n"+p2)), Baseline, 1) + h.expect(h.after(time.Hour).put("remote.md", page("", p1+"\n\nSmall teams should pick a place and stay there.")), Silent, 0) + h.expect(h.after(time.Hour).put("remote.md", page("Changed my mind.", "The hybrid model works with fixed office days.\n\n"+p2)), Published, 2) + + revs := h.revisions("remote.md") + if len(revs) != 2 || revs[1].Note != "Changed my mind." { + t.Fatalf("revisions %+v", revs) + } + // The silent edit to p2 was undone; the published diff is against v1, + // so only the first paragraph shows as changed. + f := h.diffOf(revs[1]) + if f.Stats.BlocksChanged != 1 || f.Ops[0].Op != diff.OpModify || f.Ops[1].Op != diff.OpEq { + t.Fatalf("diff %+v", f) + } +} + +func TestSilentEditsBatchIntoOneRevision(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("", "One.\n\nTwo.\n\nThree.")) + h.after(time.Hour).put("a.md", page("", "One!\n\nTwo.\n\nThree.")) + h.after(time.Hour).put("a.md", page("", "One!\n\nTwo.\n\nThree!")) + h.expect(h.after(time.Hour).put("a.md", page("Punctuation pass.", "One!\n\nTwo.\n\nThree!")), Published, 2) + f := h.diffOf(h.revisions("a.md")[1]) + if f.Stats.BlocksChanged != 2 { + t.Fatalf("want both silent edits in the published diff, got %+v", f.Stats) + } +} + +func TestUnchangedTextIsNotASnapshot(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("", p1)) + h.expect(h.after(time.Hour).put("a.md", page("", p1)), Unchanged, 0) + crlf := strings.ReplaceAll(page("", p1+" \n"), "\n", "\r\n") + h.expect(h.after(time.Hour).put("a.md", crlf), Unchanged, 0) + p, _ := h.e.Store.PageByPath(context.Background(), "folder", "a.md") + if n, _ := h.e.Store.SnapshotCount(context.Background(), p.ID); n != 1 { + t.Fatalf("snapshots %d", n) + } +} + +func TestNoteOnlyChangeAmends(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("", p1)) + h.expect(h.after(time.Hour).put("a.md", page("First tak.", p2)), Published, 2) + h.expect(h.after(time.Hour).put("a.md", page("First take.", p2)), Amended, 2) + revs := h.revisions("a.md") + if len(revs) != 2 || revs[1].Note != "First take." || !revs[1].UpdatedAt.After(revs[1].PublishedAt) { + t.Fatalf("revisions %+v", revs) + } +} + +func TestNoteWithoutChangeSinceBaselineWarns(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("", p1)) + res := h.after(time.Hour).put("a.md", page("A note", p1)) + h.expect(res, Silent, 0) + if len(res.Warnings) != 1 { + t.Fatalf("warnings %q", res.Warnings) + } +} + +func TestBaselineNoteIsNotRepublished(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("Old note from before Patchwork.", p1)) + h.expect(h.after(time.Hour).put("a.md", page("Old note from before Patchwork.", p2)), Silent, 0) +} + +func TestFoldWindow(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("", p1)) + h.expect(h.after(time.Hour).put("a.md", page("Rewrote it.", "The hybrid modle works.")), Published, 2) + h.expect(h.after(5*time.Minute).put("a.md", page("Rewrote it.", "The hybrid model works.")), Folded, 2) + + revs := h.revisions("a.md") + f := h.diffOf(revs[1]) + var ins string + for _, s := range f.Ops[0].Segments { + if s.T == diff.SegIns { + ins += s.S + } + } + if strings.Contains(ins, "modle") { + t.Fatalf("fold should recompute the diff against v1: %+v", f.Ops) + } + if !revs[1].PublishedAt.Equal(revs[1].UpdatedAt.Add(-5 * time.Minute)) { + t.Fatal("fold keeps published_at and moves updated_at") + } + + // The window counts from published_at: 25 minutes after publishing, + // even right after the fold, the edit is silent. + h.expect(h.after(20*time.Minute).put("a.md", page("Rewrote it.", "The hybrid model works well.")), Silent, 0) +} + +func TestFoldDisabled(t *testing.T) { + h := newHarness(t) + h.e.Cfg.Publish.FoldWindow.Duration = 0 + h.put("a.md", page("", p1)) + h.after(time.Hour).put("a.md", page("Note.", p2)) + h.expect(h.after(time.Minute).put("a.md", page("Note.", p2+" More.")), Silent, 0) +} + +func TestNoteRemovedAndRestoredDoesNotRepublish(t *testing.T) { + h := newHarness(t) + h.put("a.md", page("", p1)) + h.after(time.Hour).put("a.md", page("Note A.", p2)) + h.expect(h.after(time.Hour).put("a.md", page("", p2+" Edit.")), Silent, 0) + h.expect(h.after(time.Hour).put("a.md", page("Note A.", p2+" Edit.")), Silent, 0) + if n := len(h.revisions("a.md")); n != 2 { + t.Fatalf("revisions %d", n) + } +} + +func TestMaturityChangeWithNoteIsARevision(t *testing.T) { + h := newHarness(t) + h.put("a.md", "---\nmaturity: seed\n---\n"+p1) + h.expect(h.after(time.Hour).put("a.md", "---\nmaturity: sprout\nrevision: It grew.\n---\n"+p1), Published, 2) + r := h.revisions("a.md")[1] + f := h.diffOf(r) + if r.Maturity != "sprout" || len(f.Meta) != 1 || f.Meta[0].Key != "maturity" || f.Stats.BlocksChanged != 0 { + t.Fatalf("rev %+v diff %+v", r, f) + } + // Maturity alone, without a note, stays silent. + h.expect(h.after(time.Hour).put("a.md", "---\nmaturity: bloom\nrevision: It grew.\n---\n"+p1), Silent, 0) +} + +func TestInvalidEnumsAreDropped(t *testing.T) { + h := newHarness(t) + h.put("a.md", p1) + res := h.after(time.Hour).put("a.md", "---\nrevision: X\nrevision_kind: rant\nmaturity: tree\n---\n"+p2) + h.expect(res, Published, 2) + r := h.revisions("a.md")[1] + if r.Kind != "" || r.Maturity != "" || len(res.Warnings) != 2 { + t.Fatalf("rev %+v warnings %q", r, res.Warnings) + } +} + +func TestMalformedFrontmatterNeverPublishes(t *testing.T) { + h := newHarness(t) + h.put("a.md", p1) + res := h.after(time.Hour).put("a.md", "---\nrevision: [broken\n---\n"+p2) + h.expect(res, Silent, 0) + if len(res.Warnings) == 0 { + t.Fatal("want a YAML warning") + } +} + +func TestOriginalDate(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + h.put("2026-06-21-bem-vindo.md", "x") + h.put("dated.md", "---\ndate: 2026-03-02\n---\nx") + h.put("instant.md", "---\ndate: 2026-03-02T01:30:00Z\n---\nx") + h.put("plain.md", "x") + want := map[string]string{ + "2026-06-21-bem-vindo.md": "2026-06-21", + "dated.md": "2026-03-02", + "instant.md": "2026-03-01", // 01:30 UTC is still March 1 in São Paulo + "plain.md": "2026-10-01", + } + for path, date := range want { + p, err := h.e.Store.PageByPath(ctx, "folder", path) + if err != nil { + t.Fatal(err) + } + if p.OriginalDate != date { + t.Errorf("%s: original date %s, want %s", path, p.OriginalDate, date) + } + } + p, _ := h.e.Store.PageByPath(ctx, "folder", "2026-06-21-bem-vindo.md") + if p.Slug != "bem-vindo" { + t.Fatalf("slug %s", p.Slug) + } +} + +func TestReconcile(t *testing.T) { + h := newHarness(t) + ctx := context.Background() + h.put("a.md", p1) + h.put("b.md", p2) + present := map[string]bool{"a.md": true} + + res, _ := h.e.Reconcile(ctx, present, false) + if len(res) != 0 { + t.Fatalf("one miss should not remove: %+v", res) + } + res, _ = h.e.Reconcile(ctx, present, false) + if len(res) != 1 || res[0].Action != Removed || res[0].Path != "b.md" { + t.Fatalf("second miss should remove b.md: %+v", res) + } + if n, _ := h.e.ActivePages(ctx); n != 1 { + t.Fatalf("active %d", n) + } + + res, _ = h.e.Reconcile(ctx, map[string]bool{"a.md": true, "b.md": true}, false) + if len(res) != 1 || !res[0].Restored { + t.Fatalf("b.md should come back: %+v", res) + } + h.after(time.Hour) + if r := h.put("b.md", p2); r.Action != Unchanged || r.Restored { + t.Fatalf("restored page with same text: %+v", r) + } +} + +func TestDryRunWritesNothing(t *testing.T) { + h := newHarness(t) + res, err := h.e.Ingest(context.Background(), Observation{Path: "a.md", Raw: []byte(p1)}, true) + if err != nil || res.Action != Baseline { + t.Fatalf("%+v %v", res, err) + } + if n, _ := h.e.ActivePages(context.Background()); n != 0 { + t.Fatal("dry run wrote a page") + } +}
@@ -0,0 +1,157 @@+package engine + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "flag" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "testing" + "time" + + "git.pablomurad.com/pablo/patchwork/internal/diff" +) + +var update = flag.Bool("update", false, "rewrite testdata/cases/*/expected.json") + +// A golden case is a directory of versions v1.md, v2.md, ... ingested in +// order as the same file, one hour apart. An optional steps.txt changes the +// schedule: one line per step, "<file> [+duration]". +// +// expected.json records every step's action and each published revision's +// excerpt, so a change in the publish rules or the diff shows up as a +// reviewable change in the golden files. +type goldenStep struct { + File string `json:"file"` + Action Action `json:"action"` + Number int `json:"number,omitempty"` + Warnings []string `json:"warnings,omitempty"` +} + +type goldenRevision struct { + Number int `json:"number"` + Note string `json:"note"` + Kind string `json:"kind,omitempty"` + Maturity string `json:"maturity,omitempty"` + Stats diff.Stats `json:"stats"` + Meta []diff.MetaChange `json:"meta_changes,omitempty"` + Excerpt []diff.Op `json:"excerpt"` + Truncated bool `json:"truncated,omitempty"` +} + +type golden struct { + Steps []goldenStep `json:"steps"` + Revisions []goldenRevision `json:"revisions"` +} + +var versionFile = regexp.MustCompile(`^v(\d+)\.md$`) + +func TestGolden(t *testing.T) { + dirs, err := filepath.Glob("testdata/cases/*") + if err != nil { + t.Fatal(err) + } + if len(dirs) == 0 { + t.Fatal("no golden cases") + } + for _, dir := range dirs { + t.Run(filepath.Base(dir), func(t *testing.T) { runGolden(t, dir) }) + } +} + +type step struct { + file string + after time.Duration +} + +func schedule(t *testing.T, dir string) []step { + if f, err := os.Open(filepath.Join(dir, "steps.txt")); err == nil { + defer f.Close() + var steps []step + sc := bufio.NewScanner(f) + for sc.Scan() { + fields := strings.Fields(sc.Text()) + if len(fields) == 0 || strings.HasPrefix(fields[0], "#") { + continue + } + s := step{file: fields[0], after: time.Hour} + if len(fields) > 1 { + d, err := time.ParseDuration(strings.TrimPrefix(fields[1], "+")) + if err != nil { + t.Fatal(err) + } + s.after = d + } + steps = append(steps, s) + } + return steps + } + entries, _ := os.ReadDir(dir) + var nums []int + for _, e := range entries { + if m := versionFile.FindStringSubmatch(e.Name()); m != nil { + n, _ := strconv.Atoi(m[1]) + nums = append(nums, n) + } + } + sort.Ints(nums) + var steps []step + for _, n := range nums { + steps = append(steps, step{file: "v" + strconv.Itoa(n) + ".md", after: time.Hour}) + } + return steps +} + +func runGolden(t *testing.T, dir string) { + h := newHarness(t) + var got golden + for _, s := range schedule(t, dir) { + raw, err := os.ReadFile(filepath.Join(dir, s.file)) + if err != nil { + t.Fatal(err) + } + h.after(s.after) + res, err := h.e.Ingest(context.Background(), Observation{Path: "idea.md", Raw: raw}, false) + if err != nil { + t.Fatal(err) + } + got.Steps = append(got.Steps, goldenStep{File: s.file, Action: res.Action, Number: res.Number, Warnings: res.Warnings}) + } + for _, r := range h.revisions("idea.md") { + if r.Baseline() { + continue + } + f := h.diffOf(r) + ex, cut := diff.Excerpt(f.Ops, 1, h.e.Cfg.Feeds.ExcerptBlocks) + got.Revisions = append(got.Revisions, goldenRevision{ + Number: r.Number, Note: r.Note, Kind: r.Kind, Maturity: r.Maturity, + Stats: f.Stats, Meta: f.Meta, Excerpt: ex, Truncated: cut, + }) + } + + out, err := json.MarshalIndent(got, "", " ") + if err != nil { + t.Fatal(err) + } + out = append(out, '\n') + path := filepath.Join(dir, "expected.json") + if *update { + if err := os.WriteFile(path, out, 0o644); err != nil { + t.Fatal(err) + } + return + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatalf("%v (run go test ./internal/engine -run TestGolden -update)", err) + } + if !bytes.Equal(bytes.ReplaceAll(want, []byte("\r\n"), []byte("\n")), out) { + t.Errorf("%s differs; got:\n%s", path, out) + } +}
@@ -0,0 +1,62 @@+package engine + +import ( + "context" + + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +// missingScansToRemove: a file must be absent from this many consecutive +// scans before its page leaves the feeds. One miss is often an upload that +// replaces the file by deleting it first. +const missingScansToRemove = 2 + +// Reconcile is called after a full scan with every path that exists. Pages +// whose file is absent move toward removal; present pages that were missing +// come back. +func (e *Engine) Reconcile(ctx context.Context, present map[string]bool, dryRun bool) ([]Result, error) { + var out []Result + err := e.Store.Tx(ctx, dryRun, func(q *store.Queries) error { + pages, err := q.Pages(ctx, e.Source, true) + if err != nil { + return err + } + now := e.Now() + for i := range pages { + p := &pages[i] + if present[p.Path] { + if p.RemovedAt == nil && p.MissingScans == 0 { + continue + } + restored := p.RemovedAt != nil + p.RemovedAt, p.MissingScans = nil, 0 + if err := q.UpdatePage(ctx, p); err != nil { + return err + } + if restored { + out = append(out, Result{Path: p.Path, Slug: p.Slug, Action: Unchanged, Restored: true}) + } + continue + } + if p.RemovedAt != nil { + continue + } + p.MissingScans++ + if p.MissingScans >= missingScansToRemove { + p.RemovedAt = &now + out = append(out, Result{Path: p.Path, Slug: p.Slug, Action: Removed}) + } + if err := q.UpdatePage(ctx, p); err != nil { + return err + } + } + return nil + }) + return out, err +} + +// ActivePages counts pages that haven't been removed. +func (e *Engine) ActivePages(ctx context.Context) (int, error) { + pages, err := e.Store.Pages(ctx, e.Source, false) + return len(pages), err +}
@@ -0,0 +1,52 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "The watcher needs the config flag.", + "kind": "correction", + "stats": { + "words_added": 4, + "words_removed": 0, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "eq", + "kind": "paragraph", + "text": "Run the watcher:" + }, + { + "op": "modify", + "kind": "code", + "segments": [ + { + "t": "eq", + "s": "patchwork watch" + }, + { + "t": "ins", + "s": " --config site/patchwork.yaml" + } + ] + }, + { + "op": "eq", + "kind": "paragraph", + "text": "That is all." + } + ] + } + ] +}
@@ -0,0 +1,7 @@+Run the watcher: + +```sh +patchwork watch +``` + +That is all.
@@ -0,0 +1,11 @@+--- +revision: The watcher needs the config flag. +revision_kind: correction +--- +Run the watcher: + +```sh +patchwork watch --config site/patchwork.yaml +``` + +That is all.
@@ -0,0 +1,14 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "unchanged" + } + ], + "revisions": null +}
@@ -0,0 +1,3 @@+Ação e reação. + +Second line.
@@ -0,0 +1,4 @@+Ação e reação. + +Second line. +
@@ -0,0 +1,32 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "First words.", + "stats": { + "words_added": 3, + "words_removed": 0, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "insert", + "kind": "paragraph", + "text": "Something at last." + } + ] + } + ] +}
@@ -0,0 +1,4 @@+--- +revision: First words. +--- +Something at last.
@@ -0,0 +1,52 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + }, + { + "file": "v3.md", + "action": "folded", + "number": 2 + }, + { + "file": "v4.md", + "action": "silent" + } + ], + "revisions": [ + { + "number": 2, + "note": "Changed my mind.", + "kind": "position", + "stats": { + "words_added": 12, + "words_removed": 9, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "del", + "s": "The hybrid model is the worst of both worlds." + }, + { + "t": "ins", + "s": "The hybrid model works when the team agrees on fixed office days." + } + ], + "replaced": true + } + ] + } + ] +}
@@ -0,0 +1,5 @@+# v3 lands 5 minutes after the publish (folded); v4 an hour later (silent) +v1.md +v2.md +v3.md +5m +v4.md +1h
@@ -0,0 +1,1 @@+The hybrid model is the worst of both worlds.
@@ -0,0 +1,5 @@+--- +revision: Changed my mind. +revision_kind: position +--- +The hybrid model wroks when the team agrees on fixed office days.
@@ -0,0 +1,5 @@+--- +revision: Changed my mind. +revision_kind: position +--- +The hybrid model works when the team agrees on fixed office days.
@@ -0,0 +1,5 @@+--- +revision: Changed my mind. +revision_kind: position +--- +The hybrid model works when the team agrees on fixed office days. Mostly.
@@ -0,0 +1,56 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "Better section name.", + "kind": "correction", + "stats": { + "words_added": 4, + "words_removed": 0, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "gap", + "count": 1 + }, + { + "op": "eq", + "kind": "paragraph", + "text": "Notes on gardens that only open after dark." + }, + { + "op": "modify", + "kind": "heading", + "segments": [ + { + "t": "eq", + "s": "Flowers" + }, + { + "t": "ins", + "s": " that open at dusk" + } + ] + }, + { + "op": "eq", + "kind": "paragraph", + "text": "Moonflowers, evening primrose, four o'clocks." + } + ] + } + ] +}
@@ -0,0 +1,7 @@+# Night gardens + +Notes on gardens that only open after dark. + +## Flowers + +Moonflowers, evening primrose, four o'clocks.
@@ -0,0 +1,11 @@+--- +revision: Better section name. +revision_kind: correction +--- +# Night gardens + +Notes on gardens that only open after dark. + +## Flowers that open at dusk + +Moonflowers, evening primrose, four o'clocks.
@@ -0,0 +1,17 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "silent", + "warnings": [ + "new revision note but no visible change since the first version; nothing published" + ] + } + ], + "revisions": null +}
@@ -0,0 +1,5 @@+An essay. + +<!-- patchwork:ignore --> +Last checked: 2026-10-01 +<!-- /patchwork:ignore -->
@@ -0,0 +1,8 @@+--- +revision: Checked the links. +--- +An essay. + +<!-- patchwork:ignore --> +Last checked: 2026-10-20 +<!-- /patchwork:ignore -->
@@ -0,0 +1,14 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "silent" + } + ], + "revisions": null +}
@@ -0,0 +1,4 @@+--- +flowerColor: "#ffffff" +--- +A white flower.
@@ -0,0 +1,4 @@+--- +flowerColor: "#ffd6e8" +--- +A white flower.
@@ -0,0 +1,13 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1, + "warnings": [ + "the file is not valid UTF-8" + ] + } + ], + "revisions": null +}
@@ -0,0 +1,1 @@+Bytes ¤ÿ here.
@@ -0,0 +1,47 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "Added the thermos.", + "kind": "expansion", + "stats": { + "words_added": 4, + "words_removed": 0, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "gap", + "count": 2 + }, + { + "op": "eq", + "kind": "list_item", + "text": "a notebook" + }, + { + "op": "insert", + "kind": "list_item", + "text": "a thermos of coffee" + }, + { + "op": "eq", + "kind": "list_item", + "text": "patience" + } + ] + } + ] +}
@@ -0,0 +1,8 @@+--- +title: Garden tools +--- +What I use at night: + +- a red lamp +- a notebook +- patience
@@ -0,0 +1,11 @@+--- +title: Garden tools +revision: Added the thermos. +revision_kind: expansion +--- +What I use at night: + +- a red lamp +- a notebook +- a thermos of coffee +- patience
@@ -0,0 +1,160 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "A pass over every paragraph.", + "stats": { + "words_added": 10, + "words_removed": 10, + "blocks_changed": 10 + }, + "excerpt": [ + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "Paragraph 1 talks about the garden at " + }, + { + "t": "del", + "s": "night" + }, + { + "t": "ins", + "s": "dusk" + }, + { + "t": "eq", + "s": " and what grows there." + } + ] + }, + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "Paragraph 2 talks about the garden at " + }, + { + "t": "del", + "s": "night" + }, + { + "t": "ins", + "s": "dusk" + }, + { + "t": "eq", + "s": " and what grows there." + } + ] + }, + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "Paragraph 3 talks about the garden at " + }, + { + "t": "del", + "s": "night" + }, + { + "t": "ins", + "s": "dusk" + }, + { + "t": "eq", + "s": " and what grows there." + } + ] + }, + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "Paragraph 4 talks about the garden at " + }, + { + "t": "del", + "s": "night" + }, + { + "t": "ins", + "s": "dusk" + }, + { + "t": "eq", + "s": " and what grows there." + } + ] + }, + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "Paragraph 5 talks about the garden at " + }, + { + "t": "del", + "s": "night" + }, + { + "t": "ins", + "s": "dusk" + }, + { + "t": "eq", + "s": " and what grows there." + } + ] + }, + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "Paragraph 6 talks about the garden at " + }, + { + "t": "del", + "s": "night" + }, + { + "t": "ins", + "s": "dusk" + }, + { + "t": "eq", + "s": " and what grows there." + } + ] + } + ], + "truncated": true + } + ] +}
@@ -0,0 +1,20 @@+Paragraph 1 talks about the garden at night and what grows there. + +Paragraph 2 talks about the garden at night and what grows there. + +Paragraph 3 talks about the garden at night and what grows there. + +Paragraph 4 talks about the garden at night and what grows there. + +Paragraph 5 talks about the garden at night and what grows there. + +Paragraph 6 talks about the garden at night and what grows there. + +Paragraph 7 talks about the garden at night and what grows there. + +Paragraph 8 talks about the garden at night and what grows there. + +Paragraph 9 talks about the garden at night and what grows there. + +Paragraph 10 talks about the garden at night and what grows there. +
@@ -0,0 +1,23 @@+--- +revision: A pass over every paragraph. +--- +Paragraph 1 talks about the garden at dusk and what grows there. + +Paragraph 2 talks about the garden at dusk and what grows there. + +Paragraph 3 talks about the garden at dusk and what grows there. + +Paragraph 4 talks about the garden at dusk and what grows there. + +Paragraph 5 talks about the garden at dusk and what grows there. + +Paragraph 6 talks about the garden at dusk and what grows there. + +Paragraph 7 talks about the garden at dusk and what grows there. + +Paragraph 8 talks about the garden at dusk and what grows there. + +Paragraph 9 talks about the garden at dusk and what grows there. + +Paragraph 10 talks about the garden at dusk and what grows there. +
@@ -0,0 +1,52 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "silent", + "warnings": [ + "frontmatter is not valid YAML, so no revision note is read: yaml: line 1: did not find expected ',' or ']'" + ] + }, + { + "file": "v3.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "Fixed the frontmatter.", + "stats": { + "words_added": 1, + "words_removed": 0, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "modify", + "kind": "paragraph", + "segments": [ + { + "t": "eq", + "s": "A calm page" + }, + { + "t": "ins", + "s": ", edited" + }, + { + "t": "eq", + "s": "." + } + ] + } + ] + } + ] +}
@@ -0,0 +1,1 @@+A calm page.
@@ -0,0 +1,4 @@+--- +revision: [oops +--- +A calm page, edited.
@@ -0,0 +1,4 @@+--- +revision: "Fixed the frontmatter." +--- +A calm page, edited.
@@ -0,0 +1,38 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "silent" + }, + { + "file": "v3.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "It bloomed.", + "maturity": "bloom", + "stats": { + "words_added": 0, + "words_removed": 0, + "blocks_changed": 0 + }, + "meta_changes": [ + { + "key": "maturity", + "old": "seed", + "new": "bloom" + } + ], + "excerpt": null + } + ] +}
@@ -0,0 +1,4 @@+--- +maturity: seed +--- +An idea.
@@ -0,0 +1,4 @@+--- +maturity: sprout +--- +An idea.
@@ -0,0 +1,5 @@+--- +maturity: bloom +revision: It bloomed. +--- +An idea.
@@ -0,0 +1,44 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "Moved the ending to the top.", + "kind": "reorganization", + "stats": { + "words_added": 0, + "words_removed": 0, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "move", + "kind": "paragraph", + "text": "Finally, write down what bloomed.", + "from": 2, + "to": 0 + }, + { + "op": "eq", + "kind": "paragraph", + "text": "First, water the seeds." + }, + { + "op": "gap", + "count": 1 + } + ] + } + ] +}
@@ -0,0 +1,5 @@+First, water the seeds. + +Then wait for the moon. + +Finally, write down what bloomed.
@@ -0,0 +1,9 @@+--- +revision: Moved the ending to the top. +revision_kind: reorganization +--- +Finally, write down what bloomed. + +First, water the seeds. + +Then wait for the moon.
@@ -0,0 +1,14 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "silent" + } + ], + "revisions": null +}
@@ -0,0 +1,4 @@+--- +revision: Written before Patchwork existed. +--- +Old page.
@@ -0,0 +1,4 @@+--- +revision: Written before Patchwork existed. +--- +Old page, quietly edited.
@@ -0,0 +1,14 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "silent" + } + ], + "revisions": null +}
@@ -0,0 +1,3 @@+# Jardim de agentes autônomos + +E se cada protótipo fosse cuidado por um pequeno agente autônomo?
@@ -0,0 +1,3 @@+# Jardim de agentes autônomos + +E se cada rascunho fosse cuidado por um pequeno agente autônomo?
@@ -0,0 +1,49 @@+{ + "steps": [ + { + "file": "v1.md", + "action": "baseline", + "number": 1 + }, + { + "file": "v2.md", + "action": "published", + "number": 2 + } + ], + "revisions": [ + { + "number": 2, + "note": "Measured again.", + "stats": { + "words_added": 2, + "words_removed": 2, + "blocks_changed": 1 + }, + "excerpt": [ + { + "op": "modify", + "kind": "table", + "segments": [ + { + "t": "eq", + "s": "plant | opens\nmoonflower | " + }, + { + "t": "del", + "s": "21:00" + }, + { + "t": "ins", + "s": "20:40" + }, + { + "t": "eq", + "s": "\nprimrose | 19:30" + } + ] + } + ] + } + ] +}
@@ -0,0 +1,4 @@+| plant | opens | +|---|---| +| moonflower | 21:00 | +| primrose | 19:30 |
@@ -0,0 +1,7 @@+--- +revision: Measured again. +--- +| plant | opens | +|---|---| +| moonflower | 20:40 | +| primrose | 19:30 |
@@ -0,0 +1,138 @@+// Package slug turns a content file's relative path into the slug the host +// site uses in its URLs. Each rule must match the host byte for byte, so every +// rule is pinned by a parity table generated from the host's own code. +package slug + +import ( + "fmt" + "path" + "strings" + "time" + + "golang.org/x/text/unicode/norm" +) + +// Rule computes a slug from a '/'-separated relative path. +type Rule func(relPath string) string + +// Lookup returns the named rule. +func Lookup(name string) (Rule, error) { + switch name { + case "datestrip-slugify": + return DatestripSlugify, nil + case "path": + return Path, nil + } + return nil, fmt.Errorf("unknown slug rule %q (want datestrip-slugify or path)", name) +} + +// Slugify mirrors the JavaScript chain +// +// value.normalize('NFD').replace(/[̀-ͯ]/g, '').toLowerCase() +// .replace(/[^a-z0-9]+/g, '-').replace(/^-+|-+$/g, '') +// +// Lowercasing can run after the ASCII filter because, once NFD has split +// compatibility singletons like the Kelvin sign, the only runes that +// lowercase into [a-z] are ASCII capitals. +func Slugify(s string) string { + var b strings.Builder + pendingDash := false + for _, r := range norm.NFD.String(s) { + if r >= 0x300 && r <= 0x36f { + continue + } + if r >= 'A' && r <= 'Z' { + r += 'a' - 'A' + } + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + if pendingDash && b.Len() > 0 { + b.WriteByte('-') + } + pendingDash = false + b.WriteRune(r) + continue + } + pendingDash = true + } + return b.String() +} + +// DatestripSlugify is the moonflowersociety.org rule: drop the extension and +// a leading YYYY-MM-DD date, slugify, and fall back to the whole base name +// when nothing is left. +func DatestripSlugify(relPath string) string { + base := path.Base(relPath) + base = base[:len(base)-len(ext(base))] + if s := Slugify(stripDatePrefix(base)); s != "" { + return s + } + return Slugify(base) +} + +// Path keeps the directory structure: every segment is slugified and the +// extension is dropped from the last one. +func Path(relPath string) string { + segs := strings.Split(relPath, "/") + last := segs[len(segs)-1] + segs[len(segs)-1] = last[:len(last)-len(ext(last))] + out := segs[:0] + for _, s := range segs { + if s = Slugify(s); s != "" { + out = append(out, s) + } + } + return strings.Join(out, "/") +} + +// ext matches Node's path.extname: a leading dot does not start an extension. +func ext(base string) string { + i := strings.LastIndexByte(base, '.') + if i <= 0 { + return "" + } + return base[i:] +} + +// stripDatePrefix matches /^(\d{4})-(\d{2})-(\d{2})[-_\s]+/ with JavaScript's +// definitions of \d (ASCII only) and \s (Unicode spaces plus U+FEFF). +func stripDatePrefix(s string) string { + const shape = "dddd-dd-dd" + if len(s) < len(shape) { + return s + } + for i := 0; i < len(shape); i++ { + c := s[i] + if shape[i] == 'd' && (c < '0' || c > '9') || shape[i] == '-' && c != '-' { + return s + } + } + rest := s[len(shape):] + trimmed := strings.TrimLeftFunc(rest, func(r rune) bool { return r == '-' || r == '_' || isJSSpace(r) }) + if len(trimmed) == len(rest) { + return s + } + return trimmed +} + +// DatePrefix returns the YYYY-MM-DD date a file name starts with, when the +// name follows the datestrip convention and the date is a real one. +func DatePrefix(relPath string) (string, bool) { + base := path.Base(relPath) + base = base[:len(base)-len(ext(base))] + if stripDatePrefix(base) == base { + return "", false + } + d := base[:10] + if _, err := time.Parse("2006-01-02", d); err != nil { + return "", false + } + return d, true +} + +func isJSSpace(r rune) bool { + switch r { + case '\t', '\n', '\v', '\f', '\r', ' ', 0xa0, 0x1680, 0x2028, 0x2029, 0x202f, 0x205f, 0x3000, 0xfeff: + return true + } + return r >= 0x2000 && r <= 0x200a +}
@@ -0,0 +1,67 @@+package slug + +import ( + "encoding/json" + "os" + "testing" +) + +// testdata/parity.json is generated by running moonflower's JavaScript slug +// code over tricky file names; the Go rule must agree on every row. +func TestDatestripSlugifyParity(t *testing.T) { + raw, err := os.ReadFile("testdata/parity.json") + if err != nil { + t.Fatal(err) + } + var rows []struct{ File, Slug string } + if err := json.Unmarshal(raw, &rows); err != nil { + t.Fatal(err) + } + if len(rows) == 0 { + t.Fatal("empty parity table") + } + for _, row := range rows { + if got := DatestripSlugify(row.File); got != row.Slug { + t.Errorf("DatestripSlugify(%q) = %q, want %q", row.File, got, row.Slug) + } + } +} + +func TestPath(t *testing.T) { + cases := map[string]string{ + "notes/Ação.md": "notes/acao", + "a/b/2026-01-01-x.md": "a/b/2026-01-01-x", + "top.md": "top", + "🌙/only-dir-dropped.md": "only-dir-dropped", + } + for in, want := range cases { + if got := Path(in); got != want { + t.Errorf("Path(%q) = %q, want %q", in, got, want) + } + } +} + +func TestDatePrefix(t *testing.T) { + cases := map[string]string{ + "2026-06-21-bem-vindo.md": "2026-06-21", + "2026-06-21 título.md": "2026-06-21", + "2026-13-45-invalid.md": "", + "2026-06-21.md": "", + "jardim-de-agentes-autonomos.md": "", + } + for in, want := range cases { + got, ok := DatePrefix(in) + if got != want || ok != (want != "") { + t.Errorf("DatePrefix(%q) = %q, %v", in, got, ok) + } + } +} + +func TestLookup(t *testing.T) { + if _, err := Lookup("datestrip-slugify"); err != nil { + t.Fatal(err) + } + if _, err := Lookup("nope"); err == nil { + t.Fatal("want error for unknown rule") + } +}
@@ -0,0 +1,122 @@+[ + { + "file": "jardim-de-agentes-autonomos.md", + "slug": "jardim-de-agentes-autonomos" + }, + { + "file": "2026-06-21-bem-vindo-ao-jardim.md", + "slug": "bem-vindo-ao-jardim" + }, + { + "file": "2026-06-21_bem vindo.md", + "slug": "bem-vindo" + }, + { + "file": "2026-06-21 Título com Acentos.md", + "slug": "titulo-com-acentos" + }, + { + "file": "2026-06-21.md", + "slug": "2026-06-21" + }, + { + "file": "2026-06-21-.md", + "slug": "2026-06-21" + }, + { + "file": "Ação e Reação.md", + "slug": "acao-e-reacao" + }, + { + "file": "Straße.md", + "slug": "stra-e" + }, + { + "file": "İstanbul notes.md", + "slug": "istanbul-notes" + }, + { + "file": "ÉCOLE Ñandú.MD", + "slug": "ecole-nandu" + }, + { + "file": "emoji 🌙 moon.md", + "slug": "emoji-moon" + }, + { + "file": "🌙.md", + "slug": "" + }, + { + "file": "___.md", + "slug": "" + }, + { + "file": "v1.2 release.markdown", + "slug": "v1-2-release" + }, + { + "file": "C++ & Go.md", + "slug": "c-go" + }, + { + "file": "naïve café.md", + "slug": "naive-cafe" + }, + { + "file": "Kelvin K test.md", + "slug": "kelvin-k-test" + }, + { + "file": "Ångström.md", + "slug": "angstrom" + }, + { + "file": "Œuvre.md", + "slug": "uvre" + }, + { + "file": "Ελληνικά.md", + "slug": "" + }, + { + "file": "full width space.md", + "slug": "full-width-space" + }, + { + "file": "2026-06-21 nbsp prefix.md", + "slug": "nbsp-prefix" + }, + { + "file": "financial ligature.md", + "slug": "nancial-ligature" + }, + { + "file": "123 fullwidth digits.md", + "slug": "fullwidth-digits" + }, + { + "file": "mixed--dashes---here.md", + "slug": "mixed-dashes-here" + }, + { + "file": "-leading and trailing-.md", + "slug": "leading-and-trailing" + }, + { + "file": "camelCaseName.md", + "slug": "camelcasename" + }, + { + "file": "UPPER.md", + "slug": "upper" + }, + { + "file": "2026-13-45-invalid-date-still-stripped.md", + "slug": "invalid-date-still-stripped" + }, + { + "file": "12-01-2026-not-a-prefix.md", + "slug": "12-01-2026-not-a-prefix" + } +]
@@ -0,0 +1,185 @@+// Package folder watches a directory of content files and feeds changes to +// the engine. +package folder + +import ( + "context" + "errors" + "fmt" + "io/fs" + "log/slog" + "os" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "golang.org/x/text/unicode/norm" + + "git.pablomurad.com/pablo/patchwork/internal/engine" +) + +// File is one content file found by a scan. +type File struct { + Rel string // relative to the root, '/'-separated, NFC + Abs string + Size int64 + Mtime time.Time +} + +// Matcher decides which files are content. +type Matcher struct { + Include []string // "*.md" top level only, "**/*.md" any depth + Exclude []string // matched against the file or directory name +} + +func (m Matcher) excluded(name string) bool { + for _, p := range m.Exclude { + if ok, _ := path.Match(p, name); ok { + return true + } + } + return false +} + +func (m Matcher) included(rel string) bool { + for _, p := range m.Include { + target := rel + if rest, deep := strings.CutPrefix(p, "**/"); deep { + p, target = rest, path.Base(rel) + } + if ok, _ := path.Match(p, target); ok { + return true + } + } + return false +} + +// List walks root and returns the content files, sorted by path. +func List(root string, m Matcher) ([]File, error) { + var out []File + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + if p == root { + return err + } + return nil // an unreadable entry shouldn't stop the scan + } + if p == root { + return nil + } + if m.excluded(d.Name()) { + if d.IsDir() { + return filepath.SkipDir + } + return nil + } + if d.IsDir() || !d.Type().IsRegular() { + return nil + } + rel, err := filepath.Rel(root, p) + if err != nil { + return err + } + rel = norm.NFC.String(filepath.ToSlash(rel)) + if !m.included(rel) { + return nil + } + info, err := d.Info() + if err != nil { + return nil + } + out = append(out, File{Rel: rel, Abs: p, Size: info.Size(), Mtime: info.ModTime()}) + return nil + }) + sort.Slice(out, func(i, j int) bool { return out[i].Rel < out[j].Rel }) + return out, err +} + +// Report summarizes one scan. +type Report struct { + Results []engine.Result + Pending []string // files still being written; scanned again later + Errors []error +} + +// Runner scans a folder and hands changed files to the engine. +type Runner struct { + Root string + Matcher Matcher + Engine *engine.Engine + Log *slog.Logger + // Settle is how long a file's mtime must be in the past before it is + // read, so a half-written upload isn't snapshotted. + Settle time.Duration + Now func() time.Time + // PollOnly skips fsnotify, for mounts where file events never arrive. + PollOnly bool + + seen map[string]File +} + +// NewRunner builds a runner with the usual defaults. +func NewRunner(root string, m Matcher, e *engine.Engine, log *slog.Logger) *Runner { + return &Runner{Root: root, Matcher: m, Engine: e, Log: log, Settle: time.Second, Now: time.Now} +} + +// Scan runs one full pass. Unchanged files (same size and mtime as last +// time) are not read again. +func (r *Runner) Scan(ctx context.Context, dryRun bool) (Report, error) { + var rep Report + files, err := List(r.Root, r.Matcher) + if err != nil { + return rep, fmt.Errorf("scan %s: %w", r.Root, err) + } + if r.seen == nil { + r.seen = map[string]File{} + } + present := map[string]bool{} + now := r.Now() + for _, f := range files { + present[f.Rel] = true + if old, ok := r.seen[f.Rel]; ok && old.Size == f.Size && old.Mtime.Equal(f.Mtime) { + continue + } + if now.Sub(f.Mtime) < r.Settle { + rep.Pending = append(rep.Pending, f.Rel) + continue + } + raw, err := os.ReadFile(f.Abs) + if err != nil { + rep.Errors = append(rep.Errors, fmt.Errorf("%s: %w", f.Rel, err)) + continue + } + res, err := r.Engine.Ingest(ctx, engine.Observation{Path: f.Rel, Raw: raw, Mtime: f.Mtime}, dryRun) + if err != nil { + rep.Errors = append(rep.Errors, fmt.Errorf("%s: %w", f.Rel, err)) + continue + } + if !dryRun { + r.seen[f.Rel] = f + } + rep.Results = append(rep.Results, res) + } + for rel := range r.seen { + if !present[rel] { + delete(r.seen, rel) + } + } + + // A scan that finds nothing while the store still has pages is far more + // likely a dropped mount than a deleted site: don't remove anything. + if len(files) == 0 { + if n, err := r.Engine.ActivePages(ctx); err == nil && n > 0 { + rep.Errors = append(rep.Errors, errors.New("no content files found but the store has active pages; not marking anything removed")) + return rep, nil + } + } + removed, err := r.Engine.Reconcile(ctx, present, dryRun) + if err != nil { + return rep, err + } + rep.Results = append(rep.Results, removed...) + return rep, nil +}
@@ -0,0 +1,175 @@+package folder + +import ( + "context" + "io" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "testing/synctest" + "time" + + "git.pablomurad.com/pablo/patchwork/internal/config" + "git.pablomurad.com/pablo/patchwork/internal/engine" + "git.pablomurad.com/pablo/patchwork/internal/store" +) + +func writeFile(t *testing.T, root, rel, body string, mtime time.Time) { + t.Helper() + p := filepath.Join(root, filepath.FromSlash(rel)) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + if err := os.Chtimes(p, mtime, mtime); err != nil { + t.Fatal(err) + } +} + +func TestListMatchesIncludeAndExclude(t *testing.T) { + root := t.TempDir() + old := time.Now().Add(-time.Hour) + for _, rel := range []string{"a.md", "b.markdown", "notes/deep.md", ".hidden.md", "draft.md~", "upload.md.filepart", ".obsidian/x.md", "image.png"} { + writeFile(t, root, rel, "x", old) + } + m := Matcher{Include: []string{"**/*.md", "**/*.markdown"}, Exclude: []string{".*", "*~", "*.filepart"}} + files, err := List(root, m) + if err != nil { + t.Fatal(err) + } + var got []string + for _, f := range files { + got = append(got, f.Rel) + } + if strings.Join(got, ",") != "a.md,b.markdown,notes/deep.md" { + t.Fatalf("got %v", got) + } + + top := Matcher{Include: []string{"*.md"}, Exclude: []string{".*"}} + files, _ = List(root, top) + if len(files) != 1 || files[0].Rel != "a.md" { + t.Fatalf("top-level only: %+v", files) + } +} + +func newRunner(t *testing.T, root string) *Runner { + t.Helper() + st, err := store.Open(filepath.Join(t.TempDir(), "pw.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { st.Close() }) + e, err := engine.New(st, config.Default()) + if err != nil { + t.Fatal(err) + } + m := Matcher{Include: []string{"**/*.md"}, Exclude: []string{".*"}} + return NewRunner(root, m, e, slog.New(slog.NewTextHandler(io.Discard, nil))) +} + +func actions(rep Report) string { + var out []string + for _, r := range rep.Results { + out = append(out, r.Path+":"+string(r.Action)) + } + return strings.Join(out, ",") +} + +func TestScanFlow(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + r := newRunner(t, root) + past := time.Now().Add(-time.Hour) + + writeFile(t, root, "a.md", "One.", past) + rep, err := r.Scan(ctx, false) + if err != nil || actions(rep) != "a.md:baseline" { + t.Fatalf("first scan %q %v %v", actions(rep), rep.Errors, err) + } + + // Nothing changed on disk: the file isn't even read. + if rep, _ := r.Scan(ctx, false); len(rep.Results) != 0 { + t.Fatalf("unchanged scan %q", actions(rep)) + } + + writeFile(t, root, "a.md", "---\nrevision: Expanded.\n---\nOne. Two.", past.Add(time.Minute)) + if rep, _ := r.Scan(ctx, false); actions(rep) != "a.md:published" { + t.Fatalf("edit scan %q", actions(rep)) + } + + // A file modified a moment ago is still being written. + writeFile(t, root, "b.md", "Half", time.Now()) + rep, _ = r.Scan(ctx, false) + if len(rep.Pending) != 1 || rep.Pending[0] != "b.md" { + t.Fatalf("pending %v", rep.Pending) + } +} + +func TestEmptyFolderDoesNotRemoveEverything(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + r := newRunner(t, root) + writeFile(t, root, "a.md", "One.", time.Now().Add(-time.Hour)) + r.Scan(ctx, false) + os.Remove(filepath.Join(root, "a.md")) + for i := 0; i < 3; i++ { + rep, _ := r.Scan(ctx, false) + if len(rep.Results) != 0 || len(rep.Errors) != 1 { + t.Fatalf("scan %d: %q %v", i, actions(rep), rep.Errors) + } + } +} + +func TestRemovalNeedsTwoScans(t *testing.T) { + ctx := context.Background() + root := t.TempDir() + r := newRunner(t, root) + past := time.Now().Add(-time.Hour) + writeFile(t, root, "a.md", "One.", past) + writeFile(t, root, "b.md", "Two.", past) + r.Scan(ctx, false) + os.Remove(filepath.Join(root, "b.md")) + if rep, _ := r.Scan(ctx, false); len(rep.Results) != 0 { + t.Fatalf("first miss %q", actions(rep)) + } + if rep, _ := r.Scan(ctx, false); actions(rep) != "b.md:removed" { + t.Fatalf("second miss %q", actions(rep)) + } +} + +// TestWatchDebounce runs the watch loop on a fake clock: a burst of events +// produces one scan after the debounce, and polling scans on its own. +func TestWatchDebounce(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + root := t.TempDir() + r := newRunner(t, root) + r.Now = time.Now // synctest's fake clock + // fsnotify blocks in real syscalls, which a synctest bubble can't + // wait on; the loop's timing is what's under test here. + r.PollOnly = true + writeFile(t, root, "a.md", "One.", time.Now().Add(-time.Hour)) + + ctx, cancel := context.WithCancel(context.Background()) + scans := 0 + done := make(chan struct{}) + go func() { + r.Watch(ctx, 3*time.Second, time.Minute, func(Report, error) { scans++ }) + close(done) + }() + synctest.Wait() + if scans != 1 { + t.Fatalf("initial scans %d", scans) + } + time.Sleep(time.Minute) + synctest.Wait() + if scans != 2 { + t.Fatalf("after one poll: %d", scans) + } + cancel() + <-done + }) +}
@@ -0,0 +1,114 @@+package folder + +import ( + "context" + "io/fs" + "os" + "path/filepath" + "time" + + "github.com/fsnotify/fsnotify" +) + +// Watch scans once, then again after file events settle (debounce), on +// every poll tick, and shortly after any scan that left files pending. +// fsnotify is a hint, not a source of truth: it misses events on network +// mounts and some Docker bind mounts, which is why polling always runs. +// onReport is called after every scan. +func (r *Runner) Watch(ctx context.Context, debounce, poll time.Duration, onReport func(Report, error)) error { + events := make(chan struct{}, 1) + if r.PollOnly { + r.Log.Info("file events disabled; polling only") + } else if w, err := fsnotify.NewWatcher(); err != nil { + r.Log.Warn("file events unavailable; polling only", "err", err) + } else { + defer w.Close() + r.addDirs(w) + go r.forward(ctx, w, events) + } + + var tick <-chan time.Time + if poll > 0 { + t := time.NewTicker(poll) + defer t.Stop() + tick = t.C + } + var timer *time.Timer + var fire <-chan time.Time + schedule := func(d time.Duration) { + if timer != nil { + timer.Stop() + } + timer = time.NewTimer(d) + fire = timer.C + } + + scan := func() { + rep, err := r.Scan(ctx, false) + onReport(rep, err) + if len(rep.Pending) > 0 { + schedule(r.Settle + 100*time.Millisecond) + } + } + scan() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-events: + schedule(debounce) + case <-fire: + fire = nil + scan() + case <-tick: + scan() + } + } +} + +func (r *Runner) forward(ctx context.Context, w *fsnotify.Watcher, events chan<- struct{}) { + for { + select { + case <-ctx.Done(): + return + case ev, ok := <-w.Events: + if !ok { + return + } + // New directories need their own watch. + if ev.Has(fsnotify.Create) { + r.addDir(w, ev.Name) + } + select { + case events <- struct{}{}: + default: + } + case err, ok := <-w.Errors: + if !ok { + return + } + r.Log.Warn("file watcher error", "err", err) + } + } +} + +func (r *Runner) addDirs(w *fsnotify.Watcher) { + _ = filepath.WalkDir(r.Root, func(p string, d fs.DirEntry, err error) error { + if err != nil || !d.IsDir() { + return nil + } + if p != r.Root && r.Matcher.excluded(d.Name()) { + return filepath.SkipDir + } + if err := w.Add(p); err != nil { + r.Log.Warn("cannot watch directory", "dir", p, "err", err) + } + return nil + }) +} + +func (r *Runner) addDir(w *fsnotify.Watcher, p string) { + if fi, err := os.Stat(p); err == nil && fi.IsDir() && !r.Matcher.excluded(fi.Name()) { + _ = w.Add(p) + } +}
@@ -0,0 +1,24 @@+package store + +import ( + "context" + "database/sql" + "errors" +) + +// Meta reads a value from the meta table; missing keys return "". +func (q *Queries) Meta(ctx context.Context, key string) (string, error) { + var v string + err := q.q.QueryRowContext(ctx, `SELECT value FROM meta WHERE key = ?`, key).Scan(&v) + if errors.Is(err, sql.ErrNoRows) { + return "", nil + } + return v, err +} + +// SetMeta writes a value to the meta table. +func (q *Queries) SetMeta(ctx context.Context, key, value string) error { + _, err := q.q.ExecContext(ctx, + `INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`, key, value) + return err +}
@@ -0,0 +1,111 @@+package store + +import ( + "context" + "crypto/rand" + "database/sql" + "encoding/base32" + "errors" + "strings" + "time" +) + +// Page is one content file, identified by source and path. +type Page struct { + ID int64 + UID string // random and permanent; feed entry ids are built from it + Source string // "folder" or "git" + Path string // relative, '/'-separated, NFC + Slug string + // OriginalDate is the calendar date the page was first published, + // YYYY-MM-DD in the site's time zone. + OriginalDate string + FirstSeenAt time.Time + RemovedAt *time.Time + MissingScans int +} + +// NewUID returns a random page uid such as "p_k3x9q2m7d4tb". +func NewUID() string { + b := make([]byte, 8) + if _, err := rand.Read(b); err != nil { + panic(err) + } + return "p_" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b))[:12] +} + +const pageCols = `id, uid, source, path, slug, original_date, first_seen_at, removed_at, missing_scans` + +func scanPage(row interface{ Scan(...any) error }) (*Page, error) { + var p Page + var first string + var removed sql.NullString + if err := row.Scan(&p.ID, &p.UID, &p.Source, &p.Path, &p.Slug, &p.OriginalDate, &first, &removed, &p.MissingScans); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + return nil, err + } + var err error + if p.FirstSeenAt, err = parseTime(first); err != nil { + return nil, err + } + if p.RemovedAt, err = parseTimePtr(removed); err != nil { + return nil, err + } + return &p, nil +} + +// PageByPath finds a page, removed or not. +func (q *Queries) PageByPath(ctx context.Context, source, path string) (*Page, error) { + return scanPage(q.q.QueryRowContext(ctx, `SELECT `+pageCols+` FROM pages WHERE source = ? AND path = ?`, source, path)) +} + +// PageBySlug finds the active page with this slug. When two files share a +// slug (doctor reports it), the one seen first wins. +func (q *Queries) PageBySlug(ctx context.Context, slug string) (*Page, error) { + return scanPage(q.q.QueryRowContext(ctx, + `SELECT `+pageCols+` FROM pages WHERE slug = ? AND removed_at IS NULL ORDER BY first_seen_at, id LIMIT 1`, slug)) +} + +// Pages lists pages ordered by path. +func (q *Queries) Pages(ctx context.Context, source string, includeRemoved bool) ([]Page, error) { + query := `SELECT ` + pageCols + ` FROM pages WHERE (? = '' OR source = ?)` + if !includeRemoved { + query += ` AND removed_at IS NULL` + } + rows, err := q.q.QueryContext(ctx, query+` ORDER BY path`, source, source) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Page + for rows.Next() { + p, err := scanPage(rows) + if err != nil { + return nil, err + } + out = append(out, *p) + } + return out, rows.Err() +} + +// InsertPage stores a new page and sets its ID. +func (q *Queries) InsertPage(ctx context.Context, p *Page) error { + res, err := q.q.ExecContext(ctx, + `INSERT INTO pages (uid, source, path, slug, original_date, first_seen_at, removed_at, missing_scans) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + p.UID, p.Source, p.Path, p.Slug, p.OriginalDate, fmtTime(p.FirstSeenAt), fmtTimePtr(p.RemovedAt), p.MissingScans) + if err != nil { + return err + } + p.ID, err = res.LastInsertId() + return err +} + +// UpdatePage saves the mutable fields of a page. +func (q *Queries) UpdatePage(ctx context.Context, p *Page) error { + _, err := q.q.ExecContext(ctx, + `UPDATE pages SET slug = ?, original_date = ?, removed_at = ?, missing_scans = ? WHERE id = ?`, + p.Slug, p.OriginalDate, fmtTimePtr(p.RemovedAt), p.MissingScans, p.ID) + return err +}
@@ -0,0 +1,147 @@+package store + +import ( + "context" + "database/sql" + "errors" + "strings" + "time" +) + +// Revision is a public version of a page. Number 1 is the baseline. +type Revision struct { + ID int64 + PageID int64 + Number int + SnapshotID int64 + Note string + Kind string + Maturity string + PublishedAt time.Time + UpdatedAt time.Time + RetractedAt *time.Time + // DiffJSON caches the diff against the previous public version, encoded + // by diff version DiffVersion; rebuild-diffs regenerates it. + DiffJSON string + DiffVersion int + WordsAdded int + WordsRemoved int +} + +// Baseline reports whether this is the first, unannounced version. +func (r *Revision) Baseline() bool { return r.Number == 1 } + +const revisionCols = `id, page_id, number, snapshot_id, note, kind, maturity, published_at, updated_at, retracted_at, diff_json, diff_version, words_added, words_removed` + +// revisionColsR is revisionCols qualified with the alias r, for joins. +var revisionColsR = "r." + strings.ReplaceAll(revisionCols, ", ", ", r.") + +func scanRevision(row interface{ Scan(...any) error }) (*Revision, error) { + var r Revision + var published, updated string + var retracted sql.NullString + if err := row.Scan(&r.ID, &r.PageID, &r.Number, &r.SnapshotID, &r.Note, &r.Kind, &r.Maturity, &published, &updated, + &retracted, &r.DiffJSON, &r.DiffVersion, &r.WordsAdded, &r.WordsRemoved); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + return nil, err + } + var err error + if r.PublishedAt, err = parseTime(published); err != nil { + return nil, err + } + if r.UpdatedAt, err = parseTime(updated); err != nil { + return nil, err + } + if r.RetractedAt, err = parseTimePtr(retracted); err != nil { + return nil, err + } + return &r, nil +} + +func (q *Queries) revisions(ctx context.Context, query string, args ...any) ([]Revision, error) { + rows, err := q.q.QueryContext(ctx, query, args...) + if err != nil { + return nil, err + } + defer rows.Close() + var out []Revision + for rows.Next() { + r, err := scanRevision(rows) + if err != nil { + return nil, err + } + out = append(out, *r) + } + return out, rows.Err() +} + +// InsertRevision stores a revision and sets its ID. +func (q *Queries) InsertRevision(ctx context.Context, r *Revision) error { + res, err := q.q.ExecContext(ctx, + `INSERT INTO revisions (page_id, number, snapshot_id, note, kind, maturity, published_at, updated_at, retracted_at, diff_json, diff_version, words_added, words_removed) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + r.PageID, r.Number, r.SnapshotID, r.Note, r.Kind, r.Maturity, fmtTime(r.PublishedAt), fmtTime(r.UpdatedAt), + fmtTimePtr(r.RetractedAt), r.DiffJSON, r.DiffVersion, r.WordsAdded, r.WordsRemoved) + if err != nil { + return err + } + r.ID, err = res.LastInsertId() + return err +} + +// UpdateRevision saves every mutable field of a revision. +func (q *Queries) UpdateRevision(ctx context.Context, r *Revision) error { + _, err := q.q.ExecContext(ctx, + `UPDATE revisions SET snapshot_id = ?, note = ?, kind = ?, maturity = ?, updated_at = ?, retracted_at = ?, + diff_json = ?, diff_version = ?, words_added = ?, words_removed = ? WHERE id = ?`, + r.SnapshotID, r.Note, r.Kind, r.Maturity, fmtTime(r.UpdatedAt), fmtTimePtr(r.RetractedAt), + r.DiffJSON, r.DiffVersion, r.WordsAdded, r.WordsRemoved, r.ID) + return err +} + +// LastRevision returns the page's latest revision that isn't retracted. +func (q *Queries) LastRevision(ctx context.Context, pageID int64) (*Revision, error) { + return scanRevision(q.q.QueryRowContext(ctx, + `SELECT `+revisionCols+` FROM revisions WHERE page_id = ? AND retracted_at IS NULL ORDER BY number DESC LIMIT 1`, pageID)) +} + +// RevisionBefore returns the latest revision older than number that isn't +// retracted: the base a revision's diff is computed against. +func (q *Queries) RevisionBefore(ctx context.Context, pageID int64, number int) (*Revision, error) { + return scanRevision(q.q.QueryRowContext(ctx, + `SELECT `+revisionCols+` FROM revisions WHERE page_id = ? AND number < ? AND retracted_at IS NULL ORDER BY number DESC LIMIT 1`, + pageID, number)) +} + +// RevisionByNumber returns one revision, retracted or not. +func (q *Queries) RevisionByNumber(ctx context.Context, pageID int64, number int) (*Revision, error) { + return scanRevision(q.q.QueryRowContext(ctx, + `SELECT `+revisionCols+` FROM revisions WHERE page_id = ? AND number = ?`, pageID, number)) +} + +// MaxRevisionNumber counts retracted revisions too, so numbers never repeat. +func (q *Queries) MaxRevisionNumber(ctx context.Context, pageID int64) (int, error) { + var n int + err := q.q.QueryRowContext(ctx, `SELECT COALESCE(MAX(number), 0) FROM revisions WHERE page_id = ?`, pageID).Scan(&n) + return n, err +} + +// Revisions lists all of a page's revisions by number, retracted included. +func (q *Queries) Revisions(ctx context.Context, pageID int64) ([]Revision, error) { + return q.revisions(ctx, `SELECT `+revisionCols+` FROM revisions WHERE page_id = ? ORDER BY number`, pageID) +} + +// RecentRevisions lists published, non-retracted revisions of active pages, +// newest first. before, when non-zero, pages back through older ones. +func (q *Queries) RecentRevisions(ctx context.Context, limit int, before time.Time) ([]Revision, error) { + cutoff := "" + if !before.IsZero() { + cutoff = fmtTime(before) + } + return q.revisions(ctx, + `SELECT `+revisionColsR+` FROM revisions r JOIN pages p ON p.id = r.page_id + WHERE r.number > 1 AND r.retracted_at IS NULL AND p.removed_at IS NULL AND (? = '' OR r.published_at < ?) + ORDER BY r.published_at DESC, r.id DESC LIMIT ?`, cutoff, cutoff, limit) +}
@@ -0,0 +1,64 @@+package store + +// migrations[i] moves the schema from user_version i to i+1. Never edit a +// migration that has shipped; append a new one. +var migrations = []string{ + ` +CREATE TABLE pages ( + id INTEGER PRIMARY KEY, + uid TEXT NOT NULL UNIQUE, + source TEXT NOT NULL, + path TEXT NOT NULL, + slug TEXT NOT NULL, + original_date TEXT NOT NULL DEFAULT '', + first_seen_at TEXT NOT NULL, + removed_at TEXT, + missing_scans INTEGER NOT NULL DEFAULT 0, + UNIQUE (source, path) +); +CREATE INDEX pages_slug ON pages (slug); + +CREATE TABLE snapshots ( + id INTEGER PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages (id), + seq INTEGER NOT NULL, + observed_at TEXT NOT NULL, + source_mtime TEXT, + source_ref TEXT NOT NULL DEFAULT '', + raw_hash TEXT NOT NULL, + content_hash TEXT NOT NULL, + raw TEXT NOT NULL, + frontmatter_json TEXT NOT NULL, + title TEXT NOT NULL DEFAULT '', + note TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL DEFAULT '', + maturity TEXT NOT NULL DEFAULT '', + UNIQUE (page_id, seq) +); + +CREATE TABLE revisions ( + id INTEGER PRIMARY KEY, + page_id INTEGER NOT NULL REFERENCES pages (id), + number INTEGER NOT NULL, + snapshot_id INTEGER NOT NULL REFERENCES snapshots (id), + note TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL DEFAULT '', + maturity TEXT NOT NULL DEFAULT '', + published_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + retracted_at TEXT, + diff_json TEXT NOT NULL DEFAULT '', + diff_version INTEGER NOT NULL DEFAULT 0, + words_added INTEGER NOT NULL DEFAULT 0, + words_removed INTEGER NOT NULL DEFAULT 0, + UNIQUE (page_id, number) +); +CREATE INDEX revisions_feed ON revisions (published_at DESC) + WHERE number > 1 AND retracted_at IS NULL; + +CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +); +`, +}
@@ -0,0 +1,84 @@+package store + +import ( + "context" + "database/sql" + "errors" + "time" +) + +// Snapshot is one observed state of a page's file. +type Snapshot struct { + ID int64 + PageID int64 + Seq int + ObservedAt time.Time + SourceMtime *time.Time + SourceRef string // git commit in git mode + RawHash string + ContentHash string + Raw string // normalized file text + FrontmatterJSON string + Title string + Note string + Kind string + Maturity string +} + +const snapshotCols = `id, page_id, seq, observed_at, source_mtime, source_ref, raw_hash, content_hash, raw, frontmatter_json, title, note, kind, maturity` + +func scanSnapshot(row interface{ Scan(...any) error }) (*Snapshot, error) { + var s Snapshot + var observed string + var mtime sql.NullString + if err := row.Scan(&s.ID, &s.PageID, &s.Seq, &observed, &mtime, &s.SourceRef, &s.RawHash, &s.ContentHash, + &s.Raw, &s.FrontmatterJSON, &s.Title, &s.Note, &s.Kind, &s.Maturity); err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, ErrNotFound + } + return nil, err + } + var err error + if s.ObservedAt, err = parseTime(observed); err != nil { + return nil, err + } + if s.SourceMtime, err = parseTimePtr(mtime); err != nil { + return nil, err + } + return &s, nil +} + +// InsertSnapshot stores a snapshot as the page's next one and sets ID and Seq. +func (q *Queries) InsertSnapshot(ctx context.Context, s *Snapshot) error { + if err := q.q.QueryRowContext(ctx, `SELECT COALESCE(MAX(seq), 0) + 1 FROM snapshots WHERE page_id = ?`, s.PageID).Scan(&s.Seq); err != nil { + return err + } + res, err := q.q.ExecContext(ctx, + `INSERT INTO snapshots (page_id, seq, observed_at, source_mtime, source_ref, raw_hash, content_hash, raw, frontmatter_json, title, note, kind, maturity) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + s.PageID, s.Seq, fmtTime(s.ObservedAt), fmtTimePtr(s.SourceMtime), s.SourceRef, s.RawHash, s.ContentHash, + s.Raw, s.FrontmatterJSON, s.Title, s.Note, s.Kind, s.Maturity) + if err != nil { + return err + } + s.ID, err = res.LastInsertId() + return err +} + +// Snapshot returns a snapshot by id. +func (q *Queries) Snapshot(ctx context.Context, id int64) (*Snapshot, error) { + return scanSnapshot(q.q.QueryRowContext(ctx, `SELECT `+snapshotCols+` FROM snapshots WHERE id = ?`, id)) +} + +// LastSnapshot returns the page's most recent snapshot. +func (q *Queries) LastSnapshot(ctx context.Context, pageID int64) (*Snapshot, error) { + return scanSnapshot(q.q.QueryRowContext(ctx, + `SELECT `+snapshotCols+` FROM snapshots WHERE page_id = ? ORDER BY seq DESC LIMIT 1`, pageID)) +} + +// SnapshotCount counts a page's snapshots, silent ones included. +func (q *Queries) SnapshotCount(ctx context.Context, pageID int64) (int, error) { + var n int + err := q.q.QueryRowContext(ctx, `SELECT COUNT(*) FROM snapshots WHERE page_id = ?`, pageID).Scan(&n) + return n, err +}
@@ -0,0 +1,145 @@+// Package store keeps pages, snapshots and revisions in SQLite. +// +// Every observed state of a file is a snapshot. A revision is a public +// version: number 1 is the baseline, never announced, and 2.. are the ones +// the author published. Silent snapshots are history only and never leave +// the database. +package store + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "time" + + _ "modernc.org/sqlite" +) + +// Store is a Patchwork database. +type Store struct { + Queries + db *sql.DB +} + +// Open opens or creates the database at path and migrates it. +func Open(path string) (*Store, error) { + if dir := filepath.Dir(path); dir != "" { + if err := os.MkdirAll(dir, 0o755); err != nil { + return nil, err + } + } + q := url.Values{} + for _, p := range []string{"busy_timeout(5000)", "journal_mode(WAL)", "foreign_keys(1)", "synchronous(NORMAL)", "temp_store(MEMORY)"} { + q.Add("_pragma", p) + } + db, err := sql.Open("sqlite", "file:"+filepath.ToSlash(path)+"?"+q.Encode()) + if err != nil { + return nil, err + } + // One connection: SQLite has a single writer anyway, and this keeps + // every read consistent with the last write without lock juggling. + db.SetMaxOpenConns(1) + s := &Store{Queries: Queries{q: db}, db: db} + if err := s.migrate(); err != nil { + db.Close() + return nil, fmt.Errorf("migrate %s: %w", path, err) + } + return s, nil +} + +// Close closes the database. +func (s *Store) Close() error { return s.db.Close() } + +func (s *Store) migrate() error { + var v int + if err := s.db.QueryRow(`PRAGMA user_version`).Scan(&v); err != nil { + return err + } + if v > len(migrations) { + return fmt.Errorf("database schema %d is newer than this binary (%d)", v, len(migrations)) + } + for ; v < len(migrations); v++ { + tx, err := s.db.Begin() + if err != nil { + return err + } + if _, err := tx.Exec(migrations[v]); err != nil { + tx.Rollback() + return err + } + if _, err := tx.Exec(fmt.Sprintf(`PRAGMA user_version = %d`, v+1)); err != nil { + tx.Rollback() + return err + } + if err := tx.Commit(); err != nil { + return err + } + } + return nil +} + +// Tx runs fn in a transaction and commits when it returns nil. With dryRun +// the transaction is always rolled back, so fn can show what it would do. +func (s *Store) Tx(ctx context.Context, dryRun bool, fn func(q *Queries) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := fn(&Queries{q: tx}); err != nil { + tx.Rollback() + return err + } + if dryRun { + return tx.Rollback() + } + return tx.Commit() +} + +// Backup writes a consistent copy of the database to path. +func (s *Store) Backup(ctx context.Context, path string) error { + if _, err := os.Stat(path); err == nil { + return fmt.Errorf("%s already exists", path) + } + _, err := s.db.ExecContext(ctx, `VACUUM INTO ?`, path) + return err +} + +type querier interface { + ExecContext(ctx context.Context, query string, args ...any) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +} + +// Queries run against the database or inside a transaction. +type Queries struct{ q querier } + +// ErrNotFound is returned when a lookup matches nothing. +var ErrNotFound = errors.New("not found") + +const timeLayout = time.RFC3339Nano + +func fmtTime(t time.Time) string { return t.UTC().Format(timeLayout) } + +func fmtTimePtr(t *time.Time) any { + if t == nil { + return nil + } + return fmtTime(*t) +} + +func parseTime(s string) (time.Time, error) { return time.Parse(timeLayout, s) } + +func parseTimePtr(ns sql.NullString) (*time.Time, error) { + if !ns.Valid { + return nil, nil + } + t, err := parseTime(ns.String) + if err != nil { + return nil, err + } + return &t, nil +}
@@ -0,0 +1,121 @@+package store + +import ( + "context" + "errors" + "path/filepath" + "testing" + "time" +) + +func open(t *testing.T) *Store { + t.Helper() + s, err := Open(filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { s.Close() }) + return s +} + +var t0 = time.Date(2026, 10, 21, 22, 14, 3, 0, time.UTC) + +func TestRoundTrip(t *testing.T) { + ctx := context.Background() + s := open(t) + + p := &Page{UID: NewUID(), Source: "folder", Path: "a.md", Slug: "a", OriginalDate: "2026-06-21", FirstSeenAt: t0} + if err := s.InsertPage(ctx, p); err != nil { + t.Fatal(err) + } + snap := &Snapshot{PageID: p.ID, ObservedAt: t0, RawHash: "r1", ContentHash: "c1", Raw: "x", FrontmatterJSON: "{}", Note: "n"} + if err := s.InsertSnapshot(ctx, snap); err != nil { + t.Fatal(err) + } + snap2 := &Snapshot{PageID: p.ID, ObservedAt: t0.Add(time.Hour), RawHash: "r2", ContentHash: "c2", Raw: "y", FrontmatterJSON: "{}"} + if err := s.InsertSnapshot(ctx, snap2); err != nil { + t.Fatal(err) + } + if snap.Seq != 1 || snap2.Seq != 2 { + t.Fatalf("seq %d %d", snap.Seq, snap2.Seq) + } + last, err := s.LastSnapshot(ctx, p.ID) + if err != nil || last.RawHash != "r2" || !last.ObservedAt.Equal(t0.Add(time.Hour)) { + t.Fatalf("last snapshot %+v %v", last, err) + } + + base := &Revision{PageID: p.ID, Number: 1, SnapshotID: snap.ID, PublishedAt: t0, UpdatedAt: t0} + pub := &Revision{PageID: p.ID, Number: 2, SnapshotID: snap2.ID, Note: "changed", Kind: "position", PublishedAt: t0.Add(time.Hour), UpdatedAt: t0.Add(time.Hour)} + for _, r := range []*Revision{base, pub} { + if err := s.InsertRevision(ctx, r); err != nil { + t.Fatal(err) + } + } + lr, err := s.LastRevision(ctx, p.ID) + if err != nil || lr.Number != 2 || lr.Note != "changed" { + t.Fatalf("last revision %+v %v", lr, err) + } + recent, err := s.RecentRevisions(ctx, 10, time.Time{}) + if err != nil || len(recent) != 1 || recent[0].Number != 2 || recent[0].ID != pub.ID { + t.Fatalf("recent %+v %v", recent, err) + } + + now := t0.Add(2 * time.Hour) + pub.RetractedAt = &now + if err := s.UpdateRevision(ctx, pub); err != nil { + t.Fatal(err) + } + if lr, _ := s.LastRevision(ctx, p.ID); lr.Number != 1 { + t.Fatalf("retracted revision still last: %d", lr.Number) + } + if max, _ := s.MaxRevisionNumber(ctx, p.ID); max != 2 { + t.Fatalf("max %d", max) + } + if recent, _ := s.RecentRevisions(ctx, 10, time.Time{}); len(recent) != 0 { + t.Fatal("retracted revisions must leave the feed") + } +} + +func TestLookupsAndTx(t *testing.T) { + ctx := context.Background() + s := open(t) + if _, err := s.PageByPath(ctx, "folder", "missing.md"); !errors.Is(err, ErrNotFound) { + t.Fatalf("want ErrNotFound, got %v", err) + } + + // A dry-run transaction leaves nothing behind. + err := s.Tx(ctx, true, func(q *Queries) error { + return q.InsertPage(ctx, &Page{UID: NewUID(), Source: "folder", Path: "dry.md", Slug: "dry", FirstSeenAt: t0}) + }) + if err != nil { + t.Fatal(err) + } + if pages, _ := s.Pages(ctx, "", true); len(pages) != 0 { + t.Fatal("dry run committed") + } + + if err := s.SetMeta(ctx, "k", "v1"); err != nil { + t.Fatal(err) + } + s.SetMeta(ctx, "k", "v2") + if v, _ := s.Meta(ctx, "k"); v != "v2" { + t.Fatalf("meta %q", v) + } +} + +func TestReopenKeepsSchema(t *testing.T) { + path := filepath.Join(t.TempDir(), "re.db") + s, err := Open(path) + if err != nil { + t.Fatal(err) + } + s.Close() + s, err = Open(path) + if err != nil { + t.Fatal(err) + } + defer s.Close() + if err := s.Backup(context.Background(), filepath.Join(filepath.Dir(path), "copy.db")); err != nil { + t.Fatal(err) + } +}