code.pablomurad.com

patchwork

Publishes the revisions of your pages as a feed, with notes and readable diffs, written in Go

internal/engine/engine.go raw
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
// Package engine applies Patchwork's publish rules: it turns observed file
// states into snapshots, and decides which snapshots become public
// revisions.
package engine

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"path"
	"strings"
	"time"
	"unicode/utf8"

	"git.pablomurad.com/pablo/patchwork/internal/config"
	"git.pablomurad.com/pablo/patchwork/internal/diff"
	"git.pablomurad.com/pablo/patchwork/internal/doc"
	"git.pablomurad.com/pablo/patchwork/internal/slug"
	"git.pablomurad.com/pablo/patchwork/internal/store"
)

// Action is what ingesting one observation did.
type Action string

const (
	Unchanged Action = "unchanged" // same text as the last snapshot
	Baseline  Action = "baseline"  // first sight of the file: version 1
	Silent    Action = "silent"    // stored as history only
	Published Action = "published" // a new public revision
	Amended   Action = "amended"   // the last revision's note was rewritten
	Folded    Action = "folded"    // a quick follow-up joined the last revision
	Removed   Action = "removed"   // the file is gone
)

// Observation is the content of one file at one moment.
type Observation struct {
	Path  string // relative to the source root, '/'-separated
	Raw   []byte
	Mtime time.Time
	// At overrides the observation time (snapshot --at); zero means now.
	At time.Time
}

// Result reports what happened to one file.
type Result struct {
	Path     string
	Slug     string
	Action   Action
	Number   int // the revision created or changed; 1 for a baseline
	Note     string
	Restored bool // the page had been missing or removed
	Warnings []string
}

// Engine ingests observations into a store.
type Engine struct {
	Store  *store.Store
	Cfg    *config.Config
	Source string
	Now    func() time.Time

	slugRule slug.Rule
	opts     doc.Options
	loc      *time.Location
}

// New builds an engine for one source.
func New(st *store.Store, cfg *config.Config) (*Engine, error) {
	rule, err := slug.Lookup(cfg.Source.Slug)
	if err != nil {
		return nil, err
	}
	return &Engine{
		Store: st, Cfg: cfg, Source: cfg.Source.Mode, Now: time.Now,
		slugRule: rule, opts: cfg.DocOptions(), loc: cfg.Location(),
	}, nil
}

// Ingest stores an observation and applies the publish rules. With dryRun
// nothing is written, but the result says what would have happened.
func (e *Engine) Ingest(ctx context.Context, obs Observation, dryRun bool) (Result, error) {
	var res Result
	err := e.Store.Tx(ctx, dryRun, func(q *store.Queries) error {
		var err error
		res, err = e.ingest(ctx, q, obs)
		return err
	})
	return res, err
}

func (e *Engine) ingest(ctx context.Context, q *store.Queries, obs Observation) (Result, error) {
	at := obs.At
	if at.IsZero() {
		at = e.Now()
	}
	d := doc.Parse(obs.Raw, e.opts)
	res := Result{Path: obs.Path, Note: d.Note}
	res.Warnings = e.check(d, obs)
	kind, maturity := e.enum(d, &res)

	page, err := q.PageByPath(ctx, e.Source, obs.Path)
	if errors.Is(err, store.ErrNotFound) {
		return e.baseline(ctx, q, obs, d, at, kind, maturity, res)
	}
	if err != nil {
		return res, err
	}
	res.Slug = page.Slug

	if page.RemovedAt != nil || page.MissingScans > 0 {
		res.Restored = page.RemovedAt != nil
		page.RemovedAt, page.MissingScans = nil, 0
	}
	page.Slug = e.slugRule(obs.Path)
	page.OriginalDate = e.originalDate(d, obs.Path, page.FirstSeenAt)
	res.Slug = page.Slug
	if err := q.UpdatePage(ctx, page); err != nil {
		return res, err
	}

	prev, err := q.LastSnapshot(ctx, page.ID)
	if err != nil {
		return res, err
	}
	if prev.RawHash == d.RawHash {
		res.Action = Unchanged
		return res, nil
	}
	snap := e.snapshot(page.ID, obs, d, at, kind, maturity)
	if err := q.InsertSnapshot(ctx, snap); err != nil {
		return res, err
	}

	last, err := q.LastRevision(ctx, page.ID)
	if err != nil {
		return res, fmt.Errorf("%s has no revision: %w", obs.Path, err)
	}
	lastSnap, err := q.Snapshot(ctx, last.SnapshotID)
	if err != nil {
		return res, err
	}

	noteNew := d.Note != "" && d.Note != last.Note
	sameContent := d.ContentHash == lastSnap.ContentHash && maturity == last.Maturity
	fold := e.Cfg.Publish.FoldWindow.Duration

	switch {
	case noteNew && sameContent && last.Baseline():
		res.Action = Silent
		res.Warnings = append(res.Warnings, "new revision note but no visible change since the first version; nothing published")

	case noteNew && sameContent:
		// Only the note changed: the author is fixing the note.
		last.Note, last.Kind, last.SnapshotID, last.UpdatedAt = d.Note, kind, snap.ID, at
		if err := q.UpdateRevision(ctx, last); err != nil {
			return res, err
		}
		res.Action, res.Number = Amended, last.Number

	case noteNew:
		n, err := q.MaxRevisionNumber(ctx, page.ID)
		if err != nil {
			return res, err
		}
		rev := &store.Revision{
			PageID: page.ID, Number: n + 1, SnapshotID: snap.ID,
			Note: d.Note, Kind: kind, Maturity: maturity, PublishedAt: at, UpdatedAt: at,
		}
		e.setDiff(rev, lastSnap, d)
		if err := q.InsertRevision(ctx, rev); err != nil {
			return res, err
		}
		res.Action, res.Number = Published, rev.Number

	case d.Note != "" && d.Note == last.Note && !last.Baseline() && fold > 0 && at.Sub(last.PublishedAt) <= fold:
		// Same note, shortly after publishing: the typo spotted right
		// after the upload belongs to that revision.
		base, err := q.RevisionBefore(ctx, page.ID, last.Number)
		if err != nil {
			return res, err
		}
		baseSnap, err := q.Snapshot(ctx, base.SnapshotID)
		if err != nil {
			return res, err
		}
		last.SnapshotID, last.Kind, last.Maturity, last.UpdatedAt = snap.ID, kind, maturity, at
		e.setDiff(last, baseSnap, d)
		if err := q.UpdateRevision(ctx, last); err != nil {
			return res, err
		}
		res.Action, res.Number = Folded, last.Number

	default:
		res.Action = Silent
	}
	return res, nil
}

func (e *Engine) baseline(ctx context.Context, q *store.Queries, obs Observation, d *doc.Doc, at time.Time, kind, maturity string, res Result) (Result, error) {
	page := &store.Page{
		UID: store.NewUID(), Source: e.Source, Path: obs.Path,
		Slug: e.slugRule(obs.Path), FirstSeenAt: at,
	}
	page.OriginalDate = e.originalDate(d, obs.Path, at)
	if page.Slug == "" {
		res.Warnings = append(res.Warnings, "the file name produces an empty slug; the page has no URL")
	}
	if err := q.InsertPage(ctx, page); err != nil {
		return res, err
	}
	snap := e.snapshot(page.ID, obs, d, at, kind, maturity)
	if err := q.InsertSnapshot(ctx, snap); err != nil {
		return res, err
	}
	// The baseline keeps the note already in the file, unpublished, so a
	// later upload that still carries it doesn't announce an old note.
	rev := &store.Revision{
		PageID: page.ID, Number: 1, SnapshotID: snap.ID,
		Note: d.Note, Kind: kind, Maturity: maturity, PublishedAt: at, UpdatedAt: at,
	}
	if err := q.InsertRevision(ctx, rev); err != nil {
		return res, err
	}
	res.Slug, res.Action, res.Number = page.Slug, Baseline, 1
	return res, nil
}

func (e *Engine) snapshot(pageID int64, obs Observation, d *doc.Doc, at time.Time, kind, maturity string) *store.Snapshot {
	fm, err := json.Marshal(d.Frontmatter)
	if err != nil {
		fm = []byte("{}")
	}
	s := &store.Snapshot{
		PageID: pageID, ObservedAt: at, RawHash: d.RawHash, ContentHash: d.ContentHash,
		Raw: d.Text, FrontmatterJSON: string(fm), Title: d.Title, Note: d.Note, Kind: kind, Maturity: maturity,
	}
	if !obs.Mtime.IsZero() {
		m := obs.Mtime
		s.SourceMtime = &m
	}
	return s
}

func (e *Engine) setDiff(rev *store.Revision, base *store.Snapshot, cur *doc.Doc) {
	full := diff.Compare(doc.Parse([]byte(base.Raw), e.opts), cur, e.opts.DiffKeys)
	b, err := json.Marshal(full)
	if err != nil {
		panic(err) // the diff types always marshal
	}
	rev.DiffJSON, rev.DiffVersion = string(b), diff.Version
	rev.WordsAdded, rev.WordsRemoved = full.Stats.WordsAdded, full.Stats.WordsRemoved
}

func (e *Engine) check(d *doc.Doc, obs Observation) []string {
	var w []string
	if !utf8.Valid(obs.Raw) {
		w = append(w, "the file is not valid UTF-8")
	}
	if d.FrontmatterErr != nil {
		w = append(w, "frontmatter is not valid YAML, so no revision note is read: "+firstLine(d.FrontmatterErr.Error()))
	}
	return w
}

// enum keeps only declared values of revision_kind and maturity; anything
// else is dropped with a warning rather than published.
func (e *Engine) enum(d *doc.Doc, res *Result) (kind, maturity string) {
	kind, maturity = d.Kind, d.Maturity
	if !doc.Valid(kind, doc.Kinds) {
		res.Warnings = append(res.Warnings, fmt.Sprintf("%s %q is not one of %s; ignored", e.opts.KindKey, kind, strings.Join(doc.Kinds, ", ")))
		kind = ""
	}
	if !doc.Valid(maturity, doc.Maturities) {
		res.Warnings = append(res.Warnings, fmt.Sprintf("%s %q is not one of %s; ignored", e.opts.MaturityKey, maturity, strings.Join(doc.Maturities, ", ")))
		maturity = ""
	}
	return kind, maturity
}

// originalDate is the page's first publication date: frontmatter first,
// then a date at the start of the file name, then the day Patchwork first
// saw the file.
func (e *Engine) originalDate(d *doc.Doc, relPath string, firstSeen time.Time) string {
	switch v := d.Frontmatter[e.Cfg.Frontmatter.DateKey].(type) {
	case time.Time:
		// YAML reads `date: 2026-06-21` as midnight UTC; that is a calendar
		// date, not an instant, so it must not shift into the site's zone.
		if v.Hour() == 0 && v.Minute() == 0 && v.Second() == 0 && v.Location() == time.UTC {
			return v.Format("2006-01-02")
		}
		return v.In(e.loc).Format("2006-01-02")
	case string:
		s := strings.TrimSpace(v)
		if t, err := time.Parse("2006-01-02", s); err == nil {
			return t.Format("2006-01-02")
		}
		if t, err := time.Parse(time.RFC3339, s); err == nil {
			return t.In(e.loc).Format("2006-01-02")
		}
	}
	if date, ok := slug.DatePrefix(path.Base(relPath)); ok {
		return date
	}
	return firstSeen.In(e.loc).Format("2006-01-02")
}

func firstLine(s string) string {
	line, _, _ := strings.Cut(s, "\n")
	return line
}