1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327 |
// Package engine applies Patchwork's publish rules: it turns observed file
// states into snapshots, and decides which snapshots become public
// revisions.
package engine
import (
"context"
"encoding/json"
"errors"
"fmt"
"path"
"strings"
"time"
"unicode/utf8"
"git.pablomurad.com/pablo/patchwork/internal/config"
"git.pablomurad.com/pablo/patchwork/internal/diff"
"git.pablomurad.com/pablo/patchwork/internal/doc"
"git.pablomurad.com/pablo/patchwork/internal/slug"
"git.pablomurad.com/pablo/patchwork/internal/store"
)
// Action is what ingesting one observation did.
type Action string
const (
Unchanged Action = "unchanged" // same text as the last snapshot
Baseline Action = "baseline" // first sight of the file: version 1
Silent Action = "silent" // stored as history only
Published Action = "published" // a new public revision
Amended Action = "amended" // the last revision's note was rewritten
Folded Action = "folded" // a quick follow-up joined the last revision
Removed Action = "removed" // the file is gone
)
// Observation is the content of one file at one moment.
type Observation struct {
Path string // relative to the source root, '/'-separated
Raw []byte
Mtime time.Time
// At overrides the observation time (snapshot --at); zero means now.
At time.Time
}
// Result reports what happened to one file.
type Result struct {
Path string
Slug string
Action Action
Number int // the revision created or changed; 1 for a baseline
Note string
Restored bool // the page had been missing or removed
Warnings []string
}
// Engine ingests observations into a store.
type Engine struct {
Store *store.Store
Cfg *config.Config
Source string
Now func() time.Time
// NewUID makes page uids; tests replace it to get stable feed ids.
NewUID func() string
slugRule slug.Rule
opts doc.Options
loc *time.Location
}
// New builds an engine for one source.
func New(st *store.Store, cfg *config.Config) (*Engine, error) {
rule, err := slug.Lookup(cfg.Source.Slug)
if err != nil {
return nil, err
}
return &Engine{
Store: st, Cfg: cfg, Source: cfg.Source.Mode, Now: time.Now, NewUID: store.NewUID,
slugRule: rule, opts: cfg.DocOptions(), loc: cfg.Location(),
}, nil
}
// Ingest stores an observation and applies the publish rules. With dryRun
// nothing is written, but the result says what would have happened.
func (e *Engine) Ingest(ctx context.Context, obs Observation, dryRun bool) (Result, error) {
var res Result
err := e.Store.Tx(ctx, dryRun, func(q *store.Queries) error {
var err error
res, err = e.ingest(ctx, q, obs)
return err
})
return res, err
}
func (e *Engine) ingest(ctx context.Context, q *store.Queries, obs Observation) (Result, error) {
at := obs.At
if at.IsZero() {
at = e.Now()
}
d := doc.Parse(obs.Raw, e.opts)
res := Result{Path: obs.Path, Note: d.Note}
res.Warnings = e.check(d, obs)
kind, maturity := e.enum(d, &res)
page, err := q.PageByPath(ctx, e.Source, obs.Path)
if errors.Is(err, store.ErrNotFound) {
return e.baseline(ctx, q, obs, d, at, kind, maturity, res)
}
if err != nil {
return res, err
}
res.Slug = page.Slug
if page.RemovedAt != nil || page.MissingScans > 0 {
res.Restored = page.RemovedAt != nil
page.RemovedAt, page.MissingScans = nil, 0
}
page.Slug = e.slugRule(obs.Path)
// The fallback stays whatever the baseline settled on: later uploads
// change the mtime, not the day the page was planted.
page.OriginalDate = e.originalDate(d, obs.Path, page.OriginalDate)
res.Slug = page.Slug
if err := q.UpdatePage(ctx, page); err != nil {
return res, err
}
prev, err := q.LastSnapshot(ctx, page.ID)
if err != nil {
return res, err
}
if prev.RawHash == d.RawHash {
res.Action = Unchanged
return res, nil
}
snap := e.snapshot(page.ID, obs, d, at, kind, maturity)
if err := q.InsertSnapshot(ctx, snap); err != nil {
return res, err
}
last, err := q.LastRevision(ctx, page.ID)
if err != nil {
return res, fmt.Errorf("%s has no revision: %w", obs.Path, err)
}
lastSnap, err := q.Snapshot(ctx, last.SnapshotID)
if err != nil {
return res, err
}
noteNew := d.Note != "" && d.Note != last.Note
sameContent := d.ContentHash == lastSnap.ContentHash && maturity == last.Maturity
fold := e.Cfg.Publish.FoldWindow.Duration
switch {
case noteNew && sameContent && last.Baseline():
res.Action = Silent
res.Warnings = append(res.Warnings, "new revision note but no visible change since the first version; nothing published")
case noteNew && sameContent:
// Only the note changed: the author is fixing the note.
last.Note, last.Kind, last.SnapshotID, last.UpdatedAt = d.Note, kind, snap.ID, at
if err := q.UpdateRevision(ctx, last); err != nil {
return res, err
}
res.Action, res.Number = Amended, last.Number
case noteNew:
n, err := q.MaxRevisionNumber(ctx, page.ID)
if err != nil {
return res, err
}
rev := &store.Revision{
PageID: page.ID, Number: n + 1, SnapshotID: snap.ID,
Note: d.Note, Kind: kind, Maturity: maturity, PublishedAt: at, UpdatedAt: at,
}
e.setDiff(rev, lastSnap, d)
if err := q.InsertRevision(ctx, rev); err != nil {
return res, err
}
res.Action, res.Number = Published, rev.Number
case d.Note != "" && d.Note == last.Note && !last.Baseline() && fold > 0 && at.Sub(last.PublishedAt) <= fold:
// Same note, shortly after publishing: the typo spotted right
// after the upload belongs to that revision.
base, err := q.RevisionBefore(ctx, page.ID, last.Number)
if err != nil {
return res, err
}
baseSnap, err := q.Snapshot(ctx, base.SnapshotID)
if err != nil {
return res, err
}
last.SnapshotID, last.Kind, last.Maturity, last.UpdatedAt = snap.ID, kind, maturity, at
e.setDiff(last, baseSnap, d)
if err := q.UpdateRevision(ctx, last); err != nil {
return res, err
}
res.Action, res.Number = Folded, last.Number
default:
res.Action = Silent
}
return res, nil
}
func (e *Engine) baseline(ctx context.Context, q *store.Queries, obs Observation, d *doc.Doc, at time.Time, kind, maturity string, res Result) (Result, error) {
page := &store.Page{
UID: e.NewUID(), Source: e.Source, Path: obs.Path,
Slug: e.slugRule(obs.Path), FirstSeenAt: at,
}
page.OriginalDate = e.originalDate(d, obs.Path, e.baselineDate(obs, at))
if page.Slug == "" {
res.Warnings = append(res.Warnings, "the file name produces an empty slug; the page has no URL")
}
if err := q.InsertPage(ctx, page); err != nil {
return res, err
}
snap := e.snapshot(page.ID, obs, d, at, kind, maturity)
if err := q.InsertSnapshot(ctx, snap); err != nil {
return res, err
}
// The baseline keeps the note already in the file, unpublished, so a
// later upload that still carries it doesn't announce an old note.
rev := &store.Revision{
PageID: page.ID, Number: 1, SnapshotID: snap.ID,
Note: d.Note, Kind: kind, Maturity: maturity, PublishedAt: at, UpdatedAt: at,
}
if err := q.InsertRevision(ctx, rev); err != nil {
return res, err
}
res.Slug, res.Action, res.Number = page.Slug, Baseline, 1
return res, nil
}
func (e *Engine) snapshot(pageID int64, obs Observation, d *doc.Doc, at time.Time, kind, maturity string) *store.Snapshot {
fm, err := json.Marshal(d.Frontmatter)
if err != nil {
fm = []byte("{}")
}
s := &store.Snapshot{
PageID: pageID, ObservedAt: at, RawHash: d.RawHash, ContentHash: d.ContentHash,
Raw: d.Text, FrontmatterJSON: string(fm), Title: d.Title, Note: d.Note, Kind: kind, Maturity: maturity,
}
if !obs.Mtime.IsZero() {
m := obs.Mtime
s.SourceMtime = &m
}
return s
}
func (e *Engine) setDiff(rev *store.Revision, base *store.Snapshot, cur *doc.Doc) {
full := diff.Compare(doc.Parse([]byte(base.Raw), e.opts), cur, e.opts.DiffKeys)
b, err := json.Marshal(full)
if err != nil {
panic(err) // the diff types always marshal
}
rev.DiffJSON, rev.DiffVersion = string(b), diff.Version
rev.WordsAdded, rev.WordsRemoved = full.Stats.WordsAdded, full.Stats.WordsRemoved
}
func (e *Engine) check(d *doc.Doc, obs Observation) []string {
var w []string
if !utf8.Valid(obs.Raw) {
w = append(w, "the file is not valid UTF-8")
}
if d.FrontmatterErr != nil {
w = append(w, "frontmatter is not valid YAML, so no revision note is read: "+firstLine(d.FrontmatterErr.Error()))
}
return w
}
// enum keeps only declared values of revision_kind and maturity; anything
// else is dropped with a warning rather than published.
func (e *Engine) enum(d *doc.Doc, res *Result) (kind, maturity string) {
kind, maturity = d.Kind, d.Maturity
if !doc.Valid(kind, doc.Kinds) {
res.Warnings = append(res.Warnings, fmt.Sprintf("%s %q is not one of %s; ignored", e.opts.KindKey, kind, strings.Join(doc.Kinds, ", ")))
kind = ""
}
if !doc.Valid(maturity, doc.Maturities) {
res.Warnings = append(res.Warnings, fmt.Sprintf("%s %q is not one of %s; ignored", e.opts.MaturityKey, maturity, strings.Join(doc.Maturities, ", ")))
maturity = ""
}
return kind, maturity
}
// originalDate is the page's first publication date: frontmatter first,
// then a date at the start of the file name, then fallback.
func (e *Engine) originalDate(d *doc.Doc, relPath string, fallback string) string {
switch v := d.Frontmatter[e.Cfg.Frontmatter.DateKey].(type) {
case time.Time:
// YAML reads `date: 2026-06-21` as midnight UTC; that is a calendar
// date, not an instant, so it must not shift into the site's zone.
if v.Hour() == 0 && v.Minute() == 0 && v.Second() == 0 && v.Location() == time.UTC {
return v.Format("2006-01-02")
}
return v.In(e.loc).Format("2006-01-02")
case string:
s := strings.TrimSpace(v)
if t, err := time.Parse("2006-01-02", s); err == nil {
return t.Format("2006-01-02")
}
if t, err := time.Parse(time.RFC3339, s); err == nil {
return t.In(e.loc).Format("2006-01-02")
}
}
if date, ok := slug.DatePrefix(path.Base(relPath)); ok {
return date
}
return fallback
}
// baselineDate is the fallback original date of a page seen for the first
// time: the file's modification time when it's older than the observation.
// When Patchwork starts on an existing site, that is when each page was
// written or uploaded; the first scan's own date would give every undated
// page the day of the deploy.
func (e *Engine) baselineDate(obs Observation, at time.Time) string {
t := at
if !obs.Mtime.IsZero() && obs.Mtime.Before(at) {
t = obs.Mtime
}
return t.In(e.loc).Format("2006-01-02")
}
func firstLine(s string) string {
line, _, _ := strings.Cut(s, "\n")
return line
}
|