patchwork-turbo

package track

import (
	"context"
	"errors"
	"fmt"
	"net/url"

	"git.pablomurad.com/pablo/patchwork-turbo/internal/config"
	"git.pablomurad.com/pablo/patchwork-turbo/internal/feedin"
	"git.pablomurad.com/pablo/patchwork-turbo/internal/fetch"
	"git.pablomurad.com/pablo/patchwork-turbo/internal/htmlmd"
	"git.pablomurad.com/pablo/patchwork-turbo/internal/out"
	"git.pablomurad.com/pablo/patchwork-turbo/internal/sitemap"
	"git.pablomurad.com/pablo/patchwork-turbo/internal/slug"
)

// thinPage is the word count under which a page probably needs JavaScript
// to show its content, or the selector missed it.
const thinPage = 30

// page tracks one URL, as the track's single page.
func (r *Runner) page(ctx context.Context, t config.Track) (Result, error) {
	const path = out.Single
	var res Result
	resp, err := r.Fetch.Get(ctx, t.URL, r.Writer.Has(t.Name, path))
	switch {
	case fetch.Gone(err):
		res.Complete = true // the page is gone, and so is its file
		res.warn("%v; page removed", err)
		return res, nil
	case err != nil:
		return res, err
	case resp.NotModified:
		res.Keep = []string{path}
		return res, nil
	}
	p, err := r.convert(resp, t)
	if err != nil {
		return res, err
	}
	p.Path = path
	if p.Title == "" {
		p.Title = t.Name
	}
	r.checkThin(&res, p, t.URL)
	res.Pages, res.Complete = []out.Page{p}, true
	return res, nil
}

// sitemap tracks every page a sitemap lists, fetching again only the pages
// whose lastmod moved.
func (r *Runner) sitemap(ctx context.Context, t config.Track) (Result, error) {
	var res Result
	entries, err := r.sitemapEntries(ctx, t)
	if err != nil {
		return res, err
	}
	if len(entries) == 0 {
		// An empty sitemap is far more often a broken one than a site with
		// no pages; never let it wipe the track.
		return res, errors.New("the sitemap lists no pages")
	}
	if len(entries) > t.MaxPages {
		res.warn("the sitemap lists %d pages; tracking the first %d (max_pages)", len(entries), t.MaxPages)
		entries = entries[:t.MaxPages]
	}
	for _, e := range entries {
		if !sameSite(t.Sitemap, e.Loc) {
			res.warn("%s is on another host than the sitemap; skipped", e.Loc)
			continue
		}
		path := slug.FromURL(e.Loc)
		have := r.Writer.Has(t.Name, path)
		prev := r.State.Resource(e.Loc)
		if have && e.Lastmod != "" && e.Lastmod == prev.Lastmod {
			res.Keep = append(res.Keep, path)
			continue
		}
		resp, err := r.Fetch.Get(ctx, e.Loc, have)
		switch {
		case ctx.Err() != nil:
			return res, ctx.Err()
		case fetch.Gone(err):
			res.warn("%v; page removed", err)
			continue
		case err != nil:
			// Keep the last version: one failed fetch isn't a deletion.
			res.warn("%v", err)
			if have {
				res.Keep = append(res.Keep, path)
			}
			continue
		}
		r.setLastmod(e.Loc, e.Lastmod)
		if resp.NotModified {
			res.Keep = append(res.Keep, path)
			continue
		}
		p, err := r.convert(resp, t)
		if err != nil {
			res.warn("%s: %v", e.Loc, err)
			if have {
				res.Keep = append(res.Keep, path)
			}
			continue
		}
		p.Path = path
		r.checkThin(&res, p, e.Loc)
		res.Pages = append(res.Pages, p)
	}
	res.Complete = true
	return res, nil
}

// maxSitemaps bounds how many files a sitemap index may send us to.
const maxSitemaps = 50

func (r *Runner) sitemapEntries(ctx context.Context, t config.Track) ([]sitemap.Entry, error) {
	var entries []sitemap.Entry
	queue, seen := []string{t.Sitemap}, map[string]bool{}
	for len(queue) > 0 && len(seen) < maxSitemaps {
		u := queue[0]
		queue = queue[1:]
		if seen[u] {
			continue
		}
		seen[u] = true
		resp, err := r.Fetch.Get(ctx, u, false)
		if err != nil {
			return nil, err
		}
		f, err := sitemap.Parse(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("%s: %w", u, err)
		}
		entries = append(entries, f.Pages...)
		queue = append(queue, f.Sitemaps...)
	}
	return entries, nil
}

func (r *Runner) setLastmod(u, lastmod string) {
	res := r.State.Resource(u)
	res.Lastmod = lastmod
	r.State.SetResource(u, res)
}

// feed tracks the items of a feed. Items that fall off the feed keep their
// last version: a feed shows the latest items, not all of them.
func (r *Runner) feed(ctx context.Context, t config.Track) (Result, error) {
	var res Result
	resp, err := r.Fetch.Get(ctx, t.Feed, r.Writer.HasAny(t.Name))
	if err != nil || resp.NotModified {
		return res, err
	}
	_, items, err := feedin.Parse(resp.Body)
	if err != nil {
		return res, fmt.Errorf("%s: %w", t.Feed, err)
	}
	base, _ := url.Parse(resp.URL) // Get returned it, so it parses
	if len(items) > t.MaxPages {
		items = items[:t.MaxPages]
	}
	for _, it := range items {
		link := it.Link
		if link != "" {
			if u, err := base.Parse(link); err == nil {
				link = u.String()
			}
		}
		path := slug.Hash(it.Key())
		if link != "" {
			path = slug.FromURL(link)
		}
		itemBase := base
		if u, err := url.Parse(link); err == nil && u.IsAbs() {
			itemBase = u
		}
		title := it.Title
		if title == "" {
			title = path
		}
		res.Pages = append(res.Pages, out.Page{Path: path, Title: title, URL: link, Body: htmlmd.Fragment(it.HTML, itemBase)})
	}
	return res, nil
}

// convert turns a fetched HTML page into a page with the track's selectors.
func (r *Runner) convert(resp *fetch.Response, t config.Track) (out.Page, error) {
	base, _ := url.Parse(resp.URL) // Get returned it, so it parses
	hp, err := htmlmd.Convert(resp.Body, resp.ContentType, htmlmd.Options{Base: base, Select: t.Select, Ignore: t.Ignore})
	if err != nil {
		return out.Page{}, fmt.Errorf("%s: %w", resp.URL, err)
	}
	return out.Page{Title: hp.Title, URL: resp.URL, Body: hp.Body}, nil
}

func (r *Runner) checkThin(res *Result, p out.Page, u string) {
	if n := wordCount(p.Body); n < thinPage {
		res.warn("%s has only %d words of content; the page may need JavaScript, or select may miss it", u, n)
	}
}