Maven/internal/crawl/crawl.go

// Package crawl reads a web page: fetch, robots check, HTML to text.
//
// It is the LAST place Maven looks for an answer, and that ordering is the whole
// design. "Never phones home" is deprecated, but what replaced it puts local
// sources first: the resident model, then his own memory, then the Kiwix ZIMs on
// the box (internal/kiwix), and only then the network. A local read costs
// nothing and leaks nothing; a fetch costs a round-trip and puts a URL in
// someone's access log. So this package exists to be the fallback, not the
// front door — see the querySources chain in cmd/mavend/actions_query.go for
// where it actually sits.
//
// What never leaves the box: his notes, his facts, the persona block, the
// conversation history. Only the URL is requested and, for the on-demand path,
// only because he said it out loud. Nothing here reads the store.
//
// The limits are not in this package — they are in internal/webfetch, which is
// the only way anything here touches a socket: http(s) only, host allow/deny,
// private-address refusal, size cap, redirect cap, per-host rate limit. What
// this package adds is politeness (robots.txt) and dedup.
package crawl

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"errors"
	"fmt"
	"net/url"
	"strings"
	"time"
)

// Errors callers distinguish.
var (
	ErrRobots  = errors.New("crawl: robots.txt disallows this path")
	ErrNotHTML = errors.New("crawl: response is not html or text")
)

// Fetcher is the guarded HTTP door (internal/webfetch adapted by the daemon). An
// interface so this package constructs no http.Client of its own and can be
// tested without a network.
type Fetcher interface {
	Get(ctx context.Context, url string) (*Response, error)
}

// Response is the minimum a crawl needs from a fetch.
type Response struct {
	URL         string
	ContentType string
	Body        []byte
}

// Config — crawler knobs.
type Config struct {
	// UserAgent is the name matched against robots.txt groups. It must be the
	// same string the fetcher sends, or Maven would be claiming one identity
	// and obeying the rules for another.
	UserAgent string
	// MaxRunes caps extracted text. 0 ⇒ DefaultMaxRunes.
	MaxRunes int
	// RobotsTTL — how long a parsed robots.txt is trusted. 0 ⇒ 1h.
	RobotsTTL time.Duration
	// Now is injectable for tests. nil ⇒ time.Now.
	Now func() time.Time
}

// Crawler fetches and extracts pages. Safe for concurrent use.
type Crawler struct {
	fetch  Fetcher
	cfg    Config
	robots *robotsCache
}

// New builds a crawler. Returns nil when there is no fetcher, which is how the
// daemon expresses "crawling is off unless configured".
func New(fetch Fetcher, cfg Config) *Crawler {
	if fetch == nil {
		return nil
	}
	if cfg.UserAgent == "" {
		cfg.UserAgent = "Maven"
	}
	if cfg.MaxRunes <= 0 {
		cfg.MaxRunes = DefaultMaxRunes
	}
	if cfg.RobotsTTL <= 0 {
		cfg.RobotsTTL = time.Hour
	}
	if cfg.Now == nil {
		cfg.Now = time.Now
	}
	return &Crawler{fetch: fetch, cfg: cfg, robots: newRobotsCache(cfg.RobotsTTL)}
}

// Page fetches rawURL and returns its text. It checks robots.txt first and
// refuses a disallowed path with ErrRobots — there is no override.
func (c *Crawler) Page(ctx context.Context, rawURL string) (Page, error) {
	u, err := url.Parse(strings.TrimSpace(rawURL))
	if err != nil {
		return Page{}, fmt.Errorf("crawl: bad url %q: %w", rawURL, err)
	}
	ok, err := c.allowed(ctx, u)
	if err != nil {
		return Page{}, err
	}
	if !ok {
		return Page{}, fmt.Errorf("%w: %s", ErrRobots, u.Path)
	}
	resp, err := c.fetch.Get(ctx, u.String())
	if err != nil {
		return Page{}, err
	}
	// A PDF or an image is bytes Maven cannot read; saying so beats storing
	// binary garbage as a "note".
	ct := strings.ToLower(resp.ContentType)
	if ct != "" && !strings.Contains(ct, "html") && !strings.Contains(ct, "text/") &&
		!strings.Contains(ct, "xml") && !strings.Contains(ct, "json") {
		return Page{}, fmt.Errorf("%w: %s", ErrNotHTML, resp.ContentType)
	}
	return Extract(resp.URL, resp.Body, c.cfg.MaxRunes), nil
}

// allowed consults robots.txt for u's host, reading it at most once per TTL.
//
// A robots.txt that cannot be fetched (404, a timeout, a blocked host) means
// allow, per the standard. The one thing that is NOT fail-open is an explicit
// Disallow.
func (c *Crawler) allowed(ctx context.Context, u *url.URL) (bool, error) {
	host := u.Host
	now := c.cfg.Now()
	rules, ok := c.robots.get(host, now)
	if !ok {
		robotsURL := u.Scheme + "://" + host + "/robots.txt"
		resp, err := c.fetch.Get(ctx, robotsURL)
		switch {
		case err != nil:
			// Note what is NOT swallowed: a refusal from the guarded fetcher.
			// If webfetch says this host is denied or private, the page fetch
			// would fail the same way, and reporting the real reason beats
			// reporting a robots verdict we never got.
			if isFatalFetchError(err) {
				return false, err
			}
			rules = Rules{}
		default:
			rules = ParseRobots(string(resp.Body), c.cfg.UserAgent)
		}
		c.robots.put(host, rules, now)
	}
	path := u.EscapedPath()
	if u.RawQuery != "" {
		path += "?" + u.RawQuery
	}
	return rules.Allowed(path), nil
}

// isFatalFetchError — a fetch failure that means "this host is off limits"
// rather than "there is no robots.txt here". The sentinel set is webfetch's, but
// this package must not import it (the interface exists precisely so it does
// not), so the check is on the message. Ugly and honest: the alternative is a
// dependency inversion for two strings.
func isFatalFetchError(err error) bool {
	s := err.Error()
	return strings.Contains(s, "not allowed") || strings.Contains(s, "private address") ||
		strings.Contains(s, "only http and https")
}

// Hash is the dedup key for a crawl result: the sha256 of the extracted text,
// hex, first 16 chars. Text and not raw HTML, because a page whose only change
// is a rotating ad slot or a CSRF token has not changed.
func Hash(text string) string {
	sum := sha256.Sum256([]byte(strings.TrimSpace(text)))
	return hex.EncodeToString(sum[:])[:16]
}