Maven/internal/webfetch/webfetch.go

// Package webfetch is the one door Maven uses to read something off the
// network, and it is a narrow one.
//
// "Never phones home" stopped being a hard constraint on 2026-07-31, but what
// replaced it is not "she may fetch anything": local sources come first (Kiwix
// on the box), external fetching is off unless configured, and only the
// utterance ever leaves — never his notes, facts or history. That policy is
// enforced by the callers. What THIS package enforces is the part that must be
// code rather than a paragraph in a plan, because it protects the homelab from
// its own assistant:
//
//   - http/https only — no file://, no ftp://, no gopher;
//   - no private address, ever: loopback, RFC1918 (which is what makes the
//     10.42.0.0/24 wireguard tunnel and the 192.168.1.0/24 LAN unreachable),
//     link-local incl. the 169.254.169.254 cloud metadata address, CGNAT,
//     unique-local v6. Checked in the dialer's Control hook, so it holds for
//     every address the resolver returns AND for every hop of a redirect
//     chain — a DNS name that resolves to 127.0.0.1 is refused at connect
//     time, which a pre-flight lookup could not promise (rebinding);
//   - an allowlist, when one is configured, and a denylist that always wins;
//   - a response size cap, a total timeout, a redirect cap;
//   - one request per host per interval, so a poll loop with a bug is slow
//     rather than an outbound flood.
//
// Everything above is on by default with sane numbers: a zero Config is a
// usable, conservative fetcher. There is no cache and no retry — a feed poll
// or a page read that fails is simply not answered this round.
package webfetch

import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"syscall"
	"time"
)

// Defaults. Small on purpose: this reads feeds and article pages, not ISOs.
const (
	DefaultTimeout      = 20 * time.Second
	DefaultMaxBytes     = 2 << 20 // 2 MiB
	DefaultMaxRedirects = 3
	DefaultHostInterval = time.Second
	DefaultUserAgent    = "Maven/1.0 (self-hosted personal assistant)"
)

// Errors callers distinguish. Everything else is wrapped transport error.
var (
	ErrScheme    = errors.New("webfetch: only http and https are allowed")
	ErrBlocked   = errors.New("webfetch: host is not allowed")
	ErrPrivate   = errors.New("webfetch: refusing to connect to a private address")
	ErrTooLarge  = errors.New("webfetch: response exceeds the size cap")
	ErrRedirects = errors.New("webfetch: too many redirects")
	ErrStatus    = errors.New("webfetch: non-2xx status")
)

// Config are the limits. Every zero value means "the default above", so
// Config{} is safe; the only field that changes behaviour by being empty is
// AllowHosts (empty ⇒ any public host that is not denied).
type Config struct {
	// AllowHosts — when non-empty, the ONLY hosts that may be fetched. An
	// entry matches the host itself and its subdomains ("example.com" allows
	// "news.example.com"). This is the knob to reach for when a capability
	// should read two feeds and nothing else.
	AllowHosts []string
	// DenyHosts — same matching, checked first and always winning.
	DenyHosts []string

	Timeout      time.Duration // whole request, including redirects and body read
	MaxBytes     int64         // response body cap
	MaxRedirects int           // 0 ⇒ default; negative ⇒ no redirects followed
	HostInterval time.Duration // minimum spacing between requests to one host
	UserAgent    string

	// AllowPrivate disables the private-address guard. It exists for tests
	// (httptest listens on 127.0.0.1) and for an explicitly configured
	// on-box mirror. Nothing in deploy/mavend.json sets it, and it should
	// stay that way: with it on, any URL Maven is handed becomes an SSRF
	// probe of the LAN and the wireguard range.
	AllowPrivate bool
}

// Response is a fetched body, already bounded by MaxBytes.
type Response struct {
	URL         string // final URL after redirects
	Status      int
	ContentType string
	Body        []byte
	// Header — the response headers, one value each (the first). Populated for
	// every request; MCP needs Mcp-Session-Id, nothing else reads it.
	Header map[string]string
}

// Fetcher performs guarded GETs. Safe for concurrent use; the per-host rate
// limiter is shared, which is the point of sharing one Fetcher.
type Fetcher struct {
	cfg  Config
	http *http.Client

	mu   sync.Mutex
	last map[string]time.Time // host → when we last dialed it
}

// New builds a fetcher from cfg, filling in defaults.
func New(cfg Config) *Fetcher {
	if cfg.Timeout <= 0 {
		cfg.Timeout = DefaultTimeout
	}
	if cfg.MaxBytes <= 0 {
		cfg.MaxBytes = DefaultMaxBytes
	}
	if cfg.MaxRedirects == 0 {
		cfg.MaxRedirects = DefaultMaxRedirects
	}
	if cfg.HostInterval <= 0 {
		cfg.HostInterval = DefaultHostInterval
	}
	if cfg.UserAgent == "" {
		cfg.UserAgent = DefaultUserAgent
	}
	f := &Fetcher{cfg: cfg, last: map[string]time.Time{}}

	dialer := &net.Dialer{Timeout: 10 * time.Second}
	if !cfg.AllowPrivate {
		// The guard lives here rather than in a pre-flight net.LookupHost so
		// that it sees the address actually being connected to: every A/AAAA
		// the resolver handed back, on every redirect hop, with no window in
		// which the name could be re-pointed at the LAN.
		dialer.Control = func(_, address string, _ syscall.RawConn) error {
			host, _, err := net.SplitHostPort(address)
			if err != nil {
				return err
			}
			ip := net.ParseIP(host)
			if ip == nil || IsPrivateIP(ip) {
				return fmt.Errorf("%w: %s", ErrPrivate, host)
			}
			return nil
		}
	}
	f.http = &http.Client{
		Timeout:   cfg.Timeout,
		Transport: &http.Transport{DialContext: dialer.DialContext},
		CheckRedirect: func(req *http.Request, via []*http.Request) error {
			if len(via) > f.cfg.MaxRedirects {
				return ErrRedirects
			}
			// A redirect is a fresh URL and gets the full check: an allowed
			// host must not be able to bounce us onto a denied one.
			return f.checkURL(req.URL)
		},
	}
	return f
}

// Get fetches rawURL. The body is capped: a larger response is an error, not a
// truncation, because half an XML document is worse than none.
func (f *Fetcher) Get(ctx context.Context, rawURL string) (*Response, error) {
	return f.do(ctx, http.MethodGet, rawURL, nil, nil)
}

// Post sends body to rawURL and returns the reply, under exactly the same
// guards as Get: scheme rule, host lists, the dialer's private-address check on
// every hop, the size cap and the per-host rate limit.
//
// It exists for JSON-RPC over HTTP (internal/mcp), which cannot be expressed as
// a GET. That an outbound request now carries a body does not widen the
// address policy one bit — a POST to the LAN is refused for the same reason a
// GET is, unless AllowPrivate was set for that specific fetcher.
//
// hdr is merged over the defaults; a caller may not override User-Agent or
// Accept-Encoding, because identity encoding and an honest UA are part of the
// contract with whatever is on the other end.
func (f *Fetcher) Post(ctx context.Context, rawURL, contentType string, body []byte, hdr map[string]string) (*Response, error) {
	if contentType == "" {
		contentType = "application/json"
	}
	if hdr == nil {
		hdr = map[string]string{}
	}
	merged := make(map[string]string, len(hdr)+1)
	for k, v := range hdr {
		merged[k] = v
	}
	merged["Content-Type"] = contentType
	return f.do(ctx, http.MethodPost, rawURL, body, merged)
}

func (f *Fetcher) do(ctx context.Context, method, rawURL string, body []byte, hdr map[string]string) (*Response, error) {
	u, err := url.Parse(strings.TrimSpace(rawURL))
	if err != nil {
		return nil, fmt.Errorf("webfetch: bad url %q: %w", rawURL, err)
	}
	if err := f.checkURL(u); err != nil {
		return nil, err
	}
	if err := f.waitTurn(ctx, u.Hostname()); err != nil {
		return nil, err
	}

	var rdr io.Reader
	if body != nil {
		rdr = bytes.NewReader(body)
	}
	req, err := http.NewRequestWithContext(ctx, method, u.String(), rdr)
	if err != nil {
		return nil, err
	}
	for k, v := range hdr {
		req.Header.Set(k, v)
	}
	req.Header.Set("User-Agent", f.cfg.UserAgent)
	req.Header.Set("Accept-Encoding", "identity")

	resp, err := f.http.Do(req)
	if err != nil {
		// http.Client wraps our sentinels in *url.Error; unwrap so callers can
		// still tell "blocked" from "the network is down".
		for _, sentinel := range []error{ErrPrivate, ErrBlocked, ErrRedirects, ErrScheme} {
			if errors.Is(err, sentinel) {
				return nil, err
			}
		}
		return nil, err
	}
	defer resp.Body.Close()

	respBody, err := io.ReadAll(io.LimitReader(resp.Body, f.cfg.MaxBytes+1))
	if err != nil {
		return nil, err
	}
	if int64(len(respBody)) > f.cfg.MaxBytes {
		return nil, fmt.Errorf("%w (%d bytes)", ErrTooLarge, f.cfg.MaxBytes)
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return nil, fmt.Errorf("%w: %d", ErrStatus, resp.StatusCode)
	}
	out := &Response{
		URL:         resp.Request.URL.String(),
		Status:      resp.StatusCode,
		ContentType: resp.Header.Get("Content-Type"),
		Body:        respBody,
		Header:      map[string]string{},
	}
	for k := range resp.Header {
		out.Header[k] = resp.Header.Get(k)
	}
	return out, nil
}

// checkURL applies the scheme rule and the host lists. The address rule is the
// dialer's job (see New).
func (f *Fetcher) checkURL(u *url.URL) error {
	switch u.Scheme {
	case "http", "https":
	default:
		return fmt.Errorf("%w: %q", ErrScheme, u.Scheme)
	}
	host := strings.ToLower(u.Hostname())
	if host == "" {
		return fmt.Errorf("%w: no host", ErrBlocked)
	}
	if HostMatches(host, f.cfg.DenyHosts) {
		return fmt.Errorf("%w: %s is denied", ErrBlocked, host)
	}
	if len(f.cfg.AllowHosts) > 0 && !HostMatches(host, f.cfg.AllowHosts) {
		return fmt.Errorf("%w: %s is not on the allowlist", ErrBlocked, host)
	}
	// A literal private address is refused here as well as in the dialer, so
	// the error is the specific one even when no connection is attempted.
	if !f.cfg.AllowPrivate {
		if ip := net.ParseIP(host); ip != nil && IsPrivateIP(ip) {
			return fmt.Errorf("%w: %s", ErrPrivate, host)
		}
	}
	return nil
}

// waitTurn blocks until this host's rate-limit interval has elapsed. It holds
// no lock while sleeping, so two hosts never wait on each other.
func (f *Fetcher) waitTurn(ctx context.Context, host string) error {
	for {
		f.mu.Lock()
		now := time.Now()
		earliest := f.last[host].Add(f.cfg.HostInterval)
		if !now.Before(earliest) {
			f.last[host] = now
			f.mu.Unlock()
			return nil
		}
		f.mu.Unlock()
		wait := time.NewTimer(earliest.Sub(now))
		select {
		case <-ctx.Done():
			wait.Stop()
			return ctx.Err()
		case <-wait.C:
		}
	}
}

// HostMatches reports whether host equals one of pats or is a subdomain of one.
// Exported because the crawler applies the same rule to links it decides not to
// follow, before it ever builds a request.
func HostMatches(host string, pats []string) bool {
	host = strings.ToLower(strings.TrimSuffix(host, "."))
	for _, p := range pats {
		p = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(p, "*.")))
		if p == "" {
			continue
		}
		if host == p || strings.HasSuffix(host, "."+p) {
			return true
		}
	}
	return false
}

// cgnat is 100.64.0.0/10 — carrier NAT, not covered by net.IP's helpers and not
// somewhere a personal assistant has business connecting.
var cgnat = &net.IPNet{IP: net.IPv4(100, 64, 0, 0).To4(), Mask: net.CIDRMask(10, 32)}

// IsPrivateIP reports whether ip is somewhere Maven must never reach out to:
// the box itself, the LAN, the wireguard range (10.42.0.0/24 ⊂ 10/8), the cloud
// metadata address (169.254.169.254 ⊂ link-local), or anything unroutable.
func IsPrivateIP(ip net.IP) bool {
	if ip.IsLoopback() || ip.IsPrivate() || ip.IsUnspecified() ||
		ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() ||
		ip.IsInterfaceLocalMulticast() || ip.IsMulticast() {
		return true
	}
	if v4 := ip.To4(); v4 != nil && cgnat.Contains(v4) {
		return true
	}
	// IPv4-mapped/compatible forms of the above are handled by To4() inside the
	// stdlib helpers; what is left is v6 unique-local (fc00::/7).
	if len(ip) == net.IPv6len && ip.To4() == nil && ip[0]&0xfe == 0xfc {
		return true
	}
	return false
}