Maven/internal/config/crawl.go

package config

// The two scheduled outbound readers: RSS/Atom feeds and watched web pages.
// Both are off unless configured, both write notes and neither may speak —
// nothing they fetch is dispatched, nudged or announced on arrival. That is the
// "not a nag" constraint, and it is why there is no severity or channel field in
// either block to reach for.
//
// Only the URL leaves the box. His notes, facts, persona block and history are
// never part of a request; neither package can read the store.

// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md).
//
// Absent ⇒ off. Present with an empty `sources` list is also off — a poller with
// nothing to poll is not wired, and normaliseFeeds folds that back to nil.
type FeedsConfig struct {
	// Sources — the feeds to read. Empty ⇒ the reader stays down.
	Sources []FeedSourceConfig `json:"sources,omitempty"`

	// PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m).
	PollInterval Duration `json:"poll_interval,omitempty"`

	// MaxItems — most items kept from one feed in one poll. 0 ⇒
	// rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob.
	MaxItems int `json:"max_items,omitempty"`

	// MaxAge — on a first poll (no saved mark), how far back to take items.
	// 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not
	// the archive.
	MaxAge Duration `json:"max_age,omitempty"`

	// AllowHosts — when set, the reader may only connect to these hosts (and
	// their subdomains). The feed URLs' own hosts are added automatically, so
	// this is only needed to be stricter than that.
	AllowHosts []string `json:"allow_hosts,omitempty"`

	// Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout.
	Timeout Duration `json:"timeout,omitempty"`

	// MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB).
	MaxBytes int64 `json:"max_bytes,omitempty"`
}

// FeedSourceConfig — one feed.
type FeedSourceConfig struct {
	Name     string   `json:"name"`               // note source is "rss:<name>"
	URL      string   `json:"url"`                // http(s) only
	Category string   `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches
	Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval
	Include  []string `json:"include,omitempty"`  // keep only items containing one of these
	Exclude  []string `json:"exclude,omitempty"`  // drop items containing any of these
}

// normaliseFeeds folds a block with no sources back to nil: it is the same as
// no block, and keeping that "off" in one place is the point.
func (c *Config) normaliseFeeds() {
	if c.Feeds != nil && len(c.Feeds.Sources) == 0 {
		c.Feeds = nil
	}
}

// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md).
//
// Absent ⇒ off, and off means no page is ever fetched. Present with neither
// `on_demand` nor a `watches` entry is also off: there would be nothing to do.
//
// The crawler is the LAST place an answer is looked for, behind the model, his
// own memory, the live search and the local Kiwix ZIMs. That ordering lives in
// the query-source chain (cmd/mavend/actions_query.go), not here, but it is the
// reason this block is small: it is a fallback, not a search engine.
type CrawlConfig struct {
	// OnDemand — may he ask her to read a page he names out loud
	// ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer
	// source stays off and only the watches below run.
	OnDemand bool `json:"on_demand,omitempty"`

	// Watches — pages re-read on a schedule. A page whose text changed is
	// written as a note (source "crawl:<name>"); nothing is announced.
	Watches []CrawlWatchConfig `json:"watches,omitempty"`

	// Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h).
	Interval Duration `json:"interval,omitempty"`

	// AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains
	// included). Setting this is how "she may read the arch wiki and nothing
	// else" is expressed.
	//
	// A watched page's own host is reachable by the scheduled crawler whether
	// or not it is listed here, because configuring a watch is already saying
	// she may read it. That does NOT extend to on-demand reading: a watch is
	// not an allowlist entry for pages he pastes.
	AllowHosts []string `json:"allow_hosts,omitempty"`

	// DenyHosts — never reachable, checked first. Private addresses do not need
	// to be listed: they are refused unconditionally (see internal/webfetch).
	DenyHosts []string `json:"deny_hosts,omitempty"`

	// UserAgent — sent on every request AND matched against robots.txt groups.
	// Empty ⇒ webfetch.DefaultUserAgent.
	UserAgent string `json:"user_agent,omitempty"`

	// Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout.
	Timeout Duration `json:"timeout,omitempty"`

	// MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB).
	MaxBytes int64 `json:"max_bytes,omitempty"`

	// MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes
	// (4000), which is what fits a 4096-token context alongside a prompt.
	MaxRunes int `json:"max_runes,omitempty"`
}

// CrawlWatchConfig — one page kept an eye on.
type CrawlWatchConfig struct {
	Name     string   `json:"name"` // note source is "crawl:<name>"
	URL      string   `json:"url"`
	Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval
}

// normaliseCrawl folds a block that neither answers on demand nor watches
// anything back to nil: it has nothing to do.
func (c *Config) normaliseCrawl() {
	if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 {
		c.Crawl = nil
	}
}