diff --git a/internal/config/config.go b/internal/config/config.go index e08897a..88d68bc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -674,114 +674,6 @@ func (p *PatternProposalConfig) AnnounceProposals() bool { return p != nil && p.Notify } -// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md). -// -// Absent ⇒ off, and off means no outbound request at all. Present with an empty -// `sources` list is also off — a poller with nothing to poll is not wired. -// -// What a feed may NOT do here: speak. Items are written as notes with source -// "rss:" and read back when he asks; nothing is dispatched, nudged or -// announced on arrival. That is the "not a nag" constraint, and it is why there -// is no severity or channel field in this block to reach for. -type FeedsConfig struct { - // Sources — the feeds to read. Empty ⇒ the reader stays down. - Sources []FeedSourceConfig `json:"sources,omitempty"` - - // PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m). - PollInterval Duration `json:"poll_interval,omitempty"` - - // MaxItems — most items kept from one feed in one poll. 0 ⇒ - // rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob. - MaxItems int `json:"max_items,omitempty"` - - // MaxAge — on a first poll (no saved mark), how far back to take items. - // 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not - // the archive. - MaxAge Duration `json:"max_age,omitempty"` - - // AllowHosts — when set, the reader may only connect to these hosts (and - // their subdomains). The feed URLs' own hosts are added automatically, so - // this is only needed to be stricter than that. - AllowHosts []string `json:"allow_hosts,omitempty"` - - // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. - Timeout Duration `json:"timeout,omitempty"` - - // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). - MaxBytes int64 `json:"max_bytes,omitempty"` -} - -// FeedSourceConfig — one feed. -type FeedSourceConfig struct { - Name string `json:"name"` // note source is "rss:" - URL string `json:"url"` // http(s) only - Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches - Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval - Include []string `json:"include,omitempty"` // keep only items containing one of these - Exclude []string `json:"exclude,omitempty"` // drop items containing any of these -} - -// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md). -// -// Absent ⇒ off, and off means no page is ever fetched. Present with neither -// `on_demand` nor a `watches` entry is also off: there would be nothing to do. -// -// The crawler is the LAST place an answer is looked for, behind the model, his -// own memory and the local Kiwix ZIMs. That ordering lives in the query-source -// chain (cmd/mavend/actions_query.go), not here, but it is the reason this block -// is small: it is a fallback, not a search engine. -// -// Only the URL leaves the box. His notes, facts, persona block and history are -// never part of a request — the crawler package cannot even read the store. -type CrawlConfig struct { - // OnDemand — may he ask her to read a page he names out loud - // ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer - // source stays off and only the watches below run. - OnDemand bool `json:"on_demand,omitempty"` - - // Watches — pages re-read on a schedule. A page whose text changed is - // written as a note (source "crawl:"); nothing is announced. - Watches []CrawlWatchConfig `json:"watches,omitempty"` - - // Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h). - Interval Duration `json:"interval,omitempty"` - - // AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains - // included). Setting this is how "she may read the arch wiki and nothing - // else" is expressed. - // - // A watched page's own host is reachable by the scheduled crawler whether - // or not it is listed here, because configuring a watch is already saying - // she may read it. That does NOT extend to on-demand reading: a watch is - // not an allowlist entry for pages he pastes. - AllowHosts []string `json:"allow_hosts,omitempty"` - - // DenyHosts — never reachable, checked first. Private addresses do not need - // to be listed: they are refused unconditionally (see internal/webfetch). - DenyHosts []string `json:"deny_hosts,omitempty"` - - // UserAgent — sent on every request AND matched against robots.txt groups. - // Empty ⇒ webfetch.DefaultUserAgent. - UserAgent string `json:"user_agent,omitempty"` - - // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. - Timeout Duration `json:"timeout,omitempty"` - - // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). - MaxBytes int64 `json:"max_bytes,omitempty"` - - // MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes - // (4000), which is what fits a 4096-token context alongside a prompt. - MaxRunes int `json:"max_runes,omitempty"` -} - -// CrawlWatchConfig — one page kept an eye on. -type CrawlWatchConfig struct { - Name string `json:"name"` // note source is "crawl:" - URL string `json:"url"` - Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval -} - // MemoryEvalConfig — the background memory-evaluation loop (Vikunja #248). // Absent ⇒ off, like every other capability that costs something the owner did // not ask for. Each evaluation is a full LLM round-trip on the one resident @@ -1047,11 +939,7 @@ func (c *Config) applyDefaults() { c.Email.Timeout = Duration(DefaultEmailTimeout) } - // A feeds block with no sources is the same as no block: nothing to poll, - // nothing wired. Normalising it to nil keeps that "off" in one place. - if c.Feeds != nil && len(c.Feeds.Sources) == 0 { - c.Feeds = nil - } + c.normaliseFeeds() c.normaliseMCP() @@ -1059,11 +947,7 @@ func (c *Config) applyDefaults() { c.normaliseNetScan() - // Same rule for the crawler: a block that neither answers on demand nor - // watches anything has nothing to do, so it is normalised to "off". - if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 { - c.Crawl = nil - } + c.normaliseCrawl() c.normaliseKiwix() c.normaliseSearch() diff --git a/internal/config/crawl.go b/internal/config/crawl.go new file mode 100644 index 0000000..904cfd4 --- /dev/null +++ b/internal/config/crawl.go @@ -0,0 +1,126 @@ +package config + +// The two scheduled outbound readers: RSS/Atom feeds and watched web pages. +// Both are off unless configured, both write notes and neither may speak — +// nothing they fetch is dispatched, nudged or announced on arrival. That is the +// "not a nag" constraint, and it is why there is no severity or channel field in +// either block to reach for. +// +// Only the URL leaves the box. His notes, facts, persona block and history are +// never part of a request; neither package can read the store. + +// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md). +// +// Absent ⇒ off. Present with an empty `sources` list is also off — a poller with +// nothing to poll is not wired, and normaliseFeeds folds that back to nil. +type FeedsConfig struct { + // Sources — the feeds to read. Empty ⇒ the reader stays down. + Sources []FeedSourceConfig `json:"sources,omitempty"` + + // PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m). + PollInterval Duration `json:"poll_interval,omitempty"` + + // MaxItems — most items kept from one feed in one poll. 0 ⇒ + // rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob. + MaxItems int `json:"max_items,omitempty"` + + // MaxAge — on a first poll (no saved mark), how far back to take items. + // 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not + // the archive. + MaxAge Duration `json:"max_age,omitempty"` + + // AllowHosts — when set, the reader may only connect to these hosts (and + // their subdomains). The feed URLs' own hosts are added automatically, so + // this is only needed to be stricter than that. + AllowHosts []string `json:"allow_hosts,omitempty"` + + // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. + Timeout Duration `json:"timeout,omitempty"` + + // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` +} + +// FeedSourceConfig — one feed. +type FeedSourceConfig struct { + Name string `json:"name"` // note source is "rss:" + URL string `json:"url"` // http(s) only + Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches + Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval + Include []string `json:"include,omitempty"` // keep only items containing one of these + Exclude []string `json:"exclude,omitempty"` // drop items containing any of these +} + +// normaliseFeeds folds a block with no sources back to nil: it is the same as +// no block, and keeping that "off" in one place is the point. +func (c *Config) normaliseFeeds() { + if c.Feeds != nil && len(c.Feeds.Sources) == 0 { + c.Feeds = nil + } +} + +// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md). +// +// Absent ⇒ off, and off means no page is ever fetched. Present with neither +// `on_demand` nor a `watches` entry is also off: there would be nothing to do. +// +// The crawler is the LAST place an answer is looked for, behind the model, his +// own memory, the live search and the local Kiwix ZIMs. That ordering lives in +// the query-source chain (cmd/mavend/actions_query.go), not here, but it is the +// reason this block is small: it is a fallback, not a search engine. +type CrawlConfig struct { + // OnDemand — may he ask her to read a page he names out loud + // ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer + // source stays off and only the watches below run. + OnDemand bool `json:"on_demand,omitempty"` + + // Watches — pages re-read on a schedule. A page whose text changed is + // written as a note (source "crawl:"); nothing is announced. + Watches []CrawlWatchConfig `json:"watches,omitempty"` + + // Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h). + Interval Duration `json:"interval,omitempty"` + + // AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains + // included). Setting this is how "she may read the arch wiki and nothing + // else" is expressed. + // + // A watched page's own host is reachable by the scheduled crawler whether + // or not it is listed here, because configuring a watch is already saying + // she may read it. That does NOT extend to on-demand reading: a watch is + // not an allowlist entry for pages he pastes. + AllowHosts []string `json:"allow_hosts,omitempty"` + + // DenyHosts — never reachable, checked first. Private addresses do not need + // to be listed: they are refused unconditionally (see internal/webfetch). + DenyHosts []string `json:"deny_hosts,omitempty"` + + // UserAgent — sent on every request AND matched against robots.txt groups. + // Empty ⇒ webfetch.DefaultUserAgent. + UserAgent string `json:"user_agent,omitempty"` + + // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. + Timeout Duration `json:"timeout,omitempty"` + + // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` + + // MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes + // (4000), which is what fits a 4096-token context alongside a prompt. + MaxRunes int `json:"max_runes,omitempty"` +} + +// CrawlWatchConfig — one page kept an eye on. +type CrawlWatchConfig struct { + Name string `json:"name"` // note source is "crawl:" + URL string `json:"url"` + Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval +} + +// normaliseCrawl folds a block that neither answers on demand nor watches +// anything back to nil: it has nothing to do. +func (c *Config) normaliseCrawl() { + if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 { + c.Crawl = nil + } +}