From 7f411656c214765391cf732da8049cb6e7d4a4d9 Mon Sep 17 00:00:00 2001 From: claude Date: Thu, 6 Aug 2026 01:21:57 +0400 Subject: [PATCH] config: the two scheduled readers move to crawl.go (V-410) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure move. Feeds and Crawl are the same seam — a webfetch on a timer that writes notes and may not speak — so the constraint is stated once at the top of the file instead of twice inside the blocks. The crawler's own comment still put it behind "the model, his memory and Kiwix" and did not know about the live search; corrected. Co-Authored-By: Claude Opus 5 --- internal/config/config.go | 120 +----------------------------------- internal/config/crawl.go | 126 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 118 deletions(-) create mode 100644 internal/config/crawl.go diff --git a/internal/config/config.go b/internal/config/config.go index e08897a..88d68bc 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -674,114 +674,6 @@ func (p *PatternProposalConfig) AnnounceProposals() bool { return p != nil && p.Notify } -// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md). -// -// Absent ⇒ off, and off means no outbound request at all. Present with an empty -// `sources` list is also off — a poller with nothing to poll is not wired. -// -// What a feed may NOT do here: speak. Items are written as notes with source -// "rss:" and read back when he asks; nothing is dispatched, nudged or -// announced on arrival. That is the "not a nag" constraint, and it is why there -// is no severity or channel field in this block to reach for. -type FeedsConfig struct { - // Sources — the feeds to read. Empty ⇒ the reader stays down. - Sources []FeedSourceConfig `json:"sources,omitempty"` - - // PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m). - PollInterval Duration `json:"poll_interval,omitempty"` - - // MaxItems — most items kept from one feed in one poll. 0 ⇒ - // rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob. - MaxItems int `json:"max_items,omitempty"` - - // MaxAge — on a first poll (no saved mark), how far back to take items. - // 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not - // the archive. - MaxAge Duration `json:"max_age,omitempty"` - - // AllowHosts — when set, the reader may only connect to these hosts (and - // their subdomains). The feed URLs' own hosts are added automatically, so - // this is only needed to be stricter than that. - AllowHosts []string `json:"allow_hosts,omitempty"` - - // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. - Timeout Duration `json:"timeout,omitempty"` - - // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). - MaxBytes int64 `json:"max_bytes,omitempty"` -} - -// FeedSourceConfig — one feed. -type FeedSourceConfig struct { - Name string `json:"name"` // note source is "rss:" - URL string `json:"url"` // http(s) only - Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches - Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval - Include []string `json:"include,omitempty"` // keep only items containing one of these - Exclude []string `json:"exclude,omitempty"` // drop items containing any of these -} - -// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md). -// -// Absent ⇒ off, and off means no page is ever fetched. Present with neither -// `on_demand` nor a `watches` entry is also off: there would be nothing to do. -// -// The crawler is the LAST place an answer is looked for, behind the model, his -// own memory and the local Kiwix ZIMs. That ordering lives in the query-source -// chain (cmd/mavend/actions_query.go), not here, but it is the reason this block -// is small: it is a fallback, not a search engine. -// -// Only the URL leaves the box. His notes, facts, persona block and history are -// never part of a request — the crawler package cannot even read the store. -type CrawlConfig struct { - // OnDemand — may he ask her to read a page he names out loud - // ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer - // source stays off and only the watches below run. - OnDemand bool `json:"on_demand,omitempty"` - - // Watches — pages re-read on a schedule. A page whose text changed is - // written as a note (source "crawl:"); nothing is announced. - Watches []CrawlWatchConfig `json:"watches,omitempty"` - - // Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h). - Interval Duration `json:"interval,omitempty"` - - // AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains - // included). Setting this is how "she may read the arch wiki and nothing - // else" is expressed. - // - // A watched page's own host is reachable by the scheduled crawler whether - // or not it is listed here, because configuring a watch is already saying - // she may read it. That does NOT extend to on-demand reading: a watch is - // not an allowlist entry for pages he pastes. - AllowHosts []string `json:"allow_hosts,omitempty"` - - // DenyHosts — never reachable, checked first. Private addresses do not need - // to be listed: they are refused unconditionally (see internal/webfetch). - DenyHosts []string `json:"deny_hosts,omitempty"` - - // UserAgent — sent on every request AND matched against robots.txt groups. - // Empty ⇒ webfetch.DefaultUserAgent. - UserAgent string `json:"user_agent,omitempty"` - - // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. - Timeout Duration `json:"timeout,omitempty"` - - // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). - MaxBytes int64 `json:"max_bytes,omitempty"` - - // MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes - // (4000), which is what fits a 4096-token context alongside a prompt. - MaxRunes int `json:"max_runes,omitempty"` -} - -// CrawlWatchConfig — one page kept an eye on. -type CrawlWatchConfig struct { - Name string `json:"name"` // note source is "crawl:" - URL string `json:"url"` - Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval -} - // MemoryEvalConfig — the background memory-evaluation loop (Vikunja #248). // Absent ⇒ off, like every other capability that costs something the owner did // not ask for. Each evaluation is a full LLM round-trip on the one resident @@ -1047,11 +939,7 @@ func (c *Config) applyDefaults() { c.Email.Timeout = Duration(DefaultEmailTimeout) } - // A feeds block with no sources is the same as no block: nothing to poll, - // nothing wired. Normalising it to nil keeps that "off" in one place. - if c.Feeds != nil && len(c.Feeds.Sources) == 0 { - c.Feeds = nil - } + c.normaliseFeeds() c.normaliseMCP() @@ -1059,11 +947,7 @@ func (c *Config) applyDefaults() { c.normaliseNetScan() - // Same rule for the crawler: a block that neither answers on demand nor - // watches anything has nothing to do, so it is normalised to "off". - if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 { - c.Crawl = nil - } + c.normaliseCrawl() c.normaliseKiwix() c.normaliseSearch() diff --git a/internal/config/crawl.go b/internal/config/crawl.go new file mode 100644 index 0000000..904cfd4 --- /dev/null +++ b/internal/config/crawl.go @@ -0,0 +1,126 @@ +package config + +// The two scheduled outbound readers: RSS/Atom feeds and watched web pages. +// Both are off unless configured, both write notes and neither may speak — +// nothing they fetch is dispatched, nudged or announced on arrival. That is the +// "not a nag" constraint, and it is why there is no severity or channel field in +// either block to reach for. +// +// Only the URL leaves the box. His notes, facts, persona block and history are +// never part of a request; neither package can read the store. + +// FeedsConfig — the RSS/Atom reader (Vikunja #258, docs/plans/13-rss-news-feeds.md). +// +// Absent ⇒ off. Present with an empty `sources` list is also off — a poller with +// nothing to poll is not wired, and normaliseFeeds folds that back to nil. +type FeedsConfig struct { + // Sources — the feeds to read. Empty ⇒ the reader stays down. + Sources []FeedSourceConfig `json:"sources,omitempty"` + + // PollInterval — default per-feed cadence. 0 ⇒ rss.DefaultPollInterval (30m). + PollInterval Duration `json:"poll_interval,omitempty"` + + // MaxItems — most items kept from one feed in one poll. 0 ⇒ + // rss.DefaultMaxItems (5). This is the "не завали мне /dash" knob. + MaxItems int `json:"max_items,omitempty"` + + // MaxAge — on a first poll (no saved mark), how far back to take items. + // 0 ⇒ rss.DefaultMaxAge (24h), so switching a feed on imports today, not + // the archive. + MaxAge Duration `json:"max_age,omitempty"` + + // AllowHosts — when set, the reader may only connect to these hosts (and + // their subdomains). The feed URLs' own hosts are added automatically, so + // this is only needed to be stricter than that. + AllowHosts []string `json:"allow_hosts,omitempty"` + + // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. + Timeout Duration `json:"timeout,omitempty"` + + // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` +} + +// FeedSourceConfig — one feed. +type FeedSourceConfig struct { + Name string `json:"name"` // note source is "rss:" + URL string `json:"url"` // http(s) only + Category string `json:"category,omitempty"` // "технологии" — what "что нового по X?" matches + Interval Duration `json:"interval,omitempty"` // 0 ⇒ FeedsConfig.PollInterval + Include []string `json:"include,omitempty"` // keep only items containing one of these + Exclude []string `json:"exclude,omitempty"` // drop items containing any of these +} + +// normaliseFeeds folds a block with no sources back to nil: it is the same as +// no block, and keeping that "off" in one place is the point. +func (c *Config) normaliseFeeds() { + if c.Feeds != nil && len(c.Feeds.Sources) == 0 { + c.Feeds = nil + } +} + +// CrawlConfig — the web crawler (Vikunja #259, docs/plans/14-web-crawler.md). +// +// Absent ⇒ off, and off means no page is ever fetched. Present with neither +// `on_demand` nor a `watches` entry is also off: there would be nothing to do. +// +// The crawler is the LAST place an answer is looked for, behind the model, his +// own memory, the live search and the local Kiwix ZIMs. That ordering lives in +// the query-source chain (cmd/mavend/actions_query.go), not here, but it is the +// reason this block is small: it is a fallback, not a search engine. +type CrawlConfig struct { + // OnDemand — may he ask her to read a page he names out loud + // ("посмотри https://… — что там пишут?"). false ⇒ the on-demand answer + // source stays off and only the watches below run. + OnDemand bool `json:"on_demand,omitempty"` + + // Watches — pages re-read on a schedule. A page whose text changed is + // written as a note (source "crawl:"); nothing is announced. + Watches []CrawlWatchConfig `json:"watches,omitempty"` + + // Interval — default watch cadence. 0 ⇒ crawl.DefaultWatchInterval (6h). + Interval Duration `json:"interval,omitempty"` + + // AllowHosts — when set, the ONLY hosts the crawler may reach (subdomains + // included). Setting this is how "she may read the arch wiki and nothing + // else" is expressed. + // + // A watched page's own host is reachable by the scheduled crawler whether + // or not it is listed here, because configuring a watch is already saying + // she may read it. That does NOT extend to on-demand reading: a watch is + // not an allowlist entry for pages he pastes. + AllowHosts []string `json:"allow_hosts,omitempty"` + + // DenyHosts — never reachable, checked first. Private addresses do not need + // to be listed: they are refused unconditionally (see internal/webfetch). + DenyHosts []string `json:"deny_hosts,omitempty"` + + // UserAgent — sent on every request AND matched against robots.txt groups. + // Empty ⇒ webfetch.DefaultUserAgent. + UserAgent string `json:"user_agent,omitempty"` + + // Timeout — per-request budget. 0 ⇒ webfetch.DefaultTimeout. + Timeout Duration `json:"timeout,omitempty"` + + // MaxBytes — response size cap. 0 ⇒ webfetch.DefaultMaxBytes (2 MiB). + MaxBytes int64 `json:"max_bytes,omitempty"` + + // MaxRunes — how much extracted text is kept. 0 ⇒ crawl.DefaultMaxRunes + // (4000), which is what fits a 4096-token context alongside a prompt. + MaxRunes int `json:"max_runes,omitempty"` +} + +// CrawlWatchConfig — one page kept an eye on. +type CrawlWatchConfig struct { + Name string `json:"name"` // note source is "crawl:" + URL string `json:"url"` + Interval Duration `json:"interval,omitempty"` // 0 ⇒ CrawlConfig.Interval +} + +// normaliseCrawl folds a block that neither answers on demand nor watches +// anything back to nil: it has nothing to do. +func (c *Config) normaliseCrawl() { + if c.Crawl != nil && !c.Crawl.OnDemand && len(c.Crawl.Watches) == 0 { + c.Crawl = nil + } +}