Merge branch 'fix/g07' into fix/integrated

# Conflicts:
#	internal/ipc/api.go
#	internal/ipc/client.go
#	internal/llm/client.go

cmd/mavend/actions_query.go

@@ -93,10 +93,12 @@ var querySources = []querySource{
 	{"memory", (*reactiveHandler).queryMemory},
 	{"notes", (*reactiveHandler).queryNotes},
 	// LAST before the model answers from memory, and that position is the whole
-	// design (Vikunja #259): local sources first. The model, his own notes and
-	// facts, and — once internal/kiwix is wired into this chain — the offline
-	// ZIMs all get their turn before anything touches the network. This source
-	// only claims a turn where he named a URL out loud, so it never competes
+	// design (Vikunja #259): local sources first. His memory, his notes and —
+	// once internal/kiwix is wired into this chain — the offline ZIMs all get
+	// their turn before anything touches the network. The model does NOT: it
+	// answers after this, because a URL he said out loud is an instruction and
+	// a 1.7B guessing at a page it cannot read is how contents get invented.
+	// This source only claims a turn where he named a URL, so it never competes
 	// with a local answer.
 	{"web", (*reactiveHandler).queryWeb},
 	{"general-knowledge", (*reactiveHandler).queryGeneral},
@@ -218,7 +220,7 @@ func (h *reactiveHandler) queryHabits(ctx context.Context, t *queryTurn) (string
 	return profile.FormatOverallRU(), true
 }
 
-// feedNoteWindow — how many recent notes are scanned for feed items, and
+// feedNoteWindow — how many recent FEED notes are scanned, and
 // feedReadOut — how many headlines she actually reads back. She summarises the
 // top of the pile, she does not recite a river.
 const (
@@ -243,25 +245,23 @@ func (h *reactiveHandler) queryFeeds(ctx context.Context, t *queryTurn) (string,
 		// news bulletin.
 		return "я пока не читаю ленты — они не настроены.", true
 	}
-	notes, err := h.api.RecentNotes(ctx, feedNoteWindow)
+	// By source, not the last 200 notes of any kind: a busy day of voice notes
+	// used to push the newest headline out of the window, and she answered "в
+	// лентах пока ничего нового" while the poller was working fine.
+	notes, err := h.api.RecentNotesFromSource(ctx, rss.SourcePrefix, feedNoteWindow)
 	if err != nil {
 		log.Printf("voice: feeds: recent notes: %v", err)
 		return "не получилось посмотреть ленты.", true
 	}
 	var picked []string
 	for _, n := range notes {
-		if !strings.HasPrefix(n.Source, rss.SourcePrefix) {
+		if !router.CategoryMatches(rss.NoteCategory(n.Text), q.Category) {
 			continue
 		}
-		if !router.CategoryMatches(n.Text, q.Category) {
-			continue
-		}
-		// The note carries title, summary and link; she reads the title.
-		title := n.Text
-		if i := strings.IndexByte(title, '\n'); i > 0 {
-			title = title[:i]
-		}
-		picked = append(picked, strings.TrimSpace(title))
+		// The note carries title, summary, category tag and link; she reads the
+		// title alone. The tag is for the match above, and piper reads brackets
+		// out loud.
+		picked = append(picked, rss.NoteHeadline(n.Text))
 		if len(picked) == feedReadOut {
 			break
 		}
@@ -462,10 +462,12 @@ func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, b
 		return "", false
 	}
 	if h.crawler == nil {
-		// Claim rather than fall through: he asked about a specific page, and
-		// letting the model answer from the URL's spelling alone is how a small
-		// model invents a page's contents.
-		return "я не читаю страницы — это не настроено.", true
+		// Fall through. Reading pages is off unless configured, and on a daemon
+		// where it was never turned on the older behaviour is right: the model
+		// answers the question as if the URL had not been said. Announcing a
+		// configuration status is for a capability that exists and failed, not
+		// for one he never asked for.
+		return "", false
 	}
 	ctxFetch, cancel := context.WithTimeout(ctx, 30*time.Second)
 	defer cancel()

cmd/mavend/crawls.go

@@ -20,6 +20,8 @@ package main
 
 import (
 	"context"
+	"errors"
+	"fmt"
 	"log"
 	"net/url"
 	"time"
@@ -39,21 +41,36 @@ func newCrawler(cfg *config.Config) *crawl.Crawler {
 		return nil
 	}
 	cc := cfg.Crawl
+	// The WATCH crawler, and only it, reaches the watched hosts. webfetch reads
+	// a non-empty allow list as "these and nothing else", so folding the watch
+	// hosts in turned a single watch into an allowlist for everything: a config
+	// with one watch and on_demand true silently refused every other page he
+	// pasted, with "не получилось прочитать страницу." and no clue why.
+	return crawlerWithHosts(cc, crawlHosts(cc, true))
+}
 
+// crawlHosts — the allowlist for one of the two crawlers. forWatches adds the
+// watched pages' own hosts, so a watch does not have to be allowlisted by hand.
+//
+// The on-demand crawler gets his allow_hosts and nothing else. webfetch reads a
+// non-empty list as "these and nothing else", so adding the watch hosts there
+// would silently narrow on-demand reading to the watched sites.
+func crawlHosts(cc *config.CrawlConfig, forWatches bool) []string {
 	hosts := append([]string(nil), cc.AllowHosts...)
-	// A watched page's own host is always reachable; otherwise an allowlist and
-	// a watch list would have to be kept in sync by hand.
+	if !forWatches {
+		return hosts
+	}
 	for _, w := range cc.Watches {
 		if u, err := url.Parse(w.URL); err == nil && u.Hostname() != "" {
 			hosts = append(hosts, u.Hostname())
 		}
 	}
-	// An allowlist plus on-demand is a contradiction worth logging rather than
-	// silently resolving: he asked for arbitrary pages AND for a fixed list.
-	// The allowlist wins, because it is the narrower instruction.
-	if len(hosts) > 0 && cc.OnDemand && len(cc.AllowHosts) > 0 {
-		log.Printf("crawl: allow_hosts is set, so on-demand reading is limited to those hosts")
-	}
+	return hosts
+}
+
+// crawlerWithHosts builds a crawler over one allowlist. Two callers, two lists:
+// see newCrawler and onDemandCrawler.
+func crawlerWithHosts(cc *config.CrawlConfig, hosts []string) *crawl.Crawler {
 	ua := cc.UserAgent
 	if ua == "" {
 		ua = webfetch.DefaultUserAgent
@@ -81,7 +98,14 @@ func onDemandCrawler(cfg *config.Config) *crawl.Crawler {
 	if cfg.Crawl == nil || !cfg.Crawl.OnDemand {
 		return nil
 	}
-	return newCrawler(cfg)
+	cc := cfg.Crawl
+	// His own allow_hosts, and nothing added behind his back. Empty means "any
+	// host that is not denied and not private", which is what on-demand reading
+	// of a URL he just said out loud has to mean.
+	if len(cc.AllowHosts) > 0 {
+		log.Printf("crawl: allow_hosts is set, so on-demand reading is limited to those %d host(s)", len(cc.AllowHosts))
+	}
+	return crawlerWithHosts(cc, crawlHosts(cc, false))
 }
 
 // crawlWorker — ticker + watcher for the scheduled half.
@@ -137,9 +161,20 @@ func (w *crawlWorker) run(ctx context.Context) {
 // net/http out of the crawler package.
 type crawlFetcher struct{ f *webfetch.Fetcher }
 
+// Get maps webfetch's sentinels onto crawl's. This adapter is the one place
+// that imports both packages, so the mapping belongs here; the crawler used to
+// match on three substrings of a message it could not see the definition of,
+// and a reworded error would have quietly turned a blocked host into "there is
+// no robots.txt here".
 func (a *crawlFetcher) Get(ctx context.Context, u string) (*crawl.Response, error) {
 	resp, err := a.f.Get(ctx, u)
 	if err != nil {
+		switch {
+		case errors.Is(err, webfetch.ErrBlocked), errors.Is(err, webfetch.ErrPrivate), errors.Is(err, webfetch.ErrScheme):
+			return nil, fmt.Errorf("%w: %v", crawl.ErrFetchRefused, err)
+		case errors.Is(err, webfetch.ErrStatus):
+			return nil, fmt.Errorf("%w: %v", crawl.ErrFetchStatus, err)
+		}
 		return nil, err
 	}
 	return &crawl.Response{URL: resp.URL, ContentType: resp.ContentType, Body: resp.Body}, nil

cmd/mavend/crawls_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"errors"
 	"net/http"
 	"net/http/httptest"
 	"strings"
@@ -13,6 +14,7 @@ import (
 	"github.com/kami/maven/internal/phraser"
 	"github.com/kami/maven/internal/router"
 	"github.com/kami/maven/internal/voice"
+	"github.com/kami/maven/internal/webfetch"
 )
 
 // The default config reads nothing. This is the whole "off unless configured"
@@ -123,16 +125,13 @@ func TestQueryWebPassesWithoutAURL(t *testing.T) {
 	}
 }
 
-// Not configured is said out loud rather than falling through, so a small model
-// never invents a page's contents from its URL.
-func TestQueryWebSaysWhenNotConfigured(t *testing.T) {
+// A daemon where page reading was never turned on — the default — answers the
+// question the way it did before the capability existed. Claiming the turn to
+// report a configuration status is for something that exists and failed.
+func TestQueryWebPassesWhenNotConfigured(t *testing.T) {
 	h := buildWebHandler(nil)
-	reply, ok := askWeb(h, "посмотри https://example.org/page")
-	if !ok {
-		t.Fatal("the web source did not claim a question with a URL")
-	}
-	if !strings.Contains(reply, "не настроено") {
-		t.Errorf("reply = %q, want the not-configured answer", reply)
+	if reply, ok := askWeb(h, "посмотри https://example.org/page"); ok {
+		t.Fatalf("an unconfigured crawler claimed the turn with %q", reply)
 	}
 }
 
@@ -184,3 +183,58 @@ func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, erro
 	}
 	return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("<html>nope</html>")}, nil
 }
+
+// TestCrawlHostsKeepsAWatchOutOfTheOnDemandAllowlist — the on-demand crawler
+// used to be built over allow_hosts PLUS every watched host. webfetch reads a
+// non-empty allow list as "these and nothing else", so one watch on a config
+// with no allow_hosts at all turned unrestricted on-demand reading into
+// "the watched site only", and every other URL he pasted came back as
+// "не получилось прочитать страницу." with nothing in the log to explain it.
+func TestCrawlHostsKeepsAWatchOutOfTheOnDemandAllowlist(t *testing.T) {
+	cc := &config.CrawlConfig{
+		OnDemand: true,
+		Watches:  []config.CrawlWatchConfig{{Name: "p", URL: "https://watched.example/p"}},
+	}
+	if got := crawlHosts(cc, false); len(got) != 0 {
+		t.Errorf("on-demand allowlist = %v; a watch is not an allowlist entry, and an empty list is what means \"anything public\"", got)
+	}
+	if got := crawlHosts(cc, true); len(got) != 1 || got[0] != "watched.example" {
+		t.Errorf("watch allowlist = %v; want the watched host so a watch needs no hand-written entry", got)
+	}
+
+	// With allow_hosts set, his list is what on-demand gets, unchanged.
+	cc.AllowHosts = []string{"wiki.example"}
+	on := crawlHosts(cc, false)
+	if len(on) != 1 || on[0] != "wiki.example" {
+		t.Errorf("on-demand allowlist = %v; want exactly his allow_hosts", on)
+	}
+	if got := crawlHosts(cc, true); len(got) != 2 {
+		t.Errorf("watch allowlist = %v; want his hosts plus the watched one", got)
+	}
+}
+
+// TestCrawlFetcherReportsARefusalAsARefusal — internal/crawl cannot import
+// webfetch, so it used to recognise a guard refusal by matching substrings of
+// webfetch's message text. This adapter owns both packages and is where the
+// translation belongs.
+func TestCrawlFetcherReportsARefusalAsARefusal(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "boom", http.StatusBadGateway)
+	}))
+	defer srv.Close()
+
+	blocked := &crawlFetcher{f: webfetch.New(webfetch.Config{AllowHosts: []string{"wiki.example"}})}
+	if _, err := blocked.Get(context.Background(), "https://other.example/a"); !errors.Is(err, crawl.ErrFetchRefused) {
+		t.Errorf("a host outside allow_hosts = %v; want crawl.ErrFetchRefused", err)
+	}
+	if _, err := blocked.Get(context.Background(), "file:///etc/passwd"); !errors.Is(err, crawl.ErrFetchRefused) {
+		t.Errorf("a non-http scheme = %v; want crawl.ErrFetchRefused", err)
+	}
+
+	// A 5xx is a different thing: the server answered, badly. robots.txt over
+	// this must refuse the crawl rather than read it as "no rules".
+	open := &crawlFetcher{f: webfetch.New(webfetch.Config{AllowHosts: []string{"127.0.0.1"}, AllowPrivate: true})}
+	if _, err := open.Get(context.Background(), srv.URL+"/robots.txt"); !errors.Is(err, crawl.ErrFetchStatus) {
+		t.Errorf("a 502 = %v; want crawl.ErrFetchStatus", err)
+	}
+}

cmd/mavend/feeds_test.go

@@ -88,12 +88,12 @@ func TestQueryFeedsByCategory(t *testing.T) {
 // answered by the model inventing a bulletin.
 func TestQueryFeedsOffAndEmptyDiffer(t *testing.T) {
 	off := buildFeedHandler(t, false)
-	reply, ok := askFeeds(t, off, "что нового?")
+	reply, ok := askFeeds(t, off, "что нового в лентах?")
 	if !ok || !strings.Contains(reply, "не настроены") {
 		t.Fatalf("feeds off: reply = %q, ok = %v", reply, ok)
 	}
 	on := buildFeedHandler(t, true)
-	reply, ok = askFeeds(t, on, "что нового?")
+	reply, ok = askFeeds(t, on, "что нового в лентах?")
 	if !ok || !strings.Contains(reply, "ничего нового") {
 		t.Fatalf("feeds on but empty: reply = %q, ok = %v", reply, ok)
 	}
@@ -104,6 +104,25 @@ func TestQueryFeedsPassesOnANonFeedQuestion(t *testing.T) {
 	if reply, ok := askFeeds(t, h, "напомни полить цветы"); ok {
 		t.Fatalf("claimed an unrelated question with %q", reply)
 	}
+	// The bare greeting is not a request for headlines. It used to be answered
+	// with a configuration status.
+	if reply, ok := askFeeds(t, h, "что нового?"); ok {
+		t.Fatalf("claimed a greeting with %q", reply)
+	}
+}
+
+// A busy day of his own notes must not push the newest headline out of the
+// window the feed answer scans.
+func TestQueryFeedsIsNotCrowdedOutByHisOwnNotes(t *testing.T) {
+	notes := []ipc.Note{{Text: "Релиз ядра [технологии]", Source: "rss:habr"}}
+	for i := 0; i < feedNoteWindow+10; i++ {
+		notes = append(notes, ipc.Note{Text: "мысль вслух", Source: "tap:voice"})
+	}
+	h := buildFeedHandler(t, true, notes...)
+	reply, ok := askFeeds(t, h, "что нового в лентах?")
+	if !ok || !strings.Contains(reply, "ядра") {
+		t.Fatalf("reply = %q, ok = %v; the headline fell out of the window", reply, ok)
+	}
 }
 
 // The mark is what stops a restart from re-noting yesterday's headlines, so the

cmd/mavend/main.go

@@ -377,7 +377,11 @@ func run(args []string) error {
 	if locked {
 		srv.Check = func(ctx context.Context, m ipc.Method, _ json.RawMessage) error {
 			switch m {
-			case ipc.MethodAssertStepUp, ipc.MethodUnlock:
+			case ipc.MethodAssertStepUp, ipc.MethodUnlock, ipc.MethodPing:
+				// Ping is allowed for the same reason the two unlock methods
+				// are: it never reaches CoreAPI. It answers "she is up and
+				// locked", which is what mavupdate needs to tell a daemon
+				// waiting for a passkey apart from one that failed to start.
 				return nil // allowed in locked mode
 			default:
 				return errLocked
@@ -388,6 +392,7 @@ func run(args []string) error {
 	}
 
 	srv.StepUp = func(ctx context.Context) error { return passkeySess.Assert(ctx, auth.Scope{}) }
+	srv.LockedFn = dl.isLocked
 
 	// Mail ingestion (Vikunja #246): the hook stays nil unless an email block is
 	// configured and there is a llama-server to extract with, in which case

cmd/mavend/modelswap.go

@@ -58,6 +58,7 @@ func wireModelSwap(srv *ipc.Server, phr phraser.Phraser, cfg *config.Config) {
 			ModelPath:  res.ModelPath,
 			BaseURL:    res.BaseURL,
 			RolledBack: res.RolledBack,
+			NoBackend:  res.NoBackend,
 			TookMs:     res.Took.Milliseconds(),
 		}
 		if err != nil {
@@ -106,9 +107,15 @@ func wireModelSwap(srv *ipc.Server, phr phraser.Phraser, cfg *config.Config) {
 // the port of a server that no longer exists, and the daemon would degrade to
 // the classifier permanently after the first swap. The client is re-pointed, not
 // rebuilt, so nothing that holds it has to know a swap happened.
+// SetSwapGate is the other half, and on the deploy shape it is the load-bearing
+// one:
+// llama-server is relaunched on the same fixed port, so SetBaseURL is usually a
+// no-op, while the gate is what makes the swap's drain count these callers at
+// all. Without it a swap can kill the server mid-routing-decision.
 func llmClientFor(lp *phraser.LLMPhraser, timeout time.Duration) *llm.Client {
 	c := llm.New(lp.BaseURL(), timeout)
 	c.SetGate(residentGate, false)
+	c.SetSwapGate(lp)
 	lp.OnSwap(func(base string) { c.SetBaseURL(base) })
 	return c
 }
@@ -136,6 +143,7 @@ var residentGate = llm.NewGate(backgroundQuiet)
 func llmBackgroundClientFor(lp *phraser.LLMPhraser, timeout time.Duration) *llm.Client {
 	c := llm.New(lp.BaseURL(), timeout)
 	c.SetGate(residentGate, true)
+	c.SetSwapGate(lp)
 	lp.OnSwap(func(base string) { c.SetBaseURL(base) })
 	return c
 }