kami
2c1b0eede0
The network fallback behind the local sources, off unless configured. internal/crawl is pure: a stdlib robots.txt parser (group specificity, wildcards, Crawl-delay, cached per host), HTML-to-plaintext extraction, and a watcher that notes a watched page only when its text changed. It has no store access and no net/http; cmd/mavend/crawls.go is the impure half. Every limit is code and tested: the guarded fetcher from #258 enforces the host allowlist/denylist, refuses private addresses in the dialer Control hook (so DNS rebinding and each redirect hop are covered), caps size and redirects, times out, and spaces requests per host. A robots.txt Disallow is refused with no override. On demand, reading is a query source placed last in the chain, after his memory, his notes, and the local Kiwix ZIMs once those are wired: no URL in the utterance means no fetch, and only the URL ever leaves the box. Scheduled watches write notes and announce nothing. The vendored tree has no x/net/html, goquery or temoto/robotstxt, so the parsers are stdlib. No new dependency.
187 lines
6.3 KiB
Go
187 lines
6.3 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/kami/maven/internal/config"
|
|
"github.com/kami/maven/internal/crawl"
|
|
"github.com/kami/maven/internal/ipc"
|
|
"github.com/kami/maven/internal/phraser"
|
|
"github.com/kami/maven/internal/router"
|
|
"github.com/kami/maven/internal/voice"
|
|
)
|
|
|
|
// The default config reads nothing. This is the whole "off unless configured"
|
|
// contract for the crawler, asserted at the wiring level rather than trusted.
|
|
func TestCrawlOffByDefault(t *testing.T) {
|
|
cfg := &config.Config{}
|
|
if c := newCrawler(cfg); c != nil {
|
|
t.Error("newCrawler with no crawl block returned a crawler")
|
|
}
|
|
if c := onDemandCrawler(cfg); c != nil {
|
|
t.Error("onDemandCrawler with no crawl block returned a crawler")
|
|
}
|
|
if w := newCrawlWorker(nil, nil, nil, cfg); w != nil {
|
|
t.Error("newCrawlWorker with no crawl block returned a worker")
|
|
}
|
|
// Watches configured but on_demand off ⇒ the answer path still reads
|
|
// nothing: a timer over a fixed list is not permission for arbitrary URLs.
|
|
withWatch := &config.Config{Crawl: &config.CrawlConfig{
|
|
Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://example.org/p"}},
|
|
}}
|
|
if c := onDemandCrawler(withWatch); c != nil {
|
|
t.Error("onDemandCrawler honoured a watch list as on-demand permission")
|
|
}
|
|
if c := newCrawler(withWatch); c == nil {
|
|
t.Error("newCrawler returned nil for a configured watch")
|
|
}
|
|
}
|
|
|
|
// The wired fetcher must refuse a private address, because the crawler on this
|
|
// box sits one hop from the whole homelab. Same guard the webfetch tests cover;
|
|
// this asserts the daemon actually wires it.
|
|
func TestCrawlerRefusesPrivateAddress(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write([]byte("<html><body>secret</body></html>"))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
c := newCrawler(&config.Config{Crawl: &config.CrawlConfig{OnDemand: true}})
|
|
if c == nil {
|
|
t.Fatal("newCrawler returned nil for an on-demand config")
|
|
}
|
|
if _, err := c.Page(context.Background(), srv.URL); err == nil {
|
|
t.Fatalf("reading %s succeeded; a loopback address must be refused", srv.URL)
|
|
}
|
|
}
|
|
|
|
func TestFactHashesRoundTrip(t *testing.T) {
|
|
ctx := context.Background()
|
|
st := newTestStore(t)
|
|
h := &factHashes{api: ipc.NewStoreAPI(st)}
|
|
|
|
got, err := h.LastHash(ctx, "page")
|
|
if err != nil {
|
|
t.Fatalf("LastHash on a fresh store: %v", err)
|
|
}
|
|
if got != "" {
|
|
t.Errorf("LastHash = %q, want empty for a never-read page", got)
|
|
}
|
|
if err := h.SetHash(ctx, "page", "deadbeef"); err != nil {
|
|
t.Fatalf("SetHash: %v", err)
|
|
}
|
|
got, err = h.LastHash(ctx, "page")
|
|
if err != nil {
|
|
t.Fatalf("LastHash: %v", err)
|
|
}
|
|
if got != "deadbeef" {
|
|
t.Errorf("LastHash = %q, want deadbeef", got)
|
|
}
|
|
if key := hashKey("page"); key != "crawl:hash:page" {
|
|
t.Errorf("hashKey = %q", key)
|
|
}
|
|
}
|
|
|
|
// stubCrawlFetcher serves one fixed page to every URL, so queryWeb can be
|
|
// exercised without a network or an allowlist.
|
|
type stubCrawlFetcher struct{ body, ctype string }
|
|
|
|
func (s *stubCrawlFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
|
ct := s.ctype
|
|
if ct == "" {
|
|
ct = "text/html"
|
|
}
|
|
if strings.HasSuffix(u, "/robots.txt") {
|
|
return &crawl.Response{URL: u, ContentType: "text/plain", Body: []byte("")}, nil
|
|
}
|
|
return &crawl.Response{URL: u, ContentType: ct, Body: []byte(s.body)}, nil
|
|
}
|
|
|
|
func buildWebHandler(c *crawl.Crawler) *reactiveHandler {
|
|
return &reactiveHandler{
|
|
replier: voice.NewStubReplier(),
|
|
phraser: phraser.NewStub(),
|
|
crawler: c,
|
|
}
|
|
}
|
|
|
|
func askWeb(h *reactiveHandler, q string) (string, bool) {
|
|
return h.queryWeb(context.Background(), &queryTurn{
|
|
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
|
})
|
|
}
|
|
|
|
func TestQueryWebPassesWithoutAURL(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&stubCrawlFetcher{body: "<html><body>x</body></html>"}, crawl.Config{}))
|
|
if reply, ok := askWeb(h, "почему небо синее?"); ok {
|
|
t.Errorf("the web source claimed a question with no URL: %q", reply)
|
|
}
|
|
}
|
|
|
|
// Not configured is said out loud rather than falling through, so a small model
|
|
// never invents a page's contents from its URL.
|
|
func TestQueryWebSaysWhenNotConfigured(t *testing.T) {
|
|
h := buildWebHandler(nil)
|
|
reply, ok := askWeb(h, "посмотри https://example.org/page")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "не настроено") {
|
|
t.Errorf("reply = %q, want the not-configured answer", reply)
|
|
}
|
|
}
|
|
|
|
func TestQueryWebReadsThePage(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
|
body: "<html><head><title>Заголовок</title></head><body><p>текст страницы</p></body></html>",
|
|
}, crawl.Config{}))
|
|
reply, ok := askWeb(h, "посмотри https://example.org/page — что там?")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "текст страницы") {
|
|
t.Errorf("reply = %q, want the page text read back", reply)
|
|
}
|
|
}
|
|
|
|
func TestQueryWebRefusesNonHTML(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
|
body: "\x00\x01binary", ctype: "application/octet-stream",
|
|
}, crawl.Config{}))
|
|
reply, ok := askWeb(h, "почитай https://example.org/blob.bin")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "не получилось") {
|
|
t.Errorf("reply = %q, want the read-failed answer", reply)
|
|
}
|
|
}
|
|
|
|
// robots.txt is honoured on the answer path too, and she says so instead of
|
|
// reporting a generic failure.
|
|
func TestQueryWebObeysRobots(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&robotsDenyFetcher{}, crawl.Config{}))
|
|
reply, ok := askWeb(h, "посмотри https://example.org/private")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "robots.txt") {
|
|
t.Errorf("reply = %q, want the robots answer", reply)
|
|
}
|
|
}
|
|
|
|
type robotsDenyFetcher struct{}
|
|
|
|
func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
|
if strings.HasSuffix(u, "/robots.txt") {
|
|
return &crawl.Response{URL: u, ContentType: "text/plain",
|
|
Body: []byte("User-agent: *\nDisallow: /private\n")}, nil
|
|
}
|
|
return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("<html>nope</html>")}, nil
|
|
}
|