kami
327726a06a
The on-demand crawler was built over allow_hosts plus every watched host. webfetch reads a non-empty allow list as these and nothing else, so a config with one watch and no allow_hosts at all silently narrowed on-demand reading to the watched site. Every other url he pasted came back as a flat refusal with nothing in the log to explain it. The two crawlers now take two host lists from one crawlHosts helper. Crawl-delay was parsed into Rules and never read. The only pacing was the fetcher's flat one request per host per second, which cannot express what a site asked for, and deploy/README claimed the field was honoured. Page now waits it out between the robots fetch and the page fetch, and a delay longer than the turn fails the read instead of hanging it. A robots.txt that failed was treated as no rules, so a site whose server was having a bad minute became a site with no restrictions. A 5xx now refuses the crawl. A 404 still means unrestricted, which is what the standard says. The refusal check matched substrings of webfetch's message text from a package that cannot import webfetch, so a reworded error would have silently turned into a robots verdict. internal/crawl now exports ErrFetchRefused and ErrFetchStatus and the adapter in cmd/mavend maps the webfetch sentinels onto them. Robots group selection picks the longest matching agent prefix instead of the first one in file order. queryWeb passed a claim it could not serve when no crawler was configured, so an unconfigured deployment answered a web question with an apology instead of falling through to the model. Found in review of #67.
241 lines
9.1 KiB
Go
241 lines
9.1 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/kami/maven/internal/config"
|
|
"github.com/kami/maven/internal/crawl"
|
|
"github.com/kami/maven/internal/ipc"
|
|
"github.com/kami/maven/internal/phraser"
|
|
"github.com/kami/maven/internal/router"
|
|
"github.com/kami/maven/internal/voice"
|
|
"github.com/kami/maven/internal/webfetch"
|
|
)
|
|
|
|
// The default config reads nothing. This is the whole "off unless configured"
|
|
// contract for the crawler, asserted at the wiring level rather than trusted.
|
|
func TestCrawlOffByDefault(t *testing.T) {
|
|
cfg := &config.Config{}
|
|
if c := newCrawler(cfg); c != nil {
|
|
t.Error("newCrawler with no crawl block returned a crawler")
|
|
}
|
|
if c := onDemandCrawler(cfg); c != nil {
|
|
t.Error("onDemandCrawler with no crawl block returned a crawler")
|
|
}
|
|
if w := newCrawlWorker(nil, nil, nil, cfg); w != nil {
|
|
t.Error("newCrawlWorker with no crawl block returned a worker")
|
|
}
|
|
// Watches configured but on_demand off ⇒ the answer path still reads
|
|
// nothing: a timer over a fixed list is not permission for arbitrary URLs.
|
|
withWatch := &config.Config{Crawl: &config.CrawlConfig{
|
|
Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://example.org/p"}},
|
|
}}
|
|
if c := onDemandCrawler(withWatch); c != nil {
|
|
t.Error("onDemandCrawler honoured a watch list as on-demand permission")
|
|
}
|
|
if c := newCrawler(withWatch); c == nil {
|
|
t.Error("newCrawler returned nil for a configured watch")
|
|
}
|
|
}
|
|
|
|
// The wired fetcher must refuse a private address, because the crawler on this
|
|
// box sits one hop from the whole homelab. Same guard the webfetch tests cover;
|
|
// this asserts the daemon actually wires it.
|
|
func TestCrawlerRefusesPrivateAddress(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.Header().Set("Content-Type", "text/html")
|
|
w.Write([]byte("<html><body>secret</body></html>"))
|
|
}))
|
|
defer srv.Close()
|
|
|
|
c := newCrawler(&config.Config{Crawl: &config.CrawlConfig{OnDemand: true}})
|
|
if c == nil {
|
|
t.Fatal("newCrawler returned nil for an on-demand config")
|
|
}
|
|
if _, err := c.Page(context.Background(), srv.URL); err == nil {
|
|
t.Fatalf("reading %s succeeded; a loopback address must be refused", srv.URL)
|
|
}
|
|
}
|
|
|
|
func TestFactHashesRoundTrip(t *testing.T) {
|
|
ctx := context.Background()
|
|
st := newTestStore(t)
|
|
h := &factHashes{api: ipc.NewStoreAPI(st)}
|
|
|
|
got, err := h.LastHash(ctx, "page")
|
|
if err != nil {
|
|
t.Fatalf("LastHash on a fresh store: %v", err)
|
|
}
|
|
if got != "" {
|
|
t.Errorf("LastHash = %q, want empty for a never-read page", got)
|
|
}
|
|
if err := h.SetHash(ctx, "page", "deadbeef"); err != nil {
|
|
t.Fatalf("SetHash: %v", err)
|
|
}
|
|
got, err = h.LastHash(ctx, "page")
|
|
if err != nil {
|
|
t.Fatalf("LastHash: %v", err)
|
|
}
|
|
if got != "deadbeef" {
|
|
t.Errorf("LastHash = %q, want deadbeef", got)
|
|
}
|
|
if key := hashKey("page"); key != "crawl:hash:page" {
|
|
t.Errorf("hashKey = %q", key)
|
|
}
|
|
}
|
|
|
|
// stubCrawlFetcher serves one fixed page to every URL, so queryWeb can be
|
|
// exercised without a network or an allowlist.
|
|
type stubCrawlFetcher struct{ body, ctype string }
|
|
|
|
func (s *stubCrawlFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
|
ct := s.ctype
|
|
if ct == "" {
|
|
ct = "text/html"
|
|
}
|
|
if strings.HasSuffix(u, "/robots.txt") {
|
|
return &crawl.Response{URL: u, ContentType: "text/plain", Body: []byte("")}, nil
|
|
}
|
|
return &crawl.Response{URL: u, ContentType: ct, Body: []byte(s.body)}, nil
|
|
}
|
|
|
|
func buildWebHandler(c *crawl.Crawler) *reactiveHandler {
|
|
return &reactiveHandler{
|
|
replier: voice.NewStubReplier(),
|
|
phraser: phraser.NewStub(),
|
|
crawler: c,
|
|
}
|
|
}
|
|
|
|
func askWeb(h *reactiveHandler, q string) (string, bool) {
|
|
return h.queryWeb(context.Background(), &queryTurn{
|
|
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
|
|
})
|
|
}
|
|
|
|
func TestQueryWebPassesWithoutAURL(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&stubCrawlFetcher{body: "<html><body>x</body></html>"}, crawl.Config{}))
|
|
if reply, ok := askWeb(h, "почему небо синее?"); ok {
|
|
t.Errorf("the web source claimed a question with no URL: %q", reply)
|
|
}
|
|
}
|
|
|
|
// A daemon where page reading was never turned on — the default — answers the
|
|
// question the way it did before the capability existed. Claiming the turn to
|
|
// report a configuration status is for something that exists and failed.
|
|
func TestQueryWebPassesWhenNotConfigured(t *testing.T) {
|
|
h := buildWebHandler(nil)
|
|
if reply, ok := askWeb(h, "посмотри https://example.org/page"); ok {
|
|
t.Fatalf("an unconfigured crawler claimed the turn with %q", reply)
|
|
}
|
|
}
|
|
|
|
func TestQueryWebReadsThePage(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
|
body: "<html><head><title>Заголовок</title></head><body><p>текст страницы</p></body></html>",
|
|
}, crawl.Config{}))
|
|
reply, ok := askWeb(h, "посмотри https://example.org/page — что там?")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "текст страницы") {
|
|
t.Errorf("reply = %q, want the page text read back", reply)
|
|
}
|
|
}
|
|
|
|
func TestQueryWebRefusesNonHTML(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
|
|
body: "\x00\x01binary", ctype: "application/octet-stream",
|
|
}, crawl.Config{}))
|
|
reply, ok := askWeb(h, "почитай https://example.org/blob.bin")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "не получилось") {
|
|
t.Errorf("reply = %q, want the read-failed answer", reply)
|
|
}
|
|
}
|
|
|
|
// robots.txt is honoured on the answer path too, and she says so instead of
|
|
// reporting a generic failure.
|
|
func TestQueryWebObeysRobots(t *testing.T) {
|
|
h := buildWebHandler(crawl.New(&robotsDenyFetcher{}, crawl.Config{}))
|
|
reply, ok := askWeb(h, "посмотри https://example.org/private")
|
|
if !ok {
|
|
t.Fatal("the web source did not claim a question with a URL")
|
|
}
|
|
if !strings.Contains(reply, "robots.txt") {
|
|
t.Errorf("reply = %q, want the robots answer", reply)
|
|
}
|
|
}
|
|
|
|
type robotsDenyFetcher struct{}
|
|
|
|
func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
|
|
if strings.HasSuffix(u, "/robots.txt") {
|
|
return &crawl.Response{URL: u, ContentType: "text/plain",
|
|
Body: []byte("User-agent: *\nDisallow: /private\n")}, nil
|
|
}
|
|
return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("<html>nope</html>")}, nil
|
|
}
|
|
|
|
// TestCrawlHostsKeepsAWatchOutOfTheOnDemandAllowlist — the on-demand crawler
|
|
// used to be built over allow_hosts PLUS every watched host. webfetch reads a
|
|
// non-empty allow list as "these and nothing else", so one watch on a config
|
|
// with no allow_hosts at all turned unrestricted on-demand reading into
|
|
// "the watched site only", and every other URL he pasted came back as
|
|
// "не получилось прочитать страницу." with nothing in the log to explain it.
|
|
func TestCrawlHostsKeepsAWatchOutOfTheOnDemandAllowlist(t *testing.T) {
|
|
cc := &config.CrawlConfig{
|
|
OnDemand: true,
|
|
Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://watched.example/p"}},
|
|
}
|
|
if got := crawlHosts(cc, false); len(got) != 0 {
|
|
t.Errorf("on-demand allowlist = %v; a watch is not an allowlist entry, and an empty list is what means \"anything public\"", got)
|
|
}
|
|
if got := crawlHosts(cc, true); len(got) != 1 || got[0] != "watched.example" {
|
|
t.Errorf("watch allowlist = %v; want the watched host so a watch needs no hand-written entry", got)
|
|
}
|
|
|
|
// With allow_hosts set, his list is what on-demand gets, unchanged.
|
|
cc.AllowHosts = []string{"wiki.example"}
|
|
on := crawlHosts(cc, false)
|
|
if len(on) != 1 || on[0] != "wiki.example" {
|
|
t.Errorf("on-demand allowlist = %v; want exactly his allow_hosts", on)
|
|
}
|
|
if got := crawlHosts(cc, true); len(got) != 2 {
|
|
t.Errorf("watch allowlist = %v; want his hosts plus the watched one", got)
|
|
}
|
|
}
|
|
|
|
// TestCrawlFetcherReportsARefusalAsARefusal — internal/crawl cannot import
|
|
// webfetch, so it used to recognise a guard refusal by matching substrings of
|
|
// webfetch's message text. This adapter owns both packages and is where the
|
|
// translation belongs.
|
|
func TestCrawlFetcherReportsARefusalAsARefusal(t *testing.T) {
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
http.Error(w, "boom", http.StatusBadGateway)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
blocked := &crawlFetcher{f: webfetch.New(webfetch.Config{AllowHosts: []string{"wiki.example"}})}
|
|
if _, err := blocked.Get(context.Background(), "https://other.example/a"); !errors.Is(err, crawl.ErrFetchRefused) {
|
|
t.Errorf("a host outside allow_hosts = %v; want crawl.ErrFetchRefused", err)
|
|
}
|
|
if _, err := blocked.Get(context.Background(), "file:///etc/passwd"); !errors.Is(err, crawl.ErrFetchRefused) {
|
|
t.Errorf("a non-http scheme = %v; want crawl.ErrFetchRefused", err)
|
|
}
|
|
|
|
// A 5xx is a different thing: the server answered, badly. robots.txt over
|
|
// this must refuse the crawl rather than read it as "no rules".
|
|
open := &crawlFetcher{f: webfetch.New(webfetch.Config{AllowHosts: []string{"127.0.0.1"}, AllowPrivate: true})}
|
|
if _, err := open.Get(context.Background(), srv.URL+"/robots.txt"); !errors.Is(err, crawl.ErrFetchStatus) {
|
|
t.Errorf("a 502 = %v; want crawl.ErrFetchStatus", err)
|
|
}
|
|
}
|