package main
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/kami/maven/internal/config"
"github.com/kami/maven/internal/crawl"
"github.com/kami/maven/internal/ipc"
"github.com/kami/maven/internal/phraser"
"github.com/kami/maven/internal/router"
"github.com/kami/maven/internal/voice"
)
// The default config reads nothing. This is the whole "off unless configured"
// contract for the crawler, asserted at the wiring level rather than trusted.
func TestCrawlOffByDefault(t *testing.T) {
cfg := &config.Config{}
if c := newCrawler(cfg); c != nil {
t.Error("newCrawler with no crawl block returned a crawler")
}
if c := onDemandCrawler(cfg); c != nil {
t.Error("onDemandCrawler with no crawl block returned a crawler")
}
if w := newCrawlWorker(nil, nil, nil, cfg); w != nil {
t.Error("newCrawlWorker with no crawl block returned a worker")
}
// Watches configured but on_demand off ⇒ the answer path still reads
// nothing: a timer over a fixed list is not permission for arbitrary URLs.
withWatch := &config.Config{Crawl: &config.CrawlConfig{
Watches: []config.CrawlWatchConfig{{Name: "p", URL: "https://example.org/p"}},
}}
if c := onDemandCrawler(withWatch); c != nil {
t.Error("onDemandCrawler honoured a watch list as on-demand permission")
}
if c := newCrawler(withWatch); c == nil {
t.Error("newCrawler returned nil for a configured watch")
}
}
// The wired fetcher must refuse a private address, because the crawler on this
// box sits one hop from the whole homelab. Same guard the webfetch tests cover;
// this asserts the daemon actually wires it.
func TestCrawlerRefusesPrivateAddress(t *testing.T) {
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "text/html")
w.Write([]byte("secret"))
}))
defer srv.Close()
c := newCrawler(&config.Config{Crawl: &config.CrawlConfig{OnDemand: true}})
if c == nil {
t.Fatal("newCrawler returned nil for an on-demand config")
}
if _, err := c.Page(context.Background(), srv.URL); err == nil {
t.Fatalf("reading %s succeeded; a loopback address must be refused", srv.URL)
}
}
func TestFactHashesRoundTrip(t *testing.T) {
ctx := context.Background()
st := newTestStore(t)
h := &factHashes{api: ipc.NewStoreAPI(st)}
got, err := h.LastHash(ctx, "page")
if err != nil {
t.Fatalf("LastHash on a fresh store: %v", err)
}
if got != "" {
t.Errorf("LastHash = %q, want empty for a never-read page", got)
}
if err := h.SetHash(ctx, "page", "deadbeef"); err != nil {
t.Fatalf("SetHash: %v", err)
}
got, err = h.LastHash(ctx, "page")
if err != nil {
t.Fatalf("LastHash: %v", err)
}
if got != "deadbeef" {
t.Errorf("LastHash = %q, want deadbeef", got)
}
if key := hashKey("page"); key != "crawl:hash:page" {
t.Errorf("hashKey = %q", key)
}
}
// stubCrawlFetcher serves one fixed page to every URL, so queryWeb can be
// exercised without a network or an allowlist.
type stubCrawlFetcher struct{ body, ctype string }
func (s *stubCrawlFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
ct := s.ctype
if ct == "" {
ct = "text/html"
}
if strings.HasSuffix(u, "/robots.txt") {
return &crawl.Response{URL: u, ContentType: "text/plain", Body: []byte("")}, nil
}
return &crawl.Response{URL: u, ContentType: ct, Body: []byte(s.body)}, nil
}
func buildWebHandler(c *crawl.Crawler) *reactiveHandler {
return &reactiveHandler{
replier: voice.NewStubReplier(),
phraser: phraser.NewStub(),
crawler: c,
}
}
func askWeb(h *reactiveHandler, q string) (string, bool) {
return h.queryWeb(context.Background(), &queryTurn{
dec: router.Decision{Intent: router.IntentQuery, Utterance: q},
})
}
func TestQueryWebPassesWithoutAURL(t *testing.T) {
h := buildWebHandler(crawl.New(&stubCrawlFetcher{body: "x"}, crawl.Config{}))
if reply, ok := askWeb(h, "почему небо синее?"); ok {
t.Errorf("the web source claimed a question with no URL: %q", reply)
}
}
// Not configured is said out loud rather than falling through, so a small model
// never invents a page's contents from its URL.
func TestQueryWebSaysWhenNotConfigured(t *testing.T) {
h := buildWebHandler(nil)
reply, ok := askWeb(h, "посмотри https://example.org/page")
if !ok {
t.Fatal("the web source did not claim a question with a URL")
}
if !strings.Contains(reply, "не настроено") {
t.Errorf("reply = %q, want the not-configured answer", reply)
}
}
func TestQueryWebReadsThePage(t *testing.T) {
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
body: "Заголовоктекст страницы
",
}, crawl.Config{}))
reply, ok := askWeb(h, "посмотри https://example.org/page — что там?")
if !ok {
t.Fatal("the web source did not claim a question with a URL")
}
if !strings.Contains(reply, "текст страницы") {
t.Errorf("reply = %q, want the page text read back", reply)
}
}
func TestQueryWebRefusesNonHTML(t *testing.T) {
h := buildWebHandler(crawl.New(&stubCrawlFetcher{
body: "\x00\x01binary", ctype: "application/octet-stream",
}, crawl.Config{}))
reply, ok := askWeb(h, "почитай https://example.org/blob.bin")
if !ok {
t.Fatal("the web source did not claim a question with a URL")
}
if !strings.Contains(reply, "не получилось") {
t.Errorf("reply = %q, want the read-failed answer", reply)
}
}
// robots.txt is honoured on the answer path too, and she says so instead of
// reporting a generic failure.
func TestQueryWebObeysRobots(t *testing.T) {
h := buildWebHandler(crawl.New(&robotsDenyFetcher{}, crawl.Config{}))
reply, ok := askWeb(h, "посмотри https://example.org/private")
if !ok {
t.Fatal("the web source did not claim a question with a URL")
}
if !strings.Contains(reply, "robots.txt") {
t.Errorf("reply = %q, want the robots answer", reply)
}
}
type robotsDenyFetcher struct{}
func (robotsDenyFetcher) Get(_ context.Context, u string) (*crawl.Response, error) {
if strings.HasSuffix(u, "/robots.txt") {
return &crawl.Response{URL: u, ContentType: "text/plain",
Body: []byte("User-agent: *\nDisallow: /private\n")}, nil
}
return &crawl.Response{URL: u, ContentType: "text/html", Body: []byte("nope")}, nil
}