kami
2c1b0eede0
The network fallback behind the local sources, off unless configured. internal/crawl is pure: a stdlib robots.txt parser (group specificity, wildcards, Crawl-delay, cached per host), HTML-to-plaintext extraction, and a watcher that notes a watched page only when its text changed. It has no store access and no net/http; cmd/mavend/crawls.go is the impure half. Every limit is code and tested: the guarded fetcher from #258 enforces the host allowlist/denylist, refuses private addresses in the dialer Control hook (so DNS rebinding and each redirect hop are covered), caps size and redirects, times out, and spaces requests per host. A robots.txt Disallow is refused with no override. On demand, reading is a query source placed last in the chain, after his memory, his notes, and the local Kiwix ZIMs once those are wired: no URL in the utterance means no fetch, and only the URL ever leaves the box. Scheduled watches write notes and announce nothing. The vendored tree has no x/net/html, goquery or temoto/robotstxt, so the parsers are stdlib. No new dependency.
156 lines
4.9 KiB
Go
156 lines
4.9 KiB
Go
package crawl
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
// fakeFetcher serves canned pages by URL and counts requests, so a test can
|
|
// assert that robots.txt was read once and that a refusal never reached the page.
|
|
type fakeFetcher struct {
|
|
pages map[string]Response
|
|
err error
|
|
calls []string
|
|
}
|
|
|
|
func (f *fakeFetcher) Get(_ context.Context, u string) (*Response, error) {
|
|
f.calls = append(f.calls, u)
|
|
if f.err != nil {
|
|
return nil, f.err
|
|
}
|
|
r, ok := f.pages[u]
|
|
if !ok {
|
|
return nil, errors.New("http 404")
|
|
}
|
|
if r.URL == "" {
|
|
r.URL = u
|
|
}
|
|
if r.ContentType == "" {
|
|
r.ContentType = "text/html; charset=utf-8"
|
|
}
|
|
return &r, nil
|
|
}
|
|
|
|
const htmlPage = `<html><head><title>Почему небо синее</title>
|
|
<style>body{color:red}</style><script>track()</script></head>
|
|
<body><nav>меню</nav><h1>Небо</h1>
|
|
<p>Свет рассеивается на молекулах воздуха.</p>
|
|
<p>Короткие волны рассеиваются сильнее.</p>
|
|
<footer>© 2026</footer></body></html>`
|
|
|
|
func newTestCrawler(f *fakeFetcher) *Crawler {
|
|
return New(f, Config{UserAgent: "Maven/1.0", Now: func() time.Time { return time.Unix(0, 0) }})
|
|
}
|
|
|
|
func TestPageExtractsText(t *testing.T) {
|
|
f := &fakeFetcher{pages: map[string]Response{
|
|
"https://example.org/sky": {Body: []byte(htmlPage)},
|
|
}}
|
|
page, err := newTestCrawler(f).Page(context.Background(), "https://example.org/sky")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if page.Title != "Почему небо синее" {
|
|
t.Errorf("title = %q", page.Title)
|
|
}
|
|
if !strings.Contains(page.Text, "Свет рассеивается") {
|
|
t.Errorf("body text missing: %q", page.Text)
|
|
}
|
|
for _, junk := range []string{"track()", "color:red", "меню", "© 2026"} {
|
|
if strings.Contains(page.Text, junk) {
|
|
t.Errorf("%q survived extraction: %q", junk, page.Text)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestRobotsIsCheckedAndObeyed(t *testing.T) {
|
|
f := &fakeFetcher{pages: map[string]Response{
|
|
"https://example.org/robots.txt": {Body: []byte("User-agent: *\nDisallow: /secret\n"), ContentType: "text/plain"},
|
|
"https://example.org/secret/x": {Body: []byte(htmlPage)},
|
|
"https://example.org/open": {Body: []byte(htmlPage)},
|
|
}}
|
|
c := newTestCrawler(f)
|
|
if _, err := c.Page(context.Background(), "https://example.org/secret/x"); !errors.Is(err, ErrRobots) {
|
|
t.Fatalf("error = %v, want ErrRobots", err)
|
|
}
|
|
for _, u := range f.calls {
|
|
if strings.Contains(u, "/secret") {
|
|
t.Fatal("the disallowed page was fetched anyway")
|
|
}
|
|
}
|
|
if _, err := c.Page(context.Background(), "https://example.org/open"); err != nil {
|
|
t.Fatalf("allowed page: %v", err)
|
|
}
|
|
// robots.txt was read once for the host, not once per page.
|
|
robotsReads := 0
|
|
for _, u := range f.calls {
|
|
if strings.HasSuffix(u, "/robots.txt") {
|
|
robotsReads++
|
|
}
|
|
}
|
|
if robotsReads != 1 {
|
|
t.Fatalf("robots.txt read %d times, want 1", robotsReads)
|
|
}
|
|
}
|
|
|
|
// No robots.txt means allow — that is the standard, and the alternative makes
|
|
// most of the web unreadable.
|
|
func TestMissingRobotsAllows(t *testing.T) {
|
|
f := &fakeFetcher{pages: map[string]Response{
|
|
"https://example.org/page": {Body: []byte(htmlPage)},
|
|
}}
|
|
if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/page"); err != nil {
|
|
t.Fatalf("err = %v, want the page", err)
|
|
}
|
|
}
|
|
|
|
// A refusal from the guarded fetcher must surface as itself, not be laundered
|
|
// into "no robots.txt, go ahead".
|
|
func TestFetcherRefusalIsNotSwallowed(t *testing.T) {
|
|
f := &fakeFetcher{err: errors.New("webfetch: refusing to connect to a private address: 127.0.0.1")}
|
|
_, err := newTestCrawler(f).Page(context.Background(), "http://127.0.0.1:9100/mcp")
|
|
if err == nil || !strings.Contains(err.Error(), "private address") {
|
|
t.Fatalf("error = %v, want the fetcher's refusal", err)
|
|
}
|
|
}
|
|
|
|
func TestNonTextIsRefused(t *testing.T) {
|
|
f := &fakeFetcher{pages: map[string]Response{
|
|
"https://example.org/f.pdf": {Body: []byte("%PDF-1.7"), ContentType: "application/pdf"},
|
|
}}
|
|
if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/f.pdf"); !errors.Is(err, ErrNotHTML) {
|
|
t.Fatalf("error = %v, want ErrNotHTML", err)
|
|
}
|
|
}
|
|
|
|
func TestMaxRunesCapsText(t *testing.T) {
|
|
long := "<html><body><p>" + strings.Repeat("привет ", 2000) + "</p></body></html>"
|
|
f := &fakeFetcher{pages: map[string]Response{"https://example.org/l": {Body: []byte(long)}}}
|
|
c := New(f, Config{MaxRunes: 50})
|
|
page, err := c.Page(context.Background(), "https://example.org/l")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := len([]rune(page.Text)); n > 51 {
|
|
t.Fatalf("text = %d runes, want the 50-rune cap", n)
|
|
}
|
|
}
|
|
|
|
func TestNewWithoutFetcherIsNil(t *testing.T) {
|
|
if New(nil, Config{}) != nil {
|
|
t.Fatal("a crawler with no fetcher must be nil — crawling is off unless configured")
|
|
}
|
|
}
|
|
|
|
func TestHashIgnoresNothingButText(t *testing.T) {
|
|
if Hash("a") == Hash("b") {
|
|
t.Fatal("different text hashed the same")
|
|
}
|
|
if Hash(" same \n") != Hash("same") {
|
|
t.Fatal("surrounding whitespace changed the hash")
|
|
}
|
|
}
|