package crawl import ( "context" "errors" "strings" "testing" "time" ) // fakeFetcher serves canned pages by URL and counts requests, so a test can // assert that robots.txt was read once and that a refusal never reached the page. type fakeFetcher struct { pages map[string]Response err error calls []string } func (f *fakeFetcher) Get(_ context.Context, u string) (*Response, error) { f.calls = append(f.calls, u) if f.err != nil { return nil, f.err } r, ok := f.pages[u] if !ok { return nil, errors.New("http 404") } if r.URL == "" { r.URL = u } if r.ContentType == "" { r.ContentType = "text/html; charset=utf-8" } return &r, nil } const htmlPage = `Почему небо синее

Небо

Свет рассеивается на молекулах воздуха.

Короткие волны рассеиваются сильнее.

` func newTestCrawler(f *fakeFetcher) *Crawler { return New(f, Config{UserAgent: "Maven/1.0", Now: func() time.Time { return time.Unix(0, 0) }}) } func TestPageExtractsText(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{ "https://example.org/sky": {Body: []byte(htmlPage)}, }} page, err := newTestCrawler(f).Page(context.Background(), "https://example.org/sky") if err != nil { t.Fatal(err) } if page.Title != "Почему небо синее" { t.Errorf("title = %q", page.Title) } if !strings.Contains(page.Text, "Свет рассеивается") { t.Errorf("body text missing: %q", page.Text) } for _, junk := range []string{"track()", "color:red", "меню", "© 2026"} { if strings.Contains(page.Text, junk) { t.Errorf("%q survived extraction: %q", junk, page.Text) } } } func TestRobotsIsCheckedAndObeyed(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{ "https://example.org/robots.txt": {Body: []byte("User-agent: *\nDisallow: /secret\n"), ContentType: "text/plain"}, "https://example.org/secret/x": {Body: []byte(htmlPage)}, "https://example.org/open": {Body: []byte(htmlPage)}, }} c := newTestCrawler(f) if _, err := c.Page(context.Background(), "https://example.org/secret/x"); !errors.Is(err, ErrRobots) { t.Fatalf("error = %v, want ErrRobots", err) } for _, u := range f.calls { if strings.Contains(u, "/secret") { t.Fatal("the disallowed page was fetched anyway") } } if _, err := c.Page(context.Background(), "https://example.org/open"); err != nil { t.Fatalf("allowed page: %v", err) } // robots.txt was read once for the host, not once per page. robotsReads := 0 for _, u := range f.calls { if strings.HasSuffix(u, "/robots.txt") { robotsReads++ } } if robotsReads != 1 { t.Fatalf("robots.txt read %d times, want 1", robotsReads) } } // No robots.txt means allow — that is the standard, and the alternative makes // most of the web unreadable. func TestMissingRobotsAllows(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{ "https://example.org/page": {Body: []byte(htmlPage)}, }} if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/page"); err != nil { t.Fatalf("err = %v, want the page", err) } } // A refusal from the guarded fetcher must surface as itself, not be laundered // into "no robots.txt, go ahead". func TestFetcherRefusalIsNotSwallowed(t *testing.T) { f := &fakeFetcher{err: errors.New("webfetch: refusing to connect to a private address: 127.0.0.1")} _, err := newTestCrawler(f).Page(context.Background(), "http://127.0.0.1:9100/mcp") if err == nil || !strings.Contains(err.Error(), "private address") { t.Fatalf("error = %v, want the fetcher's refusal", err) } } func TestNonTextIsRefused(t *testing.T) { f := &fakeFetcher{pages: map[string]Response{ "https://example.org/f.pdf": {Body: []byte("%PDF-1.7"), ContentType: "application/pdf"}, }} if _, err := newTestCrawler(f).Page(context.Background(), "https://example.org/f.pdf"); !errors.Is(err, ErrNotHTML) { t.Fatalf("error = %v, want ErrNotHTML", err) } } func TestMaxRunesCapsText(t *testing.T) { long := "

" + strings.Repeat("привет ", 2000) + "

" f := &fakeFetcher{pages: map[string]Response{"https://example.org/l": {Body: []byte(long)}}} c := New(f, Config{MaxRunes: 50}) page, err := c.Page(context.Background(), "https://example.org/l") if err != nil { t.Fatal(err) } if n := len([]rune(page.Text)); n > 51 { t.Fatalf("text = %d runes, want the 50-rune cap", n) } } func TestNewWithoutFetcherIsNil(t *testing.T) { if New(nil, Config{}) != nil { t.Fatal("a crawler with no fetcher must be nil — crawling is off unless configured") } } func TestHashIgnoresNothingButText(t *testing.T) { if Hash("a") == Hash("b") { t.Fatal("different text hashed the same") } if Hash(" same \n") != Hash("same") { t.Fatal("surrounding whitespace changed the hash") } }