package crawl

import (
	"context"
	"errors"
	"fmt"
	"testing"
	"time"
)

// timedFetcher records when each request was made, so a test can assert a wait
// actually happened rather than that a field was parsed.
type timedFetcher struct {
	pages map[string]Response
	errs  map[string]error
	at    []time.Time
	urls  []string
}

func (f *timedFetcher) Get(_ context.Context, u string) (*Response, error) {
	f.at = append(f.at, time.Now())
	f.urls = append(f.urls, u)
	if err, ok := f.errs[u]; ok {
		return nil, err
	}
	r, ok := f.pages[u]
	if !ok {
		return nil, errors.New("http 404: no such page")
	}
	if r.URL == "" {
		r.URL = u
	}
	if r.ContentType == "" {
		r.ContentType = "text/html"
	}
	return &r, nil
}

func TestPage_HonoursCrawlDelay(t *testing.T) {
	// deploy/README says Crawl-delay is honoured. It was parsed into Rules and
	// never read: the only pacing was the fetcher's flat one request per host
	// per second, which cannot express what a site asked for.
	const delay = 120 * time.Millisecond
	f := &timedFetcher{pages: map[string]Response{
		"https://example.org/robots.txt": {Body: []byte(fmt.Sprintf("User-agent: *\nCrawl-delay: %.3f\n", delay.Seconds())), ContentType: "text/plain"},
		"https://example.org/a":          {Body: []byte("<html><body>a</body></html>")},
	}}
	c := New(f, Config{UserAgent: "Maven/1.0"})
	if _, err := c.Page(context.Background(), "https://example.org/a"); err != nil {
		t.Fatal(err)
	}
	if len(f.at) != 2 {
		t.Fatalf("requests = %v; want robots.txt then the page", f.urls)
	}
	if gap := f.at[1].Sub(f.at[0]); gap < delay {
		t.Errorf("the page was fetched %s after robots.txt; the site asked for %s", gap, delay)
	}
}

func TestPage_ACrawlDelayLongerThanTheTurnFailsInsteadOfBlocking(t *testing.T) {
	f := &timedFetcher{pages: map[string]Response{
		"https://example.org/robots.txt": {Body: []byte("User-agent: *\nCrawl-delay: 30\n"), ContentType: "text/plain"},
		"https://example.org/a":          {Body: []byte("<html><body>a</body></html>")},
	}}
	c := New(f, Config{UserAgent: "Maven/1.0"})
	ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond)
	defer cancel()
	if _, err := c.Page(ctx, "https://example.org/a"); err == nil {
		t.Fatal("a 30-second Crawl-delay was ignored inside a turn that cannot wait that long")
	}
	if len(f.urls) != 1 {
		t.Errorf("requests = %v; the page must not be fetched before the wait it refused", f.urls)
	}
}

func TestPage_ABrokenRobotsServerIsNotPermissionToCrawl(t *testing.T) {
	// A 404 means unrestricted, per the standard. A 500 does not: the standard
	// asks for the opposite, and "the site is broken, so read it" is the wrong
	// way to resolve an unknown.
	f := &timedFetcher{
		pages: map[string]Response{"https://example.org/a": {Body: []byte("<html><body>a</body></html>")}},
		errs:  map[string]error{"https://example.org/robots.txt": &StatusError{Code: 503}},
	}
	c := New(f, Config{UserAgent: "Maven/1.0"})
	if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchStatus) {
		t.Fatalf("Page over a 503 robots.txt = %v; want a refusal", err)
	}
	if len(f.urls) != 1 {
		t.Errorf("requests = %v; the page was read anyway", f.urls)
	}
}

func TestPage_AFetcherRefusalIsReportedAsItself(t *testing.T) {
	// The old check matched three substrings of webfetch's message from a
	// package that cannot import webfetch. The sentinel is mapped by the
	// adapter that owns both (cmd/mavend/crawls.go).
	f := &timedFetcher{errs: map[string]error{
		"https://example.org/robots.txt": fmt.Errorf("%w: host is not allowed", ErrFetchRefused),
	}}
	c := New(f, Config{UserAgent: "Maven/1.0"})
	if _, err := c.Page(context.Background(), "https://example.org/a"); !errors.Is(err, ErrFetchRefused) {
		t.Fatalf("Page = %v; want the fetcher's own refusal, not a robots verdict", err)
	}
}

func TestParseRobots_MostSpecificAgentWinsRegardlessOfOrder(t *testing.T) {
	body := "User-agent: maven\nDisallow: /private\n\nUser-agent: mav\nDisallow: /\n"
	r := ParseRobots(body, "maven/1.0")
	if !r.Allowed("/public") {
		t.Error("the shorter agent group won by file order; the longer prefix is the more specific match")
	}
	if r.Allowed("/private") {
		t.Error("the group that names us was not applied")
	}
}
