package crawl

import (
	"testing"
	"time"
)

const robotsBody = `# a comment
User-agent: *
Disallow: /private
Disallow: /tmp/
Crawl-delay: 5

User-agent: Maven
Disallow: /
Allow: /public
`

func TestParseRobotsPicksTheMostSpecificGroup(t *testing.T) {
	// The Maven group applies to us even though we send a longer UA string.
	r := ParseRobots(robotsBody, "Maven/1.0 (self-hosted personal assistant)")
	if r.Allowed("/anything") {
		t.Error("Disallow: / in our own group was ignored")
	}
	if !r.Allowed("/public/page") {
		t.Error("Allow: /public must beat the shorter Disallow: /")
	}

	// A different agent falls into the * group.
	star := ParseRobots(robotsBody, "SomeoneElse/2")
	if !star.Allowed("/anything") {
		t.Error("the * group disallows nothing but /private and /tmp/")
	}
	if star.Allowed("/private/x") || star.Allowed("/tmp/") {
		t.Error("the * group's disallows were not applied")
	}
	if star.Delay != 5*time.Second {
		t.Errorf("crawl-delay = %v, want 5s", star.Delay)
	}
}

func TestParseRobotsEmptyMeansAllowAll(t *testing.T) {
	for _, body := range []string{"", "# nothing here\n", "User-agent: *\nDisallow:\n"} {
		if !ParseRobots(body, "Maven").Allowed("/whatever") {
			t.Errorf("body %q must allow everything", body)
		}
	}
}

func TestRobotsWildcards(t *testing.T) {
	r := ParseRobots("User-agent: *\nDisallow: /*.pdf$\nDisallow: /a/*/secret\n", "Maven")
	if r.Allowed("/docs/manual.pdf") {
		t.Error("*.pdf$ did not match")
	}
	if !r.Allowed("/docs/manual.pdf.html") {
		t.Error("$ must anchor at the end")
	}
	if r.Allowed("/a/b/secret") {
		t.Error("/a/*/secret did not match")
	}
	if !r.Allowed("/a/b/public") {
		t.Error("unrelated path was refused")
	}
}

// Consecutive User-agent lines share one group, which is common in the wild.
func TestRobotsSharedGroup(t *testing.T) {
	r := ParseRobots("User-agent: Googlebot\nUser-agent: Maven\nDisallow: /x\n", "Maven/1.0")
	if r.Allowed("/x/y") {
		t.Fatal("a shared group's rules were not applied to the second agent")
	}
}

func TestRobotsCacheTTL(t *testing.T) {
	c := newRobotsCache(time.Minute)
	now := time.Now()
	c.put("example.com", ParseRobots("User-agent: *\nDisallow: /\n", "Maven"), now)
	if _, ok := c.get("example.com", now.Add(30*time.Second)); !ok {
		t.Error("a fresh entry must be served from cache")
	}
	if _, ok := c.get("example.com", now.Add(2*time.Minute)); ok {
		t.Error("an expired entry must be re-read")
	}
}