package crawl

import (
	"html"
	"regexp"
	"strings"
)

// HTML → text, with a regexp and no tokenizer.
//
// golang.org/x/net/html is not vendored and the network is not assumed, so this
// is stdlib. That is less of a compromise than it sounds: the unit of context
// here is a few hundred words for a 4096-token model to read, exactly like the
// Kiwix snippet, so what matters is dropping script/style/nav noise and keeping
// paragraph boundaries. A DOM would buy correctness on malformed markup that is
// then thrown away by truncation anyway.
//
// What this deliberately does NOT do: run JavaScript, follow links, or extract
// structured fields with CSS selectors or an LLM prompt. The plan's step 2 asked
// for the last of those; see docs/plans/14-web-crawler.md for why it was left
// out for now.

var (
	// RE2 has no backreferences, so each tag pair is spelled out rather than
	// captured and matched against itself.
	dropRE  = regexp.MustCompile(pairsRE("script", "style", "noscript", "svg", "head", "nav", "footer", "form"))
	titleRE = regexp.MustCompile(`(?is)<title\b[^>]*>(.*?)</title>`)
	h1RE    = regexp.MustCompile(`(?is)<h1\b[^>]*>(.*?)</h1>`)
	// Block-level tags become newlines so paragraphs survive as paragraphs.
	blockRE   = regexp.MustCompile(`(?is)</?(p|div|br|li|tr|h[1-6]|section|article|blockquote|pre)\b[^>]*>`)
	tagRE     = regexp.MustCompile(`(?s)<[^>]*>`)
	commentRE = regexp.MustCompile(`(?s)<!--.*?-->`)
	spaceRE   = regexp.MustCompile(`[ \t\f\v]+`)
	blankRE   = regexp.MustCompile(`\n{2,}`)
)

// pairsRE builds `(?is)<tag …>…</tag>|…` for the given tags.
func pairsRE(tags ...string) string {
	parts := make([]string, 0, len(tags))
	for _, t := range tags {
		parts = append(parts, `<`+t+`\b[^>]*>.*?</`+t+`>`)
	}
	return `(?is)` + strings.Join(parts, "|")
}

// Page is an extracted page.
type Page struct {
	URL   string
	Title string
	Text  string // plain text, paragraphs separated by single newlines
}

// Extract turns a fetched HTML document into a Page. maxRunes caps the text (0 ⇒
// DefaultMaxRunes); the cap is on runes, not bytes, because a Russian page cut
// at a byte boundary ends in half a letter.
func Extract(url string, body []byte, maxRunes int) Page {
	if maxRunes <= 0 {
		maxRunes = DefaultMaxRunes
	}
	s := string(body)
	s = commentRE.ReplaceAllString(s, " ")

	title := firstGroup(titleRE, s)
	if title == "" {
		title = firstGroup(h1RE, s)
	}

	s = dropRE.ReplaceAllString(s, "\n")
	s = blockRE.ReplaceAllString(s, "\n")
	s = tagRE.ReplaceAllString(s, " ")
	s = html.UnescapeString(s)
	s = spaceRE.ReplaceAllString(s, " ")

	var lines []string
	for _, l := range strings.Split(s, "\n") {
		if l = strings.TrimSpace(l); l != "" {
			lines = append(lines, l)
		}
	}
	text := blankRE.ReplaceAllString(strings.Join(lines, "\n"), "\n")

	return Page{URL: url, Title: title, Text: TrimRunes(text, maxRunes)}
}

// DefaultMaxRunes — how much of a page is kept. ~4000 runes is a long answer's
// worth of context and still leaves room in a 4096-token window for the prompt
// and the reply.
const DefaultMaxRunes = 4000

func firstGroup(re *regexp.Regexp, s string) string {
	m := re.FindStringSubmatch(s)
	if len(m) < 2 {
		return ""
	}
	t := tagRE.ReplaceAllString(m[1], " ")
	return strings.TrimSpace(strings.Join(strings.Fields(html.UnescapeString(t)), " "))
}

// TrimRunes cuts s to at most max runes, on a rune boundary.
func TrimRunes(s string, max int) string {
	r := []rune(s)
	if len(r) <= max {
		return s
	}
	return strings.TrimSpace(string(r[:max])) + "…"
}