kami
2c1b0eede0
The network fallback behind the local sources, off unless configured. internal/crawl is pure: a stdlib robots.txt parser (group specificity, wildcards, Crawl-delay, cached per host), HTML-to-plaintext extraction, and a watcher that notes a watched page only when its text changed. It has no store access and no net/http; cmd/mavend/crawls.go is the impure half. Every limit is code and tested: the guarded fetcher from #258 enforces the host allowlist/denylist, refuses private addresses in the dialer Control hook (so DNS rebinding and each redirect hop are covered), caps size and redirects, times out, and spaces requests per host. A robots.txt Disallow is refused with no override. On demand, reading is a query source placed last in the chain, after his memory, his notes, and the local Kiwix ZIMs once those are wired: no URL in the utterance means no fetch, and only the URL ever leaves the box. Scheduled watches write notes and announce nothing. The vendored tree has no x/net/html, goquery or temoto/robotstxt, so the parsers are stdlib. No new dependency.
107 lines
3.5 KiB
Go
107 lines
3.5 KiB
Go
package crawl
|
|
|
|
import (
|
|
"html"
|
|
"regexp"
|
|
"strings"
|
|
)
|
|
|
|
// HTML → text, with a regexp and no tokenizer.
|
|
//
|
|
// golang.org/x/net/html is not vendored and the network is not assumed, so this
|
|
// is stdlib. That is less of a compromise than it sounds: the unit of context
|
|
// here is a few hundred words for a 4096-token model to read, exactly like the
|
|
// Kiwix snippet, so what matters is dropping script/style/nav noise and keeping
|
|
// paragraph boundaries. A DOM would buy correctness on malformed markup that is
|
|
// then thrown away by truncation anyway.
|
|
//
|
|
// What this deliberately does NOT do: run JavaScript, follow links, or extract
|
|
// structured fields with CSS selectors or an LLM prompt. The plan's step 2 asked
|
|
// for the last of those; see docs/plans/14-web-crawler.md for why it was left
|
|
// out for now.
|
|
|
|
var (
|
|
// RE2 has no backreferences, so each tag pair is spelled out rather than
|
|
// captured and matched against itself.
|
|
dropRE = regexp.MustCompile(pairsRE("script", "style", "noscript", "svg", "head", "nav", "footer", "form"))
|
|
titleRE = regexp.MustCompile(`(?is)<title\b[^>]*>(.*?)</title>`)
|
|
h1RE = regexp.MustCompile(`(?is)<h1\b[^>]*>(.*?)</h1>`)
|
|
// Block-level tags become newlines so paragraphs survive as paragraphs.
|
|
blockRE = regexp.MustCompile(`(?is)</?(p|div|br|li|tr|h[1-6]|section|article|blockquote|pre)\b[^>]*>`)
|
|
tagRE = regexp.MustCompile(`(?s)<[^>]*>`)
|
|
commentRE = regexp.MustCompile(`(?s)<!--.*?-->`)
|
|
spaceRE = regexp.MustCompile(`[ \t\f\v]+`)
|
|
blankRE = regexp.MustCompile(`\n{2,}`)
|
|
)
|
|
|
|
// pairsRE builds `(?is)<tag …>…</tag>|…` for the given tags.
|
|
func pairsRE(tags ...string) string {
|
|
parts := make([]string, 0, len(tags))
|
|
for _, t := range tags {
|
|
parts = append(parts, `<`+t+`\b[^>]*>.*?</`+t+`>`)
|
|
}
|
|
return `(?is)` + strings.Join(parts, "|")
|
|
}
|
|
|
|
// Page is an extracted page.
|
|
type Page struct {
|
|
URL string
|
|
Title string
|
|
Text string // plain text, paragraphs separated by single newlines
|
|
}
|
|
|
|
// Extract turns a fetched HTML document into a Page. maxRunes caps the text (0 ⇒
|
|
// DefaultMaxRunes); the cap is on runes, not bytes, because a Russian page cut
|
|
// at a byte boundary ends in half a letter.
|
|
func Extract(url string, body []byte, maxRunes int) Page {
|
|
if maxRunes <= 0 {
|
|
maxRunes = DefaultMaxRunes
|
|
}
|
|
s := string(body)
|
|
s = commentRE.ReplaceAllString(s, " ")
|
|
|
|
title := firstGroup(titleRE, s)
|
|
if title == "" {
|
|
title = firstGroup(h1RE, s)
|
|
}
|
|
|
|
s = dropRE.ReplaceAllString(s, "\n")
|
|
s = blockRE.ReplaceAllString(s, "\n")
|
|
s = tagRE.ReplaceAllString(s, " ")
|
|
s = html.UnescapeString(s)
|
|
s = spaceRE.ReplaceAllString(s, " ")
|
|
|
|
var lines []string
|
|
for _, l := range strings.Split(s, "\n") {
|
|
if l = strings.TrimSpace(l); l != "" {
|
|
lines = append(lines, l)
|
|
}
|
|
}
|
|
text := blankRE.ReplaceAllString(strings.Join(lines, "\n"), "\n")
|
|
|
|
return Page{URL: url, Title: title, Text: TrimRunes(text, maxRunes)}
|
|
}
|
|
|
|
// DefaultMaxRunes — how much of a page is kept. ~4000 runes is a long answer's
|
|
// worth of context and still leaves room in a 4096-token window for the prompt
|
|
// and the reply.
|
|
const DefaultMaxRunes = 4000
|
|
|
|
func firstGroup(re *regexp.Regexp, s string) string {
|
|
m := re.FindStringSubmatch(s)
|
|
if len(m) < 2 {
|
|
return ""
|
|
}
|
|
t := tagRE.ReplaceAllString(m[1], " ")
|
|
return strings.TrimSpace(strings.Join(strings.Fields(html.UnescapeString(t)), " "))
|
|
}
|
|
|
|
// TrimRunes cuts s to at most max runes, on a rune boundary.
|
|
func TrimRunes(s string, max int) string {
|
|
r := []rune(s)
|
|
if len(r) <= max {
|
|
return s
|
|
}
|
|
return strings.TrimSpace(string(r[:max])) + "…"
|
|
}
|