claude
96d97e8964
A ZIM title carries a leading capital and the utterance does not: /A/фотосинтез is a 404 and /A/Фотосинтез is a 200. TitleCandidates tries the spoken form first, so a title that begins lowercase on purpose keeps its chance. That takes the measurement from four right to five, and the fifth is the one that mattered. "столица Франции" returned "Список столиц Олимпийских игр" and now returns Париж, through a title redirect the ZIM already held. The 2026-08-05 measurement named that case as the one no lexical signal could reach. Retrieval by title reaches it. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_013ptwopxyo3Z2kwFckHkLvN
119 lines
3.8 KiB
Go
119 lines
3.8 KiB
Go
package kiwix
|
||
|
||
import (
|
||
"strings"
|
||
"unicode"
|
||
|
||
"github.com/kami/maven/internal/lexicon"
|
||
"github.com/kami/maven/internal/morph"
|
||
)
|
||
|
||
// Topic reduces a question to the thing it is about, because Kiwix ranks by
|
||
// keyword overlap and a whole sentence buries the keyword that matters.
|
||
//
|
||
// This package's own doc says it: "why is the sky blue" finds a TV episode.
|
||
// Measured against the Russian ZIM on 2026-08-09, the sentence and the topic
|
||
// return different articles for the same question. "кто написал Войну и мир"
|
||
// returns "Радуйся, мир (Доктор Кто)"; "Войну и мир" returns the novel first.
|
||
// "что такое TCP" returns "Перехват TCP-соединения"; "TCP" returns TCP. The
|
||
// English path had a rewriter doing this with a model call. The Russian path
|
||
// reads the book verbatim (V-508) and had nothing.
|
||
//
|
||
// It drops three things off the front and stops: the narrative request, the
|
||
// interrogative, and a verb sitting between them and the noun. Everything else
|
||
// is kept, because a word this cannot classify is more likely the topic than
|
||
// noise. An empty return means the utterance was question words alone, and the
|
||
// caller searches the sentence as before.
|
||
func Topic(utterance string) string {
|
||
words := strings.Fields(strings.TrimSpace(utterance))
|
||
cut := 0
|
||
for cut < len(words) {
|
||
w := strings.Trim(strings.ToLower(words[cut]), ".,!?…:;\"'«»")
|
||
if w == "" {
|
||
cut++
|
||
continue
|
||
}
|
||
switch {
|
||
case inList(lexicon.NarrativeRequests(), w),
|
||
inList(lexicon.Interrogatives(), w),
|
||
inList(lexicon.FirstPerson(), w),
|
||
// "что ТАКОЕ x", "кто ТАКОЙ x" — the copula that only ever follows
|
||
// an interrogative, and never a topic on its own.
|
||
cut > 0 && isCopula(w),
|
||
// "расскажи ПРО x", "о x". One-letter and two-letter prepositions
|
||
// are not a closed class worth a lexicon set of their own.
|
||
cut > 0 && isLeadingPreposition(w),
|
||
// "кто НАПИСАЛ Войну и мир". A verb here is the question's own
|
||
// verb, not part of the title. Only after something was already
|
||
// dropped, so "написал отчёт" as a topic survives intact.
|
||
cut > 0 && morph.IsVerbForm(w):
|
||
cut++
|
||
default:
|
||
// The question mark is the sentence's, not the title's, and Kiwix
|
||
// carries it into the keyword match.
|
||
topic := strings.TrimRight(strings.Join(words[cut:], " "), " .,!?…:;\"'«»")
|
||
if !hasLetter(topic) {
|
||
return ""
|
||
}
|
||
return topic
|
||
}
|
||
}
|
||
return ""
|
||
}
|
||
|
||
// TitleCandidates is the topic as it might be titled, best first.
|
||
//
|
||
// A ZIM title is capitalized and the utterance is not: measured on 2026-08-09,
|
||
// `/A/фотосинтез` is a 404 and `/A/Фотосинтез` is a 200. The spoken form is
|
||
// tried first anyway, because a title that begins lowercase on purpose
|
||
// ("iPhone") would not survive capitalizing it. Both are one request each
|
||
// against a server on the same box, and a miss is a 404 rather than a wrong
|
||
// article.
|
||
func TitleCandidates(topic string) []string {
|
||
if topic == "" {
|
||
return nil
|
||
}
|
||
r := []rune(topic)
|
||
up := unicode.ToUpper(r[0])
|
||
if up == r[0] {
|
||
return []string{topic}
|
||
}
|
||
return []string{topic, string(up) + string(r[1:])}
|
||
}
|
||
|
||
func isCopula(w string) bool {
|
||
switch w {
|
||
case "такое", "такой", "такая", "такие", "is", "are", "was", "were":
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func isLeadingPreposition(w string) bool {
|
||
switch w {
|
||
case "про", "о", "об", "обо", "по", "about", "of", "on":
|
||
return true
|
||
}
|
||
return false
|
||
}
|
||
|
||
func inList(list []string, w string) bool {
|
||
for _, x := range list {
|
||
if x == w {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// hasLetter is the guard against a topic that reduced to punctuation or digits
|
||
// alone, which no ZIM title matches.
|
||
func hasLetter(s string) bool {
|
||
for _, r := range s {
|
||
if unicode.IsLetter(r) {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|