Merge the eval checks sweep (#228)
Named three repeated thresholds in the persona checks: the shortest a word can be and still carry a matched suffix, the word window around a self-reference marker, and the plural-verb length floor. Five bare length tests and three bare window bounds now read from them. One doubled sentence in checkAddress's doc block, left by an edit that did not fully replace the old text, is now stated once. No word list moved. They are scoring data and moving one changes what the eval measures. The brief asked for a word-boundary defect and there is none here. The cringe patterns already omit the ASCII-only \b around their Cyrillic alternatives and say why, and the tokenizers match Cyrillic character classes rather than boundaries. (V-581)
This commit is contained in:
@@ -126,6 +126,17 @@ func checkLength(body string) Result {
|
|||||||
// correct, "я напомнил" is not. Both directions matter, which is why this is a
|
// correct, "я напомнил" is not. Both directions matter, which is why this is a
|
||||||
// windowed scan around "я" and not a bare search for masculine endings.
|
// windowed scan around "я" and not a bare search for masculine endings.
|
||||||
|
|
||||||
|
// minInflectedRunes — the shortest a word can be and still carry one of the
|
||||||
|
// verb/adjective endings this file matches (masculine -л, feminine -ла, …).
|
||||||
|
// Below this length a suffix match would be coincidence, not grammar.
|
||||||
|
const minInflectedRunes = 3
|
||||||
|
|
||||||
|
// selfRefWindow — how many words past a self-reference marker ("я"/"ты", …)
|
||||||
|
// still count as belonging to that reference before the sentence has moved
|
||||||
|
// on. Same window on both the forward scan after "я" and the backward scans
|
||||||
|
// in governedByYou and hersNotHis.
|
||||||
|
const selfRefWindow = 3
|
||||||
|
|
||||||
var wordRE = regexp.MustCompile(`[\p{Cyrillic}]+|[,.;:!?…—-]`)
|
var wordRE = regexp.MustCompile(`[\p{Cyrillic}]+|[,.;:!?…—-]`)
|
||||||
|
|
||||||
// secondPerson — pronouns that end the self-reference window. Everything after
|
// secondPerson — pronouns that end the self-reference window. Everything after
|
||||||
@@ -154,7 +165,7 @@ var masculinePredicative = map[string]bool{
|
|||||||
// ends in -л AND is a form of a verb (Vikunja #526). Every noun the list held is
|
// ends in -л AND is a form of a verb (Vikunja #526). Every noun the list held is
|
||||||
// correctly not a verb, and so are the ones it had not got round to.
|
// correctly not a verb, and so are the ones it had not got round to.
|
||||||
func masculinePast(w string) bool {
|
func masculinePast(w string) bool {
|
||||||
if len([]rune(w)) < 3 {
|
if len([]rune(w)) < minInflectedRunes {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
if !strings.HasSuffix(w, "л") && !strings.HasSuffix(w, "лся") {
|
if !strings.HasSuffix(w, "л") && !strings.HasSuffix(w, "лся") {
|
||||||
@@ -172,7 +183,7 @@ func checkFeminine(body string) Result {
|
|||||||
// Scan the next few words. Stop at punctuation or at a second-person
|
// Scan the next few words. Stop at punctuation or at a second-person
|
||||||
// pronoun: past that point the sentence is about him and masculine is
|
// pronoun: past that point the sentence is about him and masculine is
|
||||||
// correct.
|
// correct.
|
||||||
for j := i + 1; j < len(words) && j <= i+3; j++ {
|
for j := i + 1; j < len(words) && j <= i+selfRefWindow; j++ {
|
||||||
nw := words[j]
|
nw := words[j]
|
||||||
if len(nw) == 1 && !unicode.Is(unicode.Cyrillic, []rune(nw)[0]) {
|
if len(nw) == 1 && !unicode.Is(unicode.Cyrillic, []rune(nw)[0]) {
|
||||||
break
|
break
|
||||||
@@ -259,7 +270,7 @@ var notFeminineVerb = map[string]bool{
|
|||||||
// femininePast reports whether a word looks like a feminine past-tense verb:
|
// femininePast reports whether a word looks like a feminine past-tense verb:
|
||||||
// "отдыхала", "поела", "выспалась".
|
// "отдыхала", "поела", "выспалась".
|
||||||
func femininePast(w string) bool {
|
func femininePast(w string) bool {
|
||||||
if len([]rune(w)) < 3 || notFeminineVerb[w] {
|
if len([]rune(w)) < minInflectedRunes || notFeminineVerb[w] {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return strings.HasSuffix(w, "ла") || strings.HasSuffix(w, "лась")
|
return strings.HasSuffix(w, "ла") || strings.HasSuffix(w, "лась")
|
||||||
@@ -268,7 +279,7 @@ func femininePast(w string) bool {
|
|||||||
// looksFeminineNoun — a crude guard against "зарядка была": a word right before
|
// looksFeminineNoun — a crude guard against "зарядка была": a word right before
|
||||||
// the verb that ends in "а"/"я" and is not itself a verb is probably the subject.
|
// the verb that ends in "а"/"я" and is not itself a verb is probably the subject.
|
||||||
func looksFeminineNoun(w string) bool {
|
func looksFeminineNoun(w string) bool {
|
||||||
if femininePast(w) || len([]rune(w)) < 3 {
|
if femininePast(w) || len([]rune(w)) < minInflectedRunes {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return strings.HasSuffix(w, "а") || strings.HasSuffix(w, "я")
|
return strings.HasSuffix(w, "а") || strings.HasSuffix(w, "я")
|
||||||
@@ -307,7 +318,7 @@ func checkHisGender(body string) Result {
|
|||||||
// hersNotHis — the verb is Maven's own if "я" comes shortly before it, or if the
|
// hersNotHis — the verb is Maven's own if "я" comes shortly before it, or if the
|
||||||
// thing she did was done to him ("напомнила тебе", "проверила за тебя").
|
// thing she did was done to him ("напомнила тебе", "проверила за тебя").
|
||||||
func hersNotHis(words []string, i int) bool {
|
func hersNotHis(words []string, i int) bool {
|
||||||
for j := i - 1; j >= 0 && j >= i-3; j-- {
|
for j := i - 1; j >= 0 && j >= i-selfRefWindow; j-- {
|
||||||
if words[j] == "я" {
|
if words[j] == "я" {
|
||||||
return true
|
return true
|
||||||
}
|
}
|
||||||
@@ -368,12 +379,10 @@ func prevWord(words []string, i int) string {
|
|||||||
// after the pronoun and is flagged wrongly.
|
// after the pronoun and is flagged wrongly.
|
||||||
// - any noun earlier in the message counts as an antecedent, even when it is
|
// - any noun earlier in the message counts as an antecedent, even when it is
|
||||||
// not one ("после обеда он не ел", "выпей воды, он не пил" — both missed).
|
// not one ("после обеда он не ел", "выпей воды, он не пил" — both missed).
|
||||||
// Verbs and time words no longer count, which covers the usual nudge, but a
|
// Verbs and the common time words are stoplisted so the usual nudge opening
|
||||||
// plain noun before the pronoun still blinds it. The
|
// does not blind it, but a message with any other noun in front still
|
||||||
// common time words are stoplisted so the usual nudge opening does not
|
// slips through. This is the check's real hole; widening it further would
|
||||||
// blind it, but a message with any other noun in front still slips through.
|
// start flagging legitimate third-party messages, so it stops here.
|
||||||
// This is the check's real hole; widening it further would start flagging
|
|
||||||
// legitimate third-party messages, so it stops here.
|
|
||||||
// - a message that opens with "ты" and only later slips into "он" is missed,
|
// - a message that opens with "ты" and only later slips into "он" is missed,
|
||||||
// because "ты" itself is skipped but the words around it are not.
|
// because "ты" itself is skipped but the words around it are not.
|
||||||
// - formal address outside these endings (short adjectives, "вашими" style
|
// - formal address outside these endings (short adjectives, "вашими" style
|
||||||
@@ -400,10 +409,14 @@ var prepositions = map[string]bool{
|
|||||||
"под": true, "про": true, "без": true, "для": true, "через": true,
|
"под": true, "про": true, "без": true, "для": true, "через": true,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// minPluralVerbRunes — the -ите/-ете/-йте/-ьте endings pluralVerb matches are
|
||||||
|
// three runes on their own, so anything shorter can't carry a stem plus one.
|
||||||
|
const minPluralVerbRunes = 5
|
||||||
|
|
||||||
// pluralVerb reports whether a word looks like a plural/formal verb form:
|
// pluralVerb reports whether a word looks like a plural/formal verb form:
|
||||||
// "приходите", "выпейте", "забудьте", "хотите".
|
// "приходите", "выпейте", "забудьте", "хотите".
|
||||||
func pluralVerb(w string) bool {
|
func pluralVerb(w string) bool {
|
||||||
if len([]rune(w)) < 5 {
|
if len([]rune(w)) < minPluralVerbRunes {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return strings.HasSuffix(w, "ите") || strings.HasSuffix(w, "ете") ||
|
return strings.HasSuffix(w, "ите") || strings.HasSuffix(w, "ете") ||
|
||||||
@@ -443,7 +456,7 @@ var notAnAntecedent = map[string]bool{
|
|||||||
// slipped through with "попробуй" taken for the person being talked about).
|
// slipped through with "попробуй" taken for the person being talked about).
|
||||||
func looksVerb(w string) bool {
|
func looksVerb(w string) bool {
|
||||||
r := []rune(w)
|
r := []rune(w)
|
||||||
if len(r) < 3 {
|
if len(r) < minInflectedRunes {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
for _, suf := range []string{
|
for _, suf := range []string{
|
||||||
@@ -681,11 +694,11 @@ func checkEllipsis(body string) Result {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// governedByYou reports whether "ты" stands close enough in front of the verb
|
// governedByYou reports whether "ты" stands close enough in front of the verb
|
||||||
// at index i to be its subject. Three words, the same window checkFeminine's
|
// at index i to be its subject. selfRefWindow words, the same window
|
||||||
// first pass uses after "я", and it stops at a first-person pronoun so "ты
|
// checkFeminine's first pass uses after "я", and it stops at a first-person
|
||||||
// просил, я напомнил" still trips.
|
// pronoun so "ты просил, я напомнил" still trips.
|
||||||
func governedByYou(words []string, i int) bool {
|
func governedByYou(words []string, i int) bool {
|
||||||
for j := i - 1; j >= 0 && j >= i-3; j-- {
|
for j := i - 1; j >= 0 && j >= i-selfRefWindow; j-- {
|
||||||
switch words[j] {
|
switch words[j] {
|
||||||
case "ты":
|
case "ты":
|
||||||
return true
|
return true
|
||||||
|
|||||||
Reference in New Issue
Block a user