Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 0b994ff1c3 | |||
| 0b1efe4911 | |||
| 580959f856 | |||
| bf2587c7fa | |||
| d7a43afd90 |
@@ -15,6 +15,7 @@ import (
|
|||||||
"encoding/base64"
|
"encoding/base64"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"log"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"time"
|
"time"
|
||||||
@@ -329,7 +330,15 @@ func Load(path string) (*Config, error) {
|
|||||||
// Expand ${VAR} or $VAR patterns from environment variables. This lets
|
// Expand ${VAR} or $VAR patterns from environment variables. This lets
|
||||||
// secrets live in env (docker-compose env_file) rather than the config
|
// secrets live in env (docker-compose env_file) rather than the config
|
||||||
// file committed to git.
|
// file committed to git.
|
||||||
expanded := os.ExpandEnv(string(b))
|
expanded, missing := expandEnv(string(b))
|
||||||
|
if len(missing) > 0 {
|
||||||
|
// An unset variable expands to "", which every block reads as "not
|
||||||
|
// configured" and none of them complains about. That is the intended
|
||||||
|
// behaviour and it stays: CI parses this same file with no secrets
|
||||||
|
// present. What was missing is the line telling the operator which
|
||||||
|
// capability he just turned off by forgetting an env file.
|
||||||
|
log.Printf("config: %s references unset environment variables %v — those settings are empty, so whatever they configure is off", path, missing)
|
||||||
|
}
|
||||||
var c Config
|
var c Config
|
||||||
if err := json.Unmarshal([]byte(expanded), &c); err != nil {
|
if err := json.Unmarshal([]byte(expanded), &c); err != nil {
|
||||||
return nil, fmt.Errorf("config: parse %s: %w", path, err)
|
return nil, fmt.Errorf("config: parse %s: %w", path, err)
|
||||||
@@ -341,6 +350,23 @@ func Load(path string) (*Config, error) {
|
|||||||
return &c, nil
|
return &c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// expandEnv is os.ExpandEnv plus the names it could not resolve, each reported
|
||||||
|
// once and in the order the file mentions them. A variable set to the empty
|
||||||
|
// string counts as set: the operator wrote it down, so he meant it.
|
||||||
|
func expandEnv(s string) (string, []string) {
|
||||||
|
var missing []string
|
||||||
|
seen := map[string]bool{}
|
||||||
|
out := os.Expand(s, func(name string) string {
|
||||||
|
v, ok := os.LookupEnv(name)
|
||||||
|
if !ok && !seen[name] {
|
||||||
|
seen[name] = true
|
||||||
|
missing = append(missing, name)
|
||||||
|
}
|
||||||
|
return v
|
||||||
|
})
|
||||||
|
return out, missing
|
||||||
|
}
|
||||||
|
|
||||||
func (c *Config) applyDefaults() {
|
func (c *Config) applyDefaults() {
|
||||||
if c.IntakeJournal == 0 {
|
if c.IntakeJournal == 0 {
|
||||||
c.IntakeJournal = DefaultIntakeJournal
|
c.IntakeJournal = DefaultIntakeJournal
|
||||||
|
|||||||
@@ -65,7 +65,7 @@ func mustLoad() lexiconFile {
|
|||||||
"day_offsets", "weekdays", "weekdays_english", "months_genitive", "hours_spoken",
|
"day_offsets", "weekdays", "weekdays_english", "months_genitive", "hours_spoken",
|
||||||
"not_place_after_v", "parts_of_day", "reminder_verbs", "half_hour",
|
"not_place_after_v", "parts_of_day", "reminder_verbs", "half_hour",
|
||||||
"filler_particles", "task_done_words", "task_drop_words",
|
"filler_particles", "task_done_words", "task_drop_words",
|
||||||
"confirm_yes", "confirm_no",
|
"confirm_yes", "confirm_no", "hour_units", "minute_units",
|
||||||
} {
|
} {
|
||||||
s, ok := f.Sets[name]
|
s, ok := f.Sets[name]
|
||||||
if !ok || (len(s.Words) == 0 && len(s.Values) == 0) {
|
if !ok || (len(s.Words) == 0 && len(s.Values) == 0) {
|
||||||
@@ -139,7 +139,40 @@ func TaskDropWords() []string { return words("task_drop_words") }
|
|||||||
// making the utterance a request of its own. A caller strips these (along with
|
// making the utterance a request of its own. A caller strips these (along with
|
||||||
// the numbers and the other closed time sets) to see whether an utterance
|
// the numbers and the other closed time sets) to see whether an utterance
|
||||||
// carries any content beside the value it was asked for. See the set's note.
|
// carries any content beside the value it was asked for. See the set's note.
|
||||||
func SlotValueFrame() []string { return words("slot_value_frame") }
|
// The hour and the minute nouns are part of the frame and are kept in their own
|
||||||
|
// sets, so there is one copy of each closed class rather than a copy per caller.
|
||||||
|
func SlotValueFrame() []string {
|
||||||
|
out := words("slot_value_frame")
|
||||||
|
out = append(out, HourUnits()...)
|
||||||
|
out = append(out, MinuteUnits()...)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
// HourUnits returns every form of the hour noun, and MinuteUnits every form of
|
||||||
|
// the minute noun. One home for each, because four router sets used to list the
|
||||||
|
// hour and all four stopped at "часу" (V-609). A caller folding time words into
|
||||||
|
// one set reads these; a caller asking about a single word reads IsHourUnit or
|
||||||
|
// IsMinuteUnit.
|
||||||
|
func HourUnits() []string { return words("hour_units") }
|
||||||
|
|
||||||
|
// MinuteUnits — see HourUnits.
|
||||||
|
func MinuteUnits() []string { return words("minute_units") }
|
||||||
|
|
||||||
|
// IsHourUnit reports whether a word is the hour noun in any form.
|
||||||
|
func IsHourUnit(word string) bool { return inSet("hour_units", word) }
|
||||||
|
|
||||||
|
// IsMinuteUnit reports whether a word is the minute noun in any form.
|
||||||
|
func IsMinuteUnit(word string) bool { return inSet("minute_units", word) }
|
||||||
|
|
||||||
|
func inSet(set, word string) bool {
|
||||||
|
w := norm(word)
|
||||||
|
for _, s := range ru.Sets[set].Words {
|
||||||
|
if w == s {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
// DialogueCancel returns the ways he calls off the request Maven is assembling.
|
// DialogueCancel returns the ways he calls off the request Maven is assembling.
|
||||||
// Distinct from TaskDropWords, which abandons an item that already exists.
|
// Distinct from TaskDropWords, which abandons an item that already exists.
|
||||||
|
|||||||
@@ -210,6 +210,20 @@
|
|||||||
"передумал", "передумала", "неактуально"
|
"передумал", "передумала", "неактуально"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"hour_units": {
|
||||||
|
"note": "Every form of the hour noun, Russian and English (V-609). One home for a closed class that four router sets used to list separately, and all four stopped at \"часу\": \"напомни к двум часам\" lost its hour and the reminder was left asking \"Когда?\". Russian declines, so the dative plural is as ordinary a way to say an hour as the accusative singular. A caller that folds time words into one set reads HourUnits; a caller asking about one word reads IsHourUnit.",
|
||||||
|
"words": [
|
||||||
|
"час", "часа", "часов", "часу", "часам", "часами", "часах",
|
||||||
|
"hour", "hours"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"minute_units": {
|
||||||
|
"note": "Every form of the minute noun, Russian and English (V-609). Same class as hour_units one noun over, and it had the same gap: the dative plural \"минутам\" was missing everywhere \"минут\" and \"минуты\" were present.",
|
||||||
|
"words": [
|
||||||
|
"минута", "минуты", "минуту", "минут", "минуте", "минутам", "минутами", "минутах",
|
||||||
|
"minute", "minutes"
|
||||||
|
]
|
||||||
|
},
|
||||||
"slot_value_frame": {
|
"slot_value_frame": {
|
||||||
"note": "The words that can stand around a bare slot value without making the utterance a request of its own (Vikunja #560). Prepositions, hedges and the nouns a spoken time is built from: strip these, the numbers, the interrogatives, the filler particles and the other time sets, and whatever is left is the utterance's OWN content. \"а что если в 11:00\" leaves nothing and is an answer; \"какая сейчас погода в Риме\" leaves \"погода\" and \"Риме\" and is not. Closed because each part of it is closed — Russian has a fixed list of prepositions, and a clock is built from a fixed list of nouns. It is not a stopword list: a word goes in only if it can never be the thing he is asking about.",
|
"note": "The words that can stand around a bare slot value without making the utterance a request of its own (Vikunja #560). Prepositions, hedges and the nouns a spoken time is built from: strip these, the numbers, the interrogatives, the filler particles and the other time sets, and whatever is left is the utterance's OWN content. \"а что если в 11:00\" leaves nothing and is an answer; \"какая сейчас погода в Риме\" leaves \"погода\" and \"Риме\" and is not. Closed because each part of it is closed — Russian has a fixed list of prepositions, and a clock is built from a fixed list of nouns. It is not a stopword list: a word goes in only if it can never be the thing he is asking about.",
|
||||||
"words": [
|
"words": [
|
||||||
@@ -217,10 +231,10 @@
|
|||||||
"at", "on", "in", "by", "to", "till", "until", "after", "before", "about", "for",
|
"at", "on", "in", "by", "to", "till", "until", "after", "before", "about", "for",
|
||||||
"нет", "не", "да", "ага", "угу", "ой", "ох", "тогда", "лучше", "может", "можно", "наверное", "наверно", "пожалуй", "точнее", "скорее", "если", "пусть", "прости", "извини", "слушай", "значит", "как-то", "типа", "вообще-то",
|
"нет", "не", "да", "ага", "угу", "ой", "ох", "тогда", "лучше", "может", "можно", "наверное", "наверно", "пожалуй", "точнее", "скорее", "если", "пусть", "прости", "извини", "слушай", "значит", "как-то", "типа", "вообще-то",
|
||||||
"no", "yes", "yeah", "ok", "okay", "sorry", "maybe", "actually", "rather", "then", "well",
|
"no", "yes", "yeah", "ok", "okay", "sorry", "maybe", "actually", "rather", "then", "well",
|
||||||
"час", "часа", "часов", "часу", "часам", "минут", "минута", "минуты", "минуту", "минутах", "полдень", "полночь", "полдня",
|
"полдень", "полночь", "полдня",
|
||||||
"утра", "утро", "утру", "дня", "день", "днями", "вечера", "вечер", "вечеру", "ночи", "ночь", "ночью",
|
"утра", "утро", "утру", "дня", "день", "днями", "вечера", "вечер", "вечеру", "ночи", "ночь", "ночью",
|
||||||
"сейчас", "теперь", "сегодняшний", "ближайший", "ближайшее",
|
"сейчас", "теперь", "сегодняшний", "ближайший", "ближайшее",
|
||||||
"hour", "hours", "minute", "minutes", "noon", "midnight", "am", "pm", "oclock", "now"
|
"noon", "midnight", "am", "pm", "oclock", "now"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
"dialogue_cancel": {
|
"dialogue_cancel": {
|
||||||
|
|||||||
+33
-7
@@ -5,6 +5,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"log"
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
@@ -46,6 +47,7 @@ type Pair struct {
|
|||||||
interval time.Duration
|
interval time.Duration
|
||||||
http *http.Client
|
http *http.Client
|
||||||
stop chan struct{}
|
stop chan struct{}
|
||||||
|
stopOnce sync.Once
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrRemoteUnavailable — the workstation model was required and is not
|
// ErrRemoteUnavailable — the workstation model was required and is not
|
||||||
@@ -107,13 +109,11 @@ func (p *Pair) Start(ctx context.Context) {
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
|
|
||||||
// Stop ends the prober. Idempotent.
|
// Stop ends the prober. Idempotent, and safe from two goroutines at once. The
|
||||||
|
// check-then-close it replaced let both callers see an open channel and the
|
||||||
|
// second close panicked, which turned a shutdown race into a crash.
|
||||||
func (p *Pair) Stop() {
|
func (p *Pair) Stop() {
|
||||||
select {
|
p.stopOnce.Do(func() { close(p.stop) })
|
||||||
case <-p.stop:
|
|
||||||
default:
|
|
||||||
close(p.stop)
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Available reports whether the workstation will take work right now. It reads
|
// Available reports whether the workstation will take work right now. It reads
|
||||||
@@ -170,7 +170,9 @@ func (p *Pair) Complete(ctx context.Context, r Req) (string, error) {
|
|||||||
}
|
}
|
||||||
why := "workstation down"
|
why := "workstation down"
|
||||||
if p.Available() {
|
if p.Available() {
|
||||||
out, err := p.remote.Complete(ctx, r)
|
rctx, cancel := remoteBudget(ctx)
|
||||||
|
out, err := p.remote.Complete(rctx, r)
|
||||||
|
cancel()
|
||||||
if err == nil {
|
if err == nil {
|
||||||
log.Print("llm: served by the workstation model")
|
log.Print("llm: served by the workstation model")
|
||||||
return out, nil
|
return out, nil
|
||||||
@@ -184,6 +186,30 @@ func (p *Pair) Complete(ctx context.Context, r Req) (string, error) {
|
|||||||
return p.floor.Complete(ctx, r)
|
return p.floor.Complete(ctx, r)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// remoteBudget bounds the workstation attempt so the floor still has time to
|
||||||
|
// answer. A turn carrying a deadline used to hand the whole of it to the
|
||||||
|
// remote, so a workstation that accepted the connection and then hung ate the
|
||||||
|
// budget and the fallback ran on an already-expired context: the floor
|
||||||
|
// returned the deadline error and the turn broke on the workstation being
|
||||||
|
// slow, which docs/offload.md says must never happen. Half is the split
|
||||||
|
// because both halves have to be able to finish, and there is no reason to
|
||||||
|
// prefer either one when the remote is the part that failed.
|
||||||
|
//
|
||||||
|
// A context with no deadline is left alone. The remote client's own timeout
|
||||||
|
// (workstation.timeout, 90s by default) bounds it there, and shortening that
|
||||||
|
// silently would change the configured budget.
|
||||||
|
func remoteBudget(ctx context.Context) (context.Context, context.CancelFunc) {
|
||||||
|
dl, ok := ctx.Deadline()
|
||||||
|
if !ok {
|
||||||
|
return ctx, func() {}
|
||||||
|
}
|
||||||
|
left := time.Until(dl)
|
||||||
|
if left <= 0 {
|
||||||
|
return ctx, func() {}
|
||||||
|
}
|
||||||
|
return context.WithTimeout(ctx, left/2)
|
||||||
|
}
|
||||||
|
|
||||||
// CompleteRemote runs r on the workstation or refuses. It never falls back,
|
// CompleteRemote runs r on the workstation or refuses. It never falls back,
|
||||||
// because for a world question the resident 1.7B does not answer worse, it
|
// because for a world question the resident 1.7B does not answer worse, it
|
||||||
// invents. Callers turn ErrRemoteUnavailable into a named gap.
|
// invents. Callers turn ErrRemoteUnavailable into a named gap.
|
||||||
|
|||||||
@@ -154,6 +154,48 @@ func TestRemoteErrorMidRequestFallsBack(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A workstation that accepts the connection and then hangs must not spend the
|
||||||
|
// whole turn budget. It used to: the remote got the caller's context unchanged,
|
||||||
|
// so the fallback ran on an expired one and the floor returned the deadline
|
||||||
|
// error instead of an answer. The turn broke on the workstation being slow,
|
||||||
|
// which is the one outcome docs/offload.md rules out.
|
||||||
|
func TestHangingRemoteLeavesTheFloorABudget(t *testing.T) {
|
||||||
|
var floorHits atomic.Int64
|
||||||
|
// released, not r.Context().Done(): httptest.Server.Close waits for the
|
||||||
|
// handler, and a handler that only watches the request context can outlive
|
||||||
|
// the test when the client hangs up without the server noticing.
|
||||||
|
released := make(chan struct{})
|
||||||
|
hang := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
select {
|
||||||
|
case <-released:
|
||||||
|
case <-r.Context().Done():
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer hang.Close()
|
||||||
|
defer close(released)
|
||||||
|
floor := completionServer(t, "floor", &floorHits)
|
||||||
|
up := &atomic.Bool{}
|
||||||
|
up.Store(true)
|
||||||
|
health := healthServer(t, up)
|
||||||
|
|
||||||
|
p := NewPair(New(hang.URL, time.Minute), New(floor.URL, time.Minute), health.URL, time.Hour)
|
||||||
|
p.Start(context.Background())
|
||||||
|
defer p.Stop()
|
||||||
|
if !waitFor(t, p.Available) {
|
||||||
|
t.Fatal("prober never saw the remote come up")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)
|
||||||
|
defer cancel()
|
||||||
|
out, err := p.Complete(ctx, Req{User: "привет"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("complete: %v", err)
|
||||||
|
}
|
||||||
|
if out != "floor" || floorHits.Load() != 1 {
|
||||||
|
t.Fatalf("out = %q, floor hits = %d", out, floorHits.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The naming half of the degradation rule. A world question must not be handed
|
// The naming half of the degradation rule. A world question must not be handed
|
||||||
// to the resident model, because it answers by inventing.
|
// to the resident model, because it answers by inventing.
|
||||||
func TestCompleteRemoteNamesTheGap(t *testing.T) {
|
func TestCompleteRemoteNamesTheGap(t *testing.T) {
|
||||||
|
|||||||
+22
-4
@@ -265,6 +265,16 @@ func (s *Store) Put(kind Kind, mime, source string, data []byte) (Blob, error) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Every return past the reservation has to give it back, so the defer owns
|
||||||
|
// that rather than each error path: a path that forgot over-counted the
|
||||||
|
// store until the next Open re-walked the directory.
|
||||||
|
stored := false
|
||||||
|
defer func() {
|
||||||
|
if fresh && !stored {
|
||||||
|
s.release(b.Size)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
// The sidecar goes first. Written second, a full disk or a crash between
|
// The sidecar goes first. Written second, a full disk or a crash between
|
||||||
// the two left the bytes on disk with no sidecar, and List only sees
|
// the two left the bytes on disk with no sidecar, and List only sees
|
||||||
// sidecars, so Prune could never collect them: Put returned an error and an
|
// sidecars, so Prune could never collect them: Put returned an error and an
|
||||||
@@ -274,11 +284,9 @@ func (s *Store) Put(kind Kind, mime, source string, data []byte) (Blob, error) {
|
|||||||
}
|
}
|
||||||
if err := writeFile(blobPath, data); err != nil {
|
if err := writeFile(blobPath, data); err != nil {
|
||||||
_ = os.Remove(metaPath)
|
_ = os.Remove(metaPath)
|
||||||
if fresh {
|
|
||||||
s.release(b.Size)
|
|
||||||
}
|
|
||||||
return Blob{}, err
|
return Blob{}, err
|
||||||
}
|
}
|
||||||
|
stored = true
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -331,12 +339,22 @@ func (s *Store) PutFile(kind Kind, mime, source, src string) (Blob, error) {
|
|||||||
return Blob{}, err
|
return Blob{}, err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Same reasoning as Put: the reservation is released by one defer, not by
|
||||||
|
// whichever error path remembered to.
|
||||||
|
stored := false
|
||||||
|
defer func() {
|
||||||
|
if fresh && !stored {
|
||||||
|
s.release(b.Size)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
if err := writeMeta(metaPath, b); err != nil {
|
if err := writeMeta(metaPath, b); err != nil {
|
||||||
return Blob{}, err
|
return Blob{}, err
|
||||||
}
|
}
|
||||||
if !fresh {
|
if !fresh {
|
||||||
// Same bytes already here. Drop the spool copy.
|
// Same bytes already here. Drop the spool copy.
|
||||||
_ = os.Remove(src)
|
_ = os.Remove(src)
|
||||||
|
stored = true
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
if err := os.Chmod(src, filePerm); err != nil {
|
if err := os.Chmod(src, filePerm); err != nil {
|
||||||
@@ -344,9 +362,9 @@ func (s *Store) PutFile(kind Kind, mime, source, src string) (Blob, error) {
|
|||||||
}
|
}
|
||||||
if err := os.Rename(src, blobPath); err != nil {
|
if err := os.Rename(src, blobPath); err != nil {
|
||||||
_ = os.Remove(metaPath)
|
_ = os.Remove(metaPath)
|
||||||
s.release(b.Size)
|
|
||||||
return Blob{}, fmt.Errorf("media: move spool: %w", err)
|
return Blob{}, fmt.Errorf("media: move spool: %w", err)
|
||||||
}
|
}
|
||||||
|
stored = true
|
||||||
return b, nil
|
return b, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -287,6 +287,73 @@ func TestPutLeavesNothingWhenTheBytesCannotBeWritten(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// An over-counted store answers ErrStoreFull while the disk has room, and only
|
||||||
|
// the next Open corrects it. So every failed write has to give its reservation
|
||||||
|
// back, not just the one that remembered to.
|
||||||
|
func TestPutReleasesTheBudgetWhenTheSidecarCannotBeWritten(t *testing.T) {
|
||||||
|
s := testStore(t)
|
||||||
|
data := []byte("no sidecar for this")
|
||||||
|
blockSidecar(t, s, KindImage, data)
|
||||||
|
if _, err := s.Put(KindImage, "image/png", "web:upload", data); err == nil {
|
||||||
|
t.Fatal("put must fail")
|
||||||
|
}
|
||||||
|
if s.Total() != 0 {
|
||||||
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPutFileReleasesTheBudgetWhenTheSidecarCannotBeWritten(t *testing.T) {
|
||||||
|
s := testStore(t)
|
||||||
|
data := []byte("no sidecar for this either")
|
||||||
|
blockSidecar(t, s, KindAudio, data)
|
||||||
|
src := filepath.Join(t.TempDir(), "capture.wav")
|
||||||
|
if err := os.WriteFile(src, data, 0o600); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := s.PutFile(KindAudio, "audio/wav", "meeting", src); err == nil {
|
||||||
|
t.Fatal("put file must fail")
|
||||||
|
}
|
||||||
|
if s.Total() != 0 {
|
||||||
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// The chmod arm is PutFile's alone: Put never touches a spool file.
|
||||||
|
func TestPutFileReleasesTheBudgetWhenTheSpoolCannotBeChmodded(t *testing.T) {
|
||||||
|
if os.Geteuid() == 0 {
|
||||||
|
t.Skip("root can chmod a file it does not own")
|
||||||
|
}
|
||||||
|
// A symlink to a file owned by somebody else. Stat and the hash follow it
|
||||||
|
// and succeed; chmod follows it too and is refused.
|
||||||
|
src := filepath.Join(t.TempDir(), "capture.wav")
|
||||||
|
if err := os.Symlink("/etc/hosts", src); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
info, err := os.Stat(src)
|
||||||
|
if err != nil || info.Size() == 0 {
|
||||||
|
t.Skip("no readable /etc/hosts to point at")
|
||||||
|
}
|
||||||
|
s := testStore(t)
|
||||||
|
if _, err := s.PutFile(KindAudio, "audio/wav", "meeting", src); err == nil {
|
||||||
|
t.Fatal("put file must fail")
|
||||||
|
}
|
||||||
|
if s.Total() != 0 {
|
||||||
|
t.Errorf("total = %d, want the failed put not counted", s.Total())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// blockSidecar puts a directory where the sidecar for data has to go, so
|
||||||
|
// writeMeta fails while the blob path is still free.
|
||||||
|
func blockSidecar(t *testing.T, s *Store, kind Kind, data []byte) {
|
||||||
|
t.Helper()
|
||||||
|
sum := sha256.Sum256(data)
|
||||||
|
id := hex.EncodeToString(sum[:])
|
||||||
|
bucket := filepath.Join(s.dir, string(kind), id[:2])
|
||||||
|
if err := os.MkdirAll(filepath.Join(bucket, id+".json"), 0o700); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// The per-blob cap bounds one call and nothing bounded their sum. 64 MiB per
|
// The per-blob cap bounds one call and nothing bounded their sum. 64 MiB per
|
||||||
// call times unlimited calls inside a seven-day window fills the disk mavend's
|
// call times unlimited calls inside a seven-day window fills the disk mavend's
|
||||||
// database lives on.
|
// database lives on.
|
||||||
|
|||||||
@@ -45,10 +45,10 @@ try:
|
|||||||
now = datetime.fromisoformat(sys.argv[2])
|
now = datetime.fromisoformat(sys.argv[2])
|
||||||
# Pre-process: replace Russian time qualifiers with AM/PM.
|
# Pre-process: replace Russian time qualifiers with AM/PM.
|
||||||
# Handles "9 утра", "10 часов утра", "3 часа дня" etc.
|
# Handles "9 утра", "10 часов утра", "3 часа дня" etc.
|
||||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?утра\b', r'\1 am', text, flags=re.IGNORECASE)
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов|у|ам)?\s+)?утра\b', r'\1 am', text, flags=re.IGNORECASE)
|
||||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?вечера\b', r'\1 pm', text, flags=re.IGNORECASE)
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов|у|ам)?\s+)?вечера\b', r'\1 pm', text, flags=re.IGNORECASE)
|
||||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?дня\b', r'\1 pm', text, flags=re.IGNORECASE)
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов|у|ам)?\s+)?дня\b', r'\1 pm', text, flags=re.IGNORECASE)
|
||||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?ночи\b', r'\1 am', text, flags=re.IGNORECASE)
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов|у|ам)?\s+)?ночи\b', r'\1 am', text, flags=re.IGNORECASE)
|
||||||
# A bare hour after a preposition is dropped on the floor by dateparser:
|
# A bare hour after a preposition is dropped on the floor by dateparser:
|
||||||
# "завтра в 7" resolves to tomorrow at the CURRENT clock, and "завтра в 7
|
# "завтра в 7" resolves to tomorrow at the CURRENT clock, and "завтра в 7
|
||||||
# часов" is read as seven hours from now. Only a qualifier (already an
|
# часов" is read as seven hours from now. Only a qualifier (already an
|
||||||
@@ -56,7 +56,9 @@ try:
|
|||||||
# English "at 7" fails identically, so both prepositions are rewritten.
|
# English "at 7" fails identically, so both prepositions are rewritten.
|
||||||
# "на 9" is the same hour said with the other preposition, and it was not
|
# "на 9" is the same hour said with the other preposition, and it was not
|
||||||
# read at all until V-579: "в 9" set the reminder and "на 9" did not.
|
# read at all until V-579: "в 9" set the reminder and "на 9" did not.
|
||||||
text = re.sub(r'(?<![\w:])(в|во|на|at)\s+([01]?\d|2[0-3])(?:\s+час(?:а|ов)?)?(?![\d:.\w])',
|
# "к двум часам" is a third preposition and the dative that goes with it,
|
||||||
|
# and it was read as no time at all until V-609.
|
||||||
|
text = re.sub(r'(?<![\w:])(в|во|на|к|ко|at|by)\s+([01]?\d|2[0-3])(?:\s+час(?:а|ов|у|ам)?)?(?![\d:.\w])',
|
||||||
lambda m: '%s %02d:00' % (m.group(1), int(m.group(2))), text, flags=re.IGNORECASE)
|
lambda m: '%s %02d:00' % (m.group(1), int(m.group(2))), text, flags=re.IGNORECASE)
|
||||||
settings = {'PREFER_DATES_FROM': 'future', 'RELATIVE_BASE': now}
|
settings = {'PREFER_DATES_FROM': 'future', 'RELATIVE_BASE': now}
|
||||||
# Two-step: search_dates finds the date substring in text,
|
# Two-step: search_dates finds the date substring in text,
|
||||||
|
|||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package router
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestDativePluralHourIsAnHour — "напомни к двум часам позвонить маме" reached
|
||||||
|
// the daemon with no time at all and she asked the open "Когда?", while "к трём"
|
||||||
|
// one word over read fine (V-609). The word that lost it was "часам", the dative
|
||||||
|
// plural of "час", which four separate hour sets in this package left out.
|
||||||
|
func TestDativePluralHourIsAnHour(t *testing.T) {
|
||||||
|
const s = "напомни к двум часам позвонить маме"
|
||||||
|
if !MentionsTime(s) {
|
||||||
|
t.Errorf("MentionsTime(%q) = false; the sentence names two o'clock", s)
|
||||||
|
}
|
||||||
|
if !NamesAnHour(s) {
|
||||||
|
t.Errorf("NamesAnHour(%q) = false; the sentence names two o'clock", s)
|
||||||
|
}
|
||||||
|
if got, want := SpellOutDigits(s), "напомни к 2 часам позвонить маме"; got != want {
|
||||||
|
t.Errorf("SpellOutDigits(%q) = %q, want %q", s, got, want)
|
||||||
|
}
|
||||||
|
// The slot itself, which is what the daemon reads. It was empty, so
|
||||||
|
// whenGapOf named the hour missing and she asked "Когда?".
|
||||||
|
now := time.Date(2026, 8, 6, 3, 39, 0, 0, time.UTC)
|
||||||
|
ex := Extractor{Time: StubDateTimeParser{}}
|
||||||
|
got := ex.Extract(context.Background(), IntentReminder, s, now)
|
||||||
|
if !got.HasTime {
|
||||||
|
t.Fatalf("the hour was spoken, so the slot must be filled: %+v", got)
|
||||||
|
}
|
||||||
|
if h := got.Time.Hour(); h != 2 && h != 14 {
|
||||||
|
t.Errorf("fire time = %s, want two o'clock in one half of the day or the other", got.Time.Format("15:04"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestHourUnitReachesEverySite — the four sets that read the hour noun now read
|
||||||
|
// one lexicon key, so a form added there is a form all four know. "часам" is the
|
||||||
|
// form that was missing from every one of them.
|
||||||
|
func TestHourUnitReachesEverySite(t *testing.T) {
|
||||||
|
for _, w := range []string{"час", "часа", "часов", "часу", "часам"} {
|
||||||
|
if !numeralContext[w] {
|
||||||
|
t.Errorf("numeralContext is missing %q", w)
|
||||||
|
}
|
||||||
|
if !hourMarkers[w] {
|
||||||
|
t.Errorf("hourMarkers is missing %q", w)
|
||||||
|
}
|
||||||
|
if !timeMarkers[w] {
|
||||||
|
t.Errorf("timeMarkers is missing %q", w)
|
||||||
|
}
|
||||||
|
if _, ok := unitToDuration(2, w); !ok {
|
||||||
|
t.Errorf("unitToDuration does not know %q", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMinuteUnitHasTheSameForms — the same defect one noun over: "минутам" was
|
||||||
|
// missing everywhere "минут" and "минуты" were present.
|
||||||
|
func TestMinuteUnitHasTheSameForms(t *testing.T) {
|
||||||
|
for _, w := range []string{"минут", "минуты", "минуту", "минутам"} {
|
||||||
|
if !numeralContext[w] {
|
||||||
|
t.Errorf("numeralContext is missing %q", w)
|
||||||
|
}
|
||||||
|
if !hourMarkers[w] {
|
||||||
|
t.Errorf("hourMarkers is missing %q", w)
|
||||||
|
}
|
||||||
|
if !timeMarkers[w] {
|
||||||
|
t.Errorf("timeMarkers is missing %q", w)
|
||||||
|
}
|
||||||
|
if _, ok := unitToDuration(20, w); !ok {
|
||||||
|
t.Errorf("unitToDuration does not know %q", w)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -34,12 +34,21 @@ func numeralDigit(word string) (string, bool) {
|
|||||||
// numeralContext — the words that make a numeral a time. A numeral is only
|
// numeralContext — the words that make a numeral a time. A numeral is only
|
||||||
// rewritten when one of these sits next to it, so "три яблока" in a note is
|
// rewritten when one of these sits next to it, so "три яблока" in a note is
|
||||||
// left alone and "в три часа" is not.
|
// left alone and "в три часа" is not.
|
||||||
var numeralContext = map[string]bool{
|
var numeralContext = buildNumeralContext()
|
||||||
"в": true, "во": true, "к": true, "около": true, "на": true,
|
|
||||||
"часа": true, "часов": true, "час": true, "часу": true,
|
func buildNumeralContext() map[string]bool {
|
||||||
"утра": true, "вечера": true, "дня": true, "ночи": true,
|
m := map[string]bool{
|
||||||
"минут": true, "минуты": true, "минуту": true,
|
"в": true, "во": true, "к": true, "около": true, "на": true,
|
||||||
"at": true, "by": true,
|
"утра": true, "вечера": true, "дня": true, "ночи": true,
|
||||||
|
"at": true, "by": true,
|
||||||
|
}
|
||||||
|
for _, w := range lexicon.HourUnits() {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
|
for _, w := range lexicon.MinuteUnits() {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
|
return m
|
||||||
}
|
}
|
||||||
|
|
||||||
// SpellOutDigits rewrites spoken numbers as digits so the date parsers see the
|
// SpellOutDigits rewrites spoken numbers as digits so the date parsers see the
|
||||||
|
|||||||
@@ -175,10 +175,14 @@ func afterWord(s, w string) string {
|
|||||||
return ""
|
return ""
|
||||||
}
|
}
|
||||||
|
|
||||||
// hourPrepositions — the words a spoken hour sits behind. Three, and no more:
|
// hourPrepositions — the words a spoken hour sits behind. Five, and no more:
|
||||||
// the lexicon's frame set is much wider, and a word goes in here only when the
|
// the lexicon's frame set is much wider, and a word goes in here only when the
|
||||||
// number after it is an hour of the day rather than a count of anything.
|
// number after it is an hour of the day rather than a count of anything.
|
||||||
var hourPrepositions = map[string]bool{"в": true, "во": true, "на": true}
|
//
|
||||||
|
// "к" and "ко" joined the three on V-609. "напомни к двум часам" named an hour
|
||||||
|
// and parsed to nothing, so the reminder reached the daemon with no time and she
|
||||||
|
// asked the open question about an hour he had just said.
|
||||||
|
var hourPrepositions = map[string]bool{"в": true, "во": true, "на": true, "к": true, "ко": true}
|
||||||
|
|
||||||
// StubDateTimeParser — a tiny relative/absolute parser standing in for
|
// StubDateTimeParser — a tiny relative/absolute parser standing in for
|
||||||
// `dateparser` until the i18n module lands. Handles "in Nh"/"in Nm"/"in Ns" and
|
// `dateparser` until the i18n module lands. Handles "in Nh"/"in Nm"/"in Ns" and
|
||||||
@@ -398,10 +402,18 @@ func leadingWordNumber(s string) (int, string, bool) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func unitToDuration(n int, unit string) (time.Duration, bool) {
|
func unitToDuration(n int, unit string) (time.Duration, bool) {
|
||||||
switch unit {
|
// The hour and the minute nouns are closed classes with one home in the
|
||||||
case "h", "hour", "hours", "hr", "hrs":
|
// lexicon, and the list here used to be short of the oblique forms (V-609).
|
||||||
|
if lexicon.IsHourUnit(unit) {
|
||||||
return time.Duration(n) * time.Hour, true
|
return time.Duration(n) * time.Hour, true
|
||||||
case "m", "min", "mins", "minute", "minutes":
|
}
|
||||||
|
if lexicon.IsMinuteUnit(unit) {
|
||||||
|
return time.Duration(n) * time.Minute, true
|
||||||
|
}
|
||||||
|
switch unit {
|
||||||
|
case "h", "hr", "hrs":
|
||||||
|
return time.Duration(n) * time.Hour, true
|
||||||
|
case "m", "min", "mins":
|
||||||
return time.Duration(n) * time.Minute, true
|
return time.Duration(n) * time.Minute, true
|
||||||
case "s", "sec", "secs", "second", "seconds":
|
case "s", "sec", "secs", "second", "seconds":
|
||||||
return time.Duration(n) * time.Second, true
|
return time.Duration(n) * time.Second, true
|
||||||
@@ -409,10 +421,6 @@ func unitToDuration(n int, unit string) (time.Duration, bool) {
|
|||||||
case "day", "days":
|
case "day", "days":
|
||||||
return time.Duration(n) * 24 * time.Hour, true
|
return time.Duration(n) * 24 * time.Hour, true
|
||||||
// Russian units (inflected forms)
|
// Russian units (inflected forms)
|
||||||
case "час", "часа", "часов":
|
|
||||||
return time.Duration(n) * time.Hour, true
|
|
||||||
case "минута", "минуты", "минут":
|
|
||||||
return time.Duration(n) * time.Minute, true
|
|
||||||
case "день", "дня", "дней":
|
case "день", "дня", "дней":
|
||||||
return time.Duration(n) * 24 * time.Hour, true
|
return time.Duration(n) * 24 * time.Hour, true
|
||||||
case "неделя", "недели", "недель":
|
case "неделя", "недели", "недель":
|
||||||
|
|||||||
@@ -206,12 +206,16 @@ var hourMarkers = buildHourMarkers()
|
|||||||
func buildHourMarkers() map[string]bool {
|
func buildHourMarkers() map[string]bool {
|
||||||
m := map[string]bool{
|
m := map[string]bool{
|
||||||
"утра": true, "вечера": true, "дня": true, "ночи": true,
|
"утра": true, "вечера": true, "дня": true, "ночи": true,
|
||||||
"часа": true, "часов": true, "час": true, "часу": true,
|
|
||||||
"минут": true, "минуты": true, "минуту": true,
|
|
||||||
"через": true, "спустя": true, "полчаса": true,
|
"через": true, "спустя": true, "полчаса": true,
|
||||||
"полдень": true, "полночь": true,
|
"полдень": true, "полночь": true,
|
||||||
"am": true, "pm": true, "noon": true, "midnight": true, "in": true,
|
"am": true, "pm": true, "noon": true, "midnight": true, "in": true,
|
||||||
}
|
}
|
||||||
|
for _, w := range lexicon.HourUnits() {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
|
for _, w := range lexicon.MinuteUnits() {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
for _, w := range lexicon.PartsOfDay() {
|
for _, w := range lexicon.PartsOfDay() {
|
||||||
m[w] = true
|
m[w] = true
|
||||||
}
|
}
|
||||||
@@ -274,11 +278,15 @@ var timeMarkers = buildTimeMarkers()
|
|||||||
func buildTimeMarkers() map[string]bool {
|
func buildTimeMarkers() map[string]bool {
|
||||||
m := map[string]bool{
|
m := map[string]bool{
|
||||||
"утра": true, "вечера": true, "дня": true, "ночи": true,
|
"утра": true, "вечера": true, "дня": true, "ночи": true,
|
||||||
"часа": true, "часов": true, "час": true, "часу": true,
|
|
||||||
"минут": true, "минуты": true, "минуту": true,
|
|
||||||
"через": true, "полчаса": true, "сейчас": true,
|
"через": true, "полчаса": true, "сейчас": true,
|
||||||
"am": true, "pm": true, "noon": true, "midnight": true,
|
"am": true, "pm": true, "noon": true, "midnight": true,
|
||||||
}
|
}
|
||||||
|
for _, w := range lexicon.HourUnits() {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
|
for _, w := range lexicon.MinuteUnits() {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
for _, w := range lexicon.PartsOfDay() {
|
for _, w := range lexicon.PartsOfDay() {
|
||||||
m[w] = true
|
m[w] = true
|
||||||
}
|
}
|
||||||
|
|||||||
+18
-14
@@ -37,8 +37,9 @@ type Server struct {
|
|||||||
addr netaddr.Addr
|
addr netaddr.Addr
|
||||||
ln net.Listener
|
ln net.Listener
|
||||||
|
|
||||||
wg sync.WaitGroup
|
wg sync.WaitGroup
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
|
closeOnce sync.Once
|
||||||
|
|
||||||
// connCount — assigned per accepted conn, used in logs to distinguish
|
// connCount — assigned per accepted conn, used in logs to distinguish
|
||||||
// concurrent connections. Monotonic; not load-bearing for correctness.
|
// concurrent connections. Monotonic; not load-bearing for correctness.
|
||||||
@@ -180,20 +181,23 @@ func (srv *Server) dispatch(ctx context.Context, req Request) (json.RawMessage,
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Close stops accepting and waits for in-flight connections to drain. The
|
// Close stops accepting and waits for in-flight connections to drain. The
|
||||||
// socket file is removed so a restart can rebind cleanly. Idempotent.
|
// socket file is removed so a restart can rebind cleanly.
|
||||||
|
//
|
||||||
|
// Idempotent, and safe from two goroutines at once. The check-then-close it
|
||||||
|
// replaced let both callers see an open channel and the second close panicked,
|
||||||
|
// so a shutdown racing a signal handler took the process down the one way a
|
||||||
|
// clean shutdown is supposed to prevent.
|
||||||
func (srv *Server) Close() error {
|
func (srv *Server) Close() error {
|
||||||
select {
|
var err error
|
||||||
case <-srv.done:
|
srv.closeOnce.Do(func() {
|
||||||
return nil
|
|
||||||
default:
|
|
||||||
close(srv.done)
|
close(srv.done)
|
||||||
}
|
if srv.ln == nil {
|
||||||
if srv.ln == nil {
|
return
|
||||||
return nil
|
}
|
||||||
}
|
err = srv.ln.Close()
|
||||||
err := srv.ln.Close()
|
srv.wg.Wait()
|
||||||
srv.wg.Wait()
|
netaddr.Cleanup(srv.addr)
|
||||||
netaddr.Cleanup(srv.addr)
|
})
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user