From d493be34b2400a3a9e16c7a0dfb986047ac500d8 Mon Sep 17 00:00:00 2001 From: kami Date: Mon, 6 Jul 2026 19:09:09 +0400 Subject: [PATCH] dateparser: python shell-out with two-step parse + russian qualifier pre-processing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PythonDateParser shells out to python3 with the dateparser library for full natural-language date/time extraction. Two-step approach: 1. search_dates() finds the date substring in surrounding text 2. parse() re-parses the substring for correct time resolution Russian time qualifiers (утра/вечера/дня/ночи) are pre-processed to AM/PM before parsing — dateparser drops them during substring extraction. Falls back to StubDateTimeParser when python3 or dateparser isn't available (graceful degradation, no hard runtime dependency). Dockerfile updated: python3 + dateparser==1.4.1 in runtime stage. --- Dockerfile | 5 +- cmd/mavend/voice.go | 4 +- internal/router/dateparser.go | 114 +++++++++++++++++++++ internal/router/dateparser_test.go | 157 +++++++++++++++++++++++++++++ 4 files changed, 277 insertions(+), 3 deletions(-) create mode 100644 internal/router/dateparser.go create mode 100644 internal/router/dateparser_test.go diff --git a/Dockerfile b/Dockerfile index 0e4ab58..76fd677 100644 --- a/Dockerfile +++ b/Dockerfile @@ -51,7 +51,10 @@ FROM debian:trixie-slim AS runtime # zone and time.Now() stays UTC, and mavend answers clock/date queries and # evaluates quiet-hours in UTC. RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates libvulkan1 mesa-vulkan-drivers libgomp1 tzdata && \ + ca-certificates libvulkan1 mesa-vulkan-drivers libgomp1 tzdata \ + python3 python3-pip && \ + pip3 install --no-cache-dir --break-system-packages 'dateparser==1.4.1' && \ + apt-get purge -y --auto-remove python3-pip && \ rm -rf /var/lib/apt/lists/* # runtime native libs: whisper/ggml (incl. vulkan) are real files in deps/lib. diff --git a/cmd/mavend/voice.go b/cmd/mavend/voice.go index 33c3a2d..93ae70f 100644 --- a/cmd/mavend/voice.go +++ b/cmd/mavend/voice.go @@ -229,7 +229,7 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem memStore: memStore, dialogueSessions: dialogueSessions, queryMinScore: cfg.Voice.QueryMinScore, - timeParser: router.StubDateTimeParser{}, + timeParser: router.NewPythonDateParser(), } // ----- the server (TCP listener) ----- @@ -803,7 +803,7 @@ func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64) Grammars: grammars, Classifier: cls, Extractor: router.Extractor{ - Time: router.StubDateTimeParser{}, + Time: router.NewPythonDateParser(), Acts: acts, Facts: router.DefaultFactParser{}, }, diff --git a/internal/router/dateparser.go b/internal/router/dateparser.go new file mode 100644 index 0000000..c51db10 --- /dev/null +++ b/internal/router/dateparser.go @@ -0,0 +1,114 @@ +package router + +// PythonDateParser — the production DateTimeParser. Shells out to python3 +// with the `dateparser` library (BSD-3, 200+ locales, ru+en native) for full +// natural-language date/time extraction: "через два часа", "в следующую +// пятницу", "завтра в 9 утра", "in 30 minutes", "on friday at noon". +// +// Falls back to StubDateTimeParser when python3 or dateparser is unavailable +// (e.g. dev runs without the Docker runtime image). The stub handles the +// common patterns; this handles the long tail. +// +// The interface is the seam — wireVoice swaps implementations without +// touching the extractor or the daemon. This struct is safe for concurrent +// use: each Parse call spawns an independent python process. + +import ( + "context" + "log" + "os/exec" + "strconv" + "strings" + "time" +) + +// dateparserScript — the python inline script. Receives text as argv[1] and +// the current time as an ISO 8601 string (argv[2]). Prints the parsed +// datetime as a Unix timestamp (seconds.micros) on stdout, or nothing if no +// date was found. Exits non-zero on import failure (dateparser not +// installed). +// +// Two-step parse: search_dates finds the date substring in surrounding text +// ("напомни через час" → "через час"), then parse() re-parses that substring +// for correct time resolution (search_dates mishandles AM/PM). Russian time +// qualifiers (утра/вечера/дня/ночи) are pre-processed to AM/PM because +// dateparser drops them during substring extraction. +const dateparserScript = `import sys, re +from datetime import datetime +try: + import dateparser + from dateparser.search import search_dates +except ImportError: + sys.exit(1) +try: + text = sys.argv[1] + now = datetime.fromisoformat(sys.argv[2]) + # Pre-process: replace Russian time qualifiers with AM/PM. + # Handles "9 утра", "10 часов утра", "3 часа дня" etc. + text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?утра\b', r'\1 am', text, flags=re.IGNORECASE) + text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?вечера\b', r'\1 pm', text, flags=re.IGNORECASE) + text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?дня\b', r'\1 pm', text, flags=re.IGNORECASE) + text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?ночи\b', r'\1 am', text, flags=re.IGNORECASE) + settings = {'PREFER_DATES_FROM': 'future', 'RELATIVE_BASE': now} + # Two-step: search_dates finds the date substring in text, + # parse() gets the time right (search_dates mishandles AM/PM). + results = search_dates(text, languages=['ru', 'en'], settings=settings) + if results: + substr, raw_dt = results[0] + dt = dateparser.parse(substr, languages=['ru', 'en'], settings=settings) + if dt is None: + dt = raw_dt + if dt is not None: + print(dt.timestamp()) +except Exception: + pass +` + +// PythonDateParser implements DateTimeParser via the python dateparser +// library, with a StubDateTimeParser fallback for environments where +// python3/dateparser isn't installed. +type PythonDateParser struct { + fallback DateTimeParser +} + +// NewPythonDateParser constructs a PythonDateParser with StubDateTimeParser +// as the fallback. +func NewPythonDateParser() *PythonDateParser { + return &PythonDateParser{fallback: StubDateTimeParser{}} +} + +// Parse extracts a datetime from text using python's dateparser. If python3 +// or dateparser is unavailable, falls back to the stub parser. Returns +// (time, true, nil) on success; (zero, false, nil) when no date is found. +func (p *PythonDateParser) Parse(ctx context.Context, text string, now time.Time) (time.Time, bool, error) { + t, ok, err := p.parseWithPython(ctx, text, now) + if err != nil { + // python3 missing, dateparser not installed, or process failure — + // degrade to the stub which handles the common cases. + log.Printf("router: python dateparser unavailable, falling back to stub: %v", err) + return p.fallback.Parse(ctx, text, now) + } + return t, ok, nil +} + +// parseWithPython runs the dateparser script and parses the timestamp output. +// Returns (zero, false, error) on process/exec failure; (zero, false, nil) +// when the script ran but found no date. +func (p *PythonDateParser) parseWithPython(ctx context.Context, text string, now time.Time) (time.Time, bool, error) { + cmd := exec.CommandContext(ctx, "python3", "-c", dateparserScript, text, now.Format(time.RFC3339)) + output, err := cmd.Output() + if err != nil { + return time.Time{}, false, err + } + s := strings.TrimSpace(string(output)) + if s == "" { + return time.Time{}, false, nil + } + f, err := strconv.ParseFloat(s, 64) + if err != nil { + return time.Time{}, false, nil + } + sec := int64(f) + nsec := int64((f - float64(sec)) * 1e9) + return time.Unix(sec, nsec).Local(), true, nil +} diff --git a/internal/router/dateparser_test.go b/internal/router/dateparser_test.go new file mode 100644 index 0000000..cb98073 --- /dev/null +++ b/internal/router/dateparser_test.go @@ -0,0 +1,157 @@ +package router + +import ( + "context" + "os/exec" + "testing" + "time" +) + +// TestPythonDateParser requires python3 + dateparser installed. Skips +// gracefully in environments without them (e.g. bare go test, dev machines +// without the Docker runtime image). +func TestPythonDateParser(t *testing.T) { + // Skip if python3 isn't on PATH. + if _, err := exec.LookPath("python3"); err != nil { + t.Skip("python3 not on PATH — skipping dateparser tests") + } + // Skip if dateparser isn't installed. + cmd := exec.Command("python3", "-c", "import dateparser") + if err := cmd.Run(); err != nil { + t.Skip("python dateparser not installed — skipping dateparser tests") + } + + p := NewPythonDateParser() + now := time.Date(2026, 7, 6, 18, 0, 0, 0, time.Local) + ctx := context.Background() + + tests := []struct { + name string + text string + wantOK bool + checkT func(t *testing.T, got, now time.Time) + }{ + { + name: "ru relative — через час", + text: "напомни через час выпить воды", + wantOK: true, + checkT: func(t *testing.T, got, now time.Time) { + d := got.Sub(now) + if d < 50*time.Minute || d > 70*time.Minute { + t.Errorf("через час: got %v from now, want ~1h", d) + } + }, + }, + { + name: "ru relative — через 30 минут", + text: "напомни через 30 минут снять бельё", + wantOK: true, + checkT: func(t *testing.T, got, now time.Time) { + d := got.Sub(now) + if d < 25*time.Minute || d > 35*time.Minute { + t.Errorf("через 30 минут: got %v from now, want ~30m", d) + } + }, + }, + { + name: "ru absolute — завтра в 9 утра", + text: "напомни завтра в 9 утра позвонить", + wantOK: true, + checkT: func(t *testing.T, got, now time.Time) { + if got.Hour() != 9 { + t.Errorf("завтра в 9 утра: hour=%d, want 9", got.Hour()) + } + if got.Sub(now) < 12*time.Hour { + t.Errorf("завтра в 9 утра: only %v from now, want >12h (tomorrow)", got.Sub(now)) + } + }, + }, + { + name: "ru complex — в следующую пятницу", + text: "напомни в следующую пятницу оплатить счёт", + wantOK: true, + checkT: func(t *testing.T, got, now time.Time) { + if got.Weekday() != time.Friday { + t.Errorf("в следующую пятницу: weekday=%v, want Friday", got.Weekday()) + } + if got.Sub(now) < 24*time.Hour { + t.Errorf("в следующую пятницу: only %v from now, want >24h (future)", got.Sub(now)) + } + }, + }, + { + name: "en relative — in 30 minutes", + text: "remind me in 30 minutes to drink water", + wantOK: true, + checkT: func(t *testing.T, got, now time.Time) { + d := got.Sub(now) + if d < 25*time.Minute || d > 35*time.Minute { + t.Errorf("in 30 minutes: got %v from now, want ~30m", d) + } + }, + }, + { + name: "en absolute — tomorrow at 8am", + text: "remind me tomorrow at 8am to call the doctor", + wantOK: true, + checkT: func(t *testing.T, got, now time.Time) { + if got.Hour() != 8 { + t.Errorf("tomorrow at 8am: hour=%d, want 8", got.Hour()) + } + if got.Sub(now) < 12*time.Hour { + t.Errorf("tomorrow at 8am: only %v from now, want >12h (tomorrow)", got.Sub(now)) + } + }, + }, + { + name: "no date — напомни мне", + text: "напомни мне", + wantOK: false, + }, + { + name: "no date — remind me", + text: "remind me to do something", + wantOK: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, ok, err := p.Parse(ctx, tt.text, now) + if err != nil { + t.Fatalf("Parse(%q) error: %v", tt.text, err) + } + if ok != tt.wantOK { + t.Fatalf("Parse(%q) ok=%v, want %v (got time=%v)", tt.text, ok, tt.wantOK, got) + } + if ok && tt.checkT != nil { + tt.checkT(t, got, now) + } + }) + } +} + +// TestPythonDateParser_FallbackToStub verifies that when python3 is not +// available, the parser falls back to StubDateTimeParser instead of +// returning an error. +func TestPythonDateParser_FallbackToStub(t *testing.T) { + if _, err := exec.LookPath("python3"); err == nil { + t.Skip("python3 is available — fallback path not exercised") + } + p := NewPythonDateParser() + ctx := context.Background() + now := time.Date(2026, 7, 6, 18, 0, 0, 0, time.Local) + + // The stub handles "in 1 hour" — should still work via fallback. + got, ok, err := p.Parse(ctx, "in 1 hour", now) + if err != nil { + t.Fatalf("fallback Parse error: %v", err) + } + if !ok { + t.Fatal("fallback Parse: ok=false, want true (stub should handle 'in 1 hour')") + } + d := got.Sub(now) + if d < 50*time.Minute || d > 70*time.Minute { + t.Errorf("fallback 'in 1 hour': got %v from now, want ~1h", d) + } +}