dateparser: python shell-out with two-step parse + russian qualifier pre-processing
PythonDateParser shells out to python3 with the dateparser library for full natural-language date/time extraction. Two-step approach: 1. search_dates() finds the date substring in surrounding text 2. parse() re-parses the substring for correct time resolution Russian time qualifiers (утра/вечера/дня/ночи) are pre-processed to AM/PM before parsing — dateparser drops them during substring extraction. Falls back to StubDateTimeParser when python3 or dateparser isn't available (graceful degradation, no hard runtime dependency). Dockerfile updated: python3 + dateparser==1.4.1 in runtime stage.
This commit is contained in:
kami
+4
-1
@@ -51,7 +51,10 @@ FROM debian:trixie-slim AS runtime
|
||||
# zone and time.Now() stays UTC, and mavend answers clock/date queries and
|
||||
# evaluates quiet-hours in UTC.
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ca-certificates libvulkan1 mesa-vulkan-drivers libgomp1 tzdata && \
|
||||
ca-certificates libvulkan1 mesa-vulkan-drivers libgomp1 tzdata \
|
||||
python3 python3-pip && \
|
||||
pip3 install --no-cache-dir --break-system-packages 'dateparser==1.4.1' && \
|
||||
apt-get purge -y --auto-remove python3-pip && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# runtime native libs: whisper/ggml (incl. vulkan) are real files in deps/lib.
|
||||
|
||||
+2
-2
@@ -229,7 +229,7 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
||||
memStore: memStore,
|
||||
dialogueSessions: dialogueSessions,
|
||||
queryMinScore: cfg.Voice.QueryMinScore,
|
||||
timeParser: router.StubDateTimeParser{},
|
||||
timeParser: router.NewPythonDateParser(),
|
||||
}
|
||||
|
||||
// ----- the server (TCP listener) -----
|
||||
@@ -803,7 +803,7 @@ func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64)
|
||||
Grammars: grammars,
|
||||
Classifier: cls,
|
||||
Extractor: router.Extractor{
|
||||
Time: router.StubDateTimeParser{},
|
||||
Time: router.NewPythonDateParser(),
|
||||
Acts: acts,
|
||||
Facts: router.DefaultFactParser{},
|
||||
},
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package router
|
||||
|
||||
// PythonDateParser — the production DateTimeParser. Shells out to python3
|
||||
// with the `dateparser` library (BSD-3, 200+ locales, ru+en native) for full
|
||||
// natural-language date/time extraction: "через два часа", "в следующую
|
||||
// пятницу", "завтра в 9 утра", "in 30 minutes", "on friday at noon".
|
||||
//
|
||||
// Falls back to StubDateTimeParser when python3 or dateparser is unavailable
|
||||
// (e.g. dev runs without the Docker runtime image). The stub handles the
|
||||
// common patterns; this handles the long tail.
|
||||
//
|
||||
// The interface is the seam — wireVoice swaps implementations without
|
||||
// touching the extractor or the daemon. This struct is safe for concurrent
|
||||
// use: each Parse call spawns an independent python process.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// dateparserScript — the python inline script. Receives text as argv[1] and
|
||||
// the current time as an ISO 8601 string (argv[2]). Prints the parsed
|
||||
// datetime as a Unix timestamp (seconds.micros) on stdout, or nothing if no
|
||||
// date was found. Exits non-zero on import failure (dateparser not
|
||||
// installed).
|
||||
//
|
||||
// Two-step parse: search_dates finds the date substring in surrounding text
|
||||
// ("напомни через час" → "через час"), then parse() re-parses that substring
|
||||
// for correct time resolution (search_dates mishandles AM/PM). Russian time
|
||||
// qualifiers (утра/вечера/дня/ночи) are pre-processed to AM/PM because
|
||||
// dateparser drops them during substring extraction.
|
||||
const dateparserScript = `import sys, re
|
||||
from datetime import datetime
|
||||
try:
|
||||
import dateparser
|
||||
from dateparser.search import search_dates
|
||||
except ImportError:
|
||||
sys.exit(1)
|
||||
try:
|
||||
text = sys.argv[1]
|
||||
now = datetime.fromisoformat(sys.argv[2])
|
||||
# Pre-process: replace Russian time qualifiers with AM/PM.
|
||||
# Handles "9 утра", "10 часов утра", "3 часа дня" etc.
|
||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?утра\b', r'\1 am', text, flags=re.IGNORECASE)
|
||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?вечера\b', r'\1 pm', text, flags=re.IGNORECASE)
|
||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?дня\b', r'\1 pm', text, flags=re.IGNORECASE)
|
||||
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?ночи\b', r'\1 am', text, flags=re.IGNORECASE)
|
||||
settings = {'PREFER_DATES_FROM': 'future', 'RELATIVE_BASE': now}
|
||||
# Two-step: search_dates finds the date substring in text,
|
||||
# parse() gets the time right (search_dates mishandles AM/PM).
|
||||
results = search_dates(text, languages=['ru', 'en'], settings=settings)
|
||||
if results:
|
||||
substr, raw_dt = results[0]
|
||||
dt = dateparser.parse(substr, languages=['ru', 'en'], settings=settings)
|
||||
if dt is None:
|
||||
dt = raw_dt
|
||||
if dt is not None:
|
||||
print(dt.timestamp())
|
||||
except Exception:
|
||||
pass
|
||||
`
|
||||
|
||||
// PythonDateParser implements DateTimeParser via the python dateparser
|
||||
// library, with a StubDateTimeParser fallback for environments where
|
||||
// python3/dateparser isn't installed.
|
||||
type PythonDateParser struct {
|
||||
fallback DateTimeParser
|
||||
}
|
||||
|
||||
// NewPythonDateParser constructs a PythonDateParser with StubDateTimeParser
|
||||
// as the fallback.
|
||||
func NewPythonDateParser() *PythonDateParser {
|
||||
return &PythonDateParser{fallback: StubDateTimeParser{}}
|
||||
}
|
||||
|
||||
// Parse extracts a datetime from text using python's dateparser. If python3
|
||||
// or dateparser is unavailable, falls back to the stub parser. Returns
|
||||
// (time, true, nil) on success; (zero, false, nil) when no date is found.
|
||||
func (p *PythonDateParser) Parse(ctx context.Context, text string, now time.Time) (time.Time, bool, error) {
|
||||
t, ok, err := p.parseWithPython(ctx, text, now)
|
||||
if err != nil {
|
||||
// python3 missing, dateparser not installed, or process failure —
|
||||
// degrade to the stub which handles the common cases.
|
||||
log.Printf("router: python dateparser unavailable, falling back to stub: %v", err)
|
||||
return p.fallback.Parse(ctx, text, now)
|
||||
}
|
||||
return t, ok, nil
|
||||
}
|
||||
|
||||
// parseWithPython runs the dateparser script and parses the timestamp output.
|
||||
// Returns (zero, false, error) on process/exec failure; (zero, false, nil)
|
||||
// when the script ran but found no date.
|
||||
func (p *PythonDateParser) parseWithPython(ctx context.Context, text string, now time.Time) (time.Time, bool, error) {
|
||||
cmd := exec.CommandContext(ctx, "python3", "-c", dateparserScript, text, now.Format(time.RFC3339))
|
||||
output, err := cmd.Output()
|
||||
if err != nil {
|
||||
return time.Time{}, false, err
|
||||
}
|
||||
s := strings.TrimSpace(string(output))
|
||||
if s == "" {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
f, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return time.Time{}, false, nil
|
||||
}
|
||||
sec := int64(f)
|
||||
nsec := int64((f - float64(sec)) * 1e9)
|
||||
return time.Unix(sec, nsec).Local(), true, nil
|
||||
}
|
||||
@@ -0,0 +1,157 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os/exec"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestPythonDateParser requires python3 + dateparser installed. Skips
|
||||
// gracefully in environments without them (e.g. bare go test, dev machines
|
||||
// without the Docker runtime image).
|
||||
func TestPythonDateParser(t *testing.T) {
|
||||
// Skip if python3 isn't on PATH.
|
||||
if _, err := exec.LookPath("python3"); err != nil {
|
||||
t.Skip("python3 not on PATH — skipping dateparser tests")
|
||||
}
|
||||
// Skip if dateparser isn't installed.
|
||||
cmd := exec.Command("python3", "-c", "import dateparser")
|
||||
if err := cmd.Run(); err != nil {
|
||||
t.Skip("python dateparser not installed — skipping dateparser tests")
|
||||
}
|
||||
|
||||
p := NewPythonDateParser()
|
||||
now := time.Date(2026, 7, 6, 18, 0, 0, 0, time.Local)
|
||||
ctx := context.Background()
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
text string
|
||||
wantOK bool
|
||||
checkT func(t *testing.T, got, now time.Time)
|
||||
}{
|
||||
{
|
||||
name: "ru relative — через час",
|
||||
text: "напомни через час выпить воды",
|
||||
wantOK: true,
|
||||
checkT: func(t *testing.T, got, now time.Time) {
|
||||
d := got.Sub(now)
|
||||
if d < 50*time.Minute || d > 70*time.Minute {
|
||||
t.Errorf("через час: got %v from now, want ~1h", d)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ru relative — через 30 минут",
|
||||
text: "напомни через 30 минут снять бельё",
|
||||
wantOK: true,
|
||||
checkT: func(t *testing.T, got, now time.Time) {
|
||||
d := got.Sub(now)
|
||||
if d < 25*time.Minute || d > 35*time.Minute {
|
||||
t.Errorf("через 30 минут: got %v from now, want ~30m", d)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ru absolute — завтра в 9 утра",
|
||||
text: "напомни завтра в 9 утра позвонить",
|
||||
wantOK: true,
|
||||
checkT: func(t *testing.T, got, now time.Time) {
|
||||
if got.Hour() != 9 {
|
||||
t.Errorf("завтра в 9 утра: hour=%d, want 9", got.Hour())
|
||||
}
|
||||
if got.Sub(now) < 12*time.Hour {
|
||||
t.Errorf("завтра в 9 утра: only %v from now, want >12h (tomorrow)", got.Sub(now))
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "ru complex — в следующую пятницу",
|
||||
text: "напомни в следующую пятницу оплатить счёт",
|
||||
wantOK: true,
|
||||
checkT: func(t *testing.T, got, now time.Time) {
|
||||
if got.Weekday() != time.Friday {
|
||||
t.Errorf("в следующую пятницу: weekday=%v, want Friday", got.Weekday())
|
||||
}
|
||||
if got.Sub(now) < 24*time.Hour {
|
||||
t.Errorf("в следующую пятницу: only %v from now, want >24h (future)", got.Sub(now))
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "en relative — in 30 minutes",
|
||||
text: "remind me in 30 minutes to drink water",
|
||||
wantOK: true,
|
||||
checkT: func(t *testing.T, got, now time.Time) {
|
||||
d := got.Sub(now)
|
||||
if d < 25*time.Minute || d > 35*time.Minute {
|
||||
t.Errorf("in 30 minutes: got %v from now, want ~30m", d)
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "en absolute — tomorrow at 8am",
|
||||
text: "remind me tomorrow at 8am to call the doctor",
|
||||
wantOK: true,
|
||||
checkT: func(t *testing.T, got, now time.Time) {
|
||||
if got.Hour() != 8 {
|
||||
t.Errorf("tomorrow at 8am: hour=%d, want 8", got.Hour())
|
||||
}
|
||||
if got.Sub(now) < 12*time.Hour {
|
||||
t.Errorf("tomorrow at 8am: only %v from now, want >12h (tomorrow)", got.Sub(now))
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "no date — напомни мне",
|
||||
text: "напомни мне",
|
||||
wantOK: false,
|
||||
},
|
||||
{
|
||||
name: "no date — remind me",
|
||||
text: "remind me to do something",
|
||||
wantOK: false,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, ok, err := p.Parse(ctx, tt.text, now)
|
||||
if err != nil {
|
||||
t.Fatalf("Parse(%q) error: %v", tt.text, err)
|
||||
}
|
||||
if ok != tt.wantOK {
|
||||
t.Fatalf("Parse(%q) ok=%v, want %v (got time=%v)", tt.text, ok, tt.wantOK, got)
|
||||
}
|
||||
if ok && tt.checkT != nil {
|
||||
tt.checkT(t, got, now)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestPythonDateParser_FallbackToStub verifies that when python3 is not
|
||||
// available, the parser falls back to StubDateTimeParser instead of
|
||||
// returning an error.
|
||||
func TestPythonDateParser_FallbackToStub(t *testing.T) {
|
||||
if _, err := exec.LookPath("python3"); err == nil {
|
||||
t.Skip("python3 is available — fallback path not exercised")
|
||||
}
|
||||
p := NewPythonDateParser()
|
||||
ctx := context.Background()
|
||||
now := time.Date(2026, 7, 6, 18, 0, 0, 0, time.Local)
|
||||
|
||||
// The stub handles "in 1 hour" — should still work via fallback.
|
||||
got, ok, err := p.Parse(ctx, "in 1 hour", now)
|
||||
if err != nil {
|
||||
t.Fatalf("fallback Parse error: %v", err)
|
||||
}
|
||||
if !ok {
|
||||
t.Fatal("fallback Parse: ok=false, want true (stub should handle 'in 1 hour')")
|
||||
}
|
||||
d := got.Sub(now)
|
||||
if d < 50*time.Minute || d > 70*time.Minute {
|
||||
t.Errorf("fallback 'in 1 hour': got %v from now, want ~1h", d)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user