dateparser: python shell-out with two-step parse + russian qualifier pre-processing
PythonDateParser shells out to python3 with the dateparser library for full natural-language date/time extraction. Two-step approach: 1. search_dates() finds the date substring in surrounding text 2. parse() re-parses the substring for correct time resolution Russian time qualifiers (утра/вечера/дня/ночи) are pre-processed to AM/PM before parsing — dateparser drops them during substring extraction. Falls back to StubDateTimeParser when python3 or dateparser isn't available (graceful degradation, no hard runtime dependency). Dockerfile updated: python3 + dateparser==1.4.1 in runtime stage.
This commit is contained in:
+4
-1
@@ -51,7 +51,10 @@ FROM debian:trixie-slim AS runtime
|
|||||||
# zone and time.Now() stays UTC, and mavend answers clock/date queries and
|
# zone and time.Now() stays UTC, and mavend answers clock/date queries and
|
||||||
# evaluates quiet-hours in UTC.
|
# evaluates quiet-hours in UTC.
|
||||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
ca-certificates libvulkan1 mesa-vulkan-drivers libgomp1 tzdata && \
|
ca-certificates libvulkan1 mesa-vulkan-drivers libgomp1 tzdata \
|
||||||
|
python3 python3-pip && \
|
||||||
|
pip3 install --no-cache-dir --break-system-packages 'dateparser==1.4.1' && \
|
||||||
|
apt-get purge -y --auto-remove python3-pip && \
|
||||||
rm -rf /var/lib/apt/lists/*
|
rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
# runtime native libs: whisper/ggml (incl. vulkan) are real files in deps/lib.
|
# runtime native libs: whisper/ggml (incl. vulkan) are real files in deps/lib.
|
||||||
|
|||||||
+2
-2
@@ -229,7 +229,7 @@ func wireVoice(cfg *config.Config, coreAPI ipc.CoreAPI, phr phraser.Phraser, mem
|
|||||||
memStore: memStore,
|
memStore: memStore,
|
||||||
dialogueSessions: dialogueSessions,
|
dialogueSessions: dialogueSessions,
|
||||||
queryMinScore: cfg.Voice.QueryMinScore,
|
queryMinScore: cfg.Voice.QueryMinScore,
|
||||||
timeParser: router.StubDateTimeParser{},
|
timeParser: router.NewPythonDateParser(),
|
||||||
}
|
}
|
||||||
|
|
||||||
// ----- the server (TCP listener) -----
|
// ----- the server (TCP listener) -----
|
||||||
@@ -803,7 +803,7 @@ func buildRouter(emb router.Embedder, acts router.ActMatcher, threshold float64)
|
|||||||
Grammars: grammars,
|
Grammars: grammars,
|
||||||
Classifier: cls,
|
Classifier: cls,
|
||||||
Extractor: router.Extractor{
|
Extractor: router.Extractor{
|
||||||
Time: router.StubDateTimeParser{},
|
Time: router.NewPythonDateParser(),
|
||||||
Acts: acts,
|
Acts: acts,
|
||||||
Facts: router.DefaultFactParser{},
|
Facts: router.DefaultFactParser{},
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -0,0 +1,114 @@
|
|||||||
|
package router
|
||||||
|
|
||||||
|
// PythonDateParser — the production DateTimeParser. Shells out to python3
|
||||||
|
// with the `dateparser` library (BSD-3, 200+ locales, ru+en native) for full
|
||||||
|
// natural-language date/time extraction: "через два часа", "в следующую
|
||||||
|
// пятницу", "завтра в 9 утра", "in 30 minutes", "on friday at noon".
|
||||||
|
//
|
||||||
|
// Falls back to StubDateTimeParser when python3 or dateparser is unavailable
|
||||||
|
// (e.g. dev runs without the Docker runtime image). The stub handles the
|
||||||
|
// common patterns; this handles the long tail.
|
||||||
|
//
|
||||||
|
// The interface is the seam — wireVoice swaps implementations without
|
||||||
|
// touching the extractor or the daemon. This struct is safe for concurrent
|
||||||
|
// use: each Parse call spawns an independent python process.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"os/exec"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// dateparserScript — the python inline script. Receives text as argv[1] and
|
||||||
|
// the current time as an ISO 8601 string (argv[2]). Prints the parsed
|
||||||
|
// datetime as a Unix timestamp (seconds.micros) on stdout, or nothing if no
|
||||||
|
// date was found. Exits non-zero on import failure (dateparser not
|
||||||
|
// installed).
|
||||||
|
//
|
||||||
|
// Two-step parse: search_dates finds the date substring in surrounding text
|
||||||
|
// ("напомни через час" → "через час"), then parse() re-parses that substring
|
||||||
|
// for correct time resolution (search_dates mishandles AM/PM). Russian time
|
||||||
|
// qualifiers (утра/вечера/дня/ночи) are pre-processed to AM/PM because
|
||||||
|
// dateparser drops them during substring extraction.
|
||||||
|
const dateparserScript = `import sys, re
|
||||||
|
from datetime import datetime
|
||||||
|
try:
|
||||||
|
import dateparser
|
||||||
|
from dateparser.search import search_dates
|
||||||
|
except ImportError:
|
||||||
|
sys.exit(1)
|
||||||
|
try:
|
||||||
|
text = sys.argv[1]
|
||||||
|
now = datetime.fromisoformat(sys.argv[2])
|
||||||
|
# Pre-process: replace Russian time qualifiers with AM/PM.
|
||||||
|
# Handles "9 утра", "10 часов утра", "3 часа дня" etc.
|
||||||
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?утра\b', r'\1 am', text, flags=re.IGNORECASE)
|
||||||
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?вечера\b', r'\1 pm', text, flags=re.IGNORECASE)
|
||||||
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?дня\b', r'\1 pm', text, flags=re.IGNORECASE)
|
||||||
|
text = re.sub(r'(\d+)\s+(?:час(?:а|ов)?\s+)?ночи\b', r'\1 am', text, flags=re.IGNORECASE)
|
||||||
|
settings = {'PREFER_DATES_FROM': 'future', 'RELATIVE_BASE': now}
|
||||||
|
# Two-step: search_dates finds the date substring in text,
|
||||||
|
# parse() gets the time right (search_dates mishandles AM/PM).
|
||||||
|
results = search_dates(text, languages=['ru', 'en'], settings=settings)
|
||||||
|
if results:
|
||||||
|
substr, raw_dt = results[0]
|
||||||
|
dt = dateparser.parse(substr, languages=['ru', 'en'], settings=settings)
|
||||||
|
if dt is None:
|
||||||
|
dt = raw_dt
|
||||||
|
if dt is not None:
|
||||||
|
print(dt.timestamp())
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
`
|
||||||
|
|
||||||
|
// PythonDateParser implements DateTimeParser via the python dateparser
|
||||||
|
// library, with a StubDateTimeParser fallback for environments where
|
||||||
|
// python3/dateparser isn't installed.
|
||||||
|
type PythonDateParser struct {
|
||||||
|
fallback DateTimeParser
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewPythonDateParser constructs a PythonDateParser with StubDateTimeParser
|
||||||
|
// as the fallback.
|
||||||
|
func NewPythonDateParser() *PythonDateParser {
|
||||||
|
return &PythonDateParser{fallback: StubDateTimeParser{}}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse extracts a datetime from text using python's dateparser. If python3
|
||||||
|
// or dateparser is unavailable, falls back to the stub parser. Returns
|
||||||
|
// (time, true, nil) on success; (zero, false, nil) when no date is found.
|
||||||
|
func (p *PythonDateParser) Parse(ctx context.Context, text string, now time.Time) (time.Time, bool, error) {
|
||||||
|
t, ok, err := p.parseWithPython(ctx, text, now)
|
||||||
|
if err != nil {
|
||||||
|
// python3 missing, dateparser not installed, or process failure —
|
||||||
|
// degrade to the stub which handles the common cases.
|
||||||
|
log.Printf("router: python dateparser unavailable, falling back to stub: %v", err)
|
||||||
|
return p.fallback.Parse(ctx, text, now)
|
||||||
|
}
|
||||||
|
return t, ok, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseWithPython runs the dateparser script and parses the timestamp output.
|
||||||
|
// Returns (zero, false, error) on process/exec failure; (zero, false, nil)
|
||||||
|
// when the script ran but found no date.
|
||||||
|
func (p *PythonDateParser) parseWithPython(ctx context.Context, text string, now time.Time) (time.Time, bool, error) {
|
||||||
|
cmd := exec.CommandContext(ctx, "python3", "-c", dateparserScript, text, now.Format(time.RFC3339))
|
||||||
|
output, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, false, err
|
||||||
|
}
|
||||||
|
s := strings.TrimSpace(string(output))
|
||||||
|
if s == "" {
|
||||||
|
return time.Time{}, false, nil
|
||||||
|
}
|
||||||
|
f, err := strconv.ParseFloat(s, 64)
|
||||||
|
if err != nil {
|
||||||
|
return time.Time{}, false, nil
|
||||||
|
}
|
||||||
|
sec := int64(f)
|
||||||
|
nsec := int64((f - float64(sec)) * 1e9)
|
||||||
|
return time.Unix(sec, nsec).Local(), true, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,157 @@
|
|||||||
|
package router
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"os/exec"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestPythonDateParser requires python3 + dateparser installed. Skips
|
||||||
|
// gracefully in environments without them (e.g. bare go test, dev machines
|
||||||
|
// without the Docker runtime image).
|
||||||
|
func TestPythonDateParser(t *testing.T) {
|
||||||
|
// Skip if python3 isn't on PATH.
|
||||||
|
if _, err := exec.LookPath("python3"); err != nil {
|
||||||
|
t.Skip("python3 not on PATH — skipping dateparser tests")
|
||||||
|
}
|
||||||
|
// Skip if dateparser isn't installed.
|
||||||
|
cmd := exec.Command("python3", "-c", "import dateparser")
|
||||||
|
if err := cmd.Run(); err != nil {
|
||||||
|
t.Skip("python dateparser not installed — skipping dateparser tests")
|
||||||
|
}
|
||||||
|
|
||||||
|
p := NewPythonDateParser()
|
||||||
|
now := time.Date(2026, 7, 6, 18, 0, 0, 0, time.Local)
|
||||||
|
ctx := context.Background()
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
text string
|
||||||
|
wantOK bool
|
||||||
|
checkT func(t *testing.T, got, now time.Time)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "ru relative — через час",
|
||||||
|
text: "напомни через час выпить воды",
|
||||||
|
wantOK: true,
|
||||||
|
checkT: func(t *testing.T, got, now time.Time) {
|
||||||
|
d := got.Sub(now)
|
||||||
|
if d < 50*time.Minute || d > 70*time.Minute {
|
||||||
|
t.Errorf("через час: got %v from now, want ~1h", d)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ru relative — через 30 минут",
|
||||||
|
text: "напомни через 30 минут снять бельё",
|
||||||
|
wantOK: true,
|
||||||
|
checkT: func(t *testing.T, got, now time.Time) {
|
||||||
|
d := got.Sub(now)
|
||||||
|
if d < 25*time.Minute || d > 35*time.Minute {
|
||||||
|
t.Errorf("через 30 минут: got %v from now, want ~30m", d)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ru absolute — завтра в 9 утра",
|
||||||
|
text: "напомни завтра в 9 утра позвонить",
|
||||||
|
wantOK: true,
|
||||||
|
checkT: func(t *testing.T, got, now time.Time) {
|
||||||
|
if got.Hour() != 9 {
|
||||||
|
t.Errorf("завтра в 9 утра: hour=%d, want 9", got.Hour())
|
||||||
|
}
|
||||||
|
if got.Sub(now) < 12*time.Hour {
|
||||||
|
t.Errorf("завтра в 9 утра: only %v from now, want >12h (tomorrow)", got.Sub(now))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "ru complex — в следующую пятницу",
|
||||||
|
text: "напомни в следующую пятницу оплатить счёт",
|
||||||
|
wantOK: true,
|
||||||
|
checkT: func(t *testing.T, got, now time.Time) {
|
||||||
|
if got.Weekday() != time.Friday {
|
||||||
|
t.Errorf("в следующую пятницу: weekday=%v, want Friday", got.Weekday())
|
||||||
|
}
|
||||||
|
if got.Sub(now) < 24*time.Hour {
|
||||||
|
t.Errorf("в следующую пятницу: only %v from now, want >24h (future)", got.Sub(now))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "en relative — in 30 minutes",
|
||||||
|
text: "remind me in 30 minutes to drink water",
|
||||||
|
wantOK: true,
|
||||||
|
checkT: func(t *testing.T, got, now time.Time) {
|
||||||
|
d := got.Sub(now)
|
||||||
|
if d < 25*time.Minute || d > 35*time.Minute {
|
||||||
|
t.Errorf("in 30 minutes: got %v from now, want ~30m", d)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "en absolute — tomorrow at 8am",
|
||||||
|
text: "remind me tomorrow at 8am to call the doctor",
|
||||||
|
wantOK: true,
|
||||||
|
checkT: func(t *testing.T, got, now time.Time) {
|
||||||
|
if got.Hour() != 8 {
|
||||||
|
t.Errorf("tomorrow at 8am: hour=%d, want 8", got.Hour())
|
||||||
|
}
|
||||||
|
if got.Sub(now) < 12*time.Hour {
|
||||||
|
t.Errorf("tomorrow at 8am: only %v from now, want >12h (tomorrow)", got.Sub(now))
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no date — напомни мне",
|
||||||
|
text: "напомни мне",
|
||||||
|
wantOK: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no date — remind me",
|
||||||
|
text: "remind me to do something",
|
||||||
|
wantOK: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, ok, err := p.Parse(ctx, tt.text, now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Parse(%q) error: %v", tt.text, err)
|
||||||
|
}
|
||||||
|
if ok != tt.wantOK {
|
||||||
|
t.Fatalf("Parse(%q) ok=%v, want %v (got time=%v)", tt.text, ok, tt.wantOK, got)
|
||||||
|
}
|
||||||
|
if ok && tt.checkT != nil {
|
||||||
|
tt.checkT(t, got, now)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestPythonDateParser_FallbackToStub verifies that when python3 is not
|
||||||
|
// available, the parser falls back to StubDateTimeParser instead of
|
||||||
|
// returning an error.
|
||||||
|
func TestPythonDateParser_FallbackToStub(t *testing.T) {
|
||||||
|
if _, err := exec.LookPath("python3"); err == nil {
|
||||||
|
t.Skip("python3 is available — fallback path not exercised")
|
||||||
|
}
|
||||||
|
p := NewPythonDateParser()
|
||||||
|
ctx := context.Background()
|
||||||
|
now := time.Date(2026, 7, 6, 18, 0, 0, 0, time.Local)
|
||||||
|
|
||||||
|
// The stub handles "in 1 hour" — should still work via fallback.
|
||||||
|
got, ok, err := p.Parse(ctx, "in 1 hour", now)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("fallback Parse error: %v", err)
|
||||||
|
}
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("fallback Parse: ok=false, want true (stub should handle 'in 1 hour')")
|
||||||
|
}
|
||||||
|
d := got.Sub(now)
|
||||||
|
if d < 50*time.Minute || d > 70*time.Minute {
|
||||||
|
t.Errorf("fallback 'in 1 hour': got %v from now, want ~1h", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user