kami/Maven
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
overnight/email-poller
Maven/Makefile
T
kami ee7bec11e3 Add mavmaild, the read-only IMAP poller that feeds mail intake (#246) 
The extraction seam landed on the previous branch but nothing fed it. This
adds the daemon that does: every interval it opens one mailbox read-only
(EXAMINE + BODY.PEEK, so reading leaves no \Seen behind), fetches the UIDs
it has not handed over yet, and posts each message to core over
ingest_mail. Core runs the model and writes task candidates; this daemon
writes nothing and cannot create a reminder.

It is a separate daemon because of the credential. mavpoll set the
precedent with the zenmoney token (#125): the module talking to the third
party holds the secret, reads it from a file so it never lands in argv, in
docker-compose.yml or in shell history, and core never sees it. There is
deliberately no -password flag, and a test asserts that.

Off unless configured at both ends: without -password-file the daemon
refuses to start, and if core has no email block the first ingest returns
ErrUnknownMethod, which disables the reader instead of hammering a socket
that will keep refusing. A seen-UID state file (0600, atomic write) keeps a
restart from re-extracting the whole lookback window; correctness does not
depend on it, since capture dedupes on normalised text. Logs are counts and
UIDs — no subject, sender or body.

Verified with an in-process IMAP server and a fake core: bulk mail is
filtered before core is asked, seen UIDs are not re-fetched, a failed
ingest is retried next poll, ErrUnknownMethod stops at the first message,
and state survives a restart. The live half is untested by design — no IMAP
credential exists on this box; setup is written up as QA steps.

Vikunja #246
2026-08-01 03:13:18 +04:00

192 lines
8.4 KiB
Makefile
Raw Permalink Blame History

# GOTOOLCHAIN=local pins us to the vendored deps/go tree. Without it, a go.mod
# `go` directive newer than deps/go re-execs into a downloaded toolchain module,
# and those ship only 7 of the 15 GOROOT tools (no covdata) -- which makes
# `test` below fail on the two packages that have no test files. deps-go builds
# the missing tools in, so the vendored tree is self-sufficient. Keep the version
# here in step with the `go` directive in go.mod.
GO_VERSION := 1.25.5
GO := $(shell pwd)/deps/go/go/bin/go
export GOTOOLCHAIN := local
GOFLAGS :=
CGO_LDFLAGS := -L$(shell pwd)/deps/lib -Wl,-rpath,$(shell pwd)/deps/lib
CGO_CFLAGS := -I$(shell pwd)/deps/include -I$(shell pwd)/deps/whisper.cpp/ggml/include
WHISPER_MODEL := $(shell pwd)/models/stt/ggml-small.bin
PIPER_BIN := $(shell pwd)/deps/piper/piper
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models
all: build
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail
build-stt:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavsttd ./cmd/mavsttd/
build-tts:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavttsd ./cmd/mavttsd/
build-daemon:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavend ./cmd/mavend/
build-client:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavenclient ./cmd/mavenclient/
build-waked:
$(GO) build $(GOFLAGS) -o mavwaked ./cmd/mavwaked/
build-web:
$(GO) build $(GOFLAGS) -o mavweb ./cmd/mavweb/
build-poll:
$(GO) build $(GOFLAGS) -o mavpoll ./cmd/mavpoll/
build-caldav:
$(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/
build-mail:
$(GO) build $(GOFLAGS) -o mavmaild ./cmd/mavmaild/
run-web: build-web
./mavweb -addr :9200 -voice 127.0.0.1:9100
# Install the vendored Go toolchain from scratch. Go 1.24+ release tarballs no
# longer ship covdata/pprof/test2json/nm/objdump/trace prebuilt -- `go tool X`
# builds them on demand, but `go test -coverprofile` calls covdata through
# base.Tool(), which only stats pkg/tool and exits. So build them in once here.
GO_TARBALL := go$(GO_VERSION).linux-amd64.tar.gz
GO_SHA256 := 9e9b755d63b36acf30c12a9a3fc379243714c1c6d3dd72861da637f336ebb35b
deps-go:
@mkdir -p deps/go
cd deps/go && curl -fLO 'https://go.dev/dl/$(GO_TARBALL)'
cd deps/go && echo '$(GO_SHA256) $(GO_TARBALL)' | sha256sum -c -
cd deps/go && rm -rf go && tar xzf $(GO_TARBALL) && rm $(GO_TARBALL)
cd deps/go/go/src && for t in covdata pprof test2json nm objdump trace addr2line buildid; do \
GOROOT="$(shell pwd)/deps/go/go" $(GO) build -o "$(shell pwd)/deps/go/go/pkg/tool/linux_amd64/$$t" "cmd/$$t" || exit 1; \
done
$(GO) version
# fmt-check fails if any file needs gofmt. DESIGN.md has always said `make
# test` gates on gofmt and vet; it did not, so nine files quietly drifted.
# Run `gofmt -w` on whatever this prints.
fmt-check:
@bad=$$(gofmt -l internal cmd); \
if [ -n "$$bad" ]; then \
echo "these files need gofmt:"; echo "$$bad"; exit 1; \
fi
vet:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) vet ./internal/... ./cmd/...
test: fmt-check vet
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/...
# eval-router — score the held-out RU routing fixture (internal/router/eval).
# Verbose so the report tables land in the terminal. MAVEN_ONNX_LIB points the
# prod-representative baseline at the vendored runtime; override it or set it
# empty to run only the deterministic hash ratchet. This is the measurement
# Vikunja #319 compares before #320 flips the route decider.
MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxruntime.so
eval-router:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
# Answers "can she find the note again when it matters": recall@1, recall@3,
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
# eval-router; without it only the deterministic hash ratchet runs.
eval-recall:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/memory/recalleval/
# eval-phrasing -- score nudge phrasing AND the conversational paths (chat,
# query, general knowledge) in internal/phraser/eval. Verbose so the
# report and every generated message land in the terminal. With no environment
# it scores the deterministic Stub only, which is what CI runs. Set
# MAVEN_LLM_URL to add the resident model:
# MAVEN_LLM_URL=http://127.0.0.1:18099 make eval-phrasing
# The model run is slow (minutes) -- the timeout is raised to match. It covers
# two fixtures now (15 nudges + 27 conversational cases, and the chat replies are
# the long ones), hence 90m rather than 40m.
eval-phrasing:
$(GO) test -v -count=1 -timeout 90m ./internal/phraser/eval/
# eval-models — score ONE llama-server against the same fixture, for the
# resident-model bake-off (#278, #250). Start a server with the gguf you want,
# then:
#
# make eval-models MAVEN_LLM_URL=http://127.0.0.1:18100
#
# The report names carry the model llama-server reports, so runs from two
# checkpoints stay apart. Only the LLM test runs — the classifier baselines do
# not depend on the model and take the ONNX runtime with them.
MAVEN_LLM_URL ?= http://127.0.0.1:18099
eval-models:
MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \
-run TestLLMRouterBaseline ./internal/router/eval/
run-stt: build-stt
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
run-tts: build-tts
LD_LIBRARY_PATH="$(shell pwd)/deps/piper" \
./mavttsd -socket /tmp/maven/tts.sock \
-piper $(PIPER_BIN) -model $(PIPER_MODEL) -espeak_data $(PIPER_ESPEAK)
deps: deps-whisper deps-piper
deps-whisper:
cd deps/whisper.cpp && cmake -B build -DCMAKE_BUILD_TYPE=Release \
-DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_SERVER=OFF && \
cmake --build build --config Release -j$$(nproc)
cp deps/whisper.cpp/build/bin/libwhisper.so* deps/lib/
cp deps/whisper.cpp/build/bin/libggml*.so* deps/lib/
cp deps/whisper.cpp/build/bin/libparakeet.so* deps/lib/
deps-piper:
mkdir -p deps/
curl -sL "https://github.com/rhasspy/piper/releases/download/2023.11.14-2/piper_linux_x86_64.tar.gz" \
-o /tmp/piper.tar.gz
tar -xzf /tmp/piper.tar.gz -C deps/
# multilingual-e5-small: an asymmetric retrieval model. It is trained to match
# a short question against a longer passage, which is what note recall is.
# The quantized file is the one we download, deploy and measure — see
# RECALL-EVAL-31-07-2026.md.
EMBEDDER_DIR := $(shell pwd)/models/embedder/multilingual-e5-small
EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/onnx/model_quantized.onnx
EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/tokenizer.json
download-embedder:
mkdir -p $(EMBEDDER_DIR)
curl -sL "$(EMBEDDER_MODEL_URL)" -o "$(EMBEDDER_DIR)/model_quantized.onnx"
curl -sL "$(EMBEDDER_TOKENIZER_URL)" -o "$(EMBEDDER_DIR)/tokenizer.json"
@echo ""
@echo "embedder model downloaded to $(EMBEDDER_DIR)/"
@echo "To use it, add to mavend.json:"
@echo ' "voice": {'
@echo ' ...'
@echo ' "embedder": {'
@echo ' "model_path": "$(EMBEDDER_DIR)/model_quantized.onnx",'
@echo ' "tokenizer_path": "$(EMBEDDER_DIR)/tokenizer.json",'
@echo ' "lib_path": "/path/to/libonnxruntime.so"'
@echo ' }'
@echo ' }'
@echo ""
@echo "Install libonnxruntime.so from: https://github.com/microsoft/onnxruntime/releases"
@echo "e.g. on x86_64 Linux:"
@echo ' curl -sL "https://github.com/microsoft/onnxruntime/releases/download/v1.15.1/onnxruntime-linux-x64-1.15.1.tgz" | tar xz'
@echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/'
clean:
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked mavmaild
Reference in New Issue View Git Blame Copy Permalink