claude
246db4e609
The swap itself already landed: deploy loads models/embedder/multilingual-e5-small/model_quantized.onnx, and onnxembedder.go grew EmbedQuery/EmbedPassage with the query:/passage: prefixes the model was trained with. What was missing is the half of #371 that says "re-run make eval-recall and compare against the recorded numbers", so nothing in the repo says whether it worked. It worked, on every axis at once. recall@1 60.0% → 70.4%, recall@3 80.0% → 85.2%, answered after the gate 48.0% → 63.0%, false recall 1/5 → 0/5, and latency p50 59ms → 23ms because the quantized file is 118MB against the 470MB fp32 one the old config loaded. The guitar-chords note no longer beats the docker-logs note. One premise of the task did not come true and the new doc says so. #371 expected a better retriever to separate the score distributions and make query_min_score tunable. It did not: right-first top-1 runs 0.791-0.890 and must-stay-silent runs 0.795-0.835, still overlapping, just higher and tighter. The margin separates them instead — 0.024 median against 0.002 — and 0.008 is the knee where all five silent cases are silenced at no cost. The score gate is close to inert now; the margin is the live dial. Neither is changed here, since #412 is where a sweep belongs. docs/evals/2026-08-04-recall-e5-small.md is the dated measurement. rearchitecture.md's "upgrade MiniLM → bge-m3 later" is now done and says so, CLAUDE.md names the retriever and the prefix rule where it already promises the embedder never leaves homesrv, and the Makefile comment points at this eval instead of the one that asked for the swap. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
228 lines
10 KiB
Makefile
228 lines
10 KiB
Makefile
# GOTOOLCHAIN=local pins us to the vendored deps/go tree. Without it, a go.mod
|
|
# `go` directive newer than deps/go re-execs into a downloaded toolchain module,
|
|
# and those ship only 7 of the 15 GOROOT tools (no covdata) -- which makes
|
|
# `test` below fail on the two packages that have no test files. deps-go builds
|
|
# the missing tools in, so the vendored tree is self-sufficient. Keep the version
|
|
# here in step with the `go` directive in go.mod.
|
|
GO_VERSION := 1.25.5
|
|
GO := $(shell pwd)/deps/go/go/bin/go
|
|
export GOTOOLCHAIN := local
|
|
GOFLAGS :=
|
|
CGO_LDFLAGS := -L$(shell pwd)/deps/lib -Wl,-rpath,$(shell pwd)/deps/lib
|
|
CGO_CFLAGS := -I$(shell pwd)/deps/include -I$(shell pwd)/deps/whisper.cpp/ggml/include
|
|
|
|
WHISPER_MODEL := $(shell pwd)/models/stt/ggml-small.bin
|
|
PIPER_BIN := $(shell pwd)/deps/piper/piper
|
|
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
|
|
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
|
|
|
|
.PHONY: simulate stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models build-gpud
|
|
|
|
all: build
|
|
|
|
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update build-gpud
|
|
|
|
build-stt:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) build $(GOFLAGS) -o mavsttd ./cmd/mavsttd/
|
|
|
|
build-tts:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) build $(GOFLAGS) -o mavttsd ./cmd/mavttsd/
|
|
|
|
build-daemon:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) build $(GOFLAGS) -o mavend ./cmd/mavend/
|
|
|
|
build-client:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) build $(GOFLAGS) -o mavenclient ./cmd/mavenclient/
|
|
|
|
build-waked:
|
|
$(GO) build $(GOFLAGS) -o mavwaked ./cmd/mavwaked/
|
|
|
|
build-web:
|
|
$(GO) build $(GOFLAGS) -o mavweb ./cmd/mavweb/
|
|
|
|
build-poll:
|
|
$(GO) build $(GOFLAGS) -o mavpoll ./cmd/mavpoll/
|
|
|
|
build-caldav:
|
|
$(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/
|
|
|
|
build-mail:
|
|
$(GO) build $(GOFLAGS) -o mavmaild ./cmd/mavmaild/
|
|
|
|
# mavupdate is an operator CLI, not a daemon: nothing runs it but a human on the
|
|
# box. It is built with the rest so a broken update path is caught by `make
|
|
# build` rather than the first time it is needed.
|
|
build-update:
|
|
$(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/
|
|
|
|
# mavgpud runs on the workstation, not here. It is built with the rest so a
|
|
# broken supervisor is caught by `make build` on homesrv rather than by the
|
|
# workstation refusing to serve. Copy the binary over, do not `make deploy` it.
|
|
build-gpud:
|
|
$(GO) build $(GOFLAGS) -o mavgpud ./cmd/mavgpud/
|
|
|
|
run-web: build-web
|
|
./mavweb -addr :9200 -voice 127.0.0.1:9100
|
|
|
|
# Install the vendored Go toolchain from scratch. Go 1.24+ release tarballs no
|
|
# longer ship covdata/pprof/test2json/nm/objdump/trace prebuilt -- `go tool X`
|
|
# builds them on demand, but `go test -coverprofile` calls covdata through
|
|
# base.Tool(), which only stats pkg/tool and exits. So build them in once here.
|
|
GO_TARBALL := go$(GO_VERSION).linux-amd64.tar.gz
|
|
GO_SHA256 := 9e9b755d63b36acf30c12a9a3fc379243714c1c6d3dd72861da637f336ebb35b
|
|
deps-go:
|
|
@mkdir -p deps/go
|
|
cd deps/go && curl -fLO 'https://go.dev/dl/$(GO_TARBALL)'
|
|
cd deps/go && echo '$(GO_SHA256) $(GO_TARBALL)' | sha256sum -c -
|
|
cd deps/go && rm -rf go && tar xzf $(GO_TARBALL) && rm $(GO_TARBALL)
|
|
cd deps/go/go/src && for t in covdata pprof test2json nm objdump trace addr2line buildid; do \
|
|
GOROOT="$(shell pwd)/deps/go/go" $(GO) build -o "$(shell pwd)/deps/go/go/pkg/tool/linux_amd64/$$t" "cmd/$$t" || exit 1; \
|
|
done
|
|
$(GO) version
|
|
|
|
# fmt-check fails if any file needs gofmt. docs/design.md has always said `make
|
|
# test` gates on gofmt and vet; it did not, so nine files quietly drifted.
|
|
# Run `gofmt -w` on whatever this prints.
|
|
fmt-check:
|
|
@bad=$$(gofmt -l internal cmd); \
|
|
if [ -n "$$bad" ]; then \
|
|
echo "these files need gofmt:"; echo "$$bad"; exit 1; \
|
|
fi
|
|
|
|
vet:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) vet ./internal/... ./cmd/...
|
|
|
|
# simulate — replay every scripted day under cmd/mavend/testdata/scenarios
|
|
# through the real router, store, tick loop and intake journal, on a fake clock
|
|
# (Vikunja #284). Verbose so the transcript of each scenario lands in the
|
|
# terminal. Also runs as part of `make test`; this target is for reading it.
|
|
simulate:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) test -v -count=1 -run TestSimulator ./cmd/mavend/
|
|
|
|
test: fmt-check vet
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/...
|
|
|
|
# eval-router — score the held-out RU routing fixture (internal/router/eval).
|
|
# Verbose so the report tables land in the terminal. MAVEN_ONNX_LIB points the
|
|
# prod-representative baseline at the vendored runtime; override it or set it
|
|
# empty to run only the deterministic hash ratchet. This is the measurement
|
|
# Vikunja #319 compares before #320 flips the route decider.
|
|
MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxruntime.so
|
|
|
|
eval-router:
|
|
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
|
|
|
|
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
|
|
# Answers "can she find the note again when it matters": recall@1, recall@3,
|
|
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
|
|
# eval-router; without it only the deterministic hash ratchet runs.
|
|
eval-recall:
|
|
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/memory/recalleval/
|
|
|
|
# eval-phrasing -- score nudge phrasing AND the conversational paths (chat,
|
|
# query, general knowledge) in internal/phraser/eval. Verbose so the
|
|
# report and every generated message land in the terminal. With no environment
|
|
# it scores the deterministic Stub only, which is what CI runs. Set
|
|
# MAVEN_LLM_URL to add the resident model:
|
|
# MAVEN_LLM_URL=http://127.0.0.1:18099 make eval-phrasing
|
|
# The model run is slow (minutes) -- the timeout is raised to match. It covers
|
|
# two fixtures now (15 nudges + 27 conversational cases, and the chat replies are
|
|
# the long ones), hence 90m rather than 40m.
|
|
eval-phrasing:
|
|
$(GO) test -v -count=1 -timeout 90m ./internal/phraser/eval/
|
|
|
|
# eval-models — score ONE llama-server against the same fixture, for the
|
|
# resident-model bake-off (#278, #250). Start a server with the gguf you want,
|
|
# then:
|
|
#
|
|
# make eval-models MAVEN_LLM_URL=http://127.0.0.1:18100
|
|
#
|
|
# The report names carry the model llama-server reports, so runs from two
|
|
# checkpoints stay apart. Only the LLM test runs — the classifier baselines do
|
|
# not depend on the model and take the ONNX runtime with them.
|
|
MAVEN_LLM_URL ?= http://127.0.0.1:18099
|
|
|
|
eval-models:
|
|
MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \
|
|
-run TestLLMRouterBaseline ./internal/router/eval/
|
|
|
|
# stt-fixtures — regenerate the golden STT audio in cmd/mavsttd/testdata from
|
|
# the piper voices (#288). The committed WAVs are synthesised, never recorded,
|
|
# so this is the only way they should ever change. The spoken text is read out
|
|
# of testdata/golden_v1.json, so edit the transcript there and rerun this.
|
|
#
|
|
# test-stt-golden runs both golden tests: TestGoldenAudioTranscription, which
|
|
# scores the fixtures against ggml-small and self-skips when the model is
|
|
# absent, and TestGoldenFixturesAreCanonical, which checks the committed audio
|
|
# and the manifest with no model at all.
|
|
stt-fixtures:
|
|
./scripts/gen-stt-fixtures.sh
|
|
|
|
test-stt-golden:
|
|
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
$(GO) test -v -count=1 -run TestGolden ./cmd/mavsttd/
|
|
|
|
run-stt: build-stt
|
|
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
|
|
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
|
|
|
|
run-tts: build-tts
|
|
LD_LIBRARY_PATH="$(shell pwd)/deps/piper" \
|
|
./mavttsd -socket /tmp/maven/tts.sock \
|
|
-piper $(PIPER_BIN) -model $(PIPER_MODEL) -espeak_data $(PIPER_ESPEAK)
|
|
|
|
deps: deps-whisper deps-piper
|
|
|
|
deps-whisper:
|
|
cd deps/whisper.cpp && cmake -B build -DCMAKE_BUILD_TYPE=Release \
|
|
-DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_SERVER=OFF && \
|
|
cmake --build build --config Release -j$$(nproc)
|
|
cp deps/whisper.cpp/build/bin/libwhisper.so* deps/lib/
|
|
cp deps/whisper.cpp/build/bin/libggml*.so* deps/lib/
|
|
cp deps/whisper.cpp/build/bin/libparakeet.so* deps/lib/
|
|
|
|
deps-piper:
|
|
mkdir -p deps/
|
|
curl -sL "https://github.com/rhasspy/piper/releases/download/2023.11.14-2/piper_linux_x86_64.tar.gz" \
|
|
-o /tmp/piper.tar.gz
|
|
tar -xzf /tmp/piper.tar.gz -C deps/
|
|
|
|
# multilingual-e5-small: an asymmetric retrieval model. It is trained to match
|
|
# a short question against a longer passage, which is what note recall is.
|
|
# The quantized file is the one we download, deploy and measure — see
|
|
# docs/evals/2026-08-04-recall-e5-small.md for what the swap bought.
|
|
EMBEDDER_DIR := $(shell pwd)/models/embedder/multilingual-e5-small
|
|
EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/onnx/model_quantized.onnx
|
|
EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/tokenizer.json
|
|
|
|
download-embedder:
|
|
mkdir -p $(EMBEDDER_DIR)
|
|
curl -sL "$(EMBEDDER_MODEL_URL)" -o "$(EMBEDDER_DIR)/model_quantized.onnx"
|
|
curl -sL "$(EMBEDDER_TOKENIZER_URL)" -o "$(EMBEDDER_DIR)/tokenizer.json"
|
|
@echo ""
|
|
@echo "embedder model downloaded to $(EMBEDDER_DIR)/"
|
|
@echo "To use it, add to mavend.json:"
|
|
@echo ' "voice": {'
|
|
@echo ' ...'
|
|
@echo ' "embedder": {'
|
|
@echo ' "model_path": "$(EMBEDDER_DIR)/model_quantized.onnx",'
|
|
@echo ' "tokenizer_path": "$(EMBEDDER_DIR)/tokenizer.json",'
|
|
@echo ' "lib_path": "/path/to/libonnxruntime.so"'
|
|
@echo ' }'
|
|
@echo ' }'
|
|
@echo ""
|
|
@echo "Install libonnxruntime.so from: https://github.com/microsoft/onnxruntime/releases"
|
|
@echo "e.g. on x86_64 Linux:"
|
|
@echo ' curl -sL "https://github.com/microsoft/onnxruntime/releases/download/v1.15.1/onnxruntime-linux-x64-1.15.1.tgz" | tar xz'
|
|
@echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/'
|
|
|
|
clean:
|
|
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked mavmaild
|