kami/Maven
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
overnight/coldstart-unlock
Maven/Makefile
T
kami 62cc072f8c Add golden-audio STT tests against real whisper.cpp (#288) 
Four committed WAV fixtures go through the real whisper.cpp binding in
cmd/mavsttd, so a wrong model, a wrong language hint, a broken resample
or a regressed silence gate fails `make test` instead of surfacing as
Maven mishearing him.

The fixtures are piper-synthesised, not recorded: scripts/gen-stt-fixtures.sh
drives the vendored piper with the ru_RU-irina voice Maven already speaks
with, so nothing of the owner's voice is committed and every fixture is
reproducible. 360K total for three Russian clips and one English.

Matching is tolerant on purpose. Golden transcripts move with the model,
so each case asserts intent-carrying keywords (prefix match, so Russian
inflection does not fail it) plus a word error rate ceiling, not an exact
string. The matcher is unit-tested on its own and needs no model.

TestGoldenAudioTranscription skips when models/stt/ggml-small.bin is
absent, so `make test` still passes on a box without models.
TestGoldenFixturesAreCanonical runs everywhere and checks the WAVs are
16k mono s16le and would clear mavsttd's own silence gate.
2026-08-01 05:32:10 +04:00

209 lines
9.3 KiB
Makefile
Raw Permalink Blame History

# GOTOOLCHAIN=local pins us to the vendored deps/go tree. Without it, a go.mod
# `go` directive newer than deps/go re-execs into a downloaded toolchain module,
# and those ship only 7 of the 15 GOROOT tools (no covdata) -- which makes
# `test` below fail on the two packages that have no test files. deps-go builds
# the missing tools in, so the vendored tree is self-sufficient. Keep the version
# here in step with the `go` directive in go.mod.
GO_VERSION := 1.25.5
GO := $(shell pwd)/deps/go/go/bin/go
export GOTOOLCHAIN := local
GOFLAGS :=
CGO_LDFLAGS := -L$(shell pwd)/deps/lib -Wl,-rpath,$(shell pwd)/deps/lib
CGO_CFLAGS := -I$(shell pwd)/deps/include -I$(shell pwd)/deps/whisper.cpp/ggml/include
WHISPER_MODEL := $(shell pwd)/models/stt/ggml-small.bin
PIPER_BIN := $(shell pwd)/deps/piper/piper
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
.PHONY: stt-fixtures test-stt-golden all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models
all: build
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav build-mail build-update
build-stt:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavsttd ./cmd/mavsttd/
build-tts:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavttsd ./cmd/mavttsd/
build-daemon:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavend ./cmd/mavend/
build-client:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavenclient ./cmd/mavenclient/
build-waked:
$(GO) build $(GOFLAGS) -o mavwaked ./cmd/mavwaked/
build-web:
$(GO) build $(GOFLAGS) -o mavweb ./cmd/mavweb/
build-poll:
$(GO) build $(GOFLAGS) -o mavpoll ./cmd/mavpoll/
build-caldav:
$(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/
build-mail:
$(GO) build $(GOFLAGS) -o mavmaild ./cmd/mavmaild/
# mavupdate is an operator CLI, not a daemon: nothing runs it but a human on the
# box. It is built with the rest so a broken update path is caught by `make
# build` rather than the first time it is needed.
build-update:
$(GO) build $(GOFLAGS) -o mavupdate ./cmd/mavupdate/
run-web: build-web
./mavweb -addr :9200 -voice 127.0.0.1:9100
# Install the vendored Go toolchain from scratch. Go 1.24+ release tarballs no
# longer ship covdata/pprof/test2json/nm/objdump/trace prebuilt -- `go tool X`
# builds them on demand, but `go test -coverprofile` calls covdata through
# base.Tool(), which only stats pkg/tool and exits. So build them in once here.
GO_TARBALL := go$(GO_VERSION).linux-amd64.tar.gz
GO_SHA256 := 9e9b755d63b36acf30c12a9a3fc379243714c1c6d3dd72861da637f336ebb35b
deps-go:
@mkdir -p deps/go
cd deps/go && curl -fLO 'https://go.dev/dl/$(GO_TARBALL)'
cd deps/go && echo '$(GO_SHA256) $(GO_TARBALL)' | sha256sum -c -
cd deps/go && rm -rf go && tar xzf $(GO_TARBALL) && rm $(GO_TARBALL)
cd deps/go/go/src && for t in covdata pprof test2json nm objdump trace addr2line buildid; do \
GOROOT="$(shell pwd)/deps/go/go" $(GO) build -o "$(shell pwd)/deps/go/go/pkg/tool/linux_amd64/$$t" "cmd/$$t" || exit 1; \
done
$(GO) version
# fmt-check fails if any file needs gofmt. DESIGN.md has always said `make
# test` gates on gofmt and vet; it did not, so nine files quietly drifted.
# Run `gofmt -w` on whatever this prints.
fmt-check:
@bad=$$(gofmt -l internal cmd); \
if [ -n "$$bad" ]; then \
echo "these files need gofmt:"; echo "$$bad"; exit 1; \
fi
vet:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) vet ./internal/... ./cmd/...
test: fmt-check vet
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/...
# eval-router — score the held-out RU routing fixture (internal/router/eval).
# Verbose so the report tables land in the terminal. MAVEN_ONNX_LIB points the
# prod-representative baseline at the vendored runtime; override it or set it
# empty to run only the deterministic hash ratchet. This is the measurement
# Vikunja #319 compares before #320 flips the route decider.
MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxruntime.so
eval-router:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
# Answers "can she find the note again when it matters": recall@1, recall@3,
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
# eval-router; without it only the deterministic hash ratchet runs.
eval-recall:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/memory/recalleval/
# eval-phrasing -- score nudge phrasing AND the conversational paths (chat,
# query, general knowledge) in internal/phraser/eval. Verbose so the
# report and every generated message land in the terminal. With no environment
# it scores the deterministic Stub only, which is what CI runs. Set
# MAVEN_LLM_URL to add the resident model:
# MAVEN_LLM_URL=http://127.0.0.1:18099 make eval-phrasing
# The model run is slow (minutes) -- the timeout is raised to match. It covers
# two fixtures now (15 nudges + 27 conversational cases, and the chat replies are
# the long ones), hence 90m rather than 40m.
eval-phrasing:
$(GO) test -v -count=1 -timeout 90m ./internal/phraser/eval/
# eval-models — score ONE llama-server against the same fixture, for the
# resident-model bake-off (#278, #250). Start a server with the gguf you want,
# then:
#
# make eval-models MAVEN_LLM_URL=http://127.0.0.1:18100
#
# The report names carry the model llama-server reports, so runs from two
# checkpoints stay apart. Only the LLM test runs — the classifier baselines do
# not depend on the model and take the ONNX runtime with them.
MAVEN_LLM_URL ?= http://127.0.0.1:18099
eval-models:
MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \
-run TestLLMRouterBaseline ./internal/router/eval/
# stt-fixtures — regenerate the golden STT audio in cmd/mavsttd/testdata from
# the piper voices (#288). The committed WAVs are synthesised, never recorded,
# so this is the only way they should ever change. TestGoldenAudioTranscription
# then scores them against ggml-small; it self-skips when the model is absent.
stt-fixtures:
./scripts/gen-stt-fixtures.sh
test-stt-golden:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) test -v -count=1 -run TestGolden ./cmd/mavsttd/
run-stt: build-stt
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
run-tts: build-tts
LD_LIBRARY_PATH="$(shell pwd)/deps/piper" \
./mavttsd -socket /tmp/maven/tts.sock \
-piper $(PIPER_BIN) -model $(PIPER_MODEL) -espeak_data $(PIPER_ESPEAK)
deps: deps-whisper deps-piper
deps-whisper:
cd deps/whisper.cpp && cmake -B build -DCMAKE_BUILD_TYPE=Release \
-DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_SERVER=OFF && \
cmake --build build --config Release -j$$(nproc)
cp deps/whisper.cpp/build/bin/libwhisper.so* deps/lib/
cp deps/whisper.cpp/build/bin/libggml*.so* deps/lib/
cp deps/whisper.cpp/build/bin/libparakeet.so* deps/lib/
deps-piper:
mkdir -p deps/
curl -sL "https://github.com/rhasspy/piper/releases/download/2023.11.14-2/piper_linux_x86_64.tar.gz" \
-o /tmp/piper.tar.gz
tar -xzf /tmp/piper.tar.gz -C deps/
# multilingual-e5-small: an asymmetric retrieval model. It is trained to match
# a short question against a longer passage, which is what note recall is.
# The quantized file is the one we download, deploy and measure — see
# RECALL-EVAL-31-07-2026.md.
EMBEDDER_DIR := $(shell pwd)/models/embedder/multilingual-e5-small
EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/onnx/model_quantized.onnx
EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/tokenizer.json
download-embedder:
mkdir -p $(EMBEDDER_DIR)
curl -sL "$(EMBEDDER_MODEL_URL)" -o "$(EMBEDDER_DIR)/model_quantized.onnx"
curl -sL "$(EMBEDDER_TOKENIZER_URL)" -o "$(EMBEDDER_DIR)/tokenizer.json"
@echo ""
@echo "embedder model downloaded to $(EMBEDDER_DIR)/"
@echo "To use it, add to mavend.json:"
@echo ' "voice": {'
@echo ' ...'
@echo ' "embedder": {'
@echo ' "model_path": "$(EMBEDDER_DIR)/model_quantized.onnx",'
@echo ' "tokenizer_path": "$(EMBEDDER_DIR)/tokenizer.json",'
@echo ' "lib_path": "/path/to/libonnxruntime.so"'
@echo ' }'
@echo ' }'
@echo ""
@echo "Install libonnxruntime.so from: https://github.com/microsoft/onnxruntime/releases"
@echo "e.g. on x86_64 Linux:"
@echo ' curl -sL "https://github.com/microsoft/onnxruntime/releases/download/v1.15.1/onnxruntime-linux-x64-1.15.1.tgz" | tar xz'
@echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/'
clean:
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked mavmaild
Reference in New Issue View Git Blame Copy Permalink