kami/Maven
1
0
Fork 0
Code Issues 1 Pull Requests 1 Actions Packages Projects Releases Wiki Activity
Files
67a5eb38059bd48fda1f8c4a8609b16ecb904bd7
Maven/Makefile
T
kami 50ca8c8b5a Score the chat, query and knowledge phrasing paths (#395) 
The phrasing fixture was 15 nudge cases, so every prompt change we
measured only told us about nudges. But the shared context block sits in
front of five prompts, and three of them — chat, note query, general
knowledge — had no scorer at all. Those are the long free-form replies,
where a persona break is most likely and where nothing could see one.

27 cases, nine per path. Nine rather than five because the nudge fixture
already cannot resolve a change smaller than about three cases, and a
per-path score off five would be worse.

Reuses the persona checks instead of copying them. Length, mood and
"no questions" are left out on purpose: these paths return no mood, and
a follow-up question is a feature in chat, not a fault.

The run refuses to score unless the model answers before and after it.
PhraseChat and PhraseQuery swallow model errors and return a canned
string, so without that guard a dead server produces a full report with
zero errors and a bad score — which reads as bad phrasing rather than as
nothing measured. Vikunja #397 is the real fix.
2026-07-31 16:51:52 +04:00

189 lines
8.4 KiB
Makefile
Raw Blame History

# GOTOOLCHAIN=local pins us to the vendored deps/go tree. Without it, a go.mod
# `go` directive newer than deps/go re-execs into a downloaded toolchain module,
# and those ship only 7 of the 15 GOROOT tools (no covdata) -- which makes
# `test` below fail on the two packages that have no test files. deps-go builds
# the missing tools in, so the vendored tree is self-sufficient. Keep the version
# here in step with the `go` directive in go.mod.
GO_VERSION := 1.25.5
GO := $(shell pwd)/deps/go/go/bin/go
export GOTOOLCHAIN := local
GOFLAGS :=
CGO_LDFLAGS := -L$(shell pwd)/deps/lib -Wl,-rpath,$(shell pwd)/deps/lib
CGO_CFLAGS := -I$(shell pwd)/deps/include -I$(shell pwd)/deps/whisper.cpp/ggml/include
WHISPER_MODEL := $(shell pwd)/models/stt/ggml-small.bin
PIPER_BIN := $(shell pwd)/deps/piper/piper
PIPER_MODEL := $(shell pwd)/models/tts/ru_RU-irina-medium.onnx
PIPER_ESPEAK := $(shell pwd)/deps/piper/espeak-ng-data
.PHONY: all build build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav clean test fmt-check vet run-stt run-tts run-web download-embedder deps-go eval-router eval-recall eval-phrasing eval-models
all: build
build: build-stt build-tts build-daemon build-client build-waked build-web build-poll build-caldav
build-stt:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavsttd ./cmd/mavsttd/
build-tts:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavttsd ./cmd/mavttsd/
build-daemon:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavend ./cmd/mavend/
build-client:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) build $(GOFLAGS) -o mavenclient ./cmd/mavenclient/
build-waked:
$(GO) build $(GOFLAGS) -o mavwaked ./cmd/mavwaked/
build-web:
$(GO) build $(GOFLAGS) -o mavweb ./cmd/mavweb/
build-poll:
$(GO) build $(GOFLAGS) -o mavpoll ./cmd/mavpoll/
build-caldav:
$(GO) build $(GOFLAGS) -o mavcaldav ./cmd/mavcaldav/
run-web: build-web
./mavweb -addr :9200 -voice 127.0.0.1:9100
# Install the vendored Go toolchain from scratch. Go 1.24+ release tarballs no
# longer ship covdata/pprof/test2json/nm/objdump/trace prebuilt -- `go tool X`
# builds them on demand, but `go test -coverprofile` calls covdata through
# base.Tool(), which only stats pkg/tool and exits. So build them in once here.
GO_TARBALL := go$(GO_VERSION).linux-amd64.tar.gz
GO_SHA256 := 9e9b755d63b36acf30c12a9a3fc379243714c1c6d3dd72861da637f336ebb35b
deps-go:
@mkdir -p deps/go
cd deps/go && curl -fLO 'https://go.dev/dl/$(GO_TARBALL)'
cd deps/go && echo '$(GO_SHA256) $(GO_TARBALL)' | sha256sum -c -
cd deps/go && rm -rf go && tar xzf $(GO_TARBALL) && rm $(GO_TARBALL)
cd deps/go/go/src && for t in covdata pprof test2json nm objdump trace addr2line buildid; do \
GOROOT="$(shell pwd)/deps/go/go" $(GO) build -o "$(shell pwd)/deps/go/go/pkg/tool/linux_amd64/$$t" "cmd/$$t" || exit 1; \
done
$(GO) version
# fmt-check fails if any file needs gofmt. DESIGN.md has always said `make
# test` gates on gofmt and vet; it did not, so nine files quietly drifted.
# Run `gofmt -w` on whatever this prints.
fmt-check:
@bad=$$(gofmt -l internal cmd); \
if [ -n "$$bad" ]; then \
echo "these files need gofmt:"; echo "$$bad"; exit 1; \
fi
vet:
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) vet ./internal/... ./cmd/...
test: fmt-check vet
CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
$(GO) test -race -coverprofile=coverage.out ./internal/... ./cmd/...
# eval-router — score the held-out RU routing fixture (internal/router/eval).
# Verbose so the report tables land in the terminal. MAVEN_ONNX_LIB points the
# prod-representative baseline at the vendored runtime; override it or set it
# empty to run only the deterministic hash ratchet. This is the measurement
# Vikunja #319 compares before #320 flips the route decider.
MAVEN_ONNX_LIB ?= $(shell pwd)/deps/onnxruntime-linux-x64-1.26.0/lib/libonnxruntime.so
eval-router:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/router/eval/
# eval-recall — score the held-out note-recall fixture (internal/memory/recalleval).
# Answers "can she find the note again when it matters": recall@1, recall@3,
# false recall and the query_min_score sweep. Same MAVEN_ONNX_LIB deal as
# eval-router; without it only the deterministic hash ratchet runs.
eval-recall:
MAVEN_ONNX_LIB="$(MAVEN_ONNX_LIB)" $(GO) test -v -count=1 ./internal/memory/recalleval/
# eval-phrasing -- score nudge phrasing AND the conversational paths (chat,
# query, general knowledge) in internal/phraser/eval. Verbose so the
# report and every generated message land in the terminal. With no environment
# it scores the deterministic Stub only, which is what CI runs. Set
# MAVEN_LLM_URL to add the resident model:
# MAVEN_LLM_URL=http://127.0.0.1:18099 make eval-phrasing
# The model run is slow (minutes) -- the timeout is raised to match. It covers
# two fixtures now (15 nudges + 27 conversational cases, and the chat replies are
# the long ones), hence 90m rather than 40m.
eval-phrasing:
$(GO) test -v -count=1 -timeout 90m ./internal/phraser/eval/
# eval-models — score ONE llama-server against the same fixture, for the
# resident-model bake-off (#278, #250). Start a server with the gguf you want,
# then:
#
# make eval-models MAVEN_LLM_URL=http://127.0.0.1:18100
#
# The report names carry the model llama-server reports, so runs from two
# checkpoints stay apart. Only the LLM test runs — the classifier baselines do
# not depend on the model and take the ONNX runtime with them.
MAVEN_LLM_URL ?= http://127.0.0.1:18099
eval-models:
MAVEN_LLM_URL="$(MAVEN_LLM_URL)" $(GO) test -v -count=1 -timeout 60m \
-run TestLLMRouterBaseline ./internal/router/eval/
run-stt: build-stt
LD_LIBRARY_PATH="$(shell pwd)/deps/lib" \
./mavsttd -socket /tmp/maven/stt.sock -model $(WHISPER_MODEL)
run-tts: build-tts
LD_LIBRARY_PATH="$(shell pwd)/deps/piper" \
./mavttsd -socket /tmp/maven/tts.sock \
-piper $(PIPER_BIN) -model $(PIPER_MODEL) -espeak_data $(PIPER_ESPEAK)
deps: deps-whisper deps-piper
deps-whisper:
cd deps/whisper.cpp && cmake -B build -DCMAKE_BUILD_TYPE=Release \
-DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_SERVER=OFF && \
cmake --build build --config Release -j$$(nproc)
cp deps/whisper.cpp/build/bin/libwhisper.so* deps/lib/
cp deps/whisper.cpp/build/bin/libggml*.so* deps/lib/
cp deps/whisper.cpp/build/bin/libparakeet.so* deps/lib/
deps-piper:
mkdir -p deps/
curl -sL "https://github.com/rhasspy/piper/releases/download/2023.11.14-2/piper_linux_x86_64.tar.gz" \
-o /tmp/piper.tar.gz
tar -xzf /tmp/piper.tar.gz -C deps/
# multilingual-e5-small: an asymmetric retrieval model. It is trained to match
# a short question against a longer passage, which is what note recall is.
# The quantized file is the one we download, deploy and measure — see
# RECALL-EVAL-31-07-2026.md.
EMBEDDER_DIR := $(shell pwd)/models/embedder/multilingual-e5-small
EMBEDDER_MODEL_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/onnx/model_quantized.onnx
EMBEDDER_TOKENIZER_URL := https://huggingface.co/Xenova/multilingual-e5-small/resolve/main/tokenizer.json
download-embedder:
mkdir -p $(EMBEDDER_DIR)
curl -sL "$(EMBEDDER_MODEL_URL)" -o "$(EMBEDDER_DIR)/model_quantized.onnx"
curl -sL "$(EMBEDDER_TOKENIZER_URL)" -o "$(EMBEDDER_DIR)/tokenizer.json"
@echo ""
@echo "embedder model downloaded to $(EMBEDDER_DIR)/"
@echo "To use it, add to mavend.json:"
@echo ' "voice": {'
@echo ' ...'
@echo ' "embedder": {'
@echo ' "model_path": "$(EMBEDDER_DIR)/model_quantized.onnx",'
@echo ' "tokenizer_path": "$(EMBEDDER_DIR)/tokenizer.json",'
@echo ' "lib_path": "/path/to/libonnxruntime.so"'
@echo ' }'
@echo ' }'
@echo ""
@echo "Install libonnxruntime.so from: https://github.com/microsoft/onnxruntime/releases"
@echo "e.g. on x86_64 Linux:"
@echo ' curl -sL "https://github.com/microsoft/onnxruntime/releases/download/v1.15.1/onnxruntime-linux-x64-1.15.1.tgz" | tar xz'
@echo ' sudo cp onnxruntime-linux-x64-1.15.1/lib/libonnxruntime.so* /usr/local/lib/'
clean:
rm -f mavend mavenclient mavsttd mavttsd mavweb mavpoll mavcaldav mavwaked
Reference in New Issue View Git Blame Copy Permalink