shutdown: close the sockets, or the database never gets sealed
mavend seals its encrypted database in `defer st.Close()` when run() returns. It had not returned since 2026-07-21. Every restart since then decrypted the same eleven-day-old ciphertext and rolled back everything written in between: the Telegram nudge that kept firing was a fact being un-written on each boot. The goroutine dump named it. main → srv.Close() → ipc.(*Server).Close → wg.Wait(), waiting on per-connection goroutines parked in readFrame. Close shut the listener and nothing else, so the idle persistent sockets held by mavweb, mavpoll, mavcaldav and mavmaild blocked shutdown forever. `docker compose stop -t 60` spent the whole sixty seconds and then took a SIGKILL. So: track the accepted conns and close them, in ipc and in voice, which had the identical defect. Bound all three waits — the two per-server ones and the worker wait in main — because the seal matters more than any single in-flight call. A dropped RPC costs one reply; a missed seal costs a session. The regression test leaves a client connected and idle, which is the case the old tests avoided by closing the client first. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01TrVSBKe3RFDF4fGYKWYQnX
This commit is contained in:
@@ -508,7 +508,6 @@ func (h *reactiveHandler) queryWeb(ctx context.Context, t *queryTurn) (string, b
|
||||
// model's own answer than by more waiting.
|
||||
const kiwixTimeout = 20 * time.Second
|
||||
|
||||
|
||||
// queryKiwix — the offline encyclopedia. The last local source: everything of
|
||||
// his has already had its turn and found nothing, so the question is a world
|
||||
// question, and reading beats recalling for a 1.7B.
|
||||
|
||||
+29
-1
@@ -751,7 +751,15 @@ func run(args []string) error {
|
||||
if voiceW != nil {
|
||||
voiceW.close()
|
||||
}
|
||||
wg.Wait()
|
||||
// Bounded. Every worker below watches ctx, but one parked in a model call
|
||||
// or an HTTP fetch can outlast the supervisor's patience, and run() has to
|
||||
// return for `defer st.Close()` to seal the database. A worker abandoned
|
||||
// mid-tick loses one tick; a shutdown that never returns loses every write
|
||||
// since the last clean stop — which is how the deployed ciphertext went
|
||||
// eleven days stale in July 2026.
|
||||
if !waitWorkers(&wg, workerGrace) {
|
||||
log.Printf("mavend: workers still running after %s, sealing anyway", workerGrace)
|
||||
}
|
||||
log.Printf("mavend: bye")
|
||||
return nil
|
||||
}
|
||||
@@ -782,3 +790,23 @@ func contextBlockFn(cfg *config.Config, now func() time.Time) func() string {
|
||||
f := personaFacts(cfg)
|
||||
return func() string { return f.Block(now()) }
|
||||
}
|
||||
|
||||
// workerGrace — how long shutdown waits for the background workers before it
|
||||
// goes ahead and seals without them. Comfortably inside docker's ten-second
|
||||
// default so the seal still lands before SIGKILL.
|
||||
const workerGrace = 4 * time.Second
|
||||
|
||||
// waitWorkers waits on wg for at most d. Reports whether they all finished.
|
||||
func waitWorkers(wg *sync.WaitGroup, d time.Duration) bool {
|
||||
done := make(chan struct{})
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(done)
|
||||
}()
|
||||
select {
|
||||
case <-done:
|
||||
return true
|
||||
case <-time.After(d):
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
// Command mavseal encrypts a live tmpfs working copy back to the ciphertext
|
||||
// file, for the case mavend could not do it itself.
|
||||
//
|
||||
// mavend seals its database in `defer st.Close()` when run() returns. A daemon
|
||||
// that is killed rather than shut down never gets there, and because the
|
||||
// working copy lives in the container's /dev/shm it dies with the container:
|
||||
// everything written since the last clean shutdown is lost, and the next boot
|
||||
// silently rolls back to the stale ciphertext. That is not hypothetical — on
|
||||
// 2026-08-01 the deployed ciphertext was eleven days old.
|
||||
//
|
||||
// This is a recovery tool, not part of the daemon. It is safe to run against a
|
||||
// live database: it takes a consistent snapshot with VACUUM INTO rather than
|
||||
// mutating the working copy the daemon owns.
|
||||
//
|
||||
// Usage:
|
||||
//
|
||||
// mavseal -plain /dev/shm/maven-plain.db -cipher /var/lib/maven/maven.db.enc
|
||||
//
|
||||
// The key is read from MAVEN_DB_KEY (base64, 32 bytes decoded), the same
|
||||
// variable the daemon uses. It is never taken as an argument: an argument ends
|
||||
// up in the shell history and in ps.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"github.com/kami/maven/internal/store"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func main() {
|
||||
log.SetFlags(0)
|
||||
if err := run(); err != nil {
|
||||
log.Fatalf("mavseal: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func run() error {
|
||||
plain := flag.String("plain", "", "path to the plaintext working copy (required)")
|
||||
cipher := flag.String("cipher", "", "path to write the ciphertext to (required)")
|
||||
keep := flag.Bool("keep-snapshot", false, "leave the intermediate snapshot on disk for inspection")
|
||||
flag.Parse()
|
||||
|
||||
if *plain == "" || *cipher == "" {
|
||||
flag.Usage()
|
||||
return fmt.Errorf("both -plain and -cipher are required")
|
||||
}
|
||||
key, err := readKey()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// VACUUM INTO rather than a WAL checkpoint on the file itself. The daemon
|
||||
// is usually still running and still writing when this is needed, and
|
||||
// checkpointing its working copy mutates a database it owns. VACUUM INTO
|
||||
// reads a consistent snapshot into a new file and touches nothing else, so
|
||||
// the worst case is a snapshot a few seconds stale instead of a torn one.
|
||||
snap := *plain + ".mavseal-snapshot"
|
||||
os.Remove(snap)
|
||||
if err := snapshot(*plain, snap); err != nil {
|
||||
return err
|
||||
}
|
||||
if !*keep {
|
||||
defer os.Remove(snap)
|
||||
}
|
||||
|
||||
before := fileSize(*cipher)
|
||||
if err := store.SealPlaintext(snap, *cipher, key); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("sealed %s → %s (%d bytes, was %d)", *plain, *cipher, fileSize(*cipher), before)
|
||||
return nil
|
||||
}
|
||||
|
||||
// readKey pulls the same base64 key the daemon reads. Fails closed: a short or
|
||||
// unparseable key must not silently produce a file nothing can open.
|
||||
func readKey() ([]byte, error) {
|
||||
raw := os.Getenv("MAVEN_DB_KEY")
|
||||
if raw == "" {
|
||||
return nil, fmt.Errorf("MAVEN_DB_KEY is not set")
|
||||
}
|
||||
key, err := base64.StdEncoding.DecodeString(raw)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("MAVEN_DB_KEY is not valid base64: %w", err)
|
||||
}
|
||||
if len(key) != 32 {
|
||||
return nil, fmt.Errorf("MAVEN_DB_KEY decodes to %d bytes, want 32", len(key))
|
||||
}
|
||||
return key, nil
|
||||
}
|
||||
|
||||
// snapshot writes a consistent copy of src to dst with VACUUM INTO. The copy
|
||||
// includes everything committed to the write-ahead log, which is most of what
|
||||
// is worth saving on a daemon that has been up for hours.
|
||||
func snapshot(src, dst string) error {
|
||||
db, err := sql.Open("sqlite", src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("open working copy: %w", err)
|
||||
}
|
||||
defer db.Close()
|
||||
if _, err := db.ExecContext(context.Background(), "VACUUM INTO ?", dst); err != nil {
|
||||
return fmt.Errorf("snapshot: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fileSize(path string) int64 {
|
||||
fi, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return fi.Size()
|
||||
}
|
||||
Reference in New Issue
Block a user