From 7f25dc43e44faa6931099be14b93fd012f5970b0 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 13:39:06 +0400 Subject: [PATCH] Catch a worker serving code older than its file A full vision + identity + reconcile cycle ran today against a worker that had been started before the fix it was supposed to prove. It reproduced the defect exactly: 46 of 110 boxes past the 900px panel width, coordinates clamping at 1000, the same fingerprint measured before the fix. vision worker started 12:00:09 worker_vision.py modified 12:11:35 8113bdf, carrying _bbox_to_pixels 12:16:22 Python binds a module once, at process start. Editing the file afterwards changes nothing until the process restarts, and nothing in the result says so: the stage reported completed 116/116, the orchestrator recorded no error, and identity and reconcile ran to completion on top of it. Cost was one cycle plus a registry reset to undo the 8 characters it minted. This was already known as advice. The previous handoff said the render worker "must be restarted by hand to pick up an edit". Advice did not stop it. check_stale.sh compares every running worker's process start against its module's mtime and exits non-zero if any is stale, so it can gate a script. Mutation-tested by touching worker_tts.py, which it caught. decisions/identity-bbox.md#stale-worker-invalidates carries the evidence and is committed separately with the rest of the session's notes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011FmkHKFtuhppZE3B4VvNNE --- check_stale.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 check_stale.sh diff --git a/check_stale.sh b/check_stale.sh new file mode 100755 index 0000000..95bda04 --- /dev/null +++ b/check_stale.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Which running workers are serving code older than the file on disk. +# +# Python loads a worker module once, at process start. Editing the file afterwards changes nothing +# until that process restarts, and nothing in the stage output says so: the stage completes, the +# counters read 116/116, and the result is what the OLD code produced. On 2026-08-12 this ate a full +# vision + identity + reconcile cycle — the bbox fix was committed 16 minutes after the worker that +# was supposed to apply it had already started. +# +# Run this before any stage run that is meant to prove a code change. +# Exits non-zero if anything is stale, so it can gate a script. +set -uo pipefail +cd "$(dirname "$0")" + +stale=0 +# `ps -o lstart` is always exactly five fields: "Wed Aug 12 13:20:00 2026". Read them positionally +# rather than trying to split the line with a regex. +while read -r pid _dow mon day time year cmd; do + mod=$(grep -oP '(?:worker_\w+|session_manager)(?=:app)' <<<"$cmd" | head -1) + [ -n "$mod" ] && [ -f "$mod.py" ] || continue + p_epoch=$(date -d "$mon $day $time $year" +%s 2>/dev/null) || continue + f_epoch=$(stat -c %Y "$mod.py") + if [ "$f_epoch" -gt "$p_epoch" ]; then + printf 'STALE %-22s pid %-7s edited %dm after it started\n' \ + "$mod.py" "$pid" "$(( (f_epoch - p_epoch) / 60 ))" + stale=1 + else + printf 'ok %-22s pid %s\n' "$mod.py" "$pid" + fi +done < <(ps -eo pid,lstart,cmd | grep -E 'uvicorn (worker_|session_manager)' | grep -v grep) + +if [ "$stale" -ne 0 ]; then + echo + echo "Restart the stale ones before trusting a stage result. Window numbers:" + tmux list-windows -t manga-workers -F ' #{window_index} #{window_name}' 2>/dev/null \ + || echo " (tmux session manga-workers is not running; ./start_workers.sh)" +fi +exit "$stale"