diff --git a/check_stale.sh b/check_stale.sh new file mode 100755 index 0000000..95bda04 --- /dev/null +++ b/check_stale.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Which running workers are serving code older than the file on disk. +# +# Python loads a worker module once, at process start. Editing the file afterwards changes nothing +# until that process restarts, and nothing in the stage output says so: the stage completes, the +# counters read 116/116, and the result is what the OLD code produced. On 2026-08-12 this ate a full +# vision + identity + reconcile cycle — the bbox fix was committed 16 minutes after the worker that +# was supposed to apply it had already started. +# +# Run this before any stage run that is meant to prove a code change. +# Exits non-zero if anything is stale, so it can gate a script. +set -uo pipefail +cd "$(dirname "$0")" + +stale=0 +# `ps -o lstart` is always exactly five fields: "Wed Aug 12 13:20:00 2026". Read them positionally +# rather than trying to split the line with a regex. +while read -r pid _dow mon day time year cmd; do + mod=$(grep -oP '(?:worker_\w+|session_manager)(?=:app)' <<<"$cmd" | head -1) + [ -n "$mod" ] && [ -f "$mod.py" ] || continue + p_epoch=$(date -d "$mon $day $time $year" +%s 2>/dev/null) || continue + f_epoch=$(stat -c %Y "$mod.py") + if [ "$f_epoch" -gt "$p_epoch" ]; then + printf 'STALE %-22s pid %-7s edited %dm after it started\n' \ + "$mod.py" "$pid" "$(( (f_epoch - p_epoch) / 60 ))" + stale=1 + else + printf 'ok %-22s pid %s\n' "$mod.py" "$pid" + fi +done < <(ps -eo pid,lstart,cmd | grep -E 'uvicorn (worker_|session_manager)' | grep -v grep) + +if [ "$stale" -ne 0 ]; then + echo + echo "Restart the stale ones before trusting a stage result. Window numbers:" + tmux list-windows -t manga-workers -F ' #{window_index} #{window_name}' 2>/dev/null \ + || echo " (tmux session manga-workers is not running; ./start_workers.sh)" +fi +exit "$stale"