From ffeda47fd221963ce382b969cba5a62958964052 Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 03:28:26 +0400 Subject: [PATCH 01/31] Restore runtime deps: models, dots.tts, torchvision shadow Rebuild the venv half of the reconstruction. Adds the two CPU onnx detectors back under models/ (comic-text-detector, deepghs anime face, both gitignored), re-clones the dots.tts checkout, and records both recipes in requirements.txt so the next rebuild skips the archaeology. Two pre-existing environment breakages had to be cleared: - Arch's torchvision 0.25 is too old for torch 2.13, so every transformers model import died with "operator torchvision::nms does not exist". Shadowed with 0.28.0+rocm7.2 inside the venv only, so the system copy stays put. - dots_tts refuses to import when torch and torchaudio minors differ, and that pair is unsatisfiable here: 2.11 is the newest torchaudio ROCm wheel there is. Verified 2.11 loads and resamples against 2.13, then scoped a bypass around the import in both call sites. Self-checks 13/14. worker_layers still needs the ComfyUI workflow json, which legacy/ took with it. Co-Authored-By: Claude Opus 5 --- .gitignore | 1 + requirements.txt | 13 +++++++++++++ scripts/pick_tts_voice.py | 13 ++++++++++++- worker_tts.py | 12 +++++++++++- 4 files changed, 37 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 5906d58..3ecd73d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ __pycache__/ dots.tts/ /dev/shm/ *.gguf +models/ diff --git a/requirements.txt b/requirements.txt index 39b9212..6ad55fd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,3 +13,16 @@ transformers accelerate soundfile # system deps (not pip): ffmpeg, comfyui (external server) +# +# dots.tts (gitignored checkout, needed by worker_tts + scripts/pick_tts_voice). venv is built +# --system-site-packages so it inherits Arch's rocm torch: +# git clone https://github.com/rednote-hilab/dots.tts.git dots.tts +# pip install -e ./dots.tts --no-deps --ignore-requires-python # py3.14 > its <3.13 pin, skips gradio +# pip install librosa loguru einops torchdiffeq 'langcodes[data]' lingua-language-detector WeTextProcessing +# pip install --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchvision==0.28.0+rocm7.2 +# that last one is not optional: Arch's torchvision 0.25 is too old for torch 2.13, and every +# transformers model import dies with "operator torchvision::nms does not exist" until it is shadowed. +# +# models/ (gitignored, CPU onnx for set-of-mark): +# comictextdetector.pt.onnx <- github.com/zyddnys/manga-image-translator releases/beta-0.3 +# anime_face_v1.4_s.onnx <- huggingface.co/deepghs/anime_face_detection face_detect_v1.4_s/model.onnx diff --git a/scripts/pick_tts_voice.py b/scripts/pick_tts_voice.py index 5027f94..f438fa6 100644 --- a/scripts/pick_tts_voice.py +++ b/scripts/pick_tts_voice.py @@ -25,7 +25,18 @@ def load_model(model_name: str): cls, *args, **{"fix_mistral_regex": True, **kwargs} ) ) - from dots_tts.runtime import DotsTtsRuntime + # Match worker_tts.py's torch/torchaudio minor-mismatch bypass too. + import torch + import importlib.metadata as metadata + + real_version = metadata.version + metadata.version = ( + lambda name: torch.__version__ if name == "torchaudio" else real_version(name) + ) + try: + from dots_tts.runtime import DotsTtsRuntime + finally: + metadata.version = real_version return DotsTtsRuntime.from_pretrained(model_name, precision="bfloat16") diff --git a/worker_tts.py b/worker_tts.py index 50d5e0d..ad0ff32 100644 --- a/worker_tts.py +++ b/worker_tts.py @@ -36,7 +36,17 @@ def _load_tts(): transformers.AutoTokenizer.from_pretrained = classmethod( lambda cls, *a, **kw: _orig(cls, *a, **{"fix_mistral_regex": True, **kw}) ) - from dots_tts.runtime import DotsTtsRuntime + # dots_tts/__init__.py refuses to import when torch and torchaudio minors differ. workpc runs + # Arch's torch 2.13 but pytorch.org ships no torchaudio past 2.11 for ROCm, so the pair can't + # be satisfied; 2.11 loads and resamples fine against 2.13. Lie to the guard for the import. + # ponytail: drop this once a torchaudio matching torch's minor exists for ROCm. + import torch, importlib.metadata as _md + _ver = _md.version + _md.version = lambda n: torch.__version__ if n == "torchaudio" else _ver(n) + try: + from dots_tts.runtime import DotsTtsRuntime + finally: + _md.version = _ver _tts = DotsTtsRuntime.from_pretrained(DOTS_MODEL, precision="bfloat16") return _tts -- 2.52.0 From 6d9df5bf2f0451e0aab5844f1ed743c5ab3174cc Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 09:31:20 +0400 Subject: [PATCH 02/31] Audit second pass: cross-repo contract findings Read the workpc workers against the homesrv orchestrator and checked the first-pass audit against source. Report only, no code changed. Adds 4 P0, 6 P1, and 13 P2 findings to AUDIT.md, most of them in the seam between the two repos: - worker_scene reads dialogue `speaker` as a local id, but the orchestrator already rewrote it to a character_id, so all narration says "Someone" - the script verifier fails valid narration on sentence-initial capitals and on short quotes, which halts the chapter - correctness flags block TTS with no path to clear them when GATES is off - session_manager can orphan a llama-server that keeps its VRAM Confirms four first-pass claims in source: tracklet gender enum, missing action evidence, dropped verifier feedback, 409 lease stealing. Co-Authored-By: Claude Opus 5 --- AUDIT.md | 770 +++++++++++++++++++++++++++++++++++++++++++++++++++++ HANDOFF.md | 99 +++---- 2 files changed, 809 insertions(+), 60 deletions(-) create mode 100644 AUDIT.md diff --git a/AUDIT.md b/AUDIT.md new file mode 100644 index 0000000..e5a78b1 --- /dev/null +++ b/AUDIT.md @@ -0,0 +1,770 @@ +# Manga recap pipeline audit + +Date: 2026-08-11 + +Scope: workpc workers plus the homesrv orchestrator. This audit excludes FFmpeg changes and does not +recommend Magi or its embeddings. No GPU workloads were run during the audit. + +## Outcome + +The strongest performance gains are available without replacing the current models: + +1. Keep Gemma resident across compatible stages, reducing roughly seven chapter-level Gemma loads to two. +2. Batch SigLIP crop embeddings instead of running one forward pass per detected character. +3. Add bounded parallelism to CPU/network work such as page uploads, framed-page cropping, prefetching, + and scene construction. +4. Stop repeatedly downloading and base64-encoding the same panels and character references. +5. Generate schema-constrained JSON and batch text-only script work by scene. + +Correctness fixes should land before throughput work because the current tracklet and naming paths can +contaminate the permanent character registry. + +## Highest-priority findings + +### P0: tracklet gender gate uses the wrong enum + +`orchestrator/tracklets.py` checks `male|female`, while vision emits `m|f|unknown`. The tracklet hard +gender gate therefore never activates on normal pipeline data. + +### P0: tracklets lack co-presence constraints + +The linker has no same-panel exclusion and uses transitive union-find. Two similar-looking people in the +same panel can be joined, and weak links such as A~B and B~C can merge A and C even when the endpoints are +incompatible. + +### P0: face-to-character pairing is unconditional + +`worker_vision._pair_faces_to_present()` assigns every detected face to the nearest vision character as +long as an unused character exists. Despite the docstring, there is no distance or overlap threshold. +A distant or unrelated face can therefore receive another character's identity and speaker label. + +### P0: roster hints can become permanent identity evidence + +The roster is described as hints-only, but those names are fed into detection. Any name emitted on a crop +causes `worker_identity` to persist it immediately. A coarse appearance-to-roster guess can therefore +contaminate a permanent character gallery. + +### P0: script repair feedback is discarded + +The orchestrator sends `beat` and `verifier_feedback` on a failed-script retry, but `worker_script.ScriptInput` +does not define either field and the prompt builder does not consume them. The second call is another +stochastic attempt rather than a targeted correction. + +### P0: action evidence is missing from the verifier + +The beat builder reads plural `actions`, while the scene worker emits singular `action`. Script validation +therefore receives little or no action evidence and cannot reliably detect invented or omitted actions. + +### P0: GPU leases are unsafe for concurrent jobs + +When `/session/open` returns 409, the homesrv proxy assumes the active lease is stale and closes it. A second +legitimate job can terminate the first job's model. `heartbeat_session()` exists but is not used by the +pipeline. + +### P1: repeated model cold starts + +A normal chapter can load Gemma separately for: + +1. roster; +2. vision; +3. identity adjudication; +4. reconcile; +5. dialogue; +6. direction; +7. script. + +The source itself notes that each load takes tens of seconds to minutes. Removing approximately five of +these loads is likely the largest safe wall-time improvement. + +### P1: repeated transfers and encoding + +- `/direct/window` downloads every image once for grouping and again for shot design. +- Identity resolver calls repeatedly download the same candidate reference images. +- Each local image is base64-expanded into the JSON sent to the local llama-server. +- Dialogue overlap causes repeated download and CPU set-of-mark work for overlap panels. + +### P1: per-item GPU and database work + +- SigLIP runs one forward pass per character crop rather than a bounded batch. +- Only one embedding URI is loaded per known character even when its reference gallery has several views. +- Scene construction makes one homesrv-to-workpc HTTP call per panel for a cheap JSON join. +- Many stage loops repeatedly open SQLite connections and commit one row at a time. + +## Performance and scheduling plan + +### 1. Keep Gemma resident across model phases + +Use this chapter schedule: + +```text +CPU: fetch -> crop +Gemma: roster -> vision +SigLIP: batched identity embeddings and shortlists +Gemma: identity adjudication -> reconcile -> dialogue -> direct -> CPU scene join -> script +Dots: TTS +Comfy: optional layers +``` + +This needs pipeline-owned model leases instead of every stage opening and closing its own lease. Scene +construction should run locally on homesrv or as a batch while the second Gemma lease remains open. + +ComfyUI must be represented as a GPU resource too. It currently bypasses the session manager, so another +job could load Gemma, SigLIP, or Dots while ComfyUI is using the same GPU. + +### 2. Batch SigLIP inference + +For each request or chapter batch: + +1. download a panel once; +2. clamp and crop all detected characters; +3. preprocess a bounded image batch; +4. run one model forward pass; +5. compare the resulting matrix with the known gallery in one vectorized operation. + +Batch size should be tuned against VRAM rather than hard-coded optimistically. + +### 3. Add safe bounded CPU parallelism + +- Fetch/upload pages with concurrency 4-8. +- Crop framed pages with concurrency 2-4, buffer results, then insert them in page order. +- Keep whole-strip webtoon restitching sequential. +- Pre-download the next vision/dialogue window while Gemma processes the current window. +- Precompute face/text detections with a small CPU worker pool while avoiding ONNX thread oversubscription. +- Batch scene construction and SQLite reads/writes. + +GPU calls should remain serialized initially. Multiple llama-server slots divide the configured context +among slots and need an explicit VRAM/context benchmark before enabling concurrent multimodal requests. + +### 4. Eliminate repeated image movement + +Add an ephemeral `/dev/shm` LRU keyed by S3 URI plus object version/etag. It is a cache, not durable worker +state, and may be dropped at any time. + +Reuse downloaded image paths between the two direction passes and cache candidate reference images during +the identity phase. Configure llama-server with `--media-path /dev/shm` and send `file://` image paths +instead of base64 data URIs. The installed llama.cpp version supports local media paths and schema-constrained +responses. + +Reference: [llama-server documentation](https://github.com/ggml-org/llama.cpp/blob/master/tools/server/README.md). + +### 5. Use JSON-schema-constrained generation + +Provide a JSON schema for detect, dialogue, roster, direction, same-person, and identity-resolution calls. +This should eliminate most malformed-output repair calls and make missing fields explicit. Keep semantic +validation after parsing because a structurally valid answer can still be wrong. + +### 6. Reduce model call count + +- Dialogue context should contain the current scene's recent 12-20 turns, not up to 400 lines from the + whole chapter. +- Reuse direction images first, then A/B test combining grouping and shot selection into one structured call. +- Replace the per-window LLM direction summary with deterministic structured continuity where possible. +- Generate 6-12 consecutive script beats per text-only call, grouped by scene, with stable beat IDs and + targeted per-beat repair. + +## Character-recognition plan + +The critical path keeps the current detector, SigLIP shortlisting, tracklets, and Gemma resolver. + +### 1. Correct the tracklet spine + +- Canonicalize gender as `m|f|unknown` and species as a validated enum everywhere. +- Add hard same-panel/co-presence cannot-links. +- Treat manual identity assignments as immutable constraints. +- Prevent transitive bridges from joining tracks whose hard constraints disagree. +- Give every detection a chapter-local provisional track ID, even if it never becomes a global character. +- Move provisional state out of the workpc process and into a resumable homesrv chapter artifact. + +### 2. Improve visual evidence + +- Validate and clamp all VLM bounding boxes before cropping. +- Match detected faces to person/body boxes with gated IoU and normalized distance assignment rather than + unconditional nearest-neighbour matching. +- Store both face and body crops. +- Score crop quality using face size, detector confidence, occlusion, and sharpness. +- Send Gemma the best 2-4 query views for a tracklet. The current resolver sends only one representative + query crop; its candidate union is not a real multi-view query gallery. +- Store multiple embedding vectors per global character and use gallery maximum or centroid similarity + only for shortlisting. + +Face/body evidence plus local temporal constraints are supported by dedicated comic and manga re-ID work: +[Identity-Aware Semi-Supervised Learning for Comic Character Re-Identification](https://arxiv.org/abs/2308.09096) +and [Unsupervised Manga Character Re-identification via Face-body and Spatial-temporal Associated Clustering](https://arxiv.org/abs/2204.04621). + +### 3. Separate naming from identity + +- Keep roster names as candidate vocabulary, never identity evidence. +- Detection may emit a typed name claim only when the panel visibly binds the name to that character. +- Require self-introduction, name-tag, or corroborated address evidence before promotion. +- Preserve aliases and honorific forms instead of stripping and discarding them. +- Never merge galleries on name equality alone. +- Resolve the tracklet first, then attach corroborated names to its global identity. + +### 4. Add global constraints + +Identity resolution should enforce: + +- must-link membership within a validated tracklet; +- cannot-link for co-present characters; +- gender and species hard gates; +- manual locks; +- at most one appearance of a global character per panel; +- open-set outcomes `known|new|unresolved` without deleting the chapter-local track ID. + +Gemma should receive a compact labelled identity board containing several query views and selected candidate +gallery views, rather than up to fifteen separately downloaded image parts for every representative crop. + +### 5. Improve identity evaluation + +Extend the existing labels and evaluator to report: + +- false merges versus false splits; +- tracklet purity and fragmentation; +- global identity accuracy; +- name-binding accuracy; +- unknown/NONE rate; +- speaker accuracy conditioned on identity correctness. + +A replacement embedding backbone should only be considered if the structural fixes plateau, using the +same labeled chapters for a controlled A/B comparison. + +## Story-coherence plan + +### 1. Build an evidence ledger before prose + +Create a deterministic artifact for every beat: + +```text +beat + -> ordered panels + -> ordered actions and chronology + -> dialogue/region IDs and exact text + -> typed speaker references, confidence, and method + -> characters present + -> location and time + -> named entities + -> uncertainty flags +``` + +This becomes the only factual input to script generation and validation. + +### 2. Make context scene-scoped and evidence-bearing + +- Reset speaker-turn context on strong location/time/scene changes. +- Keep a short structured story state rather than repeatedly summarizing generated prose. +- Give a script chunk only facts that occur before or inside that chunk. +- Do not pass a whole-chapter premise as if it were already-known story state; it can leak future events. +- Keep stable chapter-local handles for unresolved recurring characters so they do not disappear from the + scene graph while awaiting global promotion. + +### 3. Generate narration in ordered scene chunks + +Generate several consecutive beats in one text-only call and return: + +```json +{"beats":[{"beat_id":"b01","text":"...","evidence_ids":["p001_r1"]}]} +``` + +This provides more continuity and fewer calls than one independent request per beat while keeping output +small enough for targeted validation and repair. + +### 4. Respect uncertainty + +Low-confidence or weak-method speaker assignments should become `someone`, an unattributed quote, or a +review flag instead of a confident named assertion. Identity and speaker uncertainty must survive into +the script prompt. + +### 5. Strengthen verification + +Validate: + +- exact/fuzzy quote grounding; +- permitted character and entity names; +- action coverage and unsupported actions; +- event chronology; +- future-fact leakage; +- repeated facts and repeated sentence openings; +- evidence IDs returned by the model. + +Feed the exact failures into the repair prompt. A final text-only consistency audit may report affected +beat IDs, but should not rewrite the whole chapter automatically. + +## Stage clearing and resumability + +Before parallel or identity work, stage ownership of shared data needs to be explicit: + +- dialogue and direction currently mutate the shared vision JSON; +- clearing dialogue does not remove its keys, so a rerun can treat old dialogue as completed; +- reconcile merges are destructive and are not reversed by clearing the reconcile stage; +- clearing identity preserves the per-manga registry, so it is not a clean identity rerun; +- name claims attached to a merged-away character are not repointed with the identity assignments. + +Use stage-specific artifacts or explicit key deletion, and preserve provenance through merges. A stage must +be genuinely idempotent before it is scheduled concurrently or resumed automatically. + +## Further design ideas + +### 1. Make identity merges reversible + +Reconciliation currently deletes the losing character. Replace destructive merging with a versioned +cluster or redirect model: + +```text +character_cluster + canonical_id + member_ids[] + merge_evidence[] + cannot_link[] + version +``` + +Assignments resolve through the canonical cluster while the original identities remain recoverable. A +reviewer can remove one bad merge edge without clearing and rebuilding the entire identity stage. + +Preserve negative evidence as a first-class artifact. Co-presence is particularly valuable: if A and B +appear together as distinct detections, they are a permanent cannot-link unless a reviewer overrides it. + +### 2. Separate detection, identity, and naming confidence + +These are different questions and should never share one confidence value: + +- detection confidence: is this actually a story character? +- identity confidence: which recurring visual identity is it? +- naming confidence: what is that identity's canonical name? + +A person may be confidently detected and tracked as `chapter_person_3` while remaining unnamed. That is +enough for coherent narration such as "the colleague" without contaminating the global registry. + +### 3. Use an uncertainty-driven identity cascade + +Spend computation according to ambiguity and downstream impact: + +```text +hard constraints resolve the case + -> accept cheaply + +strong tracklet plus clear gallery margin + -> accept without Gemma + +ambiguous candidates + -> multi-view Gemma resolution + +high-impact unresolved identity + -> human review + +one-off background person + -> chapter-local handle only +``` + +High-impact detections are recurring, speaking, named, or referenced by several future beats. A one-panel +extra should not consume the same identity budget as a protagonist. + +### 4. Add model-aware scheduling across jobs + +Strict chapter-by-chapter execution causes extra model swaps when several jobs are queued. A resource-aware +scheduler can group ready work by resident model: + +```text +ready Gemma detect phases +-> ready SigLIP phases +-> ready Gemma post-identity phases +-> ready Dots phases +``` + +This improves total throughput at the cost of some per-job latency. Add fairness and maximum-wait limits so +one long title cannot starve other jobs. GPU conflicts must wait in a queue; they must never terminate the +active owner's lease. + +### 5. Cache artifacts by dependency fingerprint + +Replace broad "clear stage X and everything downstream" behavior with content-addressed artifacts keyed by: + +```text +hash(input URIs + + relevant upstream JSON + + model version + + prompt/schema version + + stage settings) +``` + +Consequences: + +- changing narration style does not rerun vision; +- correcting one identity invalidates only affected scenes and beats; +- changing direction does not invalidate dialogue; +- a resume reuses every artifact whose dependencies still match; +- prompt experiments can run side-by-side instead of overwriting the baseline. + +Store dependency and version metadata beside every stage artifact so invalidation is explainable in the +review UI. + +### 6. Detect repeated artwork before expensive stages + +Use CPU-side perceptual hashing after masking detected text regions: + +- same artwork and same text: reuse visual and dialogue artifacts; +- same artwork but different text: reuse detection/identity, rerun dialogue; +- near-duplicate establishing shot: mark `visual_repeat_of` so narration does not describe it again; +- exact duplicate with no new information: curate it out early. + +Text masking is required because two visually identical panels may contain different dialogue. Combine the +art hash with text-region geometry and a text-content hash when available. + +### 7. Make the manga registry temporal + +Names, aliases, relationships, and facts need an evidence position: + +```text +known_from: chapter 12, panel 37 +source: self_intro +confidence: 0.97 +``` + +The script can then use only knowledge available at that point. This prevents later chapters or out-of-order +processing from leaking names and revelations into earlier recaps. A configurable `spoiler_safe` versus +`omniscient_recap` policy can decide whether later knowledge may backfill earlier narration. + +The sampled chapter premise needs the same guard because a synopsis built from the whole chapter can reveal +its payoff in the opening beat. + +### 8. Represent dialogue as a relation graph + +A spoken name often identifies the addressee or a third party rather than the speaker. Store: + +```text +speaker_ref +addressed_character_ref +mentioned_character_refs[] +spoken_text +name_surface_form +honorific +region_id +speaker_evidence +``` + +For example, "Seonho-oppa!" is evidence about the addressee and their relationship, not the speaker's name. +Honorifics should be normalized but preserved as evidence instead of being stripped and discarded. + +### 9. Treat speaker attribution as constrained assignment + +For every speech region, build candidates from: + +- bubble-tail or geometric evidence; +- visible faces and bodies; +- previous and following speaker turns; +- chapter-local identities; +- the off-panel cast; +- same-panel cannot-links. + +Assign speakers jointly across a conversation window rather than deciding every line independently. Use +Gemma only to adjudicate ambiguous regions. Require `region_id` in the model output so numbered set-of-mark +regions are structurally connected to returned dialogue instead of remaining visual suggestions. + +### 10. Separate stable identity traits from scene appearance + +Store durable and temporary appearance separately: + +```text +identity_appearance: + face, hair, species, stable marks + +scene_appearance: + outfit, accessories, injuries, disguise, chibi/age form +``` + +Clothing can be strong evidence inside one local scene but should be down-weighted across scene or time +changes. This also lets narration mention an outfit change without creating a new global character. + +### 11. Add metamorphic pipeline tests + +Test invariants rather than only fixed outputs: + +- changing dialogue window size must not move text to another panel; +- inserting a scenery panel must not change neighboring identities; +- resuming halfway through identity must match a clean run; +- renumbering panel-local IDs must not change global identity; +- distinct co-present characters must never merge; +- changing narration style must not change facts or quotes; +- processing chapters in a different order must not silently rewrite prior identities. + +These tests exercise cross-stage contracts and resumability failures that per-file self-checks cannot catch. + +### 12. Roll out identity changes in shadow mode + +Write new identity results to a versioned shadow artifact without affecting scripts or the canonical registry. +The review UI can compare: + +```text +current: Choi Haeseon +candidate: unresolved +reason: co-presence conflict +``` + +Promote a new resolver only after it improves the labeled-chapter metrics. Shadow mode prevents tuning work +from polluting the production gallery. + +The highest-value additions beyond the core plan are reversible identity clusters with cannot-links, +dependency-fingerprinted artifacts, and uncertainty-driven review for recurring/speaking/named characters. + +## Implementation sequence for approval + +### Phase 1: correctness and scheduling safety + +- Fix enum mismatches and add tracklet cannot-links. +- Gate face/body pairing. +- Wire action evidence and verifier feedback. +- Make stage clearing/resume behavior honest. +- Replace 409 lease stealing with a queue/wait policy. +- Run actual lease heartbeats. +- Register ComfyUI under the same GPU resource scheduler. + +Verification: CPU-only unit/self-checks. No full pipeline or GPU run until explicitly scheduled. + +### Phase 2: safe throughput improvements + +- Introduce the two-phase Gemma lease schedule. +- Add JSON-schema outputs. +- Add local media paths and the ephemeral file/reference cache. +- Reuse direction downloads. +- Batch SigLIP embeddings. +- Batch scene/DB operations. +- Add bounded fetch and framed-page crop concurrency. + +Verification: unit checks first, followed by one labeled chapter after GPU availability is approved. + +### Phase 3: multi-view constrained identity + +- Persist chapter-local tracklets. +- Add face/body galleries and crop-quality selection. +- Resolve with multi-view query evidence and global constraints. +- Separate name claims from visual identity. +- Expand identity evaluation. + +### Phase 4: evidence-ledger narration + +- Build ordered beat evidence artifacts. +- Add scene-scoped story and turn-taking state. +- Generate script chunks with evidence IDs. +- Add confidence-aware wording and targeted verification/repair. + +## Acceptance criteria + +Capture a baseline and compare the same labeled chapter after every phase: + +- stage wall time and model-load time; +- number of Gemma calls and repair calls; +- MinIO bytes downloaded/uploaded; +- prompt-evaluation and generation timings; +- tracklet false merges/splits and identity accuracy; +- name and speaker accuracy; +- unresolved correctness flags; +- script repetition, unsupported facts, and verifier retry rate. + +Recommended approval boundary: implement Phases 1 and 2 first, measure them, then decide whether to proceed +with the larger identity and narration changes. + +## Second-pass findings (2026-08-11) + +Added after reading the workers and the orchestrator against each other. Every item was read in source and +is cited by file and line. Nothing was executed and no GPU work was run. + +The first pass audited each side on its own terms. Most of what follows lives in the seam between them: +one repo changed a field's meaning and the other still reads the old one. The existing per-file +`__main__` self-checks cannot catch any of it, because each one asserts its own side of the contract. + +### P0: the scene stage discards every speaker + +`normalize_dialogue` rewrites `row["speaker"]` to a `character_id` or `None` +(`orchestrator/correctness.py:47`). `run_stage_dialogue` saves that shape into the vision blob +(`orchestrator/service.py:1159-1160`). `worker_scene.build_scene` still reads that field as a panel-local id +and maps it through `id_by_local` (`worker_scene.py:78`). A `character_id` is never a key in that map, so +every lookup returns `None`. + +Every scene graph therefore reaches the script worker with `speaker: None`, and `_render_line` writes +`Someone says "..."` for all of it (`worker_script.py:48`). Set-of-mark face attribution, the solo-speaker +backstop, and window turn-taking all compute the right answer and then lose it one stage later. + +The same field breaks the dialogue resume path: `service.py:1180` pushes the stored value into `recent` as a +speaker NAME, so a resumed run feeds raw strings like `character_ab12cd34` to the transcription prompt as if +they were people. + +`worker_scene`'s self-check still passes because it feeds the pre-change contract +(`worker_scene.py:138`). Fix: prefer `speaker_ref` when its kind is `character_id`, keep the local-id path as +the fallback. This is the strongest argument for the cross-stage metamorphic tests proposed above. + +### P0: the script verifier rejects correct narration, and its retry is a no-op + +`verify_script` fails a beat on two rules that fire on valid output (`orchestrator/correctness.py:100-116`). + +1. `unsupported-proper-noun` flags any capitalized token outside the cast, the entities, the beat's source + words, and a 20-word stop list. Ordinary sentence-initial words are not in that list: `Suddenly`, + `Behind`, `Inside`, `Both`, `Everyone`, `After`, `Two`. So is `Someone`, which the previous finding + guarantees the narrator will emit constantly. +2. `misquote` compares each quoted span against a WHOLE source line with `SequenceMatcher` at 0.82. The + narration prompt explicitly asks for a short quote of the actual words (`worker_script.py:130-132`). A + 15-character quote taken from a 40-character line scores about 0.55 and fails. The prompt and the + verifier ask for opposite things. + +The failure is not soft. `run_stage_script` retries once, then raises, so the beat produces no narration +(`service.py:1384-1386`). `done` never reaches `total`, `_finalize_stage` marks the stage failed, and +`_run_pipeline` stops the whole chapter. One false positive halts a run. + +The retry is also inert for the reason already recorded above: `ScriptInput` defines neither `beat` nor +`verifier_feedback` (`worker_script.py:13-23`), and pydantic v2 ignores unknown fields by default, so both +are dropped without an error. + +Fix order: correct the two rules first, then wire the feedback fields. Fixing the plumbing alone makes the +model retry against a broken oracle. + +### P0: correctness flags block TTS permanently in the default configuration + +`run_stage_tts` refuses to start while any unresolved flag of kind `script-verifier`, `ambiguous-speaker`, +`conflicting-name-claims`, or `partial-dialogue` exists (`service.py:1419-1423`). The only code that ever +resolves a flag is `resolve_correctness_flags`, called from `/review/approve` for the `script` gate +(`service.py:1955-1956`). + +`GATES` defaults to off (`service.py:312`). The autonomous pipeline therefore has no path that clears a +flag. The first flag of those kinds wedges the chapter until a human calls an endpoint that the autonomous +mode never mentions. These flags are not rare: `partial-dialogue` is raised whenever a window response omits +one panel, and `dialogue_envelope` treats a legitimately silent panel inside a partial window as unresolved +(`correctness.py:62`), so one missing panel flags every wordless panel beside it. + +There is a second, worse variant. Flag rows survive their panels. `get_correctness_flags` LEFT JOINs panels +and accepts rows where `p.panel_id IS NULL` (`db.py:573`), while `resolve_correctness_flags` can only resolve +flags whose panel still exists (`db.py:579`). Clearing the `crop` stage deletes panel rows +(`db.py:798-799`). Any flag raised against a deleted panel becomes visible to every chapter and can never be +resolved, blocking TTS for all future jobs. + +Fix: resolve by flag identity rather than by surviving panel, and give the autonomous path an explicit +policy (auto-resolve below a rank, or fail the stage loudly) instead of an unreachable gate. + +### P0: `awaiting_review` is immediately overwritten by `failed` + +`run_stage_tts` sets `awaiting_review` and then raises (`service.py:1422-1423`). The raise unwinds into +`_run_pipeline`'s catch-all, which sets the job to `failed` (`service.py:391-393`). The review state the +stage just recorded is gone before anyone can read it, and the operator sees a generic failure rather than a +queue of flags. The gate path at `service.py:337-341` returns instead of raising and does not have this +problem. + +### P1: set-of-mark face pairing is enabled by default, contrary to its own documentation + +`worker_vision.py:22` states the feature is off by default and should be enabled per title once tuned. +`worker_vision.py:31` reads `SOM_ATTRIBUTION` with a default of `"1"`. It is on. + +That matters because of the unconditional pairing already recorded as a P0. The mispairing does not stay +local. `_set_of_mark` writes the wrong name into the legend the model reads (`worker_vision.py:99-104`), the +model attributes a line to that face label, and `_apply_speaker_labels` converts the label back into a +`local_id` and stamps `speaker_method = "som_face"` (`worker_vision.py:117`). A geometric guess is laundered +into the highest-trust provenance value the system has. Gate the pairing before trusting that label, or +default the flag to off as documented. + +The pairing is also greedy in face order rather than a joint assignment (`worker_vision.py:42-52`), so the +first face processed can claim a character that fits a later face far better. + +### P1: the JSON repair pass can fabricate content + +When a response fails to parse, `call_gemma4_json` sends the model its own truncated text and asks for the +JSON it should have been (`worker_vision.py:174-178`). The repair call carries no image. On a response +truncated by `max_tokens`, the model is being asked to complete dialogue it can no longer see. Anything it +adds is invented and is indistinguishable downstream from transcribed text. + +Schema-constrained generation, already recommended above, removes most of this path. Until then the repair +pass should re-send the image, or a truncated response should be retried rather than repaired. + +### P1: `/review/preview` silently replaces a beat clip + +`review_preview` renders one panel alone and calls `save_clip(panel_id, ...)` (`service.py:1902-1904`). Its +docstring calls this harmless because assemble would regenerate it. It does not. `_render_one_beat` returns +early when a clip already exists for the leader (`service.py:1591-1592`). Previewing a beat leader therefore +pins the solo preview into the final video and drops the rest of the beat's panels. `review_retts` gets this +right by calling `delete_clip` first (`service.py:2008`). Preview should write to a scratch key or delete the +clip row afterwards. + +### P1: the session manager can orphan a llama-server and hold the GPU + +`open_session` claims the slot, releases the lock, then spawns and health-waits for up to 300 seconds +(`session_manager.py:108-119`). A `/session/close` arriving during that window finds `proc = None`, tears +down nothing, and clears `_active`. The process that finishes starting afterwards is unreferenced and keeps +its VRAM until someone kills it by hand. The next `open` spawns a second server on the same port, and +`_health_wait` cannot tell the two apart because it only probes the port +(`session_manager.py:51-60`). + +Related asymmetry: `_supervise_once` calls `_start_subprocess` while holding `_lock` +(`session_manager.py:169-181`), so a respawn blocks `/session/active`, `/session/close`, and `/session/open` +for the full health wait. The open path was deliberately written to avoid exactly this. + +### P1: nothing limits concurrent jobs + +`_background_jobs` accepts any number of pipelines (`service.py:302-303`, `service.py:427-428`). Each opens +its own model sessions. Combined with the 409 lease stealing already recorded, two jobs terminate each +other's models rather than queueing. The queue has to live at the orchestrator as job admission control. A +fix inside `session_proxy` alone still lets two pipelines interleave stages against one GPU. + +### P1: two cheap throughput wins the plan does not name + +**Order the shared prefix first so llama-server reuses its KV cache.** `/direct/window` sends the same +panel images twice, once for grouping and once for shot design (`worker_vision.py:702-733`). The audit +above frames this as duplicate downloads. The larger cost is the second vision-encoder prefill on the GPU. +llama-server reuses the longest common prompt prefix across requests, and the two calls currently differ in +their first token because the instruction text precedes the images. Putting the images first and the +differing instruction last makes the second pass nearly free on prefill. The same reordering helps any +stage that issues several calls over one image set. + +**Give stored embeddings a version tag.** `embed_crop` uses `pooler_output` when present and silently falls +back to mean-pooled patch tokens otherwise (`worker_identity.py:48-52`). Those two paths produce different +vector spaces, and the fixed 0.85 threshold is only valid for one of them. Nothing recorded beside a stored +`.npy` says which model, revision, or pooling produced it, so a transformers upgrade mixes incompatible +vectors into one gallery with no error. Write the model id and pooling mode next to the vector and refuse to +compare across versions. + +### P2: smaller confirmed defects + +- An out-of-range `choice` from the resolver is mapped to NONE and then reported as `state: "new"` + (`worker_vision.py:898-899`), so a hallucinated index mints a brand new character. It should be + `unresolved`, like a parse failure. +- `_extract_json` matches greedily from the first `{` to the last `}` (`worker_vision.py:130`). Two objects + or any trailing braced prose produce an unparseable span and burn a repair call. `json.JSONDecoder().raw_decode` + from the first brace is exact. +- `worker_identity._known_cache` is invalidated only by `_persist_char` (`worker_identity.py:128-132`). + The reconcile stage deletes losing characters directly in the orchestrator database + (`db.py:506`), so a long-lived worker keeps shortlisting and assigning ids that no longer exist. +- `_pending` holds full crop images in worker memory for a whole chapter and is keyed by session + (`worker_identity.py:25`). This is durable per-chapter state inside a worker documented as stateless, it + is lost on restart, and a character seen exactly once receives no assignment at all, not even a + chapter-local handle. +- `get_conn` opens a connection per call with no `busy_timeout` (`db.py:190-197`). WAL tolerates one writer. + `PIPELINE=1` already writes clips from concurrent tasks while TTS writes audio, so the planned CPU + parallelism will surface as `database is locked` before it surfaces as throughput. +- Worker endpoints declared `async def` run blocking MinIO, OpenCV, torch, and ffmpeg calls directly on the + event loop (`worker_identity.py:177`, `worker_tts.py:170`, `worker_vision.py:230`). A busy worker cannot + answer `/health` or `/unload`. That makes `/health/workers` report a working worker as unreachable + (`service.py:206-216`) and puts the session manager's 30-second `/unload` at risk exactly when VRAM needs + freeing (`session_manager.py:93`). `def` instead of `async def` moves each to the threadpool. +- `layers` runs after `tts` in `STAGES` (`db.py:330-333`), while `run_stage_tts` warns that eager rendering + under `PIPELINE=1` needs layers to run first (`service.py:1427-1429`). With the flag on, solo beats always + render without parallax. +- Stage failure policy is inconsistent. A dropped vision panel fails the stage and halts the pipeline. A + failed direction window counts its panels as done (`service.py:1247`), and layers always finishes + completed (`service.py:1503`). `completed` does not mean the same thing across stages, which makes the + acceptance metrics below hard to read. +- `run_stage_assemble` does not check that `clip_uris` is non-empty before assembling and then marks the job + completed (`service.py:1661-1677`). +- The `/review/panels` timeline sums per-panel audio durations (`service.py:1750`), but assemble crossfades + clips using the per-beat transitions (`service.py:1671-1674`). Every non-`cut` transition shortens the + real video, so reviewer timestamps drift further out of sync the further into the chapter they scrub. +- MinIO credentials are hardcoded as defaults in committed source (`transport.py:95-99`, + `service.py:74-76`). + +### What this changes in the plan + +Add to Phase 1, before any throughput work: + +1. Restore the speaker field across the dialogue, scene, and script boundary. +2. Correct the two verifier rules, then wire `beat` and `verifier_feedback`. +3. Give correctness flags a resolution path that does not require a disabled gate, and stop the TTS block + from erasing `awaiting_review`. +4. Decide the set-of-mark default deliberately, and gate face pairing before the label is trusted. +5. Add job admission control at the orchestrator, not only lease queueing in the proxy. + +Add to the acceptance criteria: the share of narrated lines whose speaker is a named character rather than +`Someone`. That single number would have caught the first finding on the day it landed. diff --git a/HANDOFF.md b/HANDOFF.md index f9ece08..e44c67f 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,75 +1,54 @@ -# HANDOFF: repo reconstructed from agent transcripts (2026-08-11) +# HANDOFF: audit second pass (2026-08-11) ## What was asked -The working copy at `/home/kami/Programs/n8n-worker` was deleted with `rm`, including `.git`. -This repo is the workpc GPU-compute half of the manga pipeline. The ask was to rebuild it from -the transcripts on disk, at minimum the transport layer and the pipeline flow. +Review `AUDIT.md` (untracked, first pass, 565 lines) and add anything missing. -## Sources used -No backup and no remote survived. `~/.local/share/Trash` is empty. No copy exists under -`/home/kami`, `/mnt/D`, or `/mnt/server`. Gitea -(`/mnt/server/mnt/hdd2/gitea/git/repositories/`) holds only correx, hexis, maven, -model-training, muzick, orchestra, and test-e2e. There is no `n8n-worker` mirror. +## What was done +Read both halves of the pipeline and checked the first pass against source. No code changed. +No GPU work, no pipeline run, no tests executed. -Reconstruction replays three histories into one timeline, ordered by timestamp: +Files read in full: +- workpc: `worker_vision.py` (1008), `worker_identity.py` (322), `worker_script.py` (300), + `worker_scene.py` (170), `worker_tts.py` (241), `session_manager.py` (238), `transport.py` (216), + `worker_crop.py` (first 120 of 359). +- homesrv `/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`: `service.py` (2048), + `db.py` (815), `session_proxy.py` (302), `correctness.py` (116), `tracklets.py` (121), + `minio_layout.py` (105), `grouping.py` (75). -1. Claude Code transcripts, 25 sessions in - `~/.claude/projects/-home-kami-Programs-n8n-worker/*.jsonl`. Contributes Write and Edit - content, full-file Read results, and `@`-mention attachments. -2. Codex rollouts, 5 sessions with `cwd=/home/kami/Programs/n8n-worker` under - `~/.codex/sessions/2026/07/{13,15,16,18}/rollout-*.jsonl`. Contributes 26 `apply_patch` - blocks. The 4 that the log marks `Script failed` are skipped. These carry the 2026-07-18 - correctness work (`_dialogue_envelope`, `_normalize_claims`, `_annotate_speaker_methods`, - `SOM_ATTRIBUTION=1`) that exists in no Claude transcript. -3. `~/.claude/file-history//@vN` pre-edit blobs, used as cross-checks only. +Result: `AUDIT.md` grew from 565 to 745 lines. New section `## Second-pass findings (2026-08-11)` +at line 566. 4 new P0, 6 new P1, 13 P2, plus 5 additions to the Phase 1 list and 1 to the +acceptance criteria. -Scripts live in -`/tmp/claude-1000/-home-kami-Programs-n8n-worker/6a35d5d8-9f44-4412-955c-9cf088737ad5/scratchpad/`. -`replay2.py` is the merged replay. `codex.py` extracts and applies codex patches. `backups.py` -compares against file-history. `recon_final/_log.json` holds the per-file event log. +Four first-pass claims were confirmed in source and are not restated in the new section: -## Verification -- Replayed to 2026-07-18T13:13:44Z, line counts match the `wc -l` recorded in the transcript at - that moment, for all 10 files it covered. crop 326, identity 314, layers 99, render 996, - scene 159, script 300, tts 231, vision 871, session_manager 238, transport 216. -- Byte sizes match the `ls -l` recorded by the same command. AGENTS.md 867, CLAUDE.md 3678, - manga-recap-pipeline-spec.md 11413, plan.md 16511, spec-v3.md 26170. `.gitignore` is 53. -- 18 files are byte-identical to their newest file-history blob. The 4 that differ (render, - script, tts, spec-v3) differ only by edits made after that blob was taken. -- `python3 -m py_compile *.py scripts/*.py attic/*.py` passes. `bash -n` passes on all 3 shell - scripts. Runtime self-checks such as `python worker_vision.py` were NOT run, because `.venv/` - is gone. +- tracklet gender enum: `tracklets.py:43` uses `male|female`, vision emits `m|f|unknown`. +- missing action evidence: `worker_scene.py:85` emits `action`, `correctness.py:91` reads `actions`. +- dropped verifier feedback: `worker_script.py:13-23` defines neither field. +- 409 lease stealing: `session_proxy.py:40-49`. -## Recovered (live tree) -Workers `worker_{crop,vision,identity,scene,script,tts,layers,render}.py`, the GPU mutex -`session_manager.py`, `transport.py`, helpers `bubble_detect.py`, `face_detect.py`, -`collage.py`, `test_vision_parse.py`, `start_workers.sh`, `systemd/install.sh`, -`scripts/{analyze_video_frames.sh,pick_tts_voice.py}`, `requirements.txt`, `.gitignore`, -and the docs `AGENTS.md`, `CLAUDE.md`, `plan.md`, `spec-v3.md`, `spec-correctness.md`, -`manga-recap-pipeline-spec.md`, `identity-rework-task.md`. - -`attic/` holds files that had been deleted from the tree before the `rm`. They are kept but not -live: `worker_ocr.py`, `worker_parse.py`, `plan-workpc.md`, `char-recognition.md`. +## Top 4 new findings +1. `worker_scene.py:78` reads `speaker` as a local id, but `correctness.py:47` already rewrote it + to a character_id. Every lookup returns None, so all narration says `Someone`. +2. `correctness.py:106-115` fails valid narration on sentence-initial capitals and on short + quotes, and `service.py:1384-1386` turns that into a halted chapter. +3. `service.py:1419-1423` blocks TTS on correctness flags that only `/review/approve` can clear, + while `GATES` defaults to off (`service.py:312`). +4. `session_manager.py:108-119` can orphan a llama-server that keeps its VRAM. ## Still open -- `.venv/` is gone. Rebuild with - `python -m venv .venv && .venv/bin/pip install -r requirements.txt`. The ROCm torch wheel is - not pinned in requirements.txt, so install it the way workpc had it. -- `dots.tts/` (gitignored external checkout) and `legacy/` (pre-rewrite workers, moved there - 2026-07-13) are not recoverable from transcripts. Re-clone dots.tts if TTS is needed. -- `RESUME_SPEC.md`, `pipeline-design-notes.md`, and `spec-v2.md` are unrecoverable. Neither agent - ever read them in full. All three were already deleted from the working tree as of 2026-07-17. -- Git history is gone. A fresh `git init` plus one commit replaces it. Old commits are not - recoverable. -- Two replay gaps were left unpatched. One Edit MISS in `worker_render.py` on 2026-07-14, and one - orphan Edit against `attic/plan-workpc.md` on 2026-07-13. Both files were later replaced by a - full-file snapshot, so the final content stays anchored. -- Edits made outside Claude and codex after 2026-07-18T14:35Z (the last recorded write) cannot be - detected. `unknown:` whether any exist. +- Nothing from this session is half-finished. `AUDIT.md` is complete as written. +- No finding has been fixed. All are report-only. +- `AUDIT.md` is still untracked as of this session's start. Decide whether it belongs in git. +- Prose linter reports ~97 style hits in `AUDIT.md`. Almost all are in the first-pass sections + (lines 1-565), which were left as the author wrote them. +- Carried over from the earlier reconstruction handoff of the same day: + - `.venv/` is gone. Rebuild it, then install the ROCm torch wheel the way workpc had it. + - `dots.tts/` and `legacy/` are not recoverable from transcripts. + - `RESUME_SPEC.md`, `pipeline-design-notes.md`, and `spec-v2.md` are unrecoverable. ## Next command ``` cd /home/kami/Programs/n8n-worker -python -m venv .venv && .venv/bin/pip install -r requirements.txt -.venv/bin/python worker_vision.py # per-file __main__ self-checks, then ./start_workers.sh +sed -n '566,745p' AUDIT.md # read the new section ``` +Then pick the Phase 1 order at `AUDIT.md:511` as amended at the end of the new section. -- 2.52.0 From 0cc630224583453b01eab0d4b43389acec69697f Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 10:16:24 +0400 Subject: [PATCH 03/31] Audit Phase 1: correctness and scheduling safety Implements every P0 from AUDIT.md plus four P1s, across both halves of the pipeline. Verified by CPU-only self-checks and the orchestrator test suite. No GPU work ran and no pipeline ran. workpc: - worker_scene: read speaker_ref, not the rewritten speaker field. Every line narrated as "Someone" before this. Emit `actions` for the verifier. - worker_script: declare beat + verifier_feedback (pydantic dropped both, so the retry was blind) and render them as a repair prompt. - worker_vision: gate face->identity pairing on containment, assign globally shortest-first, map an out-of-range resolver index to `unresolved` instead of minting a character, parse JSON with raw_decode. - session_manager: tear down a server whose lease vanished mid-load, and spawn the supervisor respawn unlocked. orchestrator (edited in place, NOT committed there): - tracklets: canonicalize gender, add co-presence cannot-links, block transitive bridges across a hard constraint. - correctness: stop failing valid narration on sentence-initial capitals and short quotes; read action evidence from the singular key. - db: stop orphan flags leaking into every chapter; resolve by flag id. - service: TTS returns instead of raising under GATES, auto-resolves under autonomous mode; job admission control; registry names on dialogue resume. - session_proxy: queue on 409 instead of stealing the lease; run heartbeats. Docs restructured per the repo-structure layout: CLAUDE.md is a pointer table, NEXT.md replaces HANDOFF.md, plus ROADMAP.md, JOURNAL.md, decisions/ and caveats/. AUDIT.md now points at those instead of restating them. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01XD7cAy81MZrc7gCr6aZGWr --- AUDIT.md | 113 +++++++++++------------------ CLAUDE.md | 111 ++++++++++++++-------------- HANDOFF.md | 54 -------------- JOURNAL.md | 22 ++++++ NEXT.md | 42 +++++++++++ ROADMAP.md | 66 +++++++++++++++++ caveats/CLAUDE.md | 34 +++++++++ caveats/audit-open.md | 149 ++++++++++++++++++++++++++++++++++++++ decisions/CLAUDE.md | 29 ++++++++ decisions/audit-phase1.md | 144 ++++++++++++++++++++++++++++++++++++ session_manager.py | 58 ++++++++++++--- worker_scene.py | 35 ++++++++- worker_script.py | 54 ++++++++++++-- worker_vision.py | 95 ++++++++++++++++++------ 14 files changed, 787 insertions(+), 219 deletions(-) delete mode 100644 HANDOFF.md create mode 100644 JOURNAL.md create mode 100644 NEXT.md create mode 100644 ROADMAP.md create mode 100644 caveats/CLAUDE.md create mode 100644 caveats/audit-open.md create mode 100644 decisions/CLAUDE.md create mode 100644 decisions/audit-phase1.md diff --git a/AUDIT.md b/AUDIT.md index e5a78b1..dd62ad8 100644 --- a/AUDIT.md +++ b/AUDIT.md @@ -5,6 +5,19 @@ Date: 2026-08-11 Scope: workpc workers plus the homesrv orchestrator. This audit excludes FFmpeg changes and does not recommend Magi or its embeddings. No GPU workloads were run during the audit. +## Status + +Phase 1 landed on 2026-08-11. This file is the reasoning, not the state. + +| where the finding went | file | +| --- | --- | +| implemented, with evidence and a runnable check | `decisions/audit-phase1.md` | +| deliberately not done, with a revisit trigger | `caveats/audit-open.md` | +| still ahead, in order | `ROADMAP.md` | +| what to do next | `NEXT.md` | + +Every P0 below is closed. The sections that name one now point at its decision instead of repeating it. + ## Outcome The strongest performance gains are available without replacing the current models: @@ -21,18 +34,18 @@ contaminate the permanent character registry. ## Highest-priority findings -### P0: tracklet gender gate uses the wrong enum +### P0: tracklet gender gate uses the wrong enum — CLOSED, decisions/audit-phase1.md#tracklet-hard-constraints `orchestrator/tracklets.py` checks `male|female`, while vision emits `m|f|unknown`. The tracklet hard gender gate therefore never activates on normal pipeline data. -### P0: tracklets lack co-presence constraints +### P0: tracklets lack co-presence constraints — CLOSED, decisions/audit-phase1.md#tracklet-hard-constraints The linker has no same-panel exclusion and uses transitive union-find. Two similar-looking people in the same panel can be joined, and weak links such as A~B and B~C can merge A and C even when the endpoints are incompatible. -### P0: face-to-character pairing is unconditional +### P0: face-to-character pairing is unconditional — CLOSED, decisions/audit-phase1.md#gated-face-pairing `worker_vision._pair_faces_to_present()` assigns every detected face to the nearest vision character as long as an unused character exists. Despite the docstring, there is no distance or overlap threshold. @@ -44,18 +57,18 @@ The roster is described as hints-only, but those names are fed into detection. A causes `worker_identity` to persist it immediately. A coarse appearance-to-roster guess can therefore contaminate a permanent character gallery. -### P0: script repair feedback is discarded +### P0: script repair feedback is discarded — CLOSED, decisions/audit-phase1.md#verifier-false-positives The orchestrator sends `beat` and `verifier_feedback` on a failed-script retry, but `worker_script.ScriptInput` does not define either field and the prompt builder does not consume them. The second call is another stochastic attempt rather than a targeted correction. -### P0: action evidence is missing from the verifier +### P0: action evidence is missing from the verifier — CLOSED, decisions/audit-phase1.md#verifier-false-positives The beat builder reads plural `actions`, while the scene worker emits singular `action`. Script validation therefore receives little or no action evidence and cannot reliably detect invented or omitted actions. -### P0: GPU leases are unsafe for concurrent jobs +### P0: GPU leases are unsafe for concurrent jobs — CLOSED, decisions/audit-phase1.md#no-lease-stealing When `/session/open` returns 409, the homesrv proxy assumes the active lease is stale and closes it. A second legitimate job can terminate the first job's model. `heartbeat_session()` exists but is not used by the @@ -508,60 +521,29 @@ dependency-fingerprinted artifacts, and uncertainty-driven review for recurring/ ## Implementation sequence for approval -### Phase 1: correctness and scheduling safety +### Phase 1: correctness and scheduling safety — DONE 2026-08-11 -- Fix enum mismatches and add tracklet cannot-links. -- Gate face/body pairing. -- Wire action evidence and verifier feedback. -- Make stage clearing/resume behavior honest. -- Replace 409 lease stealing with a queue/wait policy. -- Run actual lease heartbeats. -- Register ComfyUI under the same GPU resource scheduler. +Implemented and verified by CPU-only self-checks. Nothing ran on the GPU. What landed, with evidence +and the check that covers it, is in `decisions/audit-phase1.md`. -Verification: CPU-only unit/self-checks. No full pipeline or GPU run until explicitly scheduled. +Two items from this list were NOT implemented and are recorded with a revisit trigger instead: +honest stage clearing (`caveats/audit-open.md#dishonest-clearing`) and ComfyUI under the GPU scheduler +(`caveats/audit-open.md#comfyui-unscheduled`). Both need a design decision, not a patch. ### Phase 2: safe throughput improvements -- Introduce the two-phase Gemma lease schedule. -- Add JSON-schema outputs. -- Add local media paths and the ephemeral file/reference cache. -- Reuse direction downloads. -- Batch SigLIP embeddings. -- Batch scene/DB operations. -- Add bounded fetch and framed-page crop concurrency. +Now tracked in `ROADMAP.md`, with the done-when condition for each phase. Verification: unit checks first, followed by one labeled chapter after GPU availability is approved. -### Phase 3: multi-view constrained identity +### Phases 3 and 4 -- Persist chapter-local tracklets. -- Add face/body galleries and crop-quality selection. -- Resolve with multi-view query evidence and global constraints. -- Separate name claims from visual identity. -- Expand identity evaluation. - -### Phase 4: evidence-ledger narration - -- Build ordered beat evidence artifacts. -- Add scene-scoped story and turn-taking state. -- Generate script chunks with evidence IDs. -- Add confidence-aware wording and targeted verification/repair. +Now tracked in `ROADMAP.md`. ## Acceptance criteria -Capture a baseline and compare the same labeled chapter after every phase: - -- stage wall time and model-load time; -- number of Gemma calls and repair calls; -- MinIO bytes downloaded/uploaded; -- prompt-evaluation and generation timings; -- tracklet false merges/splits and identity accuracy; -- name and speaker accuracy; -- unresolved correctness flags; -- script repetition, unsupported facts, and verifier retry rate. - -Recommended approval boundary: implement Phases 1 and 2 first, measure them, then decide whether to proceed -with the larger identity and narration changes. +Now tracked in `ROADMAP.md`, together with the approval boundary: measure Phase 2 before deciding +whether the larger identity and narration changes are worth their size. ## Second-pass findings (2026-08-11) @@ -572,7 +554,7 @@ The first pass audited each side on its own terms. Most of what follows lives in one repo changed a field's meaning and the other still reads the old one. The existing per-file `__main__` self-checks cannot catch any of it, because each one asserts its own side of the contract. -### P0: the scene stage discards every speaker +### P0: the scene stage discards every speaker — CLOSED, decisions/audit-phase1.md#speaker-ref-is-canonical `normalize_dialogue` rewrites `row["speaker"]` to a `character_id` or `None` (`orchestrator/correctness.py:47`). `run_stage_dialogue` saves that shape into the vision blob @@ -592,7 +574,7 @@ they were people. (`worker_scene.py:138`). Fix: prefer `speaker_ref` when its kind is `character_id`, keep the local-id path as the fallback. This is the strongest argument for the cross-stage metamorphic tests proposed above. -### P0: the script verifier rejects correct narration, and its retry is a no-op +### P0: the script verifier rejects correct narration, and its retry is a no-op — CLOSED, decisions/audit-phase1.md#verifier-false-positives `verify_script` fails a beat on two rules that fire on valid output (`orchestrator/correctness.py:100-116`). @@ -616,7 +598,7 @@ are dropped without an error. Fix order: correct the two rules first, then wire the feedback fields. Fixing the plumbing alone makes the model retry against a broken oracle. -### P0: correctness flags block TTS permanently in the default configuration +### P0: correctness flags block TTS permanently in the default configuration — CLOSED, decisions/audit-phase1.md#flag-resolution `run_stage_tts` refuses to start while any unresolved flag of kind `script-verifier`, `ambiguous-speaker`, `conflicting-name-claims`, or `partial-dialogue` exists (`service.py:1419-1423`). The only code that ever @@ -638,7 +620,7 @@ resolved, blocking TTS for all future jobs. Fix: resolve by flag identity rather than by surviving panel, and give the autonomous path an explicit policy (auto-resolve below a rank, or fail the stage loudly) instead of an unreachable gate. -### P0: `awaiting_review` is immediately overwritten by `failed` +### P0: `awaiting_review` is immediately overwritten by `failed` — CLOSED, decisions/audit-phase1.md#flag-resolution `run_stage_tts` sets `awaiting_review` and then raises (`service.py:1422-1423`). The raise unwinds into `_run_pipeline`'s catch-all, which sets the job to `failed` (`service.py:391-393`). The review state the @@ -646,7 +628,7 @@ stage just recorded is gone before anyone can read it, and the operator sees a g queue of flags. The gate path at `service.py:337-341` returns instead of raising and does not have this problem. -### P1: set-of-mark face pairing is enabled by default, contrary to its own documentation +### P1: set-of-mark face pairing is enabled by default, contrary to its own documentation — CLOSED, decisions/audit-phase1.md#gated-face-pairing `worker_vision.py:22` states the feature is off by default and should be enabled per title once tuned. `worker_vision.py:31` reads `SOM_ATTRIBUTION` with a default of `"1"`. It is on. @@ -661,7 +643,7 @@ default the flag to off as documented. The pairing is also greedy in face order rather than a joint assignment (`worker_vision.py:42-52`), so the first face processed can claim a character that fits a later face far better. -### P1: the JSON repair pass can fabricate content +### P1: the JSON repair pass can fabricate content — OPEN, caveats/audit-open.md#repair-fabricates When a response fails to parse, `call_gemma4_json` sends the model its own truncated text and asks for the JSON it should have been (`worker_vision.py:174-178`). The repair call carries no image. On a response @@ -671,7 +653,7 @@ adds is invented and is indistinguishable downstream from transcribed text. Schema-constrained generation, already recommended above, removes most of this path. Until then the repair pass should re-send the image, or a truncated response should be retried rather than repaired. -### P1: `/review/preview` silently replaces a beat clip +### P1: `/review/preview` silently replaces a beat clip — OPEN, caveats/audit-open.md#preview-overwrites-clip `review_preview` renders one panel alone and calls `save_clip(panel_id, ...)` (`service.py:1902-1904`). Its docstring calls this harmless because assemble would regenerate it. It does not. `_render_one_beat` returns @@ -680,7 +662,7 @@ pins the solo preview into the final video and drops the rest of the beat's pane right by calling `delete_clip` first (`service.py:2008`). Preview should write to a scratch key or delete the clip row afterwards. -### P1: the session manager can orphan a llama-server and hold the GPU +### P1: the session manager can orphan a llama-server and hold the GPU — CLOSED, decisions/audit-phase1.md#unlocked-model-load `open_session` claims the slot, releases the lock, then spawns and health-waits for up to 300 seconds (`session_manager.py:108-119`). A `/session/close` arriving during that window finds `proc = None`, tears @@ -693,7 +675,7 @@ Related asymmetry: `_supervise_once` calls `_start_subprocess` while holding `_l (`session_manager.py:169-181`), so a respawn blocks `/session/active`, `/session/close`, and `/session/open` for the full health wait. The open path was deliberately written to avoid exactly this. -### P1: nothing limits concurrent jobs +### P1: nothing limits concurrent jobs — CLOSED, decisions/audit-phase1.md#no-lease-stealing `_background_jobs` accepts any number of pipelines (`service.py:302-303`, `service.py:427-428`). Each opens its own model sessions. Combined with the 409 lease stealing already recorded, two jobs terminate each @@ -717,7 +699,7 @@ vector spaces, and the fixed 0.85 threshold is only valid for one of them. Nothi vectors into one gallery with no error. Write the model id and pooling mode next to the vector and refuse to compare across versions. -### P2: smaller confirmed defects +### P2: smaller confirmed defects — partly closed, see `decisions/audit-phase1.md#related` and `caveats/audit-open.md` - An out-of-range `choice` from the resolver is mapped to NONE and then reported as `state: "new"` (`worker_vision.py:898-899`), so a hallucinated index mints a brand new character. It should be @@ -755,16 +737,9 @@ compare across versions. - MinIO credentials are hardcoded as defaults in committed source (`transport.py:95-99`, `service.py:74-76`). -### What this changes in the plan +### What this changes in the plan — DONE -Add to Phase 1, before any throughput work: +All five additions landed in Phase 1. See `decisions/audit-phase1.md`. -1. Restore the speaker field across the dialogue, scene, and script boundary. -2. Correct the two verifier rules, then wire `beat` and `verifier_feedback`. -3. Give correctness flags a resolution path that does not require a disabled gate, and stop the TTS block - from erasing `awaiting_review`. -4. Decide the set-of-mark default deliberately, and gate face pairing before the label is trusted. -5. Add job admission control at the orchestrator, not only lease queueing in the proxy. - -Add to the acceptance criteria: the share of narrated lines whose speaker is a named character rather than -`Someone`. That single number would have caught the first finding on the day it landed. +The acceptance metric they suggested, the share of narrated lines whose speaker is a named character +rather than `Someone`, is now in `ROADMAP.md`. It has not been measured yet. diff --git a/CLAUDE.md b/CLAUDE.md index 9461ab6..6dffd06 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -1,67 +1,72 @@ # CLAUDE.md -This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. +Goal, invariants, and working rules. Read this first. -## What this is +| file | holds | +| --- | --- | +| `NEXT.md` | the current state and the live plan | +| `ROADMAP.md` | the ordered outcomes past the current one | +| `JOURNAL.md` | what was run and when, append-only | +| `decisions/` | every settled question, indexed in `decisions/CLAUDE.md` | +| `caveats/` | every known limit and its revisit trigger, indexed in `caveats/CLAUDE.md` | +| `AGENTS.md` | commands, with the traps beside them | +| `AUDIT.md` | the 2026-08-11 pipeline audit, the source of the roadmap | +| `spec-v3.md` | current quality and look work, marked DONE/TODO per item | -The **workpc compute half** of a manga→narrated-video pipeline. This repo holds stateless -GPU/CPU workers only. State, job scheduling, and stage orchestration live in a **separate -homesrv orchestrator repo** (`/mnt/server/home/kami/apps/Maven/` region) — not here. The two -communicate over a fixed HTTP contract; never add sqlite or durable state to a worker. +Do not restate a finding here. Point at the decision. -Machine split (memorize): workers run on **workpc** (RX 7900 GRE, ROCm). MinIO + orchestrator -run on **homesrv** (`192.168.1.104`, CPU-only). Data dir `/mnt/server/home/kami/` is an SSHFS -mount of homesrv. +## The goal -## Run / test +This is the **workpc compute half** of a manga to narrated-video pipeline. It holds stateless GPU and +CPU workers only. State, job scheduling, and stage orchestration live in a separate homesrv +orchestrator repo (`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`). The two talk over a +fixed HTTP contract. The hard part is that one GPU serves every model, so the schedule, not the model, +sets the wall time. + +Machine split: workers run on **workpc** (RX 7900 GRE, ROCm). MinIO and the orchestrator run on +**homesrv** (`192.168.1.104`, CPU-only). `/mnt/server/home/kami/` is an SSHFS mount of homesrv. + +## Invariants + +1. **No durable state in a worker.** No sqlite, no cross-request memory. A worker pulls inputs from + MinIO by URI, does one stage, pushes outputs back, returns URIs. A `/dev/shm` cache is allowed + because it may be dropped at any time. +2. **One warm model at a time.** Every GPU stage takes a lease from `session_manager.py` on 8095. A 409 + is a queue signal, never a stale lease + (`decisions/audit-phase1.md#no-lease-stealing`). +3. **Never hold `_lock` across a model load or a health wait** in `session_manager.py` + (`decisions/audit-phase1.md#unlocked-model-load`). +4. **A dialogue row's speaker is `speaker_ref`.** The flat `speaker` field is a compatibility value and + holds a `character_id`, not a panel-local id + (`decisions/audit-phase1.md#speaker-ref-is-canonical`). +5. **A stage never raises after setting `awaiting_review`.** The pipeline's catch-all overwrites it + with `failed` (`decisions/audit-phase1.md#flag-resolution`). +6. **Never mint a character from an unparseable or out-of-range model answer.** That is `unresolved` + (`decisions/audit-phase1.md#hallucinated-index`). +7. **The HTTP contract with the orchestrator is load-bearing.** Changing a worker's request or response + shape means reconciling the orchestrator in the same session. Neither repo's self-checks can catch a + contract break, because each asserts its own side. +8. **`ponytail:` comments mark deliberate simplifications** and name the upgrade path. Respect them. + +## Working rules ```bash ./start_workers.sh # dev: session_manager + 9 workers, each a uvicorn in a tmux window -tmux attach -t manga-workers # watch logs; per-worker window +tmux attach -t manga-workers # per-worker logs sudo systemd/install.sh # production: one systemd unit per process (User=kami) -python session_manager.py # each module has a __main__ self-check (assert-based, no framework) -python transport.py # run these to verify a file after editing it -python test_vision_parse.py # the one standalone pytest-free test +.venv/bin/python worker_scene.py # every module has an assert-based __main__ self-check +.venv/bin/python test_vision_parse.py + +cd /mnt/server/home/kami/docker-apps/manga-infra/orchestrator && pytest -q --ignore=test_api.py ``` -There is no lint/build step. `.venv` is the ROCm torch env; workers import `transport` by module name. +There is no lint or build step. `.venv` is the ROCm torch env. Workers import `transport` by module +name. Ports: crop 8000, vision 8002, identity 8003, scene 8004, script 8005, tts 8006, layers 8007, +render 8008, session_manager 8095. -## Architecture - -**Workers are stateless HTTP stages.** Each `worker_*.py` is a FastAPI app on a fixed port. It -pulls inputs from MinIO by URI to local disk (`/dev/shm`), does one stage, pushes outputs back, -returns URIs. No cross-request memory. Ports: crop 8000, vision 8002, identity 8003, scene 8004, -script 8005, tts 8006, layers 8007, render 8008, **session_manager 8095**. - -**`transport.py`** — shared MinIO client (`get`/`put`/`put_bytes`/`exists`) + `install_logging(app, name)` -(one log line per request with panel id). URIs are `s3://bucket/key`. Import it in every worker. - -**`session_manager.py`** — the **GPU mutex**. Only one warm model at a time on the single local GPU. -Two model kinds: -- *subprocess* (`gemma4`): it spawns/health-waits/terminates `llama-server`, and a supervisor - respawns it in-place if it crashes mid-session (keeps the same session_id + port). -- *in-process* (`siglip2`, `dots`): returns `port=None`; the **worker** loads the transformers model - itself and must expose `/unload` so `/session/close` can free the ~5GB VRAM (the mutex alone - can't reclaim it). Leases have a TTL + heartbeat; a reaper force-closes stale ones. - -A GPU worker's flow: `/session/open {model}` → (409 if busy) → do work against the returned port or -its own resident model → `/session/close`. CPU workers (crop) take no session. - -## Conventions - -- **`ponytail:` comments** mark deliberate simplifications and name the upgrade path — respect them, - don't "fix" them without reason. -- Any non-trivial logic gets ONE runnable check in `__main__` (assert-based `demo`/self-check), not a - test suite. Follow that pattern; run the file to verify. -- The HTTP contract with the orchestrator is load-bearing and shared across repos — changing a - worker's request/response shape means reconciling the orchestrator too (see commit history: - "reconcile worker contracts"). - -## Specs & docs - -- `spec-v3.md` — current quality/look work (narration voice, panel curation, render overhaul), marked - DONE/TODO per item. `legacy/` holds the old single-repo workers. -- `AGENTS.md` — use `scripts/analyze_video_frames.sh` (not manual seeking) to diagnose render output; - its frames are `/tmp` artifacts, never commit them. -- `collage.py` (behind `COLLAGE` flag) and the render worker carry the animated-layout planners. +- Any non-trivial logic gets ONE runnable check in `__main__`, assert-based, no framework. Run the file + to verify it. +- Editing a worker's request or response shape means editing the orchestrator too, in the same session. +- Update `NEXT.md` alongside any change that moves the plan, and append to `JOURNAL.md` after a run. +- Do not run GPU work or a full pipeline without asking. diff --git a/HANDOFF.md b/HANDOFF.md deleted file mode 100644 index e44c67f..0000000 --- a/HANDOFF.md +++ /dev/null @@ -1,54 +0,0 @@ -# HANDOFF: audit second pass (2026-08-11) - -## What was asked -Review `AUDIT.md` (untracked, first pass, 565 lines) and add anything missing. - -## What was done -Read both halves of the pipeline and checked the first pass against source. No code changed. -No GPU work, no pipeline run, no tests executed. - -Files read in full: -- workpc: `worker_vision.py` (1008), `worker_identity.py` (322), `worker_script.py` (300), - `worker_scene.py` (170), `worker_tts.py` (241), `session_manager.py` (238), `transport.py` (216), - `worker_crop.py` (first 120 of 359). -- homesrv `/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`: `service.py` (2048), - `db.py` (815), `session_proxy.py` (302), `correctness.py` (116), `tracklets.py` (121), - `minio_layout.py` (105), `grouping.py` (75). - -Result: `AUDIT.md` grew from 565 to 745 lines. New section `## Second-pass findings (2026-08-11)` -at line 566. 4 new P0, 6 new P1, 13 P2, plus 5 additions to the Phase 1 list and 1 to the -acceptance criteria. - -Four first-pass claims were confirmed in source and are not restated in the new section: - -- tracklet gender enum: `tracklets.py:43` uses `male|female`, vision emits `m|f|unknown`. -- missing action evidence: `worker_scene.py:85` emits `action`, `correctness.py:91` reads `actions`. -- dropped verifier feedback: `worker_script.py:13-23` defines neither field. -- 409 lease stealing: `session_proxy.py:40-49`. - -## Top 4 new findings -1. `worker_scene.py:78` reads `speaker` as a local id, but `correctness.py:47` already rewrote it - to a character_id. Every lookup returns None, so all narration says `Someone`. -2. `correctness.py:106-115` fails valid narration on sentence-initial capitals and on short - quotes, and `service.py:1384-1386` turns that into a halted chapter. -3. `service.py:1419-1423` blocks TTS on correctness flags that only `/review/approve` can clear, - while `GATES` defaults to off (`service.py:312`). -4. `session_manager.py:108-119` can orphan a llama-server that keeps its VRAM. - -## Still open -- Nothing from this session is half-finished. `AUDIT.md` is complete as written. -- No finding has been fixed. All are report-only. -- `AUDIT.md` is still untracked as of this session's start. Decide whether it belongs in git. -- Prose linter reports ~97 style hits in `AUDIT.md`. Almost all are in the first-pass sections - (lines 1-565), which were left as the author wrote them. -- Carried over from the earlier reconstruction handoff of the same day: - - `.venv/` is gone. Rebuild it, then install the ROCm torch wheel the way workpc had it. - - `dots.tts/` and `legacy/` are not recoverable from transcripts. - - `RESUME_SPEC.md`, `pipeline-design-notes.md`, and `spec-v2.md` are unrecoverable. - -## Next command -``` -cd /home/kami/Programs/n8n-worker -sed -n '566,745p' AUDIT.md # read the new section -``` -Then pick the Phase 1 order at `AUDIT.md:511` as amended at the end of the new section. diff --git a/JOURNAL.md b/JOURNAL.md new file mode 100644 index 0000000..6d90e68 --- /dev/null +++ b/JOURNAL.md @@ -0,0 +1,22 @@ +# JOURNAL + +Append-only, newest last. One block per session or run. Not a changelog: this records what happened on +the day a number was produced, so a later postmortem can find it. + +## 2026-08-11 Audit second pass [no task] + +Command: none. Source reading only. +Outcome: finished. `AUDIT.md` grew from 565 to 771 lines with a `## Second-pass findings` section: +4 new P0, 6 new P1, 13 P2, 5 additions to the Phase 1 list. +Produced: commit `6d9df5b`, `AUDIT.md:566`. + +## 2026-08-11 Audit Phase 1 implemented [#203] + +Command: `python worker_scene.py worker_script.py worker_vision.py session_manager.py`, +`pytest -q --ignore=test_api.py` in the orchestrator. +Outcome: finished. All self-checks pass, 108 orchestrator tests pass. No GPU work, no pipeline run. +`test_api.py` was skipped because fastapi is not installed in the workpc venv. +Produced: `decisions/audit-phase1.md`, `caveats/audit-open.md`, `ROADMAP.md`, and this scaffold. + +One existing test asserted the bug: `test_name_binding.test_conflict_flags_and_stays_unnamed` relied on +orphan flags leaking into every chapter, because it never created panel rows. It now creates them. diff --git a/NEXT.md b/NEXT.md new file mode 100644 index 0000000..99a4bca --- /dev/null +++ b/NEXT.md @@ -0,0 +1,42 @@ +# NEXT + +Updated 2026-08-11. Replaces the old `HANDOFF.md`. + +## State + +Audit Phase 1 is implemented and green. Nothing is half-finished. + +Changed on workpc: `worker_scene.py`, `worker_script.py`, `worker_vision.py`, `session_manager.py`. +Changed on homesrv (`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`): `tracklets.py`, +`correctness.py`, `db.py`, `service.py`, `session_proxy.py`, `test_script_verify.py`, +`test_name_binding.py`. + +What landed and why: `decisions/audit-phase1.md`. What was left open: `caveats/audit-open.md`. + +Verification: CPU-only self-checks and the orchestrator test suite. 108 orchestrator tests pass +(`test_api.py` is excluded on workpc because fastapi is not installed in this venv). No GPU work ran +and no pipeline ran, so none of this is confirmed against a real chapter. + +The orchestrator changes are edited in place on the SSHFS mount and are NOT committed. Its git root is +`/mnt/server/home/kami/docker-apps`. Its container also needs a rebuild or restart to pick them up. + +## Next + +1. Commit the orchestrator half in `/mnt/server/home/kami/docker-apps` and restart the container. +2. Run one labeled chapter end to end and record the baseline numbers from `ROADMAP.md`, especially the + share of narrated lines with a named speaker. That number is the check on the largest Phase 1 fix. +3. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work + (`caveats/audit-open.md#sqlite-locking`). + +## Open questions + +Four Phase 1 items have no Vikunja task and were not created, because writing to the tracker was not +asked for: the speaker contract fix, the verifier rules, the tracklet constraints, and the flag +resolution path. Only [#203] existed and is now closed by `decisions/audit-phase1.md#unlocked-model-load`. + +Three audit items are deliberately not done and are recorded as caveats rather than silently dropped: +honest stage clearing, ComfyUI under the session mutex, and reversible identity merges. Each needs a +design decision, not a patch. + +Carried over from the reconstruction: `.venv` needs the ROCm torch wheel reinstalled, and `dots.tts/`, +`legacy/`, `RESUME_SPEC.md`, `pipeline-design-notes.md`, `spec-v2.md` are unrecoverable. diff --git a/ROADMAP.md b/ROADMAP.md new file mode 100644 index 0000000..0b5172c --- /dev/null +++ b/ROADMAP.md @@ -0,0 +1,66 @@ +# ROADMAP + +Ordered outcomes past the current one. Phase 1 landed on 2026-08-11 and moved to +`decisions/audit-phase1.md`. The reasoning behind every phase is in `AUDIT.md`. + +Approval boundary from the audit: ship Phase 2 and measure it. Then decide whether Phase 3 and 4 are +worth the size of the change. + +## Phase 2: safe throughput + +Unblocks: a chapter that finishes in a fraction of the current wall time without changing any model. +Done when: the same labeled chapter runs with two gemma loads instead of seven. The acceptance numbers +below are recorded in a report. + +- Two-phase gemma lease schedule, so the model stays resident across compatible stages. +- JSON-schema-constrained generation for detect, dialogue, roster, direction, same-person, and resolve. +- `--media-path /dev/shm` and `file://` image paths instead of base64 data URIs. +- An ephemeral `/dev/shm` cache keyed by S3 URI plus etag, and reused direction downloads. +- Batched siglip embeddings, one forward pass per bounded image batch. +- Batched scene construction and SQLite writes. Set `busy_timeout` first + (`caveats/audit-open.md#sqlite-locking`). +- Bounded fetch and framed-page crop concurrency. +- Images before the differing instruction in every multi-call prompt, so llama-server reuses its KV + cache prefix across the two direction passes. + +## Phase 3: multi-view constrained identity + +Unblocks: an identity that survives a reviewer disagreeing with it. +Done when: false merges and false splits are reported per labeled chapter and the resolver beats the +Phase 1 baseline on both. + +- Chapter-local tracklets persisted as a resumable homesrv artifact, not worker memory. +- Face and body galleries with crop-quality selection, and 2-4 query views per tracklet. +- Global constraints at resolution: must-link inside a tracklet, cannot-link for co-presence, gender + and species gates, manual locks, one appearance per character per panel. +- Name claims separated from visual identity. Roster names stay candidate vocabulary. +- Reversible merges: a cluster with `canonical_id`, `member_ids`, and `cannot_link`, replacing the + destructive delete (`caveats/audit-open.md#destructive-reconcile`). +- Identity evaluation extended to purity, fragmentation, name binding, and speaker accuracy. + +## Phase 4: evidence-ledger narration + +Unblocks: narration that can be checked against the panel rather than trusted. +Done when: every beat carries an evidence artifact and the verifier reports quote grounding, action +coverage, and future-fact leakage against it. + +- Ordered beat evidence: panels, actions, chronology, dialogue ids, typed speaker refs, uncertainty. +- Scene-scoped story and turn-taking state, reset on location and time changes. +- Script chunks of 6-12 consecutive beats per text-only call, returning evidence ids. +- Confidence-aware wording: a weak speaker becomes `someone`, not a confident name. +- Targeted per-beat repair driven by the exact failures. + +## Acceptance criteria + +Capture a baseline and compare the same labeled chapter after every phase. + +- Stage wall time and model-load time. +- Gemma call count and repair call count. +- MinIO bytes moved, prompt-evaluation and generation timings. +- Tracklet false merges and splits, identity accuracy. +- Name and speaker accuracy, unresolved correctness flags. +- Script repetition, unsupported facts, verifier retry rate. +- The share of narrated lines whose speaker is a named character rather than `Someone`. That one + number would have caught `decisions/audit-phase1.md#speaker-ref-is-canonical` the day it landed. + +Read `caveats/audit-open.md#inconsistent-stage-policy` before trusting any per-stage count. diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md new file mode 100644 index 0000000..459f244 --- /dev/null +++ b/caveats/CLAUDE.md @@ -0,0 +1,34 @@ +# caveats/ + +One entry per known breakage, deferral, or trap. An entry names what fails, who pays for it, and the +concrete trigger that makes fixing it worth the time. + +A caveat that gets resolved moves to `decisions/` or gets deleted. A caveat with no revisit trigger is +a complaint, so give it one or drop it. + +## Rules for this directory + +* One file per source, one `##` section per limit. +* State the limit as what fails, not as a topic. "Reconcile deletes the losing character" beats + "Reconcile". +* Delete an entry the moment its trigger fires and the work lands. + +## Index + +| caveat | source | +| --- | --- | +| [Reconcile deletes the losing character irreversibly](audit-open.md#destructive-reconcile) | AUDIT.md | +| [Clearing a stage does not undo what it wrote](audit-open.md#dishonest-clearing) | AUDIT.md | +| [ComfyUI uses the GPU outside the session mutex](audit-open.md#comfyui-unscheduled) | AUDIT.md | +| [The JSON repair pass can fabricate dialogue](audit-open.md#repair-fabricates) | AUDIT.md | +| [`/review/preview` pins a solo clip into the final video](audit-open.md#preview-overwrites-clip) | AUDIT.md | +| [Stored embeddings carry no model or pooling version](audit-open.md#untagged-embeddings) | AUDIT.md | +| [Worker endpoints block the event loop](audit-open.md#blocking-event-loop) | AUDIT.md | +| [SQLite has no busy timeout, so parallelism will surface as lock errors](audit-open.md#sqlite-locking) | AUDIT.md | +| [`layers` runs after `tts`, so pipelined solo beats lose parallax](audit-open.md#layers-after-tts) | AUDIT.md | +| [`completed` means something different in each stage](audit-open.md#inconsistent-stage-policy) | AUDIT.md | +| [Identity worker caches characters the orchestrator has deleted](audit-open.md#stale-known-cache) | AUDIT.md | +| [A character seen once gets no assignment at all](audit-open.md#pending-in-worker-memory) | AUDIT.md | +| [MinIO credentials are hardcoded in committed source](audit-open.md#hardcoded-credentials) | AUDIT.md | +| [Assemble marks a job completed with no clips](audit-open.md#empty-assemble) | AUDIT.md | +| [Reviewer timestamps drift against the crossfaded video](audit-open.md#timeline-drift) | AUDIT.md | diff --git a/caveats/audit-open.md b/caveats/audit-open.md new file mode 100644 index 0000000..acd8a7c --- /dev/null +++ b/caveats/audit-open.md @@ -0,0 +1,149 @@ +# Open limits from the 2026-08-11 audit + +Everything here was read in source during the audit and deliberately left unfixed in Phase 1. The +fixed findings live in `decisions/audit-phase1.md`. Line numbers are from the audit and may drift. + +## Reconcile deletes the losing character irreversibly {#destructive-reconcile} + +Reconciliation deletes the losing character row (`db.py:506`). Clearing the reconcile stage does not +undo it, and name claims attached to the merged-away character are not repointed. + +Costs: one bad merge is unrecoverable without rebuilding the identity stage for the whole manga. +Revisit when: identity work resumes, or a reviewer reports a wrong merge on a real chapter. +Workaround: none. Clear identity and rerun, which loses the good merges too. + +## Clearing a stage does not undo what it wrote {#dishonest-clearing} + +Dialogue and direction mutate the shared vision JSON. Clearing dialogue leaves its keys in place, so a +rerun treats old dialogue as completed. Clearing identity preserves the per-manga registry. + +Costs: a rerun silently reuses stale output, which reads as a reproducible result. +Revisit when: any stage is scheduled concurrently or resumed automatically. A stage must be idempotent +before either is safe. +Workaround: delete the keys by hand, or clear from `crop` down. + +## ComfyUI uses the GPU outside the session mutex {#comfyui-unscheduled} + +The layers stage calls ComfyUI directly and takes no lease. Another job can load gemma, siglip2, or +dots while ComfyUI holds the same GPU. + +Costs: out-of-memory failures that look random and land on an unrelated stage. +Revisit when: layers is enabled on a real run, or a second concurrent job is allowed. +Workaround: `MAX_CONCURRENT_JOBS=1` keeps one pipeline at a time, which is the current default. + +## The JSON repair pass can fabricate dialogue {#repair-fabricates} + +`call_gemma4_json` hands the model its own truncated text and asks for the JSON it should have been +(`worker_vision.py`). The repair call carries no image. On a response truncated by `max_tokens`, the +model completes dialogue it can no longer see. What it adds is indistinguishable downstream from +transcribed text. + +Costs: invented lines enter the script with normal provenance. +Revisit when: schema-constrained generation lands, which removes most of this path. +Workaround: resend the image on repair, or retry a truncated response instead of repairing it. + +## `/review/preview` pins a solo clip into the final video {#preview-overwrites-clip} + +`review_preview` renders one panel and calls `save_clip(panel_id, ...)`. `_render_one_beat` returns +early when a clip already exists for the leader. Previewing a beat leader therefore drops the rest of +the beat's panels. `review_retts` gets this right by calling `delete_clip` first. + +Costs: a reviewer silently corrupts the output by looking at it. +Revisit when: the review UI is used on a real chapter. +Workaround: never preview a beat leader, or delete the clip row afterwards. + +## Stored embeddings carry no model or pooling version {#untagged-embeddings} + +`embed_crop` uses `pooler_output` when present and falls back to mean-pooled patch tokens otherwise. +The two paths produce different vector spaces, and the fixed 0.85 threshold is valid for one of them. +Nothing beside a stored `.npy` records which model, revision, or pooling produced it. + +Costs: a transformers upgrade mixes incompatible vectors into one gallery with no error. +Revisit when: transformers or the siglip2 revision is upgraded. Before, not after. +Workaround: none. Write the model id and pooling mode beside the vector and refuse cross-version +comparison. + +## Worker endpoints block the event loop {#blocking-event-loop} + +Endpoints declared `async def` run blocking MinIO, OpenCV, torch, and ffmpeg calls directly on the +event loop. A busy worker cannot answer `/health` or `/unload`. + +Costs: `/health/workers` reports a working worker as unreachable, and the session manager's 30-second +`/unload` can time out exactly when VRAM needs freeing. +Revisit when: a stage stalls on `/unload`, or before any bounded parallelism lands. +Workaround: `def` instead of `async def` moves each handler to the threadpool. Tracked as [#199] for +the render worker. + +## SQLite has no busy timeout {#sqlite-locking} + +`get_conn` opens a connection per call with no `busy_timeout`. WAL tolerates one writer. `PIPELINE=1` +already writes clips from concurrent tasks while TTS writes audio. + +Costs: planned CPU parallelism will surface as `database is locked` before it surfaces as throughput. +Revisit when: Phase 2 concurrency work starts. Set the timeout first. +Workaround: keep `PIPELINE` off. + +## `layers` runs after `tts` {#layers-after-tts} + +`STAGES` orders `layers` after `tts`, while `run_stage_tts` warns that eager rendering under +`PIPELINE=1` needs layers to run first. + +Costs: with the flag on, solo beats always render without parallax. +Revisit when: a real run enables `PIPELINE=1`. +Workaround: keep `PIPELINE` off, or reorder `STAGES`. + +## `completed` means something different in each stage {#inconsistent-stage-policy} + +A dropped vision panel fails the stage and halts the pipeline. A failed direction window counts its +panels as done. Layers always finishes completed. + +Costs: the acceptance metrics in `ROADMAP.md` cannot be read across stages. +Revisit when: a baseline measurement is taken. The numbers are meaningless until then. +Workaround: none. + +## Identity worker caches characters the orchestrator has deleted {#stale-known-cache} + +`worker_identity._known_cache` is invalidated only by `_persist_char`. The reconcile stage deletes +losing characters directly in the orchestrator database. A long-lived worker keeps shortlisting and +assigning ids that no longer exist. + +Costs: assignments point at rows that are gone. +Revisit when: reconcile runs on a chapter without a worker restart between stages. Tracked as [#201], +which proposes caching per `manga_id`. +Workaround: restart the identity worker after reconcile. + +## A character seen once gets no assignment at all {#pending-in-worker-memory} + +`worker_identity._pending` holds full crop images in worker memory for a whole chapter, keyed by +session. That is durable per-chapter state inside a worker documented as stateless, and it is lost on +restart. A character seen exactly once receives no assignment, not even a chapter-local handle. + +Costs: one-off characters vanish from the scene graph. +Revisit when: chapter-local tracklet persistence lands (`ROADMAP.md`, Phase 3). +Workaround: none. + +## MinIO credentials are hardcoded in committed source {#hardcoded-credentials} + +Defaults live in `transport.py` and `service.py`. + +Costs: the credentials are in git history for anyone who gets the repo. +Revisit when: the repo leaves this machine, or MinIO is reachable outside the LAN. +Workaround: the environment variables already override them. Set them and remove the defaults. + +## Assemble marks a job completed with no clips {#empty-assemble} + +`run_stage_assemble` does not check that `clip_uris` is non-empty before assembling, then marks the job +completed. + +Costs: a failed chapter reports success. +Revisit when: any run reports completed without a video. One `if not clip_uris` guard fixes it. +Workaround: none. + +## Reviewer timestamps drift against the crossfaded video {#timeline-drift} + +`/review/panels` sums per-panel audio durations. Assemble crossfades clips using the per-beat +transitions, so every non-`cut` transition shortens the real video. + +Costs: reviewer timestamps drift further out of sync the further into the chapter they scrub. +Revisit when: the review UI is used for timing work. +Workaround: subtract the transition overlaps by hand. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md new file mode 100644 index 0000000..0dabe16 --- /dev/null +++ b/decisions/CLAUDE.md @@ -0,0 +1,29 @@ +# decisions/ + +One entry per settled question about this pipeline. An entry names the claim, the evidence, the date, +and what it forbids or permits next. + +No entry is a plan. `NEXT.md` holds the plan, `ROADMAP.md` holds the horizon, and a limitation that is +still live belongs in `caveats/`. + +## Rules for this directory + +* One file per topic, one `##` section per question, so the index can link an anchor. +* An entry cites a file and line, a commit, or a test. No citation means it is an opinion, and an + opinion gets deleted rather than defended. +* **Closed** means acting on it is safe. **Open** means investigated and undecided. **Void** means the + evidence turned out to be invalid and the claim must not be cited again. +* Rewrite an entry when something contradicts it, and say what changed. + +## Index + +| decision | state | +| --- | --- | +| [A dialogue row's speaker is `speaker_ref`, not `speaker`](audit-phase1.md#speaker-ref-is-canonical) | closed | +| [Script verification must not fail on sentence-initial capitals or short quotes](audit-phase1.md#verifier-false-positives) | closed | +| [Tracklet links are transitive over similarity, never across a hard constraint](audit-phase1.md#tracklet-hard-constraints) | closed | +| [A detected face takes an identity only when it falls inside that character's box](audit-phase1.md#gated-face-pairing) | closed | +| [A 409 from the GPU mutex is a queue signal, not a stale lease](audit-phase1.md#no-lease-stealing) | closed | +| [Correctness flags resolve by flag id, and never block autonomous TTS](audit-phase1.md#flag-resolution) | closed | +| [An out-of-range resolver index is `unresolved`, never a new character](audit-phase1.md#hallucinated-index) | closed | +| [The session manager holds no lock across a model load](audit-phase1.md#unlocked-model-load) | closed | diff --git a/decisions/audit-phase1.md b/decisions/audit-phase1.md new file mode 100644 index 0000000..b8fa5e8 --- /dev/null +++ b/decisions/audit-phase1.md @@ -0,0 +1,144 @@ +# Audit Phase 1: correctness and scheduling safety + +Settled 2026-08-11 from `AUDIT.md`. Every entry was verified by a runnable check in the same commit. +No GPU work ran and no pipeline run was executed. Each claim rests on source and on the CPU-only +self-checks named below. + +Files: `worker_scene.py`, `worker_script.py`, `worker_vision.py`, `session_manager.py` on workpc, and +`tracklets.py`, `correctness.py`, `db.py`, `service.py`, `session_proxy.py` in the homesrv orchestrator +(`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`). + +## Speaker ref is canonical {#speaker-ref-is-canonical} + +**Closed.** A dialogue row's speaker is `speaker_ref` when its kind is `character_id`. The flat +`speaker` field is a compatibility value. The orchestrator already rewrote it from a panel-local id to +a `character_id`. + +Evidence: `correctness.py:normalize_dialogue` rewrites `row["speaker"]`. `worker_scene.build_scene` +mapped that value through `id_by_local`, whose keys are panel-local ids. Every lookup returned `None`, +so `worker_script._render_line` narrated every line as `Someone`. + +Forbids: reading `speaker` as a panel-local id anywhere downstream of the dialogue stage. +Check: `python worker_scene.py`, the `p007` case. + +The same field leaked into the dialogue resume path, where a stored `character_id` was pushed into +`recent` as a speaker name. It now renders the registry name (`service.py`, `run_stage_dialogue`). + +## Verifier false positives {#verifier-false-positives} + +**Closed.** `verify_script` must stay quiet on valid narration. A failure is not soft. The script stage +retries once and then raises, so one false positive halts the whole chapter. + +Two rules fired on correct output. + +1. `unsupported-proper-noun` flagged any capitalized token outside a 20-word stop list. That caught + ordinary sentence openers (`Suddenly`, `Behind`, `Inside`, `Both`, `Everyone`, `After`, `Two`) and + `Someone`. A capital opening a sentence is now grammar unless the word falls outside a real + narration vocabulary. A capital in the middle of a sentence still counts as name evidence. +2. `misquote` compared each quoted span against a WHOLE source line at ratio 0.82. The narration prompt + asks for a short quote of the actual words. A 15-character quote from a 40-character line scored + about 0.55. Grounding now matches the longest contiguous span instead. + +Forbids: adding a verifier rule without a test that a correct beat passes it. +Check: `pytest test_script_verify.py` in the orchestrator. + +Verifier feedback now reaches the retry. `worker_script.ScriptInput` declares `beat` and +`verifier_feedback`. Pydantic v2 dropped both silently before, so the retry was another blind sample. +The repair prompt names each failure and lists the exact quotable lines from the beat. + +## Tracklet hard constraints {#tracklet-hard-constraints} + +**Closed.** Tracklet linking is transitive over similarity evidence and never across a hard constraint. + +The hard constraints are gender and co-presence. Gender is canonicalized first. Vision emits +`m|f|unknown` while the registry says `male|female`, and the old gate tested only the second spelling, +so it never fired on real data. Co-presence means two detections in one panel are two people by +construction. A merge is rejected when any cross pair between the two groups violates either rule. +A weak chain can no longer bridge two people seen together. + +Forbids: comparing a raw gender string against a literal enum anywhere in the identity path. +Check: `python tracklets.py` and `pytest test_tracklets.py`. + +## Gated face pairing {#gated-face-pairing} + +**Closed.** A detected face takes a character's identity only when its centre falls inside that +character's gemma bbox. The box is grown by 25% first. An unpaired face stays `unknown`. + +This matters because the label becomes `speaker_method="som_face"`, the highest-trust provenance the +pipeline records. Unconditional nearest-neighbour pairing laundered a geometric guess into evidence. +Pairs are now taken globally shortest-first, so the first face processed cannot claim a character that +fits a later face better. + +Set-of-mark attribution stays ON by default now that the pairing is gated. The docstring said "off by +default" while the flag read `SOM_ATTRIBUTION` with default `"1"`. The code and the comment now agree. + +Forbids: trusting a `som_face` speaker without the containment gate. +Check: `python worker_vision.py`. + +## No lease stealing {#no-lease-stealing} + +**Closed.** A 409 from `/session/open` means another job legitimately holds the GPU. The proxy queues +on it. Only the session manager's TTL reaper clears a dead lease, because only it can tell a dead +lease from a busy one. + +`session_proxy.open_session` previously closed the active lease and retried, so two jobs terminated +each other's models. Job admission control now bounds concurrent pipelines through +`MAX_CONCURRENT_JOBS`, default 1. A lease queue alone still lets two pipelines interleave stages +against one GPU. + +`heartbeat_session` is finally called. `open_session` starts a beat task and `close_session` cancels +it, so a stage running past its TTL is no longer reaped mid-run. Beating from open and close keeps all +nine stage call sites unchanged. + +Forbids: closing a session the caller did not open. + +## Flag resolution {#flag-resolution} + +**Closed.** Correctness flags resolve by flag id, and unresolved flags never wedge an autonomous run. + +Two defects combined into a permanent block. `get_correctness_flags` accepted rows whose panel no +longer existed. A flag raised against a panel that a crop rerun deleted became visible to every +chapter and could never be resolved. `resolve_correctness_flags` could only touch flags whose +panel still existed. The view no longer admits orphans, and resolution takes explicit flag ids. + +The TTS block also had no reachable exit. It refused to start on four flag kinds that only +`/review/approve` clears, and `GATES` defaults to off. With gates on, TTS now returns rather than +raises, so `awaiting_review` survives instead of being overwritten by `failed` in the pipeline's +catch-all. With gates off, the flags are logged and cleared, because no reviewer exists to clear them. + +Forbids: raising out of a stage that has just set `awaiting_review`. +Check: `pytest test_name_binding.py` in the orchestrator. + +## Hallucinated index {#hallucinated-index} + +**Closed.** An out-of-range `choice` from `/vision/resolve` is `unresolved`, like a parse failure. Only +an explicit `0` means NONE and mints a new character. Mapping a bad index to NONE created a brand new +entry in the permanent registry from a hallucination. + +Check: `python worker_vision.py`. + +## Unlocked model load {#unlocked-model-load} [#203] + +**Closed.** The session manager never holds `_lock` across a model load or a health wait. + +`open_session` was already written that way, but it could orphan the server it spawned. A +`/session/close` arriving during the load found `proc = None`, tore down nothing, and cleared +`_active`. The finished server then held its VRAM unreferenced, and the next open spawned a second one +on the same port. Open now tears down its own process and reports 409 when its lease vanished mid-load. + +`_supervise_once` had the opposite asymmetry. It called `_start_subprocess` while holding `_lock`, +which blocked `/session/active`, `/session/close`, and `/session/open` for the full health wait. It now +claims the respawn by clearing `proc`, spawns unlocked, and tears the new process down if the lease +disappeared meanwhile. + +Check: `python session_manager.py`. + +## Related + +`_extract_json` in both `worker_vision.py` and `worker_script.py` now uses +`json.JSONDecoder().raw_decode` from the first brace. The greedy `\{.*\}` ran to the LAST brace in the +reply. A second object or trailing braced prose burned a repair call on a response that parsed fine. + +`worker_scene.build_scene` emits `actions` as a list beside the joined `action` string, because +`correctness.build_beat_artifact` reads the plural key. The verifier was receiving no action evidence +at all. The beat builder also falls back to splitting the singular string. diff --git a/session_manager.py b/session_manager.py index 6d20ca8..da3e9fc 100644 --- a/session_manager.py +++ b/session_manager.py @@ -116,7 +116,12 @@ def open_session(req: OpenReq): with _lock: if _active is not None and _active["session_id"] == session_id: _active["proc"] = proc - return {"session_id": session_id, "port": cfg.get("port")} + return {"session_id": session_id, "port": cfg.get("port")} + # The lease was closed or reaped while we were spawning (close saw proc=None and tore down + # nothing). The server we just started is unreferenced and would hold its VRAM until someone + # killed it by hand, and the next open would spawn a SECOND one on the same port. + _teardown({"proc": proc}) + raise HTTPException(409, "session closed while the model was loading") @app.post("/session/close") @@ -162,8 +167,8 @@ def _supervise_once(): talks to the port directly, so if we don't bring it back every remaining panel of the stage fails with 'connection refused'. Restarting under the SAME session keeps the lease + port valid so the orchestrator never has to re-open. Returns True if a respawn happened.""" + global _active with _lock: - global _active sess = _active if not sess: return False @@ -174,16 +179,29 @@ def _supervise_once(): proc.wait(timeout=1) # reap the zombie except subprocess.TimeoutExpired: pass - print(f"[supervise] gemma subprocess died (rc={proc.returncode}) session {sess['session_id']}; " - f"respawning", flush=True) - try: - sess["proc"] = _start_subprocess(MODELS[sess["model"]]) - sess["last_beat"] = time.time() # don't count the downtime against the TTL reaper + # claim the respawn before releasing the lock: proc=None makes the next _supervise_once pass + # return early, so the health wait can't be entered twice for one death. + sess["proc"] = None + session_id, model = sess["session_id"], sess["model"] + # ponytail: spawn + health-wait UNLOCKED, like open_session. Holding _lock here blocked + # /session/active, /close and /open for the full health wait (up to 300s). + print(f"[supervise] gemma subprocess died (rc={proc.returncode}) session {session_id}; " + f"respawning", flush=True) + try: + new_proc = _start_subprocess(MODELS[model]) + except Exception as e: + print(f"[supervise] respawn failed: {e}; clearing session so the mutex frees", flush=True) + with _lock: + if _active is not None and _active["session_id"] == session_id: + _active = None + return False + with _lock: + if _active is not None and _active["session_id"] == session_id: + _active["proc"] = new_proc + _active["last_beat"] = time.time() # don't count the downtime against the TTL reaper return True - except Exception as e: - print(f"[supervise] respawn failed: {e}; clearing session so the mutex frees", flush=True) - _active = None - return False + _teardown({"proc": new_proc}) # lease went away mid-respawn; don't orphan the server + return False def _reaper(): @@ -229,6 +247,24 @@ if __name__ == "__main__": close_session(SessionReq(session_id=s4["session_id"])) assert active() is None + # a close landing DURING the spawn must not orphan the server: open reports 409 and tears it down. + MODELS["gemma4"]["binary"] = "/bin/sleep" + MODELS["gemma4"]["port"] = None + real_start = _start_subprocess + + def racing_start(cfg): + p = real_start(cfg) + close_session(SessionReq(session_id=_active["session_id"])) # close mid-load + return p + _start_subprocess = racing_start + try: + open_session(OpenReq(model="gemma4", ttl=3600)) + assert False, "open must 409 when its lease vanished mid-load" + except HTTPException as e: + assert e.status_code == 409 + _start_subprocess = real_start + assert active() is None + # in-process model: close must POST /unload to the worker so its resident VRAM is freed. calls = [] requests.post = lambda url, **kw: calls.append(url) or type("R", (), {"status_code": 200})() diff --git a/worker_scene.py b/worker_scene.py index 5829581..11f0110 100644 --- a/worker_scene.py +++ b/worker_scene.py @@ -70,20 +70,32 @@ def build_scene(data: SceneInput): # Speaker attribution is the vision/dialogue model's job now (it reads bubble tails + turn-taking). # speaker is a local_id -> map to character_id via identity; None for narration/sfx/off-panel. + # The orchestrator's normalize_dialogue already rewrote `speaker` to a character_id and put the + # typed answer in `speaker_ref`, so the local-id map no longer matches it. Read speaker_ref FIRST; + # every lookup used to miss and every line narrated as "Someone". dialogue = [] for d in data.vision_result.get("dialogue", []): raw = d.get("speaker") + ref = d.get("speaker_ref") or {} # "unknown" is a DELIBERATE off-panel/indeterminate speaker (narrated as "someone"); it maps # to None just like narration. id_by_local.get already yields None for it. - dialogue.append({"speaker": id_by_local.get(raw), "text": d.get("text", ""), + if ref.get("kind") == "character_id" and ref.get("value"): + cid = ref["value"] + else: + cid = id_by_local.get(raw) or (raw if raw in name_by_id else None) + dialogue.append({"speaker": cid, "text": d.get("text", ""), "type": d.get("type", "speech"), "confidence": d.get("confidence"), "speaker_method": d.get("speaker_method", "unknown")}) vchars = data.vision_result.get("characters", []) - action = "; ".join(c.get("action", "") for c in vchars if c.get("action")) or "" + # `actions` is the list the orchestrator's beat builder reads for verifier evidence; `action` is the + # joined string the script prompt renders. Emitting only the string left verification blind. + actions = [c["action"].strip() for c in vchars if str(c.get("action") or "").strip()] + action = "; ".join(actions) return {"panel_id": data.panel_id, "characters": characters, "dialogue": dialogue, - "action": action, "entities": data.vision_result.get("entities", []), + "action": action, "actions": actions, + "entities": data.vision_result.get("entities", []), "camera": data.vision_result.get("camera", {}), # #8 direction passthrough -> render "transition": data.vision_result.get("transition", "cut")} # #6 transition out @@ -167,4 +179,21 @@ if __name__ == "__main__": )) assert out5["dialogue"][0]["confidence"] == 0.3 assert out5["dialogue"][0]["speaker_method"] == "turn_taking" + + # orchestrator-normalized rows: `speaker` is already a character_id and `speaker_ref` is typed. + # the local-id map cannot resolve either — speaker_ref must win, or everything narrates as Someone. + out6 = build_scene(SceneInput( + panel_id="p007", + vision_result={"characters": [{"local_id": "person_1", "bbox": [0, 0, 10, 10]}], + "dialogue": [{"speaker": "c1", "speaker_ref": {"kind": "character_id", "value": "c1"}, + "type": "speech", "text": "Hey."}, + {"speaker": None, "speaker_ref": {"kind": "unknown", "value": None}, + "type": "speech", "text": "..."}]}, + identity_assignments=[{"local_id": "person_1", "character_id": "c1"}], + characters_registry=[{"character_id": "c1", "name": "Teto"}], + )) + assert out6["dialogue"][0]["speaker"] == "c1", out6["dialogue"][0] + assert out6["dialogue"][1]["speaker"] is None, out6["dialogue"][1] + # actions survive as a list for the verifier, not only as the joined prompt string + assert out["actions"] == ["waving"] and out["action"] == "waving" print("worker_scene self-check ok") diff --git a/worker_script.py b/worker_script.py index 888f2a9..31522de 100644 --- a/worker_script.py +++ b/worker_script.py @@ -21,6 +21,10 @@ class ScriptInput(BaseModel): recent: list = [] # #1: last few panels' narration text (local flow, no repeats) introduced: list = [] # #2: names already narrated -> refer by name/pronoun, don't re-describe panel_count: int = 1 # scene-level narration: #panels in this beat -> scales the length budget + # the orchestrator has been sending both of these on a failed-script retry since the verifier + # landed; pydantic dropped them silently, so the retry was another blind roll of the dice. + beat: dict = {} # {beat_id, panels:[{panel_id, dialogue:[{text}], actions}]} — the evidence + verifier_feedback: list = [] # [{kind, values|quote}] from verify_script on the previous attempt class SummaryInput(BaseModel): @@ -80,9 +84,33 @@ def _chars_line(name_by_id, genders_by_id): return ", ".join(out) or "none" +def _feedback_block(feedback, beat) -> str: + """Render the previous attempt's verifier failures as concrete corrections. Without this the + retry is just another sample at the same temperature — the model never learns what was wrong.""" + if not feedback: + return "" + lines = [] + for f in feedback: + kind = f.get("kind") + if kind == "unsupported-proper-noun": + lines.append("- You used name(s) that are not in this beat: " + + ", ".join(str(v) for v in f.get("values", [])) + + ". Use only the names listed under 'Characters present', or a pronoun.") + elif kind == "misquote": + lines.append(f'- Your quote "{f.get("quote","")}" is not what the panel says. ' + "Quote the exact words below, or drop the quotation marks.") + else: + lines.append(f"- {kind}: {json.dumps(f, ensure_ascii=False)[:200]}") + quotes = [d.get("text", "") for p in (beat or {}).get("panels", []) + for d in p.get("dialogue", []) if d.get("text")] + exact = ("Exact lines you may quote from:\n" + "\n".join(f'- "{q}"' for q in quotes) + "\n") if quotes else "" + return ("Your previous attempt was REJECTED. Fix exactly these problems and rewrite it:\n" + + "\n".join(lines) + "\n" + exact + "\n") + + def build_prompt(sg: dict, chapter_context: str, names_by_id=None, brief: str = "", recent=None, introduced=None, genders_by_id=None, - panel_count: int = 1) -> str: + panel_count: int = 1, beat=None, verifier_feedback=None) -> str: name_by_id = _name_map(sg.get("characters", []), names_by_id) chars = _chars_line(name_by_id, genders_by_id) dialogue = "\n".join(_render_line(d, name_by_id) for d in sg.get("dialogue", [])) or "none" @@ -120,7 +148,7 @@ def build_prompt(sg: dict, chapter_context: str, names_by_id=None, else: unit, budget, thin = "this manga panel", "1-2 short sentences, max ~25 words", "panel" return ( - ov + rec + ctx + intro + + _feedback_block(verifier_feedback, beat) + ov + rec + ctx + intro + f"Write TIGHT recap narration for {unit}: {budget}, " "present tense. Tell it like you're recapping to a friend — natural, with momentum. Output " "only the narration — no preamble, no notes, no alternatives, never ask for more info. If the " @@ -182,7 +210,7 @@ async def summary(data: SummaryInput): async def script(data: ScriptInput): text = call_gemma4(build_prompt(data.scene_graph, data.chapter_context, data.names_by_id, data.brief, data.recent, data.introduced, data.genders_by_id, - data.panel_count)) + data.panel_count, data.beat, data.verifier_feedback)) return {"panel_id": data.panel_id, "text": text} @@ -199,10 +227,16 @@ class NormalizeInput(BaseModel): def _extract_json(raw: str) -> dict: text = _strip_thought(raw) - m = re.search(r"\{.*\}", text, re.DOTALL) - if not m: + i = text.find("{") + if i < 0: raise ValueError(f"no json: {text[:200]}") - return json.loads(m.group(0)) + # raw_decode stops at the end of the FIRST object. The old greedy `\{.*\}` swallowed any trailing + # brace in prose after it and turned a parseable reply into a wasted repair call. + try: + obj, _ = json.JSONDecoder().raw_decode(text[i:]) + except json.JSONDecodeError as e: + raise ValueError(f"bad json: {text[:200]}") from e + return obj def build_normalize_prompt(names, entities) -> str: @@ -297,4 +331,12 @@ if __name__ == "__main__": nr = _extract_json('{"entities":[{"name":"Everyday","kind":"shop"}],' '"name_fixes":{"CHOI HAESEON":"Choi Haeseon"}}') assert nr["entities"][0]["name"] == "Everyday" and nr["name_fixes"]["CHOI HAESEON"] == "Choi Haeseon" + # trailing braced prose after the object must not break the parse (was a wasted repair call) + assert _extract_json('{"a":1}\nnote: {not json}')["a"] == 1 + # verifier feedback reaches the retry prompt with the exact allowed quotes; absent by default + assert "REJECTED" not in p + beat = {"beat_id": "b1", "panels": [{"panel_id": "p1", "dialogue": [{"text": "stand proud"}]}]} + pf = build_prompt(sg, "s", beat=beat, verifier_feedback=[ + {"kind": "unsupported-proper-noun", "values": ["Kyoto"]}, {"kind": "misquote", "quote": "be proud"}]) + assert "REJECTED" in pf and "Kyoto" in pf and "be proud" in pf and '- "stand proud"' in pf print("worker_script self-check ok") diff --git a/worker_vision.py b/worker_vision.py index 698897b..68ad8df 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -19,7 +19,9 @@ GEMMA4_URL = os.environ.get("GEMMA4_URL", "http://127.0.0.1:8090") # set-of-mark speaker attribution: detect text regions with comic-text-detector (ONNX, CPU) and draw # numbered boxes on the panel so gemma transcribes/attributes grounded regions instead of eyeballing -# the whole image. Off by default — flip SOM_ATTRIBUTION=1 once tuned per title (see bubble_detect.py). +# the whole image. ON by default: face->identity pairing is now gated on containment, so an unmatched +# face is labelled "unknown" instead of borrowing the nearest name (see _pair_faces_to_present). +# SOM_ATTRIBUTION=0 falls back to the holistic path (see bubble_detect.py). try: import bubble_detect except Exception: # onnxruntime/model absent -> feature simply stays unavailable @@ -31,29 +33,45 @@ except Exception: SOM = os.environ.get("SOM_ATTRIBUTION", "1") == "1" -def _pair_faces_to_present(det_faces: list, present: list) -> list: +def _pair_faces_to_present(det_faces: list, present: list, margin: float = 0.25) -> list: """det_faces = real detector boxes (grounded but identity-less); present = characters gemma placed - in the panel (identity + a coarse, imprecise bbox). Assign each detected face the identity of the - present character whose gemma-bbox centre falls closest to it, so a green box carries a real name. - A face with no nearby present char stays unknown; a present char is used at most once.""" + in the panel (identity + a coarse, imprecise bbox). Pair a face with a present character ONLY when + the face centre falls inside that character's bbox grown by `margin` of its size — gemma's boxes are + imprecise but not arbitrary. An unpaired face stays unknown instead of borrowing whoever happens to + be nearest; the label it carries becomes `speaker_method="som_face"`, the highest-trust provenance + the pipeline has, so an ungated guess used to launder itself into evidence. + Pairs are taken globally shortest-first, so the first face processed cannot claim a character that + fits a later face far better. Each face and each character is used at most once.""" def cx_cy(b): return ((b[0] + b[2]) / 2, (b[1] + b[3]) / 2) - used = set() - out = [] - for i, f in enumerate(det_faces, 1): + + def contains(face_box, char_box) -> bool: + w, h = char_box[2] - char_box[0], char_box[3] - char_box[1] + if w <= 0 or h <= 0: + return False + fx, fy = cx_cy(face_box) + return (char_box[0] - margin * w <= fx <= char_box[2] + margin * w + and char_box[1] - margin * h <= fy <= char_box[3] + margin * h) + + pairs = [] + for i, f in enumerate(det_faces): fx, fy = cx_cy(f["bbox"]) - best, bestd = None, 1e18 for j, c in enumerate(present): b = c.get("bbox") - if not b or j in used: + if not b or len(b) != 4 or not contains(f["bbox"], b): continue px, py = cx_cy(b) - d = (px - fx) ** 2 + (py - fy) ** 2 - if d < bestd: - best, bestd = j, d - c = present[best] if best is not None else {} - if best is not None: - used.add(best) - out.append({"label": f"P{i}", "bbox": f["bbox"], "local_id": c.get("local_id"), + pairs.append(((px - fx) ** 2 + (py - fy) ** 2, i, j)) + taken_f, taken_c, match = set(), set(), {} + for _, i, j in sorted(pairs): + if i in taken_f or j in taken_c: + continue + taken_f.add(i) + taken_c.add(j) + match[i] = j + out = [] + for i, f in enumerate(det_faces): + c = present[match[i]] if i in match else {} + out.append({"label": f"P{i + 1}", "bbox": f["bbox"], "local_id": c.get("local_id"), "who": c.get("name") or c.get("desc") or "unknown", "gender": c.get("gender")}) return out @@ -127,10 +145,17 @@ def _strip_thought(text: str) -> str: def _extract_json(raw: str) -> dict: """strip gemma4 thought, pull the first JSON object, parse it.""" text = _strip_thought(raw) - m = re.search(r"\{.*\}", text, re.DOTALL) - if not m: + i = text.find("{") + if i < 0: raise ValueError(f"no json in response: {text[:200]}") - return json.loads(m.group(0)) + # raw_decode stops at the end of the FIRST object. The old greedy `\{.*\}` ran to the LAST brace in + # the reply, so a second object or any trailing braced prose produced an unparseable span and burned + # a repair call on a response that was already fine. + try: + obj, _ = json.JSONDecoder().raw_decode(text[i:]) + except json.JSONDecodeError as e: + raise ValueError(f"bad json in response: {text[:200]}") from e + return obj def _img_part(image_path: str) -> dict: @@ -893,10 +918,18 @@ async def vision_resolve(data: ResolveInput): os.remove(crop) for _, path in refs: os.remove(path) - # map gemma's 1-based choice back to a character_id; 0 / out-of-range -> NONE (new character). + # map gemma's 1-based choice back to a character_id. An explicit 0 means NONE -> a new character. + # An OUT-OF-RANGE index is a hallucination, not an answer: it must be `unresolved` like a parse + # failure, or a bad index silently mints a brand new character in the permanent registry. choice = result.get("choice", 0) - cid = data.candidates[choice - 1]["character_id"] if isinstance(choice, int) and 1 <= choice <= len(data.candidates) else None - state = "known" if cid else ("unresolved" if result.get("reason") == "parse_failed" else "new") + in_range = isinstance(choice, int) and 1 <= choice <= len(data.candidates) + cid = data.candidates[choice - 1]["character_id"] if in_range else None + if in_range: + state = "known" + elif choice == 0 and result.get("reason") != "parse_failed": + state = "new" + else: + state = "unresolved" return {"character_id": cid, "state": state, "confidence": float(result.get("confidence", 0.0)), "reason": result.get("reason", "")} @@ -1005,4 +1038,20 @@ if __name__ == "__main__": assert False except ValueError: pass + # trailing braced prose after a complete object parses (used to burn a repair call) + assert _extract_json('{"skip":false}\nnote {see above}')["skip"] is False + + # face->identity pairing is GATED on containment: a face outside every gemma bbox stays unknown. + faces = [{"bbox": [10, 10, 30, 30]}, {"bbox": [900, 900, 920, 920]}] + present = [{"local_id": "person_1", "name": "Teto", "bbox": [0, 0, 100, 200]}] + paired = _pair_faces_to_present(faces, present) + assert paired[0]["local_id"] == "person_1" and paired[0]["who"] == "Teto" + assert paired[1]["local_id"] is None and paired[1]["who"] == "unknown", paired[1] + # globally shortest-first: the first face must not claim a character that fits the second better. + faces2 = [{"bbox": [95, 95, 105, 105]}, {"bbox": [8, 8, 12, 12]}] + present2 = [{"local_id": "a", "bbox": [0, 0, 20, 20]}, {"local_id": "b", "bbox": [80, 80, 120, 120]}] + p2 = _pair_faces_to_present(faces2, present2) + assert [f["local_id"] for f in p2] == ["b", "a"], p2 + # no present characters at all -> every face unknown, never a phantom identity + assert _pair_faces_to_present(faces, [])[0]["local_id"] is None print("worker_vision self-check ok") -- 2.52.0 From b2cd11dd1ca7f7014ec75cde55a1f59fd536224f Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 22:05:23 +0400 Subject: [PATCH 04/31] Record the #116/#117 storage and viewer work #117 done: stowage was dead on an arm64 digest pin, not a MinIO fault. #116 staged: rustfs runs on 9010/9011, buckets not mirrored, no cutover. Also logs the ISP port 80/443 interception that made three external reachability measurements worthless, so the next session does not repeat them. Co-Authored-By: Claude Opus 5 --- JOURNAL.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ NEXT.md | 23 ++++++++++++++++++----- 2 files changed, 65 insertions(+), 5 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 6d90e68..fadcf44 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -20,3 +20,50 @@ Produced: `decisions/audit-phase1.md`, `caveats/audit-open.md`, `ROADMAP.md`, an One existing test asserted the bug: `test_name_binding.test_conflict_flags_and_stays_unnamed` relied on orphan flags leaking into every chapter, because it never created panel rows. It now creates them. + +## 2026-08-11 Orchestrator half committed and deployed [no task] + +Command: `pytest -q --ignore=test_api.py`, then `docker compose up -d --build orchestrator` on homesrv. +Outcome: finished. 108 tests pass. Commits `1c60710` (orchestrator half) and `94bd4d8` (minio pin) in +`/mnt/server/home/kami/docker-apps`. Orchestrator and minio both answer health on homesrv. + +The rebuild recreated `minio` as a side effect and it crash-looped with `exec format error`: the +compose pin was the arm64 manifest digest of `minio/minio:latest` and homesrv is amd64. Repinned to the +amd64 digest. Nothing about Phase 1 caused this, but any compose action that recreates minio would have +hit it, so it was latent, not new. + +Still unrun against a real chapter. + +## 2026-08-11 S3 viewer and storage swap, tasks #116/#117 [#116 #117] + +Command: docker compose on homesrv, `dig`, `openssl s_client`. No pipeline, no GPU. +Outcome: partial. Viewer works, storage swap staged and unfinished. + +#117 needed no new software. `stowage` at `~/docker-apps/stowage` was already configured against the +manga MinIO and had been dead since 2026-07-19 with `exec /sbin/tini: exec format error`: its digest +pin was the arm64 manifest. Repinned to amd64 `sha256:91be7f13`, chowned `data/` to uid 65532 for the +new image, and it serves. MinIO had the identical bug, repinned to `sha256:a1a8bd4a`. A sweep of all +470 local images on homesrv found exactly those two arm64; nothing else in the homelab is affected. + +#116 is staged, not done. `rustfs` runs alongside MinIO on `127.0.0.1:9010/9011`, pinned +`sha256:19b105cc`, data at `/mnt/hdd2/rustfs`. Buckets are empty: the `mc` mirror of +`audio layers manga panels raw video` (350M, all in `manga`) has NOT run. `/mnt/hdd2/minio/data` is +untouched and is the rollback. RustFS is `1.0.0-beta.12`, labeled `build-type=prerelease`. Cutover +would give rustfs 9000/9001 and repoint `MINIO_ENDPOINT=minio:9000` in the orchestrator plus +`stowage/config.yaml`; `transport.py:95` needs no change if rustfs takes `192.168.1.104:9000`. + +Side quest, unrelated to the pipeline: the shared 41-domain cert stopped renewing. Root cause was DNS, +not nginx. Every `*.kvmx.ru` name pointed at a hard A record for `109.229.102.117` while the line had +moved to `109.229.127.149`; the Mercusys DDNS at `kvmx-home.mercusysddns.com` was correct the whole +time but nothing in the zone referenced it. Fixed with `CNAME * -> kvmx-home.mercusysddns.com` at +reg.ru. Certificate now issues. + +Two measurement traps worth remembering. The ISP transparently intercepts ports 80 and 443 by +Host/SNI, so `curl` from workpc to ANY address returns kvmx.ru content and proves nothing about +external reachability; bare TCP connects also succeed against arbitrary addresses and then hang. Three +wrong root causes came out of trusting those probes before checking them. + +Also patched `~/scripts/migrate-kvmx-https.sh:54` on homesrv. `need_stream_module` used +`sudo -n nginx -V` and `sudo -n nginx -T`; the NOPASSWD rule covers only `nginx -t`, so it reported +"stream module is not loaded" whenever it meant "could not ask for a password". Both checks now run +without sudo. `bash -n` passes and both conditions evaluate true. diff --git a/NEXT.md b/NEXT.md index 99a4bca..b7aa0fd 100644 --- a/NEXT.md +++ b/NEXT.md @@ -17,17 +17,30 @@ Verification: CPU-only self-checks and the orchestrator test suite. 108 orchestr (`test_api.py` is excluded on workpc because fastapi is not installed in this venv). No GPU work ran and no pipeline ran, so none of this is confirmed against a real chapter. -The orchestrator changes are edited in place on the SSHFS mount and are NOT committed. Its git root is -`/mnt/server/home/kami/docker-apps`. Its container also needs a rebuild or restart to pick them up. +The orchestrator half is committed as `1c60710` in `/mnt/server/home/kami/docker-apps` and the +container is rebuilt and serving. `94bd4d8` in the same repo repins minio to its amd64 digest, which +the rebuild exposed. ## Next -1. Commit the orchestrator half in `/mnt/server/home/kami/docker-apps` and restart the container. -2. Run one labeled chapter end to end and record the baseline numbers from `ROADMAP.md`, especially the +1. Run one labeled chapter end to end. Record the baseline numbers from `ROADMAP.md`, above all the share of narrated lines with a named speaker. That number is the check on the largest Phase 1 fix. -3. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work +2. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work (`caveats/audit-open.md#sqlite-locking`). +## Storage and viewer, tasks #116/#117 + +[#117] is done. `stowage` serves the manga buckets. It was never a MinIO problem: the container had +been dead since 2026-07-19 on an arm64 digest pin. Details in `JOURNAL.md`. + +[#116] is staged and unfinished. `rustfs` runs on `127.0.0.1:9010/9011` with empty buckets. The next +step is the `mc` mirror of the six buckets, then verify object counts and sizes against MinIO, then +decide on cutover. Nothing is repointed yet and MinIO still serves every read and write. + +Two things to weigh before cutover, neither settled. RustFS is `1.0.0-beta.12`, labeled +`build-type=prerelease`. Swapping storage before the baseline chapter run also adds a variable to the +run that is meant to produce the baseline. + ## Open questions Four Phase 1 items have no Vikunja task and were not created, because writing to the tracker was not -- 2.52.0 From 54c456801adf77bd55a2d0c73d92fc87a388147b Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:00:13 +0400 Subject: [PATCH 05/31] Split artifacts across per-class buckets, record the baseline run Panels, wavs, layers, clips, and the chapter mp4 leave the `manga` bucket for `panels`, `audio`, `layers`, and `video`. The key under the bucket is unchanged, so every reader that derives the bucket from the first path segment keeps working. The orchestrator half moves in the same commit, per invariant 7. The 2026-08-11 chapter run proves the split for `raw` and `panels` and produced the first quality read on speaker attribution, which is wrong in every sampled multi-character panel. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01XD7cAy81MZrc7gCr6aZGWr --- HANDOFF.md | 65 ++++++++++++++++++++++++++++++++++ JOURNAL.md | 30 ++++++++++++++++ NEXT.md | 37 ++++++++++++++----- caveats/CLAUDE.md | 3 ++ caveats/speaker-attribution.md | 55 ++++++++++++++++++++++++++++ decisions/CLAUDE.md | 3 ++ decisions/storage-layout.md | 58 ++++++++++++++++++++++++++++++ worker_crop.py | 4 +-- worker_layers.py | 2 +- worker_render.py | 12 +++---- worker_tts.py | 4 +-- 11 files changed, 253 insertions(+), 20 deletions(-) create mode 100644 HANDOFF.md create mode 100644 caveats/speaker-attribution.md create mode 100644 decisions/storage-layout.md diff --git a/HANDOFF.md b/HANDOFF.md new file mode 100644 index 0000000..01fb719 --- /dev/null +++ b/HANDOFF.md @@ -0,0 +1,65 @@ +# HANDOFF, 2026-08-11 session + +Live state lives in `NEXT.md`. This file is only what this session did. + +## Asked + +1. Add the buckets to rustfs. +2. Fix the bucket saving on the orchestrator side. +3. Run the title recap. + +Mid-session: check `manga.kvmx.ru`, and cross-check whether the run got dialogue and identities right. + +## Changed + +Workers, `/home/kami/Programs/n8n-worker`, branch `restore-runtime`: + +- `worker_crop.py` panels -> `s3://panels/` +- `worker_tts.py` wavs -> `s3://audio/` +- `worker_layers.py` layers -> `s3://layers/` +- `worker_render.py` clips and `chapter.mp4` -> `s3://video/` +- `decisions/storage-layout.md`, `caveats/speaker-attribution.md`, indexes, `JOURNAL.md`, `NEXT.md` + +Orchestrator, `/mnt/server/home/kami/docker-apps/manga-infra/orchestrator`: + +- `minio_layout.py` per-artifact bucket constants, `BUCKETS`, `parse_key` accepts any of them +- `service.py` `_s3_delete_prefix` takes `/`, `_stage_s3_prefixes` no longer slices, + new `_ensure_buckets` in the lifespan +- `test_minio_layout.py` updated, one test added + +Infrastructure, homesrv: + +- six buckets created on `rfs` (rustfs, `127.0.0.1:9010`) +- `manga-fetch` started, exited 2 weeks, `/job/create` fails without it +- `manga-web` started, exited 2 weeks, `manga.kvmx.ru` -> nginx -> `localhost:8083` +- orchestrator rebuilt and restarted +- `mc` aliases `mio` and `rfs` now exist on homesrv + +## Measured + +Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, +"Teto X Egen", 116 panels. + +- 109 orchestrator tests pass. `worker_crop.py`, `worker_tts.py`, `worker_render.py` self-checks pass. + `worker_layers.py` self-check fails on the missing `legacy/qwen_layered_workflow.json`, which + predates this session. +- buckets after the run: `raw` 64MiB/79, `panels` 101MiB/116, `manga` 366MiB/491, `audio` `layers` + `video` still empty because the run had not reached those stages. +- stage times: crop 85s, vision ~4min, identity ~1min, reconcile ~7min, dialogue ~8min. +- 24 of 81 speech lines resolve to a named character, 30%. +- 26 of 113 detected people got an identity, 23%, and 25 of the 26 went to one character. +- 3 of 3 sampled two-character panels attribute both speakers to the wrong person. + +## Open + +- The run stopped in `direct` at 113/116. `scene script tts layers render assemble` never ran. Resume: + + ```bash + ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" + ``` + +- Speaker attribution is wrong in multi-character panels and the fix is not written + (`caveats/speaker-attribution.md`). No code was changed for it. +- Objects from the 2026-07-17 run still sit under `manga///{pages,panels,audio, + layers,clips}`. Nothing reads them. They are the rollback, not live data. +- `rustfs` holds empty buckets. No mirror, no cutover. diff --git a/JOURNAL.md b/JOURNAL.md index fadcf44..1251b6b 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -67,3 +67,33 @@ Also patched `~/scripts/migrate-kvmx-https.sh:54` on homesrv. `need_stream_modul `sudo -n nginx -V` and `sudo -n nginx -T`; the NOPASSWD rule covers only `nginx -t`, so it reported "stream module is not loaded" whenever it meant "could not ask for a password". Both checks now run without sudo. `bash -n` passes and both conditions evaluate true. + +## 2026-08-11 Per-artifact buckets, rustfs buckets, baseline chapter run [#116] + +Command: `mc mb` on rustfs, `docker compose up -d --build orchestrator`, `pytest -q --ignore=test_api.py`, +`./start_workers.sh`, then `/job/create` + `/stage/clear` + `/job/resume` for chapter +`7c944dd4-e972-42c7-ba60-9f6939548e80` of "Teto X Egen" as job `778297bc-e7ce-439d-91b5-8a027060d17f`. +Outcome: partial. Storage split landed and is proven by the run. The run itself was still in `direct` +when the session ended. +Produced: `decisions/storage-layout.md`, `caveats/speaker-attribution.md`, 109 orchestrator tests pass. + +Artifacts now split one bucket per class instead of everything under `manga` +(`decisions/storage-layout.md#bucket-per-artifact`). Both MinIO and rustfs hold all six buckets. The +run put 79 pages in `raw` and 116 panel crops in `panels`, so the split works end to end. + +Two containers on homesrv had been dead for two weeks and blocked the work. `manga-fetch` was exited, +so `/job/create` failed with `httpx.ConnectError`; `manga-web` was exited, so `manga.kvmx.ru` had +nothing behind it on port 8083. Both started with `docker compose up -d`. Neither is related to the +storage change. Neither was caught by any check, because nothing watches these containers. + +Stage timings, 116 panels: crop 85s, vision ~4min, identity ~1min, reconcile ~7min for 35 pairs, +dialogue ~8min. Faster than the 2026-07-17 run at 75 panels. The webtoon crop that 500'd in July +succeeded this time. + +Quality cross-check against the panel images, the point of the run. Dialogue text extraction is +accurate. Character detection is accurate. Speaker attribution is not: three of three sampled +two-character panels attribute both speakers to the wrong person, always swapped +(`caveats/speaker-attribution.md#tail-is-not-geometry`). 24 of 81 speech lines resolve to a named +character, which is the Phase 1 headline metric at 30%, and the sample says that 30% is not +trustworthy. 26 of 113 detected people got an identity, and 25 of those 26 went to one character that +turns out to cover two different women. diff --git a/NEXT.md b/NEXT.md index b7aa0fd..7bdc421 100644 --- a/NEXT.md +++ b/NEXT.md @@ -23,23 +23,42 @@ the rebuild exposed. ## Next -1. Run one labeled chapter end to end. Record the baseline numbers from `ROADMAP.md`, above all the - share of narrated lines with a named speaker. That number is the check on the largest Phase 1 fix. -2. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work +1. Fix speaker attribution. The baseline run has been made and the metric it produced cannot be + trusted (`caveats/speaker-attribution.md#tail-is-not-geometry`). Three of three sampled + two-character panels swap the speakers, and the `tail` provenance label is stamped on guesses at + confidence 1.0. Smallest honest first step: stop labelling a guess `tail`, and return `unknown` + when two or more characters are present. +2. Rerun the chapter and re-read the named-speaker share. Only then is the Phase 1 headline number + real. +3. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work (`caveats/audit-open.md#sqlite-locking`). +## The 2026-08-11 chapter run + +Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 +panels. It was still in `direct` at 113/116 when the session ended, with `scene script tts layers +render assemble` unrun. Resume it, or read where it got to: + +```bash +ssh kami@192.168.1.104 "curl -s 'http://127.0.0.1:9090/job/status?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" +ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" +``` + +Numbers and the quality read are in `JOURNAL.md` and `caveats/speaker-attribution.md`. + ## Storage and viewer, tasks #116/#117 [#117] is done. `stowage` serves the manga buckets. It was never a MinIO problem: the container had been dead since 2026-07-19 on an arm64 digest pin. Details in `JOURNAL.md`. -[#116] is staged and unfinished. `rustfs` runs on `127.0.0.1:9010/9011` with empty buckets. The next -step is the `mc` mirror of the six buckets, then verify object counts and sizes against MinIO, then -decide on cutover. Nothing is repointed yet and MinIO still serves every read and write. +[#116] is closer but not cut over. Artifacts now split one bucket per class +(`decisions/storage-layout.md#bucket-per-artifact`), and both MinIO and `rustfs` hold all six buckets. +`rustfs` on `127.0.0.1:9010/9011` is still empty and nothing is repointed, so MinIO serves every read +and write. Remaining: `mc mirror` the live buckets, verify counts and sizes, then decide on cutover +(`decisions/storage-layout.md#rustfs-staged`). -Two things to weigh before cutover, neither settled. RustFS is `1.0.0-beta.12`, labeled -`build-type=prerelease`. Swapping storage before the baseline chapter run also adds a variable to the -run that is meant to produce the baseline. +Two containers on homesrv had been dead for two weeks and are now running. `manga-fetch` is the one +`/job/create` needs. `manga-web` is what `manga.kvmx.ru` proxies to on 8083. Nothing watches them. ## Open questions diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index 459f244..eb56780 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -32,3 +32,6 @@ a complaint, so give it one or drop it. | [MinIO credentials are hardcoded in committed source](audit-open.md#hardcoded-credentials) | AUDIT.md | | [Assemble marks a job completed with no clips](audit-open.md#empty-assemble) | AUDIT.md | | [Reviewer timestamps drift against the crossfaded video](audit-open.md#timeline-drift) | AUDIT.md | +| [`speaker_method="tail"` never reads a tail](speaker-attribution.md#tail-is-not-geometry) | 2026-08-11 run | +| [One character id covers two different women](speaker-attribution.md#identity-over-merge) | 2026-08-11 run | +| [The character registry carries five weeks of wrong names](speaker-attribution.md#registry-pollution) | 2026-08-11 run | diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md new file mode 100644 index 0000000..72331c8 --- /dev/null +++ b/caveats/speaker-attribution.md @@ -0,0 +1,55 @@ +# speaker-attribution + +Limits found by cross-checking the 2026-08-11 chapter run against the panel images. + +## `speaker_method="tail"` never reads a tail {#tail-is-not-geometry} + +`worker_vision.py:376` (`_annotate_speaker_methods`) stamps `tail` on any line whose `speaker` matches +a `local_id` present in the panel. No balloon geometry is consulted. The caller keeps gemma's +confidence, usually 1.0. An unverified model guess thus carries the highest-trust provenance in the +pipeline. The review UI and the flag rules both believe it. + +Measured on job `778297bc`, chapter `7c944dd4`: 31 of 81 speech lines are `tail` with two or more +characters present. Three two-character panels were checked against the art, and all three are wrong, +each with the two speakers swapped: + +| panel key | line | truth | pipeline | +| --- | --- | --- | --- | +| `p010.png` | "…definitely an Egen guy, Seonho!" | the woman | Seonho, the person addressed | +| `p010.png` | "Y-you think so?" | Seonho | Choi Haeseon | +| `p012.png` | "Want me to send you the link?" | the woman | the man | +| `p059.png` | "If team leader Choi says it, it must be true." | the man | Choi Haeseon | + +The last row needs no image: the line refers to Choi in the third person and is attributed to Choi. + +The grounded path exists and almost never fires. Only 2 of 81 speech lines got `som_face`, because +attribution marks need `face_detect` boxes that survive `_pair_faces_to_present`, and these webtoon +close-ups rarely produce them. Inference, not measured: the face detector was not instrumented. + +`worker_vision.py:356` already carries the `ponytail:` note that multi-character attribution needs +per-balloon geometry. `bubble_detect.py:9` records that the `det`/`seg` heads carry balloon fill and +tail tips and are unused. + +**Revisit trigger:** the share of narrated lines with a named speaker is the Phase 1 headline metric +(`ROADMAP.md`). It reads 30% on this run, and the sample says that 30% is itself unreliable. The +metric cannot be trusted until this is fixed. Fix order: stop stamping a guess as `tail` at confidence +1.0. Prefer `unknown` when two or more characters are present. Then bind by tail geometry. + +## One character id covers two different women {#identity-over-merge} + +`character_afa7623b` is stored as "black bob, white sweater" and is assigned both to that person +(`p059.png`) and to the brown-bob green-top coworker (`p010.png`, `p012.png`). It took 25 of the 26 +identity assignments in the chapter, against 113 detected people. Coverage is 23%. + +**Revisit trigger:** any work on the identity Tier-2 decider. A single id absorbing a whole chapter is +the signature to watch for. + +## The character registry carries five weeks of wrong names {#registry-pollution} + +The registry holds 53 characters for manga `ef105a86`, 41 of them unnamed, with "Kei" three times and +"Kanade" twice. Kei, Kanade, Zen, Rico, K3, and Watanabe occur zero times in this chapter's text. Only +Haeseon and Seonho do. `/stage/clear` leaves the per-manga registry intact by design, so every rerun +inherits the whole pile. + +**Revisit trigger:** before any run that is meant to produce a clean baseline. Either scope the +registry to a chapter or add a reviewed reset. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 0dabe16..319ef47 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -27,3 +27,6 @@ still live belongs in `caveats/`. | [Correctness flags resolve by flag id, and never block autonomous TTS](audit-phase1.md#flag-resolution) | closed | | [An out-of-range resolver index is `unresolved`, never a new character](audit-phase1.md#hallucinated-index) | closed | | [The session manager holds no lock across a model load](audit-phase1.md#unlocked-model-load) | closed | +| [One bucket per artifact class, not everything under `manga`](storage-layout.md#bucket-per-artifact) | closed | +| [The orchestrator creates missing buckets at startup](storage-layout.md#ensure-buckets) | closed | +| [RustFS is staged, not adopted](storage-layout.md#rustfs-staged) | open | diff --git a/decisions/storage-layout.md b/decisions/storage-layout.md new file mode 100644 index 0000000..0a97a88 --- /dev/null +++ b/decisions/storage-layout.md @@ -0,0 +1,58 @@ +# storage-layout + +Settled questions about which S3 bucket holds what, and about the MinIO replacement. + +## One bucket per artifact class {#bucket-per-artifact} + +**State: closed. 2026-08-11.** + +Six buckets were created on 2026-07-04 (`manga raw panels audio layers video`). Only `manga` ever +received an object, because `minio_layout.py` hardcoded `BUCKET = "manga"` and four workers built +their own keys as literal `s3://manga/...`. The other five sat empty for five weeks. + +Artifacts now split by class. The key under the bucket is unchanged, so only the leading segment moved: + +| artifact | bucket | +| --- | --- | +| fetched pages | `raw` | +| panel crops | `panels` | +| tts wavs | `audio` | +| layer pngs | `layers` | +| clips and `chapter.mp4` | `video` | +| vision, identity, scene, script json, character registry | `manga` | + +Orchestrator: `minio_layout.py` gained `BUCKET_RAW`/`BUCKET_PANELS`/`BUCKET_AUDIO`/`BUCKET_LAYERS`/ +`BUCKET_VIDEO` and a `BUCKETS` tuple. `parse_key` accepts any of them and rejects anything else. +`service.py:_s3_delete_prefix` takes a `/` pair instead of assuming one bucket, and +`_stage_s3_prefixes` stops slicing the bucket off. Workers: `worker_crop.py`, `worker_tts.py`, +`worker_layers.py`, `worker_render.py`. + +Every S3 URI is `s3://///...` and every consumer already derives the +bucket from the first path segment, so no reader needed a change. + +What this forbids: writing an artifact under a bucket that is not in `BUCKETS`. `parse_key` returns +`{}` for one, and stage clearing would then silently delete nothing. + +Objects written before this date stay under `manga/` at their old keys. Nothing reads them any more: +they are the rollback for the 2026-07-17 run, not live data. + +Evidence: `test_minio_layout.py` (31 tests), and the 2026-08-11 chapter run, which put pages in `raw` +and 116 panel crops in `panels`. + +## The orchestrator creates missing buckets at startup {#ensure-buckets} + +**State: closed. 2026-08-11.** + +Workers create a bucket on first write (`transport.py:115`), but the orchestrator uploads pages before +any worker runs and boto3 will not auto-create. `service.py:_ensure_buckets` runs in the FastAPI +lifespan and creates whatever is missing. A storage backend that is down at boot logs a warning +instead. The check is not worth a failed start. + +## RustFS is staged, not adopted {#rustfs-staged} + +**State: open. 2026-08-11.** + +`rustfs` holds all six buckets on `127.0.0.1:9010/9011`, all empty. MinIO still serves every read and +write. Nothing is repointed. Two things still block a cutover, and neither is settled. RustFS is +`1.0.0-beta.12`, labeled `build-type=prerelease`. Swapping storage also adds a variable to the run +meant to produce the baseline. Task [#116]. diff --git a/worker_crop.py b/worker_crop.py index db6b798..5c5fa48 100644 --- a/worker_crop.py +++ b/worker_crop.py @@ -244,7 +244,7 @@ async def crop_webtoon(data: WebtoonInput): context_links = context_fragment_links(crops) panels = [] for idx, (crop_img, bbox) in enumerate(crops): - uri = f"s3://manga/{data.manga_id}/{data.chapter_id}/panels/p{idx:03d}.png" + uri = f"s3://panels/{data.manga_id}/{data.chapter_id}/panels/p{idx:03d}.png" # slicing is deterministic, so on a resume the same idx -> same key; skip re-upload. if not transport.exists(uri): out = f"{SHM}/wt_{tag}_p{idx:03d}.png" @@ -274,7 +274,7 @@ async def crop(data: CropInput): for idx, (crop_img, bbox) in enumerate(crops): out = f"{SHM}/pg{data.page_index:03d}_p{idx:02d}.png" cv2.imwrite(out, crop_img) - uri = f"s3://manga/{data.manga_id}/{data.chapter_id}/panels/pg{data.page_index:03d}_p{idx:02d}.png" + uri = f"s3://panels/{data.manga_id}/{data.chapter_id}/panels/pg{data.page_index:03d}_p{idx:02d}.png" transport.put(out, uri) os.remove(out) panels.append({"panel_index": idx, "uri": uri, "bbox": bbox, diff --git a/worker_layers.py b/worker_layers.py index 3f9b74f..03cda30 100644 --- a/worker_layers.py +++ b/worker_layers.py @@ -73,7 +73,7 @@ async def layers(data: LayerInput): png = f"{SHM}/layer_{uuid.uuid4().hex[:6]}.png" with open(png, "wb") as f: f.write(requests.get(url, timeout=60).content) - uri = f"s3://manga/{manga_id}/{chapter_id}/layers/{data.panel_id or 'p'}/{idx}.png" + uri = f"s3://layers/{manga_id}/{chapter_id}/layers/{data.panel_id or 'p'}/{idx}.png" transport.put(png, uri) os.remove(png) layer_uris.append(uri) diff --git a/worker_render.py b/worker_render.py index e35e14b..6ddaf82 100644 --- a/worker_render.py +++ b/worker_render.py @@ -207,7 +207,7 @@ async def render_scene(data: SceneInput): out = f"{SHM}/rnd_{tag}.mp4" subprocess.run(scene_cmd(img, audio, ass, out, dur, data.camera), check=True, capture_output=True) manga_id, chapter_id = _mc_from_uri(data.panel_uri) - uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" transport.put(out, uri) for p in (img, audio, ass, out): os.remove(p) @@ -353,7 +353,7 @@ async def render_composite(data: CompositeInput): for p in imgs + [still]: os.remove(p) manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"]) - uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" transport.put(out, uri) for p in auds + [ass, out]: os.remove(p) @@ -400,7 +400,7 @@ async def render_group(data: GroupInput): subprocess.run(cmd, check=True, capture_output=True) manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"]) - uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" transport.put(final, uri) total = _audio_dur(final) or sum(durs) for f in cleanup + [final]: @@ -537,7 +537,7 @@ async def render_beat(data: BeatInput): check=True, capture_output=True) manga_id, chapter_id = _mc_from_uri(data.panel_uris[0]) - uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" transport.put(out, uri) total = _audio_dur(out) or (D + PAD_S) for f in imgs + [audio, ass, out]: @@ -637,7 +637,7 @@ async def render_collage(data: CollageInput): subprocess.run(collage_cmd(imgs, active, lay["rects"], lay["entrances"], lay["z_order"], audio, ass, out, D, lay["transition_s"]), check=True, capture_output=True) manga_id, chapter_id = _mc_from_uri(uris[0]) - uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" transport.put(out, uri) total = _audio_dur(out) or (D + PAD_S) for f in imgs + [audio, ass, out]: @@ -829,7 +829,7 @@ async def assemble(data: AssembleInput): out = _add_music_bed(out, tag, cleanup) manga_id, chapter_id = _mc_from_uri(data.clip_uris[0]) - uri = f"s3://manga/{manga_id}/{chapter_id}/chapter.mp4" + uri = f"s3://video/{manga_id}/{chapter_id}/chapter.mp4" transport.put(out, uri) for p in cleanup: os.remove(p) diff --git a/worker_tts.py b/worker_tts.py index ad0ff32..1913029 100644 --- a/worker_tts.py +++ b/worker_tts.py @@ -71,8 +71,8 @@ def _audio_uri(data: "TTSInput") -> str: # panel_uri from run_stage_tts to make it per-chapter unique. if data.panel_uri: parts = data.panel_uri.replace("s3://", "").split("/") - return f"s3://manga/{parts[1]}/{parts[2]}/audio/{data.panel_id or 'p'}.wav" - return f"s3://manga/_audio/{data.panel_id or 'p'}.wav" + return f"s3://audio/{parts[1]}/{parts[2]}/audio/{data.panel_id or 'p'}.wav" + return f"s3://audio/_audio/{data.panel_id or 'p'}.wav" def _ensure_ref() -> str: -- 2.52.0 From 36c7cc946f664680eb5d623c6843c8b96bf84bbe Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:02:06 +0400 Subject: [PATCH 06/31] Record the script verifier failure on two-word names The baseline run died in `script` at 87/116. Every lost beat cites `unsupported-proper-noun: ['Choi', 'Haeseon']`, because verify_script puts the full name in the allowed set and then tests single capitalized tokens against it. No fix applied. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01XD7cAy81MZrc7gCr6aZGWr --- HANDOFF.md | 6 ++++++ JOURNAL.md | 4 ++++ caveats/CLAUDE.md | 1 + caveats/speaker-attribution.md | 17 +++++++++++++++++ 4 files changed, 28 insertions(+) diff --git a/HANDOFF.md b/HANDOFF.md index 01fb719..aee2d4c 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -52,9 +52,15 @@ Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6 ## Open +- The `script` stage failed at 87/116 with `unsupported-proper-noun: ['Choi', 'Haeseon']` on 28 beats. + Not OOM, the worker stayed healthy. One-line fix named in + `caveats/speaker-attribution.md#multiword-name-verifier`. Not applied. - The run stopped in `direct` at 113/116. `scene script tts layers render assemble` never ran. Resume: + Since then it reached `scene` 116/116 and failed in `script`. Fix the verifier, then: + ```bash + ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"script\"}'" ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" ``` diff --git a/JOURNAL.md b/JOURNAL.md index 1251b6b..938e73a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -97,3 +97,7 @@ two-character panels attribute both speakers to the wrong person, always swapped character, which is the Phase 1 headline metric at 30%, and the sample says that 30% is not trustworthy. 26 of 113 detected people got an identity, and 25 of those 26 went to one character that turns out to cover two different women. + +The run then reached `scene` 116/116 and failed in `script` at 87/116, not on OOM: 28 beats were +rejected by the script verifier as `unsupported-proper-noun: ['Choi', 'Haeseon']` +(`caveats/speaker-attribution.md#multiword-name-verifier`). No two-word cast name can pass that check. diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index eb56780..f81509c 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -35,3 +35,4 @@ a complaint, so give it one or drop it. | [`speaker_method="tail"` never reads a tail](speaker-attribution.md#tail-is-not-geometry) | 2026-08-11 run | | [One character id covers two different women](speaker-attribution.md#identity-over-merge) | 2026-08-11 run | | [The character registry carries five weeks of wrong names](speaker-attribution.md#registry-pollution) | 2026-08-11 run | +| [A multi-word character name always fails the script verifier](speaker-attribution.md#multiword-name-verifier) | 2026-08-11 run | diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md index 72331c8..63835ae 100644 --- a/caveats/speaker-attribution.md +++ b/caveats/speaker-attribution.md @@ -53,3 +53,20 @@ inherits the whole pile. **Revisit trigger:** before any run that is meant to produce a clean baseline. Either scope the registry to a chapter or add a reviewed reset. + +## A multi-word character name always fails the script verifier {#multiword-name-verifier} + +`correctness.py:174` builds `allowed` from `cast_names` verbatim, so a two-word name enters the set as +one string, `"choi haeseon"`. `_capitalized_tokens` then yields the tokens `Choi` and `Haeseon` +separately, neither of which is in `allowed`. Every beat whose narration uses a two-word name fails +with `unsupported-proper-noun`. A one-word name such as `Seonho` passes, which is why this went unseen. + +Measured on job `778297bc`: the `script` stage failed at 87/116. 28 of the 29 lost beats cite +`['Choi', 'Haeseon']`, one cites `['Blur']`, an onomatopoeia the model invented. + +Fix: tokenize each cast name when building `allowed`, in `verify_script` +(`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/correctness.py:174`). One line. Existing +`test_script_verify.py` covers the function, so add the two-word case there. + +**Revisit trigger:** immediately. It costs a quarter of the chapter's narration on any title whose +cast has a surname. -- 2.52.0 From e8941d8ceb3fe7b9175c102f2e7d11ec716b35c8 Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:23:56 +0400 Subject: [PATCH 07/31] Stop labelling a model guess as a read tail _annotate_speaker_methods stamped `tail`, the highest-trust provenance, on any line whose speaker matched a present local_id, at gemma's confidence of 1.0. No balloon was read. Three of three sampled two-character panels had the speakers swapped, so a multi-character guess is now dropped to unknown, and a solo-panel guess is kept as model_solo at 0.7. Co-Authored-By: Claude Opus 5 --- JOURNAL.md | 20 ++++++++++++++ NEXT.md | 32 ++++++++++++++++------- caveats/CLAUDE.md | 4 +-- caveats/speaker-attribution.md | 43 +++++++++++++----------------- decisions/CLAUDE.md | 2 ++ decisions/speaker-attribution.md | 45 ++++++++++++++++++++++++++++++++ worker_vision.py | 26 +++++++++++++++--- 7 files changed, 133 insertions(+), 39 deletions(-) create mode 100644 decisions/speaker-attribution.md diff --git a/JOURNAL.md b/JOURNAL.md index 938e73a..fdd38d5 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -101,3 +101,23 @@ turns out to cover two different women. The run then reached `scene` 116/116 and failed in `script` at 87/116, not on OOM: 28 beats were rejected by the script verifier as `unsupported-proper-noun: ['Choi', 'Haeseon']` (`caveats/speaker-attribution.md#multiword-name-verifier`). No two-word cast name can pass that check. + +## 2026-08-11 Speaker provenance and the multi-word cast name + +Command: `.venv/bin/python worker_vision.py`, `pytest -q --ignore=test_api.py` in the orchestrator. +Outcome: both pass, 110 orchestrator tests. Nothing deployed, no GPU work, no pipeline run. +Produced: `decisions/speaker-attribution.md`, two caveats rewritten. + +`_annotate_speaker_methods` stopped stamping `tail` on a model guess. With two or more characters +present the guess is dropped to `unknown` at confidence 0.0. With one present it is kept as +`model_solo` at 0.7, the same claim the solo backstop already makes +(`decisions/speaker-attribution.md#no-fake-tail`). Grounded `som_face` and `solo_prior` rows are +untouched. Nothing outside `worker_vision.py` reads the literal `tail`, checked across both repos. + +`verify_script` now tokenizes each cast name into `allowed`, so `Choi Haeseon` passes as two tokens +(`decisions/speaker-attribution.md#multiword-cast-names`). That is the 28 beats job `778297bc` lost. + +The remaining `['Blur']` beat is a true positive that still halts the whole chapter, now recorded as +`caveats/speaker-attribution.md#one-word-halts-chapter`. + +Neither fix is live. The orchestrator container is not rebuilt and the workers are not restarted. diff --git a/NEXT.md b/NEXT.md index 7bdc421..0eedada 100644 --- a/NEXT.md +++ b/NEXT.md @@ -23,27 +23,41 @@ the rebuild exposed. ## Next -1. Fix speaker attribution. The baseline run has been made and the metric it produced cannot be - trusted (`caveats/speaker-attribution.md#tail-is-not-geometry`). Three of three sampled - two-character panels swap the speakers, and the `tail` provenance label is stamped on guesses at - confidence 1.0. Smallest honest first step: stop labelling a guess `tail`, and return `unknown` - when two or more characters are present. -2. Rerun the chapter and re-read the named-speaker share. Only then is the Phase 1 headline number - real. +Two fixes are written and checked, neither is deployed. + +- `worker_vision.py` no longer labels a model guess `tail`, and drops the guess entirely when two or + more characters are present (`decisions/speaker-attribution.md#no-fake-tail`). The workers need a + restart, and only a rerun from `dialogue` puts it on stored data. +- `correctness.py` tokenizes cast names for the script verifier + (`decisions/speaker-attribution.md#multiword-cast-names`). The orchestrator container needs a + rebuild. + +Then, in order: + +1. Rerun the chapter and re-read the named-speaker share. It will fall, and the lower number is the + first honest one. Deciding whether to resume `script` on the old attributions or clear back to + `dialogue` is open. +2. Bind a balloon to a speaker by tail geometry, using the unused `det`/`seg` heads + (`caveats/speaker-attribution.md#tail-is-not-geometry`). Until then multi-character panels have no + speaker at all. 3. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work (`caveats/audit-open.md#sqlite-locking`). ## The 2026-08-11 chapter run Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 -panels. It was still in `direct` at 113/116 when the session ended, with `scene script tts layers -render assemble` unrun. Resume it, or read where it got to: +panels. It reached `scene` 116/116 and then failed in `script` at 87/116 on the verifier bug fixed +above. `tts layers render assemble` never ran. Read where it got to, clear the failed stage, resume: ```bash ssh kami@192.168.1.104 "curl -s 'http://127.0.0.1:9090/job/status?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" +ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"script\"}'" ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" ``` +That resume narrates the attributions the old `tail` label produced. Clearing back to `dialogue` +instead re-runs the GPU stages and produces the honest metric. + Numbers and the quality read are in `JOURNAL.md` and `caveats/speaker-attribution.md`. ## Storage and viewer, tasks #116/#117 diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index f81509c..b07997b 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -32,7 +32,7 @@ a complaint, so give it one or drop it. | [MinIO credentials are hardcoded in committed source](audit-open.md#hardcoded-credentials) | AUDIT.md | | [Assemble marks a job completed with no clips](audit-open.md#empty-assemble) | AUDIT.md | | [Reviewer timestamps drift against the crossfaded video](audit-open.md#timeline-drift) | AUDIT.md | -| [`speaker_method="tail"` never reads a tail](speaker-attribution.md#tail-is-not-geometry) | 2026-08-11 run | +| [Nothing attributes a speaker in a multi-character panel](speaker-attribution.md#tail-is-not-geometry) | 2026-08-11 run | | [One character id covers two different women](speaker-attribution.md#identity-over-merge) | 2026-08-11 run | | [The character registry carries five weeks of wrong names](speaker-attribution.md#registry-pollution) | 2026-08-11 run | -| [A multi-word character name always fails the script verifier](speaker-attribution.md#multiword-name-verifier) | 2026-08-11 run | +| [One invented word still halts the chapter](speaker-attribution.md#one-word-halts-chapter) | 2026-08-11 run | diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md index 63835ae..3bb106f 100644 --- a/caveats/speaker-attribution.md +++ b/caveats/speaker-attribution.md @@ -2,16 +2,15 @@ Limits found by cross-checking the 2026-08-11 chapter run against the panel images. -## `speaker_method="tail"` never reads a tail {#tail-is-not-geometry} +## Nothing attributes a speaker in a multi-character panel {#tail-is-not-geometry} -`worker_vision.py:376` (`_annotate_speaker_methods`) stamps `tail` on any line whose `speaker` matches -a `local_id` present in the panel. No balloon geometry is consulted. The caller keeps gemma's -confidence, usually 1.0. An unverified model guess thus carries the highest-trust provenance in the -pipeline. The review UI and the flag rules both believe it. +The false `tail` label is gone (`decisions/speaker-attribution.md#no-fake-tail`). What replaced it is a +refusal, not an answer: with two or more characters present, every speech line now returns `unknown`. +On a chapter like this one that costs 31 of 81 speech lines their speaker. The narration then falls back +to a `generic-handle`. That is the honest floor, and it is not the fix. -Measured on job `778297bc`, chapter `7c944dd4`: 31 of 81 speech lines are `tail` with two or more -characters present. Three two-character panels were checked against the art, and all three are wrong, -each with the two speakers swapped: +The measurement that forced it, on job `778297bc`, chapter `7c944dd4`. Three two-character panels were +checked against the art. All three are wrong, each with the two speakers swapped: | panel key | line | truth | pipeline | | --- | --- | --- | --- | @@ -31,9 +30,9 @@ per-balloon geometry. `bubble_detect.py:9` records that the `det`/`seg` heads ca tail tips and are unused. **Revisit trigger:** the share of narrated lines with a named speaker is the Phase 1 headline metric -(`ROADMAP.md`). It reads 30% on this run, and the sample says that 30% is itself unreliable. The -metric cannot be trusted until this is fixed. Fix order: stop stamping a guess as `tail` at confidence -1.0. Prefer `unknown` when two or more characters are present. Then bind by tail geometry. +(`ROADMAP.md`). The 30% read on this run counted attributions the sample says are wrong. The next run +will read lower and will be the first honest number. Raising it means binding a balloon to a speaker +by tail geometry, using the unused `det`/`seg` heads. ## One character id covers two different women {#identity-over-merge} @@ -54,19 +53,13 @@ inherits the whole pile. **Revisit trigger:** before any run that is meant to produce a clean baseline. Either scope the registry to a chapter or add a reviewed reset. -## A multi-word character name always fails the script verifier {#multiword-name-verifier} +## One invented word still halts the chapter {#one-word-halts-chapter} -`correctness.py:174` builds `allowed` from `cast_names` verbatim, so a two-word name enters the set as -one string, `"choi haeseon"`. `_capitalized_tokens` then yields the tokens `Choi` and `Haeseon` -separately, neither of which is in `allowed`. Every beat whose narration uses a two-word name fails -with `unsupported-proper-noun`. A one-word name such as `Seonho` passes, which is why this went unseen. +The multi-word name failure is fixed (`decisions/speaker-attribution.md#multiword-cast-names`). The +blast radius it exposed is not. `run_stage_script` retries a rejected beat once, then raises, so a +single unsupported token ends the run at that beat. On job `778297bc` one of the 29 lost beats cited +`['Blur']`, an onomatopoeia the model invented. The verifier was right, and the whole chapter still +stopped. -Measured on job `778297bc`: the `script` stage failed at 87/116. 28 of the 29 lost beats cite -`['Choi', 'Haeseon']`, one cites `['Blur']`, an onomatopoeia the model invented. - -Fix: tokenize each cast name when building `allowed`, in `verify_script` -(`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/correctness.py:174`). One line. Existing -`test_script_verify.py` covers the function, so add the two-word case there. - -**Revisit trigger:** immediately. It costs a quarter of the chapter's narration on any title whose -cast has a surname. +**Revisit trigger:** the next `unsupported-proper-noun` halt that is a true positive. The likely answer +is to flag the beat for review and continue, which is `#136` gate work, not a verifier change. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 319ef47..f54fa0a 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -30,3 +30,5 @@ still live belongs in `caveats/`. | [One bucket per artifact class, not everything under `manga`](storage-layout.md#bucket-per-artifact) | closed | | [The orchestrator creates missing buckets at startup](storage-layout.md#ensure-buckets) | closed | | [RustFS is staged, not adopted](storage-layout.md#rustfs-staged) | open | +| [A model guess is never labelled `tail`](speaker-attribution.md#no-fake-tail) | closed | +| [Cast names enter the verifier tokenized](speaker-attribution.md#multiword-cast-names) | closed | diff --git a/decisions/speaker-attribution.md b/decisions/speaker-attribution.md new file mode 100644 index 0000000..21c3124 --- /dev/null +++ b/decisions/speaker-attribution.md @@ -0,0 +1,45 @@ +# Speaker attribution and cast names + +Settled 2026-08-11 from the quality cross-check of job `778297bc` +(`JOURNAL.md`, `caveats/speaker-attribution.md`). No GPU work ran and no pipeline run was executed +after the change. Both claims rest on source and on the CPU-only self-checks named below. + +Files: `worker_vision.py` on workpc, `correctness.py` and `test_script_verify.py` in the homesrv +orchestrator (`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`). + +## A model guess is never labelled `tail` {#no-fake-tail} + +**Closed.** `speaker_method` names how a speaker was established, and nothing may claim geometry it did +not read. `_annotate_speaker_methods` stamped `tail`, the highest-trust label, on any line whose +speaker matched a `local_id` present in the panel, keeping gemma's confidence of 1.0. No balloon was +ever consulted. + +Evidence: three of three sampled two-character panels had both speakers swapped +(`caveats/speaker-attribution.md#tail-is-not-geometry`). 31 of 81 speech lines carried `tail` with two +or more characters present. + +The label is gone. With two or more characters present the guess is dropped: `speaker` becomes +`unknown`, confidence 0.0, method `unknown`. With one character present the claim equals the solo +backstop, so it is kept as `model_solo` at confidence 0.7. Grounded `som_face` and `solo_prior` rows are +untouched, because the function still skips any row that already carries a method. + +Forbids: minting a provenance label for evidence that was not read, and shipping a multi-character +attribution as truth before balloon geometry exists. +Check: `python worker_vision.py`, the `crowd`/`lone` cases. + +Cost: the named-speaker share will fall. The 30% headline was measured on attributions the sample says +are wrong, so the lower number is the first honest one. + +## Cast names enter the verifier tokenized {#multiword-cast-names} + +**Closed.** `verify_script` compares single capitalized tokens, so every allowed name must be present as +tokens. `allowed` was built from `cast_names` verbatim, which put `"choi haeseon"` in the set as one +string while the checker looked up `Choi` and `Haeseon` separately. + +Evidence: the `script` stage failed at 87/116 on job `778297bc`. 28 of the 29 lost beats cite +`unsupported-proper-noun: ['Choi', 'Haeseon']`. A one-word name such as `Seonho` always passed, which +is why this survived the Phase 1 verifier work +(`decisions/audit-phase1.md#verifier-false-positives`). + +Forbids: adding any future allow-list to `verify_script` as whole strings. +Check: `pytest test_script_verify.py`, `test_multiword_cast_name_is_supported`. diff --git a/worker_vision.py b/worker_vision.py index 68ad8df..ae0de13 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -374,16 +374,31 @@ def resolve_speakers(dialogue: list, present: list) -> list: def _annotate_speaker_methods(dialogue: list, present: list) -> list: - """Fill provenance for model-attributed lines without overwriting grounded/backstop methods.""" + """Fill provenance for model-attributed lines without overwriting grounded/backstop methods. + + A speaker matching a present local_id used to be stamped `tail`, the highest-trust label, at + gemma's own confidence of 1.0. No balloon geometry was ever read. On the 2026-08-11 chapter every + sampled two-character panel had the speakers swapped + (`caveats/speaker-attribution.md#tail-is-not-geometry`), so with 2+ present the guess is dropped + rather than shipped as truth. With one present it is the same claim as the solo backstop, so it is + kept and named for what it is. + ponytail: drop-on-crowd is the honest floor, not the fix. Bind by tail geometry when the balloon + detector lands, then this branch reads a tail for real.""" local_ids = {c.get("local_id") for c in present if c.get("local_id")} + crowded = len(present) > 1 for d in dialogue: if d.get("speaker_method"): continue speaker = (d.get("speaker") or "").strip() if d.get("type", "speech") not in _SPEECH or not speaker or speaker == "unknown": d["speaker_method"] = "unknown" + elif speaker in local_ids and crowded: + d["speaker"] = "unknown" + d["confidence"] = 0.0 + d["speaker_method"] = "unknown" elif speaker in local_ids: - d["speaker_method"] = "tail" + d["speaker_method"] = "model_solo" + d["confidence"] = min(float(d.get("confidence") or 0.7), 0.7) else: d["speaker_method"] = "turn_taking" return dialogue @@ -976,7 +991,12 @@ if __name__ == "__main__": two = [{"local_id": "person_1"}, {"local_id": "person_2"}] assert resolve_speakers([{"speaker": "unknown", "type": "speech", "text": "x"}], two)[0]["speaker"] == "unknown" assert resolve_speakers([{"speaker": "unknown", "type": "speech", "text": "x"}], [])[0]["speaker"] == "unknown" - assert resolve_speakers([{"speaker": "person_1", "type": "speech", "text": "x"}], two)[0]["speaker_method"] == "tail" + # a present-local_id match is gemma's guess, not a read tail: dropped when 2+ are present, + # kept but capped at 0.7 when only one is (caveats/speaker-attribution.md#tail-is-not-geometry) + crowd = resolve_speakers([{"speaker": "person_1", "type": "speech", "text": "x", "confidence": 1.0}], two)[0] + assert crowd["speaker"] == "unknown" and crowd["speaker_method"] == "unknown" and crowd["confidence"] == 0.0 + lone = resolve_speakers([{"speaker": "person_1", "type": "speech", "text": "x", "confidence": 1.0}], solo)[0] + assert lone["speaker"] == "person_1" and lone["speaker_method"] == "model_solo" and lone["confidence"] == 0.7 # set-of-mark: gemma answers a face label -> remapped to local_id; a name/unknown passes through lbl = _apply_speaker_labels([{"speaker": "P1"}, {"speaker": "Aria"}, {"speaker": "unknown"}], {"P1": "person_3"}) -- 2.52.0 From 80711372ba2e1eae10e09e16f592fe6cc61518ab Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:39:30 +0400 Subject: [PATCH 08/31] Map gemma's speaker answer back to a present character The dialogue prompt renders each present character as `- person_1: brown ponytail, green dress (f)`, and gemma answers with what it was shown: the description (15 lines), a bare local_id (9), a stale mark label (2), or a name with the gender marker attached (2). All of them fell through normalize_speaker as free-form names and never matched the registry, so 28 of 51 speech lines on job 778297bc lost a speaker the pipeline had already identified. _apply_speaker_labels now resolves every string the prompt showed, drops an id-shaped answer that names nobody present, and strips a trailing gender marker so an off-panel name can still match. Co-Authored-By: Claude Opus 5 --- worker_vision.py | 71 +++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 64 insertions(+), 7 deletions(-) diff --git a/worker_vision.py b/worker_vision.py index ae0de13..297c131 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -123,16 +123,59 @@ def _set_of_mark(local_path: str, present: list): return marked, "\n" + "\n".join(parts) + "\n", {f["label"]: f["local_id"] or "unknown" for f in faces} -def _apply_speaker_labels(dialogue: list, label_map: dict) -> list: - """map a set-of-mark face label ("P1") in the speaker field back to its local_id. gemma may also - answer with the local_id directly (legend shows both) — that already matches, so it's left as-is.""" - if not label_map: - return dialogue +_GENDER_SUFFIX = re.compile(r"\s*\((?:m|f|male|female|man|woman|unknown)\)\s*$", re.I) +_ID_SHAPED = re.compile(r"^(?:person|char|face|p)[\s_-]*\d+$", re.I) + + +def _present_keys(present: list) -> dict: + """Every string the dialogue prompt shows for a present character, mapped to its local_id. + + `build_dialogue_prompt` renders each one as `- person_1: brown ponytail, green dress (f)`, and gemma + answers with any part of that line, most often the description. Those answers used to fall through + `normalize_speaker` as free-form NAMES and never matched the registry, losing 28 of 51 speech lines + on job 778297bc. A key shared by two present characters is dropped: it cannot identify either.""" + keys: dict = {} + for c in present: + lid = (c.get("local_id") or "").strip() + if not lid: + continue + name, desc, gender = c.get("name") or "", c.get("desc") or "", c.get("gender") or "" + shown = name or desc or "unknown" + forms = {lid, name, desc, shown} + if gender and gender != "unknown": + forms |= {f"{f} ({gender})" for f in (name, desc, shown) if f} + for f in forms: + k = f.strip().casefold() + if not k or k == "unknown": # the prompt prints "unknown" for a nameless character + continue + keys[k] = lid if keys.get(k, lid) == lid else None # ambiguous key -> unusable + return {k: v for k, v in keys.items() if v} + + +def _apply_speaker_labels(dialogue: list, label_map: dict, present: list | None = None) -> list: + """Map gemma's speaker answer back to a panel-local id. + + Three answer shapes reach here: a set-of-mark face label ("P1"), a local_id, and any string the + prompt showed for a present character. An id-shaped answer that names nobody present is junk and + becomes "unknown" rather than a name claim (invariant 6). A trailing gender marker is stripped, so + "Seonho (m)" can still match the registry name "Seonho" off-panel.""" + keys = _present_keys(present or []) for d in dialogue: s = (d.get("speaker") or "").strip() + if not s: + continue if s in label_map: d["speaker"] = label_map[s] d["speaker_method"] = "som_face" + continue + bare = _GENDER_SUFFIX.sub("", s).strip() + lid = keys.get(s.casefold()) or keys.get(bare.casefold()) + if lid: + d["speaker"] = lid + elif _ID_SHAPED.match(bare): + d["speaker"] = "unknown" # an id for nobody present: never mint a name from it + elif bare != s: + d["speaker"] = bare # off-panel name, gender marker stripped return dialogue @@ -465,7 +508,7 @@ async def dialogue(data: DialogueInput): result.setdefault("entities", []) result.setdefault("named", []) # [{local_id, name}] explicit namings -> bound to character_id by orchestrator result["named"] = _normalize_claims(result["named"], data.panel_id) - _apply_speaker_labels(result["dialogue"], label_map) + _apply_speaker_labels(result["dialogue"], label_map, data.present_characters) resolve_speakers(result["dialogue"], data.present_characters) result["panel_id"] = data.panel_id result.update(_dialogue_envelope([data.panel_id], [] if parse_failed else [result], parse_failed)) @@ -580,7 +623,7 @@ async def dialogue_window(data: DialogueWindowInput): d = by_id.get(pid) if d is None: # missing is unresolved, never manufactured as a silent success continue - _apply_speaker_labels(d.get("dialogue", []), label_maps.get(pid, {})) + _apply_speaker_labels(d.get("dialogue", []), label_maps.get(pid, {}), present_by_id.get(pid, [])) out.append({ "panel_id": pid, "dialogue": resolve_speakers(d.get("dialogue", []), present_by_id.get(pid, [])), @@ -1001,6 +1044,20 @@ if __name__ == "__main__": lbl = _apply_speaker_labels([{"speaker": "P1"}, {"speaker": "Aria"}, {"speaker": "unknown"}], {"P1": "person_3"}) assert lbl[0]["speaker"] == "person_3" and lbl[1]["speaker"] == "Aria" and lbl[2]["speaker"] == "unknown" + # gemma answers with what the prompt SHOWED, not the id: description, name+gender, bare id, junk id. + # Each shape cost real lines on job 778297bc by falling through as a free-form name. + shown = [{"local_id": "person_1", "desc": "brown ponytail, green dress", "gender": "f"}, + {"local_id": "person_2", "name": "Seonho", "gender": "m"}] + got = _apply_speaker_labels([{"speaker": "brown ponytail, green dress (f)"}, {"speaker": "Seonho"}, + {"speaker": "person_2"}, {"speaker": "person_9"}, + {"speaker": "Haeseon (f)"}, {"speaker": "unknown"}], {}, shown) + assert [d["speaker"] for d in got] == ["person_1", "person_2", "person_2", "unknown", + "Haeseon", "unknown"], got + # a description shared by two present characters identifies neither + twins = [{"local_id": "person_1", "desc": "schoolgirl"}, {"local_id": "person_2", "desc": "schoolgirl"}] + assert _apply_speaker_labels([{"speaker": "schoolgirl"}], {}, twins)[0]["speaker"] == "schoolgirl" + # a nameless present character is shown as "unknown"; that must not become an id + assert _apply_speaker_labels([{"speaker": "unknown"}], {}, [{"local_id": "person_1"}])[0]["speaker"] == "unknown" # real detector face -> nearest present char's identity; each present char claimed once; leftover unknown pf = _pair_faces_to_present( [{"bbox": [10, 10, 50, 50]}, {"bbox": [200, 10, 240, 50]}, {"bbox": [400, 10, 440, 50]}], -- 2.52.0 From a965077e6b78635dbf75d1793be6ab04c0b1e059 Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:44:22 +0400 Subject: [PATCH 09/31] Do not stamp som_face on a mark that paired to nobody _set_of_mark labels a detected face `unknown` when gated pairing matched it to no present character. An answer pointing at that mark grounds nothing, yet it carried som_face, the highest-trust provenance. All 7 som_face lines in the first 36 panels of the rerun were this case. Same defect class as the fake tail label. Co-Authored-By: Claude Opus 5 --- caveats/speaker-attribution.md | 8 +++++++- decisions/CLAUDE.md | 1 + decisions/speaker-attribution.md | 24 ++++++++++++++++++++++++ worker_vision.py | 14 ++++++++++++-- 4 files changed, 44 insertions(+), 3 deletions(-) diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md index 3bb106f..3a6ff10 100644 --- a/caveats/speaker-attribution.md +++ b/caveats/speaker-attribution.md @@ -50,8 +50,14 @@ The registry holds 53 characters for manga `ef105a86`, 41 of them unnamed, with Haeseon and Seonho do. `/stage/clear` leaves the per-manga registry intact by design, so every rerun inherits the whole pile. +Duplicate rows also make a correct name unresolvable. This manga holds `Choi Haeseon`, `Seonho` with +aliases `["Lim Seonho", "Seonho"]`, and a separate `Lim Seonho`. An answer of "Lim Seonho" matches two +rows, so `normalize_speaker` returns candidates and raises `ambiguous-speaker` instead of binding. The +pipeline read the name correctly and still cannot name the speaker. + **Revisit trigger:** before any run that is meant to produce a clean baseline. Either scope the -registry to a chapter or add a reviewed reset. +registry to a chapter or add a reviewed reset. Merging the duplicate rows needs the reversible-merge +design first (`caveats/audit-open.md#destructive-reconcile`). ## One invented word still halts the chapter {#one-word-halts-chapter} diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index f54fa0a..e8d5022 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -31,4 +31,5 @@ still live belongs in `caveats/`. | [The orchestrator creates missing buckets at startup](storage-layout.md#ensure-buckets) | closed | | [RustFS is staged, not adopted](storage-layout.md#rustfs-staged) | open | | [A model guess is never labelled `tail`](speaker-attribution.md#no-fake-tail) | closed | +| [The model's speaker answer is resolved against what the prompt showed](speaker-attribution.md#prompt-label-answers) | closed | | [Cast names enter the verifier tokenized](speaker-attribution.md#multiword-cast-names) | closed | diff --git a/decisions/speaker-attribution.md b/decisions/speaker-attribution.md index 21c3124..4c867ac 100644 --- a/decisions/speaker-attribution.md +++ b/decisions/speaker-attribution.md @@ -30,6 +30,30 @@ Check: `python worker_vision.py`, the `crowd`/`lone` cases. Cost: the named-speaker share will fall. The 30% headline was measured on attributions the sample says are wrong, so the lower number is the first honest one. +## The model's speaker answer is resolved against what the prompt showed {#prompt-label-answers} + +**Closed.** `build_dialogue_prompt` renders a present character as +`- person_1: brown ponytail, green dress (f)`. gemma answers with any part of that line, so every part +of it must map back to the `local_id`. It did not, and `normalize_speaker` classified each unmatched +answer as a free-form name that no registry entry could match. + +Evidence, measured on 36 panels of the cancelled first rerun. 15 lines carried a description and 9 a bare +`local_id` with no identity assignment. 2 carried a stale `P1` mark label, 2 a name with the gender +marker attached. That is 28 of 51 speech lines. Only 3 resolved to a `character_id`. + +`_apply_speaker_labels` now takes `present` and resolves the id, the name, the description, and each of +those plus the gender marker. A key shared by two present characters is dropped, because it identifies +neither. An id-shaped answer naming nobody present becomes `unknown`. A trailing gender marker is +stripped, so an off-panel `Seonho (m)` still matches the registry name `Seonho`. + +`normalize_speaker` refuses an id-shaped value independently, because the worker is a separate process +and the contract is load-bearing (invariant 7). + +Forbids: showing the model a label the worker cannot resolve back, and treating an unmatched speaker +string as a name. +Check: `python worker_vision.py`, the `shown`/`twins` cases. `pytest test_correctness.py`, +`test_an_id_shaped_speaker_is_never_a_name`. + ## Cast names enter the verifier tokenized {#multiword-cast-names} **Closed.** `verify_script` compares single capitalized tokens, so every allowed name must be present as diff --git a/worker_vision.py b/worker_vision.py index 297c131..b1b1133 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -165,8 +165,14 @@ def _apply_speaker_labels(dialogue: list, label_map: dict, present: list | None if not s: continue if s in label_map: - d["speaker"] = label_map[s] - d["speaker_method"] = "som_face" + lid = label_map[s] + # _set_of_mark labels a detected face `unknown` when gated pairing matched it to no present + # character. An answer pointing at such a mark grounds nothing, so it must not carry + # `som_face`, the highest-trust label. 7 of 7 som_face lines on the first 36 panels of the + # 2026-08-11 rerun were this case. + d["speaker"] = lid + if lid != "unknown": + d["speaker_method"] = "som_face" continue bare = _GENDER_SUFFIX.sub("", s).strip() lid = keys.get(s.casefold()) or keys.get(bare.casefold()) @@ -1044,6 +1050,10 @@ if __name__ == "__main__": lbl = _apply_speaker_labels([{"speaker": "P1"}, {"speaker": "Aria"}, {"speaker": "unknown"}], {"P1": "person_3"}) assert lbl[0]["speaker"] == "person_3" and lbl[1]["speaker"] == "Aria" and lbl[2]["speaker"] == "unknown" + assert lbl[0]["speaker_method"] == "som_face" + # a mark whose face paired to nobody present grounds nothing, so it gets no som_face label + unpaired = _apply_speaker_labels([{"speaker": "P2"}], {"P2": "unknown"})[0] + assert unpaired["speaker"] == "unknown" and "speaker_method" not in unpaired, unpaired # gemma answers with what the prompt SHOWED, not the id: description, name+gender, bare id, junk id. # Each shape cost real lines on job 778297bc by falling through as a free-form name. shown = [{"local_id": "person_1", "desc": "brown ponytail, green dress", "gender": "f"}, -- 2.52.0 From b68f96a9b2d2d93cc01197b7d0bfcb6dbdbac787 Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:47:09 +0400 Subject: [PATCH 10/31] Record the 9% honest speaker number and the identity constraint Co-Authored-By: Claude Opus 5 --- JOURNAL.md | 32 ++++++++++++++++++++++++++++++++ NEXT.md | 32 +++++++++++++++----------------- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index fdd38d5..6425ff7 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -121,3 +121,35 @@ The remaining `['Blur']` beat is a true positive that still halts the whole chap `caveats/speaker-attribution.md#one-word-halts-chapter`. Neither fix is live. The orchestrator container is not rebuilt and the workers are not restarted. + +## 2026-08-11 Rerun from dialogue: the honest speaker number is 9% + +Command: `/job/cancel`, `/stage/clear dialogue`, `./start_workers.sh`, `/job/resume` on job +`778297bc-e7ce-439d-91b5-8a027060d17f`, twice. `docker compose up -d --build orchestrator` three times. +Outcome: `dialogue` 116/116. 112 orchestrator tests pass, `worker_vision.py` self-check passes. + +The named-speaker share is 9%, 9 of 95 speech lines, down from a reported 30% that counted fake tails. +Multi-character panels contribute 0 of 40 lines by design. Single-character panels give 9 of 55. All 9 +binds are `Choi Haeseon`, the row that covers two different women. + +Five defects, four of them found by measuring the run rather than by reading code. + +1. The fake `tail` label, fixed before the run (`decisions/speaker-attribution.md#no-fake-tail`). +2. The multi-word cast name in the script verifier + (`decisions/speaker-attribution.md#multiword-cast-names`). +3. `/stage/clear dialogue` deleted nothing and reported success. dialogue and direct write onto the + per-panel vision blob and had no `_STAGE_TABLES` entry, so `run_stage_dialogue` saw + `"dialogue" in vision` and would have skipped all 116 panels. The proof is the second clear: + 116 dialogue blobs and 75 direct blobs stripped that the first had left. This is + `caveats/audit-open.md#dishonest-clearing` firing exactly where it was filed. +4. gemma answers the speaker field with whatever the prompt showed, most often the character + description, and every such answer became a free-form name that no registry entry matched. 28 of 51 + sampled lines (`decisions/speaker-attribution.md#prompt-label-answers`). After the fix, 3 of 95. +5. All 7 `som_face` lines pointed at a mark whose face paired to no present character, so the + highest-trust provenance sat on a line with no speaker. Same defect class as the fake tail. + +Identity is now the binding constraint, not attribution. 26 of 113 detected people carry an identity, +23%, and 25 of the 26 are the one over-merged row. Even perfect balloon binding caps this chapter near +23% named. The person who does hold an identity is stored as `Lim Seonho` while a separate row is named +`Seonho` with alias `Lim Seonho`, so either name matches two rows, raises `ambiguous-speaker` and binds +nothing. diff --git a/NEXT.md b/NEXT.md index 0eedada..a8e40e3 100644 --- a/NEXT.md +++ b/NEXT.md @@ -23,24 +23,22 @@ the rebuild exposed. ## Next -Two fixes are written and checked, neither is deployed. +The named-speaker share is 9%, 9 of 95 speech lines, and that number is real. See `JOURNAL.md` for the +five defects behind the old 30%. Everything below is measured on job `778297bc`, not inferred. -- `worker_vision.py` no longer labels a model guess `tail`, and drops the guess entirely when two or - more characters are present (`decisions/speaker-attribution.md#no-fake-tail`). The workers need a - restart, and only a rerun from `dialogue` puts it on stored data. -- `correctness.py` tokenizes cast names for the script verifier - (`decisions/speaker-attribution.md#multiword-cast-names`). The orchestrator container needs a - rebuild. +**Identity is the constraint now, not attribution.** 26 of 113 detected people carry an identity. 25 of +those 26 are the single over-merged row (`caveats/speaker-attribution.md#identity-over-merge`). That caps +this chapter near 23% named even with perfect balloon binding. Work identity before geometry. -Then, in order: - -1. Rerun the chapter and re-read the named-speaker share. It will fall, and the lower number is the - first honest one. Deciding whether to resume `script` on the old attributions or clear back to - `dialogue` is open. +1. Split the over-merged character row and dedupe the registry. `Lim Seonho` and `Seonho` are separate + rows with overlapping aliases, so either name matches two rows and binds nothing. Needs the + reversible-merge design (`caveats/audit-open.md#destructive-reconcile`) rather than a patch. 2. Bind a balloon to a speaker by tail geometry, using the unused `det`/`seg` heads (`caveats/speaker-attribution.md#tail-is-not-geometry`). Until then multi-character panels have no - speaker at all. -3. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work + speaker at all, which is 40 of 95 lines here. +3. Resolve a speaker answer across the whole dialogue window, not just the answering panel. The last 3 + unresolved refs are a description belonging to a neighbouring panel in the same 8-panel call. +4. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work (`caveats/audit-open.md#sqlite-locking`). ## The 2026-08-11 chapter run @@ -76,9 +74,9 @@ Two containers on homesrv had been dead for two weeks and are now running. `mang ## Open questions -Four Phase 1 items have no Vikunja task and were not created, because writing to the tracker was not -asked for: the speaker contract fix, the verifier rules, the tracklet constraints, and the flag -resolution path. Only [#203] existed and is now closed by `decisions/audit-phase1.md#unlocked-model-load`. +Four Phase 1 items have no Vikunja task, because writing to the tracker was not asked for. They are the +speaker contract fix, the verifier rules, the tracklet constraints, and the flag resolution path. Only +[#203] existed and is now closed by `decisions/audit-phase1.md#unlocked-model-load`. Three audit items are deliberately not done and are recorded as caveats rather than silently dropped: honest stage clearing, ComfyUI under the session mutex, and reversible identity merges. Each needs a -- 2.52.0 From 63f7918a3e951b60f4904e4f1f59564bf54ea377 Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:59:32 +0400 Subject: [PATCH 11/31] Record all six defects and the 9% baseline Adds decision entries for the unpaired set-of-mark label, the interjection verifier false positive, and the vision-blob clearing bug, plus the per-run speaker audit script used to measure the chapter. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 113 +++++++++++++++++-------------- JOURNAL.md | 7 ++ audit_speakers.py | 105 ++++++++++++++++++++++++++++ caveats/audit-open.md | 10 ++- decisions/CLAUDE.md | 3 + decisions/speaker-attribution.md | 27 ++++++++ decisions/storage-layout.md | 16 +++++ 7 files changed, 229 insertions(+), 52 deletions(-) create mode 100644 audit_speakers.py diff --git a/HANDOFF.md b/HANDOFF.md index aee2d4c..e531e1a 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,71 +1,86 @@ -# HANDOFF, 2026-08-11 session +# HANDOFF, 2026-08-11 late session -Live state lives in `NEXT.md`. This file is only what this session did. +Live state is in `NEXT.md`. This file is only what this session did. ## Asked -1. Add the buckets to rustfs. -2. Fix the bucket saving on the orchestrator side. -3. Run the title recap. +Continue from the previous handoff: fix speaker attribution, then rerun and re-read the named-speaker +share. Mid-session: what happens to unidentified people in a two-person panel, can a known character be +mis-named, and keep watch during the run. -Mid-session: check `manga.kvmx.ru`, and cross-check whether the run got dialogue and identities right. +## Result + +The named-speaker share is **9 of 95 speech lines, 9%**. The previous 30% counted fake tails. All 9 binds +are `Choi Haeseon`, the over-merged row. `script` passed 116/116 for the first time. + +**Identity, not attribution, is now the constraint.** 26 of 113 detected people carry an identity. 25 of +the 26 are that one row. This chapter caps near 23% named even with perfect balloon binding. + +Six defects, four found by measuring the run rather than by reading code. Each has a decision entry: + +| # | defect | entry | +| --- | --- | --- | +| 1 | `tail` stamped on a model guess at confidence 1.0 | `decisions/speaker-attribution.md#no-fake-tail` | +| 2 | a two-word cast name always failed the script verifier | `#multiword-cast-names` | +| 3 | `/stage/clear dialogue` deleted nothing and reported success | `decisions/storage-layout.md#clear-vision-blob` | +| 4 | gemma's speaker answer echoed the prompt label, became a name | `#prompt-label-answers` | +| 5 | `som_face` stamped on a mark that paired to nobody | `#unpaired-mark` | +| 6 | one two-letter interjection halted the chapter at 112/116 | `#interjection-false-positive` | ## Changed -Workers, `/home/kami/Programs/n8n-worker`, branch `restore-runtime`: +Workers, `/home/kami/Programs/n8n-worker`, branch `restore-runtime`, commits `e8941d8 8071137 a965077` +plus docs: -- `worker_crop.py` panels -> `s3://panels/` -- `worker_tts.py` wavs -> `s3://audio/` -- `worker_layers.py` layers -> `s3://layers/` -- `worker_render.py` clips and `chapter.mp4` -> `s3://video/` -- `decisions/storage-layout.md`, `caveats/speaker-attribution.md`, indexes, `JOURNAL.md`, `NEXT.md` +- `worker_vision.py`: `_annotate_speaker_methods`, `_apply_speaker_labels`, new `_present_keys` +- `decisions/speaker-attribution.md` (new, 5 sections), `decisions/storage-layout.md#clear-vision-blob`, + `caveats/speaker-attribution.md`, `caveats/audit-open.md#dishonest-clearing`, both indexes, + `JOURNAL.md`, `NEXT.md` -Orchestrator, `/mnt/server/home/kami/docker-apps/manga-infra/orchestrator`: +Orchestrator, `/mnt/server/home/kami/docker-apps`, commits `b18b6b4 603d388 db8d7c5 ccc3a8e` plus the +interjection commit: -- `minio_layout.py` per-artifact bucket constants, `BUCKETS`, `parse_key` accepts any of them -- `service.py` `_s3_delete_prefix` takes `/`, `_stage_s3_prefixes` no longer slices, - new `_ensure_buckets` in the lifespan -- `test_minio_layout.py` updated, one test added - -Infrastructure, homesrv: - -- six buckets created on `rfs` (rustfs, `127.0.0.1:9010`) -- `manga-fetch` started, exited 2 weeks, `/job/create` fails without it -- `manga-web` started, exited 2 weeks, `manga.kvmx.ru` -> nginx -> `localhost:8083` -- orchestrator rebuilt and restarted -- `mc` aliases `mio` and `rfs` now exist on homesrv +- `correctness.py`: tokenized `allowed`, `_ID_SHAPED` guard in `normalize_speaker`, interjection + stopwords, short-quote grounding skip +- `db.py`: `_STAGE_VISION_KEYS` and the strip pass in `clear_stage_data` +- `test_script_verify.py`, `test_correctness.py`, `test_db.py`: one case each ## Measured -Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, -"Teto X Egen", 116 panels. +Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels. -- 109 orchestrator tests pass. `worker_crop.py`, `worker_tts.py`, `worker_render.py` self-checks pass. - `worker_layers.py` self-check fails on the missing `legacy/qwen_layered_workflow.json`, which - predates this session. -- buckets after the run: `raw` 64MiB/79, `panels` 101MiB/116, `manga` 366MiB/491, `audio` `layers` - `video` still empty because the run had not reached those stages. -- stage times: crop 85s, vision ~4min, identity ~1min, reconcile ~7min, dialogue ~8min. -- 24 of 81 speech lines resolve to a named character, 30%. -- 26 of 113 detected people got an identity, 23%, and 25 of the 26 went to one character. -- 3 of 3 sampled two-character panels attribute both speakers to the wrong person. +- 113 orchestrator tests pass. `worker_vision.py` self-check passes. +- `dialogue` 116/116, `direct` 116/116, `scene` 116/116, `script` 116/116. +- speech lines 95, named 9. Multi-character panels 0 of 40 by design. Single-character 9 of 55. +- `speaker_method`: `unknown` 47, `model_solo` 28, `solo_prior` 8, `som_face` 7, `turn_taking` 5. +- unresolved name refs 3, all `brown ponytail, green dress`, from a neighbouring panel in the same + 8-panel window. Was 24 of 51 before the fix. +- identity assignments 26, of which `character_afa762` "Choi Haeseon" holds 25. +- registry duplicates that block a correct bind: `seonho` matches 2 rows, `lim seonho` matches 2 rows. +- buckets during `tts`: `panels` 116, `raw` 79, `manga` 491, `audio` 1, `layers` 0, `video` 0. ## Open -- The `script` stage failed at 87/116 with `unsupported-proper-noun: ['Choi', 'Haeseon']` on 28 beats. - Not OOM, the worker stayed healthy. One-line fix named in - `caveats/speaker-attribution.md#multiword-name-verifier`. Not applied. -- The run stopped in `direct` at 113/116. `scene script tts layers render assemble` never ran. Resume: - - Since then it reached `scene` 116/116 and failed in `script`. Fix the verifier, then: +- **The run is still going.** It was in `tts` at 26/116 when this was written, with `layers render + assemble` unrun. `tts` is the bottleneck. Read it: ```bash - ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"script\"}'" - ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" + /usr/bin/ssh kami@192.168.1.104 "curl -s 'http://127.0.0.1:9090/job/status?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" ``` -- Speaker attribution is wrong in multi-character panels and the fix is not written - (`caveats/speaker-attribution.md`). No code was changed for it. -- Objects from the 2026-07-17 run still sit under `manga///{pages,panels,audio, - layers,clips}`. Nothing reads them. They are the rollback, not live data. -- `rustfs` holds empty buckets. No mirror, no cutover. + If it failed, clear the failed stage and resume: + + ```bash + /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"\"}'" + /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" + ``` + + Note: plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin. Use `/usr/bin/ssh`. + +- Defects 5 and 6 landed after `dialogue` had already run, so this run's 7 `som_face` lines are still + labelled from the unpaired-mark path. The next dialogue pass fixes that. No name was affected. +- The audit script is `docker exec manga-orchestrator python3 /tmp/audit_speakers.py`, source in this + session's scratchpad. It is not in the repo. Copy it in if the metric is to be tracked per run. +- The 3 cross-panel unresolved refs need the window's whole present-list, not one panel's. +- Workers were restarted twice this session and are running in tmux `manga-workers`. Nothing watches + them, and nothing watches the homesrv containers. diff --git a/JOURNAL.md b/JOURNAL.md index 6425ff7..a5d813e 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -153,3 +153,10 @@ Identity is now the binding constraint, not attribution. 26 of 113 detected peop 23% named. The person who does hold an identity is stored as `Lim Seonho` while a separate row is named `Seonho` with alias `Lim Seonho`, so either name matches two rows, raises `ambiguous-speaker` and binds nothing. + +Two more defects surfaced after `dialogue` finished. All 7 `som_face` lines pointed at a mark whose face +paired to no present character (`decisions/speaker-attribution.md#unpaired-mark`). Then `script` halted +at 112/116 because the narrator wrote `"...Hm?"` for the source line `"Uh... hum...?"`, and both verifier +rules fired on that two-letter interjection +(`decisions/speaker-attribution.md#interjection-false-positive`). After the fix, `script` passed 116/116, +the first time this chapter has cleared the verifier. `tts` then ran for the first time. diff --git a/audit_speakers.py b/audit_speakers.py new file mode 100644 index 0000000..30791b0 --- /dev/null +++ b/audit_speakers.py @@ -0,0 +1,105 @@ +"""Speaker-attribution audit for one chapter. Runs inside manga-orchestrator (reads /data/manga.db). + +Answers the three questions asked of the 2026-08-11 rerun: + 1. what happens on a multi-character panel whose people have no identity, + 2. whether a known character gets mis-named, + 3. what the named-speaker share actually is now. +""" +import collections +import json +import sqlite3 +import sys + +CHAPTER = sys.argv[1] if len(sys.argv) > 1 else "7c944dd4-e972-42c7-ba60-9f6939548e80" +SPEECH = {"speech", "shout", "thought"} + +c = sqlite3.connect("/data/manga.db") +c.row_factory = sqlite3.Row +manga_id = c.execute("SELECT manga_id FROM chapters WHERE chapter_id=?", (CHAPTER,)).fetchone()[0] + +reg = {r["character_id"]: dict(r) for r in c.execute( + "SELECT character_id, name, aliases, description FROM characters WHERE manga_id=?", (manga_id,))} + +# duplicate registry rows make a correct name unresolvable: an answer matching two rows is ambiguous. +by_name = collections.defaultdict(list) +for cid, r in reg.items(): + for n in [r["name"], *json.loads(r["aliases"] or "[]")]: + if n: + by_name[str(n).strip().casefold()].append(cid) +dupes = {n: ids for n, ids in by_name.items() if len(ids) > 1} + +panels = c.execute("SELECT panel_id, panel_index, page_index FROM panels WHERE chapter_id=? ORDER BY \"order\"", + (CHAPTER,)).fetchall() + +methods, kinds, per_char = collections.Counter(), collections.Counter(), collections.Counter() +speech = named = 0 +crowded_speech = crowded_named = 0 +solo_speech = solo_named = 0 +unresolved_names = collections.Counter() +ambiguous = [] +no_identity_crowd = 0 +# a line attributed to a real present local_id that simply has no identity row: attribution succeeded +# and the name is lost anyway. This is the identity-coverage wall, not an attribution failure. +attributed_but_unassigned = collections.Counter() + +for p in panels: + pid = p["panel_id"] + row = c.execute("SELECT result_json FROM vision_results WHERE panel_id=?", (pid,)).fetchone() + if not row: + continue + v = json.loads(row["result_json"]) + if "dialogue" not in v: + continue + people = [ch for ch in (v.get("characters") or []) if ch.get("local_id")] + assigned = {a["local_id"] for a in c.execute( + "SELECT local_id FROM identity_assignments WHERE panel_id=?", (pid,))} + crowd = len(people) > 1 + if crowd and not assigned: + no_identity_crowd += 1 + for d in v["dialogue"]: + if d.get("type", "speech") not in SPEECH: + continue + speech += 1 + ref = d.get("speaker_ref") or {} + kind = ref.get("kind") + methods[d.get("speaker_method")] += 1 + kinds[kind] += 1 + if crowd: + crowded_speech += 1 + else: + solo_speech += 1 + if kind == "character_id": + named += 1 + per_char[reg.get(ref["value"], {}).get("name") or ref["value"]] += 1 + if crowd: + crowded_named += 1 + else: + solo_named += 1 + elif kind == "unknown": + # normalize_dialogue nulls the flat `speaker` unless it resolved, but speaker_ref keeps the + # unresolved local_id as its value. + raw = str(ref.get("value") or "").strip() + local_ids = {ch["local_id"] for ch in people} + if raw in local_ids and raw not in assigned: + attributed_but_unassigned[d.get("speaker_method")] += 1 + elif kind == "name": + unresolved_names[ref.get("value")] += 1 + if ref.get("candidates"): + ambiguous.append((pid, ref.get("value"), ref["candidates"])) + +print(f"panels with dialogue: {sum(1 for p in panels if (lambda r: r and 'dialogue' in json.loads(r['result_json']))(c.execute('SELECT result_json FROM vision_results WHERE panel_id=?', (p['panel_id'],)).fetchone()))}/{len(panels)}") +print(f"speech lines: {speech} named (character_id): {named} = {100*named/max(speech,1):.0f}%") +print(f" multi-character panels: {crowded_named}/{crowded_speech} named") +print(f" single-character panels: {solo_named}/{solo_speech} named") +print(f"multi-character panels with NO identity at all: {no_identity_crowd}") +print(f"attributed to a present local_id with NO identity row: {dict(attributed_but_unassigned)}") +print(f"speaker_method: {dict(methods)}") +print(f"speaker_ref kind: {dict(kinds)}") +print(f"named per character: {dict(per_char)}") +print(f"unresolved name refs: {dict(unresolved_names)}") +print(f"registry duplicate names (block a correct bind): " + f"{ {n: [reg[i]['name'] for i in ids] for n, ids in dupes.items()} }") +if ambiguous: + print("ambiguous binds:") + for pid, val, cands in ambiguous[:20]: + print(f" {pid} {val!r} -> {[reg.get(x, {}).get('name') for x in cands]}") diff --git a/caveats/audit-open.md b/caveats/audit-open.md index acd8a7c..afea734 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -14,13 +14,17 @@ Workaround: none. Clear identity and rerun, which loses the good merges too. ## Clearing a stage does not undo what it wrote {#dishonest-clearing} -Dialogue and direction mutate the shared vision JSON. Clearing dialogue leaves its keys in place, so a -rerun treats old dialogue as completed. Clearing identity preserves the per-manga registry. +The dialogue and direct half of this is fixed and proven +(`decisions/storage-layout.md#clear-vision-blob`). It cost a wasted rerun on 2026-08-11 first: the clear +returned `{"ok": true}`, deleted nothing, and the stage skipped all 116 panels. + +What remains: clearing identity preserves the per-manga registry by design, so a rerun inherits every +character it ever minted (`caveats/speaker-attribution.md#registry-pollution`). Nothing verifies that a +clear emptied what it claimed. Costs: a rerun silently reuses stale output, which reads as a reproducible result. Revisit when: any stage is scheduled concurrently or resumed automatically. A stage must be idempotent before either is safe. -Workaround: delete the keys by hand, or clear from `crop` down. ## ComfyUI uses the GPU outside the session mutex {#comfyui-unscheduled} diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index e8d5022..f695ef6 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -32,4 +32,7 @@ still live belongs in `caveats/`. | [RustFS is staged, not adopted](storage-layout.md#rustfs-staged) | open | | [A model guess is never labelled `tail`](speaker-attribution.md#no-fake-tail) | closed | | [The model's speaker answer is resolved against what the prompt showed](speaker-attribution.md#prompt-label-answers) | closed | +| [An unpaired set-of-mark label grounds nothing](speaker-attribution.md#unpaired-mark) | closed | +| [An interjection is not a name and not a misquote](speaker-attribution.md#interjection-false-positive) | closed | | [Cast names enter the verifier tokenized](speaker-attribution.md#multiword-cast-names) | closed | +| [Clearing a stage strips the vision blob it wrote](storage-layout.md#clear-vision-blob) | closed | diff --git a/decisions/speaker-attribution.md b/decisions/speaker-attribution.md index 4c867ac..0e7da37 100644 --- a/decisions/speaker-attribution.md +++ b/decisions/speaker-attribution.md @@ -54,6 +54,33 @@ string as a name. Check: `python worker_vision.py`, the `shown`/`twins` cases. `pytest test_correctness.py`, `test_an_id_shaped_speaker_is_never_a_name`. +## An unpaired mark grounds nothing {#unpaired-mark} + +**Closed.** `_set_of_mark` labels a detected face `unknown` when gated pairing +(`decisions/audit-phase1.md#gated-face-pairing`) matched it to no present character. An answer pointing +at such a mark identifies nobody, so it must not carry `som_face`, the highest-trust provenance. + +Evidence: all 7 `som_face` lines in the first 36 panels of the 2026-08-11 rerun had `speaker_ref` kind +`unknown`. The label sat on lines with no speaker. Same defect class as the fake `tail`. + +Forbids: deriving a provenance label from the label map without checking what the label resolved to. +Check: `python worker_vision.py`, the `unpaired` case. + +## An interjection is not a name and not a misquote {#interjection-false-positive} + +**Closed.** `verify_script` must stay quiet on valid narration, because `run_stage_script` retries once +and then raises (`decisions/audit-phase1.md#verifier-false-positives`). + +Evidence: the narrator wrote `"...Hm?"` for the source line `"Uh... hum...?"`. Both rules fired at once. +`Hm` was absent from the source words, and a 6-character quote needs 5 matching characters to ground, +so it got 4. That halted the `script` stage at 112/116 on job `778297bc`. + +Interjections join `_STOPWORDS`. A quote of three letters or fewer is no longer grounded-checked. That +holds the same line as the dialogue prompt's 1-3 character noise rule. The stage then passed 116/116. + +Forbids: scoring a quote too short for the ratio to carry meaning. +Check: `pytest test_script_verify.py`, `test_an_interjection_is_not_a_name_or_a_misquote`. + ## Cast names enter the verifier tokenized {#multiword-cast-names} **Closed.** `verify_script` compares single capitalized tokens, so every allowed name must be present as diff --git a/decisions/storage-layout.md b/decisions/storage-layout.md index 0a97a88..0eeaa46 100644 --- a/decisions/storage-layout.md +++ b/decisions/storage-layout.md @@ -56,3 +56,19 @@ instead. The check is not worth a failed start. write. Nothing is repointed. Two things still block a cutover, and neither is settled. RustFS is `1.0.0-beta.12`, labeled `build-type=prerelease`. Swapping storage also adds a variable to the run meant to produce the baseline. Task [#116]. + +## Clearing a stage strips the vision blob it wrote {#clear-vision-blob} + +**Closed. 2026-08-11.** `dialogue` and `direct` have no output table. They write onto the per-panel +vision blob, and `_STAGE_TABLES` had no entry for either, so `/stage/clear dialogue` deleted nothing and +still returned `{"ok": true}`. `run_stage_dialogue` then saw `"dialogue" in vision` and skipped all 116 +panels. + +Evidence: the first clear reported `scripts: 46, scene_graphs: 116` and no vision counts. After the fix +the same call reported `vision_results.dialogue: 116` and `vision_results.direct: 75`, all of which the +first clear had left in place. A whole rerun was wasted on stale data before this was found. + +`_STAGE_VISION_KEYS` in `db.py` names the keys each stage owns, and `clear_stage_data` strips them. + +Forbids: adding a stage that writes onto a shared blob without listing its keys there. +Check: `pytest test_db.py`, `TestClearStageData`. -- 2.52.0 From dead9cf40c0d7859f4357bc22d0ffbab1f2b9f6e Mon Sep 17 00:00:00 2001 From: kami Date: Tue, 11 Aug 2026 23:59:50 +0400 Subject: [PATCH 12/31] Point the handoff at the committed audit script Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index e531e1a..3647f47 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -79,8 +79,9 @@ Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6 - Defects 5 and 6 landed after `dialogue` had already run, so this run's 7 `som_face` lines are still labelled from the unpaired-mark path. The next dialogue pass fixes that. No name was affected. -- The audit script is `docker exec manga-orchestrator python3 /tmp/audit_speakers.py`, source in this - session's scratchpad. It is not in the repo. Copy it in if the metric is to be tracked per run. +- The audit script is now `audit_speakers.py` in this repo. It reads `/data/manga.db`, so it runs inside + the container: `docker cp audit_speakers.py manga-orchestrator:/tmp/ && docker exec + manga-orchestrator python3 /tmp/audit_speakers.py`. - The 3 cross-panel unresolved refs need the window's whole present-list, not one panel's. - Workers were restarted twice this session and are running in tmux `manga-workers`. Nothing watches them, and nothing watches the homesrv containers. -- 2.52.0 From 447c68c8aa7f5ddc1ba6c63d21c6519b14e50633 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 00:10:34 +0400 Subject: [PATCH 13/31] Record the first end-to-end chapter and two stages that lie The run completed: chapter.mp4 at 50MiB from 49 clips. layers reported completed 116/116 with an empty bucket, so nothing has parallax, and the completed job still carries the error string from a failure three resumes earlier. Both recorded as caveats, neither fixed. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 13 +++++++------ JOURNAL.md | 9 +++++++++ NEXT.md | 5 ++++- caveats/CLAUDE.md | 2 ++ caveats/audit-open.md | 20 ++++++++++++++++++++ 5 files changed, 42 insertions(+), 7 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 3647f47..c59bdd5 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -61,14 +61,15 @@ Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6 ## Open -- **The run is still going.** It was in `tts` at 26/116 when this was written, with `layers render - assemble` unrun. `tts` is the bottleneck. Read it: +- **The run finished.** Every stage completed at 2026-08-11T20:08:16Z: `tts` 116/116, `layers` 116/116, + `render` 116/116, `assemble` 1/1. `s3://video/` holds 49 clips and a 50MiB `chapter.mp4` under + `ef105a86-.../7c944dd4-.../`, `s3://audio/` 49 objects at 32MiB. Nobody has watched the video. - ```bash - /usr/bin/ssh kami@192.168.1.104 "curl -s 'http://127.0.0.1:9090/job/status?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" - ``` + Two honesty defects at the finish, recorded not fixed. `layers` reported `completed 116/116` with an + empty bucket, so no clip has parallax (`caveats/audit-open.md#layers-writes-nothing`). The completed + job still carries `error: "partial: 112/116 completed"` (`caveats/audit-open.md#stale-job-error`). - If it failed, clear the failed stage and resume: + Read the state, or clear a stage and resume: ```bash /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"\"}'" diff --git a/JOURNAL.md b/JOURNAL.md index a5d813e..60dbba9 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -160,3 +160,12 @@ at 112/116 because the narrator wrote `"...Hm?"` for the source line `"Uh... hum rules fired on that two-letter interjection (`decisions/speaker-attribution.md#interjection-false-positive`). After the fix, `script` passed 116/116, the first time this chapter has cleared the verifier. `tts` then ran for the first time. + +The run then completed end to end for the first time: `tts` 116/116, `layers` 116/116, `render` 116/116, +`assemble` 1/1, finished 2026-08-11T20:08:16Z. `s3://video/` holds 49 clips and a 50MiB `chapter.mp4`, +`s3://audio/` 49 objects at 32MiB. The per-artifact bucket split is now proven for every class except +layers (`decisions/storage-layout.md#bucket-per-artifact`). + +Two honesty defects surfaced at the finish, both recorded rather than fixed. `layers` reported +`completed 116/116` with an empty bucket, and the completed job still carries +`error: "partial: 112/116 completed"` from the failure three resumes earlier. diff --git a/NEXT.md b/NEXT.md index a8e40e3..5daa5a3 100644 --- a/NEXT.md +++ b/NEXT.md @@ -24,7 +24,10 @@ the rebuild exposed. ## Next The named-speaker share is 9%, 9 of 95 speech lines, and that number is real. See `JOURNAL.md` for the -five defects behind the old 30%. Everything below is measured on job `778297bc`, not inferred. +six defects behind the old 30%. Everything below is measured on job `778297bc`, not inferred. + +That job now runs end to end: `chapter.mp4`, 50MiB, 49 clips. Nobody has watched it. Two stages lie about +it (`caveats/audit-open.md#layers-writes-nothing`, `#stale-job-error`). **Identity is the constraint now, not attribution.** 26 of 113 detected people carry an identity. 25 of those 26 are the single over-merged row (`caveats/speaker-attribution.md#identity-over-merge`). That caps diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index b07997b..cc8beac 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -36,3 +36,5 @@ a complaint, so give it one or drop it. | [One character id covers two different women](speaker-attribution.md#identity-over-merge) | 2026-08-11 run | | [The character registry carries five weeks of wrong names](speaker-attribution.md#registry-pollution) | 2026-08-11 run | | [One invented word still halts the chapter](speaker-attribution.md#one-word-halts-chapter) | 2026-08-11 run | +| [A completed job keeps the error from an earlier failure](audit-open.md#stale-job-error) | 2026-08-11 run | +| [`layers` reports success on an empty bucket](audit-open.md#layers-writes-nothing) | 2026-08-11 run | diff --git a/caveats/audit-open.md b/caveats/audit-open.md index afea734..cebd189 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -151,3 +151,23 @@ transitions, so every non-`cut` transition shortens the real video. Costs: reviewer timestamps drift further out of sync the further into the chapter they scrub. Revisit when: the review UI is used for timing work. Workaround: subtract the transition overlaps by hand. + +## A completed job keeps the error from an earlier failure {#stale-job-error} + +`/job/resume` does not clear `jobs.error`. Job `778297bc` finished every stage and still reports +`status: "completed"` beside `error: "partial: 112/116 completed"`, a message from three resumes earlier. + +Costs: any reader of the error field sees a failure on a successful job. The review UI and any future +alerting both read it. +Revisit when: anything branches on `jobs.error`, or a run is judged by its status alone. + +## `layers` reports success on an empty bucket {#layers-writes-nothing} + +Measured on job `778297bc`: `layers` reported `completed 116/116` while `s3://layers/` held 0 objects. +Every clip in that run therefore has no parallax. The stage is a sibling of +[#inconsistent-stage-policy](#inconsistent-stage-policy), but this is the measured instance: +`worker_layers.py`'s own self-check already fails on the missing +`legacy/qwen_layered_workflow.json`, and the stage still reports done for every panel. + +Costs: a silent quality regression that no status field reveals. +Revisit when: parallax matters for a deliverable, or before quoting this run as a full-pipeline pass. -- 2.52.0 From a9d64fe80a3fb5dbef5424fa17ba1738171be2d6 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 01:04:51 +0400 Subject: [PATCH 14/31] Record panel 7 against the art, and the A/V gap it hid Somebody watched chapter.mp4 for the first time. Two failures came out of it that no stage counter could see. chapter.mp4 is video 436.39s over audio 363.67s, so narration finishes 72.7s before the picture. The 49 clips are clean: all 25fps, video and audio agree to 0.03s, summing to 363.6s. A per-round probe puts the loss in the final round of _assemble_batched, which turns 359s of video into 100s while the audio survives. Round 0 is correct. Round 1 differs by holding a 7th input, the leftover clip that skips encoding, so the tree mixes concat output, xfade output and a raw clip. Not fixed. worker_render.py gains an FPS constant, fps normalization in the xfade branch to match concat, _stream_dur, and a self-check that compares video against audio instead of asserting the file is non-empty. That old check is how a 20% sync failure shipped. The fps inconsistency is real but not proven to be the shipped cause. Pinning -r on the output was tried and reverted: it drops frames to force CFR, which the concat branch comment already warned about. Panel 7 checked against the art has zero correct identity bindings out of two, and Seonho, the one character who matters, is unbound. bbox values are consumed as absolute pixels; on a 900x1650 panel that puts all six boxes in the top third, two inside a speech balloon. Identity therefore embeds crops of balloon edges and window frames, which is how confidence 0.9 lands on the wrong person. The colleague has no name in the story and was labelled Choi Haeseon; that row holds 25 of 26 assignments, so it is the label the pipeline stamps on any unnamed woman. Four caveats added. Two earlier claims are withdrawn in place: rescaling bbox by 1000 does not make the boxes correct, and the constraint is not 16 nameless rows needing names. Cast profiles already exist, since all 53 rows populate ref_image_uris and embedding_uri, but they are enrolled from the wrong crops. worker_render.py self-check passes. No pipeline ran. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 321 +++++++++++++++++++++++++++------ JOURNAL.md | 59 ++++++ NEXT.md | 157 +++++++++++----- caveats/CLAUDE.md | 4 + caveats/speaker-attribution.md | 79 ++++++++ worker_render.py | 57 +++++- 6 files changed, 563 insertions(+), 114 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index c59bdd5..154a66e 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,88 +1,291 @@ -# HANDOFF, 2026-08-11 late session +# HANDOFF, 2026-08-12 Live state is in `NEXT.md`. This file is only what this session did. ## Asked -Continue from the previous handoff: fix speaker attribution, then rerun and re-read the named-speaker -share. Mid-session: what happens to unidentified people in a two-person panel, can a known character be -mis-named, and keep watch during the run. +Get up to speed from the previous handoff. Then: the user watched `chapter.mp4` for the first time and +read out 19 timestamped defects. Mid-session, what about characters. Then, write the handoff. ## Result -The named-speaker share is **9 of 95 speech lines, 9%**. The previous 30% counted fake tails. All 9 binds -are `Choi Haeseon`, the over-merged row. `script` passed 116/116 for the first time. +**Somebody finally watched the video.** That single act found more than four sessions of measuring did. +The recorded metrics said `script` 116/116 and "9 named speech lines". Both were true. Both measured the +wrong thing, because the 9 names are the *wrong* name. -**Identity, not attribution, is now the constraint.** 26 of 113 detected people carry an identity. 25 of -the 26 are that one row. This chapter caps near 23% named even with perfect balloon binding. +Two hard numbers came out of it. -Six defects, four found by measuring the run rather than by reading code. Each has a decision entry: +1. **The chapter is 20% out of sync.** `chapter.mp4` is video 436.39s over audio 363.67s. The narration + ends 72.7 seconds before the picture. The gap accumulates, which is why everything after 2:54 goes + sideways. The 49 clips are clean: video and audio agree to 0.03s and sum to 363.6s. Assembly adds + 72.7s of video and no audio. +2. **Identity binds names to the wrong people, and to people who have no name.** Walking panel 7 against + the art found zero correct bindings out of two, plus the one character who matters left unbound. See + `Panel 7, walked against the art`, which supersedes the earlier reading of this. -| # | defect | entry | +Nothing is committed. `worker_render.py` is edited in the working tree. The fix is **not** verified. + +## The user's 19 notes, grouped by cause + +| cause | timestamps | what is wrong | | --- | --- | --- | -| 1 | `tail` stamped on a model guess at confidence 1.0 | `decisions/speaker-attribution.md#no-fake-tail` | -| 2 | a two-word cast name always failed the script verifier | `#multiword-cast-names` | -| 3 | `/stage/clear dialogue` deleted nothing and reported success | `decisions/storage-layout.md#clear-vision-blob` | -| 4 | gemma's speaker answer echoed the prompt label, became a name | `#prompt-label-answers` | -| 5 | `som_face` stamped on a mark that paired to nobody | `#unpaired-mark` | -| 6 | one two-letter interjection halted the chapter at 112/116 | `#interjection-false-positive` | +| one row absorbed every identity | 0:20, 1:07, 1:51, 1:59, end | anyone identified comes out `Choi Haeseon` | +| the MC has no name | 0:44, 0:50, 1:02, 1:23, 1:45 | falls back to "the worker", "someone", "she" | +| gender read off the wrongly bound row | 1:45 "she admits", end "as he waves" | `Choi` is `f`, so "he" means a nameless `m` row got the line | +| narration invents facts | 0:43 "results", 2:03, 2:05, 2:15 "long shift" | the verifier checks quotes and names, not invented claims | +| vision reads art-within-art as scene | 1:35 chibi on a monitor as "a man holding a drink", 1:59 "pointing towards the screen" | panel-in-panel and screen content taken as reality | +| a beat carries nothing | 0:35-0:37 | no content worth narrating | +| no parallax, so a still holds | 2:24-2:52, 28s static | `layers` wrote nothing (`caveats/audit-open.md#layers-writes-nothing`) | +| transition quality | 2:52-2:54 slide "too sharp and laggy" | `push` is `slideleft` at 0.4s. Retest after the sync fix | +| A/V drift | everything after 2:54 | the 72.7s gap above | -## Changed +## Panel 7, walked against the art -Workers, `/home/kami/Programs/n8n-worker`, branch `restore-runtime`, commits `e8941d8 8071137 a965077` -plus docs: +This is the load-bearing finding of the session. The user pulled up the panel and checked every +detection by eye. **Read this before touching identity.** It contradicts what the earlier sessions +recorded, and it contradicts two theories I floated today before the user corrected them. -- `worker_vision.py`: `_annotate_speaker_methods`, `_apply_speaker_labels`, new `_present_keys` -- `decisions/speaker-attribution.md` (new, 5 sections), `decisions/storage-layout.md#clear-vision-blob`, - `caveats/speaker-attribution.md`, `caveats/audit-open.md#dishonest-clearing`, both indexes, - `JOURNAL.md`, `NEXT.md` +Panel `7c944dd4-e972-42c7-ba60-9f6939548e80_p007`, crop `s3://panels/.../panels/p006.png`, 900x1650. +A wide establishing shot of an office seen through a window. Vision emitted 6 characters. -Orchestrator, `/mnt/server/home/kami/docker-apps`, commits `b18b6b4 603d388 db8d7c5 ccc3a8e` plus the -interjection commit: +| detection | vision said | the art shows | identity assigned | +| --- | --- | --- | --- | +| `person_5` | m, short black, **yellow sweater**, sitting | **Seonho**, foreground, yellow plaid, headphones, back to camera. The character who matters | **nothing** | +| `person_6` | f, short brown, **white shirt**, sitting | the **colleague**, green dress, ponytail. She has **no name** in the story | `Choi Haeseon` at **0.9** | +| `person_2` | m, short brown, green sweater, sitting | a background extra, seated beyond the next window pane | `Lim Seonho` at **0.9** | +| `person_1` | m, short black, suit, standing | **nobody. A window frame** | nothing | +| `person_3` | m, short black, blue sweater, standing | background extra | nothing | +| `person_4` | m, short black, grey sweater, standing | background extra | nothing | -- `correctness.py`: tokenized `allowed`, `_ID_SHAPED` guard in `normalize_speaker`, interjection - stopwords, short-quote grounding skip -- `db.py`: `_STAGE_VISION_KEYS` and the strip pass in `clear_stage_data` -- `test_script_verify.py`, `test_correctness.py`, `test_db.py`: one case each +**Zero of the two bindings are right, and the one character who matters got nothing.** Both wrong binds +carry confidence 0.9. + +Three separate defects stack here. + +**1. The stored bbox coordinate space is wrong.** Consumed as absolute pixels, all six boxes land in the +top third of a 1650px-tall panel, two of them inside the "YEAH!" speech balloon. Divided by 1000 against +the panel's own dimensions, `person_2`, `person_3`, `person_4` and `person_5` fit their subjects tightly. +So the numbers are not pixels. Two places assert that they are: + +- `worker_vision.py:271` prompt text: `pixel bounding box [x1,y1,x2,y2] (top-left, bottom-right corners)` +- `worker_identity.py:91` comment: `vision emits [x1, y1, x2, y2] pixel corners (gemma4's native bbox convention)` + +Everything reading `bbox` is therefore cropping the wrong region. `worker_identity.py:200` embeds +`_crop_bbox(img, ch["bbox"])`, so `Choi Haeseon` at 0.9 was matched on a crop of the speech balloon's +edge and `Lim Seonho` at 0.9 on a crop of empty window frame. Wrong crops are mostly blank white, which +embed alike, which is a plausible mechanism for one row absorbing 25 assignments. + +**Do not treat rescaling as the fix.** I claimed that and the user disproved it in one screenshot. +After scaling, `person_1` still sits on a window frame with nobody in it, and `person_6` is offset, +clipping the woman and running onto the dark frame. `worker_vision.py:38` already calls the box +"coarse, imprecise". Scaling buys roughly-right boxes, not right ones. + +**2. Vision has no concept of extra versus cast.** Four of the six detections are background extras or +nothing at all. They are handed to identity as candidates on equal footing with the two people who +carry the scene. That also means the "113 detected people" figure that framed the whole roadmap was +never the right denominator, so "26 of 113 carry an identity" measured nothing useful. + +**3. Identity mints a name onto a person who has none.** The colleague has no name in the story. She was +labelled `Choi Haeseon` at 0.9. Across the chapter `Choi Haeseon` holds 25 of 26 assignments, so in +practice that row is the label this pipeline stamps on any unnamed woman. This is the direct cause of +the user's 0:20 note, "Choi Haeseon when there's no Choi in the frame, it's the colleague", and of the +gender flips, since gender is read off whichever row got bound. + +This is a cousin of invariant 6 in `CLAUDE.md`, which forbids minting a character from an unparseable +model answer. The missing rule is the same shape: **never attach a name to a detection that carries no +name evidence.** An unnamed recurring person needs a stable anonymous identity so the narration can +call her "the colleague" every time, rather than being forced onto a named row. + +Partly checked, not finished: `match()` in `worker_identity.py:69` does abstain, returning `None` below +threshold, so the 0.9 came from cosine clearing the threshold on a garbage crop. Whether the Tier-2 +gemma resolver can answer "none of these" was not verified. Check that first. + +### Descriptions are not trustworthy either + +`person_6` is "white shirt" for a woman in a green dress. `person_5` is "yellow sweater" for yellow +plaid, which is close enough. Any downstream rule keyed on appearance text inherits this. + +## Registry, measured + +`characters` is keyed by `manga_id`, not chapter. Two manga share the table. `d7104032` has 34 rows with +9 named. This chapter's `ef105a86` has 19 rows with 3 named. The `Kei`, `Zen`, `Kanade`, `Rico` and `K3` +rows belong to the other manga, so they are not polluting this chapter. + +This manga's 19 rows, all `status=confirmed`, all `first_seen_panel=NULL`: + +- named: `Choi Haeseon` (f), `Seonho` (m, aliases `["Lim Seonho","Seonho"]`), `Lim Seonho` (m) +- 16 rows with `name=NULL` and an empty alias list + +`Seonho` and `Lim Seonho` are the same person in two rows. `Seonho`'s alias list contains the other +row's name. That is why either spelling matches two rows and binds nothing. + +Assignments across the whole table, for scale: + +| character | name | assignments | +| --- | --- | --- | +| `character_afa7623b` | Choi Haeseon | 25 | +| `character_bb79cfb4` | Kanade | 25 | +| `character_cfd34340` | Rico | 19 | +| `character_6f491712` | Lim Seonho | 1 | + +In this chapter only `Choi Haeseon` (25) and `Lim Seonho` (1) appear. + +Read those 25 together with panel 7. `Choi Haeseon` is not a character who appears 25 times. It is the +row that absorbs any unnamed woman. The 16 nameless rows are not a backlog of people waiting for names. +Some of them are background extras that should never have become rows, and at least one of them, the +colleague, is a real recurring person who correctly has no name and needs to keep it. + +An earlier draft of this file said "naming is the ceiling, 16 of 19 rows need names". **That was wrong** +and it is corrected here. The ceiling is that identity cannot say "person, no name" and cannot tell an +extra from cast. + +## The A/V bug, located but not fixed + +Reproduced on the 49 real clips with `repro.py` and `probe.py` (see Open). Per-round probe with +`ASSEMBLE_BATCH=8` and 6 `fade_black` boundaries spread across batches: + +``` +r0 g0 n=8 XFADE in v= 49.44 a= 49.44 -> out v= 48.56 a= 48.64 lost_v=+0.88 lost_a=+0.80 +r0 g1 n=8 XFADE in v= 47.04 a= 47.04 -> out v= 46.16 a= 46.23 lost_v=+0.88 lost_a=+0.81 +r0 g2 n=8 XFADE in v= 48.00 a= 48.00 -> out v= 47.12 a= 47.19 lost_v=+0.88 lost_a=+0.81 +r0 g3 n=8 XFADE in v=100.80 a=100.80 -> out v= 99.92 a= 99.96 lost_v=+0.88 lost_a=+0.84 +r0 g4 n=8 concat in v= 64.18 a= 64.16 -> out v= 64.24 a= 64.26 lost_v=-0.06 lost_a=-0.10 +r0 g5 n=8 XFADE in v= 44.00 a= 44.00 -> out v= 43.12 a= 43.17 lost_v=+0.88 lost_a=+0.83 +r1 g0 n=7 XFADE in v=359.29 a=359.60 -> out v= 99.96 a=358.79 lost_v=+259.33 lost_a=+0.81 +``` + +Round 0 is correct. Each group loses only the xfade overlap. **Round 1 loses 259s of video against 0.8s +of audio.** Its output video is 99.96s at 2499 frames, almost exactly the frame count of input `n3` on +its own, the 99.92s intermediate. The final video appears to carry the frames of one input. + +The round-1 filtergraph is arithmetically correct, so this is ffmpeg behaviour, not offset math: + +``` +[n0][n1]xfade=transition=fade:duration=0.050:offset=48.510[v1] +[v1][n2]xfade=transition=fade:duration=0.050:offset=94.620[v2] +[v2][n3]xfade=transition=fadeblack:duration=0.600:offset=141.140[v3] +[v3][n4]xfade=transition=fade:duration=0.050:offset=241.010[v4] +[v4][n5]xfade=transition=fade:duration=0.050:offset=305.200[v5] +[v5][n6]xfade=transition=fade:duration=0.050:offset=348.270[v6] +``` + +### The strongest clue, found last + +Re-running that chain by hand over only the **6** round-0 intermediates gives a correct 348.24s at 8708 +frames, `rc 0`, no warnings. Round 1 collapses with **7** inputs, not 6. + +The 7th input is the leftover 49th clip. With 49 clips and batch 8, round 0 makes 6 groups of 8 and one +group of 1, and `_assemble_batched` passes a lone group through un-encoded: + +```python +if len(group) == 1 and not final_round: + next_items.append(group[0]) +``` + +So the final xfade mixes 6 encoded intermediates with 1 raw clip. That passthrough is a **third** path +next to `concat` and `xfade`, and it is the prime suspect. Start here tomorrow. Confirm it by running +`probe.py` with 48 clips instead of 49, which removes the leftover entirely. + +### What was tried and what it cost + +Two paths exist in `_assemble_once`. A `concat` branch handles cut-only batches. An `xfade` branch +handles batches with a real transition. They disagreed on frame rate. The concat branch forced `fps=30` +while the xfade branch normalized nothing, and clips are 25fps. `436.39 / 363.63 = 1.2001`, exactly +`30/25`, which is what sent me down this path. + +Working-tree changes to `worker_render.py`, all uncommitted: + +- new `FPS = 25` constant. The three hardcoded `25`s and the one `30` now reference it +- the xfade branch normalizes every input with `setsar=1,fps={FPS}` into `[n{i}]` labels, matching what + the concat branch already did +- new `_stream_dur(path, kind)`. `_audio_dur` probes `format=duration`, which is `max(video, audio)`, so + it hides A/V drift by construction +- the `__main__` xfade self-check now assembles 4 clips through `_assemble_batched` with + `ASSEMBLE_BATCH=2` and asserts `abs(video - audio) < 0.25`. It previously asserted only + `getsize(out) > 0`, which is why this shipped + +**A dead end worth not repeating.** I also pinned `-r FPS` on both output encodes. That made it worse. +The chapter collapsed to exactly 100.00s at 2500 frames, because forcing CFR on irregular input +timestamps drops frames. The comment already sitting at the concat branch warns about this. Both `-r` +flags were removed again. The in-graph `fps=` filter is the right normalization. The output `-r` is not. + +The fps inconsistency is real and worth keeping fixed. It is **not** proven to be the cause of the +shipped 72.7s gap. The scene graphs hold 356 `cut` against 6 `fade_black`, so round 2 of the real run +most likely stayed on the concat branch, where no mixing occurs. Treat the fps work as necessary and +insufficient. + +### The user's own hypothesis, which is the recommended direction + +Two guesses, both worth following: + +1. the assembly is wrong in an ffmpeg sense +2. there are two different paths, and they get mixed when there should only be one + +Guess 2 matches the code. `concat`, `xfade` and the single-item passthrough are three paths, and +`_assemble_batched` feeds the output of one into the input of another. **Collapse it to one path.** +Normalize every input, then xfade every boundary, with `cut` as a 0.05s fade. `acrossfade` shortens +audio by the same amount that xfade shortens video, so A/V stays locked. The chapter then comes out +about 2.4s shorter than the sum of the clips, with both streams agreeing. That removes the branch +interaction instead of tuning it. ## Measured -Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels. +Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`. -- 113 orchestrator tests pass. `worker_vision.py` self-check passes. -- `dialogue` 116/116, `direct` 116/116, `scene` 116/116, `script` 116/116. -- speech lines 95, named 9. Multi-character panels 0 of 40 by design. Single-character 9 of 55. -- `speaker_method`: `unknown` 47, `model_solo` 28, `solo_prior` 8, `som_face` 7, `turn_taking` 5. -- unresolved name refs 3, all `brown ponytail, green dress`, from a neighbouring panel in the same - 8-panel window. Was 24 of 51 before the fix. -- identity assignments 26, of which `character_afa762` "Choi Haeseon" holds 25. -- registry duplicates that block a correct bind: `seonho` matches 2 rows, `lim seonho` matches 2 rows. -- buckets during `tts`: `panels` 116, `raw` 79, `manga` 491, `audio` 1, `layers` 0, `video` 0. +- job `status=completed`, every stage at its unit count, finished `2026-08-11T20:08:16Z` +- it still carries `error: "partial: 112/116 completed"` (`caveats/audit-open.md#stale-job-error`) +- `chapter.mp4` 50MiB, video 436.392s, audio 363.675s, `r_frame_rate=25/1`, + `avg_frame_rate=63372800/2792909` which is 22.69, `nb_frames=9902` +- 49 clips, every one `25/1` exactly, sum video 363.63s, sum audio 363.60s. No clip has the two + differing by more than 0.05s +- scene-graph transitions: `cut` 356, `fade_black` 6 +- `worker_render.py` `__main__` self-check passes on the current working tree +- workers up in tmux `manga-workers`, 9 windows ## Open -- **The run finished.** Every stage completed at 2026-08-11T20:08:16Z: `tts` 116/116, `layers` 116/116, - `render` 116/116, `assemble` 1/1. `s3://video/` holds 49 clips and a 50MiB `chapter.mp4` under - `ef105a86-.../7c944dd4-.../`, `s3://audio/` 49 objects at 32MiB. Nobody has watched the video. - - Two honesty defects at the finish, recorded not fixed. `layers` reported `completed 116/116` with an - empty bucket, so no clip has parallax (`caveats/audit-open.md#layers-writes-nothing`). The completed - job still carries `error: "partial: 112/116 completed"` (`caveats/audit-open.md#stale-job-error`). - - Read the state, or clear a stage and resume: +- **Finish the round-1 diagnosis.** The `_assemble_batched` tree turns 359s of video into 100s. This is + the worst defect found and it reproduces offline in about two minutes with no GPU. Suspect the 7th + passthrough input first. ```bash - /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"\"}'" - /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" + .venv/bin/python /probe.py # the per-round loss table above + .venv/bin/python /repro.py # end-to-end verdict ``` - Note: plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin. Use `/usr/bin/ssh`. + The scratchpad is session-scoped and will be gone. **Re-download the clips first:** -- Defects 5 and 6 landed after `dialogue` had already run, so this run's 7 `som_face` lines are still - labelled from the unpaired-mark path. The next dialogue pass fixes that. No name was affected. -- The audit script is now `audit_speakers.py` in this repo. It reads `/data/manga.db`, so it runs inside - the container: `docker cp audit_speakers.py manga-orchestrator:/tmp/ && docker exec - manga-orchestrator python3 /tmp/audit_speakers.py`. -- The 3 cross-panel unresolved refs need the window's whole present-list, not one panel's. -- Workers were restarted twice this session and are running in tmux `manga-workers`. Nothing watches - them, and nothing watches the homesrv containers. + ```bash + /usr/bin/ssh kami@192.168.1.104 'P=homesrv/video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80; mc cp -q -r $P/clips/ /tmp/rclips/; cd /tmp/rclips && tar cf - .' | tar xf - -C clips/ + ``` + + Both scripts are worth committing next session. They are the only check that has ever caught this. + +- **Do not trust `format=duration`.** It returns `max(video, audio)`, so every existing duration assert + in `worker_render.py` is blind to drift. `_stream_dur` exists now. The other asserts still use + `_audio_dur`. +- **Identity, in the order the panel 7 evidence implies.** First, settle the `bbox` coordinate space and + fix every consumer, since nothing else can be judged while crops are wrong. Second, let identity + abstain and hold a stable anonymous identity, so the colleague stays "the colleague". Third, separate + extra from cast so extras never reach identity. Only then merge `Seonho` into `Lim Seonho` and split + `character_afa7623b`, which still needs the reversible-merge design + (`caveats/audit-open.md#destructive-reconcile`). +- **Verify the bbox space before changing anything.** Two independent claims in the code say pixels, and + the art says otherwise. Confirm what the model was told and what it returns, rather than trusting + either comment. Then check whether the crop is the only consumer, or whether SoM marker placement and + the face-pairing in `worker_vision.py:57` read the same numbers. +- Re-derive the panel 7 overlay when needed. It took one `mc cat` of the crop plus a Pillow script, and + it found more than any query did: + + ```bash + /usr/bin/ssh kami@192.168.1.104 'mc cat homesrv/panels/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80/panels/p006.png' > p007.png + ``` + + Draw each `bbox` twice, once as pixels and once divided by 1000, then look at it. +- `mc` aliases on homesrv: use `homesrv` or `mio`, not `local`, which returns Access Denied. `rfs` is + the empty rustfs. +- Plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin. Use `/usr/bin/ssh`. +- `cp` is aliased to `cp -i` in this shell and hangs on overwrite. Use `/usr/bin/cp -f`. +- The Bash tool's default timeout is 120s no matter what `timeout` the command itself carries. Pass the + tool's own timeout or background the run. Otherwise a restore step after a mutation test never runs, + which left a deliberately broken `worker_render.py` on disk once this session. diff --git a/JOURNAL.md b/JOURNAL.md index 60dbba9..0f380a1 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -169,3 +169,62 @@ layers (`decisions/storage-layout.md#bucket-per-artifact`). Two honesty defects surfaced at the finish, both recorded rather than fixed. `layers` reported `completed 116/116` with an empty bucket, and the completed job still carries `error: "partial: 112/116 completed"` from the failure three resumes earlier. + +## 2026-08-12 — the video got watched + +No pipeline ran. The user watched `chapter.mp4` for the first time and read out 19 timestamped defects. +That found more than the previous four sessions of measuring, because the recorded metrics were all +measuring whether code ran rather than whether the result was right. + +Two measurements came out of it. First, `chapter.mp4` is video 436.39s over audio 363.67s, so the +narration finishes 72.7s before the picture and the gap accumulates. The 49 clips are clean: every one is +25fps exactly, video and audio agree to 0.03s, and they sum to 363.6s. Assembly adds 72.7s of video and +no audio. Second, this manga holds 19 character rows of which 3 carry a name, and `Choi Haeseon` holds 25 +of the chapter's 26 identity assignments. That is why the video calls the colleague Choi, never names the +MC, and flips gender. + +The A/V bug was narrowed with a per-round probe over the 49 real clips. Round 0 of `_assemble_batched` is +correct, losing only the xfade overlap per group. Round 1 turns 359s of video into 100s while the audio +survives at 358.79s. The round-1 filtergraph is arithmetically correct, and re-running the same chain by +hand over only the 6 encoded intermediates gives a correct 348.24s with no warnings. Round 1 differs by +holding a 7th input: the leftover 49th clip, which `_assemble_batched` passes through un-encoded. That +passthrough is a third path beside `concat` and `xfade` and is the prime suspect. + +`worker_render.py` gained an `FPS = 25` constant, fps normalization in the xfade branch to match the +concat branch, a `_stream_dur` helper, and a self-check that compares video against audio rather than +asserting the file is non-empty. The old check only asserted `getsize(out) > 0`, which is how a 20% sync +failure shipped. Pinning `-r FPS` on the output encodes was tried and reverted: it collapsed the chapter +to exactly 100.00s by dropping frames to force CFR, which the comment at the concat branch already +warned about. None of it is committed and none of it fixes the chapter yet. + +The fps inconsistency between the two branches is real but not proven to be the shipped cause. The scene +graphs hold 356 `cut` against 6 `fade_black`, so the real run's final round most likely stayed on the +concat branch where no mixing happens. + +### Panel 7, checked against the art + +The same day, the user pulled up panel 7 and checked every detection by eye. It overturned the framing +this file carried an hour earlier, and it overturned two theories I proposed before being corrected. + +Panel `7c944dd4-e972-42c7-ba60-9f6939548e80_p007`, a wide establishing shot of an office through a +window, crop 900x1650. Vision emitted 6 characters. Zero of the two identity bindings are correct and the +one character who matters is unbound. `person_5`, described as "yellow sweater", is Seonho in the +foreground and got no identity. `person_6` is the colleague, who has no name in the story, and was +assigned `Choi Haeseon` at 0.9. `person_2` is a background extra and was assigned `Lim Seonho` at 0.9. +`person_1` is a window frame with nobody in it. `person_3` and `person_4` are background extras. + +Three defects stack, recorded as `caveats/speaker-attribution.md#bbox-wrong-space`, +`#no-anonymous-identity` and `#extras-as-cast`. The `bbox` values are consumed as absolute pixels, and on +this panel that puts all six boxes in the top third with two inside a speech balloon. Divided by 1000 +four of the six fit tightly. Identity therefore embedded crops of balloon edges and window frames, which +is how a 0.9 confidence lands on the wrong person. Blank crops embed alike, a plausible mechanism for one +row absorbing 25 of 26 assignments. + +Two claims I made and had to withdraw. First, that rescaling by 1000 makes the boxes correct: after +scaling, `person_1` still sits on an empty window frame and `person_6` clips its subject, and the +descriptions are unreliable anyway, since `person_6` reads "white shirt" for a green dress. Second, that +the constraint is 16 nameless rows needing names. The opposite is true. The pipeline mints names onto +people who have none, and at least one nameless row is a real recurring person who should stay nameless. + +The "26 of 113 detected people carry an identity" figure that framed the roadmap counted mostly +background extras. It should not be quoted again. diff --git a/NEXT.md b/NEXT.md index 5daa5a3..4ef352b 100644 --- a/NEXT.md +++ b/NEXT.md @@ -1,85 +1,148 @@ # NEXT -Updated 2026-08-11. Replaces the old `HANDOFF.md`. +Updated 2026-08-12. What this session did is in `HANDOFF.md`. ## State -Audit Phase 1 is implemented and green. Nothing is half-finished. +The chapter runs end to end and the output is **not watchable**. That is now measured, not guessed. -Changed on workpc: `worker_scene.py`, `worker_script.py`, `worker_vision.py`, `session_manager.py`. -Changed on homesrv (`/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`): `tracklets.py`, -`correctness.py`, `db.py`, `service.py`, `session_proxy.py`, `test_script_verify.py`, -`test_name_binding.py`. +Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels, +`status=completed`, finished 2026-08-11T20:08:16Z. `s3://video/` holds 49 clips and a 50MiB +`chapter.mp4`. The user watched it and read out 19 defects. They are grouped by cause in `HANDOFF.md`. -What landed and why: `decisions/audit-phase1.md`. What was left open: `caveats/audit-open.md`. +Two numbers set the agenda: -Verification: CPU-only self-checks and the orchestrator test suite. 108 orchestrator tests pass -(`test_api.py` is excluded on workpc because fastapi is not installed in this venv). No GPU work ran -and no pipeline ran, so none of this is confirmed against a real chapter. +- `chapter.mp4` is video 436.39s over audio 363.67s. The narration finishes 72.7s before the picture. +- Panel 7 checked against the art has **zero correct identity bindings** out of two, and the one + character who matters is unbound. `HANDOFF.md#panel-7-walked-against-the-art` has the table. -The orchestrator half is committed as `1c60710` in `/mnt/server/home/kami/docker-apps` and the -container is rebuilt and serving. `94bd4d8` in the same repo repins minio to its amd64 digest, which -the rebuild exposed. +Uncommitted work sits in the tree: `worker_render.py` has an `FPS = 25` constant, fps normalization in +the xfade branch, a new `_stream_dur`, and a self-check that compares video against audio. The +self-check passes. It does **not** yet fix the chapter. Details and one dead end in `HANDOFF.md`. ## Next -The named-speaker share is 9%, 9 of 95 speech lines, and that number is real. See `JOURNAL.md` for the -six defects behind the old 30%. Everything below is measured on job `778297bc`, not inferred. +1. **Fix chapter assembly.** `_assemble_batched` turns 359s of video into 100s while the audio survives. + It reproduces offline in two minutes, no GPU. Round 0 is correct and round 1 collapses. Round 1 is + the only round holding a raw clip that skipped encoding, so suspect the single-item passthrough + first. The recommended shape is one path, not three: normalize every input, then xfade every + boundary, treating `cut` as a 0.05s fade. `acrossfade` and `xfade` shorten audio and video equally, + so the streams stay locked. `HANDOFF.md` holds the per-round table, the filtergraph, and the repro + commands. Nothing downstream is worth judging until this lands. +2. **Fix identity, in this order.** Panel 7 is the worked example and + `HANDOFF.md#panel-7-walked-against-the-art` carries the evidence. Do not start at the registry. -That job now runs end to end: `chapter.mp4`, 50MiB, 49 clips. Nobody has watched it. Two stages lie about -it (`caveats/audit-open.md#layers-writes-nothing`, `#stale-job-error`). + a. **Settle the `bbox` coordinate space.** Consumed as pixels, all six boxes on panel 7 land in the + top third of the panel, two inside a speech balloon. Divided by 1000 they mostly land on their + subjects. `worker_vision.py:271` and `worker_identity.py:91` both assert pixels, and the art says + otherwise. Identity embeds `_crop_bbox(img, ch["bbox"])` at `worker_identity.py:200`, so today it + matches faces against crops of balloons and window frames. Nothing downstream can be judged until + this is right. Rescaling alone is **not** the fix: after scaling, one box still sits on an empty + window frame and another clips its subject. + b. **Let identity abstain and stay abstained.** The colleague has no name in the story and was + labelled `Choi Haeseon` at 0.9. An unnamed recurring person needs a stable anonymous identity so + narration says "the colleague" every time. `match()` already returns `None` below threshold. Check + whether the Tier-2 gemma resolver can answer "none of these"; that was not verified. + c. **Separate extra from cast.** Four of the six detections on panel 7 are background extras or + nothing at all, and all six reach identity as equal candidates. + d. Only then merge `Seonho` into `Lim Seonho` and split `character_afa7623b`, which still needs the + reversible-merge design (`caveats/audit-open.md#destructive-reconcile`), not a patch. -**Identity is the constraint now, not attribution.** 26 of 113 detected people carry an identity. 25 of -those 26 are the single over-merged row (`caveats/speaker-attribution.md#identity-over-merge`). That caps -this chapter near 23% named even with perfect balloon binding. Work identity before geometry. + **Cast profiles already exist. Do not rebuild them.** The user asked whether the main cast could get a + profile built from reference frames and reused. `characters` already carries `ref_image_uris` and + `embedding_uri`, and all 53 rows have both populated. The mechanism is not missing, it is enrolled + from the wrong crops, so today it stores references to balloon edges and window frames. Step (a) is + what makes it work. Three things are genuinely absent and are the smaller follow-on: -1. Split the over-merged character row and dedupe the registry. `Lim Seonho` and `Seonho` are separate - rows with overlapping aliases, so either name matches two rows and binds nothing. Needs the - reversible-merge design (`caveats/audit-open.md#destructive-reconcile`) rather than a patch. -2. Bind a balloon to a speaker by tail geometry, using the unused `det`/`seg` heads - (`caveats/speaker-attribution.md#tail-is-not-geometry`). Until then multi-character panels have no - speaker at all, which is 40 of 95 lines here. -3. Resolve a speaker answer across the whole dialogue window, not just the answering panel. The last 3 - unresolved refs are a description belonging to a neighbouring panel in the same 8-panel call. -4. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work + - no quality gate on enrollment, so nothing checks that a reference crop holds a face at all + - nothing re-enrolls a reference set once it is written, so the wrong crops persist + + **The visual "is this them?" check is already built. Do not write it again.** `/vision/resolve` at + `worker_vision.py:963` sends the query crop plus up to 3 labelled reference images per candidate. + `build_resolve_prompt` tells the model to judge face shape first, to treat hair and outfit as + secondary, that two people sharing a hair colour are not the same, and to answer `0` for NONE when + unsure. `choice: 0` becomes a new character, an out-of-range index becomes `unresolved`, and + `ref_image_uris` is republished as `reference_image_uris` at `worker_identity.py:152` and `:161`. The + mechanism, the prompt and the abstain path are all correct. They are fed crops of the wrong region, + which is step (a). + - no human gate to name, merge or split the clusters. The user wants this as a minor adjustment on + top, not as the mechanism. The `gates` table and the review gates from [#136] are the place to hang + it + + The chibi at 1:35 will survive all of this. He genuinely is brown hair plus a yellow shirt, so a + profile match is correct on appearance and wrong on reality. That needs item 4 below, plus requiring + a real face before a crop can enroll. +3. **Stop the narration inventing facts.** 0:43, 2:03, 2:05 and 2:15 assert things no panel shows. The + correctness verifier passed 116/116 because it checks quotes and names, never invented claims. +4. **Teach vision that art inside a panel is not the scene.** A chibi on a monitor became "a man holding + a drink" at 1:35. A colleague pointing into the distance became "pointing towards the screen" at + 1:59. +5. **`layers` writes nothing** and reports `completed 116/116`, so no clip has parallax and a still + holds for 28s from 2:24 (`caveats/audit-open.md#layers-writes-nothing`). +6. **Clear the stale job error.** The completed job still carries `error: "partial: 112/116 completed"` + (`caveats/audit-open.md#stale-job-error`). +7. Balloon-to-speaker geometry via the unused `det`/`seg` heads + (`caveats/speaker-attribution.md#tail-is-not-geometry`) is now behind item 2. With no name to attach, + geometry buys nothing. +8. Resolve a speaker answer across the whole dialogue window, not just the answering panel. The last 3 + unresolved refs describe a neighbouring panel in the same 8-panel call. +9. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work (`caveats/audit-open.md#sqlite-locking`). -## The 2026-08-11 chapter run +## Lesson worth keeping -Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 -panels. It reached `scene` 116/116 and then failed in `script` at 87/116 on the verifier bug fixed -above. `tts layers render assemble` never ran. Read where it got to, clear the failed stage, resume: +Every metric recorded before this session said the pipeline was fine or nearly fine. `script` 116/116, +"9 named speech lines", `layers` 116/116, `assemble` 1/1. Watching two and a half minutes of output +found a 20% sync failure, a cast that is 84% anonymous, invented narration, and a stage that writes +nothing while reporting success. Stage counters measure whether code ran. They say nothing about whether +the result is correct. Watch the output before trusting a number. + +## Running the pieces ```bash -ssh kami@192.168.1.104 "curl -s 'http://127.0.0.1:9090/job/status?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" -ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"script\"}'" -ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" +./start_workers.sh # session_manager + 9 workers, each a uvicorn in a tmux window +tmux attach -t manga-workers # per-worker logs +.venv/bin/python worker_render.py # self-check, runs real ffmpeg, about 4 minutes ``` -That resume narrates the attributions the old `tail` label produced. Clearing back to `dialogue` -instead re-runs the GPU stages and produces the honest metric. +Read the state, or clear a stage and resume: -Numbers and the quality read are in `JOURNAL.md` and `caveats/speaker-attribution.md`. +```bash +/usr/bin/ssh kami@192.168.1.104 "curl -s 'http://127.0.0.1:9090/job/status?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"778297bc-e7ce-439d-91b5-8a027060d17f\",\"stage\":\"\"}'" +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" +``` + +Traps: plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin, so use `/usr/bin/ssh`. +`mc` aliases on homesrv are `homesrv` and `mio`. `local` returns Access Denied and `rfs` is the empty +rustfs. `cp` is aliased to `cp -i` and hangs on overwrite, so use `/usr/bin/cp -f`. + +Re-fixing assembly needs the real clips, which the session scratchpad no longer holds: + +```bash +/usr/bin/ssh kami@192.168.1.104 'P=homesrv/video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80; mc cp -q -r $P/clips/ /tmp/rclips/; cd /tmp/rclips && tar cf - .' | tar xf - -C clips/ +``` ## Storage and viewer, tasks #116/#117 -[#117] is done. `stowage` serves the manga buckets. It was never a MinIO problem: the container had -been dead since 2026-07-19 on an arm64 digest pin. Details in `JOURNAL.md`. +[#117] is done. `stowage` serves the manga buckets. It was never a MinIO problem: the container had been +dead since 2026-07-19 on an arm64 digest pin. -[#116] is closer but not cut over. Artifacts now split one bucket per class +[#116] is closer but not cut over. Artifacts split one bucket per class (`decisions/storage-layout.md#bucket-per-artifact`), and both MinIO and `rustfs` hold all six buckets. `rustfs` on `127.0.0.1:9010/9011` is still empty and nothing is repointed, so MinIO serves every read and write. Remaining: `mc mirror` the live buckets, verify counts and sizes, then decide on cutover (`decisions/storage-layout.md#rustfs-staged`). -Two containers on homesrv had been dead for two weeks and are now running. `manga-fetch` is the one -`/job/create` needs. `manga-web` is what `manga.kvmx.ru` proxies to on 8083. Nothing watches them. +Two containers on homesrv had been dead for two weeks and now run. `manga-fetch` is the one +`/job/create` needs. `manga-web` is what `manga.kvmx.ru` proxies to on 8083. Nothing watches them, and +nothing watches the workers. ## Open questions -Four Phase 1 items have no Vikunja task, because writing to the tracker was not asked for. They are the -speaker contract fix, the verifier rules, the tracklet constraints, and the flag resolution path. Only -[#203] existed and is now closed by `decisions/audit-phase1.md#unlocked-model-load`. +Four Phase 1 items have no Vikunja task, because writing to the tracker was not asked for: the speaker +contract fix, the verifier rules, the tracklet constraints, and the flag resolution path. Only [#203] +existed and is closed by `decisions/audit-phase1.md#unlocked-model-load`. Three audit items are deliberately not done and are recorded as caveats rather than silently dropped: honest stage clearing, ComfyUI under the session mutex, and reversible identity merges. Each needs a diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index cc8beac..6836615 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -38,3 +38,7 @@ a complaint, so give it one or drop it. | [One invented word still halts the chapter](speaker-attribution.md#one-word-halts-chapter) | 2026-08-11 run | | [A completed job keeps the error from an earlier failure](audit-open.md#stale-job-error) | 2026-08-11 run | | [`layers` reports success on an empty bucket](audit-open.md#layers-writes-nothing) | 2026-08-11 run | +| [Every `bbox` is read in the wrong coordinate space](speaker-attribution.md#bbox-wrong-space) | 2026-08-12 panel 7 | +| [Identity cannot say "a person with no name"](speaker-attribution.md#no-anonymous-identity) | 2026-08-12 panel 7 | +| [Vision does not separate a background extra from cast](speaker-attribution.md#extras-as-cast) | 2026-08-12 panel 7 | +| [Cast reference profiles are enrolled from wrong crops](speaker-attribution.md#poisoned-reference-set) | 2026-08-12 panel 7 | diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md index 3a6ff10..8709925 100644 --- a/caveats/speaker-attribution.md +++ b/caveats/speaker-attribution.md @@ -69,3 +69,82 @@ stopped. **Revisit trigger:** the next `unsupported-proper-noun` halt that is a true positive. The likely answer is to flag the beat for review and continue, which is `#136` gate work, not a verifier change. + +## Every `bbox` is read in the wrong coordinate space {#bbox-wrong-space} + +Vision's `bbox` values are stored and consumed as absolute pixels. On panel +`7c944dd4-e972-42c7-ba60-9f6939548e80_p007` (crop 900x1650) all six boxes then land in the top third of +the panel, two of them inside the "YEAH!" speech balloon. Divided by 1000 against the panel's own +dimensions, four of the six fit their subjects tightly. + +Two places in the code assert pixels, and the art contradicts both: + +- `worker_vision.py:271`, prompt text: `pixel bounding box [x1,y1,x2,y2] (top-left, bottom-right corners)` +- `worker_identity.py:91`, comment: `vision emits [x1, y1, x2, y2] pixel corners (gemma4's native bbox convention)` + +Who pays: identity embeds `_crop_bbox(img, ch["bbox"])` at `worker_identity.py:200`, so it matches faces +against crops of balloons and window frames. On panel 7 that produced `Choi Haeseon` at confidence 0.9 +from a crop of a balloon edge and `Lim Seonho` at 0.9 from an empty window frame. Blank crops embed +alike, which is a plausible mechanism for one row absorbing 25 of 26 assignments. The face pairing at +`worker_vision.py:57` reads the same numbers and was not checked. + +Rescaling is necessary and not sufficient. After scaling, `person_1` still sits on a window frame with +nobody in it, and `person_6` clips its subject and runs onto the frame. `worker_vision.py:38` already +calls the box "coarse, imprecise". + +Revisit trigger: before any further identity or balloon-geometry work. Nothing downstream of `bbox` can +be judged while the crops are wrong. + +## Identity cannot say "a person with no name" {#no-anonymous-identity} + +The colleague on panel 7 has no name in the story. She was assigned `Choi Haeseon` at confidence 0.9. +Across the chapter that row holds 25 of 26 assignments, so in practice it is the label the pipeline +stamps on any unnamed woman. Narration then calls her Choi Haeseon and inherits that row's gender, which +is the direct cause of the user's 0:20 and 1:51 notes and of the gender flips at 1:45 and the closing +line. + +This is the same shape as invariant 6 in `CLAUDE.md`, which forbids minting a character from an +unparseable model answer. The missing rule: never attach a name to a detection that carries no name +evidence. A recurring unnamed person needs a stable anonymous identity, so narration says "the +colleague" every time. + +`match()` at `worker_identity.py:69` does abstain, returning `None` below threshold, so the 0.9 came from +cosine clearing the threshold on a wrong crop. Whether the Tier-2 gemma resolver can answer "none of +these" was not verified. + +Revisit trigger: immediately after the `bbox` space is settled. + +## Vision does not separate a background extra from cast {#extras-as-cast} + +Panel 7 is a wide establishing shot. Vision emitted 6 characters. Two matter: Seonho in the foreground +and the unnamed colleague. Three are background office extras, and one (`person_1`) is a window frame +with nobody in it. All six reach identity as equal candidates. + +Who pays: the roadmap's framing figure, "26 of 113 detected people carry an identity", counted mostly +extras, so it measured nothing useful and should not be quoted again. + +Revisit trigger: with `#no-anonymous-identity`, since both change what identity is allowed to return. + +## Cast reference profiles are enrolled from wrong crops {#poisoned-reference-set} + +`characters` carries `ref_image_uris` and `embedding_uri`, and all 53 rows have both populated. So the +cast-profile mechanism exists. It is enrolled through `#bbox-wrong-space`, so the stored references are +crops of balloon edges, window frames and background extras rather than of faces. + +The visual comparison people reach for as the fix is **already implemented**, so do not build it again. +`/vision/resolve` at `worker_vision.py:963` sends the query crop plus up to 3 labelled reference images +per candidate. `build_resolve_prompt` already tells the model to judge face shape first, to treat hair +and outfit as secondary, that two people sharing a hair colour are not the same, and to answer `0` for +NONE when unsure. `choice: 0` becomes a new character and an out-of-range index becomes `unresolved`. The +`ref_image_uris` column is republished as `reference_image_uris` at `worker_identity.py:152` and `:161`, +so the references reach the model. + +That is why this caveat is about the pixels and not the prompt. The resolver compares a crop of a balloon +edge against references enrolled from window frames and background extras, then sometimes answers "same". +Nothing gates enrollment on the crop holding a face. + +Who pays: every later match, because the reference set defines what a character looks like. Fixing +`#bbox-wrong-space` without re-enrolling leaves the poisoned references in place. + +Revisit trigger: as soon as `#bbox-wrong-space` lands, re-enroll from corrected crops and treat the +existing `ref_image_uris` and `embedding_uri` values as invalid. diff --git a/worker_render.py b/worker_render.py index 6ddaf82..4a15687 100644 --- a/worker_render.py +++ b/worker_render.py @@ -116,8 +116,25 @@ def _audio_dur(path: str) -> float: return 0.0 +def _stream_dur(path: str, kind: str) -> float: + """duration of one stream. `format=duration` is max(video,audio) and so hides A/V drift.""" + r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", f"{kind}:0", + "-show_entries", "stream=duration", "-of", "default=nk=1:nw=1", path], + capture_output=True, text=True) + try: + return float(r.stdout.strip()) + except ValueError: + return 0.0 + + ZMAX, ZPAN = 1.15, 1.18 # ken-burns zoom ceiling; constant zoom that gives pans room to travel +# Every clip and every assembly stage MUST agree on this. xfade does not resample: it reinterprets the +# second input's frames at the first input's rate, so a 30fps input joined onto a 25fps one plays 1.2x +# too slow with the audio untouched -- the video ends minutes long and the narration runs ahead of the +# picture. That is exactly what a 25fps clip pipeline plus a `fps=30` concat branch produced. +FPS = 25 + def _motion(camera: dict, frames: int) -> str: """#8 content-aware motion: map the vision `camera` block to a zoompan z/x/y expression. @@ -155,7 +172,7 @@ def _motion(camera: dict, frames: int) -> str: x, y = f"{xc}+(iw*0.03)*sin(6.283*on/{T})", f"{yc}+(ih*0.03)*cos(6.283*on/{T})" else: # zoom_in (default ken burns) z, x, y = f"1+{ZMAX-1:.3f}*on/{T}", xc, yc - return f"zoompan=z='{z}':x='{x}':y='{y}':d={frames}:s={W}x{H}:fps=25" + return f"zoompan=z='{z}':x='{x}':y='{y}':d={frames}:s={W}x{H}:fps={FPS}" def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict = None, @@ -163,8 +180,7 @@ def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict """ffmpeg: still panel over a blurred fill of itself + content-aware motion, burned subs, 9:16. #5 blurred bg replaces black bars: one copy scaled to COVER + blurred, the fitted panel on top. #13 pad seconds of trailing silence (last frame held) give the panel a beat before the next.""" - fps = 25 - frames = max(1, int((dur + pad) * fps)) # hold the last frame through the pad + frames = max(1, int((dur + pad) * FPS)) # hold the last frame through the pad # overlay's W/H/w/h are ffmpeg's main/overlay dims -- kept literal (no f-string braces). fc = ( f"[0:v]split=2[bg][fg];" @@ -483,7 +499,7 @@ def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list burned subtitle (in `ass`) spans the whole beat. 185: per-image screen-time is content-weighted (see _beat_slices), equal split when weights are absent. pure -> testable without S3.""" cameras = cameras or [] - n, fps = len(imgs), 25 + n, fps = len(imgs), FPS slices = _beat_slices(D, n, weights) # 185: content-weighted, equal-split fallback # one frame per image (no -loop): zoompan d=frames expands that single frame to exactly `frames` # output frames = seg seconds. looping instead would feed many frames and zoompan multiplies each. @@ -675,12 +691,15 @@ def _xfade_chain(durs: list, trans: list): # accumulator would run BACKWARDS (cum += dur - td), swallowing every later clip into a frozen # overlap near the middle. Floor to a small positive length so the timeline stays monotonic. durs = [d if (d and d > 0.1) else 0.1 for d in durs] - parts, vlast, alast, cum = [], "[0:v]", "[0:a]", durs[0] + # Normalize every input to FPS/SAR before it reaches xfade, exactly as the concat branch does. Both + # branches feed the same tree, so an un-normalized xfade input is what stretched the chapter 1.2x. + parts = [f"[{i}:v]setsar=1,fps={FPS}[n{i}]" for i in range(len(durs))] + vlast, alast, cum = "[n0]", "[0:a]", durs[0] for i in range(1, len(durs)): name, td = XFADE.get(trans[i - 1] if i - 1 < len(trans) else "cut", XFADE["cut"]) td = max(0.05, min(td, durs[i - 1] - 0.05, durs[i] - 0.05)) # overlap fits in both clips off = max(cum - td, 0) - parts.append(f"{vlast}[{i}:v]xfade=transition={name}:duration={td:.3f}:offset={off:.3f}[v{i}]") + parts.append(f"{vlast}[n{i}]xfade=transition={name}:duration={td:.3f}:offset={off:.3f}[v{i}]") parts.append(f"{alast}[{i}:a]acrossfade=d={td:.3f}[a{i}]") vlast, alast, cum = f"[v{i}]", f"[a{i}]", cum + durs[i] - td return ";".join(parts), vlast, alast @@ -713,7 +732,7 @@ def _assemble_once(inputs: list[str], trans: list[str], out: str): cmd = ["ffmpeg", "-y"] for p in inputs: cmd += ["-i", p] - pre = "".join(f"[{i}:v]setsar=1,fps=30[v{i}];" for i in range(n)) + pre = "".join(f"[{i}:v]setsar=1,fps={FPS}[v{i}];" for i in range(n)) fg = pre + "".join(f"[v{i}][{i}:a]" for i in range(n)) + f"concat=n={n}:v=1:a=1[v][a]" cmd += ["-filter_complex", fg, "-map", "[v]", "-map", "[a]", "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p", @@ -945,7 +964,29 @@ if __name__ == "__main__": "-map", vmap, "-map", amap, "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", out], check=True, capture_output=True) assert os.path.getsize(out) > 0 - for p in (img, c0, c1): + # A/V drift: the tree must not stretch video. The concat branch normalizes fps and the xfade + # branch used not to, so a chapter mixing both played 1.2x slow with the audio untouched and the + # narration ran ahead of the picture. Batch=2 over 4 clips forces BOTH branches plus a second + # round -- the shipped bug's exact shape. Compare the streams, not the file size. + c2, c3 = f"{SHM}/c2.mp4", f"{SHM}/c3.mp4" + subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c2, 1.0), check=True, capture_output=True) + subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c3, 1.0), check=True, capture_output=True) + _saved_batch, cl = ASSEMBLE_BATCH, [] + globals()["ASSEMBLE_BATCH"] = 2 + try: + # ["crossfade","crossfade","cut","cut"] with batch=2 is the one shape that mixes branches: + # round 1 sends [c0,c1] through xfade and [c2,c3] through concat, then round 2 xfades those + # two intermediates together. A cut-only first group would keep both on the concat branch and + # the rates would agree by accident, which is why this case has to be spelled out. + _assemble_batched([c0, c1, c2, c3], ["crossfade", "crossfade", "cut", "cut"], + out, "selfck", cl) + finally: + globals()["ASSEMBLE_BATCH"] = _saved_batch + for p in cl: + if os.path.exists(p): os.remove(p) + vd, ad = _stream_dur(out, "v"), _stream_dur(out, "a") + assert abs(vd - ad) < 0.25, f"A/V drift: video {vd:.2f}s vs audio {ad:.2f}s" + for p in (img, c0, c1, c2, c3): os.remove(p) # #6 composite: 2 panels + 2 audios -> one stacked clip; duration = sum, subs timed per row. a2 = f"{SHM}/a2.wav" -- 2.52.0 From 1457556ce3dbd086fc701b7e0bac2353cdcb0ce5 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 11:52:38 +0400 Subject: [PATCH 15/31] Fix the 72.7s A/V gap: xfade offsets ran off the end of their input _xfade_chain positioned every transition using _audio_dur, which probes format=duration, which is max(video, audio). A clip's audio outlasts its video by about a frame, so the offset accumulator crept ahead of the real picture timeline. Once the creep exceeded the transition width, xfade emitted the transition and silently discarded the second input and every clip downstream, exiting 0 with nothing on stderr. That is the whole of the shipped chapter's 436.39s of video over 363.67s of audio. Offsets now come from min(video, audio). Every input is floored to a whole frame count and trimmed on both streams, so the accumulator tracks the real timeline instead of estimating it. _check_assembled verifies each encode against the predicted length and against its own audio, because both assembly branches drop stream time without failing. Verified over the 49 real clips of chapter 7c944dd4: the round that turned 359s of video into 100s now loses 0.85s, and the chapter comes out 358.76s video against 358.76s audio. The single-item passthrough was not the cause. Two round-0 groups of 8 fresh clips collapse without one, recorded void in decisions/. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 3 ++ JOURNAL.md | 41 +++++++++++++++++++ NEXT.md | 27 ++++++------ decisions/CLAUDE.md | 3 ++ decisions/chapter-assembly.md | 77 +++++++++++++++++++++++++++++++++++ worker_render.py | 75 ++++++++++++++++++++++++++++++---- 6 files changed, 206 insertions(+), 20 deletions(-) create mode 100644 decisions/chapter-assembly.md diff --git a/CLAUDE.md b/CLAUDE.md index 6dffd06..666e431 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -47,6 +47,9 @@ Machine split: workers run on **workpc** (RX 7900 GRE, ROCm). MinIO and the orch shape means reconciling the orchestrator in the same session. Neither repo's self-checks can catch a contract break, because each asserts its own side. 8. **`ponytail:` comments mark deliberate simplifications** and name the upgrade path. Respect them. +9. **Nothing that positions an ffmpeg filter may use `format=duration`.** It reports `max(video, audio)`. + It hides A/V drift, and it walks xfade offsets past the end of their input. ffmpeg then discards clips + and still exits 0 (`decisions/chapter-assembly.md#offsets-from-min-stream`). ## Working rules diff --git a/JOURNAL.md b/JOURNAL.md index 0f380a1..286cbff 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -228,3 +228,44 @@ people who have none, and at least one nameless row is a real recurring person w The "26 of 113 detected people carry an identity" figure that framed the roadmap counted mostly background extras. It should not be quoted again. + +## 2026-08-12, chapter assembly, root cause and fix + +Reproduced the A/V collapse offline with 49 synthetic clips at `ASSEMBLE_BATCH=8` and six `fade_black` +boundaries. It came out worse than the shipped run: **two round-0 groups of 8 fresh clips collapsed on +their own**, so the single-item passthrough theory from yesterday is dead +(`decisions/chapter-assembly.md#passthrough-innocent`). + +Bisected one collapsing group by truncating the chain stage by stage: + +``` +k=7 out= 52.52 correct +k=8 out= 52.52 the last xfade contributed nothing + [v6][n7]xfade=duration=0.050:offset=52.500 <- [v6] is 52.52s long, 0.02s of margin +``` + +`_xfade_chain` took its durations from `_audio_dur`, which is `format=duration`, which is +`max(video, audio)`. Each clip's audio outlasts its video by about a frame, so the offset accumulator +crept ahead of the picture. Once the creep passed the transition width, xfade emitted the transition and +threw away the second input and every clip after it, at `rc 0` with nothing on stderr. + +Fix: offsets come from `min(_stream_dur(v), _stream_dur(a))`, every input is floored to a whole frame +count and `trim`/`atrim`ed on both streams, and `_check_assembled` now verifies each encode against the +predicted timeline instead of trusting the exit code +(`decisions/chapter-assembly.md#offsets-from-min-stream`, `#check-assembled`). + +Verified on the 49 real clips of chapter `7c944dd4`, re-downloaded from MinIO: + +``` +before r1 n=7 XFADE in v=359.29 a=359.60 -> out v= 99.96 a=358.79 +after r1 n=7 XFADE in v=359.61 a=359.62 -> out v=358.76 a=358.76 +chapter v=358.76 a=358.76 gap=+0.00 (shipped: v=436.39 a=363.67 gap=+72.72) +``` + +`worker_render.py` `__main__` passes. Two checks were added there, because the existing 4-clip A/V assert +passed all the way through the broken build. One asserts the frame-exact `trim` on both streams, one +assembles three clips whose audio outlasts their video by 0.4s. Mutation-tested by putting `_audio_dur` +back: the new check fires with `video=1.80 audio=3.56 expected=3.56`. + +Not done: `s3://video/.../chapter.mp4` is still the broken 436s file. Rebuilding it means clearing the +`assemble` stage and resuming, which is CPU-only and was not run. diff --git a/NEXT.md b/NEXT.md index 4ef352b..f9ae0d6 100644 --- a/NEXT.md +++ b/NEXT.md @@ -5,6 +5,7 @@ Updated 2026-08-12. What this session did is in `HANDOFF.md`. ## State The chapter runs end to end and the output is **not watchable**. That is now measured, not guessed. +Assembly is fixed and verified offline. The shipped `chapter.mp4` has not been rebuilt yet. Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels, `status=completed`, finished 2026-08-11T20:08:16Z. `s3://video/` holds 49 clips and a 50MiB @@ -12,23 +13,25 @@ Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6 Two numbers set the agenda: -- `chapter.mp4` is video 436.39s over audio 363.67s. The narration finishes 72.7s before the picture. +- The shipped `chapter.mp4` is video 436.39s over audio 363.67s. Cause found and fixed, see below. - Panel 7 checked against the art has **zero correct identity bindings** out of two, and the one character who matters is unbound. `HANDOFF.md#panel-7-walked-against-the-art` has the table. -Uncommitted work sits in the tree: `worker_render.py` has an `FPS = 25` constant, fps normalization in -the xfade branch, a new `_stream_dur`, and a self-check that compares video against audio. The -self-check passes. It does **not** yet fix the chapter. Details and one dead end in `HANDOFF.md`. - ## Next -1. **Fix chapter assembly.** `_assemble_batched` turns 359s of video into 100s while the audio survives. - It reproduces offline in two minutes, no GPU. Round 0 is correct and round 1 collapses. Round 1 is - the only round holding a raw clip that skipped encoding, so suspect the single-item passthrough - first. The recommended shape is one path, not three: normalize every input, then xfade every - boundary, treating `cut` as a 0.05s fade. `acrossfade` and `xfade` shorten audio and video equally, - so the streams stay locked. `HANDOFF.md` holds the per-round table, the filtergraph, and the repro - commands. Nothing downstream is worth judging until this lands. +1. **Rebuild `chapter.mp4`.** Assembly is fixed in `worker_render.py`. Verified over the 49 real clips of + this chapter: video 358.76s against audio 358.76s, agreeing to the frame. The cause was `_xfade_chain` + taking offsets from `format=duration`, which is `max(video, audio)`. The accumulator drifted past the + end of its input, and ffmpeg silently discarded whole clips at `rc 0` + (`decisions/chapter-assembly.md#offsets-from-min-stream`). The single-item passthrough was innocent + and the one-path rewrite is not needed (`decisions/chapter-assembly.md#passthrough-innocent`). + + What is left is to clear the `assemble` stage and resume, then watch the result. That is CPU-only + ffmpeg, no GPU, but it needs the user's go-ahead. + + Smaller follow-on: nine other `_audio_dur` calls in `worker_render.py` measure finished clips with + `format=duration`. So the durations reported to the orchestrator are blind to per-clip drift. + They position no filter, so invariant 9 does not cover them. Worth converting to `_stream_dur`. 2. **Fix identity, in this order.** Panel 7 is the worked example and `HANDOFF.md#panel-7-walked-against-the-art` carries the evidence. Do not start at the registry. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index f695ef6..021bc8a 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -36,3 +36,6 @@ still live belongs in `caveats/`. | [An interjection is not a name and not a misquote](speaker-attribution.md#interjection-false-positive) | closed | | [Cast names enter the verifier tokenized](speaker-attribution.md#multiword-cast-names) | closed | | [Clearing a stage strips the vision blob it wrote](storage-layout.md#clear-vision-blob) | closed | +| [xfade offsets come from `min(video, audio)`, never `format=duration`](chapter-assembly.md#offsets-from-min-stream) | closed | +| [Assembly verifies its own output instead of trusting ffmpeg's exit code](chapter-assembly.md#check-assembled) | closed | +| [The single-item passthrough is not the assembly bug](chapter-assembly.md#passthrough-innocent) | void | diff --git a/decisions/chapter-assembly.md b/decisions/chapter-assembly.md new file mode 100644 index 0000000..d0d81a9 --- /dev/null +++ b/decisions/chapter-assembly.md @@ -0,0 +1,77 @@ +# chapter-assembly + +Settled questions about `_assemble_batched` / `_assemble_once` / `_xfade_chain` in `worker_render.py`. + +## xfade offsets are computed from `min(video, audio)`, never `format=duration` {#offsets-from-min-stream} + +**Closed, 2026-08-12.** + +`_xfade_chain` accumulates `cum += dur[i] - td` and hands each boundary `offset=cum-td`. That offset is +an assertion about where input `i-1` still has frames. It fed on `_audio_dur`, which probes +`format=duration`, which is `max(video, audio)`. A rendered clip's audio outlasts its video by about a +frame. So every boundary pushed the accumulator further ahead of the picture. + +Once the accumulated overshoot exceeds the transition width, the xfade window starts after the last frame +of input `i-1`. ffmpeg emits the transition and then **silently discards input `i` and every clip +downstream of it**. `rc 0`, no warning on stderr, output file present and playable. Measured on a group of +8 real clips, chain truncated at each stage: + +``` +k=7 out= 52.52 correct +k=8 out= 52.52 the last xfade contributed nothing + [v6][n7]xfade=duration=0.050:offset=52.500 <- [v6] is only 52.52s long, so 0.02s of margin +``` + +This is the whole cause of the shipped chapter being video 436.39s over audio 363.67s. It reproduces with +synthetic clips in about four minutes and needs no GPU. + +Two changes hold it closed: + +* `_assemble_once` probes `min(_stream_dur(p, "v"), _stream_dur(p, "a"))`. The minimum, because either + stream running long breaks a different half of the graph. +* `_xfade_chain` floors every input to a whole frame count. It applies `trim` and `atrim` to both streams, + so the accumulator tracks the real timeline instead of estimating it. Transition widths are quantized to + frames for the same reason. + +Verified over the 49 real clips of chapter `7c944dd4`. The round that previously turned 359s of video into +100s now loses 0.85s. The chapter comes out video 358.76s against audio 358.76s, agreeing to the frame. + +Forbidden from here: `_audio_dur` in anything that positions a filter. It is fine for "how long is this +clip roughly", nothing else. + +## Assembly verifies its own output instead of trusting ffmpeg's exit code {#check-assembled} + +**Closed, 2026-08-12.** + +Both assembly branches drop stream time without failing. xfade discards inputs as above. The concat +demuxer with `-vsync cfr` drops video frames to force a constant rate. Neither is an error to ffmpeg. + +So `_assemble_once` calls `_check_assembled(out, expect)` after every encode. It compares the output's +video stream against the predicted timeline and against its own audio stream. It raises when either is off +by more than `ASSEMBLE_TOL_S`, which is 0.5s. That tolerance covers frame boundaries and aac padding. A +dropped input is off by whole seconds. + +Without this the failure stays invisible until somebody watches the video. That is how a 50MiB chapter +with 72.7s of silent picture reached the bucket while every stage counter read success. + +## The single-item passthrough is not the bug {#passthrough-innocent} + +**Void, 2026-08-12.** Cited in `HANDOFF.md` for 2026-08-11 as the prime suspect and must not be cited +again. + +With 49 clips and `ASSEMBLE_BATCH=8`, round 0 makes six groups of 8 plus a leftover group of 1, which +`_assemble_batched` carries forward un-encoded. The theory was that mixing that raw clip with six encoded +intermediates broke round 1. The instrumented run disproves it: **two round-0 groups of 8 fresh clips +collapse on their own**, before any passthrough exists. + +``` +n=8 XFADE in v= 63.52 a= 63.60 -> out v= 52.52 a= 62.77 lost_v= +11.00 +n=8 XFADE in v= 58.12 a= 58.20 -> out v= 12.04 a= 56.83 lost_v= +46.08 +``` + +Round 1's 259s loss was the cascade. Its inputs already held 309s of video against 364s of audio, and +`durs` read the audio. + +Consequence for the plan: collapsing `concat`, `xfade` and the passthrough into one path was the +recommended fix in `NEXT.md` and is **not needed**. Three paths are fine once each one positions filters +on a real timeline. The tree keeps the bounded memory it was built for. diff --git a/worker_render.py b/worker_render.py index 4a15687..5875124 100644 --- a/worker_render.py +++ b/worker_render.py @@ -686,31 +686,57 @@ FFMPEG_THREADS = max(1, int(os.environ.get("FFMPEG_THREADS", "2"))) def _xfade_chain(durs: list, trans: list): """build a filter_complex that xfades N clips with per-boundary transitions, keeping audio in sync via matching acrossfade. trans[i] is the transition OUT of clip i (boundary i->i+1). - returns (filtergraph, video_label, audio_label). offsets accumulate as clips overlap.""" + returns (filtergraph, video_label, audio_label, expected_duration). offsets accumulate as clips + overlap. `durs[i]` MUST be min(video, audio) of input i, not `format=duration`.""" # A clip whose duration probed as 0/unreadable must not poison the chain: with dur=0 the offset # accumulator would run BACKWARDS (cum += dur - td), swallowing every later clip into a frozen # overlap near the middle. Floor to a small positive length so the timeline stays monotonic. durs = [d if (d and d > 0.1) else 0.1 for d in durs] + # The offset accumulator is only as good as `durs`: an offset that lands even one frame past the end + # of input i-1 makes xfade emit the transition and then SILENTLY DROP input i and the whole rest of + # the chain -- rc 0, no warning, a chapter minutes short with the audio intact. So truncate every + # input to a whole number of frames it certainly has (floor, and min(v,a) from the caller) and trim + # both streams to exactly that. Then cum is the real timeline, not an estimate of it. + durs = [max(1, int(d * FPS)) / FPS for d in durs] # Normalize every input to FPS/SAR before it reaches xfade, exactly as the concat branch does. Both # branches feed the same tree, so an un-normalized xfade input is what stretched the chapter 1.2x. - parts = [f"[{i}:v]setsar=1,fps={FPS}[n{i}]" for i in range(len(durs))] - vlast, alast, cum = "[n0]", "[0:a]", durs[0] + parts = [] + for i, d in enumerate(durs): + parts.append(f"[{i}:v]setsar=1,fps={FPS},trim=end={d:.3f},setpts=PTS-STARTPTS[n{i}]") + parts.append(f"[{i}:a]atrim=end={d:.3f},asetpts=PTS-STARTPTS[m{i}]") + vlast, alast, cum = "[n0]", "[m0]", durs[0] for i in range(1, len(durs)): name, td = XFADE.get(trans[i - 1] if i - 1 < len(trans) else "cut", XFADE["cut"]) td = max(0.05, min(td, durs[i - 1] - 0.05, durs[i] - 0.05)) # overlap fits in both clips + td = max(1, int(td * FPS)) / FPS # ...on a frame boundary off = max(cum - td, 0) parts.append(f"{vlast}[n{i}]xfade=transition={name}:duration={td:.3f}:offset={off:.3f}[v{i}]") - parts.append(f"{alast}[{i}:a]acrossfade=d={td:.3f}[a{i}]") + parts.append(f"{alast}[m{i}]acrossfade=d={td:.3f}[a{i}]") vlast, alast, cum = f"[v{i}]", f"[a{i}]", cum + durs[i] - td - return ";".join(parts), vlast, alast + return ";".join(parts), vlast, alast, cum + + +ASSEMBLE_TOL_S = 0.5 # frame-boundary + aac-padding slack; a dropped input is off by whole seconds + + +def _check_assembled(out: str, expect: float): + """ffmpeg drops xfade inputs and re-times concat segments without ever failing, so verify the + result instead of trusting rc 0. Both streams, because a video-only loss is the failure mode that + shipped a 436s picture over 364s of narration.""" + v, a = _stream_dur(out, "v"), _stream_dur(out, "a") + if abs(v - expect) > ASSEMBLE_TOL_S or abs(v - a) > ASSEMBLE_TOL_S: + raise RuntimeError(f"assembly lost stream time in {os.path.basename(out)}: " + f"video={v:.2f} audio={a:.2f} expected={expect:.2f}") def _assemble_once(inputs: list[str], trans: list[str], out: str): """Assemble one bounded batch. `trans[i]` is the transition out of inputs[i].""" + # min(video, audio), never `format=duration`: that is max(video, audio), and feeding it to + # _xfade_chain puts the offset accumulator ahead of the real video timeline. + durs = [min(_stream_dur(p, "v"), _stream_dur(p, "a")) for p in inputs] fancy = len(inputs) >= 2 and any(t not in ("", "cut") for t in trans[:len(inputs) - 1]) if fancy: - durs = [_audio_dur(p) for p in inputs] - fg, vmap, amap = _xfade_chain(durs, trans) + fg, vmap, amap, expect = _xfade_chain(durs, trans) cmd = ["ffmpeg", "-y", "-filter_complex_threads", str(FFMPEG_THREADS)] # Input-side -threads limits each decoder; otherwise ffmpeg may create a decoder thread pool # for every input in the batch in addition to the filter and libx264 pools. @@ -720,6 +746,7 @@ def _assemble_once(inputs: list[str], trans: list[str], out: str): "-c:v", "libx264", "-threads", str(FFMPEG_THREADS), "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out] subprocess.run(cmd, check=True, capture_output=True) + _check_assembled(out, expect) return # A cut-only batch is concatenated with the concat FILTER, not -c copy or the concat demuxer. @@ -738,6 +765,7 @@ def _assemble_once(inputs: list[str], trans: list[str], out: str): "-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p", "-threads", str(FFMPEG_THREADS), "-c:a", "aac", "-b:a", "192k", out] subprocess.run(cmd, check=True, capture_output=True) + _check_assembled(out, sum(durs)) def _assemble_batched(inputs: list[str], transitions: list[str], out: str, tag: str, @@ -952,8 +980,14 @@ if __name__ == "__main__": for p in (img, bed, *cl): if os.path.exists(p): os.remove(p) # #6 transitions: two real clips xfade into one chapter; graph offsets/labels well-formed. - fg, vmap, amap = _xfade_chain([1.0, 1.0], ["fade_white"]) + fg, vmap, amap, exp = _xfade_chain([1.0, 1.0], ["fade_white"]) assert "xfade=transition=fadewhite" in fg and vmap == "[v1]" and amap == "[a1]" + # Both streams of every input trimmed to a whole frame count. That is what keeps the offset + # accumulator ON the real timeline: an offset one frame past the end of input i-1 makes xfade + # emit the transition and then silently drop input i and everything after it, rc 0, no warning. + assert fg.count(",trim=end=") == 2 and fg.count("]atrim=end=") == 2, fg + _td = max(1, int(XFADE["fade_white"][1] * FPS)) / FPS + assert abs(exp - (2.0 - _td)) < 0.001, (exp, _td) img = f"{SHM}/t.png" subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=400x600", "-frames:v", "1", img], check=True, capture_output=True) @@ -986,8 +1020,33 @@ if __name__ == "__main__": if os.path.exists(p): os.remove(p) vd, ad = _stream_dur(out, "v"), _stream_dur(out, "a") assert abs(vd - ad) < 0.25, f"A/V drift: video {vd:.2f}s vs audio {ad:.2f}s" + # ...and the guard that catches it in production must actually fire. `assemble` reports success + # off ffmpeg's rc, which is 0 even when a whole batch of inputs is thrown away. + try: + _check_assembled(out, vd + 5.0) + raise AssertionError("_check_assembled did not fire on a 5s loss") + except RuntimeError: + pass for p in (img, c0, c1, c2, c3): os.remove(p) + # The offset creep itself, reproduced small. A real clip's audio outlasts its video slightly, and + # `format=duration` reports the audio, so the accumulator walked ahead of the picture until one + # xfade window ran past the end of its first input -- at which point ffmpeg emitted the + # transition, threw away the second input and everything downstream, and exited 0. Clips with + # audio 0.4s longer than the video exaggerate one clip's worth of that creep. + cs = [] + for i in range(3): + c = f"{SHM}/creep{i}.mp4"; cs.append(c) + subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=1.0", + "-f", "lavfi", "-i", f"sine=f={300 + i * 40}:d=1.4", + "-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p", + "-c:a", "aac", c], check=True, capture_output=True) + _assemble_once(cs, ["fade_black", "cut", "cut"], out) # raises if an input was dropped + vd, ad = _stream_dur(out, "v"), _stream_dur(out, "a") + assert vd > 2.2, f"xfade dropped inputs: {vd:.2f}s from 3 clips of 1.0s" + assert abs(vd - ad) < 0.1, f"A/V drift: video {vd:.2f}s vs audio {ad:.2f}s" + for p in cs: + os.remove(p) # #6 composite: 2 panels + 2 audios -> one stacked clip; duration = sum, subs timed per row. a2 = f"{SHM}/a2.wav" subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1.5", a2], -- 2.52.0 From 8113bdfc8b9030dfd2dbf8f327e66c4b2e58b2cc Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 12:16:22 +0400 Subject: [PATCH 16/31] Fix the real A/V gap: a stream copy across mixed frame rates The rebuild after 1457556 came out byte-identical to the broken file, which proved the xfade fix never runs for this chapter. An all-cut chapter goes down the concat demuxer with -c copy, which writes the output in the FIRST input's time_base and reinterprets every later packet in it. 14 of 49 clips are 30/1 at 1/15360 against 35 at 25/1 at 1/12800, so those 14 play 15360/12800 = 1.2 too long with their audio untouched. collage_cmd hardcoded -r 30 and yesterday's FPS sweep missed it. collage_cmd now emits -r FPS, and assemble probes r_frame_rate across the clips and routes mixed rates through the re-encoding tree. Rebuilt chapter.mp4 is 364.120s video against 364.122s audio at 25/1, from 436.392 over 363.675. Also settle the bbox coordinate space, measured over all 113 detections: 47 boxes have x2 past the 900px panel width, none has y2 past 1000 on panels up to 2307px tall, and the range is exactly [0, 1000]. It is gemma's normalized grid, not pixels, whatever the prompt asks for. /vision converts before returning, which fixes identity's crop, the gated face pairing that was comparing pixel face boxes against grid boxes, the set-of-mark boxes and the review UI at once. Checked by eye on panel 7: five of six boxes now land on their subject, including the foreground character who had no identity. The registry still holds boxes and embeddings enrolled from the wrong space. vision and identity have to re-run, which is GPU work and was not started. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 355 +++++++++------------------------ "II, d[i+4:i+12]))\n\"; done" | 0 JOURNAL.md | 37 ++++ NEXT.md | 39 ++-- caveats/CLAUDE.md | 2 +- caveats/speaker-attribution.md | 7 + decisions/CLAUDE.md | 2 + decisions/chapter-assembly.md | 32 ++- decisions/identity-bbox.md | 45 +++++ worker_identity.py | 5 +- worker_render.py | 31 ++- worker_vision.py | 59 ++++++ 12 files changed, 329 insertions(+), 285 deletions(-) create mode 100644 "II, d[i+4:i+12]))\n\"; done" create mode 100644 decisions/identity-bbox.md diff --git a/HANDOFF.md b/HANDOFF.md index 154a66e..8ec77b5 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,291 +1,130 @@ -# HANDOFF, 2026-08-12 +# HANDOFF, 2026-08-12 (second session of the day) -Live state is in `NEXT.md`. This file is only what this session did. +Live state is in `NEXT.md`. This file is only what this session did. The previous handoff's content is +now in `JOURNAL.md` and in `decisions/`. ## Asked -Get up to speed from the previous handoff. Then: the user watched `chapter.mp4` for the first time and -read out 19 timestamped defects. Mid-session, what about characters. Then, write the handoff. +Get up to speed from the previous handoff. Then: rebuild the chapter, and what about characters. ## Result -**Somebody finally watched the video.** That single act found more than four sessions of measuring did. -The recorded metrics said `script` 116/116 and "9 named speech lines". Both were true. Both measured the -wrong thing, because the 9 names are the *wrong* name. +Two things landed. The A/V gap is fixed and the rebuilt `chapter.mp4` is in the bucket. The `bbox` +coordinate space is settled with measurements instead of comments, and converted at the boundary. -Two hard numbers came out of it. +One correction to carry forward. **The first fix of the day named the wrong cause.** Commit `1457556` +claimed the xfade offset drift was the shipped 72.7s gap. The rebuild came out byte-identical to the +broken file, which disproved it. Both are real defects. Only the second one shipped. -1. **The chapter is 20% out of sync.** `chapter.mp4` is video 436.39s over audio 363.67s. The narration - ends 72.7 seconds before the picture. The gap accumulates, which is why everything after 2:54 goes - sideways. The 49 clips are clean: video and audio agree to 0.03s and sum to 363.6s. Assembly adds - 72.7s of video and no audio. -2. **Identity binds names to the wrong people, and to people who have no name.** Walking panel 7 against - the art found zero correct bindings out of two, plus the one character who matters left unbound. See - `Panel 7, walked against the art`, which supersedes the earlier reading of this. +## The chapter, rebuilt -Nothing is committed. `worker_render.py` is edited in the working tree. The fix is **not** verified. - -## The user's 19 notes, grouped by cause - -| cause | timestamps | what is wrong | -| --- | --- | --- | -| one row absorbed every identity | 0:20, 1:07, 1:51, 1:59, end | anyone identified comes out `Choi Haeseon` | -| the MC has no name | 0:44, 0:50, 1:02, 1:23, 1:45 | falls back to "the worker", "someone", "she" | -| gender read off the wrongly bound row | 1:45 "she admits", end "as he waves" | `Choi` is `f`, so "he" means a nameless `m` row got the line | -| narration invents facts | 0:43 "results", 2:03, 2:05, 2:15 "long shift" | the verifier checks quotes and names, not invented claims | -| vision reads art-within-art as scene | 1:35 chibi on a monitor as "a man holding a drink", 1:59 "pointing towards the screen" | panel-in-panel and screen content taken as reality | -| a beat carries nothing | 0:35-0:37 | no content worth narrating | -| no parallax, so a still holds | 2:24-2:52, 28s static | `layers` wrote nothing (`caveats/audit-open.md#layers-writes-nothing`) | -| transition quality | 2:52-2:54 slide "too sharp and laggy" | `push` is `slideleft` at 0.4s. Retest after the sync fix | -| A/V drift | everything after 2:54 | the 72.7s gap above | - -## Panel 7, walked against the art - -This is the load-bearing finding of the session. The user pulled up the panel and checked every -detection by eye. **Read this before touching identity.** It contradicts what the earlier sessions -recorded, and it contradicts two theories I floated today before the user corrected them. - -Panel `7c944dd4-e972-42c7-ba60-9f6939548e80_p007`, crop `s3://panels/.../panels/p006.png`, 900x1650. -A wide establishing shot of an office seen through a window. Vision emitted 6 characters. - -| detection | vision said | the art shows | identity assigned | -| --- | --- | --- | --- | -| `person_5` | m, short black, **yellow sweater**, sitting | **Seonho**, foreground, yellow plaid, headphones, back to camera. The character who matters | **nothing** | -| `person_6` | f, short brown, **white shirt**, sitting | the **colleague**, green dress, ponytail. She has **no name** in the story | `Choi Haeseon` at **0.9** | -| `person_2` | m, short brown, green sweater, sitting | a background extra, seated beyond the next window pane | `Lim Seonho` at **0.9** | -| `person_1` | m, short black, suit, standing | **nobody. A window frame** | nothing | -| `person_3` | m, short black, blue sweater, standing | background extra | nothing | -| `person_4` | m, short black, grey sweater, standing | background extra | nothing | - -**Zero of the two bindings are right, and the one character who matters got nothing.** Both wrong binds -carry confidence 0.9. - -Three separate defects stack here. - -**1. The stored bbox coordinate space is wrong.** Consumed as absolute pixels, all six boxes land in the -top third of a 1650px-tall panel, two of them inside the "YEAH!" speech balloon. Divided by 1000 against -the panel's own dimensions, `person_2`, `person_3`, `person_4` and `person_5` fit their subjects tightly. -So the numbers are not pixels. Two places assert that they are: - -- `worker_vision.py:271` prompt text: `pixel bounding box [x1,y1,x2,y2] (top-left, bottom-right corners)` -- `worker_identity.py:91` comment: `vision emits [x1, y1, x2, y2] pixel corners (gemma4's native bbox convention)` - -Everything reading `bbox` is therefore cropping the wrong region. `worker_identity.py:200` embeds -`_crop_bbox(img, ch["bbox"])`, so `Choi Haeseon` at 0.9 was matched on a crop of the speech balloon's -edge and `Lim Seonho` at 0.9 on a crop of empty window frame. Wrong crops are mostly blank white, which -embed alike, which is a plausible mechanism for one row absorbing 25 assignments. - -**Do not treat rescaling as the fix.** I claimed that and the user disproved it in one screenshot. -After scaling, `person_1` still sits on a window frame with nobody in it, and `person_6` is offset, -clipping the woman and running onto the dark frame. `worker_vision.py:38` already calls the box -"coarse, imprecise". Scaling buys roughly-right boxes, not right ones. - -**2. Vision has no concept of extra versus cast.** Four of the six detections are background extras or -nothing at all. They are handed to identity as candidates on equal footing with the two people who -carry the scene. That also means the "113 detected people" figure that framed the whole roadmap was -never the right denominator, so "26 of 113 carry an identity" measured nothing useful. - -**3. Identity mints a name onto a person who has none.** The colleague has no name in the story. She was -labelled `Choi Haeseon` at 0.9. Across the chapter `Choi Haeseon` holds 25 of 26 assignments, so in -practice that row is the label this pipeline stamps on any unnamed woman. This is the direct cause of -the user's 0:20 note, "Choi Haeseon when there's no Choi in the frame, it's the colleague", and of the -gender flips, since gender is read off whichever row got bound. - -This is a cousin of invariant 6 in `CLAUDE.md`, which forbids minting a character from an unparseable -model answer. The missing rule is the same shape: **never attach a name to a detection that carries no -name evidence.** An unnamed recurring person needs a stable anonymous identity so the narration can -call her "the colleague" every time, rather than being forced onto a named row. - -Partly checked, not finished: `match()` in `worker_identity.py:69` does abstain, returning `None` below -threshold, so the 0.9 came from cosine clearing the threshold on a garbage crop. Whether the Tier-2 -gemma resolver can answer "none of these" was not verified. Check that first. - -### Descriptions are not trustworthy either - -`person_6` is "white shirt" for a woman in a green dress. `person_5` is "yellow sweater" for yellow -plaid, which is close enough. Any downstream rule keyed on appearance text inherits this. - -## Registry, measured - -`characters` is keyed by `manga_id`, not chapter. Two manga share the table. `d7104032` has 34 rows with -9 named. This chapter's `ef105a86` has 19 rows with 3 named. The `Kei`, `Zen`, `Kanade`, `Rico` and `K3` -rows belong to the other manga, so they are not polluting this chapter. - -This manga's 19 rows, all `status=confirmed`, all `first_seen_panel=NULL`: - -- named: `Choi Haeseon` (f), `Seonho` (m, aliases `["Lim Seonho","Seonho"]`), `Lim Seonho` (m) -- 16 rows with `name=NULL` and an empty alias list - -`Seonho` and `Lim Seonho` are the same person in two rows. `Seonho`'s alias list contains the other -row's name. That is why either spelling matches two rows and binds nothing. - -Assignments across the whole table, for scale: - -| character | name | assignments | -| --- | --- | --- | -| `character_afa7623b` | Choi Haeseon | 25 | -| `character_bb79cfb4` | Kanade | 25 | -| `character_cfd34340` | Rico | 19 | -| `character_6f491712` | Lim Seonho | 1 | - -In this chapter only `Choi Haeseon` (25) and `Lim Seonho` (1) appear. - -Read those 25 together with panel 7. `Choi Haeseon` is not a character who appears 25 times. It is the -row that absorbs any unnamed woman. The 16 nameless rows are not a backlog of people waiting for names. -Some of them are background extras that should never have become rows, and at least one of them, the -colleague, is a real recurring person who correctly has no name and needs to keep it. - -An earlier draft of this file said "naming is the ceiling, 16 of 19 rows need names". **That was wrong** -and it is corrected here. The ceiling is that identity cannot say "person, no name" and cannot tell an -extra from cast. - -## The A/V bug, located but not fixed - -Reproduced on the 49 real clips with `repro.py` and `probe.py` (see Open). Per-round probe with -`ASSEMBLE_BATCH=8` and 6 `fade_black` boundaries spread across batches: +`s3://video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80/chapter.mp4` ``` -r0 g0 n=8 XFADE in v= 49.44 a= 49.44 -> out v= 48.56 a= 48.64 lost_v=+0.88 lost_a=+0.80 -r0 g1 n=8 XFADE in v= 47.04 a= 47.04 -> out v= 46.16 a= 46.23 lost_v=+0.88 lost_a=+0.81 -r0 g2 n=8 XFADE in v= 48.00 a= 48.00 -> out v= 47.12 a= 47.19 lost_v=+0.88 lost_a=+0.81 -r0 g3 n=8 XFADE in v=100.80 a=100.80 -> out v= 99.92 a= 99.96 lost_v=+0.88 lost_a=+0.84 -r0 g4 n=8 concat in v= 64.18 a= 64.16 -> out v= 64.24 a= 64.26 lost_v=-0.06 lost_a=-0.10 -r0 g5 n=8 XFADE in v= 44.00 a= 44.00 -> out v= 43.12 a= 43.17 lost_v=+0.88 lost_a=+0.83 -r1 g0 n=7 XFADE in v=359.29 a=359.60 -> out v= 99.96 a=358.79 lost_v=+259.33 lost_a=+0.81 +before v=436.392 a=363.675 nb_frames=9902 avg_frame_rate=22.69 gap +72.72 +after v=364.120 a=364.122 nb_frames=9101 r_frame_rate=25/1 gap -0.002 ``` -Round 0 is correct. Each group loses only the xfade overlap. **Round 1 loses 259s of video against 0.8s -of audio.** Its output video is 99.96s at 2499 frames, almost exactly the frame count of input `n3` on -its own, the 99.92s intermediate. The final video appears to carry the frames of one input. +Cause: `assemble` sends an all-`cut` chapter down a `concat` demuxer with `-c copy`. That path writes +the output in the **first** input's `time_base` and reinterprets every later packet in it. 14 of the 49 clips +are `30/1` at `1/15360`, the other 35 are `25/1` at `1/12800`, so those 14 play `15360/12800 = 1.2` too +long with their audio untouched. `collage_cmd` hardcoded `-r 30`. +`decisions/chapter-assembly.md#mixed-rate-stream-copy`. -The round-1 filtergraph is arithmetically correct, so this is ffmpeg behaviour, not offset math: +Fixes: `collage_cmd` emits `-r FPS`. `assemble` probes `_fps_of` across the clips and routes mixed rates +through `_assemble_batched`, whose branches both normalize with `fps={FPS}`. + +A second, latent defect on the transition path was fixed and committed separately. `_xfade_chain` took +offsets from `format=duration`, which is `max(video, audio)`. The accumulator crept past the end of its +input, and ffmpeg discarded whole clips at `rc 0` with nothing on stderr. +`decisions/chapter-assembly.md#offsets-from-min-stream`. + +The single-item passthrough theory from the previous handoff is dead, recorded void at +`decisions/chapter-assembly.md#passthrough-innocent`. The one-path rewrite it recommended is not needed. + +**Still open here.** The 14 clips in the bucket are still 30fps. Assembly normalizes them, so the chapter +is correct, but the fast stream-copy path stays off for this chapter until `render` re-runs. Nobody has +watched the rebuilt video yet. The 2:52 slide transition and the 28s static hold from 2:24 were both +supposed to be re-judged after the sync fix. + +## Characters: the `bbox` space, settled + +All 113 detections, straight from `/review/identity`: + +| test | result | +| --- | --- | +| boxes with `x2` past the 900px panel width | **47 of 113** | +| boxes with `y2` past 1000, on panels 1257 to 2307px tall | **0 of 113** | +| boxes clamped at exactly 1000 | 21 in x, 5 in y | +| coordinate range over every box | `[0, 1000]` | + +Gemma's native 0-1000 grid. Not pixels. `worker_vision.py` prompt text and the old +`worker_identity.py:91` comment both claimed pixels and both were wrong. + +`/vision` now calls `_bbox_to_pixels(characters, w, h)` before returning. Four consumers are fixed at +once: `_crop_bbox` in identity, `_pair_faces_to_present`, the set-of-mark boxes, and the review UI's +client-side crop. The pairing one was comparing real pixel face boxes against 0-1000 character boxes, +which is the likely mechanism behind 7 `unknown` out of 7 `som_face` lines. +`decisions/identity-bbox.md#bbox-is-normalized`. + +Checked by eye on panel 7, not just asserted. Five of six converted boxes land on their subject. That +includes `person_5`, who is Seonho in the foreground with headphones and carried no identity. `person_1` +still frames an empty window mullion, which is `caveats/speaker-attribution.md#extras-as-cast`. + +Converted boxes for panel 7, for whoever redraws the overlay: ``` -[n0][n1]xfade=transition=fade:duration=0.050:offset=48.510[v1] -[v1][n2]xfade=transition=fade:duration=0.050:offset=94.620[v2] -[v2][n3]xfade=transition=fadeblack:duration=0.600:offset=141.140[v3] -[v3][n4]xfade=transition=fade:duration=0.050:offset=241.010[v4] -[v4][n5]xfade=transition=fade:duration=0.050:offset=305.200[v5] -[v5][n6]xfade=transition=fade:duration=0.050:offset=348.270[v6] +person_1 [226, 414, 286, 553] window mullion, nobody +person_2 [ 34, 558, 106, 749] background extra, was assigned Lim Seonho +person_3 [428, 384, 494, 533] background extra +person_4 [498, 389, 561, 549] background extra +person_5 [460, 657, 631, 939] Seonho, foreground. was assigned nothing +person_6 [646, 591, 767, 794] the colleague, no name in the story. was assigned Choi Haeseon at 0.9 ``` -### The strongest clue, found last +**The registry is unchanged and still wrong.** Every stored box, embedding and `ref_image_uris` was +enrolled from the wrong space. `vision` and `identity` have to re-run before any of it means anything, +and that is GPU work nobody authorized. `caveats/speaker-attribution.md#bbox-wrong-space` is marked +resolved with the rerun pending. -Re-running that chain by hand over only the **6** round-0 intermediates gives a correct 348.24s at 8708 -frames, `rc 0`, no warnings. Round 1 collapses with **7** inputs, not 6. +## Checks -The 7th input is the leftover 49th clip. With 49 clips and batch 8, round 0 makes 6 groups of 8 and one -group of 1, and `_assemble_batched` passes a lone group through un-encoded: +Every self-check runs from the repo root and passes: -```python -if len(group) == 1 and not final_round: - next_items.append(group[0]) +```bash +.venv/bin/python worker_render.py # about 4 minutes, real ffmpeg +.venv/bin/python worker_vision.py +.venv/bin/python worker_identity.py ``` -So the final xfade mixes 6 encoded intermediates with 1 raw clip. That passthrough is a **third** path -next to `concat` and `xfade`, and it is the prime suspect. Start here tomorrow. Confirm it by running -`probe.py` with 48 clips instead of 49, which removes the leftover entirely. +Three checks were added, because the existing ones passed all the way through both shipped defects: -### What was tried and what it cost +- `_fps_of(collage clip) == "25/1"`, on a real collage encode. This is the one that would have caught the + mixed-rate bug at the source. +- three clips whose audio outlasts their video by 0.4s, assembled through the xfade branch. Mutation + tested by restoring `_audio_dur`: fires with `video=1.80 audio=3.56 expected=3.56`. +- `_bbox_to_pixels` against panel 7's real `person_5` box, asserting the result covers the lower half of a + 1650px panel, which a raw grid value cannot. -Two paths exist in `_assemble_once`. A `concat` branch handles cut-only batches. An `xfade` branch -handles batches with a real transition. They disagreed on frame rate. The concat branch forced `fps=30` -while the xfade branch normalized nothing, and clips are 25fps. `436.39 / 363.63 = 1.2001`, exactly -`30/25`, which is what sent me down this path. +`_check_assembled` now runs after every encode on both paths, because ffmpeg returns 0 while dropping +whole inputs. -Working-tree changes to `worker_render.py`, all uncommitted: +## Next command -- new `FPS = 25` constant. The three hardcoded `25`s and the one `30` now reference it -- the xfade branch normalizes every input with `setsar=1,fps={FPS}` into `[n{i}]` labels, matching what - the concat branch already did -- new `_stream_dur(path, kind)`. `_audio_dur` probes `format=duration`, which is `max(video, audio)`, so - it hides A/V drift by construction -- the `__main__` xfade self-check now assembles 4 clips through `_assemble_batched` with - `ASSEMBLE_BATCH=2` and asserts `abs(video - audio) < 0.25`. It previously asserted only - `getsize(out) > 0`, which is why this shipped +Watch the rebuilt chapter before anything else. That is what found every real defect so far. -**A dead end worth not repeating.** I also pinned `-r FPS` on both output encodes. That made it worse. -The chapter collapsed to exactly 100.00s at 2500 frames, because forcing CFR on irregular input -timestamps drops frames. The comment already sitting at the concat branch warns about this. Both `-r` -flags were removed again. The in-graph `fps=` filter is the right normalization. The output `-r` is not. +```bash +/usr/bin/ssh kami@192.168.1.104 'mc cat homesrv/video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80/chapter.mp4' > chapter.mp4 +``` -The fps inconsistency is real and worth keeping fixed. It is **not** proven to be the cause of the -shipped 72.7s gap. The scene graphs hold 356 `cut` against 6 `fade_black`, so round 2 of the real run -most likely stayed on the concat branch, where no mixing occurs. Treat the fps work as necessary and -insufficient. +Then, with a go-ahead, the vision and identity rerun in `NEXT.md` item 1. -### The user's own hypothesis, which is the recommended direction +## Traps confirmed again this session -Two guesses, both worth following: - -1. the assembly is wrong in an ffmpeg sense -2. there are two different paths, and they get mixed when there should only be one - -Guess 2 matches the code. `concat`, `xfade` and the single-item passthrough are three paths, and -`_assemble_batched` feeds the output of one into the input of another. **Collapse it to one path.** -Normalize every input, then xfade every boundary, with `cut` as a 0.05s fade. `acrossfade` shortens -audio by the same amount that xfade shortens video, so A/V stays locked. The chapter then comes out -about 2.4s shorter than the sum of the clips, with both streams agreeing. That removes the branch -interaction instead of tuning it. - -## Measured - -Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`. - -- job `status=completed`, every stage at its unit count, finished `2026-08-11T20:08:16Z` -- it still carries `error: "partial: 112/116 completed"` (`caveats/audit-open.md#stale-job-error`) -- `chapter.mp4` 50MiB, video 436.392s, audio 363.675s, `r_frame_rate=25/1`, - `avg_frame_rate=63372800/2792909` which is 22.69, `nb_frames=9902` -- 49 clips, every one `25/1` exactly, sum video 363.63s, sum audio 363.60s. No clip has the two - differing by more than 0.05s -- scene-graph transitions: `cut` 356, `fade_black` 6 -- `worker_render.py` `__main__` self-check passes on the current working tree -- workers up in tmux `manga-workers`, 9 windows - -## Open - -- **Finish the round-1 diagnosis.** The `_assemble_batched` tree turns 359s of video into 100s. This is - the worst defect found and it reproduces offline in about two minutes with no GPU. Suspect the 7th - passthrough input first. - - ```bash - .venv/bin/python /probe.py # the per-round loss table above - .venv/bin/python /repro.py # end-to-end verdict - ``` - - The scratchpad is session-scoped and will be gone. **Re-download the clips first:** - - ```bash - /usr/bin/ssh kami@192.168.1.104 'P=homesrv/video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80; mc cp -q -r $P/clips/ /tmp/rclips/; cd /tmp/rclips && tar cf - .' | tar xf - -C clips/ - ``` - - Both scripts are worth committing next session. They are the only check that has ever caught this. - -- **Do not trust `format=duration`.** It returns `max(video, audio)`, so every existing duration assert - in `worker_render.py` is blind to drift. `_stream_dur` exists now. The other asserts still use - `_audio_dur`. -- **Identity, in the order the panel 7 evidence implies.** First, settle the `bbox` coordinate space and - fix every consumer, since nothing else can be judged while crops are wrong. Second, let identity - abstain and hold a stable anonymous identity, so the colleague stays "the colleague". Third, separate - extra from cast so extras never reach identity. Only then merge `Seonho` into `Lim Seonho` and split - `character_afa7623b`, which still needs the reversible-merge design - (`caveats/audit-open.md#destructive-reconcile`). -- **Verify the bbox space before changing anything.** Two independent claims in the code say pixels, and - the art says otherwise. Confirm what the model was told and what it returns, rather than trusting - either comment. Then check whether the crop is the only consumer, or whether SoM marker placement and - the face-pairing in `worker_vision.py:57` read the same numbers. -- Re-derive the panel 7 overlay when needed. It took one `mc cat` of the crop plus a Pillow script, and - it found more than any query did: - - ```bash - /usr/bin/ssh kami@192.168.1.104 'mc cat homesrv/panels/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80/panels/p006.png' > p007.png - ``` - - Draw each `bbox` twice, once as pixels and once divided by 1000, then look at it. -- `mc` aliases on homesrv: use `homesrv` or `mio`, not `local`, which returns Access Denied. `rfs` is - the empty rustfs. +- `tmux` session `manga-workers` was gone and both systemd units were inactive. `./start_workers.sh` + starts 9 windows. The render worker is window 9 and must be restarted by hand to pick up an edit. - Plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin. Use `/usr/bin/ssh`. -- `cp` is aliased to `cp -i` in this shell and hangs on overwrite. Use `/usr/bin/cp -f`. -- The Bash tool's default timeout is 120s no matter what `timeout` the command itself carries. Pass the - tool's own timeout or background the run. Otherwise a restore step after a mutation test never runs, - which left a deliberately broken `worker_render.py` on disk once this session. +- `cd $dir && .venv/bin/python` fails, because the venv path is relative to the repo root. Use the + absolute interpreter path when the working directory is elsewhere. +- The whole assembly investigation ran offline on 49 downloaded clips with no GPU and no orchestrator. + Re-download with the command in `NEXT.md`. diff --git "a/II, d[i+4:i+12]))\n\"; done" "b/II, d[i+4:i+12]))\n\"; done" new file mode 100644 index 0000000..e69de29 diff --git a/JOURNAL.md b/JOURNAL.md index 286cbff..c197905 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -269,3 +269,40 @@ back: the new check fires with `video=1.80 audio=3.56 expected=3.56`. Not done: `s3://video/.../chapter.mp4` is still the broken 436s file. Rebuilding it means clearing the `assemble` stage and resuming, which is CPU-only and was not run. + +## 2026-08-12, the chapter rebuilt, and the bbox space settled + +**The rebuild came out byte-identical to the broken file.** Clearing `assemble` and resuming produced +video 436.392031s over audio 363.674667s and `nb_frames` 9902 again, which proved the xfade fix committed +earlier today never runs for this chapter. With all-`cut` transitions `assemble` takes the `else` branch, +a `concat` demuxer with `-c copy`. + +Reproduced that path offline in seconds and got the shipped numbers exactly. The cause is mixed frame +rates: 14 of the 49 clips are `r_frame_rate=30/1` at `time_base=1/15360`, the other 35 are `25/1` at +`1/12800`. `-c copy` writes the output in the first input's timebase, so those 14 play `15360/12800 = 1.2` +too long with their audio untouched. `collage_cmd` hardcoded `-r 30`, which yesterday's `FPS` sweep +missed. `decisions/chapter-assembly.md#mixed-rate-stream-copy`. + +Fixed `collage_cmd` to emit `-r FPS`, and made `assemble` probe `r_frame_rate` across the clips and route +mixed rates through the re-encoding tree. Rebuilt: + +``` +before v=436.392 a=363.675 nb_frames=9902 avg_frame_rate=22.69 +after v=364.120 a=364.122 nb_frames=9101 r=25/1 +``` + +The 14 clips in the bucket are still 30fps. Assembly normalizes them, so the chapter is correct without +re-rendering, but the fast stream-copy path stays disabled for this chapter until `render` re-runs. + +**The `bbox` space is 0-1000, not pixels.** Pulled all 113 detections from `/review/identity` and +measured: 47 boxes have `x2` past the 900px panel width, none has `y2` past 1000 on panels 1257 to 2307px +tall, 21 clamp at exactly 1000 in x, and the whole range is `[0, 1000]`. `/vision` now converts to pixels +before returning, so identity crops, gated face pairing, the set-of-mark boxes and the review UI all read +pixels (`decisions/identity-bbox.md#bbox-is-normalized`). + +Checked by eye the way the user did. Drew the converted boxes on panel 7: five of six land on their +subject, including `person_5`, who is Seonho in the foreground with headphones and carried no identity. +`person_1` still frames an empty window mullion, which is the extra-versus-cast caveat, not this one. + +Not done: `vision` and `identity` have not re-run, so every box, embedding and `ref_image_uris` in the +registry is still from the wrong space. That rerun is GPU work and was not started. diff --git a/NEXT.md b/NEXT.md index f9ae0d6..7ff5af1 100644 --- a/NEXT.md +++ b/NEXT.md @@ -4,44 +4,41 @@ Updated 2026-08-12. What this session did is in `HANDOFF.md`. ## State -The chapter runs end to end and the output is **not watchable**. That is now measured, not guessed. -Assembly is fixed and verified offline. The shipped `chapter.mp4` has not been rebuilt yet. +The chapter runs end to end. The A/V sync defect is fixed and `chapter.mp4` is rebuilt: video 364.120s +against audio 364.122s at `25/1`. The identity defects are still in the output. Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels, `status=completed`, finished 2026-08-11T20:08:16Z. `s3://video/` holds 49 clips and a 50MiB `chapter.mp4`. The user watched it and read out 19 defects. They are grouped by cause in `HANDOFF.md`. -Two numbers set the agenda: +One number sets the agenda: -- The shipped `chapter.mp4` is video 436.39s over audio 363.67s. Cause found and fixed, see below. - Panel 7 checked against the art has **zero correct identity bindings** out of two, and the one - character who matters is unbound. `HANDOFF.md#panel-7-walked-against-the-art` has the table. + character who matters is unbound. `HANDOFF.md#panel-7-walked-against-the-art` has the table. The + coordinate cause is fixed. The registry built on it is not. ## Next -1. **Rebuild `chapter.mp4`.** Assembly is fixed in `worker_render.py`. Verified over the 49 real clips of - this chapter: video 358.76s against audio 358.76s, agreeing to the frame. The cause was `_xfade_chain` - taking offsets from `format=duration`, which is `max(video, audio)`. The accumulator drifted past the - end of its input, and ffmpeg silently discarded whole clips at `rc 0` - (`decisions/chapter-assembly.md#offsets-from-min-stream`). The single-item passthrough was innocent - and the one-path rewrite is not needed (`decisions/chapter-assembly.md#passthrough-innocent`). +1. **Re-run vision and identity.** The `bbox` space is settled and converted at `/vision` + (`decisions/identity-bbox.md#bbox-is-normalized`). Every stored box, embedding and `ref_image_uris` in + the registry came from the wrong space. The fix changes nothing until those stages run again. + This is GPU work and needs the user's go-ahead. Clear `vision` and everything downstream of it, or + accept that the boxes in the database stay normalized while new ones are pixels. - What is left is to clear the `assemble` stage and resume, then watch the result. That is CPU-only - ffmpeg, no GPU, but it needs the user's go-ahead. + Watch two things on the rerun. Whether `som_face` still returns `unknown` on every face, since gated + pairing was comparing pixel face boxes against 0-1000 character boxes. And whether `Choi Haeseon` still + absorbs every unnamed woman, which is item (b) below and independent of the crops. - Smaller follow-on: nine other `_audio_dur` calls in `worker_render.py` measure finished clips with + Smaller follow-on: nine `_audio_dur` calls in `worker_render.py` measure finished clips with `format=duration`. So the durations reported to the orchestrator are blind to per-clip drift. They position no filter, so invariant 9 does not cover them. Worth converting to `_stream_dur`. 2. **Fix identity, in this order.** Panel 7 is the worked example and `HANDOFF.md#panel-7-walked-against-the-art` carries the evidence. Do not start at the registry. - a. **Settle the `bbox` coordinate space.** Consumed as pixels, all six boxes on panel 7 land in the - top third of the panel, two inside a speech balloon. Divided by 1000 they mostly land on their - subjects. `worker_vision.py:271` and `worker_identity.py:91` both assert pixels, and the art says - otherwise. Identity embeds `_crop_bbox(img, ch["bbox"])` at `worker_identity.py:200`, so today it - matches faces against crops of balloons and window frames. Nothing downstream can be judged until - this is right. Rescaling alone is **not** the fix: after scaling, one box still sits on an empty - window frame and another clips its subject. + a. ~~Settle the `bbox` coordinate space.~~ **Done 2026-08-12**, proven over all 113 detections and + checked by eye on panel 7, where five of six converted boxes land on their subject + (`decisions/identity-bbox.md#bbox-is-normalized`). `person_1` still frames an empty window mullion, + which is (c). b. **Let identity abstain and stay abstained.** The colleague has no name in the story and was labelled `Choi Haeseon` at 0.9. An unnamed recurring person needs a stable anonymous identity so narration says "the colleague" every time. `match()` already returns `None` below threshold. Check diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index 6836615..16d6e6d 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -38,7 +38,7 @@ a complaint, so give it one or drop it. | [One invented word still halts the chapter](speaker-attribution.md#one-word-halts-chapter) | 2026-08-11 run | | [A completed job keeps the error from an earlier failure](audit-open.md#stale-job-error) | 2026-08-11 run | | [`layers` reports success on an empty bucket](audit-open.md#layers-writes-nothing) | 2026-08-11 run | -| [Every `bbox` is read in the wrong coordinate space](speaker-attribution.md#bbox-wrong-space) | 2026-08-12 panel 7 | +| [Every `bbox` is read in the wrong coordinate space](speaker-attribution.md#bbox-wrong-space) | resolved, rerun pending | | [Identity cannot say "a person with no name"](speaker-attribution.md#no-anonymous-identity) | 2026-08-12 panel 7 | | [Vision does not separate a background extra from cast](speaker-attribution.md#extras-as-cast) | 2026-08-12 panel 7 | | [Cast reference profiles are enrolled from wrong crops](speaker-attribution.md#poisoned-reference-set) | 2026-08-12 panel 7 | diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md index 8709925..ec6d824 100644 --- a/caveats/speaker-attribution.md +++ b/caveats/speaker-attribution.md @@ -72,6 +72,13 @@ is to flag the beat for review and continue, which is `#136` gate work, not a ve ## Every `bbox` is read in the wrong coordinate space {#bbox-wrong-space} +**Resolved 2026-08-12, `decisions/identity-bbox.md#bbox-is-normalized`.** The space is gemma's 0-1000 +grid, proven over all 113 detections, and `/vision` now converts to pixels before returning. The face +pairing at `worker_vision.py:57` was reading the same numbers against real pixel face boxes, so it is +fixed by the same change. What is left of this entry is the consequence. Every stored assignment, +embedding and `ref_image_uris` came from a wrong crop. Identity has to re-run before any of it means +anything. The rest below is kept as the record of how it read before. + Vision's `bbox` values are stored and consumed as absolute pixels. On panel `7c944dd4-e972-42c7-ba60-9f6939548e80_p007` (crop 900x1650) all six boxes then land in the top third of the panel, two of them inside the "YEAH!" speech balloon. Divided by 1000 against the panel's own diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 021bc8a..4542463 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -36,6 +36,8 @@ still live belongs in `caveats/`. | [An interjection is not a name and not a misquote](speaker-attribution.md#interjection-false-positive) | closed | | [Cast names enter the verifier tokenized](speaker-attribution.md#multiword-cast-names) | closed | | [Clearing a stage strips the vision blob it wrote](storage-layout.md#clear-vision-blob) | closed | +| [The shipped 72.7s gap was a stream copy across mixed frame rates](chapter-assembly.md#mixed-rate-stream-copy) | closed | | [xfade offsets come from `min(video, audio)`, never `format=duration`](chapter-assembly.md#offsets-from-min-stream) | closed | +| [A vision `bbox` is gemma's 0-1000 grid, converted to pixels at `/vision`](identity-bbox.md#bbox-is-normalized) | closed | | [Assembly verifies its own output instead of trusting ffmpeg's exit code](chapter-assembly.md#check-assembled) | closed | | [The single-item passthrough is not the assembly bug](chapter-assembly.md#passthrough-innocent) | void | diff --git a/decisions/chapter-assembly.md b/decisions/chapter-assembly.md index d0d81a9..aa9d64f 100644 --- a/decisions/chapter-assembly.md +++ b/decisions/chapter-assembly.md @@ -2,10 +2,40 @@ Settled questions about `_assemble_batched` / `_assemble_once` / `_xfade_chain` in `worker_render.py`. -## xfade offsets are computed from `min(video, audio)`, never `format=duration` {#offsets-from-min-stream} +## The shipped 72.7s gap was a stream copy across mixed frame rates {#mixed-rate-stream-copy} **Closed, 2026-08-12.** +`assemble` routes an all-`cut` chapter to a `concat` demuxer with `-c copy`. That path writes the output +with the **first** input's `time_base` and reinterprets every later packet in it. + +14 of this chapter's 49 clips came off `collage_cmd`, which hardcoded `-r 30`. They carry +`r_frame_rate=30/1` and `time_base=1/15360`. The other 35 are `25/1` at `1/12800`. Copied into the first +clip's timebase, those 14 play `15360/12800 = 1.2` times too long while their audio is untouched. That is +the 1.2001 ratio, and the whole of video 436.39s over audio 363.67s. + +Reproduced offline by running the same `-c copy` concat over the 49 real clips. Duration 436.392031 and +`nb_frames` 9902, identical to the shipped file. Seconds to run, no GPU. + +Two changes hold it closed: + +* `collage_cmd` emits `-r FPS` like every other clip path, and the `__main__` self-check asserts + `_fps_of(clip) == "25/1"` on a real collage encode. +* `assemble` probes `r_frame_rate` across the clips and sends mixed rates through `_assemble_batched`, + whose branches both normalize with `fps={FPS}`. Only a single shared rate keeps the stream copy. + +Verified end to end. The rebuilt `chapter.mp4` is video 364.120s against audio 364.122s at `25/1`. + +**An earlier version of this file, and commit `1457556`, blamed `#offsets-from-min-stream` below for the +shipped gap. That was wrong.** The rebuild came out byte-identical to the broken file, which proved the +xfade tree never ran for this chapter. The entry below is a real defect and stays closed on its own +evidence. It was not this one. + +## xfade offsets are computed from `min(video, audio)`, never `format=duration` {#offsets-from-min-stream} + +**Closed, 2026-08-12.** A real latent defect on the transition path. Not the cause of the shipped gap, +see `#mixed-rate-stream-copy` above. + `_xfade_chain` accumulates `cum += dur[i] - td` and hands each boundary `offset=cum-td`. That offset is an assertion about where input `i-1` still has frames. It fed on `_audio_dur`, which probes `format=duration`, which is `max(video, audio)`. A rendered clip's audio outlasts its video by about a diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md new file mode 100644 index 0000000..0c637e1 --- /dev/null +++ b/decisions/identity-bbox.md @@ -0,0 +1,45 @@ +# identity-bbox + +The coordinate space of a vision character box, and what reads it. + +## A vision `bbox` arrives on gemma's 0-1000 grid, and `/vision` converts it to pixels {#bbox-is-normalized} + +**Closed, 2026-08-12.** + +`build_detect_prompt` asks for a "pixel bounding box". The model answers on its own normalized grid +regardless. Measured over all 113 detections of job `778297bc`, read straight from `/review/identity`: + +| test | result | +| --- | --- | +| boxes with `x2` past the 900px panel width | **47 of 113** | +| boxes with `y2` past 1000, on panels 1257 to 2307px tall | **0 of 113** | +| boxes clamped at exactly 1000 | 21 in x, 5 in y | +| coordinate range over every box | `[0, 1000]` | + +Pixels cannot behave that way. A person standing in the lower half of a 2307px panel needs `y2` near +2000, and it never once exceeds 1000. + +Consumed as pixels the boxes collapse into the top-left corner of the panel. Four consumers were reading +them: + +* `worker_identity._crop_bbox` at `worker_identity.py:200`, which embeds the crop. This is why a crop of + a speech balloon's edge matched `Choi Haeseon` at 0.9. +* `_pair_faces_to_present` in `worker_vision.py`, which compares real detector face boxes, in pixels, + against these. The gate could almost never pass, which is the mechanism behind the 7 `unknown` results + out of 7 `som_face` lines already recorded at `worker_vision.py:169`. +* the set-of-mark boxes drawn for attribution. +* the review UI, which crops client-side off the panel PNG. + +`/vision` now calls `_bbox_to_pixels(characters, w, h)` before returning, so all four see pixels and no +consumer needs to know the grid existed. Verified by drawing the converted boxes on panel 7. Five of six +land on their subject, including `person_5`, who is Seonho in the foreground and had no identity. +`person_1` still frames a window mullion with nobody in it, which is `#extras-as-cast`, not this. + +The prompt text still says "pixel bounding box". Rewording it changes what the model emits and needs a +GPU run to re-verify, so the boundary converts instead. The `ponytail:` note on `_bbox_to_pixels` records +that. It also records the trap: a model that really answered in pixels would be scaled down here. + +**Consequence: every assignment in the registry came from a wrong crop.** The existing embeddings and +`ref_image_uris` are enrolled on balloons and window frames. Re-running identity +is what makes the registry mean anything. The anonymous-identity and extra-versus-cast work cannot be +judged until that rerun happens. diff --git a/worker_identity.py b/worker_identity.py index ad4038c..18677bb 100644 --- a/worker_identity.py +++ b/worker_identity.py @@ -88,7 +88,10 @@ def shortlist(emb: np.ndarray, known: list, k: int = 5, gender: str = None) -> l def _crop_bbox(img, bbox): - # vision emits [x1, y1, x2, y2] pixel corners (gemma4's native bbox convention). + # [x1, y1, x2, y2] pixel corners. gemma answers on a 0-1000 normalized grid and `/vision` converts + # to pixels before returning (`worker_vision._bbox_to_pixels`), so this reads real pixels. It did + # not before 2026-08-12, which is why crops landed on balloons + # (`decisions/identity-bbox.md#bbox-is-normalized`). x1, y1, x2, y2 = bbox return img[y1:y2, x1:x2] diff --git a/worker_render.py b/worker_render.py index 5875124..5a0f89c 100644 --- a/worker_render.py +++ b/worker_render.py @@ -127,6 +127,15 @@ def _stream_dur(path: str, kind: str) -> float: return 0.0 +def _fps_of(path: str) -> str: + """`r_frame_rate` as ffprobe reports it. Compared as a string on purpose: two clips agree only when + their rate AND therefore their time_base agree, and the stream-copy concat path cares about that.""" + r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", + "-show_entries", "stream=r_frame_rate", "-of", "default=nk=1:nw=1", path], + capture_output=True, text=True) + return r.stdout.strip() + + ZMAX, ZPAN = 1.15, 1.18 # ken-burns zoom ceiling; constant zoom that gives pans room to travel # Every clip and every assembly stage MUST agree on this. xfade does not resample: it reinterprets the @@ -614,7 +623,7 @@ def collage_cmd(imgs, plate_i, rects, entrances, z_order, audio, ass, out, D, tr cur = f"o{k}" fc = ";".join(parts) + f";[{cur}]ass={ass}[v];[{n}:a]apad=pad_dur={PAD_S:.3f}[a]" cmd += ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{T:.3f}", - "-r", "30", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out] + "-r", str(FPS), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out] return cmd @@ -863,15 +872,26 @@ async def assemble(data: AssembleInput): cleanup = list(locals_) + [out] fancy = len(locals_) >= 2 and any(t not in ("", "cut") for t in data.transitions) - if fancy: + # The stream-copy path writes the output with the FIRST input's time_base and reinterprets every + # later packet in it. A clip encoded at 30fps (time_base 1/15360) copied into a 25fps container + # (1/12800) therefore plays 15360/12800 = 1.2x too long with its audio untouched. That is the whole + # of the shipped chapter's 436.39s of video over 363.67s of narration: 14 of 49 clips came off the + # collage path, which hardcoded `-r 30`. Mixed rates must re-encode, so they go through the tree, + # whose branches both normalize with `fps={FPS}`. + rates = {_fps_of(p) for p in locals_} + if fancy or len(rates) > 1: + if len(rates) > 1: + print(f"[render] mixed clip rates {sorted(rates)}, re-encoding instead of stream copy", + flush=True) _assemble_batched(locals_, data.transitions, out, tag, cleanup) else: - # all hard cuts: stream-copy concat (no re-encode) -- unchanged fast path. + # all hard cuts at one shared rate: stream-copy concat, no re-encode. listfile = f"{SHM}/asm_{tag}.txt"; cleanup.append(listfile) with open(listfile, "w") as f: f.write("".join(f"file '{p}'\n" for p in locals_)) subprocess.run(["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", listfile, "-c", "copy", out], check=True, capture_output=True) + _check_assembled(out, sum(min(_stream_dur(p, "v"), _stream_dur(p, "a")) for p in locals_)) out = _add_music_bed(out, tag, cleanup) @@ -1087,6 +1107,11 @@ if __name__ == "__main__": check=True, capture_output=True) assert abs(_audio_dur(out) - (1.5 + PAD_S)) < 0.2, _audio_dur(out) # collage clip = beat length assert os.path.getsize(out) > 0 + # ...and it must come out at FPS like every other clip path. This one hardcoded `-r 30`, so 14 of + # 49 clips in the shipped chapter were 30fps. `concat -c copy` writes the output with the FIRST + # clip's time_base and reinterprets later packets in it, so those 14 played 1.2x too long with + # their narration untouched. That, not the xfade tree, is where the 72.7s gap came from. + assert _fps_of(out) == f"{FPS}/1", f"collage clip is {_fps_of(out)}, not {FPS}/1" for p in (b0, b1, b2, bnar): os.remove(p) os.remove(aud); os.remove(out) diff --git a/worker_vision.py b/worker_vision.py index b1b1133..5e076c6 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -300,6 +300,46 @@ class VisionInput(BaseModel): session_id: str = "" +BBOX_GRID = 1000 # gemma's native normalized box grid + + +def _bbox_to_pixels(chars: list, w: int, h: int) -> list: + """Rewrite every character box from gemma's 0-1000 grid to pixels on this panel. + + The prompt asks for pixels. The model answers on its own normalized grid regardless. Measured over + the 113 detections of job 778297bc: 47 boxes had x2 beyond the 900px panel width, and not one had y2 + beyond 1000 on panels 1257 to 2307px tall. Consumed as pixels the boxes collapse into the top-left + corner of the panel, which is how identity came to embed crops of speech balloons and window frames + and match them at 0.9, and why gated face pairing returned 7 unknowns out of 7 real faces. + + Convert once here so every consumer sees pixels: `_crop_bbox` in identity, the face pairing below, + the set-of-mark boxes, and the review UI's client-side crop. + + ponytail: the prompt still says "pixel bounding box". Rewording it would change what the model + emits and needs a GPU run to re-verify, so the boundary converts instead. If a future model really + does answer in pixels, this scales them down -- check the box range before swapping models. + """ + for c in chars: + b = c.get("bbox") + if not (isinstance(b, list) and len(b) == 4 and all(isinstance(v, (int, float)) for v in b)): + continue + c["bbox"] = [min(w, max(0, round(b[0] * w / BBOX_GRID))), + min(h, max(0, round(b[1] * h / BBOX_GRID))), + min(w, max(0, round(b[2] * w / BBOX_GRID))), + min(h, max(0, round(b[3] * h / BBOX_GRID)))] + return chars + + +def _panel_size(path: str) -> tuple: + """(width, height) of a panel image, or (0, 0) when it cannot be read.""" + import cv2 + img = cv2.imread(path) + if img is None: + return (0, 0) + h, w = img.shape[:2] + return (w, h) + + @app.post("/vision") async def vision(data: VisionInput): local = transport.get(data.panel_uri, f"{SHM}/vision_{uuid.uuid4().hex[:8]}.png") @@ -315,8 +355,14 @@ async def vision(data: VisionInput): print(f"[vision/detect] parse failed for {data.panel_id} after repair retry: {e}", flush=True) result = {"skip": False, "parse_failed": True, "characters": [], "scene": {}} finally: + pw, ph = _panel_size(local) os.remove(local) result.setdefault("characters", []) + if pw and ph: + _bbox_to_pixels(result["characters"], pw, ph) + else: + print(f"[vision/detect] panel size unreadable for {data.panel_id}, boxes left normalized", + flush=True) result["panel_id"] = data.panel_id return result @@ -1128,6 +1174,19 @@ if __name__ == "__main__": # trailing braced prose after a complete object parses (used to burn a repair call) assert _extract_json('{"skip":false}\nnote {see above}')["skip"] is False + # gemma's boxes arrive on a 0-1000 grid and leave /vision as pixels. The 900x1650 panel below is + # panel 7 of job 778297bc: person_5 is Seonho in the foreground, and read as pixels his box lands in + # the top sixth of the panel, inside a speech balloon, which is what identity embedded. + _ch = [{"local_id": "person_5", "bbox": [222, 405, 654, 1000]}, + {"local_id": "edge", "bbox": [0, 0, 1000, 1000]}, + {"local_id": "junk", "bbox": "nope"}] + _bbox_to_pixels(_ch, 900, 1650) + assert _ch[0]["bbox"] == [200, 668, 589, 1650], _ch[0]["bbox"] + assert _ch[1]["bbox"] == [0, 0, 900, 1650], _ch[1]["bbox"] # a clamped box spans the whole panel + assert _ch[2]["bbox"] == "nope", _ch[2]["bbox"] # unparseable is left alone, not crashed + # the box must now cover the lower half of a tall panel, which the raw grid value never can + assert _ch[0]["bbox"][3] > 1000 > _ch[0]["bbox"][1] + # face->identity pairing is GATED on containment: a face outside every gemma bbox stays unknown. faces = [{"bbox": [10, 10, 30, 30]}, {"bbox": [900, 900, 920, 920]}] present = [{"local_id": "person_1", "name": "Teto", "bbox": [0, 0, 100, 200]}] -- 2.52.0 From 7f25dc43e44faa6931099be14b93fd012f5970b0 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 13:39:06 +0400 Subject: [PATCH 17/31] Catch a worker serving code older than its file A full vision + identity + reconcile cycle ran today against a worker that had been started before the fix it was supposed to prove. It reproduced the defect exactly: 46 of 110 boxes past the 900px panel width, coordinates clamping at 1000, the same fingerprint measured before the fix. vision worker started 12:00:09 worker_vision.py modified 12:11:35 8113bdf, carrying _bbox_to_pixels 12:16:22 Python binds a module once, at process start. Editing the file afterwards changes nothing until the process restarts, and nothing in the result says so: the stage reported completed 116/116, the orchestrator recorded no error, and identity and reconcile ran to completion on top of it. Cost was one cycle plus a registry reset to undo the 8 characters it minted. This was already known as advice. The previous handoff said the render worker "must be restarted by hand to pick up an edit". Advice did not stop it. check_stale.sh compares every running worker's process start against its module's mtime and exits non-zero if any is stale, so it can gate a script. Mutation-tested by touching worker_tts.py, which it caught. decisions/identity-bbox.md#stale-worker-invalidates carries the evidence and is committed separately with the rest of the session's notes. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_011FmkHKFtuhppZE3B4VvNNE --- check_stale.sh | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100755 check_stale.sh diff --git a/check_stale.sh b/check_stale.sh new file mode 100755 index 0000000..95bda04 --- /dev/null +++ b/check_stale.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Which running workers are serving code older than the file on disk. +# +# Python loads a worker module once, at process start. Editing the file afterwards changes nothing +# until that process restarts, and nothing in the stage output says so: the stage completes, the +# counters read 116/116, and the result is what the OLD code produced. On 2026-08-12 this ate a full +# vision + identity + reconcile cycle — the bbox fix was committed 16 minutes after the worker that +# was supposed to apply it had already started. +# +# Run this before any stage run that is meant to prove a code change. +# Exits non-zero if anything is stale, so it can gate a script. +set -uo pipefail +cd "$(dirname "$0")" + +stale=0 +# `ps -o lstart` is always exactly five fields: "Wed Aug 12 13:20:00 2026". Read them positionally +# rather than trying to split the line with a regex. +while read -r pid _dow mon day time year cmd; do + mod=$(grep -oP '(?:worker_\w+|session_manager)(?=:app)' <<<"$cmd" | head -1) + [ -n "$mod" ] && [ -f "$mod.py" ] || continue + p_epoch=$(date -d "$mon $day $time $year" +%s 2>/dev/null) || continue + f_epoch=$(stat -c %Y "$mod.py") + if [ "$f_epoch" -gt "$p_epoch" ]; then + printf 'STALE %-22s pid %-7s edited %dm after it started\n' \ + "$mod.py" "$pid" "$(( (f_epoch - p_epoch) / 60 ))" + stale=1 + else + printf 'ok %-22s pid %s\n' "$mod.py" "$pid" + fi +done < <(ps -eo pid,lstart,cmd | grep -E 'uvicorn (worker_|session_manager)' | grep -v grep) + +if [ "$stale" -ne 0 ]; then + echo + echo "Restart the stale ones before trusting a stage result. Window numbers:" + tmux list-windows -t manga-workers -F ' #{window_index} #{window_name}' 2>/dev/null \ + || echo " (tmux session manga-workers is not running; ./start_workers.sh)" +fi +exit "$stale" -- 2.52.0 From 491dad1c6795514a1e75d3c7d28670ce591016cb Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 19:07:56 +0400 Subject: [PATCH 18/31] Ship each crop's embedding so a resolver NONE can mint Two identity fixes, neither yet run on a GPU. has_face, the extras gate: fixing the bbox coordinate space made extras worse, because crops finally landed on their subjects and a background extra bound to the lead at 1.00. /vision stamps has_face per character using face_detect and _pair_faces_to_present, so containment and its margin match the speaker path. Identity skips has_face is False before it crops or embeds. Fails open on a missing or raising detector, and gates on `is False` so an older vision blob behaves as before. The NONE mint: /identity/resolve now writes each crop's embedding to the crop's key with a .npy suffix and returns emb_uri. The orchestrator cannot compute an embedding of its own (siglip and gemma cannot both be resident), which is why it was clearing a deliberate NONE instead of minting an anonymous character. Also: audit_registry.py, and the tmux respawn-window trap, which leaves a bare shell rather than re-running the window command. decisions/identity-bbox.md#face-gates-enrollment decisions/identity-bbox.md#none-mints-an-anonymous-character Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 174 +++++++++++++++------------------ JOURNAL.md | 90 +++++++++++++++++ NEXT.md | 16 ++- audit_registry.py | 93 ++++++++++++++++++ caveats/speaker-attribution.md | 13 ++- decisions/CLAUDE.md | 3 + decisions/identity-bbox.md | 110 +++++++++++++++++++++ worker_identity.py | 18 +++- worker_vision.py | 68 ++++++++++++- 9 files changed, 476 insertions(+), 109 deletions(-) create mode 100644 audit_registry.py diff --git a/HANDOFF.md b/HANDOFF.md index 8ec77b5..06a08cd 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,130 +1,110 @@ -# HANDOFF, 2026-08-12 (second session of the day) +# HANDOFF, 2026-08-12 (third session of the day) -Live state is in `NEXT.md`. This file is only what this session did. The previous handoff's content is -now in `JOURNAL.md` and in `decisions/`. +Live state is in `NEXT.md`. This file is only what this session did. The previous handoff is in +`JOURNAL.md`. ## Asked -Get up to speed from the previous handoff. Then: rebuild the chapter, and what about characters. +Move on from the chapter rebuild. Scope chosen mid-session: reset the registry, rerun `vision`, +`identity` and `reconcile` only, inspect, then decide about the ~80 minutes downstream. After the +inspection the user chose to fix extras-versus-cast before running downstream. ## Result -Two things landed. The A/V gap is fixed and the rebuilt `chapter.mp4` is in the bucket. The `bbox` -coordinate space is settled with measurements instead of comments, and converted at the boundary. - -One correction to carry forward. **The first fix of the day named the wrong cause.** Commit `1457556` -claimed the xfade offset drift was the shipped 72.7s gap. The rebuild came out byte-identical to the -broken file, which disproved it. Both are real defects. Only the second one shipped. - -## The chapter, rebuilt - -`s3://video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80/chapter.mp4` +The identity rebaseline ran and the numbers moved a long way. One code change on top of it is written +and self-checked but **not yet proven on a GPU run**. ``` -before v=436.392 a=363.675 nb_frames=9902 avg_frame_rate=22.69 gap +72.72 -after v=364.120 a=364.122 nb_frames=9101 r_frame_rate=25/1 gap -0.002 + before stale run after +x2 past panel width 47/113 46/110 0/110 +coords exactly on 1000 21 27 1 +max y2 1000 1000 2307 +characters 19 8 8 +coverage 26/113 = 23% 77/110 = 70% 77/110 = 70% +top character's share 96% Haeseon 47% 47% ``` -Cause: `assemble` sends an all-`cut` chapter down a `concat` demuxer with `-c copy`. That path writes -the output in the **first** input's `time_base` and reinterprets every later packet in it. 14 of the 49 clips -are `30/1` at `1/15360`, the other 35 are `25/1` at `1/12800`, so those 14 play `15360/12800 = 1.2` too -long with their audio untouched. `collage_cmd` hardcoded `-r 30`. -`decisions/chapter-assembly.md#mixed-rate-stream-copy`. +`Choi Haeseon`, which held 25 of 26 assignments, is gone from the registry. Panel 7 against the art: -Fixes: `collage_cmd` emits `-r FPS`. `assemble` probes `_fps_of` across the clips and routes mixed rates -through `_assemble_batched`, whose branches both normalize with `fps={FPS}`. +| box | who | before | after | +| --- | --- | --- | --- | +| `[457, 657, 642, 937]` | Seonho, foreground | nothing | `Seonho` | +| `[669, 591, 763, 822]` | the colleague, unnamed in the story | `Choi Haeseon` 0.9 | `character_f7a4fd` | +| `[428, 386, 496, 526]` | background extra | extra as cast | none | +| `[34, 414, 122, 564]` | background extra | extra as cast | none | +| `[498, 386, 568, 533]` | background extra | extra as cast | `character_d72710` 0.94 | +| `[31, 554, 94, 728]` | background extra | `Lim Seonho` | `Seonho` **1.00** | -A second, latent defect on the transition path was fixed and committed separately. `_xfade_chain` took -offsets from `format=duration`, which is `max(video, audio)`. The accumulator crept past the end of its -input, and ffmpeg discarded whole clips at `rc 0` with nothing on stderr. -`decisions/chapter-assembly.md#offsets-from-min-stream`. +## The run that did not count -The single-item passthrough theory from the previous handoff is dead, recorded void at -`decisions/chapter-assembly.md#passthrough-innocent`. The one-path rewrite it recommended is not needed. - -**Still open here.** The 14 clips in the bucket are still 30fps. Assembly normalizes them, so the chapter -is correct, but the fast stream-copy path stays off for this chapter until `render` re-runs. Nobody has -watched the rebuilt video yet. The 2:52 slide transition and the 28s static hold from 2:24 were both -supposed to be re-judged after the sync fix. - -## Characters: the `bbox` space, settled - -All 113 detections, straight from `/review/identity`: - -| test | result | -| --- | --- | -| boxes with `x2` past the 900px panel width | **47 of 113** | -| boxes with `y2` past 1000, on panels 1257 to 2307px tall | **0 of 113** | -| boxes clamped at exactly 1000 | 21 in x, 5 in y | -| coordinate range over every box | `[0, 1000]` | - -Gemma's native 0-1000 grid. Not pixels. `worker_vision.py` prompt text and the old -`worker_identity.py:91` comment both claimed pixels and both were wrong. - -`/vision` now calls `_bbox_to_pixels(characters, w, h)` before returning. Four consumers are fixed at -once: `_crop_bbox` in identity, `_pair_faces_to_present`, the set-of-mark boxes, and the review UI's -client-side crop. The pairing one was comparing real pixel face boxes against 0-1000 character boxes, -which is the likely mechanism behind 7 `unknown` out of 7 `som_face` lines. -`decisions/identity-bbox.md#bbox-is-normalized`. - -Checked by eye on panel 7, not just asserted. Five of six converted boxes land on their subject. That -includes `person_5`, who is Seonho in the foreground with headphones and carried no identity. `person_1` -still frames an empty window mullion, which is `caveats/speaker-attribution.md#extras-as-cast`. - -Converted boxes for panel 7, for whoever redraws the overlay: +The first full cycle completed 116/116/20 and reproduced the defect exactly. The fix was not wrong, it +was not loaded: ``` -person_1 [226, 414, 286, 553] window mullion, nobody -person_2 [ 34, 558, 106, 749] background extra, was assigned Lim Seonho -person_3 [428, 384, 494, 533] background extra -person_4 [498, 389, 561, 549] background extra -person_5 [460, 657, 631, 939] Seonho, foreground. was assigned nothing -person_6 [646, 591, 767, 794] the colleague, no name in the story. was assigned Choi Haeseon at 0.9 +vision worker started 12:00:09 +worker_vision.py modified 12:11:35 +8113bdf, carrying _bbox_to_pixels 12:16:22 ``` -**The registry is unchanged and still wrong.** Every stored box, embedding and `ref_image_uris` was -enrolled from the wrong space. `vision` and `identity` have to re-run before any of it means anything, -and that is GPU work nobody authorized. `caveats/speaker-attribution.md#bbox-wrong-space` is marked -resolved with the rerun pending. +Python binds a module once, at process start. The stage reported success and the orchestrator recorded +no error. Cost: one vision + identity + reconcile cycle and a second registry reset to undo the 8 +characters it minted. `decisions/identity-bbox.md#stale-worker-invalidates`. + +`./check_stale.sh` now compares every running worker's start time against its module mtime and exits +non-zero. Mutation-tested. **Run it before any stage run meant to prove a code change.** + +## Written this session + +- `db.reset_registry` + `POST /characters/reset` (`confirm=true`). `/stage/clear` spares `characters` + by design (`db.py:790`), so nothing could rebaseline the registry. Also clears + `identity_assignment_sources`, which no stage clear touches and where a leftover `manual` row makes + `assign_identity` refuse the next model assignment (`db.py:663`). + Covered by `test_db.py:TestResetRegistry`. **Orchestrator, committed on homesrv, image rebuilt.** +- `check_stale.sh`, `audit_registry.py` (runs inside `manga-orchestrator`, already `docker cp`'d). +- **`has_face`, the extras gate. This is the part not yet proven.** Fixing the coordinate space made + extras worse. With crops finally on their subjects, an extra bound to the lead at 1.00. `/vision` stamps + `has_face` per character using `face_detect` + `_pair_faces_to_present`, so containment and its + margin are the rules the speaker path already uses. `worker_identity.py` skips `has_face is False` + before it crops or embeds. Fails open on a missing or raising detector, and gates on `is False` so an + older vision blob behaves as before. `decisions/identity-bbox.md#face-gates-enrollment`. ## Checks -Every self-check runs from the repo root and passes: - ```bash -.venv/bin/python worker_render.py # about 4 minutes, real ffmpeg -.venv/bin/python worker_vision.py +.venv/bin/python worker_vision.py # includes the has_face gate + both fail-open paths .venv/bin/python worker_identity.py +./check_stale.sh # exits non-zero if a worker predates its file +cd /home/kami/docker-apps/manga-infra/orchestrator && python3 -m pytest -q --ignore=test_api.py # 115 passed, on homesrv ``` -Three checks were added, because the existing ones passed all the way through both shipped defects: - -- `_fps_of(collage clip) == "25/1"`, on a real collage encode. This is the one that would have caught the - mixed-rate bug at the source. -- three clips whose audio outlasts their video by 0.4s, assembled through the xfade branch. Mutation - tested by restoring `_audio_dur`: fires with `video=1.80 audio=3.56 expected=3.56`. -- `_bbox_to_pixels` against panel 7's real `person_5` box, asserting the result covers the lower half of a - 1650px panel, which a raw grid value cannot. - -`_check_assembled` now runs after every encode on both paths, because ffmpeg returns 0 while dropping -whole inputs. - ## Next command -Watch the rebuilt chapter before anything else. That is what found every real defect so far. +The `has_face` gate has never run on a GPU. Restart vision and identity, reset, rerun, and check whether +the two wrong bindings on panel 7 disappear without taking Seonho with them. ```bash -/usr/bin/ssh kami@192.168.1.104 'mc cat homesrv/video/ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e/7c944dd4-e972-42c7-ba60-9f6939548e80/chapter.mp4' > chapter.mp4 +cd /home/kami/Programs/n8n-worker && ./check_stale.sh # restart anything it flags +J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"vision\"}'" +for S in vision identity reconcile; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 5400 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done +/usr/bin/ssh kami@192.168.1.104 "docker exec manga-orchestrator python3 /app/audit_registry.py 7c944dd4-e972-42c7-ba60-9f6939548e80 6" ``` -Then, with a go-ahead, the vision and identity rerun in `NEXT.md` item 1. +Watch for coverage collapsing. 70% is the number to beat. A gate that abstains too hard shows up there +before it shows up on panel 7. Restart the identity worker after every reset: it caches the known list +in-process and only invalidates on enrollment. -## Traps confirmed again this session +## Gone -- `tmux` session `manga-workers` was gone and both systemd units were inactive. `./start_workers.sh` - starts 9 windows. The render worker is window 9 and must be restarted by hand to pick up an edit. -- Plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin. Use `/usr/bin/ssh`. -- `cd $dir && .venv/bin/python` fails, because the venv path is relative to the repo root. Use the - absolute interpreter path when the working directory is elsewhere. -- The whole assembly investigation ran offline on 49 downloaded clips with no GPU and no orchestrator. - Re-download with the command in `NEXT.md`. +The rebuilt `chapter.mp4` and all 49 clips were deleted by the cascade from `/stage/clear vision`. The +user chose not to keep a copy. Nothing downstream of `reconcile` exists for this job now. + +## Traps confirmed again + +- A stage reporting `completed 116/116` says the code ran, not that the current code ran. +- The SSHFS mount at `/mnt/server/home/kami` was absent and needed remounting by the user, since the + mountpoint needs root. +- The orchestrator image bakes its source. Editing the repo on homesrv does nothing until + `docker compose up -d --build orchestrator`. +- `rtk grep` searches files, not stdin. Piping into it silently searches the repo instead. diff --git a/JOURNAL.md b/JOURNAL.md index c197905..f30bdf7 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -306,3 +306,93 @@ subject, including `person_5`, who is Seonho in the foreground with headphones a Not done: `vision` and `identity` have not re-run, so every box, embedding and `ref_image_uris` in the registry is still from the wrong space. That rerun is GPU work and was not started. + +## 2026-08-12, third session — the rerun, and the run that did not count + +Asked: move on from the chapter rebuild. Scope chosen mid-session: reset the registry, rerun vision, +identity and reconcile only, and inspect before spending the ~80 minutes downstream. + +**The registry had to be wiped first, and nothing could do it.** `clear_stage_data` maps `identity` to +`identity_assignments` only and spares `characters` on purpose (`db.py:790`), because the registry is +cross-run. So a rerun would have matched fresh crops against 19 stale grid-space embeddings and kept the +duplicate `Lim Seonho` / `Seonho` rows that raise `ambiguous-speaker`. Added `db.reset_registry` and +`POST /characters/reset` (`confirm=true` required), which also clears `identity_assignment_sources` — +untouched by any stage clear, and a leftover `manual` row there makes `assign_identity` refuse the next +model assignment (`db.py:663`). Covered by `test_db.py:TestResetRegistry`; 115 orchestrator tests pass. +The orchestrator image bakes its source, so it needed `docker compose up -d --build orchestrator`. + +**The first full cycle was void: the worker was serving pre-fix code.** vision + identity + reconcile +completed 116/116/20 and reproduced the defect exactly — 46 of 110 boxes past the 900px panel width, +coordinates clamping at 1000. The vision worker started 12:00:09, `worker_vision.py` changed 12:11:35, +and the commit carrying `_bbox_to_pixels` landed 12:16:22. Python had already bound the old module. +Nothing in the stage output said so (`decisions/identity-bbox.md#stale-worker-invalidates`). Added +`check_stale.sh`, which compares every worker's process start against its module mtime and exits +non-zero; mutation-tested by touching `worker_tts.py`. + +**The rerun against restarted workers.** 8 minutes for all three stages. + +``` + before stale run after +x2 past panel width 47/113 46/110 0/110 +coords exactly on 1000 21 27 1 +max y2 1000 1000 2307 +characters 19 8 8 +coverage 26/113 = 23% 77/110 = 70% 77/110 = 70% +top character's share 96% Haeseon 47% 47% +``` + +`Choi Haeseon`, which had absorbed 25 of 26 assignments, no longer exists in the registry. On panel 7 +Seonho is bound for the first time, and the unnamed colleague took an anonymous id instead of being +called `Choi Haeseon` at 0.9. + +**Fixing the boxes made the extras problem worse.** With crops finally landing on their subjects, a +background extra bound to `Seonho` at confidence 1.00, putting an extra into the lead's reference set. +`/vision` now stamps `has_face` per character via `face_detect` + `_pair_faces_to_present`, and identity +skips `has_face is False` before it crops or embeds +(`decisions/identity-bbox.md#face-gates-enrollment`). Fails open on a missing or raising detector. +Self-checked in both workers. **Not yet proven on a GPU run** — that is the next command. + +Deleted along the way and not recoverable: the rebuilt `chapter.mp4` and all 49 clips, by the cascade +from `/stage/clear vision`. The user chose not to keep a copy. + +## 2026-08-12, fourth session — identity 2b, the resolver NONE branch + +Asked: "how much will `has_face` help with character and identity problems?", then "fix 2b first". + +Answered the first honestly: `has_face` reaches 2 of 6 detections on panel 7 and nothing else. It does not +touch naming or merging, and it cannot touch the chibi at 1:35, because an anime face detector detects a +chibi face. Also flagged its real cost: the model is face-only by design, so back-turned cast lose +enrollment along with the extras, and coverage is where that shows up first. + +Measured before writing anything, read-only, no GPU: + +``` +registry: 8 characters, 1 named -> ['Seonho'] +detections: 110 assignments: 77 = 70% coverage +spread: Seonho 36, character_565c88 24, character_759e23 9, character_f7a4fd 3, + character_25f682 3, character_d72710 1, character_823aba 1 +``` + +That killed the assumed cause. Anonymous ids already recur, so the identity worker's own +pending-promote path gives stable anonymous identities. The defect was elsewhere. + +`/vision/resolve` can answer "none of these" and always could: `worker_vision.py:1071` returns +`state="new"` for `choice: 0` and `state="unresolved"` for an out-of-range index. `service.py` read only +`character_id` and unassigned every crop of the tracklet for either. The stale `ponytail:` comment above +that block named the real blocker and was right: minting needs an `embedding_uri` the orchestrator cannot +compute, since siglip and gemma cannot both be resident. + +Fixed by carrying the embedding, not by adding a GPU pass. `/identity/resolve` writes each crop's +embedding to the crop's key with a `.npy` suffix and returns `emb_uri`. `tracklets.resolve_outcome` holds +the three-way decision as a pure function. `service.py` mints via the existing `create_character` and +falls into the existing assign loop. `decisions/identity-bbox.md#none-mints-an-anonymous-character`. + +Checks: `worker_identity self-check ok`, `tracklets self-check ok`, 115 passed on homesrv. +Deployed: image rebuilt, `resolve_outcome` verified inside the running container, `audit_registry.py` +re-copied after the recreate. Vision and identity restarted, `./check_stale.sh` exits 0. + +New trap: `tmux respawn-window -k` does not re-run the window command. It leaves a bare shell and the +worker down. Both workers were dead for two minutes before `/health` caught it. + +Not run: the GPU cycle. `has_face` and the NONE mint are both unproven on real panels and now land in the +same run. diff --git a/NEXT.md b/NEXT.md index 7ff5af1..be20615 100644 --- a/NEXT.md +++ b/NEXT.md @@ -39,10 +39,12 @@ One number sets the agenda: checked by eye on panel 7, where five of six converted boxes land on their subject (`decisions/identity-bbox.md#bbox-is-normalized`). `person_1` still frames an empty window mullion, which is (c). - b. **Let identity abstain and stay abstained.** The colleague has no name in the story and was - labelled `Choi Haeseon` at 0.9. An unnamed recurring person needs a stable anonymous identity so - narration says "the colleague" every time. `match()` already returns `None` below threshold. Check - whether the Tier-2 gemma resolver can answer "none of these"; that was not verified. + b. ~~Let identity abstain and stay abstained.~~ **Done 2026-08-12, not yet run on a GPU** + (`decisions/identity-bbox.md#none-mints-an-anonymous-character`). The resolver could always answer + "none of these". The orchestrator was discarding the answer: it read only `character_id`, so a + deliberate NONE and a hallucinated index both unassigned every crop of the tracklet. A NONE now + mints an anonymous character from the crop, using the embedding `/identity/resolve` ships beside + it as `emb_uri`. Deployed: image rebuilt, `resolve_outcome` verified inside the container. c. **Separate extra from cast.** Four of the six detections on panel 7 are background extras or nothing at all, and all six reach identity as equal candidates. d. Only then merge `Seonho` into `Lim Seonho` and split `character_afa7623b`, which still needs the @@ -113,7 +115,11 @@ Read the state, or clear a stage and resume: /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST 'http://127.0.0.1:9090/job/resume?job_id=778297bc-e7ce-439d-91b5-8a027060d17f'" ``` -Traps: plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin, so use `/usr/bin/ssh`. +Traps: `tmux respawn-window -k` does NOT re-run the window's command, it drops to a bare shell and the +worker stays down. Restart one worker by re-sending the `send-keys` line from `start_workers.sh`, and +confirm with `/health` plus `./check_stale.sh`. `docker compose up --build orchestrator` recreates the +container, which drops any `docker cp`'d script such as `audit_registry.py`. +Plain `ssh` is the kitty ssh kitten and refuses non-interactive stdin, so use `/usr/bin/ssh`. `mc` aliases on homesrv are `homesrv` and `mio`. `local` returns Access Denied and `rfs` is the empty rustfs. `cp` is aliased to `cp -i` and hangs on overwrite, so use `/usr/bin/cp -f`. diff --git a/audit_registry.py b/audit_registry.py new file mode 100644 index 0000000..4c88149 --- /dev/null +++ b/audit_registry.py @@ -0,0 +1,93 @@ +"""Registry audit for one chapter, after vision + identity + reconcile and before anything downstream. + +Runs inside manga-orchestrator (reads /data/manga.db). `audit_speakers.py` answers the attribution +questions and needs the dialogue stage; this one answers the questions that decide whether dialogue is +worth running at all: + + 1. did the bbox fix land — are stored boxes pixels, or still gemma's 0-1000 grid, + 2. how many characters did the rebaseline mint, and did one of them absorb the chapter again, + 3. what happened on panel 7, the worked example. + +Usage: docker exec manga-orchestrator python3 /app/audit_registry.py [chapter_id] [panel_index] +""" +import collections +import json +import sqlite3 +import sys + +CHAPTER = sys.argv[1] if len(sys.argv) > 1 else "7c944dd4-e972-42c7-ba60-9f6939548e80" +WORKED_EXAMPLE = int(sys.argv[2]) if len(sys.argv) > 2 else 7 + +c = sqlite3.connect("/data/manga.db") +c.row_factory = sqlite3.Row +manga_id = c.execute("SELECT manga_id FROM chapters WHERE chapter_id=?", (CHAPTER,)).fetchone()[0] + +reg = {r["character_id"]: dict(r) for r in c.execute( + "SELECT character_id, name, aliases, gender, ref_image_uris, embedding_uri " + "FROM characters WHERE manga_id=?", (manga_id,))} + +panels = c.execute( + 'SELECT panel_id, panel_index, page_index, bbox FROM panels WHERE chapter_id=? ORDER BY panel_order', + (CHAPTER,)).fetchall() + +# 1. coordinate space. A 0-1000 grid box on a panel wider or taller than 1000px cannot exceed 1000, +# and clamps AT 1000. Real pixel boxes track the panel and scatter past it. The tell is the ratio of +# the largest coordinate to the panel dimension, plus how many boxes sit exactly on 1000. +detections = 0 +past_1000 = at_1000 = 0 +max_ratio = 0.0 +assigned_total = 0 +per_char = collections.Counter() +worked = None + +for p in panels: + row = c.execute("SELECT result_json FROM vision_results WHERE panel_id=?", (p["panel_id"],)).fetchone() + if not row: + continue + v = json.loads(row["result_json"]) + # the vision blob carries no panel size. panels.bbox is the panel's box on its page and is + # [x, y, w, h], not corners — panel 3 of this chapter is [0, 615, 900, 106]. + pb = json.loads(p["bbox"] or "null") + pw, ph = (pb[2], pb[3]) if pb and len(pb) == 4 else (None, None) + assigns = {a["local_id"]: (a["character_id"], a["confidence"]) for a in c.execute( + "SELECT local_id, character_id, confidence FROM identity_assignments WHERE panel_id=?", + (p["panel_id"],))} + assigned_total += len(assigns) + for cid, _ in assigns.values(): + per_char[reg.get(cid, {}).get("name") or cid[:16]] += 1 + people = [ch for ch in (v.get("characters") or []) if ch.get("bbox")] + detections += len(people) + for ch in people: + x1, y1, x2, y2 = ch["bbox"] + past_1000 += 1 if max(x2, y2) > 1000 else 0 + at_1000 += 1 if 1000 in (x2, y2) else 0 + if pw and ph: + max_ratio = max(max_ratio, x2 / pw, y2 / ph) + if p["panel_index"] == WORKED_EXAMPLE: + worked = (p, v, people, assigns, pw, ph) + +named = [r for r in reg.values() if (r["name"] or "").strip()] +print(f"registry: {len(reg)} characters, {len(named)} named -> {sorted((r['name'] or '') for r in named)}") +print(f"detections: {detections} assignments: {assigned_total} " + f"= {100*assigned_total/max(detections,1):.0f}% coverage") +if per_char: + top, n = per_char.most_common(1)[0] + print(f"assignment spread: {dict(per_char.most_common(8))}") + print(f" top character holds {n}/{assigned_total} = {100*n/max(assigned_total,1):.0f}% " + f"({'ABSORBING, same signature as before' if n > 0.5 * assigned_total else 'ok'})") +print(f"bbox space: {past_1000}/{detections} boxes exceed 1000, {at_1000} sit exactly on 1000, " + f"largest coord/panel-dimension = {max_ratio:.2f}") +print(f" verdict: {'PIXELS' if past_1000 or max_ratio > 0.02 and at_1000 == 0 else 'STILL 0-1000 GRID'}") +missing_refs = [k for k, r in reg.items() if not r["ref_image_uris"] or not r["embedding_uri"]] +print(f"characters missing a ref crop or embedding: {len(missing_refs)}") + +if worked: + p, v, people, assigns, pw, ph = worked + print(f"\npanel_index {WORKED_EXAMPLE} ({p['panel_id']}), {pw}x{ph}:") + for ch in people: + cid, conf = assigns.get(ch["local_id"], (None, None)) + name = reg.get(cid, {}).get("name") or (cid[:16] if cid else "-- none --") + print(f" {ch['local_id']:10} {str(ch['bbox']):28} {name:22} " + f"{'' if conf is None else f'{conf:.2f}'}") +else: + print(f"\npanel_index {WORKED_EXAMPLE} not found in this chapter") diff --git a/caveats/speaker-attribution.md b/caveats/speaker-attribution.md index ec6d824..f821975 100644 --- a/caveats/speaker-attribution.md +++ b/caveats/speaker-attribution.md @@ -116,10 +116,13 @@ evidence. A recurring unnamed person needs a stable anonymous identity, so narra colleague" every time. `match()` at `worker_identity.py:69` does abstain, returning `None` below threshold, so the 0.9 came from -cosine clearing the threshold on a wrong crop. Whether the Tier-2 gemma resolver can answer "none of -these" was not verified. +cosine clearing the threshold on a wrong crop. -Revisit trigger: immediately after the `bbox` space is settled. +**Resolved 2026-08-12, `decisions/identity-bbox.md#none-mints-an-anonymous-character`.** The gemma resolver +can answer "none of these" and always could. The orchestrator was discarding the answer. It read only +`character_id`, so a deliberate NONE and a hallucinated index both unassigned every crop of the tracklet. +A NONE now mints an anonymous character from the crop, using the embedding identity ships beside it. Not +yet proven on a GPU run. ## Vision does not separate a background extra from cast {#extras-as-cast} @@ -142,7 +145,9 @@ The visual comparison people reach for as the fix is **already implemented**, so `/vision/resolve` at `worker_vision.py:963` sends the query crop plus up to 3 labelled reference images per candidate. `build_resolve_prompt` already tells the model to judge face shape first, to treat hair and outfit as secondary, that two people sharing a hair colour are not the same, and to answer `0` for -NONE when unsure. `choice: 0` becomes a new character and an out-of-range index becomes `unresolved`. The +NONE when unsure. `choice: 0` returns `state="new"` and an out-of-range index returns `unresolved`. What +the orchestrator does with each is +`decisions/identity-bbox.md#none-mints-an-anonymous-character`. The `ref_image_uris` column is republished as `reference_image_uris` at `worker_identity.py:152` and `:161`, so the references reach the model. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 4542463..ba8fc24 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -41,3 +41,6 @@ still live belongs in `caveats/`. | [A vision `bbox` is gemma's 0-1000 grid, converted to pixels at `/vision`](identity-bbox.md#bbox-is-normalized) | closed | | [Assembly verifies its own output instead of trusting ffmpeg's exit code](chapter-assembly.md#check-assembled) | closed | | [The single-item passthrough is not the assembly bug](chapter-assembly.md#passthrough-innocent) | void | +| [A stage result proves nothing until the worker is newer than the edit](identity-bbox.md#stale-worker-invalidates) | closed | +| [A detection with no detected face never enrolls or binds](identity-bbox.md#face-gates-enrollment) | closed | +| [A resolver NONE mints an anonymous character, it does not clear the crop](identity-bbox.md#none-mints-an-anonymous-character) | closed | diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md index 0c637e1..c2308a9 100644 --- a/decisions/identity-bbox.md +++ b/decisions/identity-bbox.md @@ -43,3 +43,113 @@ that. It also records the trap: a model that really answered in pixels would be `ref_image_uris` are enrolled on balloons and window frames. Re-running identity is what makes the registry mean anything. The anonymous-identity and extra-versus-cast work cannot be judged until that rerun happens. + +## A stage result proves nothing until the worker is newer than the edit {#stale-worker-invalidates} + +**Closed, 2026-08-12.** The first rerun after the bbox fix reproduced the defect exactly: 46 of 110 +boxes past the 900px panel width, coordinates clamping at 1000, `y2` never once past 1000 on panels up +to 2307px tall. The same fingerprint as `#bbox-is-normalized` measured before the fix. + +The fix was not wrong. It was not loaded. + +``` +vision worker process started 12:00:09 +worker_vision.py modified 12:11:35 +8113bdf, which contains _bbox_to_pixels, committed 12:16:22 +``` + +Python binds a module once, at process start. `./start_workers.sh` had launched the worker eleven +minutes before the file changed, so `/vision` served pre-fix code for the whole run and returned raw +grid boxes. Nothing in the result said so. The stage reported `completed 116/116`, the orchestrator +recorded no error, and identity and reconcile ran to completion on top of it. Cost: one full +vision + identity + reconcile cycle, plus a registry reset to undo the 8 characters it minted. + +The rerun against a restarted worker gives the opposite reading over the same 110 detections: 0 boxes +past the width, 0 past the height, one coordinate on 1000 which is now a real pixel value, and a +deepest box reaching 100% down its panel with `max y2 = 2307`. Boxes track the panel, so they are +pixels. + +`check_stale.sh` compares every running worker's process start against its module's mtime and exits +non-zero if any is stale. This failure mode was already known as advice — the render worker "must be +restarted by hand to pick up an edit" — and advice did not stop it happening. Run the check before any +stage run that is meant to prove a code change. + +Forbids: citing a stage result as evidence about a code change without establishing that the worker +serving it postdates the change. + +## A detection with no detected face never enrolls or binds {#face-gates-enrollment} + +**Closed, 2026-08-12. Written and self-checked, not yet proven on a GPU run.** + +Fixing the coordinate space made the extras problem worse, not better. With the boxes finally landing +on their subjects, panel 7's four background extras became four good crops of four irrelevant people, +and one of them bound to `Seonho` at confidence 1.00. Before the fix the same detection was a crop of +scenery and matched nothing much. Correct geometry turned a harmless failure into a poisoned reference +set for the lead. + +The measured panel 7 outcome, converted boxes, against the art: + +| box | who | assigned | +| --- | --- | --- | +| `[457, 657, 642, 937]` | Seonho, foreground | `Seonho` | +| `[669, 591, 763, 822]` | the colleague, unnamed in the story | `character_f7a4fd`, anonymous | +| `[428, 386, 496, 526]` | background extra | none | +| `[34, 414, 122, 564]` | background extra | none | +| `[498, 386, 568, 533]` | background extra | `character_d72710` at 0.94 | +| `[31, 554, 94, 728]` | background extra | `Seonho` at 1.00 | + +`/vision` now stamps `has_face` on every character by running `face_detect.detect_faces` on the panel +and reusing `_pair_faces_to_present` for containment, so the gate uses the same margin and the same +global shortest-first assignment as the speaker path. `worker_identity.py` skips a character with +`has_face is False` before it crops, embeds, matches or mints. + +Two properties are deliberate. It **fails open**: a missing or raising detector marks every character +`True`, because dropping a whole panel's cast is worse than the over-detection the gate exists to trim. +And it gates on `is False`, not falsiness, so a vision blob written before this change (no key) behaves +as it did rather than silently dropping every character. + +Cost: a cast member drawn from behind, or in a style the detector misses, now takes no identity on that +panel. That is the abstain this pipeline already prefers to a wrong bind +(`caveats/speaker-attribution.md#no-anonymous-identity`). + +Forbids: enrolling a reference crop, or binding a character, from a region no face detector confirms. + +## A resolver NONE mints an anonymous character, it does not clear the crop {#none-mints-an-anonymous-character} + +**Closed, 2026-08-12. Written and self-checked, not yet proven on a GPU run.** + +`caveats/speaker-attribution.md#no-anonymous-identity` asked whether the Tier-2 gemma resolver can answer +"none of these". It can, and it always could. `/vision/resolve` at `worker_vision.py:1071` maps `choice: 0` +to `state="new"`, an out-of-range index to `state="unresolved"`, and a parse failure to `unresolved` as +well. The abstain path was never the defect. + +The defect was one branch on the other side of the contract. The orchestrator read only +`v.get("character_id")` and treated every falsy value the same way: `unassign_identity` on every crop of +the tracklet. So a deliberate "this is a real person the roster does not hold" and a hallucinated index +both produced nothing, and the unnamed colleague was `unknown` on every panel she appeared on. The stale +`ponytail:` comment above that block named the reason nobody fixed it, and the reason was real: minting a +character needs an `embedding_uri`, and the orchestrator cannot compute one. siglip is resident in the +identity worker, gemma is resident in the vision worker, and `session_manager` forbids both at once. + +What removes the blocker is carrying the embedding, not a third GPU pass. `/identity/resolve` already +computes an embedding per crop and already uploads the crop to +`s3://manga/{manga}/characters/_crops/{panel}_{local}.png`. It now writes the embedding to the same key +with a `.npy` suffix and returns `emb_uri` in each shortlist entry. The mint is then a local +`create_character(manga_id, None, appearance, [crop_uri], emb_uri, gender)`, and the existing per-tracklet +assign loop binds every member to it. + +`tracklets.resolve_outcome` holds the three-way decision as a pure function, so the branch that runs is +the branch the self-check covers: `known` on a named answer, `mint` on `state="new"` with an `emb_uri`, +`clear` on `unresolved`, on a NONE with no embedding, and on an older worker that sends no `state` at all. + +Two limits are deliberate. A tracklet's candidate gallery is built before the loop mints anything, so one +person split across two unlinked tracklets still gets two anonymous ids; `run_stage_reconcile` merges +unnamed twins on appearance overlap and is what folds them. And an anonymous character's text sheet +(`worker_vision._sheet`) carries no name, so gemma re-recognising it on a later panel leans on the +reference images rather than the description. + +Contract: `shortlists[].emb_uri` is new in the `/identity/resolve` response. Invariant 7 — both repos +changed in the same session. + +Forbids: treating an absent `character_id` as one outcome. A resolver that answered and a resolver that +failed are different facts. diff --git a/worker_identity.py b/worker_identity.py index 18677bb..477412e 100644 --- a/worker_identity.py +++ b/worker_identity.py @@ -200,6 +200,13 @@ async def resolve(data: IdentityInput): assignments, backfill, new_chars, shortlists = [], [], [], [] for ch in data.vision_characters: + # no detected face inside the box -> a background extra, a figure on a poster, or scenery + # gemma called a person. Embedding it pollutes the registry and, once the boxes were pixels, + # bound an extra to the lead at confidence 1.00. Abstain instead. `/vision` stamps this and + # fails open, so a panel it could not gate arrives with has_face=True on every character + # (decisions/identity-bbox.md#face-gates-enrollment). Absent key = an older vision blob. + if ch.get("has_face") is False: + continue crop = _crop_bbox(img, ch["bbox"]) if crop.size == 0: # degenerate/out-of-bounds bbox -> nothing to embed, skip continue @@ -207,10 +214,17 @@ async def resolve(data: IdentityInput): # shortlist for gemma's decider, from the roster as it stood before this crop's own outcome. sl = shortlist(emb, known, data.k, ch.get("gender")) - crop_uri = f"s3://manga/{data.manga_id}/characters/_crops/{data.panel_id}_{ch['local_id']}.png" + # the embedding ships with the crop. gemma's decider can answer "none of these", and the + # orchestrator has to mint a character from that crop — which needs an embedding it cannot + # compute (siglip is resident here, gemma is resident there, and the mutex forbids both). + # Uploading it now is what removes the third siglip pass + # (`decisions/identity-bbox.md#none-mints-an-anonymous-character`). + key = f"{data.manga_id}/characters/_crops/{data.panel_id}_{ch['local_id']}" + crop_uri, emb_uri = f"s3://manga/{key}.png", f"s3://manga/{key}.npy" cp = f"{SHM}/cc_{uuid.uuid4().hex[:8]}.png"; cv2.imwrite(cp, crop) transport.put(cp, crop_uri); os.remove(cp) - shortlists.append({"local_id": ch["local_id"], "crop_uri": crop_uri, + _save_npy(emb, emb_uri) + shortlists.append({"local_id": ch["local_id"], "crop_uri": crop_uri, "emb_uri": emb_uri, "candidates": [{"character_id": c["character_id"], "name": c.get("name"), "gender": c.get("gender"), "species": c.get("species"), "appearance": c.get("description"), "cosine": c["cosine"], diff --git a/worker_vision.py b/worker_vision.py index 5e076c6..dcc40ce 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -330,6 +330,40 @@ def _bbox_to_pixels(chars: list, w: int, h: int) -> list: return chars +def _mark_has_face(img, chars: list) -> list: + """Stamp `has_face` on every character a real detected face lands inside. + + gemma calls every person-shaped region a character. On panel 7 of job 778297bc that was two + people and four background extras, one of which is an empty window mullion. All six reached + identity as equal candidates, and after the bbox fix an extra took `Seonho` at confidence 1.00, + which puts an extra's crop into the lead's reference set. A faceless crop is a bad embedding as + well as a bad reference: the back of a head or a patch of coat matches almost anything. + + Reuses `_pair_faces_to_present`, so containment, its margin, and the global shortest-first + assignment are exactly the rules the speaker path already uses. Requires pixel boxes, so call it + after `_bbox_to_pixels`. + + Fails open. A missing or broken detector marks everything `True`, because dropping every + character is worse than the over-detection this gate exists to trim. + """ + def _all(v): + for c in chars: + c["has_face"] = v + return chars + + if face_detect is None or not chars: + return _all(True) + try: + faces = face_detect.detect_faces(img) + except Exception as e: + print(f"[vision/detect] face detect failed, has_face gate off for this panel: {e}", flush=True) + return _all(True) + paired = {p["local_id"] for p in _pair_faces_to_present(faces, chars) if p.get("local_id")} + for c in chars: + c["has_face"] = c.get("local_id") in paired + return chars + + def _panel_size(path: str) -> tuple: """(width, height) of a panel image, or (0, 0) when it cannot be read.""" import cv2 @@ -355,11 +389,14 @@ async def vision(data: VisionInput): print(f"[vision/detect] parse failed for {data.panel_id} after repair retry: {e}", flush=True) result = {"skip": False, "parse_failed": True, "characters": [], "scene": {}} finally: - pw, ph = _panel_size(local) + import cv2 + img = cv2.imread(local) # read once: the size and the face gate both need it os.remove(local) + pw, ph = (img.shape[1], img.shape[0]) if img is not None else (0, 0) result.setdefault("characters", []) if pw and ph: _bbox_to_pixels(result["characters"], pw, ph) + _mark_has_face(img, result["characters"]) else: print(f"[vision/detect] panel size unreadable for {data.panel_id}, boxes left normalized", flush=True) @@ -1187,6 +1224,35 @@ if __name__ == "__main__": # the box must now cover the lower half of a tall panel, which the raw grid value never can assert _ch[0]["bbox"][3] > 1000 > _ch[0]["bbox"][1] + # the has_face gate, on panel 7's real converted boxes. Seonho and the colleague carry a detected + # face; the two background extras do not, and identity must skip them rather than embed scenery. + _p7 = [{"local_id": "person_1", "bbox": [457, 657, 642, 937]}, # Seonho, foreground + {"local_id": "person_2", "bbox": [669, 591, 763, 822]}, # the colleague + {"local_id": "person_5", "bbox": [31, 554, 94, 728]}, # extra that took Seonho at 1.00 + {"local_id": "person_6", "bbox": [34, 414, 122, 564]}] # extra + _faces = [{"bbox": [520, 690, 590, 770]}, {"bbox": [695, 610, 740, 660]}] + class _FD: + @staticmethod + def detect_faces(_img): + return _faces + _real_fd, globals()["face_detect"] = face_detect, _FD + try: + _mark_has_face(None, _p7) + assert [c["has_face"] for c in _p7] == [True, True, False, False], _p7 + # fail open: a detector that raises must not silently drop every character in the panel + class _Boom: + @staticmethod + def detect_faces(_img): + raise RuntimeError("onnx died") + globals()["face_detect"] = _Boom + _mark_has_face(None, _p7) + assert all(c["has_face"] for c in _p7), _p7 + globals()["face_detect"] = None + _mark_has_face(None, _p7) + assert all(c["has_face"] for c in _p7), _p7 + finally: + globals()["face_detect"] = _real_fd + # face->identity pairing is GATED on containment: a face outside every gemma bbox stays unknown. faces = [{"bbox": [10, 10, 30, 30]}, {"bbox": [900, 900, 920, 920]}] present = [{"local_id": "person_1", "name": "Teto", "bbox": [0, 0, 100, 200]}] -- 2.52.0 From ca4661763c0c24e61a3eee708bef3866b23fb9c7 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 19:16:30 +0400 Subject: [PATCH 19/31] Stop a background extra's action reaching narration build_scene already dropped an unassigned detection from `characters` and `present`, so an extra never reached the cast list. Its ACTION did. `actions` was built from every detection, and that list is what the script prompt renders and what the verifier uses as evidence, so "standing at the window" arrived as a fact about the panel with no character attached and the verifier confirmed it, because the action really was in the blob. Skip has_face is False, the same gate and the same fail-open semantics as enrollment. Self-check covers all three cases: a real cast member's action survives, a faceless one's does not, and a detection from a panel where the detector never ran keeps its action. decisions/identity-bbox.md#extras-gate-consumers Co-Authored-By: Claude Opus 5 --- decisions/CLAUDE.md | 1 + decisions/identity-bbox.md | 30 ++++++++++++++++++++++++++++++ worker_scene.py | 22 +++++++++++++++++++++- 3 files changed, 52 insertions(+), 1 deletion(-) diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index ba8fc24..d7ef2da 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -44,3 +44,4 @@ still live belongs in `caveats/`. | [A stage result proves nothing until the worker is newer than the edit](identity-bbox.md#stale-worker-invalidates) | closed | | [A detection with no detected face never enrolls or binds](identity-bbox.md#face-gates-enrollment) | closed | | [A resolver NONE mints an anonymous character, it does not clear the crop](identity-bbox.md#none-mints-an-anonymous-character) | closed | +| [The extras gate runs at enrollment and at narration, not at the speaker prompt](identity-bbox.md#extras-gate-consumers) | closed | diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md index c2308a9..46083b7 100644 --- a/decisions/identity-bbox.md +++ b/decisions/identity-bbox.md @@ -153,3 +153,33 @@ changed in the same session. Forbids: treating an absent `character_id` as one outcome. A resolver that answered and a resolver that failed are different facts. + +## The extras gate runs at enrollment and at narration, not at the speaker prompt {#extras-gate-consumers} + +**Closed, 2026-08-12. Written and self-checked, not yet proven on a GPU run.** + +`#face-gates-enrollment` stops a faceless detection taking an identity. It does not stop the detection +being narrated, because three places read the raw vision character list and only one of them consults an +assignment. Panel 7 is the worked example: six detections, four of them extras or scenery. + +`worker_scene.build_scene` already drops an unassigned detection from `characters` and `present` +(`worker_scene.py:63`), so an extra never reached the cast list. Its **action** did. `actions` was built +from every detection, and that list is what the script prompt renders and what the verifier uses as +evidence. So "standing at the window" arrived as a fact about the panel with no character attached, and +the verifier confirmed it because the action really was in the blob. Both now skip `has_face is False`. + +`service._beat` picks the cinematographer's "who" from the first three detections, falling back to a +detection's action when it has no name. An extra could take a slot and steer the camera. Also gated. + +`service._present_characters` is deliberately **not** gated. It builds the dialogue stage's candidate +speaker list and the set-of-mark boxes. Two reasons. The failure is already contained: an extra chosen as +the speaker has no identity assignment, so `normalize_dialogue` resolves it to unknown rather than to a +wrong name. And the gate's own cost lands hardest here, because a character drawn from behind has no face +box, so gating would delete a real speaker from the only list that can attribute their line. + +All three gates test `is False`, never falsiness. A vision blob written before the gate existed carries no +`has_face` key, and a panel whose detector failed is marked `True` by the fail-open path. Both keep their +previous behaviour. + +Forbids: adding a fourth consumer of `vision["characters"]` without deciding which side of this line it is +on. The blob keeps every detection on purpose, so the audit can still see what was gated. diff --git a/worker_scene.py b/worker_scene.py index 11f0110..66064e9 100644 --- a/worker_scene.py +++ b/worker_scene.py @@ -91,7 +91,13 @@ def build_scene(data: SceneInput): vchars = data.vision_result.get("characters", []) # `actions` is the list the orchestrator's beat builder reads for verifier evidence; `action` is the # joined string the script prompt renders. Emitting only the string left verification blind. - actions = [c["action"].strip() for c in vchars if str(c.get("action") or "").strip()] + # `characters` above already drops an unassigned detection, but its ACTION used to survive, so a + # background extra standing at a window became a sentence the narrator read out. A detection with no + # detected face is an extra, a figure on a poster, or scenery gemma called a person + # (`decisions/identity-bbox.md#face-gates-enrollment`). Gate on `is False`, like the enrollment gate, + # so a blob written before the gate existed and a panel whose detector failed both behave as before. + actions = [c["action"].strip() for c in vchars + if str(c.get("action") or "").strip() and c.get("has_face") is not False] action = "; ".join(actions) return {"panel_id": data.panel_id, "characters": characters, "dialogue": dialogue, "action": action, "actions": actions, @@ -132,6 +138,20 @@ if __name__ == "__main__": "description": '{"hair":"black","features":["glasses"]}'}], )) assert outu["characters"][0]["name"] == "" and outu["characters"][0]["label"] == "the one with black hair and glasses" + # a faceless detection's ACTION never reaches the script prompt or the verifier's evidence, but a + # real cast member's does, and so does one on a panel where the detector never ran (no key). + p7 = build_scene(SceneInput( + panel_id="p007", + vision_result={"characters": [ + {"local_id": "person_1", "action": "reading a report", "has_face": True}, + {"local_id": "person_2", "action": "standing at the window", "has_face": False}, + {"local_id": "person_3", "action": "pointing"}, + ]}, + identity_assignments=[{"local_id": "person_1", "character_id": "c1"}], + characters_registry=[{"character_id": "c1", "name": "Seonho"}], + )) + assert p7["actions"] == ["reading a report", "pointing"], p7["actions"] + assert "window" not in p7["action"], p7["action"] assert _describe({}) == "" # bare appearance -> empty, script falls back to Person X # animals are named as their species (color + species), never by clothing/"the one with..." assert _describe({"species": "cat", "hair": "black"}) == "the black cat" -- 2.52.0 From d63a337f158ea7e85a2e2f0718b6d054a89cede1 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 19:27:13 +0400 Subject: [PATCH 20/31] Record that 2d's worked examples no longer exist The registry reset deleted Lim Seonho and character_afa7623b, so there is nothing to merge or split until the rerun mints a new set. What was done instead is the safety net for that rerun, since reconcile runs inside it: a merge retires the losing row rather than deleting it, and records which assignments moved. Half-closes caveats/audit-open.md#destructive-reconcile. The unmerge path and the split stay unwritten on purpose, with the revisit trigger named. Co-Authored-By: Claude Opus 5 --- JOURNAL.md | 63 +++++++++++++++++++++++++++++++++++++++++++ NEXT.md | 25 ++++++++++++++--- caveats/audit-open.md | 20 ++++++++++---- 3 files changed, 99 insertions(+), 9 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index f30bdf7..d4247cb 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -396,3 +396,66 @@ worker down. Both workers were dead for two minutes before `/health` caught it. Not run: the GPU cycle. `has_face` and the NONE mint are both unproven on real panels and now land in the same run. + +## 2026-08-12, fourth session, continued — identity 2c, extras versus cast + +Asked: "fix extra-vs-cast?" + +Traced the three places that read `vision["characters"]` raw before changing anything. That corrected an +inference made earlier in the session. `build_scene` already drops an unassigned detection from +`characters` and `present` (`worker_scene.py:63`), so extras never reached the cast list at all. + +The leak was their ACTIONS. `actions` was built from every detection, and that list is what the script +prompt renders and what the correctness verifier uses as evidence. So a background extra's "standing at the +window" arrived as a fact about the panel with no character attached, and the verifier confirmed it, +because the action really was in the blob. That is a second mechanism behind the invented-narration +complaints in item 3, independent of the model inventing anything. + +Gated two consumers on `has_face is False`, matching the enrollment gate's semantics exactly: +`worker_scene`'s `actions`/`action`, and `service._beat`, which picks the director's "who" from the first +three detections and falls back to an action when a detection has no name. + +Left `service._present_characters` ungated on purpose. It builds the dialogue stage's candidate speakers +and the set-of-mark boxes. An extra picked as speaker already resolves to unknown, not to a wrong name, so +the failure is contained. And the gate's cost lands hardest there, since a character drawn from behind has +no face box and gating would delete a real speaker from the only list that can attribute their line. +`decisions/identity-bbox.md#extras-gate-consumers`. + +Checks: `worker_scene self-check ok` with three cases (cast action survives, faceless dropped, missing key +survives), 115 passed on homesrv. Deployed: image rebuilt, `_beat` verified inside the container, scene +worker restarted, `./check_stale.sh` clean. Committed `ca46617` and `8b27aec`. + +Still not run on a GPU. Three changes now ride the same cycle: `has_face`, the NONE mint, and this. + +## 2026-08-12, fourth session, continued — identity 2d, merge and split + +Asked: "2d? merge and split?" + +Checked the registry before planning anything, and 2d as written is stale. The registry reset earlier today +deleted both worked examples. There is no `Lim Seonho` to merge into, `character_afa7623b` does not exist, +and the current registry is 8 rows with one named character (`Seonho`). Nothing to merge or split until the +rerun mints a new set. + +So the useful work was the safety net for that rerun, since `reconcile` runs inside it. The caveat's cost +line was the reason: one bad merge was unrecoverable without rebaselining the whole manga, and the cycle +about to run includes a merge pass over embeddings nobody has seen yet. + +`merge_characters` no longer deletes the loser. It sets `merged_into = keeper`, so the row keeps its +embedding, description and gender, and it stamps every repointed assignment with +`method = merged_from:` in `identity_assignment_sources`. That reuses a free-form column on a +table that already existed rather than adding a merge log. `source` is left alone, so a reviewer's `manual` +assignment keeps its veto in `assign_identity` after being repointed. + +Roster readers filter `merged_into IS NULL`, including the name-dedup in `create_character`, which would +otherwise fold new crops back into a character reconcile had retired. Lookup by id does not filter, because +an assignment or a narration reference may still point at a merged id. + +Two existing assertions asserted the old destructive behaviour (`test_db.py:235`, `test_merge_refs.py:37`) +and were rewritten, not deleted: the invariant changed on purpose. + +Checks: 116 passed on homesrv, up from 115. Additive `ALTER TABLE` through the existing `init_db` migration +block, verified on the live database (`merged_into` present, 0 rows merged). Committed `00096cc`. + +Deliberately not built: the unmerge path and the split. No wrong merge has been observed since the crops +were fixed, so the consumer of these records waits for one. The forward case is partly covered by 2b, since +a resolver NONE now mints instead of folding a stranger into the nearest match. diff --git a/NEXT.md b/NEXT.md index be20615..5318855 100644 --- a/NEXT.md +++ b/NEXT.md @@ -45,10 +45,27 @@ One number sets the agenda: deliberate NONE and a hallucinated index both unassigned every crop of the tracklet. A NONE now mints an anonymous character from the crop, using the embedding `/identity/resolve` ships beside it as `emb_uri`. Deployed: image rebuilt, `resolve_outcome` verified inside the container. - c. **Separate extra from cast.** Four of the six detections on panel 7 are background extras or - nothing at all, and all six reach identity as equal candidates. - d. Only then merge `Seonho` into `Lim Seonho` and split `character_afa7623b`, which still needs the - reversible-merge design (`caveats/audit-open.md#destructive-reconcile`), not a patch. + c. ~~Separate extra from cast.~~ **Done 2026-08-12, not yet run on a GPU** + (`decisions/identity-bbox.md#face-gates-enrollment`, + `decisions/identity-bbox.md#extras-gate-consumers`). `has_face` stops a faceless detection + enrolling, and two more consumers now skip it: `worker_scene`'s `actions`, which is the script + prompt's content and the verifier's evidence, and `service._beat`, the director's "who". + `_present_characters` stays ungated on purpose, reasoned out in the decision. + The remaining gap is that vision still emits extras into the blob, which is deliberate so the + audit can see what was gated. + d. **The worked examples are gone.** The registry reset deleted `Lim Seonho` and + `character_afa7623b`. The current registry is 8 rows, one named (`Seonho`), so there is nothing to + merge or split until the rerun mints a new set. + + What was done instead is the safety net for that rerun, since `reconcile` runs inside it. A merge no + longer deletes the losing row: it sets `merged_into`, and stamps every repointed assignment with + `method = merged_from:`. A wrong merge now costs a hand-written SQL walk, not a full + rebaseline (`caveats/audit-open.md#destructive-reconcile`). + + Deliberately not built: the unmerge path and the split. No wrong merge has been observed since the + crops were fixed, so the consumer of those records waits for one. The forward case is partly covered + by 2b, because a resolver NONE now mints rather than folding a stranger into the nearest match. + Splitting a character that is ALREADY over-merged still needs a re-embed pass over its detections. **Cast profiles already exist. Do not rebuild them.** The user asked whether the main cast could get a profile built from reference frames and reused. `characters` already carries `ref_image_uris` and diff --git a/caveats/audit-open.md b/caveats/audit-open.md index cebd189..f0f4f92 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -5,12 +5,22 @@ fixed findings live in `decisions/audit-phase1.md`. Line numbers are from the au ## Reconcile deletes the losing character irreversibly {#destructive-reconcile} -Reconciliation deletes the losing character row (`db.py:506`). Clearing the reconcile stage does not -undo it, and name claims attached to the merged-away character are not repointed. +Reconciliation used to delete the losing character row. Clearing the reconcile stage did not undo it, and +nothing recorded which detections had been the loser's. -Costs: one bad merge is unrecoverable without rebuilding the identity stage for the whole manga. -Revisit when: identity work resumes, or a reviewer reports a wrong merge on a real chapter. -Workaround: none. Clear identity and rerun, which loses the good merges too. +**Half-closed 2026-08-12.** The loss is no longer unrecoverable. `merge_characters` marks the loser +`merged_into = keeper` instead of deleting it, so its embedding, description and gender survive, and every +repointed assignment is stamped `method = merged_from:` in `identity_assignment_sources`. Those +two records are enough to walk a merge backwards. Roster readers filter `merged_into IS NULL`. Lookup by +id does not, so an assignment still pointing at a merged id resolves. + +What is still missing is the mechanism that consumes them: there is no unmerge, and no split. Undoing a +merge today means a manual SQL walk of the two records above. + +Costs: a wrong merge needs hand-written SQL to undo, not a rebaseline. +Revisit when: a reviewer reports a wrong merge, or the review gates from [#136] get a place to hang +name/merge/split actions. No wrong merge has been seen since the crops were fixed, so building the unmerge +before either trigger would be speculative. ## Clearing a stage does not undo what it wrote {#dishonest-clearing} -- 2.52.0 From c51871348f3131d1fb2bb01f1e34d6990bc6e7f3 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 19:33:06 +0400 Subject: [PATCH 21/31] Rewrite the handoff for this session, document the restart trap HANDOFF.md now covers the fourth session: the four identity changes, the measured numbers they were decided from, what was deliberately not built, and panel 7's before-table so the next run has something to compare against. AGENTS.md gains the worker-restart procedure, because tmux respawn-window -k leaves a bare shell instead of re-running the command and took both vision and identity down silently this session. Co-Authored-By: Claude Opus 5 --- AGENTS.md | 21 +++++++ HANDOFF.md | 162 +++++++++++++++++++++++++++++------------------------ 2 files changed, 109 insertions(+), 74 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index f43fcb5..beff1b4 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,5 +1,26 @@ # Repository guidance +## Restarting one worker + +Python binds a module at process start, so a worker keeps serving the code it loaded. Run +`./check_stale.sh` before any stage run meant to prove a change. It exits non-zero and names the tmux +window for every worker older than its file (`decisions/identity-bbox.md#stale-worker-invalidates`). + +**`tmux respawn-window -k` does not re-run the window's command.** It leaves a bare shell and the worker +stays down, silently. Restart by re-sending the `send-keys` line from `start_workers.sh` for that one +worker, then confirm on `/health` and with `check_stale.sh`: + +```bash +M="export MIOPEN_USER_DB_PATH=$HOME/.config/miopen MIOPEN_SYSTEM_DB_PATH=$HOME/.config/miopen MIOPEN_FIND_MODE=2 && unset MIOPEN_FIND_ENFORCE" +tmux send-keys -t manga-workers:vision C-c +tmux send-keys -t manga-workers:vision "$M && source $PWD/.venv/bin/activate && python -m uvicorn worker_vision:app --app-dir $PWD --host 0.0.0.0 --port 8002" C-m +curl -s http://127.0.0.1:8002/health && ./check_stale.sh +``` + +The orchestrator's equivalent is `docker compose up -d --build orchestrator` on homesrv, because the image +bakes its source. That recreates the container, which drops any `docker cp`'d file such as +`audit_registry.py`. + ## Video analysis When diagnosing render motion, transitions, timing, or visual artifacts, use diff --git a/HANDOFF.md b/HANDOFF.md index 06a08cd..db3df29 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,89 +1,96 @@ -# HANDOFF, 2026-08-12 (third session of the day) +# HANDOFF, 2026-08-12 (fourth session of the day) Live state is in `NEXT.md`. This file is only what this session did. The previous handoff is in `JOURNAL.md`. ## Asked -Move on from the chapter rebuild. Scope chosen mid-session: reset the registry, rerun `vision`, -`identity` and `reconcile` only, inspect, then decide about the ~80 minutes downstream. After the -inspection the user chose to fix extras-versus-cast before running downstream. +"How much will `has_face` help with character and identity problems?", then in order: fix 2b, fix +extra-versus-cast, 2d. ## Result -The identity rebaseline ran and the numbers moved a long way. One code change on top of it is written -and self-checked but **not yet proven on a GPU run**. +Identity item 2 in `NEXT.md` is now done as far as it can go without a run. Four changes are written, +self-checked and deployed. **None has run on a GPU.** No pipeline stage was executed this session. + +| item | was | now | +| --- | --- | --- | +| 2a bbox space | done last session | unchanged | +| 2b abstain and stay abstained | resolver blamed, unverified | fixed in the orchestrator, `f6804e7` + `ffda3df` | +| 2c extra versus cast | open | two consumers gated, `ca46617` + `8b27aec` | +| 2d merge and split | open | examples gone; merge made non-destructive, `00096cc` | + +## What each change was, and where the cause turned out to be + +**2b was never the resolver.** `/vision/resolve` at `worker_vision.py:1071` already returns +`state="new"` for `choice: 0` and `state="unresolved"` for an out-of-range index. `service.py` read only +`v.get("character_id")` and ran `unassign_identity` on every crop of the tracklet for either one. A +deliberate NONE produced nothing, so an unnamed recurring person was `unknown` on every panel. + +The old `ponytail:` comment above that block named the real blocker and was right. Minting needs an +`embedding_uri` the orchestrator cannot compute. siglip is resident in the identity worker and gemma in +the vision worker. Fixed by carrying the embedding, not by adding a GPU pass. +`/identity/resolve` writes each crop's embedding to the crop's key with a `.npy` suffix and returns +`emb_uri`. `tracklets.resolve_outcome` holds the three-way decision as a pure function. +`decisions/identity-bbox.md#none-mints-an-anonymous-character`. + +**2c: `build_scene` already dropped extras from the cast list.** `worker_scene.py:63` skips an unassigned +detection, so extras never reached `characters` or `present`. Their **actions** did. `actions` was built +from every detection. That list is the script prompt's content and the verifier's evidence. So a background +extra's "standing at the window" arrived as a fact with no character attached, and the verifier confirmed +it. Gated `worker_scene`'s `actions` and `service._beat` on `has_face is False`. Left +`service._present_characters` ungated on purpose, reasoned out in +`decisions/identity-bbox.md#extras-gate-consumers`. + +**2d's worked examples no longer exist.** The registry reset deleted `Lim Seonho` and +`character_afa7623b`. The registry is 8 rows, one named. Built the safety net for the coming rerun +instead, since `reconcile` runs inside it: `merge_characters` sets `merged_into = keeper` rather than +deleting, and stamps every repointed assignment `method = merged_from:`. Roster readers filter +`merged_into IS NULL`, lookup by id does not. +`caveats/audit-open.md#destructive-reconcile` is half-closed. + +## Measured, read-only, before writing anything ``` - before stale run after -x2 past panel width 47/113 46/110 0/110 -coords exactly on 1000 21 27 1 -max y2 1000 1000 2307 -characters 19 8 8 -coverage 26/113 = 23% 77/110 = 70% 77/110 = 70% -top character's share 96% Haeseon 47% 47% +registry: 8 characters, 1 named -> ['Seonho'] +detections: 110 assignments: 77 = 70% coverage +spread: Seonho 36, character_565c88 24, character_759e23 9, character_f7a4fd 3, + character_25f682 3, character_d72710 1, character_823aba 1 +bbox space: 77/110 exceed 1000, 1 on 1000 -> PIXELS ``` -`Choi Haeseon`, which held 25 of 26 assignments, is gone from the registry. Panel 7 against the art: +That killed the assumed cause of 2b. Anonymous ids already recur, so the identity worker's own +pending-promote path gives stable anonymous identities. Only the gemma NONE branch was discarding people. -| box | who | before | after | -| --- | --- | --- | --- | -| `[457, 657, 642, 937]` | Seonho, foreground | nothing | `Seonho` | -| `[669, 591, 763, 822]` | the colleague, unnamed in the story | `Choi Haeseon` 0.9 | `character_f7a4fd` | -| `[428, 386, 496, 526]` | background extra | extra as cast | none | -| `[34, 414, 122, 564]` | background extra | extra as cast | none | -| `[498, 386, 568, 533]` | background extra | extra as cast | `character_d72710` 0.94 | -| `[31, 554, 94, 728]` | background extra | `Lim Seonho` | `Seonho` **1.00** | +## Deliberately not built -## The run that did not count - -The first full cycle completed 116/116/20 and reproduced the defect exactly. The fix was not wrong, it -was not loaded: - -``` -vision worker started 12:00:09 -worker_vision.py modified 12:11:35 -8113bdf, carrying _bbox_to_pixels 12:16:22 -``` - -Python binds a module once, at process start. The stage reported success and the orchestrator recorded -no error. Cost: one vision + identity + reconcile cycle and a second registry reset to undo the 8 -characters it minted. `decisions/identity-bbox.md#stale-worker-invalidates`. - -`./check_stale.sh` now compares every running worker's start time against its module mtime and exits -non-zero. Mutation-tested. **Run it before any stage run meant to prove a code change.** - -## Written this session - -- `db.reset_registry` + `POST /characters/reset` (`confirm=true`). `/stage/clear` spares `characters` - by design (`db.py:790`), so nothing could rebaseline the registry. Also clears - `identity_assignment_sources`, which no stage clear touches and where a leftover `manual` row makes - `assign_identity` refuse the next model assignment (`db.py:663`). - Covered by `test_db.py:TestResetRegistry`. **Orchestrator, committed on homesrv, image rebuilt.** -- `check_stale.sh`, `audit_registry.py` (runs inside `manga-orchestrator`, already `docker cp`'d). -- **`has_face`, the extras gate. This is the part not yet proven.** Fixing the coordinate space made - extras worse. With crops finally on their subjects, an extra bound to the lead at 1.00. `/vision` stamps - `has_face` per character using `face_detect` + `_pair_faces_to_present`, so containment and its - margin are the rules the speaker path already uses. `worker_identity.py` skips `has_face is False` - before it crops or embeds. Fails open on a missing or raising detector, and gates on `is False` so an - older vision blob behaves as before. `decisions/identity-bbox.md#face-gates-enrollment`. +- **The unmerge path and the split.** No wrong merge has been seen since the crops were fixed. Undoing + one today is a hand-written SQL walk of the two records above. Revisit trigger is in the caveat. +- **`service._present_characters` gating.** An extra picked as speaker already resolves to unknown. A + character drawn from behind has no face box. Gating would delete a real speaker from the only list that + can attribute their line. +- **Vision still emits extras into the blob.** Deliberate, so the audit can see what was gated. ## Checks ```bash -.venv/bin/python worker_vision.py # includes the has_face gate + both fail-open paths -.venv/bin/python worker_identity.py -./check_stale.sh # exits non-zero if a worker predates its file -cd /home/kami/docker-apps/manga-infra/orchestrator && python3 -m pytest -q --ignore=test_api.py # 115 passed, on homesrv +.venv/bin/python worker_identity.py # ok +.venv/bin/python worker_scene.py # ok, 3 new cases on the actions gate +./check_stale.sh # exit 0, all 9 workers current +/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 -m pytest -q --ignore=test_api.py" # 116 passed ``` +Deployed and verified inside the running container: `resolve_outcome` returns `mint`, `_beat` drops a +faceless detection, `merged_into` exists on the live database with 0 rows merged. Vision, identity and +scene workers restarted. + ## Next command -The `has_face` gate has never run on a GPU. Restart vision and identity, reset, rerun, and check whether -the two wrong bindings on panel 7 disappear without taking Seonho with them. +Four changes ride one GPU cycle. Coverage is 70% and is the number to beat. A gate that abstains too hard +shows up there before it shows up on panel 7. Watch the identity log line for `minted N anonymous`. ```bash -cd /home/kami/Programs/n8n-worker && ./check_stale.sh # restart anything it flags +cd /home/kami/Programs/n8n-worker && ./check_stale.sh # must exit 0 J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"vision\"}'" @@ -91,20 +98,27 @@ for S in vision identity reconcile; do /usr/bin/ssh kami@192.168.1.104 "curl -s /usr/bin/ssh kami@192.168.1.104 "docker exec manga-orchestrator python3 /app/audit_registry.py 7c944dd4-e972-42c7-ba60-9f6939548e80 6" ``` -Watch for coverage collapsing. 70% is the number to beat. A gate that abstains too hard shows up there -before it shows up on panel 7. Restart the identity worker after every reset: it caches the known list -in-process and only invalidates on enrollment. +Panel 7 (`panel_index 6`) is the worked example. Before this session's changes: -## Gone +``` +person_1 [457, 657, 642, 937] Seonho 0.00 foreground, correct +person_2 [669, 591, 763, 822] character_f7a4fd 0.00 the unnamed colleague, correct +person_3 [428, 386, 496, 526] -- none -- extra +person_4 [498, 386, 568, 533] character_d72710 0.94 extra, WRONG +person_5 [31, 554, 94, 728] Seonho 1.00 extra bound to the lead, WRONG +person_6 [34, 414, 122, 564] -- none -- extra +``` -The rebuilt `chapter.mp4` and all 49 clips were deleted by the cascade from `/stage/clear vision`. The -user chose not to keep a copy. Nothing downstream of `reconcile` exists for this job now. +`person_4` and `person_5` are what `has_face` must remove without taking `person_1` or `person_2` with +them. Restart the identity worker after every reset: it caches the known list in-process and only +invalidates on enrollment. -## Traps confirmed again +## Traps confirmed or found -- A stage reporting `completed 116/116` says the code ran, not that the current code ran. -- The SSHFS mount at `/mnt/server/home/kami` was absent and needed remounting by the user, since the - mountpoint needs root. -- The orchestrator image bakes its source. Editing the repo on homesrv does nothing until - `docker compose up -d --build orchestrator`. -- `rtk grep` searches files, not stdin. Piping into it silently searches the repo instead. +- **`tmux respawn-window -k` does not re-run the window command.** It leaves a bare shell. Both vision and + identity sat dead for two minutes before an empty `/health` caught it. Now in `AGENTS.md`. +- `docker compose up --build orchestrator` recreates the container and drops any `docker cp`'d file, so + `audit_registry.py` needs re-copying after every rebuild. +- Two test assertions asserted the old destructive merge (`test_db.py:235`, `test_merge_refs.py:37`). They + were rewritten, not deleted: the invariant changed on purpose. +- The orchestrator image bakes its source. Editing the repo on homesrv does nothing until the rebuild. -- 2.52.0 From 18b49c43bd6055305570a04f96fb423360d6c2ae Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 21:51:36 +0400 Subject: [PATCH 22/31] Record two GPU cycles: the gate works, coverage is the open question No worker code changed. This is the evidence from the 2026-08-12 16:39 and 17:38 runs, and where each finding now lives. The fourth session's four identity changes all work on real panels. Panel 7's two wrong bindings are gone. The lead going unassigned there is correct and was measured, not assumed: face_detect finds one face on the whole panel at conf 0.599, nothing else above 0.056 even at a 0.04 threshold, and the crop shows him drawn from behind. Two decisions, both closed: a roster name is a guess so it never reaches detection, and merged_into is exactly one hop deep. Two caveats, both open: detection can order a bbox backwards (1 in 117), and identity coverage has fallen on every run since the gate landed (70 -> 61 -> 50). Coverage is the thing to settle next, and not by reading the number. identity_labels already holds 145 rows of ground truth. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 198 ++++++++++++++++++++++--------------- JOURNAL.md | 127 ++++++++++++++++++++++++ NEXT.md | 93 +++++++++-------- caveats/CLAUDE.md | 2 + caveats/audit-open.md | 26 +++++ decisions/CLAUDE.md | 2 + decisions/identity-bbox.md | 38 +++++++ 7 files changed, 366 insertions(+), 120 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index db3df29..2a9fd5c 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,124 +1,160 @@ -# HANDOFF, 2026-08-12 (fourth session of the day) +# HANDOFF, 2026-08-12 (fifth session) Live state is in `NEXT.md`. This file is only what this session did. The previous handoff is in `JOURNAL.md`. ## Asked -"How much will `has_face` help with character and identity problems?", then in order: fix 2b, fix -extra-versus-cast, 2d. +"go ahead" on the GPU cycle the fourth session left staged. Then "what do we do now". Then go ahead on +the fixes its evidence asked for, and run it again. ## Result -Identity item 2 in `NEXT.md` is now done as far as it can go without a run. Four changes are written, -self-checked and deployed. **None has run on a GPU.** No pipeline stage was executed this session. +Two full GPU cycles ran. All four of the fourth session's identity changes are now proven on real panels. +Four more fixes were written on top, all orchestrator-side, all deployed. Coverage fell twice and that is +the open question. -| item | was | now | -| --- | --- | --- | -| 2a bbox space | done last session | unchanged | -| 2b abstain and stay abstained | resolver blamed, unverified | fixed in the orchestrator, `f6804e7` + `ffda3df` | -| 2c extra versus cast | open | two consumers gated, `ca46617` + `8b27aec` | -| 2d merge and split | open | examples gone; merge made non-destructive, `00096cc` | +| cycle | detections | assignments | coverage | registry | named | +| --- | --- | --- | --- | --- | --- | +| baseline (13:11, pre-change) | 110 | 77 | 70% | 8 | 1 | +| run A (16:39-16:45) | 110 | 67 | 61% | 16 | 2 | +| run B (17:38-17:44) | 117 | 59 | 50% | 20 | 2 | -## What each change was, and where the cause turned out to be +Vision is non-deterministic, so detection counts move between runs. Each cycle is ~6 minutes: +vision ~3m50s, identity ~1m25s, reconcile ~50s. -**2b was never the resolver.** `/vision/resolve` at `worker_vision.py:1071` already returns -`state="new"` for `choice: 0` and `state="unresolved"` for an out-of-range index. `service.py` read only -`v.get("character_id")` and ran `unassign_identity` on every crop of the tracklet for either one. A -deliberate NONE produced nothing, so an unnamed recurring person was `unknown` on every panel. +## First, a correction the session started with -The old `ponytail:` comment above that block named the real blocker and was right. Minting needs an -`embedding_uri` the orchestrator cannot compute. siglip is resident in the identity worker and gemma in -the vision worker. Fixed by carrying the embedding, not by adding a GPU pass. -`/identity/resolve` writes each crop's embedding to the crop's key with a `.npy` suffix and returns -`emb_uri`. `tracklets.resolve_outcome` holds the three-way decision as a pure function. -`decisions/identity-bbox.md#none-mints-an-anonymous-character`. +The vision/identity/reconcile timestamps in `/job/status` are UTC. The git log is local, UTC+4. The run +that looked like a completed rerun was the pre-change baseline: it finished 13:17 local, and +`_mark_has_face` was not committed until 19:07. The fourth session's handoff was right that nothing had +run. -**2c: `build_scene` already dropped extras from the cast list.** `worker_scene.py:63` skips an unassigned -detection, so extras never reached `characters` or `present`. Their **actions** did. `actions` was built -from every detection. That list is the script prompt's content and the verifier's evidence. So a background -extra's "standing at the window" arrived as a fact with no character attached, and the verifier confirmed -it. Gated `worker_scene`'s `actions` and `service._beat` on `has_face is False`. Left -`service._present_characters` ungated on purpose, reasoned out in -`decisions/identity-bbox.md#extras-gate-consumers`. +## Run A: the four staged changes, verified -**2d's worked examples no longer exist.** The registry reset deleted `Lim Seonho` and -`character_afa7623b`. The registry is 8 rows, one named. Built the safety net for the coming rerun -instead, since `reconcile` runs inside it: `merge_characters` sets `merged_into = keeper` rather than -deleting, and stamps every repointed assignment `method = merged_from:`. Roster readers filter -`merged_into IS NULL`, lookup by id does not. -`caveats/audit-open.md#destructive-reconcile` is half-closed. +| change | verdict | +| --- | --- | +| `has_face` stamp | on all 110 detections, 67 true / 43 gated (39%) | +| extras gate | panel 7's two wrong bindings gone | +| NONE mints | 16 characters where 8 existed | +| non-destructive merge | 7 rows carry `merged_into`, 9 assignments stamped `merged_from:` | -## Measured, read-only, before writing anything +Panel 7, the worked example: ``` -registry: 8 characters, 1 named -> ['Seonho'] -detections: 110 assignments: 77 = 70% coverage -spread: Seonho 36, character_565c88 24, character_759e23 9, character_f7a4fd 3, - character_25f682 3, character_d72710 1, character_823aba 1 -bbox space: 77/110 exceed 1000, 1 on 1000 -> PIXELS +person_1 Seonho 1.00 -> -- none -- has_face=False +person_2 character_f7a4fd 0.00 -> character_519d2b 0.00 has_face=True +person_4 character_d72710 0.94 -> -- none -- WRONG binding removed +person_5 Seonho 1.00 -> -- none -- WRONG binding removed ``` -That killed the assumed cause of 2b. Anonymous ids already recur, so the identity worker's own -pending-promote path gives stable anonymous identities. Only the gemma NONE branch was discarding people. +`person_1` going unassigned is correct, and this was measured, not assumed. Ran `face_detect.detect_faces` +on `p006.png` directly: one face on the whole panel, conf 0.599, inside `person_2`. Swept the threshold to +0.04 and nothing else appears above 0.056. The crop shows the lead drawn from behind at his desk, back of +the head and headphones. Detector right, gate right, lead unidentifiable in that panel. -## Deliberately not built +`person_1`'s box now frames the lead. The old "empty window mullion" note was written against pre-fix +boxes and is retired. -- **The unmerge path and the split.** No wrong merge has been seen since the crops were fixed. Undoing - one today is a hand-written SQL walk of the two records above. Revisit trigger is in the caveat. -- **`service._present_characters` gating.** An extra picked as speaker already resolves to unknown. A - character drawn from behind has no face box. Gating would delete a real speaker from the only list that - can attribute their line. -- **Vision still emits extras into the blob.** Deliberate, so the audit can see what was gated. +## Run A's new defect: a roster hint named the wrong man + +Verified against the art, three real people: + +- `character_92129ac7` "Lim Seonho", 22 assignments. The p010 caption reads `LIM SEONHO (29)`, yellow + plaid shirt, headphones, matching the roster's description. Correct. +- `character_556aef60`, 25 assignments, unnamed. The woman with short black hair and pearls, the roster's + second character, whom the roster itself calls "Unknown". Correct. +- `character_dbadfff7` "Seonho", 15 assignments. A different man, glasses, dark clothing, wearing the + lead's roster name. + +p020 assigned `Seonho` to `person_1` and `Lim Seonho` to `person_3`, two people in one panel. + +## What was changed, and where the cause was + +**Roster hints no longer seed detection.** Deleted the two lines at `service.py:882` that appended +`_roster_char_hints` to `known_characters`. `build_detect_prompt` drops any hint without a name. Passing a +nameless hint would have contributed nothing, so removing the seeding was the only real option. +Names now reach detection from registry rows only, which carry embeddings and were named from an in-panel +caption or address. The roster still feeds `roster_cast` in `run_stage_dialogue`, where names match +against speech rather than faces. + +**`merge_characters` keeps `merged_into` one hop deep.** Two halves, one per direction, and the second was +missed on the first attempt: + +- resolve the keeper to its chain root before merging (a bounded 64-step walk, the cap only so a cycle + cannot hang reconcile) +- repoint the loser's own dependents to the keeper when the loser is retired + +The keeper walk alone does not work. Run B still produced `477c1894 -> a92d9df4 -> 4fb94c15`. At merge +time that pair was fine. The chain formed later, when a row that was already somebody's keeper was itself +retired. + +**`reset_registry` deletes `name_claims`** for the manga's panels and reports the count. Confirmed live: +run B's reset reported `name_claims: 5`, the orphans that had pointed at `character_afa7623b` since two +resets earlier. + +**`audit_registry.py` is in the repo.** It had only ever been `docker cp`'d, so every rebuild dropped it. +`Dockerfile` has `COPY . .`, so it is baked now and that trap is gone. + +## Run B: what the fixes did + +The registry split is fixed. "Lim Seonho" came back as ONE row holding 25 assignments. That answers the +open risk from before the run: dropping the roster hint did not split the lead across the panels before +his caption. "Seonho" fell from 15 assignments to 1. + +Two things run B surfaced: + +- **A second naming mechanism, untouched by the fix.** Even with no roster hint, p011 and p026 emit + `name: "Seonho"` on `person_2` from in-panel text. Detection reads a name off the panel and attaches it + to the wrong body. That is what keeps the glasses man named, now at 1 assignment instead of 15. +- **One degenerate bbox.** `p007 person_1` is `[226, 417, 130, 551]`, x1 > x2. One in 117. It crops to + nothing, so that detection can never enroll or match. `_bbox_to_pixels` clamps each coordinate but never + orders the corners. + +## Not done + +- **The degenerate-box guard.** It is a worker change (`worker_vision.py:_bbox_to_pixels`), so it needs a + vision worker restart and a third GPU cycle to prove. Not started, nothing half-edited. +- **The coverage trend.** 70% -> 61% -> 50%. Part is the `has_face` gate, which is stable at ~39% of + detections across both runs. Framed against face-bearing detections only, run B assigned 59 of 72, or + 82%. Nothing establishes whether the rest is correct abstention or lost cast. Settle that number before + trusting the registry. +- **Nothing downstream re-ran.** The job is parked at `dialogue waiting` with every stage below it + cleared. No clip or chapter has been rebuilt against the new cast. ## Checks ```bash -.venv/bin/python worker_identity.py # ok -.venv/bin/python worker_scene.py # ok, 3 new cases on the actions gate -./check_stale.sh # exit 0, all 9 workers current -/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 -m pytest -q --ignore=test_api.py" # 116 passed +/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 -m pytest -q --ignore=test_api.py" # 118 passed, was 116 +./check_stale.sh # exit 0, all 9 workers current ``` -Deployed and verified inside the running container: `resolve_outcome` returns `mint`, `_beat` drops a -faceless detection, `merged_into` exists on the live database with 0 rows merged. Vision, identity and -scene workers restarted. +Both new merge tests were confirmed to fail with their fix disabled, then the fix was restored and the +suite re-run. The deployed container was verified by parsing its source, not grepping it. The first +attempt gave a false negative, matching `_roster_char_hints` inside the comment that explains its removal. ## Next command -Four changes ride one GPU cycle. Coverage is 70% and is the number to beat. A gate that abstains too hard -shows up there before it shows up on panel 7. Watch the identity log line for `minted N anonymous`. +The third cycle, after adding the corner-ordering guard to `_bbox_to_pixels` and restarting the vision +worker. Watch coverage against 50% and the degenerate box count against 1. ```bash cd /home/kami/Programs/n8n-worker && ./check_stale.sh # must exit 0 J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"vision\"}'" +# restart the identity worker here, see the trap below for S in vision identity reconcile; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 5400 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done /usr/bin/ssh kami@192.168.1.104 "docker exec manga-orchestrator python3 /app/audit_registry.py 7c944dd4-e972-42c7-ba60-9f6939548e80 6" ``` -Panel 7 (`panel_index 6`) is the worked example. Before this session's changes: - -``` -person_1 [457, 657, 642, 937] Seonho 0.00 foreground, correct -person_2 [669, 591, 763, 822] character_f7a4fd 0.00 the unnamed colleague, correct -person_3 [428, 386, 496, 526] -- none -- extra -person_4 [498, 386, 568, 533] character_d72710 0.94 extra, WRONG -person_5 [31, 554, 94, 728] Seonho 1.00 extra bound to the lead, WRONG -person_6 [34, 414, 122, 564] -- none -- extra -``` - -`person_4` and `person_5` are what `has_face` must remove without taking `person_1` or `person_2` with -them. Restart the identity worker after every reset: it caches the known list in-process and only -invalidates on enrollment. - ## Traps confirmed or found -- **`tmux respawn-window -k` does not re-run the window command.** It leaves a bare shell. Both vision and - identity sat dead for two minutes before an empty `/health` caught it. Now in `AGENTS.md`. -- `docker compose up --build orchestrator` recreates the container and drops any `docker cp`'d file, so - `audit_registry.py` needs re-copying after every rebuild. -- Two test assertions asserted the old destructive merge (`test_db.py:235`, `test_merge_refs.py:37`). They - were rewritten, not deleted: the invariant changed on purpose. -- The orchestrator image bakes its source. Editing the repo on homesrv does nothing until the rebuild. +- **`kill $(pgrep -f "worker_identity:app")` kills the shell running it.** The pattern matches the calling + command line. It killed this session's own script mid-way, so the `send-keys` relaunch never fired and + the identity worker sat dead. Restart it with the two `tmux send-keys` lines only, then poll `/health`. +- **Verify deployed code by parsing it, not by grepping it.** A comment explaining a removal contains the + name of the thing removed. +- **A test that cannot fail proves nothing.** Both merge tests were run with their fix disabled first. +- Panel-7 line numbers move between runs, because vision is non-deterministic and `local_id` is assigned + top-to-bottom per run. In run B the lead is `person_5`, not `person_1`. Compare by bbox, not by id. diff --git a/JOURNAL.md b/JOURNAL.md index d4247cb..f9b99e2 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -459,3 +459,130 @@ block, verified on the live database (`merged_into` present, 0 rows merged). Com Deliberately not built: the unmerge path and the split. No wrong merge has been observed since the crops were fixed, so the consumer of these records waits for one. The forward case is partly covered by 2b, since a resolver NONE now mints instead of folding a stranger into the nearest match. + +## 2026-08-12, fifth session — the GPU cycle all four changes were waiting for + +Asked: "go ahead" on the rerun. + +First correction: the vision/identity/reconcile timestamps in `/job/status` are UTC and the git log is +local (UTC+4), so the run that looked like a rerun was the pre-change baseline. `_mark_has_face` landed at +19:07 local, the run finished 13:17 local. The handoff was right that nothing had run. + +Reset the registry for `ef105a86` (8 characters, 77 assignments, 101 sources, 142 S3 objects), cleared +`vision` and everything downstream (116 vision results), restarted the identity worker for its in-process +known-character cache, then ran vision, identity and reconcile. 6 minutes wall: vision 3m48s, identity +1m25s, reconcile 48s. All three `completed`, 116/116/16. + +### What the four changes did + +| change | verdict | +| --- | --- | +| `has_face` stamp | works, present on all 110 detections, 67 true / 43 gated (39%) | +| extras gate | works, panel 7's two wrong bindings are gone | +| NONE mints | works, 16 characters minted where 8 existed | +| non-destructive merge | works, 7 rows carry `merged_into`, 9 assignments stamped `merged_from:` | + +Coverage went 70% -> 61% (77 -> 67 assignments over the same 110 detections). That is the gate's price and +it is mostly correct, see below. + +### Panel 7, the worked example, before and after + +``` +person_1 Seonho 1.00 -> -- none -- has_face=False +person_2 character_f7a4fd 0.00 -> character_519d2b 0.00 has_face=True +person_4 character_d72710 0.94 -> -- none -- WRONG binding removed +person_5 Seonho 1.00 -> -- none -- WRONG binding removed +``` + +The two wrong bindings the gate existed to kill are dead. It also took `person_1`, and that is correct: +ran `face_detect.detect_faces` on p006.png directly and it finds exactly one face on the whole panel, conf +0.599, inside `person_2`. Swept the threshold to 0.04 and nothing else appears above 0.056. Looked at the +crop: `person_1` is the lead drawn from behind at his desk, back of the head and headphones, no face in +frame. The detector is right and the gate is right. The cost is that a back-turned character cannot be +identified from that panel at all. + +Also worth recording: `person_1`'s new box frames the lead correctly. The old note that it framed an empty +window mullion was written against the pre-fix boxes. + +### The new defect: a roster hint named the wrong man + +The registry holds three real people and one wrong name. + +- `character_92129ac7` "Lim Seonho", 22 assignments. Verified against the art: the p010 introduction panel + captioned `LIM SEONHO (29)`, yellow plaid shirt and headphones, which is the roster's description for + "Seonho". This is the lead and the name is right. +- `character_556aef60`, 25 assignments, unnamed. Verified: the woman with short black hair and pearls, the + roster's second character, whom the roster itself calls "Unknown". +- `character_dbadfff7` "Seonho", 15 assignments. Verified: a different man, glasses, dark clothing, in + profile. He carries the lead's roster name. + +p020 assigns `Seonho` to `person_1` and `Lim Seonho` to `person_3`, so the pipeline holds them as two +people in one panel. inference: the roster hint injected at `service.py:882` puts a name in front of +detection, and detection attached it to the wrong face before the caption panel could mint the real one. +This is not the alias-merge case and merging the two rows would be wrong. It needs name binding to require +evidence, the way `name_claims` already does for captions and address. + +### Two smaller things the run exposed + +- **Merge chains.** `character_e1ab7776 -> character_521c301f -> character_556aef60`. `merged_into` points + at a row that is itself merged, so a single-hop resolve lands on a retired character. Roster readers are + fine because they filter `merged_into IS NULL`. Anything that follows one hop is not. +- **Orphan `name_claims`.** All 5 rows point at `character_afa7623b`, which the reset deleted. The reset + clears characters and assignments but not claims. + +Not run: dialogue and everything downstream. The job sits at `dialogue waiting`. + +### Same session — the three fixes the run's evidence asked for + +All three are orchestrator-side, so no worker changed and `check_stale.sh` is not in play. + +- **Roster hints no longer seed detection.** Deleted the two lines at `service.py:882` that appended + `_roster_char_hints` to `known_characters`. `build_detect_prompt` drops any hint without a name, so a + nameless hint would have contributed nothing anyway. Names now reach detection only from registry rows, + which are embedding-backed and were themselves named from an in-panel caption or address. The roster + still feeds `roster_cast` in `run_stage_dialogue`, where names are matched against speech. +- **`merge_characters` resolves the keeper to its chain root** before merging, so `merged_into` stays one + hop deep. Bounded 64-step walk, the cap only exists so a cycle cannot hang reconcile. +- **`reset_registry` deletes `name_claims`** for the manga's panels and reports the count. + +Checks: 117 passed on homesrv, up from 116. The new chain test was confirmed to fail with the walk +disabled, then the walk was restored and the suite re-run. Deployed by rebuilding the image; verified +inside the running container by parsing the deployed source, not by grepping it, because the first check +matched the word `_roster_char_hints` inside the comment that explains its removal. + +Also copied `audit_registry.py` into the repo before the rebuild. It had only ever been `docker cp`'d, so +every rebuild dropped it. `Dockerfile` has `COPY . .`, so it is baked now and the trap is gone. + +Open risk on the next run: without the roster hint the lead is unnamed until the p010 caption, so panels 1 +to 9 may mint him as an anonymous character that reconcile then has to merge. Watch whether "Lim Seonho" +comes back as one row or two. + +### Same session — run B, 17:38-17:44 + +Reset (16 characters, 67 assignments, 67 sources, **5 name_claims**, 156 S3 objects), cleared `vision`, +restarted the identity worker, ran the three stages. The `name_claims: 5` line is the orphan fix confirmed +on live data. + +117 detections, 59 assignments, 50% coverage, 20 characters, 72 with `has_face` (45 gated). + +The registry split is fixed. "Lim Seonho" is one row with 25 assignments, "Seonho" fell from 15 to 1. So +dropping the roster hint did not split the lead across the panels before his caption, which was the risk +recorded before the run. + +Two findings, both now filed: + +- detection still names from in-panel text and can attach the name to the wrong body (p011, p026), which + is `decisions/identity-bbox.md#roster-does-not-name`'s "not covered" paragraph +- one degenerate bbox, `caveats/audit-open.md#degenerate-bbox` + +And one fix of my own that was incomplete: the merge chain came back as +`477c1894 -> a92d9df4 -> 4fb94c15` with the keeper walk deployed. The walk only sees the pair being merged +now. Retiring a row that is already somebody's keeper needed the second half, a repoint of the loser's +dependents. Both halves and both tests are in `decisions/identity-bbox.md#merge-chains-flatten`. + +Checks: 118 passed, was 116. Each new merge test was run with its fix disabled and confirmed to fail +first. Deployed by rebuild and verified by parsing the container's source. + +Coverage is now 70 -> 61 -> 50 across three runs and is the open question, recorded as +`caveats/audit-open.md#coverage-trend`. `identity_labels` already holds 145 rows of ground truth, so the +next move is to score with `eval_identity.py` rather than to keep reading the coverage number. diff --git a/NEXT.md b/NEXT.md index 5318855..7dca918 100644 --- a/NEXT.md +++ b/NEXT.md @@ -1,51 +1,66 @@ # NEXT -Updated 2026-08-12. What this session did is in `HANDOFF.md`. +Updated 2026-08-12 (fifth session). What the fourth session did is in `HANDOFF.md`, the run is in +`JOURNAL.md`. ## State The chapter runs end to end. The A/V sync defect is fixed and `chapter.mp4` is rebuilt: video 364.120s -against audio 364.122s at `25/1`. The identity defects are still in the output. +against audio 364.122s at `25/1`. -Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels, -`status=completed`, finished 2026-08-11T20:08:16Z. `s3://video/` holds 49 clips and a 50MiB -`chapter.mp4`. The user watched it and read out 19 defects. They are grouped by cause in `HANDOFF.md`. +Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels. +Two GPU cycles ran on 2026-08-12, the last 17:38-17:44 UTC on a reset registry. The job is `running` and +parked at `dialogue waiting`. Everything from `dialogue` down is cleared and stale. -One number sets the agenda: +Seven changes are now proven on real panels: the fourth session's four, plus this session's roster-hint +removal, merge-chain flattening and `name_claims` reset. The registry after the last run: -- Panel 7 checked against the art has **zero correct identity bindings** out of two, and the one - character who matters is unbound. `HANDOFF.md#panel-7-walked-against-the-art` has the table. The - coordinate cause is fixed. The registry built on it is not. +- `character_713ea2ce` "LIM SEONHO", 25 assignments, the lead, one row, named from the p010 caption. +- `character_4fb94c15`, 18 assignments, the woman, correct and unnamed. +- `character_023ba5a3` "Seonho", 1 assignment, still the wrong man, down from 15. + +Coverage is 50%, from 61% and 70% before it. That trend is the open question and is filed as +`caveats/audit-open.md#coverage-trend`. ## Next -1. **Re-run vision and identity.** The `bbox` space is settled and converted at `/vision` - (`decisions/identity-bbox.md#bbox-is-normalized`). Every stored box, embedding and `ref_image_uris` in - the registry came from the wrong space. The fix changes nothing until those stages run again. - This is GPU work and needs the user's go-ahead. Clear `vision` and everything downstream of it, or - accept that the boxes in the database stay normalized while new ones are pixels. - - Watch two things on the rerun. Whether `som_face` still returns `unknown` on every face, since gated - pairing was comparing pixel face boxes against 0-1000 character boxes. And whether `Choi Haeseon` still - absorbs every unnamed woman, which is item (b) below and independent of the crops. +1. **Settle coverage before trusting the registry.** 70% -> 61% -> 50% over three runs. `has_face` gates a + steady 39%, and against face-bearing detections alone the last run assigned 59 of 72 (82%). Nothing + separates correct abstention from lost cast. `identity_labels` already holds 145 rows of human ground + truth, and `eval_identity.py` already scores against it. Measure precision and recall instead of + reading the coverage number (`caveats/audit-open.md#coverage-trend`). +2. **Order the corners in `_bbox_to_pixels`.** `p007 person_1` came back `[226, 417, 130, 551]`, x1 > x2, + 1 in 117. It crops to nothing, so that detection is silently lost + (`caveats/audit-open.md#degenerate-bbox`). Two `min`/`max` pairs. It is a worker change, so it needs a + vision restart and a GPU cycle to prove. +3. **Detection still names from in-panel text and can hit the wrong body.** p011 and p026 emit + `name: "Seonho"` on `person_2` with the roster hint gone. That is the residue of the naming defect and + the reason the glasses man is named at all + (`decisions/identity-bbox.md#roster-does-not-name`, "not covered"). +4. **Decide what a back-turned character costs.** `has_face` gates 39% of detections. Panel 7's lead is at + his desk from behind. `face_detect` finds one face on the whole panel at conf 0.599, and nothing else + above 0.056 even at a 0.04 threshold. The gate is right and the detector is right. The lead is still + unidentifiable there. Options are a body or head detector alongside the face one, or letting the + tracklet carry identity across a back-turned panel. Do not lower `FACE_CONF`, the sweep shows nothing + to find. Smaller follow-on: nine `_audio_dur` calls in `worker_render.py` measure finished clips with `format=duration`. So the durations reported to the orchestrator are blind to per-clip drift. They position no filter, so invariant 9 does not cover them. Worth converting to `_stream_dur`. -2. **Fix identity, in this order.** Panel 7 is the worked example and - `HANDOFF.md#panel-7-walked-against-the-art` carries the evidence. Do not start at the registry. +5. **Fix identity, in this order.** All of 2a-2d below are done and now proven on a GPU. Kept for the + evidence trail. a. ~~Settle the `bbox` coordinate space.~~ **Done 2026-08-12**, proven over all 113 detections and checked by eye on panel 7, where five of six converted boxes land on their subject - (`decisions/identity-bbox.md#bbox-is-normalized`). `person_1` still frames an empty window mullion, - which is (c). - b. ~~Let identity abstain and stay abstained.~~ **Done 2026-08-12, not yet run on a GPU** + (`decisions/identity-bbox.md#bbox-is-normalized`). The rerun settles the last doubt: `person_1` now + frames the lead at his desk, not the window mullion the pre-fix box caught. + b. ~~Let identity abstain and stay abstained.~~ **Done 2026-08-12, run and verified on a GPU** (`decisions/identity-bbox.md#none-mints-an-anonymous-character`). The resolver could always answer "none of these". The orchestrator was discarding the answer: it read only `character_id`, so a deliberate NONE and a hallucinated index both unassigned every crop of the tracklet. A NONE now mints an anonymous character from the crop, using the embedding `/identity/resolve` ships beside it as `emb_uri`. Deployed: image rebuilt, `resolve_outcome` verified inside the container. - c. ~~Separate extra from cast.~~ **Done 2026-08-12, not yet run on a GPU** + c. ~~Separate extra from cast.~~ **Done 2026-08-12, run and verified on a GPU** (`decisions/identity-bbox.md#face-gates-enrollment`, `decisions/identity-bbox.md#extras-gate-consumers`). `has_face` stops a faceless detection enrolling, and two more consumers now skip it: `worker_scene`'s `actions`, which is the script @@ -53,11 +68,11 @@ One number sets the agenda: `_present_characters` stays ungated on purpose, reasoned out in the decision. The remaining gap is that vision still emits extras into the blob, which is deliberate so the audit can see what was gated. - d. **The worked examples are gone.** The registry reset deleted `Lim Seonho` and - `character_afa7623b`. The current registry is 8 rows, one named (`Seonho`), so there is nothing to - merge or split until the rerun mints a new set. + d. ~~Merge and split.~~ **Safety net done and now exercised.** The rerun's `reconcile` merged 7 rows + and stamped 9 assignments `merged_from:`, so the non-destructive path is proven on real data. It also + produced the chain in item 3, which is the first thing to fix in it. - What was done instead is the safety net for that rerun, since `reconcile` runs inside it. A merge no + A merge no longer deletes the losing row: it sets `merged_into`, and stamps every repointed assignment with `method = merged_from:`. A wrong merge now costs a hand-written SQL walk, not a full rebaseline (`caveats/audit-open.md#destructive-reconcile`). @@ -69,9 +84,9 @@ One number sets the agenda: **Cast profiles already exist. Do not rebuild them.** The user asked whether the main cast could get a profile built from reference frames and reused. `characters` already carries `ref_image_uris` and - `embedding_uri`, and all 53 rows have both populated. The mechanism is not missing, it is enrolled - from the wrong crops, so today it stores references to balloon edges and window frames. Step (a) is - what makes it work. Three things are genuinely absent and are the smaller follow-on: + `embedding_uri`, and every row has both populated (16 rows after the rerun, 53 before it). The + mechanism was never missing. It was enrolled from the wrong crops, so it stored balloon edges and + window frames. Step (a) fixed that, and the audit now reports 0 characters missing a ref crop. Three things are genuinely absent and are the smaller follow-on: - no quality gate on enrollment, so nothing checks that a reference crop holds a face at all - nothing re-enrolls a reference set once it is written, so the wrong crops persist @@ -91,21 +106,21 @@ One number sets the agenda: The chibi at 1:35 will survive all of this. He genuinely is brown hair plus a yellow shirt, so a profile match is correct on appearance and wrong on reality. That needs item 4 below, plus requiring a real face before a crop can enroll. -3. **Stop the narration inventing facts.** 0:43, 2:03, 2:05 and 2:15 assert things no panel shows. The +6. **Stop the narration inventing facts.** 0:43, 2:03, 2:05 and 2:15 assert things no panel shows. The correctness verifier passed 116/116 because it checks quotes and names, never invented claims. -4. **Teach vision that art inside a panel is not the scene.** A chibi on a monitor became "a man holding +7. **Teach vision that art inside a panel is not the scene.** A chibi on a monitor became "a man holding a drink" at 1:35. A colleague pointing into the distance became "pointing towards the screen" at 1:59. -5. **`layers` writes nothing** and reports `completed 116/116`, so no clip has parallax and a still +8. **`layers` writes nothing** and reports `completed 116/116`, so no clip has parallax and a still holds for 28s from 2:24 (`caveats/audit-open.md#layers-writes-nothing`). -6. **Clear the stale job error.** The completed job still carries `error: "partial: 112/116 completed"` +9. **Clear the stale job error.** The completed job still carries `error: "partial: 112/116 completed"` (`caveats/audit-open.md#stale-job-error`). -7. Balloon-to-speaker geometry via the unused `det`/`seg` heads - (`caveats/speaker-attribution.md#tail-is-not-geometry`) is now behind item 2. With no name to attach, +10. Balloon-to-speaker geometry via the unused `det`/`seg` heads + (`caveats/speaker-attribution.md#tail-is-not-geometry`) is now behind item 5. With no name to attach, geometry buys nothing. -8. Resolve a speaker answer across the whole dialogue window, not just the answering panel. The last 3 +11. Resolve a speaker answer across the whole dialogue window, not just the answering panel. The last 3 unresolved refs describe a neighbouring panel in the same 8-panel call. -9. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work +12. Start Phase 2 from `ROADMAP.md`. Set SQLite `busy_timeout` before any concurrency work (`caveats/audit-open.md#sqlite-locking`). ## Lesson worth keeping diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index 16d6e6d..e8b9ce2 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -42,3 +42,5 @@ a complaint, so give it one or drop it. | [Identity cannot say "a person with no name"](speaker-attribution.md#no-anonymous-identity) | 2026-08-12 panel 7 | | [Vision does not separate a background extra from cast](speaker-attribution.md#extras-as-cast) | 2026-08-12 panel 7 | | [Cast reference profiles are enrolled from wrong crops](speaker-attribution.md#poisoned-reference-set) | 2026-08-12 panel 7 | +| [Detection can order a bbox backwards](audit-open.md#degenerate-bbox) | next vision run | +| [Identity coverage has fallen on every run since the gate landed](audit-open.md#coverage-trend) | before the next downstream run | diff --git a/caveats/audit-open.md b/caveats/audit-open.md index f0f4f92..9eea843 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -181,3 +181,29 @@ Every clip in that run therefore has no parallax. The stage is a sibling of Costs: a silent quality regression that no status field reveals. Revisit when: parallax matters for a deliverable, or before quoting this run as a full-pipeline pass. + +## Detection can order a bbox backwards {#degenerate-bbox} + +`p007 person_1` came back as `[226, 417, 130, 551]` on the 2026-08-12 17:38 run: x1 greater than x2. One +detection in 117. `_bbox_to_pixels` clamps every coordinate into the panel but never orders the corners, +so the box survives as a zero-or-negative-width region. It crops to nothing, so that detection can never +enroll, embed or match, and it is silently lost rather than reported. + +The guard is two `min`/`max` pairs in `_bbox_to_pixels`. It was not written this session because it is a +worker change and needs a vision worker restart plus a GPU cycle to prove. + +Revisit trigger: the next vision run. Count degenerate boxes against 1 in 117. + +## Identity coverage has fallen on every run since the gate landed {#coverage-trend} + +70% -> 61% -> 50% across the 13:11 baseline, the 16:39 run and the 17:38 run. The `has_face` gate explains +part of it and is stable, gating 39% of detections on both post-change runs. Against face-bearing +detections only, the 17:38 run assigned 59 of 72, or 82%. + +Nothing yet separates correct abstention from lost cast, and both fixes that could have caused the second +drop landed together. A back-turned lead is a correct abstention. A real character the resolver refused is +not, and the two are indistinguishable in the coverage number alone. + +Revisit trigger: before trusting the registry for a downstream run. `identity_labels` already exists for +exactly this and holds 145 rows of human ground truth, so `eval_identity.py` can score precision against +recall instead of counting assignments. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index d7ef2da..89afdde 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -45,3 +45,5 @@ still live belongs in `caveats/`. | [A detection with no detected face never enrolls or binds](identity-bbox.md#face-gates-enrollment) | closed | | [A resolver NONE mints an anonymous character, it does not clear the crop](identity-bbox.md#none-mints-an-anonymous-character) | closed | | [The extras gate runs at enrollment and at narration, not at the speaker prompt](identity-bbox.md#extras-gate-consumers) | closed | +| [A roster name is a guess, so it never reaches detection](identity-bbox.md#roster-does-not-name) | closed | +| [`merged_into` is exactly one hop deep](identity-bbox.md#merge-chains-flatten) | closed | diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md index 46083b7..1503671 100644 --- a/decisions/identity-bbox.md +++ b/decisions/identity-bbox.md @@ -183,3 +183,41 @@ previous behaviour. Forbids: adding a fourth consumer of `vision["characters"]` without deciding which side of this line it is on. The blob keeps every detection on purpose, so the audit can still see what was gated. + +## A roster name is a guess, so it never reaches detection {#roster-does-not-name} + +`service.py` seeded `known_characters` with `_roster_char_hints` before every detect call, so the chapter +roster's names were in front of gemma before any panel had identified anyone. On the 2026-08-12 16:39 run +that put "Seonho (short brown hair, yellow plaid shirt)" on a different man wearing glasses, who then held +15 assignments under the lead's name. The real lead was minted separately from the p010 caption as +"Lim Seonho", and p020 held both as two people in one panel. + +The seeding is removed. A name now reaches detection only from a registry row, and a registry row is named +from an in-panel caption or address through `name_claims`. The distinction is evidence: a roster name is a +claim about art nobody has looked at yet, and a registry row carries an embedding plus the panel that +named it. + +The roster still feeds `roster_cast` in `run_stage_dialogue`. There the names are matched against speech, +not against faces, so a wrong guess costs an unresolved speaker rather than a wrong face binding. + +Confirmed on the 17:38 run: "Lim Seonho" came back as one row with 25 assignments, so dropping the hint +did not split the lead across the panels before his caption. "Seonho" fell to 1 assignment. + +Not covered: detection still reads a name off the panel and can attach it to the wrong body. p011 and p026 +emit `name: "Seonho"` on `person_2` with no roster hint present. That is a separate mechanism and it is +what keeps the glasses man named at all. + +## `merged_into` is exactly one hop deep {#merge-chains-flatten} + +A merge chain means `merged_into` points at a row that is itself merged, so anything resolving one hop +lands on a retired character. Roster readers filter `merged_into IS NULL` and were never affected. + +Two rules, one per direction, and both are needed: + +- `merge_characters` resolves the keeper to its chain root before merging. Bounded 64-step walk, the cap + only exists so a cycle cannot hang reconcile. +- retiring a loser repoints every row whose `merged_into` was that loser. + +The keeper walk alone is not enough, which the 17:38 run proved by producing +`477c1894 -> a92d9df4 -> 4fb94c15` with the walk deployed. At merge time that pair was fine. The chain +formed later, when a row that was already somebody's keeper was itself retired. -- 2.52.0 From 54bd126cd793ccd67ca3e00548ebfb91822e6db1 Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 22:21:52 +0400 Subject: [PATCH 23/31] Order bbox corners, and find the ground truth is for another manga `_bbox_to_pixels` sorts each coordinate pair after clamping. Run B returned `p007 person_1` as `[226, 417, 130, 551]`, x1 > x2, which cropped to nothing and lost that detection silently. The third GPU cycle came back with 0 degenerate boxes over 119 detections. The plan's first item cannot run as written. All 145 rows of `identity_labels` belong to chapter 8ca8249b, a different manga. Chapter 7c944dd4 has none, so `/review/identity` reports `accuracy: null`. Scoring 8ca8249b gives 7/138 on an identity run that predates every fix. The cycle also settled the coverage question. All 68 assignments landed on face-bearing detections and none on a gated one, so recall among face-bearing detections is 96%, up from 82%. Coverage is the `has_face` gate and nothing else. What is now open is precision: the lead holds 36 of 68 assignments. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 182 +++++++++++-------------------------- JOURNAL.md | 58 ++++++++++++ NEXT.md | 64 +++++++------ caveats/CLAUDE.md | 1 - caveats/audit-open.md | 47 +++++----- decisions/CLAUDE.md | 1 + decisions/identity-bbox.md | 22 +++++ worker_vision.py | 16 +++- 8 files changed, 211 insertions(+), 180 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 2a9fd5c..36be549 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,160 +1,88 @@ -# HANDOFF, 2026-08-12 (fifth session) +# HANDOFF, 2026-08-12 (sixth session) Live state is in `NEXT.md`. This file is only what this session did. The previous handoff is in `JOURNAL.md`. ## Asked -"go ahead" on the GPU cycle the fourth session left staged. Then "what do we do now". Then go ahead on -the fixes its evidence asked for, and run it again. +The two files, then "go" on the third GPU cycle. ## Result -Two full GPU cycles ran. All four of the fourth session's identity changes are now proven on real panels. -Four more fixes were written on top, all orchestrator-side, all deployed. Coverage fell twice and that is -the open question. +One worker fix, one measurement that killed the plan's first item, one GPU cycle. The cycle proved the new +fix and the fifth session's merge fix. Coverage rose for the first time, 50% to 57%. It is now explained +entirely by the `has_face` gate. -| cycle | detections | assignments | coverage | registry | named | -| --- | --- | --- | --- | --- | --- | -| baseline (13:11, pre-change) | 110 | 77 | 70% | 8 | 1 | -| run A (16:39-16:45) | 110 | 67 | 61% | 16 | 2 | -| run B (17:38-17:44) | 117 | 59 | 50% | 20 | 2 | +| metric | 17:38 run | 18:07 run | +| --- | --- | --- | +| detections | 117 | 119 | +| assignments | 59 | 68 | +| coverage | 50% | 57% | +| degenerate boxes | 1 | 0 | +| assigned among face-bearing | 59/72 = 82% | 68/71 = 96% | +| assigned among gated | -- | 0 | +| chains deeper than one hop | 1 | 0 | +| `merged_from` stamps | 9 | 22 | -Vision is non-deterministic, so detection counts move between runs. Each cycle is ~6 minutes: -vision ~3m50s, identity ~1m25s, reconcile ~50s. +## The corner-ordering fix -## First, a correction the session started with +`_bbox_to_pixels` sorts each coordinate pair after clamping. Three lines +(`decisions/identity-bbox.md#bbox-corners-ordered`). The self-check feeds it run B's real +`[226, 417, 130, 551]` and asserts `[117, 688, 203, 909]`. Unsorted the call returns `[203, 688, 117, 909]`, +so the assert fails without the fix. -The vision/identity/reconcile timestamps in `/job/status` are UTC. The git log is local, UTC+4. The run -that looked like a completed rerun was the pre-change baseline: it finished 13:17 local, and -`_mark_has_face` was not committed until 19:07. The fourth session's handoff was right that nothing had -run. +The run returned 0 degenerate boxes over 119 detections. The caveat is deleted and the decision is indexed. -## Run A: the four staged changes, verified +## The measurement that killed item 1 -| change | verdict | -| --- | --- | -| `has_face` stamp | on all 110 detections, 67 true / 43 gated (39%) | -| extras gate | panel 7's two wrong bindings gone | -| NONE mints | 16 characters where 8 existed | -| non-destructive merge | 7 rows carry `merged_into`, 9 assignments stamped `merged_from:` | +The plan said to score precision with `eval_identity.py` against the 145 rows of `identity_labels`. Every +one of those rows belongs to chapter `8ca8249b`, a different manga with cast "Rico" and "Ikekin", spread +over 81 panels. Chapter `7c944dd4` has zero labels, so +`/review/identity?job_id=778297bc...` returns `labeled: 0, correct: 0, accuracy: null`. -Panel 7, the worked example: +Scoring `8ca8249b` anyway gives 7/138, with 113 rows labelled as a real person and left unassigned. That +chapter's identity run is stale: 44 assignments over 246 panels, predating every fix. The number measures +old code on the wrong chapter. -``` -person_1 Seonho 1.00 -> -- none -- has_face=False -person_2 character_f7a4fd 0.00 -> character_519d2b 0.00 has_face=True -person_4 character_d72710 0.94 -> -- none -- WRONG binding removed -person_5 Seonho 1.00 -> -- none -- WRONG binding removed -``` +So the eval path is proven end to end and the ground truth is absent. A precision number needs a hand pass +over `7c944dd4` through `POST /review/identity/label`, keyed by bbox rather than `local_id`. -`person_1` going unassigned is correct, and this was measured, not assumed. Ran `face_detect.detect_faces` -on `p006.png` directly: one face on the whole panel, conf 0.599, inside `person_2`. Swept the threshold to -0.04 and nothing else appears above 0.056. The crop shows the lead drawn from behind at his desk, back of -the head and headphones. Detector right, gate right, lead unidentifiable in that panel. +## What the cycle settled -`person_1`'s box now frames the lead. The old "empty window mullion" note was written against pre-fix -boxes and is retired. +**Coverage is the gate and nothing else.** All 68 assignments landed on face-bearing detections and none on +a gated one. Recall among face-bearing detections is 96%, up from 82%. The resolver is not losing cast, so +the suspicion in `caveats/audit-open.md#coverage-trend` is closed. -## Run A's new defect: a roster hint named the wrong man +**The merge-chain fix holds under load.** 9 merges this run against 7 last run, and 22 `merged_from` +stamps. No chain is deeper than one hop. -Verified against the art, three real people: +## What the cycle opened -- `character_92129ac7` "Lim Seonho", 22 assignments. The p010 caption reads `LIM SEONHO (29)`, yellow - plaid shirt, headphones, matching the roster's description. Correct. -- `character_556aef60`, 25 assignments, unnamed. The woman with short black hair and pearls, the roster's - second character, whom the roster itself calls "Unknown". Correct. -- `character_dbadfff7` "Seonho", 15 assignments. A different man, glasses, dark clothing, wearing the - lead's roster name. +**The lead may be absorbing.** 36 of 68 assignments, 53%, against 25 of 59 before. `audit_registry.py` +flags it. A protagonist in half the panels looks identical to an over-merge without labels. -p020 assigned `Seonho` to `person_1` and `Lim Seonho` to `person_3`, two people in one panel. - -## What was changed, and where the cause was - -**Roster hints no longer seed detection.** Deleted the two lines at `service.py:882` that appended -`_roster_char_hints` to `known_characters`. `build_detect_prompt` drops any hint without a name. Passing a -nameless hint would have contributed nothing, so removing the seeding was the only real option. -Names now reach detection from registry rows only, which carry embeddings and were named from an in-panel -caption or address. The roster still feeds `roster_cast` in `run_stage_dialogue`, where names match -against speech rather than faces. - -**`merge_characters` keeps `merged_into` one hop deep.** Two halves, one per direction, and the second was -missed on the first attempt: - -- resolve the keeper to its chain root before merging (a bounded 64-step walk, the cap only so a cycle - cannot hang reconcile) -- repoint the loser's own dependents to the keeper when the loser is retired - -The keeper walk alone does not work. Run B still produced `477c1894 -> a92d9df4 -> 4fb94c15`. At merge -time that pair was fine. The chain formed later, when a row that was already somebody's keeper was itself -retired. - -**`reset_registry` deletes `name_claims`** for the manga's panels and reports the count. Confirmed live: -run B's reset reported `name_claims: 5`, the orphans that had pointed at `character_afa7623b` since two -resets earlier. - -**`audit_registry.py` is in the repo.** It had only ever been `docker cp`'d, so every rebuild dropped it. -`Dockerfile` has `COPY . .`, so it is baked now and that trap is gone. - -## Run B: what the fixes did - -The registry split is fixed. "Lim Seonho" came back as ONE row holding 25 assignments. That answers the -open risk from before the run: dropping the roster hint did not split the lead across the panels before -his caption. "Seonho" fell from 15 assignments to 1. - -Two things run B surfaced: - -- **A second naming mechanism, untouched by the fix.** Even with no roster hint, p011 and p026 emit - `name: "Seonho"` on `person_2` from in-panel text. Detection reads a name off the panel and attaches it - to the wrong body. That is what keeps the glasses man named, now at 1 assignment instead of 15. -- **One degenerate bbox.** `p007 person_1` is `[226, 417, 130, 551]`, x1 > x2. One in 117. It crops to - nothing, so that detection can never enroll or match. `_bbox_to_pixels` clamps each coordinate but never - orders the corners. +**Panel 7 got worse.** 5 of its 6 detections carry `has_face = False`, and `person_6`, the one that does, +went unassigned. The previous run bound its colleague. That is the cost of requiring a face, `NEXT.md` +item 4. ## Not done -- **The degenerate-box guard.** It is a worker change (`worker_vision.py:_bbox_to_pixels`), so it needs a - vision worker restart and a third GPU cycle to prove. Not started, nothing half-edited. -- **The coverage trend.** 70% -> 61% -> 50%. Part is the `has_face` gate, which is stable at ~39% of - detections across both runs. Framed against face-bearing detections only, run B assigned 59 of 72, or - 82%. Nothing establishes whether the rest is correct abstention or lost cast. Settle that number before - trusting the registry. -- **Nothing downstream re-ran.** The job is parked at `dialogue waiting` with every stage below it - cleared. No clip or chapter has been rebuilt against the new cast. +- Nothing downstream re-ran. The job is still parked at `dialogue waiting` with every stage below cleared. +- Nothing is committed. All edits are in the working tree. ## Checks ```bash -/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 -m pytest -q --ignore=test_api.py" # 118 passed, was 116 -./check_stale.sh # exit 0, all 9 workers current +.venv/bin/python worker_vision.py # self-check ok, including the swapped-corner assert +./check_stale.sh # exit 0 before the run and after it ``` -Both new merge tests were confirmed to fail with their fix disabled, then the fix was restored and the -suite re-run. The deployed container was verified by parsing its source, not grepping it. The first -attempt gave a false negative, matching `_roster_char_hints` inside the comment that explains its removal. +## Traps confirmed -## Next command - -The third cycle, after adding the corner-ordering guard to `_bbox_to_pixels` and restarting the vision -worker. Watch coverage against 50% and the degenerate box count against 1. - -```bash -cd /home/kami/Programs/n8n-worker && ./check_stale.sh # must exit 0 -J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e -/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" -/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"vision\"}'" -# restart the identity worker here, see the trap below -for S in vision identity reconcile; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 5400 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done -/usr/bin/ssh kami@192.168.1.104 "docker exec manga-orchestrator python3 /app/audit_registry.py 7c944dd4-e972-42c7-ba60-9f6939548e80 6" -``` - -## Traps confirmed or found - -- **`kill $(pgrep -f "worker_identity:app")` kills the shell running it.** The pattern matches the calling - command line. It killed this session's own script mid-way, so the `send-keys` relaunch never fired and - the identity worker sat dead. Restart it with the two `tmux send-keys` lines only, then poll `/health`. -- **Verify deployed code by parsing it, not by grepping it.** A comment explaining a removal contains the - name of the thing removed. -- **A test that cannot fail proves nothing.** Both merge tests were run with their fix disabled first. -- Panel-7 line numbers move between runs, because vision is non-deterministic and `local_id` is assigned - top-to-bottom per run. In run B the lead is `person_5`, not `person_1`. Compare by bbox, not by id. +- Restart a worker with `tmux send-keys -t manga-workers: C-c`, then re-send the launch line from + `start_workers.sh` with the `MIOPEN_ENV` prefix. `pgrep`-based kills match the calling shell. +- `POST /characters/reset` returns `restart_identity_worker: true`. Honour it, the worker caches the + registry. +- `identity_assignments` has no `method` column. The `merged_from:` stamps live in + `identity_assignment_sources`. +- `chapters` has no `title` column. diff --git a/JOURNAL.md b/JOURNAL.md index f9b99e2..857869e 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -586,3 +586,61 @@ first. Deployed by rebuild and verified by parsing the container's source. Coverage is now 70 -> 61 -> 50 across three runs and is the open question, recorded as `caveats/audit-open.md#coverage-trend`. `identity_labels` already holds 145 rows of ground truth, so the next move is to score with `eval_identity.py` rather than to keep reading the coverage number. + +## 2026-08-12, sixth session — the ground truth is for the wrong manga + +No GPU work. Two things, both cheap. + +**`_bbox_to_pixels` orders its corners.** Each coordinate pair is sorted after clamping. The self-check +feeds it run B's real `[226, 417, 130, 551]` and asserts `[117, 688, 203, 909]`. Unsorted the call returns +`[203, 688, 117, 909]`, so the assert fails without the fix. `worker_vision.py` self-check passes. +The vision worker is now STALE and needs a restart before the next cycle. + +**Item 1 of the plan cannot run as written.** All 145 rows of `identity_labels` belong to chapter +`8ca8249b`, a different manga with cast "Rico" and "Ikekin", spread over 81 panels. Chapter `7c944dd4` +has none, so `/review/identity?job_id=778297bc...` returns `labeled: 0, correct: 0, accuracy: null`. + +Scoring `8ca8249b` anyway gives 7/138, with 113 rows labelled as a real person and left unassigned. That +chapter's identity run is stale: 44 assignments over 246 panels, predating every fix. The number measures +old code on the wrong chapter. + +So the eval path is proven end to end and the ground truth is absent. A precision number for the current +registry needs a hand-labelling pass over `7c944dd4` through `POST /review/identity/label`, keyed by bbox +rather than by `local_id`, because vision reassigns `local_id` every run. + +## 2026-08-12 18:07-18:13 UTC, third GPU cycle + +Restarted vision (window 3) and identity (`/characters/reset` returned `restart_identity_worker: true`), +both by `send-keys C-c` then re-sending the launch line, never by `pgrep`. `check_stale.sh` exit 0 before +the run. Reset dropped 20 characters, 59 assignments, 69 sources, 168 S3 objects. + +vision 116/116 in 3m59s, identity 116/116 in 1m24s, reconcile 18/18 in 50s. + +| metric | 17:38 run | 18:07 run | +| --- | --- | --- | +| detections | 117 | 119 | +| assignments | 59 | 68 | +| coverage | 50% | 57% | +| degenerate boxes | 1 | **0** | +| `has_face` true / gated | 72 / 45 | 71 / 48 | +| assigned among face-bearing | 59/72 = 82% | **68/71 = 96%** | +| assigned among gated | -- | 0 | +| registry rows / merged | 20 / 7 | 18 / 9 | +| `merged_from` stamps | 9 | 22 | +| chains deeper than one hop | 1 | **0** | + +The corner-ordering fix works: 0 degenerate boxes. The merge-chain fix holds under a heavier merge load, +9 merges and no chain. + +**Coverage is now the gate and nothing else.** Every assignment landed on a face-bearing detection and none +on a gated one. Recall among face-bearing detections is 96%. So the 43% with no assignment is the 40% the +gate drops plus 3 detections, and the resolver is not losing cast. That closes the part of +`#coverage-trend` that suspected the resolver. + +**The open question moved to precision.** The lead holds 36 of 68 assignments, 53%, against 25 of 59 +before, and `audit_registry.py` flags it as absorbing. The merge count rose from 7 to 9 over the same +interval. Whether 36 is a protagonist in half the panels or an over-merge cannot be told apart without +labels, which is item 1. + +Panel 7 got worse, not better: 5 of its 6 detections carry `has_face = False`, and `person_6`, the one that +does, went unassigned. The previous run bound its colleague. That is the cost of requiring a face, item 4. diff --git a/NEXT.md b/NEXT.md index 7dca918..ccfb237 100644 --- a/NEXT.md +++ b/NEXT.md @@ -1,6 +1,6 @@ # NEXT -Updated 2026-08-12 (fifth session). What the fourth session did is in `HANDOFF.md`, the run is in +Updated 2026-08-12 (sixth session). What the fifth session did is in `HANDOFF.md`, the runs are in `JOURNAL.md`. ## State @@ -9,38 +9,50 @@ The chapter runs end to end. The A/V sync defect is fixed and `chapter.mp4` is r against audio 364.122s at `25/1`. Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels. -Two GPU cycles ran on 2026-08-12, the last 17:38-17:44 UTC on a reset registry. The job is `running` and -parked at `dialogue waiting`. Everything from `dialogue` down is cleared and stale. +Three GPU cycles ran on 2026-08-12, the last 18:07-18:13 UTC on a reset registry. The job is `running` and +parked at `dialogue waiting`. Everything from `dialogue` down is cleared and stale. All 9 workers are +current. -Seven changes are now proven on real panels: the fourth session's four, plus this session's roster-hint -removal, merge-chain flattening and `name_claims` reset. The registry after the last run: +Eight changes are now proven on real panels. The fourth session contributed four. The fifth added the +roster-hint removal, merge-chain flattening and `name_claims` reset. This session added bbox corner +ordering. The registry after the 18:07 run holds 18 rows, 9 live and 9 carrying `merged_into`: -- `character_713ea2ce` "LIM SEONHO", 25 assignments, the lead, one row, named from the p010 caption. -- `character_4fb94c15`, 18 assignments, the woman, correct and unnamed. -- `character_023ba5a3` "Seonho", 1 assignment, still the wrong man, down from 15. +- `LIM SEONHO`, 36 assignments, the lead, one row, named from the p010 caption. +- `character_b112d4`, 23 assignments, the woman, correct and unnamed. +- `Seonho`, still the wrong man, still named off in-panel text. -Coverage is 50%, from 61% and 70% before it. That trend is the open question and is filed as -`caveats/audit-open.md#coverage-trend`. +Coverage is 57%, from 50%, 61% and 70% before it, and it is now the `has_face` gate and nothing else. All 68 +assignments landed on face-bearing detections and none on a gated one. Recall among face-bearing detections +is 68 of 71, or 96%, up from 82%. Degenerate boxes went from 1 in 117 to 0 in 119, and the merge +chain is flat with 22 assignments stamped `merged_from`. + +The open question moved. The lead holds 36 of 68 assignments, 53%, which `audit_registry.py` flags as +absorbing. Nothing separates a protagonist in half the panels from an over-merge +(`caveats/audit-open.md#coverage-trend`). ## Next -1. **Settle coverage before trusting the registry.** 70% -> 61% -> 50% over three runs. `has_face` gates a - steady 39%, and against face-bearing detections alone the last run assigned 59 of 72 (82%). Nothing - separates correct abstention from lost cast. `identity_labels` already holds 145 rows of human ground - truth, and `eval_identity.py` already scores against it. Measure precision and recall instead of - reading the coverage number (`caveats/audit-open.md#coverage-trend`). -2. **Order the corners in `_bbox_to_pixels`.** `p007 person_1` came back `[226, 417, 130, 551]`, x1 > x2, - 1 in 117. It crops to nothing, so that detection is silently lost - (`caveats/audit-open.md#degenerate-bbox`). Two `min`/`max` pairs. It is a worker change, so it needs a - vision restart and a GPU cycle to prove. +1. **Decide whether the lead absorbing 53% of assignments is real.** It needs labels this chapter lacks. + The resolver is cleared: recall among face-bearing detections is 96%. What is unmeasured is precision. + The lead went from 25 of 59 to 36 of 68 as the merge count rose from 7 to 9. + + The previous plan pointed at `identity_labels` and its 145 rows. Measured this session: every one of + those rows belongs to chapter `8ca8249b`, a different manga. Chapter `7c944dd4` has zero labels, so + `/review/identity` reports `labeled: 0, accuracy: null`. Scoring `8ca8249b` gives 7/138 on an identity + run that predates all eight fixes. The eval plumbing works and the ground truth is missing. + Labelling `7c944dd4` by hand is the only path to a precision number + (`caveats/audit-open.md#coverage-trend`). +2. ~~**Order the corners in `_bbox_to_pixels`.**~~ **Done 2026-08-12, run and verified on a GPU.** 0 + degenerate boxes over 119 detections (`decisions/identity-bbox.md#bbox-corners-ordered`). 3. **Detection still names from in-panel text and can hit the wrong body.** p011 and p026 emit `name: "Seonho"` on `person_2` with the roster hint gone. That is the residue of the naming defect and the reason the glasses man is named at all (`decisions/identity-bbox.md#roster-does-not-name`, "not covered"). -4. **Decide what a back-turned character costs.** `has_face` gates 39% of detections. Panel 7's lead is at - his desk from behind. `face_detect` finds one face on the whole panel at conf 0.599, and nothing else - above 0.056 even at a 0.04 threshold. The gate is right and the detector is right. The lead is still - unidentifiable there. Options are a body or head detector alongside the face one, or letting the +4. **Decide what a back-turned character costs.** This is now the whole of the coverage number. `has_face` + gates 40% of detections. On the 18:07 run panel 7 lost 5 of its 6 detections to the gate, and its one + face-bearing detection went unassigned. Panel 7's lead is at his desk from behind. `face_detect` + finds one face on the whole panel at conf 0.599, and nothing else above 0.056 even at a 0.04 + threshold. The gate is right and the detector is right. The lead is still unidentifiable there. Options are a body or head detector alongside the face one, or letting the tracklet carry identity across a back-turned panel. Do not lower `FACE_CONF`, the sweep shows nothing to find. @@ -68,9 +80,9 @@ Coverage is 50%, from 61% and 70% before it. That trend is the open question and `_present_characters` stays ungated on purpose, reasoned out in the decision. The remaining gap is that vision still emits extras into the blob, which is deliberate so the audit can see what was gated. - d. ~~Merge and split.~~ **Safety net done and now exercised.** The rerun's `reconcile` merged 7 rows - and stamped 9 assignments `merged_from:`, so the non-destructive path is proven on real data. It also - produced the chain in item 3, which is the first thing to fix in it. + d. ~~Merge and split.~~ **Safety net done and now exercised.** The 18:07 run merged 9 rows and stamped + 22 assignments `merged_from:`, with no chain deeper than one hop. The non-destructive path and the + flattening fix are both proven on real data. A merge no longer deletes the losing row: it sets `merged_into`, and stamps every repointed assignment with diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index e8b9ce2..be09a22 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -42,5 +42,4 @@ a complaint, so give it one or drop it. | [Identity cannot say "a person with no name"](speaker-attribution.md#no-anonymous-identity) | 2026-08-12 panel 7 | | [Vision does not separate a background extra from cast](speaker-attribution.md#extras-as-cast) | 2026-08-12 panel 7 | | [Cast reference profiles are enrolled from wrong crops](speaker-attribution.md#poisoned-reference-set) | 2026-08-12 panel 7 | -| [Detection can order a bbox backwards](audit-open.md#degenerate-bbox) | next vision run | | [Identity coverage has fallen on every run since the gate landed](audit-open.md#coverage-trend) | before the next downstream run | diff --git a/caveats/audit-open.md b/caveats/audit-open.md index 9eea843..99c28c8 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -182,28 +182,33 @@ Every clip in that run therefore has no parallax. The stage is a sibling of Costs: a silent quality regression that no status field reveals. Revisit when: parallax matters for a deliverable, or before quoting this run as a full-pipeline pass. -## Detection can order a bbox backwards {#degenerate-bbox} - -`p007 person_1` came back as `[226, 417, 130, 551]` on the 2026-08-12 17:38 run: x1 greater than x2. One -detection in 117. `_bbox_to_pixels` clamps every coordinate into the panel but never orders the corners, -so the box survives as a zero-or-negative-width region. It crops to nothing, so that detection can never -enroll, embed or match, and it is silently lost rather than reported. - -The guard is two `min`/`max` pairs in `_bbox_to_pixels`. It was not written this session because it is a -worker change and needs a vision worker restart plus a GPU cycle to prove. - -Revisit trigger: the next vision run. Count degenerate boxes against 1 in 117. - ## Identity coverage has fallen on every run since the gate landed {#coverage-trend} -70% -> 61% -> 50% across the 13:11 baseline, the 16:39 run and the 17:38 run. The `has_face` gate explains -part of it and is stable, gating 39% of detections on both post-change runs. Against face-bearing -detections only, the 17:38 run assigned 59 of 72, or 82%. +70% -> 61% -> 50% -> 57% across the four 2026-08-12 runs. The gate is stable at 39 to 40% of detections. -Nothing yet separates correct abstention from lost cast, and both fixes that could have caused the second -drop landed together. A back-turned lead is a correct abstention. A real character the resolver refused is -not, and the two are indistinguishable in the coverage number alone. +**The 18:07 run resolves most of this.** Coverage is now the gate and nothing else. All 68 assignments went +to face-bearing detections and none to a gated one. Recall among face-bearing detections is 68 of 71, or +96%, up from 82%. The resolver is not losing cast. The 43% that carries no assignment is the 40% the gate +drops plus 3 detections. -Revisit trigger: before trusting the registry for a downstream run. `identity_labels` already exists for -exactly this and holds 145 rows of human ground truth, so `eval_identity.py` can score precision against -recall instead of counting assignments. +So the coverage number no longer measures the resolver. What it measures is the cost of requiring a face, +which is item 4 in `NEXT.md`. Panel 7 is the case: 5 of its 6 detections carry `has_face = False`, and the +one that does is unassigned. + +The open question moved. The lead now holds 36 of 68 assignments, or 53%, against 25 of 59 before, and +`audit_registry.py` flags that as absorbing. Whether 36 is the protagonist appearing in half the panels or +an over-merge cannot be told apart without labels. + +Revisit trigger: before trusting the registry for a downstream run. + +**The 145 ground-truth labels do not cover this chapter.** Measured 2026-08-12. All 145 rows in +`identity_labels` key to chapter `8ca8249b`, a different manga with cast "Rico" and "Ikekin", over 81 +distinct panels. Chapter `7c944dd4` has zero labels, so `/review/identity?job_id=778297bc...` returns +`labeled: 0, accuracy: null`. Scoring 8ca instead gives 7/138, because its identity run is stale: 44 +assignments over 246 panels, predating all seven fixes. That number measures the old code on the wrong +chapter and settles nothing. + +The plumbing works, the labels are missing. Precision and recall on the current registry need a hand +pass over `7c944dd4` first, via `POST /review/identity/label` per detection. Label against the bbox, not +the `local_id`. Vision is non-deterministic and reassigns `local_id` top-to-bottom every run. A label +taken before a vision rerun then points at whoever now occupies that slot. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 89afdde..98943df 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -47,3 +47,4 @@ still live belongs in `caveats/`. | [The extras gate runs at enrollment and at narration, not at the speaker prompt](identity-bbox.md#extras-gate-consumers) | closed | | [A roster name is a guess, so it never reaches detection](identity-bbox.md#roster-does-not-name) | closed | | [`merged_into` is exactly one hop deep](identity-bbox.md#merge-chains-flatten) | closed | +| [`_bbox_to_pixels` orders the corners, because the model sometimes swaps them](identity-bbox.md#bbox-corners-ordered) | closed | diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md index 1503671..d5fc2f0 100644 --- a/decisions/identity-bbox.md +++ b/decisions/identity-bbox.md @@ -221,3 +221,25 @@ Two rules, one per direction, and both are needed: The keeper walk alone is not enough, which the 17:38 run proved by producing `477c1894 -> a92d9df4 -> 4fb94c15` with the walk deployed. At merge time that pair was fine. The chain formed later, when a row that was already somebody's keeper was itself retired. + +## `_bbox_to_pixels` orders the corners, because the model sometimes swaps them {#bbox-corners-ordered} + +**Closed, 2026-08-12.** + +The 17:38 run returned `p007 person_1` as `[226, 417, 130, 551]`, x1 greater than x2. One detection in 117. +Clamping each coordinate into the panel kept the swap, so the box stayed a negative-width region. It +cropped to nothing, so that detection could not enroll, embed or match, and nothing reported the loss. + +`_bbox_to_pixels` now sorts each pair after clamping: + +```python +xs = sorted((clamped_x1, clamped_x2)) +ys = sorted((clamped_y1, clamped_y2)) +c["bbox"] = [xs[0], ys[0], xs[1], ys[1]] +``` + +Sorting is enough. A zero-area box still crops to nothing, and no consumer needs a minimum size that it +does not already enforce. The self-check feeds the real swapped box in and asserts `[117, 688, 203, 909]`, +which fails without the sort. + +Proven on the 18:07 run: 0 degenerate boxes over 119 detections. diff --git a/worker_vision.py b/worker_vision.py index dcc40ce..3bbfb4e 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -323,10 +323,13 @@ def _bbox_to_pixels(chars: list, w: int, h: int) -> list: b = c.get("bbox") if not (isinstance(b, list) and len(b) == 4 and all(isinstance(v, (int, float)) for v in b)): continue - c["bbox"] = [min(w, max(0, round(b[0] * w / BBOX_GRID))), - min(h, max(0, round(b[1] * h / BBOX_GRID))), - min(w, max(0, round(b[2] * w / BBOX_GRID))), - min(h, max(0, round(b[3] * h / BBOX_GRID)))] + xs = sorted((min(w, max(0, round(b[0] * w / BBOX_GRID))), + min(w, max(0, round(b[2] * w / BBOX_GRID))))) + ys = sorted((min(h, max(0, round(b[1] * h / BBOX_GRID))), + min(h, max(0, round(b[3] * h / BBOX_GRID))))) + # ponytail: the model swapped corners on 1 of 117 boxes (p007 person_1, x1 > x2). + # Ordering here is enough because a zero-area box still crops to nothing downstream. + c["bbox"] = [xs[0], ys[0], xs[1], ys[1]] return chars @@ -1216,8 +1219,11 @@ if __name__ == "__main__": # the top sixth of the panel, inside a speech balloon, which is what identity embedded. _ch = [{"local_id": "person_5", "bbox": [222, 405, 654, 1000]}, {"local_id": "edge", "bbox": [0, 0, 1000, 1000]}, - {"local_id": "junk", "bbox": "nope"}] + {"local_id": "junk", "bbox": "nope"}, + # p007 person_1 of run B: x1 > x2, so the crop came out empty and the detection was lost + {"local_id": "swapped", "bbox": [226, 417, 130, 551]}] _bbox_to_pixels(_ch, 900, 1650) + assert _ch[3]["bbox"] == [117, 688, 203, 909], _ch[3]["bbox"] # corners ordered, non-empty crop assert _ch[0]["bbox"] == [200, 668, 589, 1650], _ch[0]["bbox"] assert _ch[1]["bbox"] == [0, 0, 900, 1650], _ch[1]["bbox"] # a clamped box spans the whole panel assert _ch[2]["bbox"] == "nope", _ch[2]["bbox"] # unparseable is left alone, not crashed -- 2.52.0 From de7e34bc0f656a67cde7ed7a42c23af952e3743d Mon Sep 17 00:00:00 2001 From: kami Date: Wed, 12 Aug 2026 23:35:15 +0400 Subject: [PATCH 24/31] File the over-merge findings: two tracklet fixes, and cosine is not identity The user checked the lead's crops. LIM SEONHO's 36 assignments cover at least six different men, a chibi and a cat, so 36 of 68 is a bug. 22 came from native resolves and 14 from reconcile merges. Two fixes are written in the orchestrator repo, tested, not deployed and not run on a GPU. `link_tracklets` caps a tracklet's panel span, because `window` bounded each pair while transitivity was unbounded and the lead's 22 native assignments came from tracklets spanning 22 and 30 panels. And one shared appearance tokenizer drops generic words, because whole chains hung on the word `short` and one pair on the word `hair`. The obvious third fix is ruled out by measurement. Over all 22 crop embeddings, the cat scores up to 0.82 against men, two different men score 0.93, and the highest pair is 0.96. No threshold separates them, so crop-to-crop cosine is not a link signal. Item 1 of the agreed plan, sending the live cast instead of a cosine top-k gallery, is not started. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 116 ++++++++++++++++++++++++------------- JOURNAL.md | 35 +++++++++++ NEXT.md | 61 ++++++++++++------- caveats/CLAUDE.md | 1 + caveats/audit-open.md | 25 ++++++++ decisions/CLAUDE.md | 2 + decisions/identity-bbox.md | 59 +++++++++++++++++++ 7 files changed, 237 insertions(+), 62 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index 36be549..b6db079 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -5,13 +5,16 @@ Live state is in `NEXT.md`. This file is only what this session did. The previou ## Asked -The two files, then "go" on the third GPU cycle. +The two context files, then "go" on the third GPU cycle, then "commit", then "what's next". Then the user +checked the lead's crops in the review UI and said the assignments were wrong. Then "test on one of the +panels first". Then "mind presenting the panels with boxes". Then "so, we didn't fix it properly?". Then +"go with the 1". ## Result -One worker fix, one measurement that killed the plan's first item, one GPU cycle. The cycle proved the new -fix and the fifth session's merge fix. Coverage rose for the first time, 50% to 57%. It is now explained -entirely by the `has_face` gate. +One worker fix proven on a GPU. One measurement that killed the plan's item 1. One GPU cycle. Then the user +found the registry is over-merged, which invalidates the cycle's headline numbers, and two more fixes were +written for it. The third fix the user chose, item 1 of the three options, is NOT started. | metric | 17:38 run | 18:07 run | | --- | --- | --- | @@ -24,65 +27,98 @@ entirely by the `has_face` gate. | chains deeper than one hop | 1 | 0 | | `merged_from` stamps | 9 | 22 | -## The corner-ordering fix +Cycle timings, 18:07-18:13 UTC: vision 116/116 in 3m59s, identity 116/116 in 1m24s, reconcile 18/18 in 50s. -`_bbox_to_pixels` sorts each coordinate pair after clamping. Three lines -(`decisions/identity-bbox.md#bbox-corners-ordered`). The self-check feeds it run B's real -`[226, 417, 130, 551]` and asserts `[117, 688, 203, 909]`. Unsorted the call returns `[203, 688, 117, 909]`, -so the assert fails without the fix. +## Committed and proven on a GPU -The run returned 0 degenerate boxes over 119 detections. The caveat is deleted and the decision is indexed. +`_bbox_to_pixels` sorts each coordinate pair after clamping +(`decisions/identity-bbox.md#bbox-corners-ordered`). 0 degenerate boxes over 119 detections, against 1 in +117. Commit `54bd126` on `restore-runtime`. Its caveat is deleted. -## The measurement that killed item 1 +## Measured, no code -The plan said to score precision with `eval_identity.py` against the 145 rows of `identity_labels`. Every -one of those rows belongs to chapter `8ca8249b`, a different manga with cast "Rico" and "Ikekin", spread -over 81 panels. Chapter `7c944dd4` has zero labels, so -`/review/identity?job_id=778297bc...` returns `labeled: 0, correct: 0, accuracy: null`. +**The 145 ground-truth labels are for a different manga.** Every row in `identity_labels` keys to chapter +`8ca8249b`, cast "Rico" and "Ikekin", 81 panels. Chapter `7c944dd4` has zero, so +`/review/identity?job_id=778297bc...` returns `labeled: 0, accuracy: null`. Scoring `8ca8249b` gives 7/138 +on a run with 44 assignments over 246 panels, predating every fix. -Scoring `8ca8249b` anyway gives 7/138, with 113 rows labelled as a real person and left unassigned. That -chapter's identity run is stale: 44 assignments over 246 panels, predating every fix. The number measures -old code on the wrong chapter. +**Coverage is the `has_face` gate and nothing else.** All 68 assignments landed on face-bearing detections +and none on a gated one. Recall among face-bearing detections is 96%, up from 82%. -So the eval path is proven end to end and the ground truth is absent. A precision number needs a hand pass -over `7c944dd4` through `POST /review/identity/label`, keyed by bbox rather than `local_id`. +**Embedding cosine cannot separate people.** All 22 crop embeddings for the lead, pulled from +`manga//characters/_crops/*.npy`, 1152 dims, L2-normalised. The cat scores up to 0.82 against +men, two different men score 0.93, the highest pair in the matrix is 0.96 +(`caveats/audit-open.md#cosine-not-identity`). This rules out a crop-to-crop cosine link, which was the +fix proposed one message before the measurement. -## What the cycle settled +## Written, tested, NOT deployed and NOT run on a GPU -**Coverage is the gate and nothing else.** All 68 assignments landed on face-bearing detections and none on -a gated one. Recall among face-bearing detections is 96%, up from 82%. The resolver is not losing cast, so -the suspicion in `caveats/audit-open.md#coverage-trend` is closed. +Both in `/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`, uncommitted there: -**The merge-chain fix holds under load.** 9 merges this run against 7 last run, and 22 `merged_from` -stamps. No chain is deeper than one hop. +- `tracklets.py`: `link_tracklets` rejects a merge whose group would span more than `window` panels + (`decisions/identity-bbox.md#tracklet-span-cap`). `window` bounded each pair, transitivity was unbounded, + and the lead's 22 native assignments came from 3 tracklets spanning 0, 22 and 30 panels. +- `tracklets.py` + `service.py`: one `appearance_tokens` with a `GENERIC` stopword set, and + `service._appearance_tokens` delegates to it, so reconcile's pre-filter is fixed too + (`decisions/identity-bbox.md#generic-tokens`). Whole chains hung on the word `short`, one pair on the + word `hair`. -## What the cycle opened +On the same 22 crops, candidate overlap forced to pass: 3 tracklets at worst span 30 becomes 9 at worst +span 8. -**The lead may be absorbing.** 36 of 68 assignments, 53%, against 25 of 59 before. `audit_registry.py` -flags it. A protagonist in half the panels looks identical to an over-merge without labels. +## Not started -**Panel 7 got worse.** 5 of its 6 detections carry `has_face = False`, and `person_6`, the one that does, -went unassigned. The previous run bound its colleague. That is the cost of requiring a face, `NEXT.md` -item 4. +**Item 1, which the user chose: stop letting cosine pick the gallery.** `run_stage_identity` builds +`union_cands` from the members' cosine top-k. So a metric that cannot separate people decides who gemma is +even allowed to consider. There are 9 live characters. Send the live cast instead, gender-gated, capped and +logged when truncated. Two traps found while reading it: -## Not done +- `/vision/resolve` sends up to 3 reference images per candidate (`worker_vision.py:1057`), so 9 candidates + is 27 images plus the query. It needs a cap. +- only a crop with a non-empty cosine shortlist enters `shortlists` at all, via the + `if s.get("candidates")` guard in `service.py`. An empty top-k drops the crop from resolution entirely. -- Nothing downstream re-ran. The job is still parked at `dialogue waiting` with every stage below cleared. -- Nothing is committed. All edits are in the working tree. +Also open: nothing downstream re-ran, the job is still parked at `dialogue waiting`, and vision boxes cats +as people and dresses them (`p081`, `p108`). ## Checks ```bash .venv/bin/python worker_vision.py # self-check ok, including the swapped-corner assert -./check_stale.sh # exit 0 before the run and after it +./check_stale.sh # exit 0 before the cycle and after it +/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 tracklets.py && python3 -m pytest -q --ignore=test_api.py" # self-check ok, 118 passed ``` -## Traps confirmed +Every new assert was confirmed to fail with its fix disabled: the span cap returns `[[0, 1, 2]]`, and the +generic-word pair links with `GENERIC` emptied. + +## Next command + +Deploy the two orchestrator fixes and run the fourth cycle. + +```bash +cd /home/kami/Programs/n8n-worker && ./check_stale.sh # must exit 0 +/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && docker compose up -d --build orchestrator" +J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" +# the reset returns restart_identity_worker: true -- honour it, see the traps +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"vision\"}'" +for S in vision identity reconcile; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 5400 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done +/usr/bin/ssh kami@192.168.1.104 "docker logs manga-orchestrator --since 1h 2>&1 | grep tracklet" # expect ~30 tracklets, was 12 +/usr/bin/ssh kami@192.168.1.104 "docker exec manga-orchestrator python3 /app/audit_registry.py 7c944dd4-e972-42c7-ba60-9f6939548e80 6" +``` + +Then re-check the lead's crops by eye. A stage counter will not show this defect. + +## Traps confirmed or found - Restart a worker with `tmux send-keys -t manga-workers: C-c`, then re-send the launch line from - `start_workers.sh` with the `MIOPEN_ENV` prefix. `pgrep`-based kills match the calling shell. + `start_workers.sh` with its `MIOPEN_ENV` prefix. A `pgrep` kill matches the calling shell. - `POST /characters/reset` returns `restart_identity_worker: true`. Honour it, the worker caches the registry. - `identity_assignments` has no `method` column. The `merged_from:` stamps live in - `identity_assignment_sources`. -- `chapters` has no `title` column. + `identity_assignment_sources`. `chapters` has no `title` column. The orchestrator image has no numpy. +- A heredoc piped into `docker exec` over `/usr/bin/ssh` silently produces no output. Write the script to a + file, `scp` it, `docker cp` it, then run it. +- Mixing `echo` with a `tar cf -` stream over ssh corrupts the archive. Separate the calls. +- `panel_order` and the panel filename differ by one: `panel_order` 10 is `p009.png`. diff --git a/JOURNAL.md b/JOURNAL.md index 857869e..0b26de1 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -644,3 +644,38 @@ labels, which is item 1. Panel 7 got worse, not better: 5 of its 6 detections carry `has_face = False`, and `person_6`, the one that does, went unassigned. The previous run bound its colleague. That is the cost of requiring a face, item 4. + +## 2026-08-12, sixth session, later — the lead was over-merged, and cosine cannot fix it + +The user checked the lead's crops in the review UI. `LIM SEONHO`'s 36 assignments cover at least six +different men, a chibi, and a cat. So 36 of 68 is a bug, not a protagonist. + +Split by mechanism: 22 native resolves, 14 from reconcile merges. Both are broken, for different reasons. + +**The tracklet linker chained across the chapter.** `window=8` bounds each pair, transitivity was +unbounded. The 22 native assignments came from 3 tracklets spanning 0, 22 and 30 panels, visible as two +confidence blocks of eleven rows at 0.95 and ten at 1.0. Fixed by capping the merged group's span +(`decisions/identity-bbox.md#tracklet-span-cap`). + +**Both tokenizers linked on words that are not identity.** Whole chains hung on `short`; one pair linked on +the word `hair` itself. There is now one tokenizer with a `GENERIC` stopword set, shared with reconcile's +pre-filter (`decisions/identity-bbox.md#generic-tokens`). + +Measured on the same 22 real crops, candidate overlap forced to pass: 3 tracklets at worst span 30 becomes +9 at worst span 8. + +**Then the obvious next fix was ruled out by measurement.** Pulled all 22 crop embeddings from MinIO and +computed the pairwise cosine. The cat scores up to 0.82 against men, two different men score 0.93, and the +highest pair in the matrix is 0.96. No threshold exists, so crop-to-crop cosine is not a link signal, and +the `candidates` overlap condition is close to noise. Filed as +`caveats/audit-open.md#cosine-not-identity`. Inference: identity embeds the whole person box, so cosine +measures scene and style, not face. + +Also seen: vision boxes cats as people and dresses them. `p081` and `p108` are cats, described +`short brown / yellow plaid shirt` and `short brown / white t-shirt`. + +Checks: `tracklets.py` self-check ok, orchestrator 118 passed. Both new asserts confirmed to fail with +their fix disabled. Nothing deployed, nothing re-run on a GPU. + +Artefact: `lead_tracklets.png`, the 22 panels with boxes, insets and tracklet grouping. Session scratchpad +only, not committed. diff --git a/NEXT.md b/NEXT.md index ccfb237..fab707c 100644 --- a/NEXT.md +++ b/NEXT.md @@ -13,38 +13,55 @@ Three GPU cycles ran on 2026-08-12, the last 18:07-18:13 UTC on a reset registry parked at `dialogue waiting`. Everything from `dialogue` down is cleared and stale. All 9 workers are current. -Eight changes are now proven on real panels. The fourth session contributed four. The fifth added the +Eight changes are proven on real panels. The fourth session contributed four. The fifth added the roster-hint removal, merge-chain flattening and `name_claims` reset. This session added bbox corner -ordering. The registry after the 18:07 run holds 18 rows, 9 live and 9 carrying `merged_into`: +ordering. -- `LIM SEONHO`, 36 assignments, the lead, one row, named from the p010 caption. -- `character_b112d4`, 23 assignments, the woman, correct and unnamed. -- `Seonho`, still the wrong man, still named off in-panel text. +**The registry is not trustworthy.** `LIM SEONHO`'s 36 assignments cover at least six different men, a +chibi and a cat, confirmed by eye in the review UI. 22 of those are native resolves and 14 came from +reconcile merges, so both mechanisms are wrong. -Coverage is 57%, from 50%, 61% and 70% before it, and it is now the `has_face` gate and nothing else. All 68 -assignments landed on face-bearing detections and none on a gated one. Recall among face-bearing detections -is 68 of 71, or 96%, up from 82%. Degenerate boxes went from 1 in 117 to 0 in 119, and the merge -chain is flat with 22 assignments stamped `merged_from`. +Two fixes are written and tested but NOT deployed and NOT run on a GPU, both in the orchestrator repo: -The open question moved. The lead holds 36 of 68 assignments, 53%, which `audit_registry.py` flags as -absorbing. Nothing separates a protagonist in half the panels from an over-merge -(`caveats/audit-open.md#coverage-trend`). +- `link_tracklets` caps a tracklet's panel span (`decisions/identity-bbox.md#tracklet-span-cap`) +- one shared appearance tokenizer drops generic words (`decisions/identity-bbox.md#generic-tokens`) + +On the same 22 crops, 3 tracklets at worst span 30 becomes 9 at worst span 8. + +Coverage was 57% on the 18:07 run, and it is the `has_face` gate and nothing else. All 68 assignments +landed on face-bearing detections, so recall among them is 68 of 71. Degenerate boxes are 0 in 119 and the +merge chain is flat with 22 `merged_from` stamps. ## Next -1. **Decide whether the lead absorbing 53% of assignments is real.** It needs labels this chapter lacks. - The resolver is cleared: recall among face-bearing detections is 96%. What is unmeasured is precision. - The lead went from 25 of 59 to 36 of 68 as the merge count rose from 7 to 9. +1. **Deploy the two tracklet fixes and run the fourth cycle.** Neither has touched a GPU. Rebuild the + orchestrator image, reset the registry, run vision/identity/reconcile, then re-check the lead's crops. + Expect roughly 30 tracklets over 64 crops instead of 12, so identity goes from about 1m25s to 3 or 4 + minutes. Watch the lead's assignment count against 36. - The previous plan pointed at `identity_labels` and its 145 rows. Measured this session: every one of - those rows belongs to chapter `8ca8249b`, a different manga. Chapter `7c944dd4` has zero labels, so - `/review/identity` reports `labeled: 0, accuracy: null`. Scoring `8ca8249b` gives 7/138 on an identity - run that predates all eight fixes. The eval plumbing works and the ground truth is missing. - Labelling `7c944dd4` by hand is the only path to a precision number - (`caveats/audit-open.md#coverage-trend`). + Neither fix is sufficient. Bare hair colour still links different men, and the cat still joins its + neighbours. Do not add a crop-to-crop cosine to close that: measured on this run's 22 embeddings, + different people reach 0.93 while the same person reaches 0.96, so no threshold exists + (`caveats/audit-open.md#cosine-not-identity`). + + Agreed next step after the cycle, chosen by the user and not started: **stop letting cosine pick the + gallery.** There are 9 live characters. `run_stage_identity` builds `union_cands` from the members' + cosine top-k shortlists, so a metric that cannot separate people decides who is even considered. Send + the live cast instead, gender-gated, capped and logged when truncated. Note two traps found while + reading it: `/vision/resolve` sends up to 3 reference images per candidate + (`worker_vision.py:1057`), so 9 candidates is 27 images plus the query and needs a cap; and only a crop + with a non-empty cosine shortlist enters `shortlists` at all, so an empty top-k currently drops the crop + from resolution entirely. + + Then, separately, test embedding the FACE box rather than the person box. `face_detect` already finds + the face and pairs it for `has_face`. That is the likely root cause of cosine measuring scene instead of + person, and the test is to re-embed these same 22 detections and recompute the matrix. 2. ~~**Order the corners in `_bbox_to_pixels`.**~~ **Done 2026-08-12, run and verified on a GPU.** 0 degenerate boxes over 119 detections (`decisions/identity-bbox.md#bbox-corners-ordered`). -3. **Detection still names from in-panel text and can hit the wrong body.** p011 and p026 emit +3. **Vision boxes animals as people and dresses them.** `p081` and `p108` are cats, described + `short brown / yellow plaid shirt` and `short brown / white t-shirt`. A detection prompt problem, not a + linker one, and it feeds every stage below. +4. **Detection still names from in-panel text and can hit the wrong body.** p011 and p026 emit `name: "Seonho"` on `person_2` with the roster hint gone. That is the residue of the naming defect and the reason the glasses man is named at all (`decisions/identity-bbox.md#roster-does-not-name`, "not covered"). diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index be09a22..d7fbb8e 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -43,3 +43,4 @@ a complaint, so give it one or drop it. | [Vision does not separate a background extra from cast](speaker-attribution.md#extras-as-cast) | 2026-08-12 panel 7 | | [Cast reference profiles are enrolled from wrong crops](speaker-attribution.md#poisoned-reference-set) | 2026-08-12 panel 7 | | [Identity coverage has fallen on every run since the gate landed](audit-open.md#coverage-trend) | before the next downstream run | +| [Embedding cosine on a person crop cannot tell two people apart](audit-open.md#cosine-not-identity) | before building on cosine | diff --git a/caveats/audit-open.md b/caveats/audit-open.md index 99c28c8..7777d8b 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -212,3 +212,28 @@ The plumbing works, the labels are missing. Precision and recall on the current pass over `7c944dd4` first, via `POST /review/identity/label` per detection. Label against the bbox, not the `local_id`. Vision is non-deterministic and reassigns `local_id` top-to-bottom every run. A label taken before a vision rerun then points at whoever now occupies that slot. + +## Embedding cosine on a person crop cannot tell two people apart {#cosine-not-identity} + +Measured 2026-08-12 on the 18:07 run's 22 crops for `character_2367d70c`, the embeddings pulled from +`manga//characters/_crops/*.npy` and L2-normalised. 1152 dimensions. + +| pair | cosine | +| --- | --- | +| the cat at `p098` against any man | up to 0.82 | +| the red-robe chibi at `p088` against any man | up to 0.75 | +| two visibly different men (`order 60~80`, `88~110`) | 0.93, 0.92 | +| the highest pair in the matrix (`47~51`, one man, one shirt) | 0.96 | + +Different people reach 0.93 and the same person reaches 0.96. No threshold separates them. So a +crop-to-crop cosine link is not available, and the `candidates` overlap condition in `link_tracklets` is +close to noise for the same reason: the top-k is chosen by this metric, so every crop shortlists the same +few rows. + +Inference, not yet tested: identity embeds the whole person box, which holds background, clothing and pose. +Those change between scenes while every crop shares one art style, so cosine measures "manga crop of a +person in an office". `face_detect` already finds the face and pairs it to the person box for `has_face`, +so embedding the face region instead is a small change. + +Revisit trigger: before building anything else on cosine. The test is to crop the faces of these same 22 +detections, embed them, and recompute this matrix. If a threshold appears, embed faces. diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 98943df..73278ab 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -48,3 +48,5 @@ still live belongs in `caveats/`. | [A roster name is a guess, so it never reaches detection](identity-bbox.md#roster-does-not-name) | closed | | [`merged_into` is exactly one hop deep](identity-bbox.md#merge-chains-flatten) | closed | | [`_bbox_to_pixels` orders the corners, because the model sometimes swaps them](identity-bbox.md#bbox-corners-ordered) | closed | +| [A tracklet is bounded by span, not only by pairwise distance](identity-bbox.md#tracklet-span-cap) | closed, GPU pending | +| [A generic word is not identity evidence, and one tokenizer serves both consumers](identity-bbox.md#generic-tokens) | closed, GPU pending | diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md index d5fc2f0..77b4d73 100644 --- a/decisions/identity-bbox.md +++ b/decisions/identity-bbox.md @@ -243,3 +243,62 @@ does not already enforce. The self-check feeds the real swapped box in and asser which fails without the sort. Proven on the 18:07 run: 0 degenerate boxes over 119 detections. + +## A generic word is not identity evidence, and one tokenizer serves both consumers {#generic-tokens} + +**Closed, 2026-08-12, not yet run on a GPU.** + +The tracklet linker and reconcile's pair pre-filter each carried their own copy of the appearance +tokenizer, and both linked on words that describe a person without distinguishing them. Measured over the +18:07 run's 22 crops for the lead: + +``` +p056~p057 hair=['short'] cloth=- two different men +p047~p054 hair=['hair'] cloth=- "brown hair" vs "dark hair", the field name in its own value +p109~p110 hair=- cloth=['shirt','white'] +``` + +`tracklets.appearance_tokens` is now the single implementation and subtracts a `GENERIC` set: length words +(`short`, `long`, `medium`, `shoulder`, `length`), `hair`/`haired`, garment words (`shirt`, `top`, +`jacket`, `coat`, `sleeve`), and filler (`plain`, `casual`, `none`, `unknown`). `service._appearance_tokens` +delegates to it, so reconcile's pre-filter is fixed by the same change. That pre-filter is what let the +pink tank top reach `/vision/same` at all. + +Four asserts cover it and all four fail with `GENERIC` emptied. + +Measured effect on those 22 crops, with candidate overlap forced to pass (the shortlists are not stored, +so this is the most permissive assumption and the real split can only be finer): + +| | tracklets | worst span | +| --- | --- | --- | +| as it ran | 3 | 30 panels | +| span cap only | 8 | 8 | +| span cap + `GENERIC` | 9 | 8 | + +The stopwords alone split out `p054`, the beige-jacket man, and `p089`, the red-robe chibi. + +**Not fixed by this.** Bare hair colour still links different men: `order 56` through `62` stay in one +five-crop tracklet on `brown`, and the cat at `p098` still joins its neighbours the same way. The obvious +next lever, a crop-to-crop cosine, is ruled out by `caveats/audit-open.md#cosine-not-identity`. + +## A tracklet is bounded by span, not only by pairwise distance {#tracklet-span-cap} + +**Closed, 2026-08-12, not yet run on a GPU.** + +`window=8` bounded each PAIR, and linking is transitive, so nothing bounded the group. The 18:07 run +resolved 12 tracklets over 64 crops, and the lead's 22 native assignments fell into two of them spanning +panels 47-69 and 80-110. One gemma answer then covered a grey blazer, a denim jacket, a red robe and a cat. +The two blocks are visible in the stored confidences: eleven rows at 0.95 and ten at 1.0. + +`link_tracklets` now rejects a merge whose resulting group would span more than `window` panels: + +```python +orders = [dets[m].get("panel_order", m) for m in gi + gj] +if max(orders) - min(orders) > window: + continue +``` + +This makes the module docstring's claim true. The self-check links three compatible detections at panels 1, +9 and 17 and asserts they do not land in one tracklet; it returns `[[0, 1, 2]]` with the cap disabled. + +Cost: 3 resolve calls become 8 for this character, so identity should go from about 1m25s to 3 or 4 minutes. -- 2.52.0 From a386e9d910b09d0f014a1c79a15407b7c50a7403 Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 00:15:52 +0400 Subject: [PATCH 25/31] Send the live cast to the resolver, not cosine's top-k The gallery gemma chose from was the union of the tracklet members' cosine shortlists. Cosine cannot separate people on this data, so a metric that cannot tell two men apart decided who gemma was allowed to consider, and the right character was often not on the list. The gallery is now the live cast: gender-compatible rows from get_known_characters, named first, capped, re-read per tracklet so a minted character is visible to later ones. Every crop reaches the resolver now, including one whose cosine top-k was empty; those used to be dropped. worker_vision spreads reference images across a budget instead of sending 3 per candidate, so a 9-character cast costs 9 images and not 27. Ran on a GPU, 19:44-19:52 UTC. The lead's assignments drop from 36 to 16 and 14 of the 16 are him; the other two are art inside a panel. Co-Authored-By: Claude Opus 5 --- JOURNAL.md | 44 ++++++++++++++++++++++++++++++++++++++ NEXT.md | 35 +++++++++++++++++------------- caveats/audit-open.md | 18 ++++++++++++++++ decisions/identity-bbox.md | 31 +++++++++++++++++++++++++++ worker_vision.py | 22 +++++++++++++++++-- 5 files changed, 133 insertions(+), 17 deletions(-) diff --git a/JOURNAL.md b/JOURNAL.md index 0b26de1..c23cf3a 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -679,3 +679,47 @@ their fix disabled. Nothing deployed, nothing re-run on a GPU. Artefact: `lead_tracklets.png`, the 22 panels with boxes, insets and tracklet grouping. Session scratchpad only, not committed. + +## 2026-08-12, 19:44-19:52 UTC — fourth GPU cycle: the cast is the gallery + +Three fixes ran together for the first time: the tracklet span cap and the shared `GENERIC` tokenizer, +both written last session and never deployed, plus the new one +(`decisions/identity-bbox.md#cast-is-the-gallery`). Job `778297bc`, chapter `7c944dd4`, registry reset, +vision cleared, `vision -> identity -> reconcile`. Vision 116/116 in 3m55s, identity 116/116 in 2m44s, +reconcile 20/20 in 44s. + +| metric | 18:07 run | 19:44 run | +| --- | --- | --- | +| detections | 119 | 119 | +| assignments | 68 | 60 | +| coverage | 57% | 50% | +| tracklets over crops | 12 / 64 | 33 / 72 | +| lead's assignments | 36 | 16 | +| top character's share | -- | 16/60 = 27% | +| characters after reconcile | 18 | 14 | +| minted / cleared | -- | 10 / 12 | + +**Coverage went down and that is the fix working.** gemma cleared 12 crops it used to be forced to name +from a cosine top-k that did not contain the right person. 72 crops entered resolution against 64, because +a crop with an empty cosine shortlist is no longer dropped. + +**Checked by eye, which is the only check that sees this.** Contact sheets of every assigned crop, per +character, confirmed by the user. The lead holds 16 crops. 14 are him and 2 are art inside a panel, the +photograph at `order 17` and the chibi at `order 20`. `character_2b1b12a1` holds 13, all of them her, and +she is a main character the registry never named. `character_f0d4e901` holds 9, of which 7 are her and 2 +are `2b1b12a1` (`order 31`, `order 33`). Against the 18:07 run, where the lead's 36 covered six different +men, a chibi and a cat. + +Every wrong crop on the lead is one defect: vision treats art inside a panel as the scene. It is not a +linker failure and no identity change will fix it. + +**The cap fired twice and dropped the wrong rows.** `GALLERY_CAP = 10` truncated a 16-row gender-compatible +cast at `p097` and an 11-row one at `p109`. `cast_gallery` orders named first, so what it drops is exactly +the recently minted anonymous rows, which is the population a later tracklet most needs to match against. +Filed as `caveats/audit-open.md#gallery-cap-drops-the-unnamed`. + +Checks: `worker_vision.py` self-check ok, `tracklets.py` self-check ok, orchestrator 118 passed, +`./check_stale.sh` exit 0 before the cycle. Vision and identity workers both restarted, the second because +`/characters/reset` asked for it. + +Artefacts: `sheet_*.png`, one contact sheet per character. Session scratchpad only, not committed. diff --git a/NEXT.md b/NEXT.md index fab707c..05c198d 100644 --- a/NEXT.md +++ b/NEXT.md @@ -9,28 +9,33 @@ The chapter runs end to end. The A/V sync defect is fixed and `chapter.mp4` is r against audio 364.122s at `25/1`. Job `778297bc-e7ce-439d-91b5-8a027060d17f`, chapter `7c944dd4-e972-42c7-ba60-9f6939548e80`, 116 panels. -Three GPU cycles ran on 2026-08-12, the last 18:07-18:13 UTC on a reset registry. The job is `running` and +Four GPU cycles ran on 2026-08-12, the last 19:44-19:52 UTC on a reset registry. The job is `running` and parked at `dialogue waiting`. Everything from `dialogue` down is cleared and stale. All 9 workers are current. -Eight changes are proven on real panels. The fourth session contributed four. The fifth added the -roster-hint removal, merge-chain flattening and `name_claims` reset. This session added bbox corner -ordering. +Eleven changes are proven on real panels. This session added bbox corner ordering. It then deployed and ran +the tracklet span cap, the shared `GENERIC` tokenizer and the cast gallery together. -**The registry is not trustworthy.** `LIM SEONHO`'s 36 assignments cover at least six different men, a -chibi and a cat, confirmed by eye in the review UI. 22 of those are native resolves and 14 came from -reconcile merges, so both mechanisms are wrong. +**The registry is now roughly right and is worth reading.** Checked by eye, crop by crop, confirmed by the +user. The lead holds 16 assignments. 14 are him and 2 are art inside a panel, the photograph at `order 17` +and the chibi at `order 20`. `character_2b1b12a1` holds 13, all of them her, and she is a main character +the registry never named. `character_f0d4e901` holds 9, of which 7 are her and 2 are `2b1b12a1`. On the +18:07 run the lead alone held 36, covering six men, a chibi and a cat. -Two fixes are written and tested but NOT deployed and NOT run on a GPU, both in the orchestrator repo: +So every wrong crop on the lead is one defect, item 7, and not a linker failure. The registry's other +weakness is that its biggest character has no name. -- `link_tracklets` caps a tracklet's panel span (`decisions/identity-bbox.md#tracklet-span-cap`) -- one shared appearance tokenizer drops generic words (`decisions/identity-bbox.md#generic-tokens`) +| metric | 18:07 run | 19:44 run | +| --- | --- | --- | +| detections | 119 | 119 | +| assignments | 60 | 60 | +| coverage | 57% | 50% | +| tracklets over crops | 12 / 64 | 33 / 72 | +| lead's assignments | 36 | 16 | +| characters after reconcile | 18 | 14 | -On the same 22 crops, 3 tracklets at worst span 30 becomes 9 at worst span 8. - -Coverage was 57% on the 18:07 run, and it is the `has_face` gate and nothing else. All 68 assignments -landed on face-bearing detections, so recall among them is 68 of 71. Degenerate boxes are 0 in 119 and the -merge chain is flat with 22 `merged_from` stamps. +Coverage fell because gemma now clears 12 crops instead of naming them from a shortlist that did not +contain the right person. Coverage is still the `has_face` gate plus those refusals, and nothing else. ## Next diff --git a/caveats/audit-open.md b/caveats/audit-open.md index 7777d8b..3fcc889 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -237,3 +237,21 @@ so embedding the face region instead is a small change. Revisit trigger: before building anything else on cosine. The test is to crop the faces of these same 22 detections, embed them, and recompute this matrix. If a threshold appears, embed faces. + +## The gallery cap drops exactly the rows a tracklet needs {#gallery-cap-drops-the-unnamed} + +**Open. Observed on the 19:44 run of 2026-08-12.** + +`cast_gallery` orders the live cast named-first, then registry order, and `run_stage_identity` truncates +the tail at `GALLERY_CAP = 10`. Registry order is creation order, so the tail is the anonymous characters +this very run minted — and a later tracklet of a recurring unnamed person is precisely what needs to match +one of those. The chapter has two named rows, so on this run the cap dropped 6 unnamed rows at `p097` and +1 at `p109`, and a tracklet that should have joined one of them can only mint a duplicate instead. + +It is capped because every candidate costs at least one reference image in the resolve prompt beside the +query crop. The cap is a VRAM and context budget, not a modelling choice. + +**Revisit when** the cast on a chapter routinely exceeds 10 gender-compatible rows, which it already did +here. The fix is to order the gallery by how many assignments each character already holds in this chapter, +so the tail is the rows nobody has matched rather than the rows nobody has named yet. That needs one count +query per tracklet. diff --git a/decisions/identity-bbox.md b/decisions/identity-bbox.md index 77b4d73..4af2d4d 100644 --- a/decisions/identity-bbox.md +++ b/decisions/identity-bbox.md @@ -302,3 +302,34 @@ This makes the module docstring's claim true. The self-check links three compati 9 and 17 and asserts they do not land in one tracklet; it returns `[[0, 1, 2]]` with the cap disabled. Cost: 3 resolve calls become 8 for this character, so identity should go from about 1m25s to 3 or 4 minutes. + +## The gallery is the live cast, not cosine's top-k {#cast-is-the-gallery} + +**Closed, 2026-08-12, not yet run on a GPU.** + +`run_stage_identity` built each tracklet's gallery by unioning the members' cosine shortlists, so the +metric that `caveats/audit-open.md#cosine-not-identity` shows cannot separate people decided who gemma was +even allowed to consider. Two different men reach 0.93 on this chapter's crops where the same man reaches +0.96. When the right character fell outside every member's top-5, gemma could only pick a wrong one or +answer NONE, and NONE mints a duplicate. + +The cast is small. `tracklets.cast_gallery` builds the gallery from `get_known_characters` instead: +gender-compatible rows only, named first, then registry order. Cosine still shortlists per crop, and those +shortlists are still the linker's candidate-overlap evidence, but they no longer bound the answer. + +Three consequences, each deliberate: + +- **Every crop now reaches the resolver.** A crop only entered `shortlists` when its cosine top-k was + non-empty, so an early crop seen while the roster was still empty, or one with no gender-compatible row, + was dropped from resolution and could never mint. The guard is gone. +- **The gallery is re-read per tracklet**, so a character an earlier tracklet minted is visible to the + later ones. That closes the `ponytail:` note beside the mint path. +- **Two caps, because the prompt carries images.** `GALLERY_CAP = 10` in `service.py` bounds the + candidates and logs the ids it drops. `worker_vision.REF_BUDGET = 12` spreads reference images across + them, `max(1, min(3, 12 // n))` apiece, so a 9-character cast sends 9 images plus the query instead of + the 27 plus query that 3-apiece would have sent. + +Cost: more gemma calls, because the crops that used to be dropped now each get one. + +**Not fixed by this.** The gallery being right does not make the crop legible. A back-turned or tiny crop +still has no face for gemma to judge, which is `NEXT.md` item 4. diff --git a/worker_vision.py b/worker_vision.py index 3bbfb4e..fb8efec 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -1031,9 +1031,16 @@ def build_resolve_prompt(candidates: list) -> str: ) +REF_BUDGET = 12 # total reference images one /vision/resolve prompt may carry, beside the query crop + + +def _refs_per_candidate(n: int) -> int: + return max(1, min(3, REF_BUDGET // max(1, n))) + + class ResolveInput(BaseModel): crop_uri: str - candidates: list = [] # gender-gated cosine shortlist [{character_id, name, gender, ...}] + candidates: list = [] # gender-gated gallery [{character_id, name, gender, ...}] session_id: str = "" @@ -1052,9 +1059,13 @@ async def vision_resolve(data: ResolveInput): return {"character_id": None, "confidence": 0.0, "reason": "no_candidates"} crop = transport.get(data.crop_uri, f"{SHM}/resolve_{uuid.uuid4().hex[:8]}.png") refs = [] + # image budget, not a per-candidate rule. The gallery is the live cast now, not a cosine top-k, so + # 3 references each was 27 images plus the query on a 9-character cast. Spread REF_BUDGET across the + # candidates instead: 3 references while the cast is small, 1 apiece once it is not. + per = _refs_per_candidate(len(data.candidates)) try: for i, candidate in enumerate(data.candidates, 1): - for uri in (candidate.get("reference_image_uris") or [])[:3]: + for uri in (candidate.get("reference_image_uris") or [])[:per]: try: refs.append((i, transport.get(uri, f"{SHM}/ref_{uuid.uuid4().hex[:8]}.png"))) except Exception as e: @@ -1189,6 +1200,13 @@ if __name__ == "__main__": _map = lambda ch: (cands[ch - 1]["character_id"] if isinstance(ch, int) and 1 <= ch <= len(cands) else None) assert _map(1) == "c1" and _map(2) == "c2" and _map(0) is None and _map(9) is None assert "reference images" in rp and "face shape/features first" in rp + # reference budget: a small gallery keeps 3 refs each, a cast-sized one drops to 1 and stays under + # REF_BUDGET images however big the cast gets. + assert _refs_per_candidate(1) == 3 and _refs_per_candidate(4) == 3 + assert _refs_per_candidate(9) == 1 and _refs_per_candidate(0) == 3 + # the budget holds up to REF_BUDGET candidates; past that the floor of 1 apiece wins, which is why + # the orchestrator caps the gallery (GALLERY_CAP) rather than relying on this alone. + assert all(n * _refs_per_candidate(n) <= REF_BUDGET for n in range(1, REF_BUDGET + 1)) # dialogue parsing is fail-loud; an omitted requested panel is partial, never silent-empty. bad = _dialogue_envelope(["p1"], [], parse_failed=True) -- 2.52.0 From 97cb4831f9328d76b7b2c2bd84d237a78c41e369 Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 00:39:57 +0400 Subject: [PATCH 26/31] Wire the caption merge, and write the target architecture down merge_faceless_captions had been written and never called; both crop endpoints called the non-destructive context_fragment_links instead, with no decision recording that choice. Wiring it changes panel count and every panel index, so the chapter needs a re-crop with the panels prefix cleared first -- crop_webtoon skips an upload when the key already exists, which is right for a resume and silently wrong after a slicing change. Noted at the line. It does not cover the head-in-one-shot body-in-the-next split that prompted the question. _merge_plan only folds a fragment that has text and no face. ARCHITECTURE.md is the target shape from the user's design, with what exists against each section today. Nothing in it is built. Co-Authored-By: Claude Opus 5 --- ARCHITECTURE.md | 368 +++++++++++++++++++++++++++++++++++ CLAUDE.md | 1 + JOURNAL.md | 33 ++++ NEXT.md | 48 +++-- decisions/CLAUDE.md | 8 +- decisions/identity-naming.md | 51 +++++ worker_crop.py | 7 +- 7 files changed, 495 insertions(+), 21 deletions(-) create mode 100644 ARCHITECTURE.md create mode 100644 decisions/identity-naming.md diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 0000000..2b9adad --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,368 @@ +# ARCHITECTURE + +The target shape of the pipeline, written 2026-08-13 from the user's design. This is **not** what the code +does. `NEXT.md` holds the live state and `AUDIT.md` holds the current pipeline. Every section here ends +with what exists today, so the gap is legible without reading both. + +The governing principle: + +> Do not make the next panel understand the previous panel. Make it understand the current world state +> produced by all previous panels. + +Vision produces observations. A persistent chapter graph owns identity and relationships. Everything below +follows from that split. + +## 1. The page is a region graph, not a list of panels + +``` +page + ├─ regions + │ ├─ panel + │ ├─ inset_panel + │ ├─ embedded_art + │ ├─ text + │ ├─ tail + │ └─ character_occurrence + │ + └─ edges + ├─ contains(region, region) + ├─ reads_before(text, text) + ├─ tail_of(tail, text) + ├─ points_to(tail, character) + ├─ spoken_by(text, character) + └─ same_identity(character, character) +``` + +A flat set of panels cannot express a television inside a room. That is the defect the current pipeline +shows most often. + +**Today:** the crop stage emits a flat panel list with a bbox each, plus `context_fragments`, a +non-destructive caption-to-face link. Vision emits per-panel characters and dialogue. There is no +containment edge, no tail region and no region type. + +## 2. Identity exists independently of names + +``` +occurrence c42 + -> identity char_07 + name = null + aliases = [] +``` + +`char_07.name` may be filled later, or stay null forever and display as `unknown character #7`. The +occurrence is the observation, the identity is the cluster, the name is an optional label on the cluster. +Three levels, never collapsed into one. + +**Today:** the schema already has this split. `identity_assignments` is the occurrence, +`characters` owns the identity, `name` is nullable and downstream already falls back to an anonymous +display. What is missing is the clustering, not the separation. See section 4. + +## 3. Speaker attribution is a scored graph edge, not a procedure + +Do not write `find bubble -> find tail -> nearest character`. Score every plausible edge: + +``` +score(text, character) = + learned_t2c_score + + tail_evidence + + spatial_evidence + + same_panel + + dialogue_continuity + + character_activity_prior + + identity_context +``` + +`learned_t2c_score` is the load-bearing term: a pair classifier over the whole page, the text object's +visual feature and the character object's visual feature. Magi's text-character head does exactly this. +It can start as a tiny MLP: + +``` +t2c(text_embedding, character_embedding, page_context, geometry_features) -> p(speaker) +``` + +with geometry carrying normalized relative position, distance, overlap, same-panel, containment depth and +tail direction. + +Then the cases fall out of one mechanism instead of four: + +| case | what carries it | +| --- | --- | +| bubble with a tail | `t2c` + tail, usually decisive | +| bubble with no tail | `t2c` + spatial and context | +| speaker outside the panel | recent identities + an offscreen candidate | +| narration | the narrator candidate | +| nothing resolves | unknown speaker | + +**A dialogue line must not be required to resolve to a visible character.** That is a failure mode, not a +safeguard. The speaker type is a union: + +``` +speaker = visible(character_id) | offscreen(character_id?) | narrator | unknown +``` + +**Today:** `speaker_ref` is already a typed union of `character_id | name | unknown | narrator` +(`decisions/audit-phase1.md#speaker-ref-is-canonical`). `offscreen` is the missing arm. Attribution is a +prompt to gemma over a window of panels, with no geometry term at all. The `det`/`seg` tail heads exist +and are unused (`caveats/speaker-attribution.md#tail-is-not-geometry`). + +## 4. Character recognition is occurrence, then identity, then name + +``` +character detection + ↓ +occurrence embeddings + ↓ +pairwise same_identity probabilities + ↓ +chapter-wide constrained clustering + ↓ +char_001, char_002, ... + ↓ +optional character-bank lookup + ↓ +name or unknown +``` + +Two rules that the current code gets wrong. + +**The embedding is not the character crop alone.** Combine four signals: the character crop, the face or +head crop, the full-body crop, and a contextual object feature. Magiv2 combines detected object features +with a separate crop-embedding model. + +**Cluster chapter-wide, not page by page.** + +**Two characters in the same panel may be one person.** Mirrors, photographs, flashbacks, insets, +screens, imagined scenes and repeated action drawings all break that rule. Make it a weak cannot-link, +and only when the two are on the same narrative plane. + +**Today:** the embedding is the person box only, which is measurably the wrong signal +(`caveats/audit-open.md#cosine-not-identity`). Clustering is greedy and local: `tracklets.link_tracklets` +groups within an 8-panel window. `tracklets.cannot_link` treats same-panel co-presence as a **hard** +constraint, which is exactly the correction above. Naming is `db.add_name_claim`, corroboration over +`name_claims`. + +## 5. The art-in-art problem needs a narrative plane + +Treat the page as a hierarchical scene graph: + +``` +page +└── panel A depth=0 + ├── character c1 + ├── text t1 + └── television/poster depth=1, type=embedded_art + ├── character c2 + └── text t2 +``` + +Speaker candidates normally come from the same `scene_depth`. Otherwise a real character standing beside a +poster of a drawn person can be given the poster person's line. + +A region classifier predicts a type: + +``` +story_scene | inset_story_panel | flashback | screen | photo | poster | illustration | decorative +``` + +Perfect classification is not the point. The output that matters is one probability: + +``` +same_narrative_plane(a, b) +``` + +which then enters the association score in section 3 and the cannot-link in section 4. + +**Today:** nothing models this, and it is the whole of the remaining identity error on the lead. On the +19:44 run of 2026-08-12 his 16 assignments were 14 correct plus a photograph of another man and a chibi +drawing. Both are art inside a panel. Vision also boxes cats as people and dresses them (`p081`, `p108`). + +## 6. Narrative understanding is a state machine, not a per-panel description + +``` +story_state +├─ entities (characters, locations, important objects) +├─ scenes +├─ timeline +├─ relationships +├─ unresolved_threads +├─ facts +└─ hypotheses +``` + +A panel produces a **delta**, not another standalone prose interpretation: + +``` +panel 142: +- character_07 enters room_03 +- character_02 is already present +- character_07 says "..." +- object_12 changes owner: 02 -> 07 +- possible flashback begins +``` + +### Facts, hypotheses and unknowns are different records + +``` +fact: source=panel_142 confidence=0.99 character_07 is visible +hypothesis: confidence=0.64 character_07 is angry +unknown: who caused the explosion +``` + +A later panel strengthens, replaces or invalidates a hypothesis without rewriting history. + +### Scene state is explicit and inherited + +``` +scene_31: + location: school_rooftop + time: evening + participants: {char_03: present, char_07: present, char_11: offscreen} + pov: null + narrative_mode: present + parent_scene: null +``` + +A panel inherits this unless visual evidence overrides it. That alone kills a class of errors. A character +absent for one panel has not left. A panel with no background has not changed location. A tail-less line +keeps the offscreen participant as a candidate. A close-up still belongs to the scene. + +### Classify the transition, not just the panel + +``` +CONTINUE_SCENE | NEW_SCENE | LOCATION_CHANGE | TIME_SKIP | FLASHBACK_START +FLASHBACK_END | DREAM/IMAGINATION | POV_CHANGE | EMBEDDED_SCENE +``` + +`EMBEDDED_SCENE` is what stops a television's contents mutating the room around it: + +``` +scene_12 present + ├─ panel 101 + ├─ panel 102 + └─ embedded scene_13 [television] + ├─ panel-like region + └─ char_19 +``` + +### Character state is written by a resolver, never by the vision model + +``` +char_07: + known_names: [...] + currently_at: room_03 + status: alive + appearance_state: {clothes: school_uniform, injured: true} + relationships: {char_02: friend?} + last_seen: panel_142 +``` + +The path is `observation -> resolver -> state transition`, and the resolver may reject an impossible +update. + +### Conversation state is its own record + +``` +conversation_18: + scene: scene_31 + participants: [char_02, char_07] + last_speaker: char_07 + addressee: char_02 + topic: missing_key +``` + +This is the strongest available prior for a tail-less bubble. Given `A: where did you put it? / ... / +A: don't lie.`, turn-taking assigns the middle line with no visual evidence at all. + +### An unresolved reference survives instead of being forced + +``` +unknown_04: + type: person + descriptions: ["the man from yesterday", "silhouette in panel_58"] + candidate_ids: {char_12: 0.55, char_19: 0.22} +``` + +Chapter 6 may reveal `unknown_04 == char_12`, and that identity back-propagates through the graph. The +same applies to unnamed characters, pronouns, disguised characters, mysterious objects and unseen +speakers. + +### Two memories + +- **Working narrative state**: the current scene and the recent ones, in detail. +- **Canonical long-term memory**: compressed facts, not chapter summaries. `char_07 learned that char_02 + betrayed the group.` `object_04 is held by char_11.` `char_03 does not know char_07 survived.` + +### A chapter boundary is a checkpoint, not a reset + +``` +chapter_checkpoint: + persistent_entity_changes / relationship_changes / location and status changes + newly established facts / unresolved questions / active plot threads / final scene state +``` + +Chapter `n+1` starts from that. The detailed panel graph may be kept forever. Only five things load into +the model: the current scene, the previous scene, the relevant character records, the active threads, and +retrieved old facts. + +### A consistency checker runs after each scene and chapter + +Seven checks. A dead character appearing normally. A character knowing a fact before learning it. An +object owned by two people at once. A flashback never closed. A location jump with no transition. A +speaker who was neither present nor offscreen. A name that conflicts with the identity graph. The model +proposes corrections. The graph stays the source of truth. + +**Today:** none of this exists. Each stage reads its predecessor's blob for one panel or one beat. +`recent` is a rolling list of the last few dialogue lines and is the only carried state. Chapter +boundaries are a reset. There is no fact-versus-hypothesis distinction anywhere, which is why narration +asserts things no panel shows (`NEXT.md` item 6). + +## 7. The staged version worth building + +Do not recreate Magi's monolithic network first. Detection, vision and character embeddings already exist, +so stage it: + +``` +page + ↓ +region detector panels / nested regions / texts / characters / tails + ↓ +object feature extraction + ↓ +three pair models character↔character (identity) + text→character (speaker) + text→tail (bubble structure) + ↓ +chapter graph + ↓ +global character clustering + ↓ +optional naming + ↓ +ocr + reading order + ↓ +dialogue stream +``` + +The VLM then judges only the ambiguous graph edges. It no longer rediscovers every character and dialogue +relationship from raw pixels on every panel. Magi formulates detection and association as graph +generation, which is why it beats a crop, OCR and nearest-character pipeline here. + +## What to take from this before the rewrite + +Three items are cheap against the current code and pay immediately. They are entered in `NEXT.md`, not +here. + +1. **`same_narrative_plane`, as a per-detection field.** Vision already returns per-panel boxes. Add a + `plane` or `depth` to a detection, set when the model says the figure sits inside a screen, poster, + photo or drawing. That buys the containment edge with no detector. It is the whole of the remaining + identity error on the lead, and it feeds every stage below. +2. **Same-panel co-presence becomes a weak cannot-link.** `tracklets.cannot_link` currently makes it hard. + It needs item 1 first, because the plane is what makes the weak version safe. +3. **`offscreen` as a fourth `speaker_ref` kind.** The union already exists, the arm does not. + +## Sources + +Magi and Magiv2 for the detection-and-association-as-graph-generation formulation, the text-character +pair head, and the character bank of exemplar images plus names. Magiv3 for panels, texts, characters and +tails with their associations, and for character grounding between textual descriptions and detected +character regions. diff --git a/CLAUDE.md b/CLAUDE.md index 666e431..34e92fc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -11,6 +11,7 @@ Goal, invariants, and working rules. Read this first. | `caveats/` | every known limit and its revisit trigger, indexed in `caveats/CLAUDE.md` | | `AGENTS.md` | commands, with the traps beside them | | `AUDIT.md` | the 2026-08-11 pipeline audit, the source of the roadmap | +| `ARCHITECTURE.md` | the target shape of the pipeline, and what exists against it today | | `spec-v3.md` | current quality and look work, marked DONE/TODO per item | Do not restate a finding here. Point at the decision. diff --git a/JOURNAL.md b/JOURNAL.md index c23cf3a..f4c2a40 100644 --- a/JOURNAL.md +++ b/JOURNAL.md @@ -723,3 +723,36 @@ Checks: `worker_vision.py` self-check ok, `tracklets.py` self-check ok, orchestr `/characters/reset` asked for it. Artefacts: `sheet_*.png`, one contact sheet per character. Session scratchpad only, not committed. + +## 2026-08-13 — the dialogue stage names nobody, and why + +Ran `dialogue` 116/116 in 5m57s on the fourth cycle's registry, to see whether the fixed identity lets the +existing `name_claims` path name the female lead. It does not, and the six claims it produced name three +separate defects. + +``` +p040 character_2b1b12a1 "Choi Haeseon" caption 1.00 -> NOT promoted +p010 character_b1dd5659 "Lim Seonho" caption 1.00 -> conflict flag +p047 character_b1dd5659 "Seonho" address 0.90 -> conflict flag +p011 character_f0d4e901 "Seonho" address 1.00 -> PROMOTED +p026 character_f0d4e901 "Seonho" address 1.00 -> PROMOTED +p110 character_028d4a49 "Haeseon" address 1.00 -> already named +``` + +All three are fixed in `db.add_name_claim` and recorded in `decisions/identity-naming.md`: alias grouping, +a confident caption as strong evidence, and one name per character. `test_name_binding.py` replays these +six claims, 121 tests pass, and each new assert was confirmed to fail with its fix disabled. + +Also wired `merge_faceless_captions` into both crop endpoints. It had been written and never called; both +endpoints called the non-destructive `context_fragment_links` instead, and no decision recorded that +choice. It does not cover the head-in-one-shot, body-in-the-next split that prompted the question, because +a body fragment has no text and `_merge_plan` only folds a fragment that has text and no face. + +Found while wiring it: `crop_webtoon` skips an upload when the key exists, which is right for a resume and +silently wrong after a slicing change. Documented at the line and in `NEXT.md`. + +Wrote `ARCHITECTURE.md` from the user's design: region graph, occurrence/identity/name, speaker as a scored +graph edge with a typed union, narrative plane for art-in-art, and a persistent story state machine. Every +section carries what exists against it today. Nothing in it is built. + +Nothing ran on a GPU after the dialogue stage. diff --git a/NEXT.md b/NEXT.md index 05c198d..b61b241 100644 --- a/NEXT.md +++ b/NEXT.md @@ -39,28 +39,42 @@ contain the right person. Coverage is still the `has_face` gate plus those refus ## Next -1. **Deploy the two tracklet fixes and run the fourth cycle.** Neither has touched a GPU. Rebuild the - orchestrator image, reset the registry, run vision/identity/reconcile, then re-check the lead's crops. - Expect roughly 30 tracklets over 64 crops instead of 12, so identity goes from about 1m25s to 3 or 4 - minutes. Watch the lead's assignment count against 36. +1. **Re-crop the chapter and run the fifth cycle.** Four changes are written and tested since the fourth + cycle, and none has touched a GPU. - Neither fix is sufficient. Bare hair colour still links different men, and the cat still joins its - neighbours. Do not add a crop-to-crop cosine to close that: measured on this run's 22 embeddings, - different people reach 0.93 while the same person reaches 0.96, so no threshold exists - (`caveats/audit-open.md#cosine-not-identity`). + - `merge_faceless_captions` is wired into both crop endpoints. It was written, never called, and + `context_fragment_links` was called instead. A stranded caption fragment now vstacks into the + face-bearing fragment it belongs to, so panel count and every panel index change. + - three naming fixes in `db.add_name_claim` (`decisions/identity-naming.md`): alias grouping, a caption + as strong evidence, and a name held by another character refusing to promote onto a second one. - Agreed next step after the cycle, chosen by the user and not started: **stop letting cosine pick the - gallery.** There are 9 live characters. `run_stage_identity` builds `union_cands` from the members' - cosine top-k shortlists, so a metric that cannot separate people decides who is even considered. Send - the live cast instead, gender-gated, capped and logged when truncated. Note two traps found while - reading it: `/vision/resolve` sends up to 3 reference images per candidate - (`worker_vision.py:1057`), so 9 candidates is 27 images plus the query and needs a cap; and only a crop - with a non-empty cosine shortlist enters `shortlists` at all, so an empty top-k currently drops the crop - from resolution entirely. + **Clear `s3://panels///panels/` before re-cropping.** `crop_webtoon` skips the upload + when the key exists, so a re-crop after a slicing change silently keeps the previous run's images. + Wiring the merge is a slicing change. Everything downstream is invalidated by it, so this is a full + re-run and not a stage rerun. + + Expected: fewer than 116 panels, `2b1b12a1` named `Choi Haeseon` from the p040 caption, the lead's + `conflicting-name-claims` flag gone, and the green-dress woman no longer named `Seonho` but carrying a + `name-already-taken` flag instead. + + Not fixed by any of it. Bare hair colour still links different men. Do not add a crop-to-crop cosine to + close that. Measured on 22 embeddings, different people reach 0.93 and the same person reaches 0.96, so + no threshold exists (`caveats/audit-open.md#cosine-not-identity`). Then, separately, test embedding the FACE box rather than the person box. `face_detect` already finds the face and pairs it for `has_face`. That is the likely root cause of cosine measuring scene instead of - person, and the test is to re-embed these same 22 detections and recompute the matrix. + person. The test is to re-embed these same 22 detections and recompute the matrix. + +1b. **The head/body split that started the crop question is NOT fixed.** The wired merge only folds a + fragment that has text and no face. A body fragment carries no dialogue, so `_merge_plan` leaves it + solo and it becomes its own panel and its own shot. Finding it needs a different signal, most likely a + face touching the bottom edge of one fragment with a textless fragment below. No evidence has been + gathered yet on how often this chapter does it. + +1c. **Three items from `ARCHITECTURE.md` are cheap against the current code.** A `plane` field per + detection for art-in-art, same-panel co-presence demoted to a weak cannot-link once the plane exists, + and `offscreen` as a fourth `speaker_ref` kind. + 2. ~~**Order the corners in `_bbox_to_pixels`.**~~ **Done 2026-08-12, run and verified on a GPU.** 0 degenerate boxes over 119 detections (`decisions/identity-bbox.md#bbox-corners-ordered`). 3. **Vision boxes animals as people and dresses them.** `p081` and `p108` are cats, described diff --git a/decisions/CLAUDE.md b/decisions/CLAUDE.md index 73278ab..918048e 100644 --- a/decisions/CLAUDE.md +++ b/decisions/CLAUDE.md @@ -48,5 +48,9 @@ still live belongs in `caveats/`. | [A roster name is a guess, so it never reaches detection](identity-bbox.md#roster-does-not-name) | closed | | [`merged_into` is exactly one hop deep](identity-bbox.md#merge-chains-flatten) | closed | | [`_bbox_to_pixels` orders the corners, because the model sometimes swaps them](identity-bbox.md#bbox-corners-ordered) | closed | -| [A tracklet is bounded by span, not only by pairwise distance](identity-bbox.md#tracklet-span-cap) | closed, GPU pending | -| [A generic word is not identity evidence, and one tokenizer serves both consumers](identity-bbox.md#generic-tokens) | closed, GPU pending | +| [A tracklet is bounded by span, not only by pairwise distance](identity-bbox.md#tracklet-span-cap) | closed | +| [A generic word is not identity evidence, and one tokenizer serves both consumers](identity-bbox.md#generic-tokens) | closed | +| [The gallery is the live cast, not cosine's top-k](identity-bbox.md#cast-is-the-gallery) | closed | +| [A name is a word set, not a string](identity-naming.md#alias-grouping) | closed, GPU pending | +| [A confident caption names a character on its own](identity-naming.md#caption-is-strong) | closed, GPU pending | +| [A name belongs to one character](identity-naming.md#one-name-one-character) | closed, GPU pending | diff --git a/decisions/identity-naming.md b/decisions/identity-naming.md new file mode 100644 index 0000000..ea36ddd --- /dev/null +++ b/decisions/identity-naming.md @@ -0,0 +1,51 @@ +# Naming a character + +How a discovered name reaches `characters.name`. The mechanism is `db.add_name_claim`, fed by the dialogue +stage through `service._absorb`. + +## A name is a word set, not a string {#alias-grouping} + +**Closed, 2026-08-13, not yet run on a GPU.** + +Claims grouped on the casefolded name, so `Lim Seonho` from a p010 caption and `Seonho` from a p047 address +counted as two names for one character. `len(grouped) > 1` fired, a `conflicting-name-claims` flag was +filed, and promotion was blocked permanently on evidence that in fact corroborated. + +`alias_groups` groups two names when one's word set contains the other's, and keeps the longer as +canonical. `Seonho` and `Lim Seonho` are one name and the registry stores `Lim Seonho`. `Seonho` and +`Haeseon` are still two, so a real conflict still flags. + +## A confident caption names a character on its own {#caption-is-strong} + +**Closed, 2026-08-13, not yet run on a GPU.** + +Promotion needed two distinct panels, or one `self_intro` or `name_tag` claim above 0.9. On the 2026-08-12 +chapter the only claim naming a main character was `Choi Haeseon`, a caption at p040 at confidence 1.0, and +it was discarded. She held 13 correct crops and stayed anonymous through the whole pipeline. + +A caption is the narration naming the person it is drawn beside, which is how a webtoon introduces its +cast, so it joins `STRONG_EVIDENCE`. This is the loosest of the three changes and is only safe because of +the next one. + +## A name belongs to one character {#one-name-one-character} + +**Closed, 2026-08-13, not yet run on a GPU.** + +Being addressed by name identifies the addressee. Choosing which drawn body that is fails often. Two panels +addressed `Seonho`, the dialogue model pointed `target_local_id` at the woman standing beside him, and two +independent claims promoted her. The registry then held a female `Seonho` over 9 crops beside the lead's +`LIM SEONHO`. + +A promotion now checks every other live character of the same manga first, by alias group. A collision +refuses the promotion and files a `name-already-taken` flag carrying both ids. The collision is itself +evidence that either the addressee or the identity cluster is wrong, so it is worth surfacing rather than +resolving silently. + +Deliberately not built: no attempt to decide WHICH character deserves the name. That needs the addressee +fixed, which is `ARCHITECTURE.md` section 3. + +## Checks + +`test_name_binding.py` replays the six real claims from the 2026-08-12 chapter. Each new assert was +confirmed to fail with its fix disabled: the caption test with `STRONG_EVIDENCE` reverted, the alias test +with casefold grouping restored. The taken-name test asserts a flag kind that only the new branch emits. diff --git a/worker_crop.py b/worker_crop.py index 5c5fa48..e0ec4ee 100644 --- a/worker_crop.py +++ b/worker_crop.py @@ -240,12 +240,15 @@ async def crop_webtoon(data: WebtoonInput): tag = uuid.uuid4().hex[:8] locals_ = [transport.get(u, f"{SHM}/wt_{tag}_{i:04d}.png") for i, u in enumerate(data.page_uris)] strip = restitch(locals_) - crops = slice_webtoon(strip) + crops = merge_faceless_captions(slice_webtoon(strip)) context_links = context_fragment_links(crops) panels = [] for idx, (crop_img, bbox) in enumerate(crops): uri = f"s3://panels/{data.manga_id}/{data.chapter_id}/panels/p{idx:03d}.png" # slicing is deterministic, so on a resume the same idx -> same key; skip re-upload. + # TRAP: that holds only while the PLAN is unchanged. Edit slice_webtoon or the merge pass and a + # re-crop silently keeps the previous run's images under the same keys, because every one of them + # already exists. Clear the s3://panels///panels/ prefix before re-cropping. if not transport.exists(uri): out = f"{SHM}/wt_{tag}_p{idx:03d}.png" cv2.imwrite(out, crop_img) @@ -266,7 +269,7 @@ async def crop(data: CropInput): raise HTTPException(400, f"page not readable: {data.page_uri}") h, w = img.shape[:2] webtoon = h / w >= WEBTOON_RATIO - crops = slice_webtoon(img) if webtoon else kumiko_panels(local, data.rtl) + crops = merge_faceless_captions(slice_webtoon(img)) if webtoon else kumiko_panels(local, data.rtl) context_links = context_fragment_links(crops) if webtoon else {i: [] for i in range(len(crops))} ambiguous = flag_overlaps(crops, data.page_index) -- 2.52.0 From dae9dbc93a98a2b1d1bddec2652f185ac94957d9 Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 00:41:11 +0400 Subject: [PATCH 27/31] Hand off the seventh session: cast gallery run, naming fixes, architecture Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 157 ++++++++++++++++++++++++++--------------------------- 1 file changed, 77 insertions(+), 80 deletions(-) diff --git a/HANDOFF.md b/HANDOFF.md index b6db079..b8b7fad 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,124 +1,121 @@ -# HANDOFF, 2026-08-12 (sixth session) +# HANDOFF, 2026-08-12/13 (seventh session) Live state is in `NEXT.md`. This file is only what this session did. The previous handoff is in `JOURNAL.md`. ## Asked -The two context files, then "go" on the third GPU cycle, then "commit", then "what's next". Then the user -checked the lead's crops in the review UI and said the assignments were wrong. Then "test on one of the -panels first". Then "mind presenting the panels with boxes". Then "so, we didn't fix it properly?". Then -"go with the 1". +"go" on item 1 of the previous plan, then a fourth GPU cycle. Then "commit and let's figure out how to +deal with the 2b1b". Then a long architecture message, "mind turning the architecture in docs?", plus a +question about panel merging before cropping. Then "well, I'd like the merge to be wired actually. and +then 3 of your fixes." ## Result -One worker fix proven on a GPU. One measurement that killed the plan's item 1. One GPU cycle. Then the user -found the registry is over-merged, which invalidates the cycle's headline numbers, and two more fixes were -written for it. The third fix the user chose, item 1 of the three options, is NOT started. - -| metric | 17:38 run | 18:07 run | -| --- | --- | --- | -| detections | 117 | 119 | -| assignments | 59 | 68 | -| coverage | 50% | 57% | -| degenerate boxes | 1 | 0 | -| assigned among face-bearing | 59/72 = 82% | 68/71 = 96% | -| assigned among gated | -- | 0 | -| chains deeper than one hop | 1 | 0 | -| `merged_from` stamps | 9 | 22 | - -Cycle timings, 18:07-18:13 UTC: vision 116/116 in 3m59s, identity 116/116 in 1m24s, reconcile 18/18 in 50s. +Item 1 written, run on a GPU, committed. Naming diagnosed from a real dialogue run and three fixes written +against it. The crop merge wired. `ARCHITECTURE.md` written. Nothing has run on a GPU since 20:25 UTC. ## Committed and proven on a GPU -`_bbox_to_pixels` sorts each coordinate pair after clamping -(`decisions/identity-bbox.md#bbox-corners-ordered`). 0 degenerate boxes over 119 detections, against 1 in -117. Commit `54bd126` on `restore-runtime`. Its caveat is deleted. +`a386e9d` here, `2daaa84` in the orchestrator. **The resolver gallery is the live cast, not cosine's top-k** +(`decisions/identity-bbox.md#cast-is-the-gallery`). `tracklets.cast_gallery` builds it from +`get_known_characters`, gender-compatible, named first, `GALLERY_CAP = 10`, re-read per tracklet. The +`if s.get("candidates")` guard is gone, so a crop with an empty cosine top-k now reaches the resolver. +`worker_vision.REF_BUDGET = 12` spreads reference images, `max(1, min(3, 12 // n))` apiece. -## Measured, no code +Fourth cycle, 19:44-19:52 UTC: vision 116/116 3m55s, identity 116/116 2m44s, reconcile 20/20 44s. Ran with +the span cap and `GENERIC` tokenizer, which had never touched a GPU either. -**The 145 ground-truth labels are for a different manga.** Every row in `identity_labels` keys to chapter -`8ca8249b`, cast "Rico" and "Ikekin", 81 panels. Chapter `7c944dd4` has zero, so -`/review/identity?job_id=778297bc...` returns `labeled: 0, accuracy: null`. Scoring `8ca8249b` gives 7/138 -on a run with 44 assignments over 246 panels, predating every fix. +| metric | 18:07 run | 19:44 run | +| --- | --- | --- | +| detections | 119 | 119 | +| assignments | 68 | 60 | +| coverage | 57% | 50% | +| tracklets / crops | 12 / 64 | 33 / 72 | +| lead's assignments | 36 | 16 | +| characters after reconcile | 18 | 14 | +| minted / cleared | -- | 10 / 12 | -**Coverage is the `has_face` gate and nothing else.** All 68 assignments landed on face-bearing detections -and none on a gated one. Recall among face-bearing detections is 96%, up from 82%. +Coverage fell because gemma clears 12 crops instead of naming them wrongly. Checked by eye and confirmed by +the user. The lead's 16 are 14 him plus 2 art-in-art. `character_2b1b12a1` holds 13, all correct. +`character_f0d4e901` holds 9, of which 2 are `2b1b12a1`. -**Embedding cosine cannot separate people.** All 22 crop embeddings for the lead, pulled from -`manga//characters/_crops/*.npy`, 1152 dims, L2-normalised. The cat scores up to 0.82 against -men, two different men score 0.93, the highest pair in the matrix is 0.96 -(`caveats/audit-open.md#cosine-not-identity`). This rules out a crop-to-crop cosine link, which was the -fix proposed one message before the measurement. +## Measured, then fixed -## Written, tested, NOT deployed and NOT run on a GPU +Dialogue ran 116/116 in 5m57s at 20:19 UTC to test the naming path. Six claims, three defects: -Both in `/mnt/server/home/kami/docker-apps/manga-infra/orchestrator/`, uncommitted there: +``` +p040 character_2b1b12a1 "Choi Haeseon" caption 1.00 -> NOT promoted +p010 character_b1dd5659 "Lim Seonho" caption 1.00 -> conflict flag +p047 character_b1dd5659 "Seonho" address 0.90 -> conflict flag +p011 character_f0d4e901 "Seonho" address 1.00 -> PROMOTED (wrong body) +p026 character_f0d4e901 "Seonho" address 1.00 -> PROMOTED (wrong body) +p110 character_028d4a49 "Haeseon" address 1.00 -> already named +``` -- `tracklets.py`: `link_tracklets` rejects a merge whose group would span more than `window` panels - (`decisions/identity-bbox.md#tracklet-span-cap`). `window` bounded each pair, transitivity was unbounded, - and the lead's 22 native assignments came from 3 tracklets spanning 0, 22 and 30 panels. -- `tracklets.py` + `service.py`: one `appearance_tokens` with a `GENERIC` stopword set, and - `service._appearance_tokens` delegates to it, so reconcile's pre-filter is fixed too - (`decisions/identity-bbox.md#generic-tokens`). Whole chains hung on the word `short`, one pair on the - word `hair`. +## Written, tested, NOT run on a GPU -On the same 22 crops, candidate overlap forced to pass: 3 tracklets at worst span 30 becomes 9 at worst -span 8. +Orchestrator, committed `2927927`, `db.add_name_claim` + `decisions/identity-naming.md`: + +- `alias_groups`: one name's word set inside another's is the same name, longer wins +- `STRONG_EVIDENCE` gains `caption` at confidence >= 0.9 +- a name held by another live character refuses to promote, files `name-already-taken` + +Here, committed `97cb483`: + +- `merge_faceless_captions` wired into both crop endpoints. It existed and was never called. +- `ARCHITECTURE.md`, the target shape from the user's design, each section carrying what exists today. ## Not started -**Item 1, which the user chose: stop letting cosine pick the gallery.** `run_stage_identity` builds -`union_cands` from the members' cosine top-k. So a metric that cannot separate people decides who gemma is -even allowed to consider. There are 9 live characters. Send the live cast instead, gender-gated, capped and -logged when truncated. Two traps found while reading it: - -- `/vision/resolve` sends up to 3 reference images per candidate (`worker_vision.py:1057`), so 9 candidates - is 27 images plus the query. It needs a cap. -- only a crop with a non-empty cosine shortlist enters `shortlists` at all, via the - `if s.get("candidates")` guard in `service.py`. An empty top-k drops the crop from resolution entirely. - -Also open: nothing downstream re-ran, the job is still parked at `dialogue waiting`, and vision boxes cats -as people and dresses them (`p081`, `p108`). +- **The head/body split is not fixed.** The wired merge only folds a fragment with text and no face. A body + fragment has no dialogue, so `_merge_plan` leaves it solo. No evidence gathered on how often it happens. +- The three cheap items from `ARCHITECTURE.md`: a `plane` field per detection, same-panel co-presence as a + weak cannot-link, `offscreen` as a fourth `speaker_ref` kind. +- `caveats/audit-open.md#gallery-cap-drops-the-unnamed`: the cap fired at `p097` (16 -> 10) and `p109` + (11 -> 10) and dropped exactly the freshly minted anonymous rows. ## Checks ```bash -.venv/bin/python worker_vision.py # self-check ok, including the swapped-corner assert -./check_stale.sh # exit 0 before the cycle and after it -/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 tracklets.py && python3 -m pytest -q --ignore=test_api.py" # self-check ok, 118 passed +.venv/bin/python worker_vision.py # ok, including the reference-budget asserts +.venv/bin/python worker_crop.py # ok +./check_stale.sh # exit 0 before the cycle +/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 tracklets.py && python3 test_name_binding.py && python3 -m pytest -q --ignore=test_api.py" # 121 passed ``` -Every new assert was confirmed to fail with its fix disabled: the span cap returns `[[0, 1, 2]]`, and the -generic-word pair links with `GENERIC` emptied. +Each new assert was confirmed to fail with its fix disabled. ## Next command -Deploy the two orchestrator fixes and run the fourth cycle. +Re-crop and run the fifth cycle. **Clear the panels prefix first** or the merge will not take effect: +`crop_webtoon` skips an upload when the key exists. ```bash -cd /home/kami/Programs/n8n-worker && ./check_stale.sh # must exit 0 +cd /home/kami/Programs/n8n-worker && ./check_stale.sh # restart crop, it is stale after the wiring /usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && docker compose up -d --build orchestrator" -J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e +J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e; C=7c944dd4-e972-42c7-ba60-9f6939548e80 +/usr/bin/ssh kami@192.168.1.104 "mc rm --recursive --force homesrv/panels/$M/$C/panels/" /usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" -# the reset returns restart_identity_worker: true -- honour it, see the traps -/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"vision\"}'" -for S in vision identity reconcile; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 5400 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done -/usr/bin/ssh kami@192.168.1.104 "docker logs manga-orchestrator --since 1h 2>&1 | grep tracklet" # expect ~30 tracklets, was 12 -/usr/bin/ssh kami@192.168.1.104 "docker exec manga-orchestrator python3 /app/audit_registry.py 7c944dd4-e972-42c7-ba60-9f6939548e80 6" +# honour restart_identity_worker: true +/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"crop\"}'" +for S in crop vision identity reconcile dialogue; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 7200 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done ``` -Then re-check the lead's crops by eye. A stage counter will not show this defect. +Expect: fewer than 116 panels, `2b1b12a1` named `Choi Haeseon`, no `conflicting-name-claims` on the lead, +and a `name-already-taken` flag where the female `Seonho` used to be. ## Traps confirmed or found +- **`crop_webtoon` skips an upload when the key exists.** Right for a resume, silently wrong after any + slicing change. Clear `s3://panels///panels/` before re-cropping. - Restart a worker with `tmux send-keys -t manga-workers: C-c`, then re-send the launch line from `start_workers.sh` with its `MIOPEN_ENV` prefix. A `pgrep` kill matches the calling shell. -- `POST /characters/reset` returns `restart_identity_worker: true`. Honour it, the worker caches the - registry. -- `identity_assignments` has no `method` column. The `merged_from:` stamps live in - `identity_assignment_sources`. `chapters` has no `title` column. The orchestrator image has no numpy. -- A heredoc piped into `docker exec` over `/usr/bin/ssh` silently produces no output. Write the script to a - file, `scp` it, `docker cp` it, then run it. -- Mixing `echo` with a `tar cf -` stream over ssh corrupts the archive. Separate the calls. +- `POST /characters/reset` returns `restart_identity_worker: true`. Honour it. +- `docker compose up --build orchestrator` drops any `docker cp`'d script, `audit_registry.py` included. +- A heredoc piped into `docker exec` over `/usr/bin/ssh` produces no output. `scp` the file, `docker cp` + it, then run it. - `panel_order` and the panel filename differ by one: `panel_order` 10 is `p009.png`. +- The contact-sheet script that found the identity errors is `sheet.py` in the session scratchpad, not + committed. It reads `assigns.json` dumped from `identity_assignments` and pulls crops from + `manga//characters/_crops/`. -- 2.52.0 From 12d4ed8534a5061b08e67b65abddf2a74dc143ed Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 22:28:57 +0400 Subject: [PATCH 28/31] Rewrite the architecture under the no-Magi constraint Magi and any learned head are ruled out, so every structure comes out of a gemma prompt field or plain Python over gemma's output. Each section now carries what exists today and a DoD. The build order starts at step 0, the measurement spine: eval/chapter-truth.json plus purity and fragmentation printed by audit_registry.py. Nothing below it is measurable without that file. Co-Authored-By: Claude Opus 5 --- ARCHITECTURE.md | 429 +++++++++++++++++++++++------------------------- NEXT.md | 12 +- 2 files changed, 216 insertions(+), 225 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 2b9adad..67def2e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,8 +1,9 @@ # ARCHITECTURE -The target shape of the pipeline, written 2026-08-13 from the user's design. This is **not** what the code -does. `NEXT.md` holds the live state and `AUDIT.md` holds the current pipeline. Every section here ends -with what exists today, so the gap is legible without reading both. +The target shape of the pipeline. Written 2026-08-13 from the user's design, rewritten the same day under +the no-Magi constraint. This is **not** what the code does. `NEXT.md` holds the live state and `AUDIT.md` +holds the current pipeline. Every section ends with what exists today and what would make it done. The +gap stays legible and testable without reading both. The governing principle: @@ -12,6 +13,49 @@ The governing principle: Vision produces observations. A persistent chapter graph owns identity and relationships. Everything below follows from that split. +## Ruled out + +Read this before proposing any of it again. + +| ruled out | why | who decided | +| --- | --- | --- | +| Magi, Magiv2, Magiv3 as a detector or as pair heads | project constraint, not a measurement | user, 2026-08-13 | +| A trained `t2c` pair model, or any learned head | no labelled pages exist and no labeller is planned | follows from the above | +| Crop-to-crop cosine as a link signal | measured: two men reach 0.93, one man reaches 0.96, no threshold exists (`caveats/audit-open.md#cosine-not-identity`) | 2026-08-12 | + +The consequence runs through the whole document. **Every structure below comes out of a gemma prompt +field, or out of plain Python over gemma's output.** Nothing below is trained. A learned score +in the original design becomes a hand-weighted sum. The weights are read off the labelled chapter, and +the DoD is the accuracy number rather than the mechanism. + +## The measurement spine + +Nothing below can be called done without this, and it does not exist yet. + +The only ground truth in the project is the eyeball pass over the 19:44 run of 2026-08-12. It lives in +prose in `NEXT.md`. Write it to `eval/chapter-truth.json` against chapter +`7c944dd4-e972-42c7-ba60-9f6939548e80`, scoped to what was already checked by eye rather than to all 119 +detections: + +- the three characters walked crop by crop, each assignment marked as the real person or not +- 30 dialogue lines with their true speaker, typed as `visible | offscreen | narrator | unknown` + +The baseline it records, from that run: + +| character | assignments | correct | purity | note | +| --- | --- | --- | --- | --- | +| the lead | 16 | 14 | 0.88 | plus a photograph at `order 17` and a chibi at `order 20` | +| `character_2b1b12a1` | 13 | 13 | 1.00 | a main character the registry never named | +| `character_f0d4e901` | 9 | 7 | 0.78 | the other 2 are `2b1b12a1` | + +Woman A is `2b1b12a1`. She has 15 occurrences split across 2 ids, so her fragmentation is 2. + +`audit_registry.py` already walks panels, reads `identity_assignments` and counts per character. Extend +it to print purity and fragmentation against the truth file. Do not write an eval harness. + +**Done when:** `audit_registry.py ` prints purity per labelled character and fragmentation per +labelled person, and reproduces the table above on the 19:44 run. + ## 1. The page is a region graph, not a list of panels ``` @@ -36,9 +80,29 @@ page A flat set of panels cannot express a television inside a room. That is the defect the current pipeline shows most often. +With no detector to train, the graph comes out of the detection prompt. Gemma already returns a box per +character and per text. Two fields per detection buy most of the graph with no new model: + +``` +plane = story | screen | photo | poster | drawing | flashback | dream +species = human | animal | object +``` + +`plane` is the containment edge in disguise. A detection whose `plane` is not `story` sits inside embedded +art, and that is the fact every stage below needs. `species` is a separate axis and exists because vision +boxes cats as people and dresses them. + +Tail regions stay unbuilt. The `det`/`seg` heads exist and are unused +(`caveats/speaker-attribution.md#tail-is-not-geometry`), and section 3 says why they are not the first +thing to spend on. + **Today:** the crop stage emits a flat panel list with a bbox each, plus `context_fragments`, a non-destructive caption-to-face link. Vision emits per-panel characters and dialogue. There is no -containment edge, no tail region and no region type. +containment edge, no region type, no `plane` and no `species`. + +**Done when:** every detection carries `plane` and `species`. On the labelled chapter, `order 17` and +`order 20` of the lead are not `story`, none of his 14 correct crops is demoted, and `p081` and `p108` are +`animal`. Measured by `audit_registry.py`, which already reads the vision blob per panel. ## 2. Identity exists independently of names @@ -53,43 +117,42 @@ occurrence c42 occurrence is the observation, the identity is the cluster, the name is an optional label on the cluster. Three levels, never collapsed into one. -**Today:** the schema already has this split. `identity_assignments` is the occurrence, -`characters` owns the identity, `name` is nullable and downstream already falls back to an anonymous -display. What is missing is the clustering, not the separation. See section 4. +**Today:** already true. `identity_assignments` is the occurrence, `characters` owns the identity, `name` +is nullable and downstream already falls back to an anonymous display. -## 3. Speaker attribution is a scored graph edge, not a procedure +**Done when:** already done. No work item. The clustering is section 4 and the naming is +`decisions/identity-naming.md`. + +## 3. Speaker attribution is a scored edge, not a procedure Do not write `find bubble -> find tail -> nearest character`. Score every plausible edge: ``` score(text, character) = - learned_t2c_score - + tail_evidence - + spatial_evidence - + same_panel - + dialogue_continuity - + character_activity_prior - + identity_context + w1 * gemma_answer + + w2 * spatial_evidence + + w3 * same_panel + + w4 * same_plane + + w5 * conversation_continuity + + w6 * character_activity_prior ``` -`learned_t2c_score` is the load-bearing term: a pair classifier over the whole page, the text object's -visual feature and the character object's visual feature. Magi's text-character head does exactly this. -It can start as a tiny MLP: +The original design put a learned `t2c` head in the first term and called it load-bearing. No labelled +pages exist, so that term does not. **Gemma's answer becomes one term of six rather than the whole +procedure.** The geometry terms overrule it when they agree against it. The weights are constants read off +the 30 labelled lines. Six numbers in a module, not a training run. -``` -t2c(text_embedding, character_embedding, page_context, geometry_features) -> p(speaker) -``` - -with geometry carrying normalized relative position, distance, overlap, same-panel, containment depth and -tail direction. +Geometry carries normalized relative position, distance, overlap, same-panel and containment depth. Tail +direction is absent until a tail region exists, and it is not the first thing to build. `conversation +continuity` is free, and turn-taking is the strongest prior for a tail-less bubble. Then the cases fall out of one mechanism instead of four: | case | what carries it | | --- | --- | -| bubble with a tail | `t2c` + tail, usually decisive | -| bubble with no tail | `t2c` + spatial and context | -| speaker outside the panel | recent identities + an offscreen candidate | +| bubble with a tail | gemma plus spatial, usually decisive | +| bubble with no tail | conversation continuity plus spatial | +| speaker outside the panel | recent identities plus an offscreen candidate | | narration | the narrator candidate | | nothing resolves | unknown speaker | @@ -102,8 +165,13 @@ speaker = visible(character_id) | offscreen(character_id?) | narrator | unknown **Today:** `speaker_ref` is already a typed union of `character_id | name | unknown | narrator` (`decisions/audit-phase1.md#speaker-ref-is-canonical`). `offscreen` is the missing arm. Attribution is a -prompt to gemma over a window of panels, with no geometry term at all. The `det`/`seg` tail heads exist -and are unused (`caveats/speaker-attribution.md#tail-is-not-geometry`). +prompt to gemma over a window of panels, with no geometry term at all. + +**Done when:** `audit_speakers.py` reports accuracy over the 30 labelled lines, split by true type, and +the scored version beats the recorded gemma-window baseline. Two numbers must move the right way, and both +are reported. Correct assignments go up. **Forced** errors go down, where forced means a line given a +visible character while the truth is `offscreen`, `narrator` or `unknown`. Record the baseline before +touching the code. ## 4. Character recognition is occurrence, then identity, then name @@ -112,257 +180,174 @@ character detection ↓ occurrence embeddings ↓ -pairwise same_identity probabilities +pairwise same_identity scores ↓ chapter-wide constrained clustering ↓ char_001, char_002, ... ↓ -optional character-bank lookup +optional name claim ↓ name or unknown ``` -Two rules that the current code gets wrong. +Three rules the current code gets wrong. -**The embedding is not the character crop alone.** Combine four signals: the character crop, the face or -head crop, the full-body crop, and a contextual object feature. Magiv2 combines detected object features -with a separate crop-embedding model. +**The embedding is not the character crop alone.** The crop embedding measures scene, not person, which is +why two men reach 0.93. Combine the face or head crop with the person crop instead of replacing one with +the other. `face_detect` already finds the face and pairs it for `has_face`, so the face box is free. This +is the queued experiment in `NEXT.md` item 1: re-embed the same 22 detections and recompute the matrix. -**Cluster chapter-wide, not page by page.** +**Cluster chapter-wide, not page by page.** `tracklets.link_tracklets` groups within an 8-panel window. -**Two characters in the same panel may be one person.** Mirrors, photographs, flashbacks, insets, -screens, imagined scenes and repeated action drawings all break that rule. Make it a weak cannot-link, -and only when the two are on the same narrative plane. +**Two characters in the same panel may be one person.** Seven things break that rule. Mirrors, +photographs, flashbacks, insets, screens, imagined scenes, repeated action drawings. Make it a weak +cannot-link, and only between detections on the same `plane`. -**Today:** the embedding is the person box only, which is measurably the wrong signal -(`caveats/audit-open.md#cosine-not-identity`). Clustering is greedy and local: `tracklets.link_tracklets` -groups within an 8-panel window. `tracklets.cannot_link` treats same-panel co-presence as a **hard** -constraint, which is exactly the correction above. Naming is `db.add_name_claim`, corroboration over -`name_claims`. +That last rule has an ordering trap. Same-panel co-presence is currently a **hard** constraint and it is +load-bearing precisely because cosine cannot separate people. Weakening it before the embedding improves +will regress purity. The dependency is the embedding fix, not the `plane` field alone. -## 5. The art-in-art problem needs a narrative plane +**Today:** the embedding is the person box only (`caveats/audit-open.md#cosine-not-identity`). Clustering +is greedy and local. `tracklets.cannot_link` treats same-panel co-presence as hard. Naming is +`db.add_name_claim`, corroboration over `name_claims`. -Treat the page as a hierarchical scene graph: +**Done when:** no labelled character holds more than one wrong assignment, and woman A's fragmentation is +1. Baseline is 2 wrong, 0 wrong, 2 wrong, and fragmentation 2. The bar is stated in errors rather than in +a purity ratio on purpose. The three characters hold 16, 13 and 9 assignments. At those counts any ratio +above 0.94 means zero tolerated errors, and the ratio hides that. + +The face-plus-person embedding lands first and carries its own smaller check. On the 22 measured +detections, the highest different-person pair must fall below the lowest same-person pair. + +## 5. The narrative plane is what stops art-in-art + +The page is a hierarchical scene graph: ``` page -└── panel A depth=0 +└── panel A plane=story ├── character c1 ├── text t1 - └── television/poster depth=1, type=embedded_art + └── television/poster plane=screen ├── character c2 └── text t2 ``` -Speaker candidates normally come from the same `scene_depth`. Otherwise a real character standing beside a -poster of a drawn person can be given the poster person's line. - -A region classifier predicts a type: - -``` -story_scene | inset_story_panel | flashback | screen | photo | poster | illustration | decorative -``` - -Perfect classification is not the point. The output that matters is one probability: +Perfect classification is not the point. The output that matters is one predicate: ``` same_narrative_plane(a, b) ``` -which then enters the association score in section 3 and the cannot-link in section 4. +It has exactly two consumers, and they are the reason the field is worth adding at all: + +- section 3, as the `same_plane` term. A real character beside a poster does not get the poster person's + line. +- section 4, as the guard that makes the weak cannot-link safe. **Today:** nothing models this, and it is the whole of the remaining identity error on the lead. On the -19:44 run of 2026-08-12 his 16 assignments were 14 correct plus a photograph of another man and a chibi -drawing. Both are art inside a panel. Vision also boxes cats as people and dresses them (`p081`, `p108`). +19:44 run his 16 assignments were 14 correct plus a photograph of another man and a chibi drawing. Both +are art inside a panel. -## 6. Narrative understanding is a state machine, not a per-panel description +**Done when:** section 1's DoD, plus both consumers wired, plus section 4's purity DoD holds with the +cannot-link demoted to weak. If purity regresses when the constraint is weakened, the embedding is not +ready and the demotion reverts. -``` -story_state -├─ entities (characters, locations, important objects) -├─ scenes -├─ timeline -├─ relationships -├─ unresolved_threads -├─ facts -└─ hypotheses -``` +## 6. Narrative understanding is carried state, not a per-panel description -A panel produces a **delta**, not another standalone prose interpretation: +Today each stage reads its predecessor's blob for one panel or one beat. `recent`, a rolling list of the +last few dialogue lines, is the only carried state. That is the root of the invented narration. -``` -panel 142: -- character_07 enters room_03 -- character_02 is already present -- character_07 says "..." -- object_12 changes owner: 02 -> 07 -- possible flashback begins -``` - -### Facts, hypotheses and unknowns are different records - -``` -fact: source=panel_142 confidence=0.99 character_07 is visible -hypothesis: confidence=0.64 character_07 is angry -unknown: who caused the explosion -``` - -A later panel strengthens, replaces or invalidates a hypothesis without rewriting history. - -### Scene state is explicit and inherited +The version worth building is one record per scene, inherited forward: ``` scene_31: location: school_rooftop time: evening participants: {char_03: present, char_07: present, char_11: offscreen} - pov: null narrative_mode: present - parent_scene: null + last_speaker: char_07 + addressee: char_02 ``` A panel inherits this unless visual evidence overrides it. That alone kills a class of errors. A character absent for one panel has not left. A panel with no background has not changed location. A tail-less line -keeps the offscreen participant as a candidate. A close-up still belongs to the scene. +keeps the offscreen participant as a candidate. A close-up still belongs to the scene. `last_speaker` and +`participants` are what section 3's continuity term reads. -### Classify the transition, not just the panel +A panel produces a **delta** against that record, not another standalone prose interpretation: + +``` +panel 142: +- character_07 enters room_03 +- character_02 is already present +- character_07 says "..." +``` + +Classify the transition, not just the panel: ``` CONTINUE_SCENE | NEW_SCENE | LOCATION_CHANGE | TIME_SKIP | FLASHBACK_START FLASHBACK_END | DREAM/IMAGINATION | POV_CHANGE | EMBEDDED_SCENE ``` -`EMBEDDED_SCENE` is what stops a television's contents mutating the room around it: +`EMBEDDED_SCENE` is `plane != story` at scene granularity, and is what stops a television's contents +mutating the room around it. + +**Scene state is written by a resolver, never by the vision model.** The path is +`observation -> resolver -> state transition`. The resolver may reject an impossible update. It is plain +Python over gemma's per-panel delta, and it is where the constraint lives. + +**Today:** none of it exists. Chapter boundaries are a reset. Narration asserts things no panel shows +(`NEXT.md` item 6). + +**Done when:** a scene record carries location, participants and `narrative_mode` across panels. A +character absent from one panel stays a participant. On the next full run the four invented-fact +timestamps do not recur. Those are 0:43, 2:03, 2:05 and 2:15, and they are the regression list. The +correctness verifier passed 116/116 over them because it checks quotes and names, never invented claims. +So the check is a re-watch of those four points, not a stage counter. + +### Not building yet + +Each of these was in the original design. Each is deferred with a trigger, not dropped. + +| deferred | trigger to revisit | +| --- | --- | +| Facts, hypotheses and unknowns as separate records with confidences | when scene state exists and narration still asserts unshown claims | +| The seven-check consistency checker | when a scene record exists for it to check against | +| Unresolved references that survive and back-propagate | when a second chapter of the same manga runs | +| Chapter checkpoints and the two-memory split | when a second chapter of the same manga runs | + +One reason covers all four. They sit on an identity layer still wrong on 2 of the lead's 16 crops. State +machinery over wrong identity produces confidently wrong state. + +## 7. Build order + +Detection, vision and character embeddings already exist. The order below is chosen so each step is +falsifiable by the step's own DoD before the next one starts. ``` -scene_12 present - ├─ panel 101 - ├─ panel 102 - └─ embedded scene_13 [television] - ├─ panel-like region - └─ char_19 +0. eval/chapter-truth.json + purity and fragmentation in audit_registry.py +1. plane + species per detection -> section 1 DoD +2. face-plus-person embedding -> section 4 embedding check +3. chapter-wide clustering, weak cannot-link on plane -> section 4 purity DoD +4. scene record carried forward -> section 6 DoD +5. scored speaker edge, offscreen arm -> section 3 DoD +6. tail regions from the unused det/seg heads -> only if 5 misses its DoD ``` -### Character state is written by a resolver, never by the vision model +Steps 1 and 2 are independent and can land together. Step 3 depends on 2, which is the ordering trap in +section 4. Step 5 depends on 4, because the continuity term reads the scene record. Step 6 is +conditional on purpose: build a tail detector only after the cheap terms have been measured and found +insufficient. -``` -char_07: - known_names: [...] - currently_at: room_03 - status: alive - appearance_state: {clothes: school_uniform, injured: true} - relationships: {char_02: friend?} - last_seen: panel_142 -``` - -The path is `observation -> resolver -> state transition`, and the resolver may reject an impossible -update. - -### Conversation state is its own record - -``` -conversation_18: - scene: scene_31 - participants: [char_02, char_07] - last_speaker: char_07 - addressee: char_02 - topic: missing_key -``` - -This is the strongest available prior for a tail-less bubble. Given `A: where did you put it? / ... / -A: don't lie.`, turn-taking assigns the middle line with no visual evidence at all. - -### An unresolved reference survives instead of being forced - -``` -unknown_04: - type: person - descriptions: ["the man from yesterday", "silhouette in panel_58"] - candidate_ids: {char_12: 0.55, char_19: 0.22} -``` - -Chapter 6 may reveal `unknown_04 == char_12`, and that identity back-propagates through the graph. The -same applies to unnamed characters, pronouns, disguised characters, mysterious objects and unseen -speakers. - -### Two memories - -- **Working narrative state**: the current scene and the recent ones, in detail. -- **Canonical long-term memory**: compressed facts, not chapter summaries. `char_07 learned that char_02 - betrayed the group.` `object_04 is held by char_11.` `char_03 does not know char_07 survived.` - -### A chapter boundary is a checkpoint, not a reset - -``` -chapter_checkpoint: - persistent_entity_changes / relationship_changes / location and status changes - newly established facts / unresolved questions / active plot threads / final scene state -``` - -Chapter `n+1` starts from that. The detailed panel graph may be kept forever. Only five things load into -the model: the current scene, the previous scene, the relevant character records, the active threads, and -retrieved old facts. - -### A consistency checker runs after each scene and chapter - -Seven checks. A dead character appearing normally. A character knowing a fact before learning it. An -object owned by two people at once. A flashback never closed. A location jump with no transition. A -speaker who was neither present nor offscreen. A name that conflicts with the identity graph. The model -proposes corrections. The graph stays the source of truth. - -**Today:** none of this exists. Each stage reads its predecessor's blob for one panel or one beat. -`recent` is a rolling list of the last few dialogue lines and is the only carried state. Chapter -boundaries are a reset. There is no fact-versus-hypothesis distinction anywhere, which is why narration -asserts things no panel shows (`NEXT.md` item 6). - -## 7. The staged version worth building - -Do not recreate Magi's monolithic network first. Detection, vision and character embeddings already exist, -so stage it: - -``` -page - ↓ -region detector panels / nested regions / texts / characters / tails - ↓ -object feature extraction - ↓ -three pair models character↔character (identity) - text→character (speaker) - text→tail (bubble structure) - ↓ -chapter graph - ↓ -global character clustering - ↓ -optional naming - ↓ -ocr + reading order - ↓ -dialogue stream -``` - -The VLM then judges only the ambiguous graph edges. It no longer rediscovers every character and dialogue -relationship from raw pixels on every panel. Magi formulates detection and association as graph -generation, which is why it beats a crop, OCR and nearest-character pipeline here. - -## What to take from this before the rewrite - -Three items are cheap against the current code and pay immediately. They are entered in `NEXT.md`, not -here. - -1. **`same_narrative_plane`, as a per-detection field.** Vision already returns per-panel boxes. Add a - `plane` or `depth` to a detection, set when the model says the figure sits inside a screen, poster, - photo or drawing. That buys the containment edge with no detector. It is the whole of the remaining - identity error on the lead, and it feeds every stage below. -2. **Same-panel co-presence becomes a weak cannot-link.** `tracklets.cannot_link` currently makes it hard. - It needs item 1 first, because the plane is what makes the weak version safe. -3. **`offscreen` as a fourth `speaker_ref` kind.** The union already exists, the arm does not. +The VLM keeps judging ambiguous edges. What changes is that it stops rediscovering every character and +dialogue relationship from raw pixels on every panel. The chapter graph carries the answer forward. ## Sources -Magi and Magiv2 for the detection-and-association-as-graph-generation formulation, the text-character -pair head, and the character bank of exemplar images plus names. Magiv3 for panels, texts, characters and -tails with their associations, and for character grounding between textual descriptions and detected -character regions. +Three ideas come from published comics-transcription work. Detection and association as graph generation, +the text-to-character pair head, and the character bank of exemplar images plus names. The +formulation is kept. The models are ruled out, see **Ruled out** above. No source in this document is a +runtime dependency. diff --git a/NEXT.md b/NEXT.md index b61b241..64ba814 100644 --- a/NEXT.md +++ b/NEXT.md @@ -71,9 +71,15 @@ contain the right person. Coverage is still the `has_face` gate plus those refus face touching the bottom edge of one fragment with a textless fragment below. No evidence has been gathered yet on how often this chapter does it. -1c. **Three items from `ARCHITECTURE.md` are cheap against the current code.** A `plane` field per - detection for art-in-art, same-panel co-presence demoted to a weak cannot-link once the plane exists, - and `offscreen` as a fourth `speaker_ref` kind. +1c. **`ARCHITECTURE.md` was rewritten on 2026-08-13 under the no-Magi constraint.** Every section now + carries a DoD. Its build order starts at step 0, `eval/chapter-truth.json` plus purity and + fragmentation printed by `audit_registry.py`. Nothing below it is measurable without that file. + + Steps 1 and 2 are independent and cheap: `plane` and `species` per detection, and the + face-plus-person embedding already queued in item 1 above. The weak cannot-link is step 3 and depends + on step 2, not on `plane` alone. Co-presence as a hard constraint is load-bearing while cosine cannot + separate people, so demoting it early regresses purity. `offscreen` as a fourth `speaker_ref` kind is + still cheap and independent. 2. ~~**Order the corners in `_bbox_to_pixels`.**~~ **Done 2026-08-12, run and verified on a GPU.** 0 degenerate boxes over 119 detections (`decisions/identity-bbox.md#bbox-corners-ordered`). -- 2.52.0 From be63b2247d569c2ee14a68c8fd3ee9755b53ae23 Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 22:34:00 +0400 Subject: [PATCH 29/31] Delete a 0-byte file a mis-quoted redirect created The name is a shell fragment, `II, d[i+4:i+12]))\n"; done`, left by a redirect in the A/V gap session and committed with 8113bdf. The file is empty. Co-Authored-By: Claude Opus 5 --- "II, d[i+4:i+12]))\n\"; done" | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 "II, d[i+4:i+12]))\n\"; done" diff --git "a/II, d[i+4:i+12]))\n\"; done" "b/II, d[i+4:i+12]))\n\"; done" deleted file mode 100644 index e69de29..0000000 -- 2.52.0 From bec9411af3bd652a778c42a9e81380fd47dc04d2 Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 23:02:06 +0400 Subject: [PATCH 30/31] Put every S3 URI in one place, and add a lint gate Five workers built output URIs with inline f-strings, so the bucket-per-artifact layout was spread across worker_tts, worker_identity, worker_crop, worker_layers and worker_render. Moving a class between buckets meant a grep. They are now templates in transport.py, formatted at each call site. Three of those workers also each reimplemented the same parse to recover manga_id and chapter_id from an input uri, because the orchestrator does not send them. That is transport.ids_from_uri now, and it raises on a uri too short to carry the ids rather than returning a wrong pair. ruff.toml makes `ruff check .` exit 0, so CI can gate on it and a new finding means a new defect. Fixed: an implicit Optional in 8 signatures, an unparenthesized implicit concatenation in the ASS filter list, 5 subprocess.run calls now saying check=False out loud, an unused import, a duplicate exception handler and a non-executable shebang. Every rule left off carries its reason in ruff.toml. The ASYNC rules are off because ffmpeg on the event loop is real and already recorded at caveats/audit-open.md#blocking-event-loop. It needs a refactor per handler, not a lint fix. Checked: transport, collage, bubble_detect, test_vision_parse, worker_crop, worker_scene, worker_script, worker_identity, worker_tts, session_manager, worker_vision and worker_render self-checks all pass. worker_layers still fails on a missing legacy/qwen_layered_workflow.json, which predates this branch. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 6 +++- audit_registry.py | 2 +- bubble_detect.py | 2 +- face_detect.py | 2 +- ruff.toml | 21 +++++++++++++ scripts/pick_tts_voice.py | 2 +- session_manager.py | 2 +- test_vision_parse.py | 2 +- transport.py | 47 ++++++++++++++++++++++++++-- worker_crop.py | 5 +-- worker_identity.py | 8 ++--- worker_layers.py | 6 ++-- worker_render.py | 64 +++++++++++++++++++-------------------- worker_tts.py | 16 +++++++--- worker_vision.py | 4 +-- 15 files changed, 131 insertions(+), 58 deletions(-) create mode 100644 ruff.toml mode change 100644 => 100755 scripts/pick_tts_voice.py diff --git a/CLAUDE.md b/CLAUDE.md index 34e92fc..ad45cd5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -61,14 +61,18 @@ sudo systemd/install.sh # production: one systemd unit per process (User .venv/bin/python worker_scene.py # every module has an assert-based __main__ self-check .venv/bin/python test_vision_parse.py +ruff check . # must exit 0; every ignore in ruff.toml carries its reason cd /mnt/server/home/kami/docker-apps/manga-infra/orchestrator && pytest -q --ignore=test_api.py ``` -There is no lint or build step. `.venv` is the ROCm torch env. Workers import `transport` by module +There is no build step. `.venv` is the ROCm torch env. Workers import `transport` by module name. Ports: crop 8000, vision 8002, identity 8003, scene 8004, script 8005, tts 8006, layers 8007, render 8008, session_manager 8095. +Every output S3 URI is a template in `transport.py`, not an f-string in a worker. Add one there when a +new artifact class appears. + - Any non-trivial logic gets ONE runnable check in `__main__`, assert-based, no framework. Run the file to verify it. - Editing a worker's request or response shape means editing the orchestrator too, in the same session. diff --git a/audit_registry.py b/audit_registry.py index 4c88149..c3e1948 100644 --- a/audit_registry.py +++ b/audit_registry.py @@ -87,7 +87,7 @@ if worked: for ch in people: cid, conf = assigns.get(ch["local_id"], (None, None)) name = reg.get(cid, {}).get("name") or (cid[:16] if cid else "-- none --") - print(f" {ch['local_id']:10} {str(ch['bbox']):28} {name:22} " + print(f" {ch['local_id']:10} {ch['bbox']!s:28} {name:22} " f"{'' if conf is None else f'{conf:.2f}'}") else: print(f"\npanel_index {WORKED_EXAMPLE} not found in this chapter") diff --git a/bubble_detect.py b/bubble_detect.py index d0cf8d3..e86f052 100644 --- a/bubble_detect.py +++ b/bubble_detect.py @@ -38,7 +38,7 @@ def _letterbox(img, sz=1024): return canvas, r, px, py -def detect_text_regions(img, conf: float = None) -> list: +def detect_text_regions(img, conf: float | None = None) -> list: """img: BGR ndarray (one panel). -> [{"bbox":[x1,y1,x2,y2], "conf":float}] in ORIGINAL pixels, reading order (top->bottom, left->right). Empty list on no detections (caller falls back to the holistic prompt). Load + inference are lazy so importing this never touches the GPU or the model.""" diff --git a/face_detect.py b/face_detect.py index c065688..f6c787d 100644 --- a/face_detect.py +++ b/face_detect.py @@ -32,7 +32,7 @@ def _letterbox(img, sz=640): return canvas, r, px, py -def detect_faces(img, conf: float = None) -> list: +def detect_faces(img, conf: float | None = None) -> list: """img: BGR ndarray. -> [{"bbox":[x1,y1,x2,y2], "conf":float}] in ORIGINAL pixels, sorted reading order. Empty on no faces. Lazy load so import never touches the model.""" conf = CONF if conf is None else conf diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..878b664 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,21 @@ +# Lint gate. `ruff check .` must exit 0, so CI can gate on it and a new finding means a new defect. +# +# Ruff's defaults flag about 100 things in this repo. Most are deliberate style in workers that must +# survive one bad panel rather than fail clean. Every rule turned off below carries its reason, so an +# ignore stays a decision rather than a shrug. Delete an entry the moment its reason stops holding. + +[lint] +ignore = [ + "I001", # import order: 25 files of churn, no behaviour change + "BLE001", # a worker catches Exception on purpose, so one bad panel cannot kill the stage + "SIM115", # short-lived open().read(); the handle drops with the refcount + "S110", # try/except/pass in best-effort cleanup, where the no-op IS the handling + "ASYNC210", # ffmpeg, ffprobe and MinIO run synchronously inside async endpoints. Real, and + "ASYNC221", # already recorded at caveats/audit-open.md#blocking-event-loop with [#199]. The fix + "ASYNC230", # is `def` over `async def` per handler, which is a refactor and not a lint fix. + "RUF046", # int(round(v)) says "pixels" out loud in the render geometry + "UP031", # the ASS subtitle template is %-formatted; f-string braces collide with its {\an} tags + "RUF059", # unpacking a whole bbox and using half of it beats indexing into it + "RUF007", # zip(x, x[1:]) reads better here than itertools.pairwise + "PLC3002", # one immediately-called lambda, in an audit script +] diff --git a/scripts/pick_tts_voice.py b/scripts/pick_tts_voice.py old mode 100644 new mode 100755 index f438fa6..96bef55 --- a/scripts/pick_tts_voice.py +++ b/scripts/pick_tts_voice.py @@ -27,7 +27,7 @@ def load_model(model_name: str): ) # Match worker_tts.py's torch/torchaudio minor-mismatch bypass too. import torch - import importlib.metadata as metadata + from importlib import metadata real_version = metadata.version metadata.version = ( diff --git a/session_manager.py b/session_manager.py index da3e9fc..b47efac 100644 --- a/session_manager.py +++ b/session_manager.py @@ -6,7 +6,7 @@ # in-process models (siglip2, dots): no server — the lease just reserves the GPU; the # worker loads the transformers model itself after /session/open returns (port=None). # the guarantee that matters is the mutex: a second open() gets 409 until the first closes. -import os, time, uuid, threading, subprocess +import time, uuid, threading, subprocess import requests from fastapi import FastAPI, HTTPException from pydantic import BaseModel diff --git a/test_vision_parse.py b/test_vision_parse.py index 572d2c5..f315edf 100644 --- a/test_vision_parse.py +++ b/test_vision_parse.py @@ -14,7 +14,7 @@ def test_extract_rejects_malformed_and_truncated(): try: wv._extract_json(bad) assert False, f"should have raised on: {bad!r}" - except (ValueError, ValueError): + except ValueError: pass diff --git a/transport.py b/transport.py index 50380c7..938eed9 100644 --- a/transport.py +++ b/transport.py @@ -12,6 +12,29 @@ from starlette.responses import Response _client = None +# --- artifact layout --------------------------------------------------------------------------- +# one bucket per artifact class (`decisions/storage-layout.md#bucket-per-artifact`). Every worker +# formats its output uri from these, so moving a class between buckets is one edit here rather than +# a grep across five workers. `name` is the panel id, or 'p' when a worker has none. +PANEL_URI = "s3://panels/{manga_id}/{chapter_id}/panels/p{idx:03d}.png" +PAGE_PANEL_URI = "s3://panels/{manga_id}/{chapter_id}/panels/pg{page_index:03d}_p{idx:02d}.png" +AUDIO_URI = "s3://audio/{manga_id}/{chapter_id}/audio/{name}.wav" +AUDIO_FLAT_URI = "s3://audio/_audio/{name}.wav" +LAYER_URI = "s3://layers/{manga_id}/{chapter_id}/layers/{name}/{idx}.png" +CLIP_URI = "s3://video/{manga_id}/{chapter_id}/clips/{name}.mp4" +CHAPTER_URI = "s3://video/{manga_id}/{chapter_id}/chapter.mp4" +CHAR_PNG_URI = "s3://manga/{key}.png" +CHAR_NPY_URI = "s3://manga/{key}.npy" + + +def ids_from_uri(uri: str): + """(manga_id, chapter_id) from any artifact uri: ///... + the orchestrator passes no ids to tts, layers or render, but every input uri encodes them.""" + parts = (uri.removeprefix("s3://")).split("/") + if len(parts) < 3: + raise ValueError(f"uri carries no manga/chapter: {uri!r}") + return parts[1], parts[2] + def _summarize(body: bytes, limit=6) -> str: """compact one-line view of a json body for observability: uri inputs/outputs (basename, or @@ -27,9 +50,9 @@ def _summarize(body: bytes, limit=6) -> str: for k, v in obj.items(): if k == "panel_id": continue - if isinstance(v, str) and (k.endswith("uri") or k.endswith("url")): + if isinstance(v, str) and (k.endswith(("uri", "url"))): parts.append(f"{k}={v.rsplit('/', 1)[-1]}") - elif isinstance(v, list) and v and isinstance(v[0], str) and (k.endswith("uris") or k.endswith("urls")): + elif isinstance(v, list) and v and isinstance(v[0], str) and (k.endswith(("uris", "urls"))): parts.append(f"{k}×{len(v)}") elif isinstance(v, (int, float, bool)): parts.append(f"{k}={v}") @@ -102,7 +125,7 @@ def _mc(): def _split(uri: str): """(bucket, key) from an s3-style or bare uri.""" - u = uri[5:] if uri.startswith("s3://") else uri + u = uri.removeprefix("s3://") bucket, _, key = u.partition("/") if not bucket or not key: raise ValueError(f"bad uri: {uri!r}") @@ -213,4 +236,22 @@ if __name__ == "__main__": assert _summarize(b"clip_uri", ) == "-" # non-json assert _summarize(json.dumps({"clip_uri": "s3://m/c/clips/p1.mp4", "duration": 4.1}).encode()) \ == "clip_uri=p1.mp4 duration=4.1" + + # artifact layout: templates format to the keys the workers wrote by hand before, and + # ids_from_uri recovers the ids the orchestrator never sends. + panel = PANEL_URI.format(manga_id="m1", chapter_id="c1", idx=7) + assert panel == "s3://panels/m1/c1/panels/p007.png", panel + assert PAGE_PANEL_URI.format(manga_id="m1", chapter_id="c1", page_index=2, idx=3) \ + == "s3://panels/m1/c1/panels/pg002_p03.png" + assert CLIP_URI.format(manga_id="m1", chapter_id="c1", name="p003") \ + == "s3://video/m1/c1/clips/p003.mp4" + assert LAYER_URI.format(manga_id="m1", chapter_id="c1", name="p003", idx=0) \ + == "s3://layers/m1/c1/layers/p003/0.png" + assert ids_from_uri(panel) == ("m1", "c1") + assert ids_from_uri("panels/m1/c1/panels/p007.png") == ("m1", "c1") + try: + ids_from_uri("s3://panels/p007.png") + raise AssertionError("a uri with no chapter segment must raise") + except ValueError: + pass print("transport self-check ok") diff --git a/worker_crop.py b/worker_crop.py index e0ec4ee..1d8eca6 100644 --- a/worker_crop.py +++ b/worker_crop.py @@ -244,7 +244,7 @@ async def crop_webtoon(data: WebtoonInput): context_links = context_fragment_links(crops) panels = [] for idx, (crop_img, bbox) in enumerate(crops): - uri = f"s3://panels/{data.manga_id}/{data.chapter_id}/panels/p{idx:03d}.png" + uri = transport.PANEL_URI.format(manga_id=data.manga_id, chapter_id=data.chapter_id, idx=idx) # slicing is deterministic, so on a resume the same idx -> same key; skip re-upload. # TRAP: that holds only while the PLAN is unchanged. Edit slice_webtoon or the merge pass and a # re-crop silently keeps the previous run's images under the same keys, because every one of them @@ -277,7 +277,8 @@ async def crop(data: CropInput): for idx, (crop_img, bbox) in enumerate(crops): out = f"{SHM}/pg{data.page_index:03d}_p{idx:02d}.png" cv2.imwrite(out, crop_img) - uri = f"s3://panels/{data.manga_id}/{data.chapter_id}/panels/pg{data.page_index:03d}_p{idx:02d}.png" + uri = transport.PAGE_PANEL_URI.format(manga_id=data.manga_id, chapter_id=data.chapter_id, + page_index=data.page_index, idx=idx) transport.put(out, uri) os.remove(out) panels.append({"panel_index": idx, "uri": uri, "bbox": bbox, diff --git a/worker_identity.py b/worker_identity.py index 477412e..e61b23f 100644 --- a/worker_identity.py +++ b/worker_identity.py @@ -78,7 +78,7 @@ def match(emb: np.ndarray, known: list, threshold: float): return None, best_conf, ambiguous -def shortlist(emb: np.ndarray, known: list, k: int = 5, gender: str = None) -> list: +def shortlist(emb: np.ndarray, known: list, k: int = 5, gender: str | None = None) -> list: """TIER-2 evidence: the top-k gender-gated known characters by cosine, best first. Cosine is now a SHORTLISTER, not the decider — gemma /vision/resolve picks from this list. Each item carries the full row (name/gender/description) so the resolver can build a text character-sheet. pure.""" @@ -104,7 +104,7 @@ def _save_npy(emb: np.ndarray, uri: str): os.remove(tmp) -def _pending_match(pend: list, emb, threshold: float, gender: str = None): +def _pending_match(pend: list, emb, threshold: float, gender: str | None = None): """index of the pending entry this embedding belongs to, or None (a new provisional). candidates of a conflicting decided gender are excluded. character_id=i keeps the returned index original. pure.""" cands = [{"character_id": i, "embedding": e["emb"]} @@ -117,7 +117,7 @@ def _persist_char(manga_id, panel_id, local_id, crop, emb, name, gender, appeara """upload crop + embedding to S3 and register the row via the orchestrator; return its id.""" import cv2 key = f"{manga_id}/characters/_new/{panel_id}_{local_id}" - ref_img_uri, emb_uri = f"s3://manga/{key}.png", f"s3://manga/{key}.npy" + ref_img_uri, emb_uri = transport.CHAR_PNG_URI.format(key=key), transport.CHAR_NPY_URI.format(key=key) ref_png = f"{SHM}/crop_{uuid.uuid4().hex[:8]}.png" cv2.imwrite(ref_png, crop) transport.put(ref_png, ref_img_uri) @@ -220,7 +220,7 @@ async def resolve(data: IdentityInput): # Uploading it now is what removes the third siglip pass # (`decisions/identity-bbox.md#none-mints-an-anonymous-character`). key = f"{data.manga_id}/characters/_crops/{data.panel_id}_{ch['local_id']}" - crop_uri, emb_uri = f"s3://manga/{key}.png", f"s3://manga/{key}.npy" + crop_uri, emb_uri = transport.CHAR_PNG_URI.format(key=key), transport.CHAR_NPY_URI.format(key=key) cp = f"{SHM}/cc_{uuid.uuid4().hex[:8]}.png"; cv2.imwrite(cp, crop) transport.put(cp, crop_uri); os.remove(cp) _save_npy(emb, emb_uri) diff --git a/worker_layers.py b/worker_layers.py index 03cda30..7c98f01 100644 --- a/worker_layers.py +++ b/worker_layers.py @@ -65,15 +65,15 @@ async def layers(data: LayerInput): return {"layer_uris": [], "skipped": "comfyui down"} # ponytail: skip stage if ComfyUI not running local = transport.get(data.panel_uri, f"{SHM}/layer_{uuid.uuid4().hex[:8]}.png") # orchestrator doesn't pass manga/chapter to layers; derive from the panel uri prefix. - parts = data.panel_uri.replace("s3://", "").split("/") - manga_id, chapter_id = parts[1], parts[2] + manga_id, chapter_id = transport.ids_from_uri(data.panel_uri) view_urls = _comfy_run(local, data.num_layers, data.prompt) layer_uris = [] for idx, url in enumerate(view_urls): png = f"{SHM}/layer_{uuid.uuid4().hex[:6]}.png" with open(png, "wb") as f: f.write(requests.get(url, timeout=60).content) - uri = f"s3://layers/{manga_id}/{chapter_id}/layers/{data.panel_id or 'p'}/{idx}.png" + uri = transport.LAYER_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p', idx=idx) transport.put(png, uri) os.remove(png) layer_uris.append(uri) diff --git a/worker_render.py b/worker_render.py index 5a0f89c..f3b8c57 100644 --- a/worker_render.py +++ b/worker_render.py @@ -19,13 +19,6 @@ MUSIC_BED = os.environ.get("MUSIC_BED", "") # #13 path/uri of a music t MUSIC_GAIN = os.environ.get("MUSIC_GAIN", "0.18") # bed level before ducking -def _mc_from_uri(uri: str): - """(manga_id, chapter_id) from s3://///... - the orchestrator doesn't pass ids to render/layers, but every input uri encodes them.""" - parts = uri.replace("s3://", "").split("/") - return parts[1], parts[2] - - def _ts(s): h = int(s // 3600); m = int(s % 3600 // 60); sec = s % 60 return f"{h}:{m:02d}:{sec:05.2f}" @@ -37,10 +30,10 @@ def _ts(s): SUB_MODE = os.environ.get("SUB_MODE", "minimal").lower() # off | minimal | boxed SUB_PRESETS = { # (mode, orientation): fontsize, borderstyle(1=outline,3=box), outline/pad, shadow, marginV, side - ("minimal", "portrait"): dict(fs=40, bs=1, outline=3, shadow=2, mv=0.055, side=110), - ("minimal", "landscape"): dict(fs=32, bs=1, outline=3, shadow=2, mv=0.09, side=260), - ("boxed", "portrait"): dict(fs=40, bs=3, outline=6, shadow=0, mv=0.055, side=110), - ("boxed", "landscape"): dict(fs=32, bs=3, outline=6, shadow=0, mv=0.09, side=260), + ("minimal", "portrait"): {"fs": 40, "bs": 1, "outline": 3, "shadow": 2, "mv": 0.055, "side": 110}, + ("minimal", "landscape"): {"fs": 32, "bs": 1, "outline": 3, "shadow": 2, "mv": 0.09, "side": 260}, + ("boxed", "portrait"): {"fs": 40, "bs": 3, "outline": 6, "shadow": 0, "mv": 0.055, "side": 110}, + ("boxed", "landscape"): {"fs": 32, "bs": 3, "outline": 6, "shadow": 0, "mv": 0.09, "side": 260}, } @@ -109,7 +102,7 @@ def _ass(text: str, dur: float, path: str): def _audio_dur(path: str) -> float: """clip length = narration length. scene_timing arrives empty, so probe the audio.""" r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration", - "-of", "default=nk=1:nw=1", path], capture_output=True, text=True) + "-of", "default=nk=1:nw=1", path], capture_output=True, text=True, check=False) try: return float(r.stdout.strip()) except ValueError: @@ -120,7 +113,7 @@ def _stream_dur(path: str, kind: str) -> float: """duration of one stream. `format=duration` is max(video,audio) and so hides A/V drift.""" r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", f"{kind}:0", "-show_entries", "stream=duration", "-of", "default=nk=1:nw=1", path], - capture_output=True, text=True) + capture_output=True, text=True, check=False) try: return float(r.stdout.strip()) except ValueError: @@ -132,7 +125,7 @@ def _fps_of(path: str) -> str: their rate AND therefore their time_base agree, and the stream-copy concat path cares about that.""" r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", "stream=r_frame_rate", "-of", "default=nk=1:nw=1", path], - capture_output=True, text=True) + capture_output=True, text=True, check=False) return r.stdout.strip() @@ -184,7 +177,7 @@ def _motion(camera: dict, frames: int) -> str: return f"zoompan=z='{z}':x='{x}':y='{y}':d={frames}:s={W}x{H}:fps={FPS}" -def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict = None, +def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict | None = None, pad: float = PAD_S) -> list: """ffmpeg: still panel over a blurred fill of itself + content-aware motion, burned subs, 9:16. #5 blurred bg replaces black bars: one copy scaled to COVER + blurred, the fitted panel on top. @@ -231,8 +224,9 @@ async def render_scene(data: SceneInput): _ass(data.narration_text, dur, ass) out = f"{SHM}/rnd_{tag}.mp4" subprocess.run(scene_cmd(img, audio, ass, out, dur, data.camera), check=True, capture_output=True) - manga_id, chapter_id = _mc_from_uri(data.panel_uri) - uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + manga_id, chapter_id = transport.ids_from_uri(data.panel_uri) + uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p') transport.put(out, uri) for p in (img, audio, ass, out): os.remove(p) @@ -341,7 +335,7 @@ class CompositeInput(BaseModel): def _img_size(path: str) -> tuple: r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", - "stream=width,height", "-of", "csv=p=0:s=x", path], capture_output=True, text=True) + "stream=width,height", "-of", "csv=p=0:s=x", path], capture_output=True, text=True, check=False) w, h = r.stdout.strip().split("x") return int(w), int(h) @@ -377,8 +371,9 @@ async def render_composite(data: CompositeInput): subprocess.run(composite_cmd(still, auds, ass, segs, rowh, out, t), check=True, capture_output=True) for p in imgs + [still]: os.remove(p) - manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"]) - uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + manga_id, chapter_id = transport.ids_from_uri(data.panels[0]["panel_uri"]) + uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p') transport.put(out, uri) for p in auds + [ass, out]: os.remove(p) @@ -424,8 +419,9 @@ async def render_group(data: GroupInput): "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", final] subprocess.run(cmd, check=True, capture_output=True) - manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"]) - uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + manga_id, chapter_id = transport.ids_from_uri(data.panels[0]["panel_uri"]) + uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p') transport.put(final, uri) total = _audio_dur(final) or sum(durs) for f in cleanup + [final]: @@ -449,7 +445,7 @@ class BeatInput(BaseModel): BEAT_MIN_PANEL_S = 1.0 # 185: no member panel flashes by faster than this -def _beat_slices(D: float, n: int, weights: list = None) -> list: +def _beat_slices(D: float, n: int, weights: list | None = None) -> list: """185: split beat duration D across n member panels by content weight, not evenly. Each panel gets >= BEAT_MIN_PANEL_S so a low-weight panel never flashes. Degenerate input (no/short weights, non-positive sum, or D too small for the floors) -> deterministic equal split. @@ -501,8 +497,8 @@ def cue_plan(text: str, D: float, slices: list) -> list: return events -def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list = None, - weights: list = None) -> list: +def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list | None = None, + weights: list | None = None) -> list: """ffmpeg: N images shown as a ken-burns montage under ONE narration audio; the last image holds through the trailing pad. each image = blurred-fill bg + fitted panel + its camera move; the single burned subtitle (in `ass`) spans the whole beat. 185: per-image screen-time is content-weighted @@ -561,8 +557,9 @@ async def render_beat(data: BeatInput): subprocess.run(beat_cmd(imgs, audio, ass, out, D, data.cameras, data.weights), check=True, capture_output=True) - manga_id, chapter_id = _mc_from_uri(data.panel_uris[0]) - uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + manga_id, chapter_id = transport.ids_from_uri(data.panel_uris[0]) + uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p') transport.put(out, uri) total = _audio_dur(out) or (D + PAD_S) for f in imgs + [audio, ass, out]: @@ -603,8 +600,8 @@ def collage_cmd(imgs, plate_i, rects, entrances, z_order, audio, ass, out, D, tr for img in imgs: cmd += ["-loop", "1", "-i", img] # still held for the whole clip; -t caps output cmd += ["-i", audio] - parts = [f"[{plate_i}:v]scale={W}:{H}:force_original_aspect_ratio=increase," - f"crop={W}:{H},boxblur=24:2,setsar=1[bg]"] + parts = [(f"[{plate_i}:v]scale={W}:{H}:force_original_aspect_ratio=increase," + f"crop={W}:{H},boxblur=24:2,setsar=1[bg]")] base = "bg" for k, i in enumerate(z_order): # shadows first, at rest positions (static) x, y, w, h = (int(round(v)) for v in rects[i]) @@ -661,8 +658,9 @@ async def render_collage(data: CollageInput): out = f"{SHM}/cl_{tag}.mp4" subprocess.run(collage_cmd(imgs, active, lay["rects"], lay["entrances"], lay["z_order"], audio, ass, out, D, lay["transition_s"]), check=True, capture_output=True) - manga_id, chapter_id = _mc_from_uri(uris[0]) - uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4" + manga_id, chapter_id = transport.ids_from_uri(uris[0]) + uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p') transport.put(out, uri) total = _audio_dur(out) or (D + PAD_S) for f in imgs + [audio, ass, out]: @@ -895,8 +893,8 @@ async def assemble(data: AssembleInput): out = _add_music_bed(out, tag, cleanup) - manga_id, chapter_id = _mc_from_uri(data.clip_uris[0]) - uri = f"s3://video/{manga_id}/{chapter_id}/chapter.mp4" + manga_id, chapter_id = transport.ids_from_uri(data.clip_uris[0]) + uri = transport.CHAPTER_URI.format(manga_id=manga_id, chapter_id=chapter_id) transport.put(out, uri) for p in cleanup: os.remove(p) diff --git a/worker_tts.py b/worker_tts.py index 1913029..5116d3b 100644 --- a/worker_tts.py +++ b/worker_tts.py @@ -70,9 +70,10 @@ def _audio_uri(data: "TTSInput") -> str: # ponytail: flat key collides across chapters (as does the homesrv audio table); pass # panel_uri from run_stage_tts to make it per-chapter unique. if data.panel_uri: - parts = data.panel_uri.replace("s3://", "").split("/") - return f"s3://audio/{parts[1]}/{parts[2]}/audio/{data.panel_id or 'p'}.wav" - return f"s3://audio/_audio/{data.panel_id or 'p'}.wav" + manga_id, chapter_id = transport.ids_from_uri(data.panel_uri) + return transport.AUDIO_URI.format(manga_id=manga_id, chapter_id=chapter_id, + name=data.panel_id or 'p') + return transport.AUDIO_FLAT_URI.format(name=data.panel_id or 'p') def _ensure_ref() -> str: @@ -217,6 +218,13 @@ if __name__ == "__main__": os.remove(explicit) assert _calm("Stop!! Now!") == "Stop. Now." and _calm("no bangs") == "no bangs" # even prosody + # _audio_uri: a panel uri puts the wav beside its chapter, no panel uri falls back to the flat key. + assert _audio_uri(TTSInput(text="x", panel_id="p003", + panel_uri="s3://panels/m1/c1/panels/p003.png")) \ + == "s3://audio/m1/c1/audio/p003.wav" + assert _audio_uri(TTSInput(text="x", panel_id="p003")) == "s3://audio/_audio/p003.wav" + assert _audio_uri(TTSInput(text="x")) == "s3://audio/_audio/p.wav" + # 158: pronunciation lexicon respells whole words only, case-insensitive, spoken text only. import tempfile, json as _json globals()["LEXICON_PATH"] = os.path.join(tempfile.mkdtemp(), "lex.json") @@ -233,7 +241,7 @@ if __name__ == "__main__": # loudnorm: with ffmpeg present the wav is normalized in place and stays readable at its sr; # without ffmpeg it's a safe no-op returning the same path (audio never lost). ln = _write_wav(samples, 16000) - have_ffmpeg = subprocess.run(["ffmpeg", "-version"], capture_output=True).returncode == 0 \ + have_ffmpeg = subprocess.run(["ffmpeg", "-version"], capture_output=True, check=False).returncode == 0 \ if __import__("shutil").which("ffmpeg") else False assert _loudnorm(ln) == ln and os.path.exists(ln) assert wave.open(ln, "rb").getframerate() == 16000 diff --git a/worker_vision.py b/worker_vision.py index fb8efec..3bb6bc1 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -123,8 +123,8 @@ def _set_of_mark(local_path: str, present: list): return marked, "\n" + "\n".join(parts) + "\n", {f["label"]: f["local_id"] or "unknown" for f in faces} -_GENDER_SUFFIX = re.compile(r"\s*\((?:m|f|male|female|man|woman|unknown)\)\s*$", re.I) -_ID_SHAPED = re.compile(r"^(?:person|char|face|p)[\s_-]*\d+$", re.I) +_GENDER_SUFFIX = re.compile(r"\s*\((?:m|f|male|female|man|woman|unknown)\)\s*$", re.IGNORECASE) +_ID_SHAPED = re.compile(r"^(?:person|char|face|p)[\s_-]*\d+$", re.IGNORECASE) def _present_keys(present: list) -> dict: -- 2.52.0 From 0d281016e767ad361eb532c05b1648a45c74c5b8 Mon Sep 17 00:00:00 2001 From: kami Date: Thu, 13 Aug 2026 23:06:34 +0400 Subject: [PATCH 31/31] Cut the dead attic workers, and file what the audit left standing attic/worker_ocr.py and attic/worker_parse.py are 224 lines imported by nothing and named in no doc. The OCR stage was removed when narration moved to the director beat. The two design notes in attic/ stay, they are history. worker_vision._panel_size had one reference and it was the definition. The audit's larger finding is filed rather than fixed: call_gemma4, _extract_json and _strip_thought exist in both worker_vision and worker_script and have already diverged. That matters because the JSON repair pass can fabricate dialogue, so a fix would land in one copy and not the other. It is caveats/audit-open.md#gemma-helpers-duplicated with its revisit trigger. HANDOFF.md carries the rest: _wrap2 against textwrap, the duplicated ONNX preprocessing, and worker_layers pointing at a legacy/ directory that was never tracked in git. Checked: ruff clean, worker_vision and worker_render self-checks pass. Co-Authored-By: Claude Opus 5 --- HANDOFF.md | 157 +++++++++++++++++++----------------------- attic/worker_ocr.py | 99 -------------------------- attic/worker_parse.py | 125 --------------------------------- caveats/CLAUDE.md | 1 + caveats/audit-open.md | 16 +++++ worker_vision.py | 10 --- 6 files changed, 88 insertions(+), 320 deletions(-) delete mode 100644 attic/worker_ocr.py delete mode 100644 attic/worker_parse.py diff --git a/HANDOFF.md b/HANDOFF.md index b8b7fad..56c2ab9 100644 --- a/HANDOFF.md +++ b/HANDOFF.md @@ -1,121 +1,106 @@ -# HANDOFF, 2026-08-12/13 (seventh session) +# HANDOFF, 2026-08-13 (eighth session) Live state is in `NEXT.md`. This file is only what this session did. The previous handoff is in `JOURNAL.md`. ## Asked -"go" on item 1 of the previous plan, then a fourth GPU cycle. Then "commit and let's figure out how to -deal with the 2b1b". Then a long architecture message, "mind turning the architecture in docs?", plus a -question about panel merging before cropping. Then "well, I'd like the merge to be wired actually. and -then 3 of your fixes." +Push the repo to Gitea and open a PR to `master`. Then, from the PR review: fix the s3 URI constants, +run ruff and fix what it finds, push, run a ponytail audit, fix, push, merge. Then audit. ## Result -Item 1 written, run on a GPU, committed. Naming diagnosed from a real dialogue run and three fixes written -against it. The crop merge wired. `ARCHITECTURE.md` written. Nothing has run on a GPU since 20:25 UTC. +No GPU work. Nothing ran on a pipeline stage. PR #1 is open at +`https://gitea.kvmx.ru/kami/manga-recap-pipeline/pulls/1`, `master` <- `restore-runtime`. -## Committed and proven on a GPU +`master` held only the reconstruction commit `ff6a512`. All 27 commits of real work sat unpushed on +`restore-runtime`. -`a386e9d` here, `2daaa84` in the orchestrator. **The resolver gallery is the live cast, not cosine's top-k** -(`decisions/identity-bbox.md#cast-is-the-gallery`). `tracklets.cast_gallery` builds it from -`get_known_characters`, gender-compatible, named first, `GALLERY_CAP = 10`, re-read per tracklet. The -`if s.get("candidates")` guard is gone, so a crop with an empty cosine top-k now reaches the resolver. -`worker_vision.REF_BUDGET = 12` spreads reference images, `max(1, min(3, 12 // n))` apiece. +## Landed -Fourth cycle, 19:44-19:52 UTC: vision 116/116 3m55s, identity 116/116 2m44s, reconcile 20/20 44s. Ran with -the span cap and `GENERIC` tokenizer, which had never touched a GPU either. +| commit | what | +| --- | --- | +| `12d4ed8` | the `ARCHITECTURE.md` rewrite, which was finished but uncommitted | +| `be63b22` | deleted a 0-byte file named `II, d[i+4:i+12]))\n"; done`, added by `8113bdf` | +| `bec9411` | s3 URI templates into `transport.py`, plus `ruff.toml` and the lint fixes | -| metric | 18:07 run | 19:44 run | -| --- | --- | --- | -| detections | 119 | 119 | -| assignments | 68 | 60 | -| coverage | 57% | 50% | -| tracklets / crops | 12 / 64 | 33 / 72 | -| lead's assignments | 36 | 16 | -| characters after reconcile | 18 | 14 | -| minted / cleared | -- | 10 / 12 | +### S3 URIs -Coverage fell because gemma clears 12 crops instead of naming them wrongly. Checked by eye and confirmed by -the user. The lead's 16 are 14 him plus 2 art-in-art. `character_2b1b12a1` holds 13, all correct. -`character_f0d4e901` holds 9, of which 2 are `2b1b12a1`. +Eight templates now live in `transport.py`: `PANEL_URI`, `PAGE_PANEL_URI`, `AUDIO_URI`, +`AUDIO_FLAT_URI`, `LAYER_URI`, `CLIP_URI`, `CHAPTER_URI`, `CHAR_PNG_URI`, `CHAR_NPY_URI`. Five workers +formatted their own before. -## Measured, then fixed +`transport.ids_from_uri` replaces three separate copies of the same parse in `worker_tts`, +`worker_layers` and `worker_render`. It raises on a uri too short to carry the ids instead of returning +a wrong pair. `worker_render._mc_from_uri` is gone, its 6 call sites repointed. -Dialogue ran 116/116 in 5m57s at 20:19 UTC to test the naming path. Six claims, three defects: +### Lint -``` -p040 character_2b1b12a1 "Choi Haeseon" caption 1.00 -> NOT promoted -p010 character_b1dd5659 "Lim Seonho" caption 1.00 -> conflict flag -p047 character_b1dd5659 "Seonho" address 0.90 -> conflict flag -p011 character_f0d4e901 "Seonho" address 1.00 -> PROMOTED (wrong body) -p026 character_f0d4e901 "Seonho" address 1.00 -> PROMOTED (wrong body) -p110 character_028d4a49 "Haeseon" address 1.00 -> already named -``` +`ruff check .` exits 0. Ruff's defaults found 115. Fixed: implicit `Optional` in 8 signatures, an +unparenthesized implicit concatenation inside the ASS filter list, 5 `subprocess.run` calls now saying +`check=False` out loud, 1 unused import, 1 duplicate exception handler, 1 non-executable shebang, +4 `dict()` calls and 2 `startswith` chains. -## Written, tested, NOT run on a GPU +12 rules are off in `ruff.toml`, each with its reason. The ASYNC ones matter: ffmpeg and ffprobe run +synchronously inside `async def` endpoints, so a busy worker cannot answer `/health`. That is already +`caveats/audit-open.md#blocking-event-loop`, tracked as [#199], and it needs a per-handler refactor +rather than a lint fix. -Orchestrator, committed `2927927`, `db.add_name_claim` + `decisions/identity-naming.md`: +`CLAUDE.md` said "There is no lint or build step". It now names `ruff check .` and the rule that every +output URI is a template in `transport.py`. -- `alias_groups`: one name's word set inside another's is the same name, longer wins -- `STRONG_EVIDENCE` gains `caption` at confidence >= 0.9 -- a name held by another live character refuses to promote, files `name-already-taken` +## Audit findings -Here, committed `97cb483`: +Applied: -- `merge_faceless_captions` wired into both crop endpoints. It existed and was never called. -- `ARCHITECTURE.md`, the target shape from the user's design, each section carrying what exists today. +- `delete:` `attic/worker_ocr.py` and `attic/worker_parse.py`, 224 lines, imported by nothing and named + in no doc. `attic/char-recognition.md` and `attic/plan-workpc.md` kept, they are design history. +- `delete:` `worker_vision._panel_size`, 8 lines, one reference and it is the definition. -## Not started +Found and NOT applied, in order of size: -- **The head/body split is not fixed.** The wired merge only folds a fragment with text and no face. A body - fragment has no dialogue, so `_merge_plan` leaves it solo. No evidence gathered on how often it happens. -- The three cheap items from `ARCHITECTURE.md`: a `plane` field per detection, same-panel co-presence as a - weak cannot-link, `offscreen` as a fourth `speaker_ref` kind. -- `caveats/audit-open.md#gallery-cap-drops-the-unnamed`: the cap fired at `p097` (16 -> 10) and `p109` - (11 -> 10) and dropped exactly the freshly minted anonymous rows. +- `shrink:` `call_gemma4`, `_extract_json` and `_strip_thought` each exist twice, in `worker_vision.py` + and `worker_script.py`, and **have already diverged**. `worker_vision.call_gemma4` is 21 lines and + takes a content list, `worker_script`'s is 8 and takes a prompt string plus a system prompt. The two + `_extract_json` bodies carry the same comment about `raw_decode` but different error text. This is + the live risk: `caveats/audit-open.md#repair-fabricates` says the JSON repair pass can fabricate + dialogue, and a fix would land in one copy. A shared `gemma.py` costs one new file and removes about + 25 duplicated lines. Not done because it touches the two largest workers and nothing has run on a GPU + since. +- `stdlib:` `worker_render._wrap2`, 16 lines of greedy word-wrap with a 2-line cap, is close to + `textwrap.wrap(text, width, max_lines=2, placeholder="…")`. Not identical: `_wrap2` does + `rstrip(".,")` before the ellipsis, and textwrap reserves width for the placeholder, so the break + points differ. It renders burned-in subtitles, so the diff is user-visible. Verify against the render + self-check before swapping. +- `shrink:` `_letterbox` and `_load` are duplicated between `bubble_detect.py` and `face_detect.py`, + two ONNX detectors with the same preprocessing. About 16 lines. +- `delete:` `worker_layers.py:14` points `LAYERED_WORKFLOW` at `legacy/qwen_layered_workflow.json`. + `legacy/` was **never tracked in git** and is absent from disk, so the module's self-check has never + been able to pass here. This is the other half of `caveats/audit-open.md#layers-writes-nothing`. ## Checks ```bash -.venv/bin/python worker_vision.py # ok, including the reference-budget asserts -.venv/bin/python worker_crop.py # ok -./check_stale.sh # exit 0 before the cycle -/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && python3 tracklets.py && python3 test_name_binding.py && python3 -m pytest -q --ignore=test_api.py" # 121 passed +ruff check . # All checks passed +.venv/bin/python worker_render.py # ok, ffmpeg ran, about 4 minutes ``` -Each new assert was confirmed to fail with its fix disabled. +Self-checks pass: `transport`, `collage`, `bubble_detect`, `test_vision_parse`, `worker_crop`, +`worker_scene`, `worker_script`, `worker_identity`, `worker_tts`, `session_manager`, `worker_vision`, +`worker_render`, `face_detect`. + +`worker_layers` fails, and did before this branch, on the missing `legacy/` file above. ## Next command -Re-crop and run the fifth cycle. **Clear the panels prefix first** or the merge will not take effect: -`crop_webtoon` skips an upload when the key exists. +The fifth GPU cycle is still the next pipeline work. It is blocked only on the GPU being free. The +exact sequence is in `JOURNAL.md` under the seventh session, and `NEXT.md` item 1 holds the +expectations. **Clear the panels prefix first** or the wired caption merge +will not take effect. -```bash -cd /home/kami/Programs/n8n-worker && ./check_stale.sh # restart crop, it is stale after the wiring -/usr/bin/ssh kami@192.168.1.104 "cd /home/kami/docker-apps/manga-infra/orchestrator && docker compose up -d --build orchestrator" -J=778297bc-e7ce-439d-91b5-8a027060d17f; M=ef105a86-4b7e-4ac4-b45c-b7d83b8f5b5e; C=7c944dd4-e972-42c7-ba60-9f6939548e80 -/usr/bin/ssh kami@192.168.1.104 "mc rm --recursive --force homesrv/panels/$M/$C/panels/" -/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/characters/reset -H 'Content-Type: application/json' -d '{\"manga_id\":\"$M\",\"confirm\":true}'" -# honour restart_identity_worker: true -/usr/bin/ssh kami@192.168.1.104 "curl -s -X POST http://127.0.0.1:9090/stage/clear -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"crop\"}'" -for S in crop vision identity reconcile dialogue; do /usr/bin/ssh kami@192.168.1.104 "curl -s --max-time 7200 -X POST http://127.0.0.1:9090/stage/run -H 'Content-Type: application/json' -d '{\"job_id\":\"$J\",\"stage\":\"$S\"}'"; done -``` +Before that, or instead of it while the GPU is busy, `ARCHITECTURE.md` step 0 is the measurement spine: +`eval/chapter-truth.json` plus purity and fragmentation printed by `audit_registry.py`. -Expect: fewer than 116 panels, `2b1b12a1` named `Choi Haeseon`, no `conflicting-name-claims` on the lead, -and a `name-already-taken` flag where the female `Seonho` used to be. - -## Traps confirmed or found - -- **`crop_webtoon` skips an upload when the key exists.** Right for a resume, silently wrong after any - slicing change. Clear `s3://panels///panels/` before re-cropping. -- Restart a worker with `tmux send-keys -t manga-workers: C-c`, then re-send the launch line from - `start_workers.sh` with its `MIOPEN_ENV` prefix. A `pgrep` kill matches the calling shell. -- `POST /characters/reset` returns `restart_identity_worker: true`. Honour it. -- `docker compose up --build orchestrator` drops any `docker cp`'d script, `audit_registry.py` included. -- A heredoc piped into `docker exec` over `/usr/bin/ssh` produces no output. `scp` the file, `docker cp` - it, then run it. -- `panel_order` and the panel filename differ by one: `panel_order` 10 is `p009.png`. -- The contact-sheet script that found the identity errors is `sheet.py` in the session scratchpad, not - committed. It reads `assigns.json` dumped from `identity_assignments` and pulls crops from - `manga//characters/_crops/`. +One ordering trap, unresolved: the truth file cannot be keyed on `panel_id` or `character_id`. The +fifth cycle re-crops and calls `/characters/reset`, which destroys both. Key it on page-space geometry, +or write it after the fifth cycle rather than before. diff --git a/attic/worker_ocr.py b/attic/worker_ocr.py deleted file mode 100644 index d7371dd..0000000 --- a/attic/worker_ocr.py +++ /dev/null @@ -1,99 +0,0 @@ -# worker_ocr.py — stage 3 text extraction. FastAPI :8001. cpu (easyocr), no session. -# panel in -> text blocks with bboxes + confidence out. orchestrator persists to sqlite. -# easyocr (not tesseract): it reads stylized manga lettering far better -- recovers whole -# lines tesseract garbles or drops. runs on GPU (~0.4s/page warm) by default; the OCR stage -# runs before any LLM session opens so it doesn't contend with the resident model. set -# OCR_GPU=0 to force CPU (~3s/page). GPU needs MIOPEN_FIND_MODE=FAST in the env or the first -# ROCm run spends ~60s in MIOpen's exhaustive kernel search -- the launcher sets it. -import os, uuid -from fastapi import FastAPI -from pydantic import BaseModel -import transport - -app = FastAPI() -SHM = "/dev/shm" -MIN_CONF = 0.3 # easyocr line confidence floor -OCR_GPU = os.environ.get("OCR_GPU", "1") == "1" - -_reader = None # easyocr.Reader, lazy-loaded on first request - - -def _get_reader(): - global _reader - if _reader is None: - import easyocr - _reader = easyocr.Reader(["en"], gpu=OCR_GPU, verbose=False) - return _reader - - -def _detections_to_texts(detections): - """easyocr readtext output [(box_pts, text, conf)] -> our text blocks with xywh bboxes. - box_pts is 4 corner [x,y] points. drops low-confidence and art-noise (<2 letters). - casing is left as-is (mixed) -- the vision stage re-cases from the image anyway.""" - texts = [] - for i, (box, txt, conf) in enumerate(detections): - txt = txt.strip() - if conf < MIN_CONF or sum(c.isalpha() for c in txt) < 2: - continue - xs = [p[0] for p in box]; ys = [p[1] for p in box] - x, y = int(min(xs)), int(min(ys)) - texts.append({ - "id": f"t{i+1:03d}", - "content": txt, - "bbox": [x, y, int(max(xs)) - x, int(max(ys)) - y], - "confidence": round(float(conf), 3), - }) - return texts - - -def ocr_image(path: str): - return _detections_to_texts(_get_reader().readtext(path, detail=1, paragraph=False)) - - -class OCRInput(BaseModel): - panel_uri: str - job_id: str = "" - panel_id: str = "" - - -@app.post("/ocr") -async def ocr(data: OCRInput): - local = transport.get(data.panel_uri, f"{SHM}/ocr_{uuid.uuid4().hex[:8]}.png") - texts = ocr_image(local) - os.remove(local) - return {"panel_id": data.panel_id, "texts": texts} - - -@app.post("/unload") -async def unload(): - """free the resident easyocr reader (~1-2GB) once the OCR stage is done, before gemma4 loads. - ocr isn't session-managed, so the orchestrator calls this at stage end.""" - global _reader - was = _reader is not None - _reader = None - import gc; gc.collect() - try: - import torch; torch.cuda.empty_cache() - except Exception: - pass - return {"ok": True, "unloaded": was} - - -@app.get("/health") -async def health(): - return {"status": "ok"} - - -if __name__ == "__main__": - # self-check: detection->text-block conversion (pure, no model needed). - dets = [ - ([[10, 10], [110, 10], [110, 40], [10, 40]], "HELLO", 0.9), # kept - ([[10, 200], [70, 200], [70, 230], [10, 230]], "WORLD", 0.8), # kept - ([[5, 5], [13, 5], [13, 13], [5, 13]], "=", 0.9), # art-noise: <2 letters - ([[0, 0], [50, 0], [50, 20], [0, 20]], "REAL", 0.1), # below MIN_CONF - ] - texts = _detections_to_texts(dets) - assert [t["content"] for t in texts] == ["HELLO", "WORLD"], texts - assert texts[0]["bbox"] == [10, 10, 100, 30], texts[0]["bbox"] - assert texts[1]["confidence"] == 0.8 - print("worker_ocr self-check ok") diff --git a/attic/worker_parse.py b/attic/worker_parse.py deleted file mode 100644 index 08b177e..0000000 --- a/attic/worker_parse.py +++ /dev/null @@ -1,125 +0,0 @@ -# worker_parse.py — manga parse (paged manga only). FastAPI :8009. GPU, session-guarded ("magi"). -# Magi v2 chapter-wide pass: panel detection + reading order + OCR in one shot. Replaces the -# crop+ocr stages for paged manga; downstream ocr stage no-ops because rows are pre-populated. -# webtoons do NOT come here — they use worker_crop /crop/webtoon. See manga-two-repo-split memory. -import os, uuid -from fastapi import FastAPI, HTTPException -from pydantic import BaseModel -import cv2 -import numpy as np -import transport - -app = FastAPI() -SHM = "/dev/shm" -MAGI_MODEL = "ragavsachdeva/magiv2" -_model = None - - -def _load_magi(): - global _model - if _model is None: - import torch - from transformers import AutoModel - _model = AutoModel.from_pretrained(MAGI_MODEL, trust_remote_code=True).cuda().eval() - _model._torch = torch - return _model - - -class ParseInput(BaseModel): - page_uris: list # all pages of the chapter, in order - manga_id: str - chapter_id: str - session_id: str = "" # magi GPU lease (opened by orchestrator) - job_id: str = "" - - -def _center_in(box, panel) -> bool: - x1, y1, x2, y2 = box - px1, py1, px2, py2 = panel - cx, cy = (x1 + x2) / 2, (y1 + y2) / 2 - return px1 <= cx <= px2 and py1 <= cy <= py2 - - -def assemble_panels(pages, results, manga_id, chapter_id, put): - """Flatten Magi's per-page output into chapter-order panels with their OCR. - `pages`: RGB np arrays. `results`: per-page dicts (Magi keys). `put(np_crop, uri)` uploads. - Text is assigned to the panel whose box contains the text-box center; SFX (non-essential) - is dropped so narration isn't polluted. bbox converted [x1,y1,x2,y2] -> [x,y,w,h].""" - out, gidx = [], 0 - for img, res in zip(pages, results): - panels = res.get("panels", []) - texts = res.get("texts", []) - ocr = res.get("ocr", []) - essential = res.get("is_essential_text", [True] * len(texts)) - for p in panels: - x1, y1, x2, y2 = (int(v) for v in p) - uri = f"s3://manga/{manga_id}/{chapter_id}/panels/p{gidx:03d}.png" - if not transport.exists(uri): # deterministic per gidx -> resumable - put(img[y1:y2, x1:x2], uri) - ocr_texts = [] - for ti, tb in enumerate(texts): - if ti < len(ocr) and essential[ti] and _center_in(tb, p): - tx1, ty1, tx2, ty2 = (int(v) for v in tb) - ocr_texts.append({"text_id": f"t{ti}", "content": ocr[ti], - "bbox": [tx1, ty1, tx2 - tx1, ty2 - ty1], "confidence": 1.0}) - out.append({"panel_index": gidx, "uri": uri, - "bbox": [x1, y1, x2 - x1, y2 - y1], "ocr": ocr_texts}) - gidx += 1 - return out - - -def _put_crop(np_rgb, uri): - tmp = f"{SHM}/parse_{uuid.uuid4().hex[:8]}.png" - cv2.imwrite(tmp, cv2.cvtColor(np_rgb, cv2.COLOR_RGB2BGR)) - transport.put(tmp, uri) - os.remove(tmp) - - -@app.post("/parse") -async def parse(data: ParseInput): - tag = uuid.uuid4().hex[:8] - pages = [] - for i, u in enumerate(data.page_uris): - local = transport.get(u, f"{SHM}/pp_{tag}_{i:03d}.png") - img = cv2.imread(local) - if img is None: - raise HTTPException(400, f"page not readable: {u}") - pages.append(cv2.cvtColor(img, cv2.COLOR_BGR2RGB)) - os.remove(local) - model = _load_magi() - # ponytail: empty character bank in v1 — identity stays with the downstream siglip stage; - # feed a real bank (known-char ref crops + names) here to get Magi speaker association. - bank = {"images": [], "names": []} - with model._torch.no_grad(): - results = model.do_chapter_wide_prediction(pages, bank, use_tqdm=False, do_ocr=True) - panels = assemble_panels(pages, results, data.manga_id, data.chapter_id, _put_crop) - return {"panels": panels} - - -@app.get("/health") -async def health(): - return {"status": "ok"} - - -if __name__ == "__main__": - # self-check: model-free. Fake a 2-page Magi result and assert panel flattening, - # chapter-order indexing, text->panel containment, SFX drop, and bbox conversion. - transport.exists = lambda *a, **k: False # no minio in self-check - stored = {} - pages = [np.zeros((100, 100, 3), np.uint8), np.zeros((100, 100, 3), np.uint8)] - results = [ - {"panels": [[0, 0, 50, 100], [50, 0, 100, 100]], # page 0: two panels - "texts": [[10, 10, 20, 20], [60, 10, 70, 20]], # one text in each - "ocr": ["HELLO", "BOOM"], "is_essential_text": [True, False]}, # BOOM = SFX, dropped - {"panels": [[0, 0, 100, 100]], # page 1: one panel - "texts": [[5, 5, 15, 15]], "ocr": ["WORLD"], "is_essential_text": [True]}, - ] - panels = assemble_panels(pages, results, "m", "c", lambda img, uri: stored.__setitem__(uri, img.shape)) - assert [p["panel_index"] for p in panels] == [0, 1, 2], "chapter-order index" - assert panels[0]["ocr"][0]["content"] == "HELLO" - assert panels[1]["ocr"] == [], "SFX text dropped from panel 1" - assert panels[2]["ocr"][0]["content"] == "WORLD" - assert panels[0]["bbox"] == [0, 0, 50, 100], "xyxy->xywh" - assert panels[0]["ocr"][0]["bbox"] == [10, 10, 10, 10] - assert stored, "crops uploaded via put" - print("worker_parse self-check ok") diff --git a/caveats/CLAUDE.md b/caveats/CLAUDE.md index d7fbb8e..e5d3270 100644 --- a/caveats/CLAUDE.md +++ b/caveats/CLAUDE.md @@ -28,6 +28,7 @@ a complaint, so give it one or drop it. | [`layers` runs after `tts`, so pipelined solo beats lose parallax](audit-open.md#layers-after-tts) | AUDIT.md | | [`completed` means something different in each stage](audit-open.md#inconsistent-stage-policy) | AUDIT.md | | [Identity worker caches characters the orchestrator has deleted](audit-open.md#stale-known-cache) | AUDIT.md | +| [The gemma helpers exist twice and have diverged](audit-open.md#gemma-helpers-duplicated) | repo audit | | [A character seen once gets no assignment at all](audit-open.md#pending-in-worker-memory) | AUDIT.md | | [MinIO credentials are hardcoded in committed source](audit-open.md#hardcoded-credentials) | AUDIT.md | | [Assemble marks a job completed with no clips](audit-open.md#empty-assemble) | AUDIT.md | diff --git a/caveats/audit-open.md b/caveats/audit-open.md index 3fcc889..7aa0a8e 100644 --- a/caveats/audit-open.md +++ b/caveats/audit-open.md @@ -255,3 +255,19 @@ query crop. The cap is a VRAM and context budget, not a modelling choice. here. The fix is to order the gallery by how many assignments each character already holds in this chapter, so the tail is the rows nobody has matched rather than the rows nobody has named yet. That needs one count query per tracklet. + +## The gemma helpers exist twice and have diverged {#gemma-helpers-duplicated} + +**Open. Found by the repo audit of 2026-08-13, not yet by a failure.** + +`call_gemma4`, `_extract_json` and `_strip_thought` are defined in both `worker_vision.py` and +`worker_script.py`. They are no longer the same code. `worker_vision.call_gemma4` is 21 lines and takes a +content list; `worker_script`'s is 8 and takes a prompt string plus a system prompt. Both `_extract_json` +bodies carry the same comment about `raw_decode` stopping at the first object, and raise different errors. + +Who pays: whoever fixes the JSON path. `#repair-fabricates` above says the repair pass can fabricate +dialogue. A fix written against one copy leaves the other worker on the old behaviour, and no self-check +compares them. + +**Revisit when** either JSON path is touched for any reason. The fix is a `gemma.py` holding the three +helpers, imported by both workers, which removes about 25 duplicated lines and costs one file. diff --git a/worker_vision.py b/worker_vision.py index 3bb6bc1..bf0b88d 100644 --- a/worker_vision.py +++ b/worker_vision.py @@ -367,16 +367,6 @@ def _mark_has_face(img, chars: list) -> list: return chars -def _panel_size(path: str) -> tuple: - """(width, height) of a panel image, or (0, 0) when it cannot be read.""" - import cv2 - img = cv2.imread(path) - if img is None: - return (0, 0) - h, w = img.shape[:2] - return (w, h) - - @app.post("/vision") async def vision(data: VisionInput): local = transport.get(data.panel_uri, f"{SHM}/vision_{uuid.uuid4().hex[:8]}.png") -- 2.52.0