kami
63f7918a3e
Adds decision entries for the unpaired set-of-mark label, the interjection verifier false positive, and the vision-blob clearing bug, plus the per-run speaker audit script used to measure the chapter. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
106 lines
4.7 KiB
Python
106 lines
4.7 KiB
Python
"""Speaker-attribution audit for one chapter. Runs inside manga-orchestrator (reads /data/manga.db).
|
|
|
|
Answers the three questions asked of the 2026-08-11 rerun:
|
|
1. what happens on a multi-character panel whose people have no identity,
|
|
2. whether a known character gets mis-named,
|
|
3. what the named-speaker share actually is now.
|
|
"""
|
|
import collections
|
|
import json
|
|
import sqlite3
|
|
import sys
|
|
|
|
CHAPTER = sys.argv[1] if len(sys.argv) > 1 else "7c944dd4-e972-42c7-ba60-9f6939548e80"
|
|
SPEECH = {"speech", "shout", "thought"}
|
|
|
|
c = sqlite3.connect("/data/manga.db")
|
|
c.row_factory = sqlite3.Row
|
|
manga_id = c.execute("SELECT manga_id FROM chapters WHERE chapter_id=?", (CHAPTER,)).fetchone()[0]
|
|
|
|
reg = {r["character_id"]: dict(r) for r in c.execute(
|
|
"SELECT character_id, name, aliases, description FROM characters WHERE manga_id=?", (manga_id,))}
|
|
|
|
# duplicate registry rows make a correct name unresolvable: an answer matching two rows is ambiguous.
|
|
by_name = collections.defaultdict(list)
|
|
for cid, r in reg.items():
|
|
for n in [r["name"], *json.loads(r["aliases"] or "[]")]:
|
|
if n:
|
|
by_name[str(n).strip().casefold()].append(cid)
|
|
dupes = {n: ids for n, ids in by_name.items() if len(ids) > 1}
|
|
|
|
panels = c.execute("SELECT panel_id, panel_index, page_index FROM panels WHERE chapter_id=? ORDER BY \"order\"",
|
|
(CHAPTER,)).fetchall()
|
|
|
|
methods, kinds, per_char = collections.Counter(), collections.Counter(), collections.Counter()
|
|
speech = named = 0
|
|
crowded_speech = crowded_named = 0
|
|
solo_speech = solo_named = 0
|
|
unresolved_names = collections.Counter()
|
|
ambiguous = []
|
|
no_identity_crowd = 0
|
|
# a line attributed to a real present local_id that simply has no identity row: attribution succeeded
|
|
# and the name is lost anyway. This is the identity-coverage wall, not an attribution failure.
|
|
attributed_but_unassigned = collections.Counter()
|
|
|
|
for p in panels:
|
|
pid = p["panel_id"]
|
|
row = c.execute("SELECT result_json FROM vision_results WHERE panel_id=?", (pid,)).fetchone()
|
|
if not row:
|
|
continue
|
|
v = json.loads(row["result_json"])
|
|
if "dialogue" not in v:
|
|
continue
|
|
people = [ch for ch in (v.get("characters") or []) if ch.get("local_id")]
|
|
assigned = {a["local_id"] for a in c.execute(
|
|
"SELECT local_id FROM identity_assignments WHERE panel_id=?", (pid,))}
|
|
crowd = len(people) > 1
|
|
if crowd and not assigned:
|
|
no_identity_crowd += 1
|
|
for d in v["dialogue"]:
|
|
if d.get("type", "speech") not in SPEECH:
|
|
continue
|
|
speech += 1
|
|
ref = d.get("speaker_ref") or {}
|
|
kind = ref.get("kind")
|
|
methods[d.get("speaker_method")] += 1
|
|
kinds[kind] += 1
|
|
if crowd:
|
|
crowded_speech += 1
|
|
else:
|
|
solo_speech += 1
|
|
if kind == "character_id":
|
|
named += 1
|
|
per_char[reg.get(ref["value"], {}).get("name") or ref["value"]] += 1
|
|
if crowd:
|
|
crowded_named += 1
|
|
else:
|
|
solo_named += 1
|
|
elif kind == "unknown":
|
|
# normalize_dialogue nulls the flat `speaker` unless it resolved, but speaker_ref keeps the
|
|
# unresolved local_id as its value.
|
|
raw = str(ref.get("value") or "").strip()
|
|
local_ids = {ch["local_id"] for ch in people}
|
|
if raw in local_ids and raw not in assigned:
|
|
attributed_but_unassigned[d.get("speaker_method")] += 1
|
|
elif kind == "name":
|
|
unresolved_names[ref.get("value")] += 1
|
|
if ref.get("candidates"):
|
|
ambiguous.append((pid, ref.get("value"), ref["candidates"]))
|
|
|
|
print(f"panels with dialogue: {sum(1 for p in panels if (lambda r: r and 'dialogue' in json.loads(r['result_json']))(c.execute('SELECT result_json FROM vision_results WHERE panel_id=?', (p['panel_id'],)).fetchone()))}/{len(panels)}")
|
|
print(f"speech lines: {speech} named (character_id): {named} = {100*named/max(speech,1):.0f}%")
|
|
print(f" multi-character panels: {crowded_named}/{crowded_speech} named")
|
|
print(f" single-character panels: {solo_named}/{solo_speech} named")
|
|
print(f"multi-character panels with NO identity at all: {no_identity_crowd}")
|
|
print(f"attributed to a present local_id with NO identity row: {dict(attributed_but_unassigned)}")
|
|
print(f"speaker_method: {dict(methods)}")
|
|
print(f"speaker_ref kind: {dict(kinds)}")
|
|
print(f"named per character: {dict(per_char)}")
|
|
print(f"unresolved name refs: {dict(unresolved_names)}")
|
|
print(f"registry duplicate names (block a correct bind): "
|
|
f"{ {n: [reg[i]['name'] for i in ids] for n, ids in dupes.items()} }")
|
|
if ambiguous:
|
|
print("ambiguous binds:")
|
|
for pid, val, cands in ambiguous[:20]:
|
|
print(f" {pid} {val!r} -> {[reg.get(x, {}).get('name') for x in cands]}")
|