Put every S3 URI in one place, and add a lint gate
Five workers built output URIs with inline f-strings, so the bucket-per-artifact layout was spread across worker_tts, worker_identity, worker_crop, worker_layers and worker_render. Moving a class between buckets meant a grep. They are now templates in transport.py, formatted at each call site. Three of those workers also each reimplemented the same parse to recover manga_id and chapter_id from an input uri, because the orchestrator does not send them. That is transport.ids_from_uri now, and it raises on a uri too short to carry the ids rather than returning a wrong pair. ruff.toml makes `ruff check .` exit 0, so CI can gate on it and a new finding means a new defect. Fixed: an implicit Optional in 8 signatures, an unparenthesized implicit concatenation in the ASS filter list, 5 subprocess.run calls now saying check=False out loud, an unused import, a duplicate exception handler and a non-executable shebang. Every rule left off carries its reason in ruff.toml. The ASYNC rules are off because ffmpeg on the event loop is real and already recorded at caveats/audit-open.md#blocking-event-loop. It needs a refactor per handler, not a lint fix. Checked: transport, collage, bubble_detect, test_vision_parse, worker_crop, worker_scene, worker_script, worker_identity, worker_tts, session_manager, worker_vision and worker_render self-checks all pass. worker_layers still fails on a missing legacy/qwen_layered_workflow.json, which predates this branch. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+44
-3
@@ -12,6 +12,29 @@ from starlette.responses import Response
|
||||
|
||||
_client = None
|
||||
|
||||
# --- artifact layout ---------------------------------------------------------------------------
|
||||
# one bucket per artifact class (`decisions/storage-layout.md#bucket-per-artifact`). Every worker
|
||||
# formats its output uri from these, so moving a class between buckets is one edit here rather than
|
||||
# a grep across five workers. `name` is the panel id, or 'p' when a worker has none.
|
||||
PANEL_URI = "s3://panels/{manga_id}/{chapter_id}/panels/p{idx:03d}.png"
|
||||
PAGE_PANEL_URI = "s3://panels/{manga_id}/{chapter_id}/panels/pg{page_index:03d}_p{idx:02d}.png"
|
||||
AUDIO_URI = "s3://audio/{manga_id}/{chapter_id}/audio/{name}.wav"
|
||||
AUDIO_FLAT_URI = "s3://audio/_audio/{name}.wav"
|
||||
LAYER_URI = "s3://layers/{manga_id}/{chapter_id}/layers/{name}/{idx}.png"
|
||||
CLIP_URI = "s3://video/{manga_id}/{chapter_id}/clips/{name}.mp4"
|
||||
CHAPTER_URI = "s3://video/{manga_id}/{chapter_id}/chapter.mp4"
|
||||
CHAR_PNG_URI = "s3://manga/{key}.png"
|
||||
CHAR_NPY_URI = "s3://manga/{key}.npy"
|
||||
|
||||
|
||||
def ids_from_uri(uri: str):
|
||||
"""(manga_id, chapter_id) from any artifact uri: <bucket>/<manga_id>/<chapter_id>/...
|
||||
the orchestrator passes no ids to tts, layers or render, but every input uri encodes them."""
|
||||
parts = (uri.removeprefix("s3://")).split("/")
|
||||
if len(parts) < 3:
|
||||
raise ValueError(f"uri carries no manga/chapter: {uri!r}")
|
||||
return parts[1], parts[2]
|
||||
|
||||
|
||||
def _summarize(body: bytes, limit=6) -> str:
|
||||
"""compact one-line view of a json body for observability: uri inputs/outputs (basename, or
|
||||
@@ -27,9 +50,9 @@ def _summarize(body: bytes, limit=6) -> str:
|
||||
for k, v in obj.items():
|
||||
if k == "panel_id":
|
||||
continue
|
||||
if isinstance(v, str) and (k.endswith("uri") or k.endswith("url")):
|
||||
if isinstance(v, str) and (k.endswith(("uri", "url"))):
|
||||
parts.append(f"{k}={v.rsplit('/', 1)[-1]}")
|
||||
elif isinstance(v, list) and v and isinstance(v[0], str) and (k.endswith("uris") or k.endswith("urls")):
|
||||
elif isinstance(v, list) and v and isinstance(v[0], str) and (k.endswith(("uris", "urls"))):
|
||||
parts.append(f"{k}×{len(v)}")
|
||||
elif isinstance(v, (int, float, bool)):
|
||||
parts.append(f"{k}={v}")
|
||||
@@ -102,7 +125,7 @@ def _mc():
|
||||
|
||||
def _split(uri: str):
|
||||
"""(bucket, key) from an s3-style or bare uri."""
|
||||
u = uri[5:] if uri.startswith("s3://") else uri
|
||||
u = uri.removeprefix("s3://")
|
||||
bucket, _, key = u.partition("/")
|
||||
if not bucket or not key:
|
||||
raise ValueError(f"bad uri: {uri!r}")
|
||||
@@ -213,4 +236,22 @@ if __name__ == "__main__":
|
||||
assert _summarize(b"clip_uri", ) == "-" # non-json
|
||||
assert _summarize(json.dumps({"clip_uri": "s3://m/c/clips/p1.mp4", "duration": 4.1}).encode()) \
|
||||
== "clip_uri=p1.mp4 duration=4.1"
|
||||
|
||||
# artifact layout: templates format to the keys the workers wrote by hand before, and
|
||||
# ids_from_uri recovers the ids the orchestrator never sends.
|
||||
panel = PANEL_URI.format(manga_id="m1", chapter_id="c1", idx=7)
|
||||
assert panel == "s3://panels/m1/c1/panels/p007.png", panel
|
||||
assert PAGE_PANEL_URI.format(manga_id="m1", chapter_id="c1", page_index=2, idx=3) \
|
||||
== "s3://panels/m1/c1/panels/pg002_p03.png"
|
||||
assert CLIP_URI.format(manga_id="m1", chapter_id="c1", name="p003") \
|
||||
== "s3://video/m1/c1/clips/p003.mp4"
|
||||
assert LAYER_URI.format(manga_id="m1", chapter_id="c1", name="p003", idx=0) \
|
||||
== "s3://layers/m1/c1/layers/p003/0.png"
|
||||
assert ids_from_uri(panel) == ("m1", "c1")
|
||||
assert ids_from_uri("panels/m1/c1/panels/p007.png") == ("m1", "c1")
|
||||
try:
|
||||
ids_from_uri("s3://panels/p007.png")
|
||||
raise AssertionError("a uri with no chapter segment must raise")
|
||||
except ValueError:
|
||||
pass
|
||||
print("transport self-check ok")
|
||||
|
||||
Reference in New Issue
Block a user