kami/manga-recap-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Put every S3 URI in one place, and add a lint gate

Browse Source 
Five workers built output URIs with inline f-strings, so the bucket-per-artifact
layout was spread across worker_tts, worker_identity, worker_crop, worker_layers
and worker_render. Moving a class between buckets meant a grep. They are now
templates in transport.py, formatted at each call site.

Three of those workers also each reimplemented the same parse to recover
manga_id and chapter_id from an input uri, because the orchestrator does not
send them. That is transport.ids_from_uri now, and it raises on a uri too short
to carry the ids rather than returning a wrong pair.

ruff.toml makes `ruff check .` exit 0, so CI can gate on it and a new finding
means a new defect. Fixed: an implicit Optional in 8 signatures, an unparenthesized
implicit concatenation in the ASS filter list, 5 subprocess.run calls now saying
check=False out loud, an unused import, a duplicate exception handler and a
non-executable shebang. Every rule left off carries its reason in ruff.toml.

The ASYNC rules are off because ffmpeg on the event loop is real and already
recorded at caveats/audit-open.md#blocking-event-loop. It needs a refactor per
handler, not a lint fix.

Checked: transport, collage, bubble_detect, test_vision_parse, worker_crop,
worker_scene, worker_script, worker_identity, worker_tts, session_manager,
worker_vision and worker_render self-checks all pass. worker_layers still fails
on a missing legacy/qwen_layered_workflow.json, which predates this branch.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
kami
2026-08-13 23:02:06 +04:00
parent be63b2247d
commit bec9411af3
15 changed files with 131 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker_render.py
+31 -33
View File
@@ -19,13 +19,6 @@ MUSIC_BED = os.environ.get("MUSIC_BED", "") # #13 path/uri of a music t
MUSIC_GAIN = os.environ.get("MUSIC_GAIN", "0.18") # bed level before ducking
def _mc_from_uri(uri: str):
"""(manga_id, chapter_id) from s3://<bucket>/<manga_id>/<chapter_id>/...
the orchestrator doesn't pass ids to render/layers, but every input uri encodes them."""
parts = uri.replace("s3://", "").split("/")
return parts[1], parts[2]
def _ts(s):
h = int(s // 3600); m = int(s % 3600 // 60); sec = s % 60
return f"{h}:{m:02d}:{sec:05.2f}"
@@ -37,10 +30,10 @@ def _ts(s):
SUB_MODE = os.environ.get("SUB_MODE", "minimal").lower() # off | minimal | boxed
SUB_PRESETS = {
# (mode, orientation): fontsize, borderstyle(1=outline,3=box), outline/pad, shadow, marginV, side
("minimal", "portrait"): dict(fs=40, bs=1, outline=3, shadow=2, mv=0.055, side=110),
("minimal", "landscape"): dict(fs=32, bs=1, outline=3, shadow=2, mv=0.09, side=260),
("boxed", "portrait"): dict(fs=40, bs=3, outline=6, shadow=0, mv=0.055, side=110),
("boxed", "landscape"): dict(fs=32, bs=3, outline=6, shadow=0, mv=0.09, side=260),
("minimal", "portrait"): {"fs": 40, "bs": 1, "outline": 3, "shadow": 2, "mv": 0.055, "side": 110},
("minimal", "landscape"): {"fs": 32, "bs": 1, "outline": 3, "shadow": 2, "mv": 0.09, "side": 260},
("boxed", "portrait"): {"fs": 40, "bs": 3, "outline": 6, "shadow": 0, "mv": 0.055, "side": 110},
("boxed", "landscape"): {"fs": 32, "bs": 3, "outline": 6, "shadow": 0, "mv": 0.09, "side": 260},
}
@@ -109,7 +102,7 @@ def _ass(text: str, dur: float, path: str):
def _audio_dur(path: str) -> float:
"""clip length = narration length. scene_timing arrives empty, so probe the audio."""
r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration",
"-of", "default=nk=1:nw=1", path], capture_output=True, text=True)
"-of", "default=nk=1:nw=1", path], capture_output=True, text=True, check=False)
try:
return float(r.stdout.strip())
except ValueError:
@@ -120,7 +113,7 @@ def _stream_dur(path: str, kind: str) -> float:
"""duration of one stream. `format=duration` is max(video,audio) and so hides A/V drift."""
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", f"{kind}:0",
"-show_entries", "stream=duration", "-of", "default=nk=1:nw=1", path],
capture_output=True, text=True)
capture_output=True, text=True, check=False)
try:
return float(r.stdout.strip())
except ValueError:
@@ -132,7 +125,7 @@ def _fps_of(path: str) -> str:
their rate AND therefore their time_base agree, and the stream-copy concat path cares about that."""
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0",
"-show_entries", "stream=r_frame_rate", "-of", "default=nk=1:nw=1", path],
capture_output=True, text=True)
capture_output=True, text=True, check=False)
return r.stdout.strip()
@@ -184,7 +177,7 @@ def _motion(camera: dict, frames: int) -> str:
return f"zoompan=z='{z}':x='{x}':y='{y}':d={frames}:s={W}x{H}:fps={FPS}"
def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict = None,
def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict | None = None,
pad: float = PAD_S) -> list:
"""ffmpeg: still panel over a blurred fill of itself + content-aware motion, burned subs, 9:16.
#5 blurred bg replaces black bars: one copy scaled to COVER + blurred, the fitted panel on top.
@@ -231,8 +224,9 @@ async def render_scene(data: SceneInput):
_ass(data.narration_text, dur, ass)
out = f"{SHM}/rnd_{tag}.mp4"
subprocess.run(scene_cmd(img, audio, ass, out, dur, data.camera), check=True, capture_output=True)
manga_id, chapter_id = _mc_from_uri(data.panel_uri)
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
manga_id, chapter_id = transport.ids_from_uri(data.panel_uri)
uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id,
name=data.panel_id or 'p')
transport.put(out, uri)
for p in (img, audio, ass, out):
os.remove(p)
@@ -341,7 +335,7 @@ class CompositeInput(BaseModel):
def _img_size(path: str) -> tuple:
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
"stream=width,height", "-of", "csv=p=0:s=x", path], capture_output=True, text=True)
"stream=width,height", "-of", "csv=p=0:s=x", path], capture_output=True, text=True, check=False)
w, h = r.stdout.strip().split("x")
return int(w), int(h)
@@ -377,8 +371,9 @@ async def render_composite(data: CompositeInput):
subprocess.run(composite_cmd(still, auds, ass, segs, rowh, out, t), check=True, capture_output=True)
for p in imgs + [still]:
os.remove(p)
manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"])
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
manga_id, chapter_id = transport.ids_from_uri(data.panels[0]["panel_uri"])
uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id,
name=data.panel_id or 'p')
transport.put(out, uri)
for p in auds + [ass, out]:
os.remove(p)
@@ -424,8 +419,9 @@ async def render_group(data: GroupInput):
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", final]
subprocess.run(cmd, check=True, capture_output=True)
manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"])
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
manga_id, chapter_id = transport.ids_from_uri(data.panels[0]["panel_uri"])
uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id,
name=data.panel_id or 'p')
transport.put(final, uri)
total = _audio_dur(final) or sum(durs)
for f in cleanup + [final]:
@@ -449,7 +445,7 @@ class BeatInput(BaseModel):
BEAT_MIN_PANEL_S = 1.0 # 185: no member panel flashes by faster than this
def _beat_slices(D: float, n: int, weights: list = None) -> list:
def _beat_slices(D: float, n: int, weights: list | None = None) -> list:
"""185: split beat duration D across n member panels by content weight, not evenly.
Each panel gets >= BEAT_MIN_PANEL_S so a low-weight panel never flashes. Degenerate input
(no/short weights, non-positive sum, or D too small for the floors) -> deterministic equal split.
@@ -501,8 +497,8 @@ def cue_plan(text: str, D: float, slices: list) -> list:
return events
def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list = None,
weights: list = None) -> list:
def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list | None = None,
weights: list | None = None) -> list:
"""ffmpeg: N images shown as a ken-burns montage under ONE narration audio; the last image holds
through the trailing pad. each image = blurred-fill bg + fitted panel + its camera move; the single
burned subtitle (in `ass`) spans the whole beat. 185: per-image screen-time is content-weighted
@@ -561,8 +557,9 @@ async def render_beat(data: BeatInput):
subprocess.run(beat_cmd(imgs, audio, ass, out, D, data.cameras, data.weights),
check=True, capture_output=True)
manga_id, chapter_id = _mc_from_uri(data.panel_uris[0])
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
manga_id, chapter_id = transport.ids_from_uri(data.panel_uris[0])
uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id,
name=data.panel_id or 'p')
transport.put(out, uri)
total = _audio_dur(out) or (D + PAD_S)
for f in imgs + [audio, ass, out]:
@@ -603,8 +600,8 @@ def collage_cmd(imgs, plate_i, rects, entrances, z_order, audio, ass, out, D, tr
for img in imgs:
cmd += ["-loop", "1", "-i", img] # still held for the whole clip; -t caps output
cmd += ["-i", audio]
parts = [f"[{plate_i}:v]scale={W}:{H}:force_original_aspect_ratio=increase,"
f"crop={W}:{H},boxblur=24:2,setsar=1[bg]"]
parts = [(f"[{plate_i}:v]scale={W}:{H}:force_original_aspect_ratio=increase,"
f"crop={W}:{H},boxblur=24:2,setsar=1[bg]")]
base = "bg"
for k, i in enumerate(z_order): # shadows first, at rest positions (static)
x, y, w, h = (int(round(v)) for v in rects[i])
@@ -661,8 +658,9 @@ async def render_collage(data: CollageInput):
out = f"{SHM}/cl_{tag}.mp4"
subprocess.run(collage_cmd(imgs, active, lay["rects"], lay["entrances"], lay["z_order"],
audio, ass, out, D, lay["transition_s"]), check=True, capture_output=True)
manga_id, chapter_id = _mc_from_uri(uris[0])
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
manga_id, chapter_id = transport.ids_from_uri(uris[0])
uri = transport.CLIP_URI.format(manga_id=manga_id, chapter_id=chapter_id,
name=data.panel_id or 'p')
transport.put(out, uri)
total = _audio_dur(out) or (D + PAD_S)
for f in imgs + [audio, ass, out]:
@@ -895,8 +893,8 @@ async def assemble(data: AssembleInput):
out = _add_music_bed(out, tag, cleanup)
manga_id, chapter_id = _mc_from_uri(data.clip_uris[0])
uri = f"s3://video/{manga_id}/{chapter_id}/chapter.mp4"
manga_id, chapter_id = transport.ids_from_uri(data.clip_uris[0])
uri = transport.CHAPTER_URI.format(manga_id=manga_id, chapter_id=chapter_id)
transport.put(out, uri)
for p in cleanup:
os.remove(p)