kami
ff6a512630
Working tree (including .git) was lost to an rm. Rebuilt by replaying Write/Edit/ Read/attachment events from 25 Claude sessions and 22 successful codex apply_patch blocks into one timestamp-ordered timeline. Verified against ground truth recorded in the transcripts: wc -l on 10 files and ls -l on 5 files at 2026-07-18T13:13:44Z both match exactly; 18 files are byte-identical to their newest ~/.claude/file-history blob. See HANDOFF.md for sources, gaps, and how to rebuild .venv. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
997 lines
52 KiB
Python
997 lines
52 KiB
Python
# worker_render.py — stage 9 part 2 rendering. FastAPI :8008.
|
||
# per-scene: motion (ken burns, or parallax when layers present) + burned subtitles -> mp4 clip.
|
||
# chapter assembly: concat clips with crossfades -> chapter.mp4. all outputs to minio.
|
||
# ponytail: ken burns for both cases for now; layer-parallax is the upgrade path when the
|
||
# depth renderer is dialed in — the layer_uris are already threaded through the input.
|
||
import os, re, uuid, subprocess
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from fastapi import FastAPI, HTTPException
|
||
from pydantic import BaseModel
|
||
import transport
|
||
from collage import plan_layout
|
||
|
||
app = FastAPI()
|
||
transport.install_logging(app, "render")
|
||
SHM = "/dev/shm"
|
||
W, H = 1080, 1920 # vertical 9:16
|
||
PAD_S = float(os.environ.get("PANEL_PAD_S", "0.4")) # #13 trailing silence per panel for pacing
|
||
MUSIC_BED = os.environ.get("MUSIC_BED", "") # #13 path/uri of a music track; empty -> no bed
|
||
MUSIC_GAIN = os.environ.get("MUSIC_GAIN", "0.18") # bed level before ducking
|
||
|
||
|
||
def _mc_from_uri(uri: str):
|
||
"""(manga_id, chapter_id) from s3://<bucket>/<manga_id>/<chapter_id>/...
|
||
the orchestrator doesn't pass ids to render/layers, but every input uri encodes them."""
|
||
parts = uri.replace("s3://", "").split("/")
|
||
return parts[1], parts[2]
|
||
|
||
|
||
def _ts(s):
|
||
h = int(s // 3600); m = int(s % 3600 // 60); sec = s % 60
|
||
return f"{h}:{m:02d}:{sec:05.2f}"
|
||
|
||
|
||
# 188: readable narration subtitles. Modes: off (burn nothing), minimal (small text, thin outline +
|
||
# soft shadow, NO box — least art occlusion), boxed (small text on a restrained ~50% box). Presets carry
|
||
# portrait/landscape safe-areas. ASS colour is &HAABBGGRR (alpha 00=opaque..FF=clear).
|
||
SUB_MODE = os.environ.get("SUB_MODE", "minimal").lower() # off | minimal | boxed
|
||
SUB_PRESETS = {
|
||
# (mode, orientation): fontsize, borderstyle(1=outline,3=box), outline/pad, shadow, marginV, side
|
||
("minimal", "portrait"): dict(fs=40, bs=1, outline=3, shadow=2, mv=0.055, side=110),
|
||
("minimal", "landscape"): dict(fs=32, bs=1, outline=3, shadow=2, mv=0.09, side=260),
|
||
("boxed", "portrait"): dict(fs=40, bs=3, outline=6, shadow=0, mv=0.055, side=110),
|
||
("boxed", "landscape"): dict(fs=32, bs=3, outline=6, shadow=0, mv=0.09, side=260),
|
||
}
|
||
|
||
|
||
def _wrap2(text: str, width: int) -> str:
|
||
"""188: keep a cue to at most 2 short lines. Greedy word-wrap to `width`; if it still needs a 3rd
|
||
line, truncate the 2nd with an ellipsis so a long cue never grows back into a paragraph block."""
|
||
words, lines, cur = (text or "").split(), [], ""
|
||
for w in words:
|
||
if cur and len(cur) + 1 + len(w) > width:
|
||
lines.append(cur); cur = w
|
||
if len(lines) == 2:
|
||
break
|
||
else:
|
||
cur = f"{cur} {w}".strip()
|
||
if len(lines) < 2:
|
||
lines.append(cur)
|
||
elif cur or len(words) > sum(len(l.split()) for l in lines):
|
||
lines[1] = lines[1].rstrip(".,") + "…" # more words remained than fit two lines
|
||
return "\\N".join(l for l in lines if l)
|
||
|
||
|
||
def _ass_multi(events, path, mode=None, orientation="portrait"):
|
||
"""Burn caption events. events: [(text, start, end)] or [(text, start, end, align)] where align is an
|
||
ASS numpad alignment (2=bottom-center default, 8=top-center to dodge a low subject). mode off -> no
|
||
Dialogue lines (nothing burned). Style comes from SUB_PRESETS (188)."""
|
||
mode = (mode or SUB_MODE)
|
||
if mode == "off":
|
||
events = []
|
||
p = SUB_PRESETS.get((mode, orientation), SUB_PRESETS[("minimal", "portrait")])
|
||
fill = "&H80000000" if p["bs"] == 3 else "&H00000000" # boxed: ~50% box; minimal: opaque outline
|
||
fmt = ("Name, Fontname, Fontsize, PrimaryColour, OutlineColour, BackColour, "
|
||
"Bold, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV")
|
||
mv = int(p["mv"] * (W if orientation == "landscape" else H))
|
||
style = ("Def,DejaVu Sans,%d,&H00FFFFFF,%s,&H80000000,0,%d,%d,%d,2,%d,%d,%d"
|
||
% (p["fs"], fill, p["bs"], p["outline"], p["shadow"], p["side"], p["side"], mv))
|
||
width = 34 if orientation == "portrait" else 46
|
||
lines = ""
|
||
for ev in events:
|
||
t, s, e = ev[0], ev[1], ev[2]
|
||
align = ev[3] if len(ev) > 3 else 2
|
||
# {\anN} overrides alignment per line so a cue can jump to the top over a low subject.
|
||
lines += "Dialogue: 0,%s,%s,Def,{\\an%d}%s\n" % (_ts(s), _ts(e), align, _wrap2(t, width))
|
||
with open(path, "w") as f:
|
||
f.write(
|
||
"[Script Info]\nScriptType: v4.00+\nPlayResX: %d\nPlayResY: %d\nWrapStyle: 0\n\n"
|
||
"[V4+ Styles]\nFormat: %s\nStyle: %s\n\n"
|
||
"[Events]\nFormat: Layer, Start, End, Style, Text\n%s" % (W, H, fmt, style, lines)
|
||
)
|
||
|
||
|
||
def _sub_align(camera: dict) -> int:
|
||
"""188: dodge the subject where we can. The director's focus point (camera.to=[x,y], y normalized
|
||
top->bottom) is the one subject location we already know — if it sits low in frame, put the caption
|
||
at the TOP (an8) so it doesn't cover the face; otherwise keep it bottom (an2).
|
||
ponytail: no face/bubble detector yet — upgrade to real bbox avoidance when one is wired in."""
|
||
to = (camera or {}).get("to")
|
||
if to and len(to) >= 2 and float(to[1]) > 0.6:
|
||
return 8
|
||
return 2
|
||
|
||
|
||
def _ass(text: str, dur: float, path: str):
|
||
_ass_multi([(text, 0.0, dur)], path)
|
||
|
||
|
||
def _audio_dur(path: str) -> float:
|
||
"""clip length = narration length. scene_timing arrives empty, so probe the audio."""
|
||
r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||
"-of", "default=nk=1:nw=1", path], capture_output=True, text=True)
|
||
try:
|
||
return float(r.stdout.strip())
|
||
except ValueError:
|
||
return 0.0
|
||
|
||
|
||
ZMAX, ZPAN = 1.15, 1.18 # ken-burns zoom ceiling; constant zoom that gives pans room to travel
|
||
|
||
|
||
def _motion(camera: dict, frames: int) -> str:
|
||
"""#8 content-aware motion: map the vision `camera` block to a zoompan z/x/y expression.
|
||
vocab (spec-v3 direction schema): static|hold, zoom_in|zoom_out, pan_left/right/up/down,
|
||
dolly_to_subject (uses camera.to=[x,y] normalized), orbit, shake. default = gentle zoom_in
|
||
(the old ken-burns look) so panels without direction are unchanged.
|
||
x/y reference `zoom` for the live crop-window size; z is driven linearly by `on` (frame index)
|
||
over T frames so the move completes across the whole clip regardless of length."""
|
||
T = max(1, frames - 1)
|
||
eff = (camera or {}).get("effect", "zoom_in")
|
||
xc, yc = "iw/2-(iw/zoom/2)", "ih/2-(ih/zoom/2)" # centered crop
|
||
mx, my = "(iw-iw/zoom)", "(ih-ih/zoom)" # pan travel margin
|
||
if eff in ("static", "hold"):
|
||
z, x, y = "1.0", xc, yc
|
||
elif eff == "zoom_out":
|
||
z, x, y = f"{ZMAX}-{ZMAX-1:.3f}*on/{T}", xc, yc
|
||
elif eff == "pan_left":
|
||
z, x, y = f"{ZPAN}", f"{mx}*(1-on/{T})", yc
|
||
elif eff == "pan_right":
|
||
z, x, y = f"{ZPAN}", f"{mx}*on/{T}", yc
|
||
elif eff == "pan_up":
|
||
z, x, y = f"{ZPAN}", xc, f"{my}*(1-on/{T})"
|
||
elif eff == "pan_down":
|
||
z, x, y = f"{ZPAN}", xc, f"{my}*on/{T}"
|
||
elif eff == "dolly_to_subject":
|
||
to = (camera or {}).get("to") or [0.5, 0.35]
|
||
tx, ty = min(max(float(to[0]), 0.0), 1.0), min(max(float(to[1]), 0.0), 1.0)
|
||
z = f"1+{ZMAX-1:.3f}*on/{T}"
|
||
x = f"(iw/2+({tx}*iw-iw/2)*on/{T})-(iw/zoom/2)" # crop center eases toward subject
|
||
y = f"(ih/2+({ty}*ih-ih/2)*on/{T})-(ih/zoom/2)"
|
||
elif eff == "shake":
|
||
z, x, y = "1.06", f"{xc}+8*sin(on*1.5)", f"{yc}+8*cos(on*1.3)"
|
||
elif eff == "orbit":
|
||
z = "1.12"
|
||
x, y = f"{xc}+(iw*0.03)*sin(6.283*on/{T})", f"{yc}+(ih*0.03)*cos(6.283*on/{T})"
|
||
else: # zoom_in (default ken burns)
|
||
z, x, y = f"1+{ZMAX-1:.3f}*on/{T}", xc, yc
|
||
return f"zoompan=z='{z}':x='{x}':y='{y}':d={frames}:s={W}x{H}:fps=25"
|
||
|
||
|
||
def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict = None,
|
||
pad: float = PAD_S) -> list:
|
||
"""ffmpeg: still panel over a blurred fill of itself + content-aware motion, burned subs, 9:16.
|
||
#5 blurred bg replaces black bars: one copy scaled to COVER + blurred, the fitted panel on top.
|
||
#13 pad seconds of trailing silence (last frame held) give the panel a beat before the next."""
|
||
fps = 25
|
||
frames = max(1, int((dur + pad) * fps)) # hold the last frame through the pad
|
||
# overlay's W/H/w/h are ffmpeg's main/overlay dims -- kept literal (no f-string braces).
|
||
fc = (
|
||
f"[0:v]split=2[bg][fg];"
|
||
f"[bg]scale={W}:{H}:force_original_aspect_ratio=increase,crop={W}:{H},boxblur=20:2[bgb];"
|
||
f"[fg]scale={W}:{H}:force_original_aspect_ratio=decrease[fgs];"
|
||
f"[bgb][fgs]overlay=(W-w)/2:(H-h)/2,"
|
||
f"{_motion(camera, frames)},"
|
||
f"ass={ass}[v];"
|
||
f"[1:a]apad=pad_dur={pad:.3f}[a]" # trailing silence to match the held frames
|
||
)
|
||
return ["ffmpeg", "-y", "-loop", "1", "-i", img, "-i", audio,
|
||
"-filter_complex", fc, "-map", "[v]", "-map", "[a]",
|
||
"-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", "-b:a", "192k", "-shortest", out]
|
||
|
||
|
||
class SceneInput(BaseModel):
|
||
panel_uri: str
|
||
layer_uris: list = []
|
||
audio_uri: str = ""
|
||
narration_text: str = ""
|
||
panel_id: str = ""
|
||
scene_timing: dict = {}
|
||
camera: dict = {} # #8 direction block from vision: {"effect": "...", "to": [x,y]}
|
||
manga_id: str = ""
|
||
chapter_id: str = ""
|
||
|
||
|
||
@app.post("/render/scene")
|
||
async def render_scene(data: SceneInput):
|
||
tag = uuid.uuid4().hex[:8]
|
||
img = transport.get(data.panel_uri, f"{SHM}/rnd_{tag}.png")
|
||
audio = transport.get(data.audio_uri, f"{SHM}/rnd_{tag}.wav")
|
||
# #1: real duration = narration length; scene_timing is empty in practice, 4.0 last-resort.
|
||
dur = (_audio_dur(audio)
|
||
or float(data.scene_timing.get("end", 0)) - float(data.scene_timing.get("start", 0))
|
||
or 4.0)
|
||
ass = f"{SHM}/rnd_{tag}.ass"
|
||
_ass(data.narration_text, dur, ass)
|
||
out = f"{SHM}/rnd_{tag}.mp4"
|
||
subprocess.run(scene_cmd(img, audio, ass, out, dur, data.camera), check=True, capture_output=True)
|
||
manga_id, chapter_id = _mc_from_uri(data.panel_uri)
|
||
uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
for p in (img, audio, ass, out):
|
||
os.remove(p)
|
||
return {"clip_uri": uri, "duration": dur}
|
||
|
||
|
||
# #6 multi-panel composite: a group of small panels shares ONE shot (vertical stack). each panel
|
||
# keeps its own narration + audio (script/tts stay per-panel); they play in sequence while a moving
|
||
# highlight marks the active panel -- that's the "swap the front panel while audio plays" beat.
|
||
def _stack_still_cmd(images: list, out: str, rowh: int) -> list:
|
||
"""compose N panels fitted into equal vertical rows on one W×H still (black gutters)."""
|
||
n = len(images)
|
||
cmd = ["ffmpeg", "-y"]
|
||
for im in images:
|
||
cmd += ["-i", im]
|
||
parts = [f"[{i}:v]scale={W}:{rowh}:force_original_aspect_ratio=decrease,"
|
||
f"pad={W}:{rowh}:(ow-iw)/2:(oh-ih)/2:color=black[p{i}]" for i in range(n)]
|
||
stacked = "".join(f"[p{i}]" for i in range(n))
|
||
fc = ";".join(parts) + f";{stacked}vstack=inputs={n},pad={W}:{H}:0:0:color=black[v]"
|
||
return cmd + ["-filter_complex", fc, "-map", "[v]", "-frames:v", "1", out]
|
||
|
||
|
||
# D#6 revisited: camera-traversal composite. instead of vstacking cropped panels (black gutters,
|
||
# "wrong look"), pan across the REAL page — hold on each member's panel, quick-ease to the next,
|
||
# partial neighbours stay visible. groups are always single-page (grouping.py), so one page image.
|
||
TRAVERSE_EASE = 0.5 # seconds of quick slide between held panels
|
||
WIN_MARGIN = 1.6 # crop window height = tallest member bbox * this (room for neighbour reveal)
|
||
|
||
|
||
def _framed_window(pw: int, ph: int, bboxes: list) -> tuple:
|
||
"""one constant 9:16 crop window (px) sized to hold the tallest member with margin, aspect
|
||
preserved, clamped to the page. constant size => only x,y animate (crop supports per-frame x/y,
|
||
not per-frame w/h)."""
|
||
ar = W / H
|
||
ch = max((b[3] for b in bboxes), default=ph) * WIN_MARGIN
|
||
cw = ch * ar
|
||
if cw > pw:
|
||
cw, ch = pw, pw / ar
|
||
if ch > ph:
|
||
ch, cw = ph, ph * ar
|
||
return int(cw), int(ch)
|
||
|
||
|
||
def _win_tl(bbox: list, cw: int, ch: int, pw: int, ph: int) -> tuple:
|
||
"""top-left of the window centered on a bbox, clamped so it stays inside the page."""
|
||
cx, cy = bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2
|
||
return (min(max(cx - cw / 2, 0), pw - cw), min(max(cy - ch / 2, 0), ph - ch))
|
||
|
||
|
||
def _pan_expr(vals: list, segs: list, ease: float) -> str:
|
||
"""piecewise ffmpeg expr in t: hold vals[i] through beat i, linear-ease to vals[i+1] over the
|
||
beat's last `ease`s; final beat just holds. segs=[(start,end)] per beat."""
|
||
n = len(vals)
|
||
expr = f"{vals[-1]:.1f}"
|
||
for i in range(n - 2, -1, -1):
|
||
s, e = segs[i]
|
||
he = e - ease
|
||
a, b = vals[i], vals[i + 1]
|
||
beat = f"if(lt(t,{he:.3f}),{a:.1f},({a:.1f}+({b - a:.1f})*(t-{he:.3f})/{ease:.3f}))"
|
||
expr = f"if(lt(t,{e:.3f}),{beat},{expr})"
|
||
return expr
|
||
|
||
|
||
def traverse_cmd(page: str, audios: list, ass: str, segs: list, bboxes: list,
|
||
pw: int, ph: int, out: str, dur: float) -> list:
|
||
"""pan a constant 9:16 window across the page: hold on each member, quick-ease to the next."""
|
||
ease = min(TRAVERSE_EASE, min((e - s for s, e in segs), default=1.0) / 2)
|
||
cw, ch = _framed_window(pw, ph, bboxes)
|
||
tls = [_win_tl(b, cw, ch, pw, ph) for b in bboxes]
|
||
xexpr = _pan_expr([t[0] for t in tls], segs, ease)
|
||
yexpr = _pan_expr([t[1] for t in tls], segs, ease)
|
||
n = len(audios)
|
||
aconcat = "".join(f"[{i + 1}:a]" for i in range(n)) + f"concat=n={n}:v=0:a=1[a]"
|
||
fc = (f"[0:v]crop={cw}:{ch}:x='{xexpr}':y='{yexpr}',scale={W}:{H},setsar=1,"
|
||
f"ass={ass}[v];{aconcat}")
|
||
cmd = ["ffmpeg", "-y", "-loop", "1", "-i", page]
|
||
for a in audios:
|
||
cmd += ["-i", a]
|
||
return cmd + ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{dur:.3f}",
|
||
"-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", "-b:a", "192k", out]
|
||
|
||
|
||
def composite_cmd(still: str, audios: list, ass: str, segs: list, rowh: int, out: str, dur: float) -> list:
|
||
"""loop the stacked still for the whole group, concat each panel's audio in order, burn the
|
||
per-segment subtitles, and outline the active row during its narration. segs=[(start,end)]."""
|
||
n = len(audios)
|
||
hl = "".join(
|
||
f"drawbox=x=0:y={i*rowh}:w={W}:h={rowh}:color=yellow@0.85:t=6:enable='between(t,{s:.2f},{e:.2f})',"
|
||
for i, (s, e) in enumerate(segs))
|
||
cmd = ["ffmpeg", "-y", "-loop", "1", "-i", still]
|
||
for a in audios:
|
||
cmd += ["-i", a]
|
||
aconcat = "".join(f"[{i+1}:a]" for i in range(n)) + f"concat=n={n}:v=0:a=1[a]"
|
||
fc = f"[0:v]{hl}ass={ass}[v];{aconcat}"
|
||
return cmd + ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{dur:.3f}",
|
||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||
|
||
|
||
class CompositeInput(BaseModel):
|
||
# ordered members: [{panel_uri, audio_uri, narration_text, bbox?, page_uri?}]. bbox=[x,y,w,h] on
|
||
# the source page + page_uri enable the camera-traversal look; absent -> vstack fallback.
|
||
panels: list = []
|
||
panel_id: str = "" # leader panel id -> clip key (assemble picks up one clip per group)
|
||
|
||
|
||
def _img_size(path: str) -> tuple:
|
||
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
|
||
"stream=width,height", "-of", "csv=p=0:s=x", path], capture_output=True, text=True)
|
||
w, h = r.stdout.strip().split("x")
|
||
return int(w), int(h)
|
||
|
||
|
||
@app.post("/render/composite")
|
||
async def render_composite(data: CompositeInput):
|
||
if len(data.panels) < 2:
|
||
raise HTTPException(400, "composite needs >=2 panels")
|
||
tag = uuid.uuid4().hex[:8]
|
||
auds = [transport.get(p["audio_uri"], f"{SHM}/cmp_{tag}_{i}.wav") for i, p in enumerate(data.panels)]
|
||
durs = [_audio_dur(a) or 3.0 for a in auds]
|
||
segs, t = [], 0.0
|
||
for d in durs:
|
||
segs.append((t, t + d)); t += d
|
||
n = len(data.panels)
|
||
ass, out = f"{SHM}/cmp_{tag}.ass", f"{SHM}/cmp_{tag}.mp4"
|
||
_ass_multi([(data.panels[i].get("narration_text", ""), segs[i][0], segs[i][1]) for i in range(n)], ass)
|
||
|
||
# camera-traversal path: needs a shared page + a bbox per member (grouping keeps groups single-page)
|
||
bboxes = [p.get("bbox") for p in data.panels]
|
||
page_uri = data.panels[0].get("page_uri")
|
||
if page_uri and all(b and len(b) == 4 for b in bboxes):
|
||
page = transport.get(page_uri, f"{SHM}/cmp_{tag}_pg.png")
|
||
pw, ph = _img_size(page)
|
||
subprocess.run(traverse_cmd(page, auds, ass, segs, bboxes, pw, ph, out, t),
|
||
check=True, capture_output=True)
|
||
os.remove(page)
|
||
else: # fallback: legacy vstack (black gutters) when geometry is unavailable
|
||
imgs = [transport.get(p["panel_uri"], f"{SHM}/cmp_{tag}_{i}.png") for i, p in enumerate(data.panels)]
|
||
rowh = H // n
|
||
still = f"{SHM}/cmp_{tag}_s.png"
|
||
subprocess.run(_stack_still_cmd(imgs, still, rowh), check=True, capture_output=True)
|
||
subprocess.run(composite_cmd(still, auds, ass, segs, rowh, out, t), check=True, capture_output=True)
|
||
for p in imgs + [still]:
|
||
os.remove(p)
|
||
manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"])
|
||
uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
for p in auds + [ass, out]:
|
||
os.remove(p)
|
||
return {"clip_uri": uri, "duration": t}
|
||
|
||
|
||
# director groups: render a beat's member panels as sequential FULL-FRAME shots (each its own
|
||
# scene_cmd clip: blurred bg + ken-burns + its narration), then xfade-chain them into one clip with
|
||
# the per-member transition. replaces the vstack/traverse composite look with real transitioned crops.
|
||
class GroupInput(BaseModel):
|
||
# ordered members: [{panel_uri, audio_uri, narration_text, camera, transition}]
|
||
panels: list = []
|
||
panel_id: str = "" # leader id -> clip key
|
||
|
||
|
||
@app.post("/render/group")
|
||
async def render_group(data: GroupInput):
|
||
if len(data.panels) < 2:
|
||
raise HTTPException(400, "group needs >=2 panels")
|
||
tag = uuid.uuid4().hex[:8]
|
||
clips, durs, cleanup = [], [], []
|
||
for i, p in enumerate(data.panels):
|
||
img = transport.get(p["panel_uri"], f"{SHM}/grp_{tag}_{i}.png")
|
||
aud = transport.get(p["audio_uri"], f"{SHM}/grp_{tag}_{i}.wav")
|
||
dur = _audio_dur(aud) or 3.0
|
||
ass = f"{SHM}/grp_{tag}_{i}.ass"
|
||
_ass(p.get("narration_text", ""), dur, ass)
|
||
out = f"{SHM}/grp_{tag}_{i}.mp4"
|
||
subprocess.run(scene_cmd(img, aud, ass, out, dur, p.get("camera") or {}),
|
||
check=True, capture_output=True)
|
||
clips.append(out)
|
||
durs.append(_audio_dur(out) or (dur + PAD_S)) # full clip length incl. trailing pad
|
||
cleanup += [img, aud, ass, out]
|
||
|
||
# transition INTO member i+1 = member i's transition (transition out of the leaving shot).
|
||
trans = [(p.get("transition") or "cut") for p in data.panels[:-1]]
|
||
fg, vmap, amap = _xfade_chain(durs, trans)
|
||
final = f"{SHM}/grp_{tag}.mp4"
|
||
cmd = ["ffmpeg", "-y"]
|
||
for c in clips:
|
||
cmd += ["-i", c]
|
||
cmd += ["-filter_complex", fg, "-map", vmap, "-map", amap,
|
||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", final]
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
|
||
manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"])
|
||
uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(final, uri)
|
||
total = _audio_dur(final) or sum(durs)
|
||
for f in cleanup + [final]:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
return {"clip_uri": uri, "duration": total}
|
||
|
||
|
||
# scene-level narration: ONE narration+audio spans a whole beat, shown over its member panels as a
|
||
# ken-burns montage. the beat's single subtitle holds across every image; images split the narration
|
||
# duration equally. (per-panel narration is gone at this granularity — the beat is the story unit.)
|
||
class BeatInput(BaseModel):
|
||
panel_uris: list = [] # ordered member images (>=1)
|
||
audio_uri: str = "" # the beat's single narration audio
|
||
narration_text: str = ""
|
||
cameras: list = [] # per-image camera block; short/empty -> default zoom_in
|
||
weights: list = [] # 185: per-image relative screen-time; short/empty/degenerate -> equal split
|
||
panel_id: str = "" # leader id -> clip key
|
||
|
||
|
||
BEAT_MIN_PANEL_S = 1.0 # 185: no member panel flashes by faster than this
|
||
|
||
|
||
def _beat_slices(D: float, n: int, weights: list = None) -> list:
|
||
"""185: split beat duration D across n member panels by content weight, not evenly.
|
||
Each panel gets >= BEAT_MIN_PANEL_S so a low-weight panel never flashes. Degenerate input
|
||
(no/short weights, non-positive sum, or D too small for the floors) -> deterministic equal split.
|
||
Pure + total: sum(result) == D always."""
|
||
if n <= 0:
|
||
return []
|
||
if not weights or len(weights) < n or D <= n * BEAT_MIN_PANEL_S:
|
||
return [D / n] * n
|
||
w = [max(0.0, float(x)) for x in weights[:n]]
|
||
s = sum(w)
|
||
if s <= 0:
|
||
return [D / n] * n
|
||
floor = n * BEAT_MIN_PANEL_S
|
||
free = D - floor # distribute only the time above the floors by weight
|
||
return [BEAT_MIN_PANEL_S + free * (wi / s) for wi in w]
|
||
|
||
|
||
def _split_cues(text: str) -> list:
|
||
"""187: break the beat's one flowing narration into sentence/phrase cues. Split on sentence-ending
|
||
punctuation (kept) and newlines so short exclamations like 'Hold up.' stay their own cue."""
|
||
text = (text or "").strip()
|
||
if not text:
|
||
return []
|
||
parts = re.split(r"(?<=[.!?…。！？])\s+|\n+", text)
|
||
return [p.strip() for p in parts if p and p.strip()]
|
||
|
||
|
||
def cue_plan(text: str, D: float, slices: list) -> list:
|
||
"""187: a timed subtitle plan for one beat. Each cue = one sentence, timed along the SAME [0,D]
|
||
timeline as the per-panel slices, so a cue never appears before the panel on screen when it starts
|
||
(no future-panel facts leak early). Cue durations are length-weighted with a readable floor (reusing
|
||
_beat_slices); each cue is mapped to the member panel whose window contains its start. Pure/testable.
|
||
Returns [{text, start, end, panel_index}] in order."""
|
||
cues = _split_cues(text)
|
||
if not cues or D <= 0:
|
||
return []
|
||
durs = _beat_slices(D, len(cues), [len(c) for c in cues]) # length-weighted, min-hold, equal fallback
|
||
bounds, t = [], 0.0
|
||
for s in slices: # panel on-screen windows
|
||
bounds.append((t, t + s)); t += s
|
||
events, t = [], 0.0
|
||
for c, d in zip(cues, durs):
|
||
start, end = t, min(t + d, D)
|
||
pi = next((k for k, (a, b) in enumerate(bounds) if a <= start < b), max(0, len(bounds) - 1))
|
||
# clamp start to the panel's reveal so the caption is never ahead of its image
|
||
events.append({"text": c, "start": max(start, bounds[pi][0]) if bounds else start,
|
||
"end": end, "panel_index": pi})
|
||
t = end
|
||
return events
|
||
|
||
|
||
def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list = None,
|
||
weights: list = None) -> list:
|
||
"""ffmpeg: N images shown as a ken-burns montage under ONE narration audio; the last image holds
|
||
through the trailing pad. each image = blurred-fill bg + fitted panel + its camera move; the single
|
||
burned subtitle (in `ass`) spans the whole beat. 185: per-image screen-time is content-weighted
|
||
(see _beat_slices), equal split when weights are absent. pure -> testable without S3."""
|
||
cameras = cameras or []
|
||
n, fps = len(imgs), 25
|
||
slices = _beat_slices(D, n, weights) # 185: content-weighted, equal-split fallback
|
||
# one frame per image (no -loop): zoompan d=frames expands that single frame to exactly `frames`
|
||
# output frames = seg seconds. looping instead would feed many frames and zoompan multiplies each.
|
||
cmd = ["ffmpeg", "-y"]
|
||
for img in imgs:
|
||
cmd += ["-i", img]
|
||
cmd += ["-i", audio]
|
||
parts = []
|
||
for i in range(n):
|
||
cam = cameras[i] if i < len(cameras) else {}
|
||
seg = slices[i] + (PAD_S if i == n - 1 else 0.0)
|
||
frames = max(1, int(seg * fps))
|
||
parts.append(
|
||
f"[{i}:v]split=2[bg{i}][fg{i}];"
|
||
f"[bg{i}]scale={W}:{H}:force_original_aspect_ratio=increase,crop={W}:{H},boxblur=20:2[bgb{i}];"
|
||
f"[fg{i}]scale={W}:{H}:force_original_aspect_ratio=decrease[fgs{i}];"
|
||
f"[bgb{i}][fgs{i}]overlay=(W-w)/2:(H-h)/2,{_motion(cam, frames)}[v{i}]"
|
||
)
|
||
concat_in = "".join(f"[v{i}]" for i in range(n))
|
||
fc = (";".join(parts) + f";{concat_in}concat=n={n}:v=1:a=0[vc];"
|
||
f"[vc]ass={ass}[v];[{n}:a]apad=pad_dur={PAD_S:.3f}[a]")
|
||
cmd += ["-filter_complex", fc, "-map", "[v]", "-map", "[a]",
|
||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||
return cmd
|
||
|
||
|
||
@app.post("/render/beat")
|
||
async def render_beat(data: BeatInput):
|
||
if not data.panel_uris:
|
||
raise HTTPException(400, "beat needs >=1 panel")
|
||
tag = uuid.uuid4().hex[:8]
|
||
audio = transport.get(data.audio_uri, f"{SHM}/bt_{tag}.wav")
|
||
D = _audio_dur(audio) or 4.0
|
||
imgs = [transport.get(u, f"{SHM}/bt_{tag}_{i}.png") for i, u in enumerate(data.panel_uris)]
|
||
ass = f"{SHM}/bt_{tag}.ass"
|
||
# 187: timed sentence cues instead of one beat-long paragraph; same slice timeline as the montage,
|
||
# so each line surfaces with its panel. Empty/unsplittable text -> single caption (old behaviour).
|
||
slices = _beat_slices(D, len(imgs), data.weights)
|
||
cues = cue_plan(data.narration_text, D, slices)
|
||
cams = data.cameras or []
|
||
if cues:
|
||
cues[-1]["end"] = D + PAD_S # hold the closing line through the trailing pad
|
||
# 188: each cue dodges its panel's subject (top vs bottom) via the director's focus point.
|
||
_ass_multi([(c["text"], c["start"], c["end"],
|
||
_sub_align(cams[c["panel_index"]] if c["panel_index"] < len(cams) else {}))
|
||
for c in cues], ass)
|
||
else:
|
||
_ass_multi([(data.narration_text, 0.0, D + PAD_S)], ass)
|
||
out = f"{SHM}/bt_{tag}.mp4"
|
||
subprocess.run(beat_cmd(imgs, audio, ass, out, D, data.cameras, data.weights),
|
||
check=True, capture_output=True)
|
||
|
||
manga_id, chapter_id = _mc_from_uri(data.panel_uris[0])
|
||
uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
total = _audio_dur(out) or (D + PAD_S)
|
||
for f in imgs + [audio, ass, out]:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
# 187: cue plan (text/timing/panel_index per line) returned so the orchestrator can persist it for
|
||
# the review UI to edit mapping+timing, and so #183's collage can key reveals off the same cues.
|
||
return {"clip_uri": uri, "duration": total, "cues": cues}
|
||
|
||
|
||
# --- 183: animated manga collage ------------------------------------------------------------------
|
||
# A beat's member panels laid out as a motion-comic collage instead of a full-frame ken-burns montage:
|
||
# a blurred plate of the dominant panel fills the frame, sharp aspect-fit panels rest in a planned
|
||
# template (collage.plan_layout), and non-dominant panels slide into place over a brief entrance while
|
||
# the dominant one resolves by scale. Holds stay crisp. Behind the pipeline's COLLAGE flag.
|
||
# ponytail ceilings (visual polish, QA-tuned against the reference video, no still to check here):
|
||
# - transition-only directional/radial MOTION BLUR is not applied (clean slide/scale entrance); the
|
||
# beat-to-beat "streak" transition still rides the assemble-stage xfade. Add tblend accumulation
|
||
# when a fixture shows the clean slide reads too flat.
|
||
# - per-panel slow "drift" during the hold is omitted (static hold); add a gentle zoompan when needed.
|
||
class CollageInput(BaseModel):
|
||
panel_uris: list = [] # ordered member images (reading order), 1..4
|
||
audio_uri: str = ""
|
||
narration_text: str = ""
|
||
weights: list = [] # 185 timing hints (also picks the dominant panel = argmax)
|
||
cameras: list = [] # for 188 subtitle subject-dodge
|
||
rtl: bool = True
|
||
active: int = -1 # dominant panel index; <0 -> argmax(weights) or 0
|
||
panel_id: str = ""
|
||
|
||
|
||
def collage_cmd(imgs, plate_i, rects, entrances, z_order, audio, ass, out, D, trans):
|
||
"""ffmpeg graph: blurred plate of imgs[plate_i] + each sharp panel scaled to its resting rect with a
|
||
restrained drop shadow, composited back-to-front (z_order), non-dominant panels sliding in from their
|
||
entrance offset over `trans` seconds. One narration audio; burned cues in `ass`. Pure -> testable."""
|
||
n, T = len(imgs), D + PAD_S
|
||
cmd = ["ffmpeg", "-y"]
|
||
for img in imgs:
|
||
cmd += ["-loop", "1", "-i", img] # still held for the whole clip; -t caps output
|
||
cmd += ["-i", audio]
|
||
parts = [f"[{plate_i}:v]scale={W}:{H}:force_original_aspect_ratio=increase,"
|
||
f"crop={W}:{H},boxblur=24:2,setsar=1[bg]"]
|
||
base = "bg"
|
||
for k, i in enumerate(z_order): # shadows first, at rest positions (static)
|
||
x, y, w, h = (int(round(v)) for v in rects[i])
|
||
parts.append(f"[{base}]drawbox=x={x + 7}:y={y + 7}:w={w}:h={h}:color=black@0.35:t=fill[sh{k}]")
|
||
base = f"sh{k}"
|
||
for i in range(n):
|
||
w, h = int(round(rects[i][2])), int(round(rects[i][3]))
|
||
parts.append(f"[{i}:v]scale={w}:{h},setsar=1[p{i}]")
|
||
cur = base
|
||
for k, i in enumerate(z_order):
|
||
x, y = int(round(rects[i][0])), int(round(rects[i][1]))
|
||
dx, dy = entrances[i]
|
||
# ease from (x+dx, y+dy) to (x, y) over `trans`s, then hold. commas safe inside the '...' quotes.
|
||
ease = f"max(0,1-t/{trans:.3f})"
|
||
parts.append(f"[{cur}][p{i}]overlay=x='{x}+({dx})*{ease}':y='{y}+({dy})*{ease}'[o{k}]")
|
||
cur = f"o{k}"
|
||
fc = ";".join(parts) + f";[{cur}]ass={ass}[v];[{n}:a]apad=pad_dur={PAD_S:.3f}[a]"
|
||
cmd += ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{T:.3f}",
|
||
"-r", "30", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||
return cmd
|
||
|
||
|
||
@app.post("/render/collage")
|
||
async def render_collage(data: CollageInput):
|
||
if not data.panel_uris:
|
||
raise HTTPException(400, "collage needs >=1 panel")
|
||
uris = data.panel_uris[:4] # planner templates cap at 4 readable panels
|
||
n = len(uris)
|
||
tag = uuid.uuid4().hex[:8]
|
||
audio = transport.get(data.audio_uri, f"{SHM}/cl_{tag}.wav")
|
||
D = _audio_dur(audio) or 4.0
|
||
imgs = [transport.get(u, f"{SHM}/cl_{tag}_{i}.png") for i, u in enumerate(uris)]
|
||
aspects = []
|
||
for p in imgs:
|
||
w, h = _img_size(p)
|
||
aspects.append(w / h if h else 1.0)
|
||
active = data.active if 0 <= data.active < n else (
|
||
max(range(n), key=lambda i: data.weights[i]) if len(data.weights) >= n else 0)
|
||
lay = plan_layout(aspects, data.rtl, active, (W, H))
|
||
|
||
# 185/187: content-weighted slices + timed cues, same as the montage path.
|
||
slices = _beat_slices(D, n, data.weights)
|
||
cues = cue_plan(data.narration_text, D, slices)
|
||
ass = f"{SHM}/cl_{tag}.ass"
|
||
if cues:
|
||
cues[-1]["end"] = D + PAD_S
|
||
cams = data.cameras or []
|
||
_ass_multi([(c["text"], c["start"], c["end"],
|
||
_sub_align(cams[c["panel_index"]] if c["panel_index"] < len(cams) else {}))
|
||
for c in cues], ass)
|
||
else:
|
||
_ass_multi([(data.narration_text, 0.0, D + PAD_S)], ass)
|
||
|
||
out = f"{SHM}/cl_{tag}.mp4"
|
||
subprocess.run(collage_cmd(imgs, active, lay["rects"], lay["entrances"], lay["z_order"],
|
||
audio, ass, out, D, lay["transition_s"]), check=True, capture_output=True)
|
||
manga_id, chapter_id = _mc_from_uri(uris[0])
|
||
uri = f"s3://manga/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
total = _audio_dur(out) or (D + PAD_S)
|
||
for f in imgs + [audio, ass, out]:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
return {"clip_uri": uri, "duration": total, "cues": cues, "template": lay["template"]}
|
||
|
||
|
||
# #6 non-generic transitions: map the direction schema's transition.type -> (xfade name, seconds).
|
||
# "cut" stays a hard cut via the concat fast-path; the rest re-encode through an xfade chain.
|
||
XFADE = {
|
||
"cut": ("fade", 0.0),
|
||
"crossfade": ("fade", 0.5),
|
||
"dissolve": ("dissolve", 0.5),
|
||
"fade_black": ("fadeblack", 0.6),
|
||
"fade_white": ("fadewhite", 0.6),
|
||
"wipe_left": ("wipeleft", 0.4),
|
||
"wipe_right": ("wiperight", 0.4),
|
||
"push": ("slideleft", 0.4),
|
||
}
|
||
|
||
# Chapter assembly used to open every clip in one ffmpeg process. A long chapter therefore created
|
||
# one enormous xfade graph: N decoders + N-1 full-frame filter stages, enough to exhaust RAM/VRAM and
|
||
# get ffmpeg SIGKILLed by the OOM killer. Assemble a bounded tree instead. Eight inputs keeps enough
|
||
# work in each encode to be efficient without letting decoder/filter threads grow with chapter size.
|
||
ASSEMBLE_BATCH = max(2, int(os.environ.get("ASSEMBLE_BATCH", "8")))
|
||
FFMPEG_THREADS = max(1, int(os.environ.get("FFMPEG_THREADS", "2")))
|
||
|
||
|
||
def _xfade_chain(durs: list, trans: list):
|
||
"""build a filter_complex that xfades N clips with per-boundary transitions, keeping audio in
|
||
sync via matching acrossfade. trans[i] is the transition OUT of clip i (boundary i->i+1).
|
||
returns (filtergraph, video_label, audio_label). offsets accumulate as clips overlap."""
|
||
# A clip whose duration probed as 0/unreadable must not poison the chain: with dur=0 the offset
|
||
# accumulator would run BACKWARDS (cum += dur - td), swallowing every later clip into a frozen
|
||
# overlap near the middle. Floor to a small positive length so the timeline stays monotonic.
|
||
durs = [d if (d and d > 0.1) else 0.1 for d in durs]
|
||
parts, vlast, alast, cum = [], "[0:v]", "[0:a]", durs[0]
|
||
for i in range(1, len(durs)):
|
||
name, td = XFADE.get(trans[i - 1] if i - 1 < len(trans) else "cut", XFADE["cut"])
|
||
td = max(0.05, min(td, durs[i - 1] - 0.05, durs[i] - 0.05)) # overlap fits in both clips
|
||
off = max(cum - td, 0)
|
||
parts.append(f"{vlast}[{i}:v]xfade=transition={name}:duration={td:.3f}:offset={off:.3f}[v{i}]")
|
||
parts.append(f"{alast}[{i}:a]acrossfade=d={td:.3f}[a{i}]")
|
||
vlast, alast, cum = f"[v{i}]", f"[a{i}]", cum + durs[i] - td
|
||
return ";".join(parts), vlast, alast
|
||
|
||
|
||
def _assemble_once(inputs: list[str], trans: list[str], out: str):
|
||
"""Assemble one bounded batch. `trans[i]` is the transition out of inputs[i]."""
|
||
fancy = len(inputs) >= 2 and any(t not in ("", "cut") for t in trans[:len(inputs) - 1])
|
||
if fancy:
|
||
durs = [_audio_dur(p) for p in inputs]
|
||
fg, vmap, amap = _xfade_chain(durs, trans)
|
||
cmd = ["ffmpeg", "-y", "-filter_complex_threads", str(FFMPEG_THREADS)]
|
||
# Input-side -threads limits each decoder; otherwise ffmpeg may create a decoder thread pool
|
||
# for every input in the batch in addition to the filter and libx264 pools.
|
||
for p in inputs:
|
||
cmd += ["-threads", "1", "-i", p]
|
||
cmd += ["-filter_complex", fg, "-map", vmap, "-map", amap,
|
||
"-c:v", "libx264", "-threads", str(FFMPEG_THREADS), "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", "-b:a", "192k", out]
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
return
|
||
|
||
# A cut-only batch is concatenated with the concat FILTER, not -c copy or the concat demuxer.
|
||
# The tree feeds xfade intermediates (irregular video timestamps) back in as inputs; the concat
|
||
# demuxer + -vsync cfr DROPS video frames to force CFR while audio survives -> the video ends up
|
||
# minutes short and freezes on a frame with narration playing on (the "one image, narration behind
|
||
# it" bug). The concat filter decodes and re-times every segment, so no frames are dropped. It needs
|
||
# N decoders, but the tree already bounds a batch to ASSEMBLE_BATCH inputs, so memory stays capped.
|
||
n = len(inputs)
|
||
cmd = ["ffmpeg", "-y"]
|
||
for p in inputs:
|
||
cmd += ["-i", p]
|
||
pre = "".join(f"[{i}:v]setsar=1,fps=30[v{i}];" for i in range(n))
|
||
fg = pre + "".join(f"[v{i}][{i}:a]" for i in range(n)) + f"concat=n={n}:v=1:a=1[v][a]"
|
||
cmd += ["-filter_complex", fg, "-map", "[v]", "-map", "[a]",
|
||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p",
|
||
"-threads", str(FFMPEG_THREADS), "-c:a", "aac", "-b:a", "192k", out]
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
|
||
|
||
def _assemble_batched(inputs: list[str], transitions: list[str], out: str, tag: str,
|
||
cleanup: list[str]):
|
||
"""Bounded tree assembly preserving every original transition.
|
||
|
||
Each intermediate carries the transition OUT of its final source clip. That becomes the boundary
|
||
transition between intermediates in the next round, so batching does not change direction. A
|
||
chapter of 100 clips with batch=8 uses at most eight simultaneous inputs instead of 100.
|
||
"""
|
||
items = [{"path": p,
|
||
"out_transition": transitions[i] if i < len(transitions) else "cut",
|
||
"temporary": False}
|
||
for i, p in enumerate(inputs)]
|
||
round_no = 0
|
||
while len(items) > 1:
|
||
final_round = len(items) <= ASSEMBLE_BATCH
|
||
next_items = []
|
||
for start in range(0, len(items), ASSEMBLE_BATCH):
|
||
group = items[start:start + ASSEMBLE_BATCH]
|
||
if len(group) == 1 and not final_round:
|
||
next_items.append(group[0])
|
||
continue
|
||
target = out if final_round else f"{SHM}/asm_{tag}_r{round_no}_{start // ASSEMBLE_BATCH}.mp4"
|
||
_assemble_once([x["path"] for x in group], [x["out_transition"] for x in group], target)
|
||
if not final_round:
|
||
cleanup.append(target)
|
||
next_items.append({"path": target, "out_transition": group[-1]["out_transition"],
|
||
"temporary": not final_round})
|
||
# Intermediates from the previous round are no longer needed. Keep cleanup idempotent: remove
|
||
# them here for low /dev/shm usage and from the cleanup list so endpoint cleanup won't retry.
|
||
for item in items:
|
||
p = item["path"]
|
||
if item["temporary"] and os.path.exists(p):
|
||
os.remove(p)
|
||
if p in cleanup:
|
||
cleanup.remove(p)
|
||
items = next_items
|
||
round_no += 1
|
||
|
||
|
||
class AssembleInput(BaseModel):
|
||
clip_uris: list
|
||
transitions: list = [] # #6 per-clip transition-out type; empty/all-"cut" -> fast concat
|
||
chapter_id: str = ""
|
||
manga_id: str = ""
|
||
|
||
|
||
def _music_filter(gain: str) -> str:
|
||
"""#13 loop a music bed under the narration, ducked by sidechain compression keyed on the
|
||
narration itself, then mix. duration=first ends the mix with the video's audio."""
|
||
return (
|
||
f"[1:a]volume={gain}[bed];"
|
||
"[0:a]asplit=2[nar][key];"
|
||
"[bed][key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400[duck];"
|
||
"[nar][duck]amix=inputs=2:duration=first:dropout_transition=0[a]"
|
||
)
|
||
|
||
|
||
def _add_music_bed(video: str, tag: str, cleanup: list) -> str:
|
||
"""mix MUSIC_BED under the assembled chapter. no-op (returns input) when unset or on failure."""
|
||
if not MUSIC_BED:
|
||
return video
|
||
music = MUSIC_BED
|
||
if MUSIC_BED.startswith("s3://"):
|
||
music = transport.get(MUSIC_BED, f"{SHM}/bed_{tag}{os.path.splitext(MUSIC_BED)[1] or '.mp3'}")
|
||
cleanup.append(music)
|
||
mixed = f"{SHM}/chapter_{tag}_mus.mp4"
|
||
try:
|
||
subprocess.run(
|
||
["ffmpeg", "-y", "-i", video, "-stream_loop", "-1", "-i", music,
|
||
"-filter_complex", _music_filter(MUSIC_GAIN), "-map", "0:v", "-map", "[a]",
|
||
"-c:v", "copy", "-c:a", "aac", "-b:a", "192k", "-shortest", mixed],
|
||
check=True, capture_output=True)
|
||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||
print(f"[render] music bed skipped: {e}", flush=True)
|
||
if os.path.exists(mixed):
|
||
os.remove(mixed)
|
||
return video
|
||
cleanup.append(mixed)
|
||
return mixed
|
||
|
||
|
||
@app.post("/render/assemble")
|
||
async def assemble(data: AssembleInput):
|
||
if not data.clip_uris:
|
||
raise HTTPException(400, "no clips to assemble")
|
||
tag = uuid.uuid4().hex[:8]
|
||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||
locals_ = list(pool.map(
|
||
lambda iu: transport.get(iu[1], f"{SHM}/asm_{tag}_{iu[0]}.mp4"),
|
||
enumerate(data.clip_uris),
|
||
))
|
||
out = f"{SHM}/chapter_{tag}.mp4"
|
||
cleanup = list(locals_) + [out]
|
||
|
||
fancy = len(locals_) >= 2 and any(t not in ("", "cut") for t in data.transitions)
|
||
if fancy:
|
||
_assemble_batched(locals_, data.transitions, out, tag, cleanup)
|
||
else:
|
||
# all hard cuts: stream-copy concat (no re-encode) -- unchanged fast path.
|
||
listfile = f"{SHM}/asm_{tag}.txt"; cleanup.append(listfile)
|
||
with open(listfile, "w") as f:
|
||
f.write("".join(f"file '{p}'\n" for p in locals_))
|
||
subprocess.run(["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", listfile, "-c", "copy", out],
|
||
check=True, capture_output=True)
|
||
|
||
out = _add_music_bed(out, tag, cleanup)
|
||
|
||
manga_id, chapter_id = _mc_from_uri(data.clip_uris[0])
|
||
uri = f"s3://manga/{manga_id}/{chapter_id}/chapter.mp4"
|
||
transport.put(out, uri)
|
||
for p in cleanup:
|
||
os.remove(p)
|
||
return {"video_uri": uri} # orchestrator save_video reads video_uri
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
return {"status": "ok"}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
# self-check: ass file well-formed; ffmpeg produces a clip from a synthetic image + silence.
|
||
import shutil
|
||
_ass("Hello\nworld", 2.0, f"{SHM}/t.ass")
|
||
txt = open(f"{SHM}/t.ass").read()
|
||
assert "Hello world" in txt and "Dialogue:" in txt # short cue -> one line
|
||
|
||
# 188: subtitle redesign. wrap to <=2 lines, off-mode burns nothing, subject-dodge picks top/bottom.
|
||
assert _wrap2("one two three four five six seven eight", 12).count("\\N") == 1 # exactly two lines
|
||
long = _wrap2(" ".join(["word"] * 40), 12)
|
||
assert long.count("\\N") == 1 and long.endswith("…") # 3rd line truncated, never a paragraph
|
||
_ass_multi([("hidden", 0.0, 2.0)], f"{SHM}/off.ass", mode="off")
|
||
assert "Dialogue:" not in open(f"{SHM}/off.ass").read() # off -> nothing burned
|
||
_ass_multi([("boxed", 0.0, 2.0)], f"{SHM}/bx.ass", mode="boxed")
|
||
assert ",3," in open(f"{SHM}/bx.ass").read() # BorderStyle 3 = box
|
||
assert _sub_align({"to": [0.5, 0.8]}) == 8 and _sub_align({"to": [0.5, 0.3]}) == 2 # dodge low subject
|
||
_ass_multi([("top", 0.0, 2.0, 8)], f"{SHM}/an.ass")
|
||
assert "{\\an8}top" in open(f"{SHM}/an.ass").read() # per-cue alignment override
|
||
for f in (f"{SHM}/off.ass", f"{SHM}/bx.ass", f"{SHM}/an.ass"):
|
||
os.remove(f)
|
||
# #8 motion: each effect yields a distinct, well-formed zoompan expr; dolly aims at its target.
|
||
assert "z='1.0'" in _motion({"effect": "static"}, 50) # truly still
|
||
assert "on/49" in _motion({"effect": "pan_left"}, 50) # travels over the clip
|
||
assert "0.8*iw" in _motion({"effect": "dolly_to_subject", "to": [0.8, 0.2]}, 50)
|
||
assert _motion({}, 50) == _motion({"effect": "zoom_in"}, 50) # default == ken burns
|
||
|
||
# 185: content-weighted beat slices. always sum to D; floor honored; degenerate -> equal split.
|
||
eq = _beat_slices(12.0, 3)
|
||
assert eq == [4.0, 4.0, 4.0], eq # no weights -> equal
|
||
w = _beat_slices(12.0, 3, [3, 1, 1]) # heavier panel gets more time
|
||
assert abs(sum(w) - 12.0) < 1e-6 and w[0] > w[1] and min(w) >= BEAT_MIN_PANEL_S, w
|
||
assert all(abs(s - 0.8) < 1e-9 for s in _beat_slices(2.4, 3, [3, 1, 1])) # below floors -> equal
|
||
assert _beat_slices(12.0, 3, [0, 0, 0]) == [4.0, 4.0, 4.0] # zero-sum -> equal split
|
||
assert _beat_slices(12.0, 3, [5]) == [4.0, 4.0, 4.0] # short weights -> equal split
|
||
|
||
# 187: timed cue plan. "Hold up." must be its own cue and must not start before its panel's window.
|
||
sl = _beat_slices(9.0, 3, [1, 1, 1]) # three 3.0s panel windows
|
||
cues = cue_plan("She sees the reflection. Those are psycho eyes. Hold up.", 9.0, sl)
|
||
assert [c["text"] for c in cues] == ["She sees the reflection.", "Those are psycho eyes.", "Hold up."]
|
||
assert cues[0]["start"] == 0.0 # first cue opens the beat
|
||
for c in cues: # every cue starts within/after its panel
|
||
assert c["start"] >= c["panel_index"] * 3.0 - 1e-6, c
|
||
assert cues[-1]["panel_index"] == 2, cues[-1] # last line maps to the last panel
|
||
assert cues == cue_plan("She sees the reflection. Those are psycho eyes. Hold up.", 9.0, sl) # deterministic
|
||
assert cue_plan("", 9.0, sl) == [] # empty narration -> no cues (paragraph fallback)
|
||
# Long chapter assembly is bounded, and the transition out of a batch's last source clip is used
|
||
# to join that batch to the next one. Mock the encoder so this remains a cheap pure orchestration test.
|
||
_real_once = _assemble_once
|
||
_calls = []
|
||
try:
|
||
globals()["_assemble_once"] = lambda ins, trs, out: _calls.append((list(ins), list(trs), out))
|
||
_n = ASSEMBLE_BATCH * 2 + 1
|
||
_batch_trans = [f"t{i}" for i in range(_n)]
|
||
_assemble_batched([f"in{i}.mp4" for i in range(_n)], _batch_trans,
|
||
"/tmp/final.mp4", "batchck", [])
|
||
assert all(len(ins) <= ASSEMBLE_BATCH for ins, _, _ in _calls), _calls
|
||
assert _calls[-1][1][0] == _batch_trans[ASSEMBLE_BATCH - 1], _calls[-1]
|
||
assert _calls[-1][1][1] == _batch_trans[ASSEMBLE_BATCH * 2 - 1], _calls[-1]
|
||
finally:
|
||
globals()["_assemble_once"] = _real_once
|
||
if shutil.which("ffmpeg"):
|
||
aud = f"{SHM}/t.wav"; out = f"{SHM}/t.mp4"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1", aud],
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(aud) - 1.0) < 0.1, _audio_dur(aud) # #1: duration from audio
|
||
# both a wide-short and a tall webtoon-style panel must render (tall used to break pad)
|
||
cams = ({}, {"effect": "pan_right"}, {"effect": "dolly_to_subject", "to": [0.7, 0.3]})
|
||
for size, cam in zip(("200x60", "900x2200", "800x1200"), cams):
|
||
img = f"{SHM}/t.png"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=black:s={size}",
|
||
"-frames:v", "1", img], check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", out, 1.0, cam), check=True, capture_output=True)
|
||
assert os.path.getsize(out) > 0, size
|
||
os.remove(img)
|
||
# #13 padding: a 1.0s narration clip runs ~1.0+PAD_S with the trailing silence held.
|
||
img = f"{SHM}/tp.png"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=400x600",
|
||
"-frames:v", "1", img], check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", out, 1.0, pad=0.4), check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - 1.4) < 0.15, _audio_dur(out)
|
||
# #13 music bed: mixing a generated tone under the clip keeps duration and produces output.
|
||
bed = f"{SHM}/bed.wav"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "sine=frequency=220:duration=0.5", bed],
|
||
check=True, capture_output=True)
|
||
_saved = globals()["MUSIC_BED"]; globals()["MUSIC_BED"] = bed
|
||
cl = []
|
||
mixed = _add_music_bed(out, "selfck", cl)
|
||
assert mixed != out and os.path.exists(mixed) and abs(_audio_dur(mixed) - 1.4) < 0.2, _audio_dur(mixed)
|
||
globals()["MUSIC_BED"] = _saved
|
||
for p in (img, bed, *cl):
|
||
if os.path.exists(p): os.remove(p)
|
||
# #6 transitions: two real clips xfade into one chapter; graph offsets/labels well-formed.
|
||
fg, vmap, amap = _xfade_chain([1.0, 1.0], ["fade_white"])
|
||
assert "xfade=transition=fadewhite" in fg and vmap == "[v1]" and amap == "[a1]"
|
||
img = f"{SHM}/t.png"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=400x600",
|
||
"-frames:v", "1", img], check=True, capture_output=True)
|
||
c0, c1 = f"{SHM}/c0.mp4", f"{SHM}/c1.mp4"
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c0, 1.0), check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c1, 1.0), check=True, capture_output=True)
|
||
subprocess.run(["ffmpeg", "-y", "-i", c0, "-i", c1, "-filter_complex", fg,
|
||
"-map", vmap, "-map", amap, "-c:v", "libx264", "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", out], check=True, capture_output=True)
|
||
assert os.path.getsize(out) > 0
|
||
for p in (img, c0, c1):
|
||
os.remove(p)
|
||
# #6 composite: 2 panels + 2 audios -> one stacked clip; duration = sum, subs timed per row.
|
||
a2 = f"{SHM}/a2.wav"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1.5", a2],
|
||
check=True, capture_output=True)
|
||
i0, i1 = f"{SHM}/i0.png", f"{SHM}/i1.png"
|
||
for im, sz in ((i0, "300x200"), (i1, "500x300")):
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=gray:s={sz}",
|
||
"-frames:v", "1", im], check=True, capture_output=True)
|
||
segs = [(0.0, 1.0), (1.0, 2.5)]
|
||
still = f"{SHM}/stk.png"
|
||
subprocess.run(_stack_still_cmd([i0, i1], still, H // 2), check=True, capture_output=True)
|
||
_ass_multi([("first beat", *segs[0]), ("second beat", *segs[1])], f"{SHM}/t.ass")
|
||
subprocess.run(composite_cmd(still, [aud, a2], f"{SHM}/t.ass", segs, H // 2, out, 2.5),
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - 2.5) < 0.2, _audio_dur(out) # duration = sum of both narrations
|
||
for p in (a2, i0, i1, still):
|
||
os.remove(p)
|
||
# scene-level narration: 3 images under ONE 1.5s narration -> clip = D + PAD_S, single subtitle.
|
||
b0, b1, b2 = f"{SHM}/b0.png", f"{SHM}/b1.png", f"{SHM}/b2.png"
|
||
for im, sz in ((b0, "300x200"), (b1, "800x1200"), (b2, "500x900")):
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=navy:s={sz}",
|
||
"-frames:v", "1", im], check=True, capture_output=True)
|
||
bnar = f"{SHM}/bnar.wav"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1.5", bnar],
|
||
check=True, capture_output=True)
|
||
_ass("one flowing beat narration", 1.5 + PAD_S, f"{SHM}/t.ass")
|
||
cams = ({}, {"effect": "pan_right"}, {"effect": "dolly_to_subject", "to": [0.6, 0.4]})
|
||
subprocess.run(beat_cmd([b0, b1, b2], bnar, f"{SHM}/t.ass", out, 1.5, list(cams)),
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - (1.5 + PAD_S)) < 0.2, _audio_dur(out) # one narration spans N images
|
||
|
||
# 183: collage of the same 3 panels -> one clip of D+PAD, laid out by plan_layout (dominant=2).
|
||
aspects = [_img_size(p)[0] / _img_size(p)[1] for p in (b0, b1, b2)]
|
||
lay = plan_layout(aspects, rtl=True, active=2, frame=(W, H))
|
||
_ass_multi([("collage cue", 0.0, 1.5 + PAD_S)], f"{SHM}/t.ass")
|
||
subprocess.run(collage_cmd([b0, b1, b2], 2, lay["rects"], lay["entrances"], lay["z_order"],
|
||
bnar, f"{SHM}/t.ass", out, 1.5, lay["transition_s"]),
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - (1.5 + PAD_S)) < 0.2, _audio_dur(out) # collage clip = beat length
|
||
assert os.path.getsize(out) > 0
|
||
for p in (b0, b1, b2, bnar):
|
||
os.remove(p)
|
||
os.remove(aud); os.remove(out)
|
||
print("worker_render self-check ok (ffmpeg ran)")
|
||
else:
|
||
print("worker_render self-check ok (ffmpeg absent, ass-only)")
|
||
os.remove(f"{SHM}/t.ass")
|