kami
a9d64fe80a
Somebody watched chapter.mp4 for the first time. Two failures came out of it that no stage counter could see. chapter.mp4 is video 436.39s over audio 363.67s, so narration finishes 72.7s before the picture. The 49 clips are clean: all 25fps, video and audio agree to 0.03s, summing to 363.6s. A per-round probe puts the loss in the final round of _assemble_batched, which turns 359s of video into 100s while the audio survives. Round 0 is correct. Round 1 differs by holding a 7th input, the leftover clip that skips encoding, so the tree mixes concat output, xfade output and a raw clip. Not fixed. worker_render.py gains an FPS constant, fps normalization in the xfade branch to match concat, _stream_dur, and a self-check that compares video against audio instead of asserting the file is non-empty. That old check is how a 20% sync failure shipped. The fps inconsistency is real but not proven to be the shipped cause. Pinning -r on the output was tried and reverted: it drops frames to force CFR, which the concat branch comment already warned about. Panel 7 checked against the art has zero correct identity bindings out of two, and Seonho, the one character who matters, is unbound. bbox values are consumed as absolute pixels; on a 900x1650 panel that puts all six boxes in the top third, two inside a speech balloon. Identity therefore embeds crops of balloon edges and window frames, which is how confidence 0.9 lands on the wrong person. The colleague has no name in the story and was labelled Choi Haeseon; that row holds 25 of 26 assignments, so it is the label the pipeline stamps on any unnamed woman. Four caveats added. Two earlier claims are withdrawn in place: rescaling bbox by 1000 does not make the boxes correct, and the constraint is not 16 nameless rows needing names. Cast profiles already exist, since all 53 rows populate ref_image_uris and embedding_uri, but they are enrolled from the wrong crops. worker_render.py self-check passes. No pipeline ran. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
1038 lines
54 KiB
Python
1038 lines
54 KiB
Python
# worker_render.py — stage 9 part 2 rendering. FastAPI :8008.
|
||
# per-scene: motion (ken burns, or parallax when layers present) + burned subtitles -> mp4 clip.
|
||
# chapter assembly: concat clips with crossfades -> chapter.mp4. all outputs to minio.
|
||
# ponytail: ken burns for both cases for now; layer-parallax is the upgrade path when the
|
||
# depth renderer is dialed in — the layer_uris are already threaded through the input.
|
||
import os, re, uuid, subprocess
|
||
from concurrent.futures import ThreadPoolExecutor
|
||
from fastapi import FastAPI, HTTPException
|
||
from pydantic import BaseModel
|
||
import transport
|
||
from collage import plan_layout
|
||
|
||
app = FastAPI()
|
||
transport.install_logging(app, "render")
|
||
SHM = "/dev/shm"
|
||
W, H = 1080, 1920 # vertical 9:16
|
||
PAD_S = float(os.environ.get("PANEL_PAD_S", "0.4")) # #13 trailing silence per panel for pacing
|
||
MUSIC_BED = os.environ.get("MUSIC_BED", "") # #13 path/uri of a music track; empty -> no bed
|
||
MUSIC_GAIN = os.environ.get("MUSIC_GAIN", "0.18") # bed level before ducking
|
||
|
||
|
||
def _mc_from_uri(uri: str):
|
||
"""(manga_id, chapter_id) from s3://<bucket>/<manga_id>/<chapter_id>/...
|
||
the orchestrator doesn't pass ids to render/layers, but every input uri encodes them."""
|
||
parts = uri.replace("s3://", "").split("/")
|
||
return parts[1], parts[2]
|
||
|
||
|
||
def _ts(s):
|
||
h = int(s // 3600); m = int(s % 3600 // 60); sec = s % 60
|
||
return f"{h}:{m:02d}:{sec:05.2f}"
|
||
|
||
|
||
# 188: readable narration subtitles. Modes: off (burn nothing), minimal (small text, thin outline +
|
||
# soft shadow, NO box — least art occlusion), boxed (small text on a restrained ~50% box). Presets carry
|
||
# portrait/landscape safe-areas. ASS colour is &HAABBGGRR (alpha 00=opaque..FF=clear).
|
||
SUB_MODE = os.environ.get("SUB_MODE", "minimal").lower() # off | minimal | boxed
|
||
SUB_PRESETS = {
|
||
# (mode, orientation): fontsize, borderstyle(1=outline,3=box), outline/pad, shadow, marginV, side
|
||
("minimal", "portrait"): dict(fs=40, bs=1, outline=3, shadow=2, mv=0.055, side=110),
|
||
("minimal", "landscape"): dict(fs=32, bs=1, outline=3, shadow=2, mv=0.09, side=260),
|
||
("boxed", "portrait"): dict(fs=40, bs=3, outline=6, shadow=0, mv=0.055, side=110),
|
||
("boxed", "landscape"): dict(fs=32, bs=3, outline=6, shadow=0, mv=0.09, side=260),
|
||
}
|
||
|
||
|
||
def _wrap2(text: str, width: int) -> str:
|
||
"""188: keep a cue to at most 2 short lines. Greedy word-wrap to `width`; if it still needs a 3rd
|
||
line, truncate the 2nd with an ellipsis so a long cue never grows back into a paragraph block."""
|
||
words, lines, cur = (text or "").split(), [], ""
|
||
for w in words:
|
||
if cur and len(cur) + 1 + len(w) > width:
|
||
lines.append(cur); cur = w
|
||
if len(lines) == 2:
|
||
break
|
||
else:
|
||
cur = f"{cur} {w}".strip()
|
||
if len(lines) < 2:
|
||
lines.append(cur)
|
||
elif cur or len(words) > sum(len(l.split()) for l in lines):
|
||
lines[1] = lines[1].rstrip(".,") + "…" # more words remained than fit two lines
|
||
return "\\N".join(l for l in lines if l)
|
||
|
||
|
||
def _ass_multi(events, path, mode=None, orientation="portrait"):
|
||
"""Burn caption events. events: [(text, start, end)] or [(text, start, end, align)] where align is an
|
||
ASS numpad alignment (2=bottom-center default, 8=top-center to dodge a low subject). mode off -> no
|
||
Dialogue lines (nothing burned). Style comes from SUB_PRESETS (188)."""
|
||
mode = (mode or SUB_MODE)
|
||
if mode == "off":
|
||
events = []
|
||
p = SUB_PRESETS.get((mode, orientation), SUB_PRESETS[("minimal", "portrait")])
|
||
fill = "&H80000000" if p["bs"] == 3 else "&H00000000" # boxed: ~50% box; minimal: opaque outline
|
||
fmt = ("Name, Fontname, Fontsize, PrimaryColour, OutlineColour, BackColour, "
|
||
"Bold, BorderStyle, Outline, Shadow, Alignment, MarginL, MarginR, MarginV")
|
||
mv = int(p["mv"] * (W if orientation == "landscape" else H))
|
||
style = ("Def,DejaVu Sans,%d,&H00FFFFFF,%s,&H80000000,0,%d,%d,%d,2,%d,%d,%d"
|
||
% (p["fs"], fill, p["bs"], p["outline"], p["shadow"], p["side"], p["side"], mv))
|
||
width = 34 if orientation == "portrait" else 46
|
||
lines = ""
|
||
for ev in events:
|
||
t, s, e = ev[0], ev[1], ev[2]
|
||
align = ev[3] if len(ev) > 3 else 2
|
||
# {\anN} overrides alignment per line so a cue can jump to the top over a low subject.
|
||
lines += "Dialogue: 0,%s,%s,Def,{\\an%d}%s\n" % (_ts(s), _ts(e), align, _wrap2(t, width))
|
||
with open(path, "w") as f:
|
||
f.write(
|
||
"[Script Info]\nScriptType: v4.00+\nPlayResX: %d\nPlayResY: %d\nWrapStyle: 0\n\n"
|
||
"[V4+ Styles]\nFormat: %s\nStyle: %s\n\n"
|
||
"[Events]\nFormat: Layer, Start, End, Style, Text\n%s" % (W, H, fmt, style, lines)
|
||
)
|
||
|
||
|
||
def _sub_align(camera: dict) -> int:
|
||
"""188: dodge the subject where we can. The director's focus point (camera.to=[x,y], y normalized
|
||
top->bottom) is the one subject location we already know — if it sits low in frame, put the caption
|
||
at the TOP (an8) so it doesn't cover the face; otherwise keep it bottom (an2).
|
||
ponytail: no face/bubble detector yet — upgrade to real bbox avoidance when one is wired in."""
|
||
to = (camera or {}).get("to")
|
||
if to and len(to) >= 2 and float(to[1]) > 0.6:
|
||
return 8
|
||
return 2
|
||
|
||
|
||
def _ass(text: str, dur: float, path: str):
|
||
_ass_multi([(text, 0.0, dur)], path)
|
||
|
||
|
||
def _audio_dur(path: str) -> float:
|
||
"""clip length = narration length. scene_timing arrives empty, so probe the audio."""
|
||
r = subprocess.run(["ffprobe", "-v", "error", "-show_entries", "format=duration",
|
||
"-of", "default=nk=1:nw=1", path], capture_output=True, text=True)
|
||
try:
|
||
return float(r.stdout.strip())
|
||
except ValueError:
|
||
return 0.0
|
||
|
||
|
||
def _stream_dur(path: str, kind: str) -> float:
|
||
"""duration of one stream. `format=duration` is max(video,audio) and so hides A/V drift."""
|
||
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", f"{kind}:0",
|
||
"-show_entries", "stream=duration", "-of", "default=nk=1:nw=1", path],
|
||
capture_output=True, text=True)
|
||
try:
|
||
return float(r.stdout.strip())
|
||
except ValueError:
|
||
return 0.0
|
||
|
||
|
||
ZMAX, ZPAN = 1.15, 1.18 # ken-burns zoom ceiling; constant zoom that gives pans room to travel
|
||
|
||
# Every clip and every assembly stage MUST agree on this. xfade does not resample: it reinterprets the
|
||
# second input's frames at the first input's rate, so a 30fps input joined onto a 25fps one plays 1.2x
|
||
# too slow with the audio untouched -- the video ends minutes long and the narration runs ahead of the
|
||
# picture. That is exactly what a 25fps clip pipeline plus a `fps=30` concat branch produced.
|
||
FPS = 25
|
||
|
||
|
||
def _motion(camera: dict, frames: int) -> str:
|
||
"""#8 content-aware motion: map the vision `camera` block to a zoompan z/x/y expression.
|
||
vocab (spec-v3 direction schema): static|hold, zoom_in|zoom_out, pan_left/right/up/down,
|
||
dolly_to_subject (uses camera.to=[x,y] normalized), orbit, shake. default = gentle zoom_in
|
||
(the old ken-burns look) so panels without direction are unchanged.
|
||
x/y reference `zoom` for the live crop-window size; z is driven linearly by `on` (frame index)
|
||
over T frames so the move completes across the whole clip regardless of length."""
|
||
T = max(1, frames - 1)
|
||
eff = (camera or {}).get("effect", "zoom_in")
|
||
xc, yc = "iw/2-(iw/zoom/2)", "ih/2-(ih/zoom/2)" # centered crop
|
||
mx, my = "(iw-iw/zoom)", "(ih-ih/zoom)" # pan travel margin
|
||
if eff in ("static", "hold"):
|
||
z, x, y = "1.0", xc, yc
|
||
elif eff == "zoom_out":
|
||
z, x, y = f"{ZMAX}-{ZMAX-1:.3f}*on/{T}", xc, yc
|
||
elif eff == "pan_left":
|
||
z, x, y = f"{ZPAN}", f"{mx}*(1-on/{T})", yc
|
||
elif eff == "pan_right":
|
||
z, x, y = f"{ZPAN}", f"{mx}*on/{T}", yc
|
||
elif eff == "pan_up":
|
||
z, x, y = f"{ZPAN}", xc, f"{my}*(1-on/{T})"
|
||
elif eff == "pan_down":
|
||
z, x, y = f"{ZPAN}", xc, f"{my}*on/{T}"
|
||
elif eff == "dolly_to_subject":
|
||
to = (camera or {}).get("to") or [0.5, 0.35]
|
||
tx, ty = min(max(float(to[0]), 0.0), 1.0), min(max(float(to[1]), 0.0), 1.0)
|
||
z = f"1+{ZMAX-1:.3f}*on/{T}"
|
||
x = f"(iw/2+({tx}*iw-iw/2)*on/{T})-(iw/zoom/2)" # crop center eases toward subject
|
||
y = f"(ih/2+({ty}*ih-ih/2)*on/{T})-(ih/zoom/2)"
|
||
elif eff == "shake":
|
||
z, x, y = "1.06", f"{xc}+8*sin(on*1.5)", f"{yc}+8*cos(on*1.3)"
|
||
elif eff == "orbit":
|
||
z = "1.12"
|
||
x, y = f"{xc}+(iw*0.03)*sin(6.283*on/{T})", f"{yc}+(ih*0.03)*cos(6.283*on/{T})"
|
||
else: # zoom_in (default ken burns)
|
||
z, x, y = f"1+{ZMAX-1:.3f}*on/{T}", xc, yc
|
||
return f"zoompan=z='{z}':x='{x}':y='{y}':d={frames}:s={W}x{H}:fps={FPS}"
|
||
|
||
|
||
def scene_cmd(img: str, audio: str, ass: str, out: str, dur: float, camera: dict = None,
|
||
pad: float = PAD_S) -> list:
|
||
"""ffmpeg: still panel over a blurred fill of itself + content-aware motion, burned subs, 9:16.
|
||
#5 blurred bg replaces black bars: one copy scaled to COVER + blurred, the fitted panel on top.
|
||
#13 pad seconds of trailing silence (last frame held) give the panel a beat before the next."""
|
||
frames = max(1, int((dur + pad) * FPS)) # hold the last frame through the pad
|
||
# overlay's W/H/w/h are ffmpeg's main/overlay dims -- kept literal (no f-string braces).
|
||
fc = (
|
||
f"[0:v]split=2[bg][fg];"
|
||
f"[bg]scale={W}:{H}:force_original_aspect_ratio=increase,crop={W}:{H},boxblur=20:2[bgb];"
|
||
f"[fg]scale={W}:{H}:force_original_aspect_ratio=decrease[fgs];"
|
||
f"[bgb][fgs]overlay=(W-w)/2:(H-h)/2,"
|
||
f"{_motion(camera, frames)},"
|
||
f"ass={ass}[v];"
|
||
f"[1:a]apad=pad_dur={pad:.3f}[a]" # trailing silence to match the held frames
|
||
)
|
||
return ["ffmpeg", "-y", "-loop", "1", "-i", img, "-i", audio,
|
||
"-filter_complex", fc, "-map", "[v]", "-map", "[a]",
|
||
"-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", "-b:a", "192k", "-shortest", out]
|
||
|
||
|
||
class SceneInput(BaseModel):
|
||
panel_uri: str
|
||
layer_uris: list = []
|
||
audio_uri: str = ""
|
||
narration_text: str = ""
|
||
panel_id: str = ""
|
||
scene_timing: dict = {}
|
||
camera: dict = {} # #8 direction block from vision: {"effect": "...", "to": [x,y]}
|
||
manga_id: str = ""
|
||
chapter_id: str = ""
|
||
|
||
|
||
@app.post("/render/scene")
|
||
async def render_scene(data: SceneInput):
|
||
tag = uuid.uuid4().hex[:8]
|
||
img = transport.get(data.panel_uri, f"{SHM}/rnd_{tag}.png")
|
||
audio = transport.get(data.audio_uri, f"{SHM}/rnd_{tag}.wav")
|
||
# #1: real duration = narration length; scene_timing is empty in practice, 4.0 last-resort.
|
||
dur = (_audio_dur(audio)
|
||
or float(data.scene_timing.get("end", 0)) - float(data.scene_timing.get("start", 0))
|
||
or 4.0)
|
||
ass = f"{SHM}/rnd_{tag}.ass"
|
||
_ass(data.narration_text, dur, ass)
|
||
out = f"{SHM}/rnd_{tag}.mp4"
|
||
subprocess.run(scene_cmd(img, audio, ass, out, dur, data.camera), check=True, capture_output=True)
|
||
manga_id, chapter_id = _mc_from_uri(data.panel_uri)
|
||
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
for p in (img, audio, ass, out):
|
||
os.remove(p)
|
||
return {"clip_uri": uri, "duration": dur}
|
||
|
||
|
||
# #6 multi-panel composite: a group of small panels shares ONE shot (vertical stack). each panel
|
||
# keeps its own narration + audio (script/tts stay per-panel); they play in sequence while a moving
|
||
# highlight marks the active panel -- that's the "swap the front panel while audio plays" beat.
|
||
def _stack_still_cmd(images: list, out: str, rowh: int) -> list:
|
||
"""compose N panels fitted into equal vertical rows on one W×H still (black gutters)."""
|
||
n = len(images)
|
||
cmd = ["ffmpeg", "-y"]
|
||
for im in images:
|
||
cmd += ["-i", im]
|
||
parts = [f"[{i}:v]scale={W}:{rowh}:force_original_aspect_ratio=decrease,"
|
||
f"pad={W}:{rowh}:(ow-iw)/2:(oh-ih)/2:color=black[p{i}]" for i in range(n)]
|
||
stacked = "".join(f"[p{i}]" for i in range(n))
|
||
fc = ";".join(parts) + f";{stacked}vstack=inputs={n},pad={W}:{H}:0:0:color=black[v]"
|
||
return cmd + ["-filter_complex", fc, "-map", "[v]", "-frames:v", "1", out]
|
||
|
||
|
||
# D#6 revisited: camera-traversal composite. instead of vstacking cropped panels (black gutters,
|
||
# "wrong look"), pan across the REAL page — hold on each member's panel, quick-ease to the next,
|
||
# partial neighbours stay visible. groups are always single-page (grouping.py), so one page image.
|
||
TRAVERSE_EASE = 0.5 # seconds of quick slide between held panels
|
||
WIN_MARGIN = 1.6 # crop window height = tallest member bbox * this (room for neighbour reveal)
|
||
|
||
|
||
def _framed_window(pw: int, ph: int, bboxes: list) -> tuple:
|
||
"""one constant 9:16 crop window (px) sized to hold the tallest member with margin, aspect
|
||
preserved, clamped to the page. constant size => only x,y animate (crop supports per-frame x/y,
|
||
not per-frame w/h)."""
|
||
ar = W / H
|
||
ch = max((b[3] for b in bboxes), default=ph) * WIN_MARGIN
|
||
cw = ch * ar
|
||
if cw > pw:
|
||
cw, ch = pw, pw / ar
|
||
if ch > ph:
|
||
ch, cw = ph, ph * ar
|
||
return int(cw), int(ch)
|
||
|
||
|
||
def _win_tl(bbox: list, cw: int, ch: int, pw: int, ph: int) -> tuple:
|
||
"""top-left of the window centered on a bbox, clamped so it stays inside the page."""
|
||
cx, cy = bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2
|
||
return (min(max(cx - cw / 2, 0), pw - cw), min(max(cy - ch / 2, 0), ph - ch))
|
||
|
||
|
||
def _pan_expr(vals: list, segs: list, ease: float) -> str:
|
||
"""piecewise ffmpeg expr in t: hold vals[i] through beat i, linear-ease to vals[i+1] over the
|
||
beat's last `ease`s; final beat just holds. segs=[(start,end)] per beat."""
|
||
n = len(vals)
|
||
expr = f"{vals[-1]:.1f}"
|
||
for i in range(n - 2, -1, -1):
|
||
s, e = segs[i]
|
||
he = e - ease
|
||
a, b = vals[i], vals[i + 1]
|
||
beat = f"if(lt(t,{he:.3f}),{a:.1f},({a:.1f}+({b - a:.1f})*(t-{he:.3f})/{ease:.3f}))"
|
||
expr = f"if(lt(t,{e:.3f}),{beat},{expr})"
|
||
return expr
|
||
|
||
|
||
def traverse_cmd(page: str, audios: list, ass: str, segs: list, bboxes: list,
|
||
pw: int, ph: int, out: str, dur: float) -> list:
|
||
"""pan a constant 9:16 window across the page: hold on each member, quick-ease to the next."""
|
||
ease = min(TRAVERSE_EASE, min((e - s for s, e in segs), default=1.0) / 2)
|
||
cw, ch = _framed_window(pw, ph, bboxes)
|
||
tls = [_win_tl(b, cw, ch, pw, ph) for b in bboxes]
|
||
xexpr = _pan_expr([t[0] for t in tls], segs, ease)
|
||
yexpr = _pan_expr([t[1] for t in tls], segs, ease)
|
||
n = len(audios)
|
||
aconcat = "".join(f"[{i + 1}:a]" for i in range(n)) + f"concat=n={n}:v=0:a=1[a]"
|
||
fc = (f"[0:v]crop={cw}:{ch}:x='{xexpr}':y='{yexpr}',scale={W}:{H},setsar=1,"
|
||
f"ass={ass}[v];{aconcat}")
|
||
cmd = ["ffmpeg", "-y", "-loop", "1", "-i", page]
|
||
for a in audios:
|
||
cmd += ["-i", a]
|
||
return cmd + ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{dur:.3f}",
|
||
"-c:v", "libx264", "-tune", "stillimage", "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", "-b:a", "192k", out]
|
||
|
||
|
||
def composite_cmd(still: str, audios: list, ass: str, segs: list, rowh: int, out: str, dur: float) -> list:
|
||
"""loop the stacked still for the whole group, concat each panel's audio in order, burn the
|
||
per-segment subtitles, and outline the active row during its narration. segs=[(start,end)]."""
|
||
n = len(audios)
|
||
hl = "".join(
|
||
f"drawbox=x=0:y={i*rowh}:w={W}:h={rowh}:color=yellow@0.85:t=6:enable='between(t,{s:.2f},{e:.2f})',"
|
||
for i, (s, e) in enumerate(segs))
|
||
cmd = ["ffmpeg", "-y", "-loop", "1", "-i", still]
|
||
for a in audios:
|
||
cmd += ["-i", a]
|
||
aconcat = "".join(f"[{i+1}:a]" for i in range(n)) + f"concat=n={n}:v=0:a=1[a]"
|
||
fc = f"[0:v]{hl}ass={ass}[v];{aconcat}"
|
||
return cmd + ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{dur:.3f}",
|
||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||
|
||
|
||
class CompositeInput(BaseModel):
|
||
# ordered members: [{panel_uri, audio_uri, narration_text, bbox?, page_uri?}]. bbox=[x,y,w,h] on
|
||
# the source page + page_uri enable the camera-traversal look; absent -> vstack fallback.
|
||
panels: list = []
|
||
panel_id: str = "" # leader panel id -> clip key (assemble picks up one clip per group)
|
||
|
||
|
||
def _img_size(path: str) -> tuple:
|
||
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries",
|
||
"stream=width,height", "-of", "csv=p=0:s=x", path], capture_output=True, text=True)
|
||
w, h = r.stdout.strip().split("x")
|
||
return int(w), int(h)
|
||
|
||
|
||
@app.post("/render/composite")
|
||
async def render_composite(data: CompositeInput):
|
||
if len(data.panels) < 2:
|
||
raise HTTPException(400, "composite needs >=2 panels")
|
||
tag = uuid.uuid4().hex[:8]
|
||
auds = [transport.get(p["audio_uri"], f"{SHM}/cmp_{tag}_{i}.wav") for i, p in enumerate(data.panels)]
|
||
durs = [_audio_dur(a) or 3.0 for a in auds]
|
||
segs, t = [], 0.0
|
||
for d in durs:
|
||
segs.append((t, t + d)); t += d
|
||
n = len(data.panels)
|
||
ass, out = f"{SHM}/cmp_{tag}.ass", f"{SHM}/cmp_{tag}.mp4"
|
||
_ass_multi([(data.panels[i].get("narration_text", ""), segs[i][0], segs[i][1]) for i in range(n)], ass)
|
||
|
||
# camera-traversal path: needs a shared page + a bbox per member (grouping keeps groups single-page)
|
||
bboxes = [p.get("bbox") for p in data.panels]
|
||
page_uri = data.panels[0].get("page_uri")
|
||
if page_uri and all(b and len(b) == 4 for b in bboxes):
|
||
page = transport.get(page_uri, f"{SHM}/cmp_{tag}_pg.png")
|
||
pw, ph = _img_size(page)
|
||
subprocess.run(traverse_cmd(page, auds, ass, segs, bboxes, pw, ph, out, t),
|
||
check=True, capture_output=True)
|
||
os.remove(page)
|
||
else: # fallback: legacy vstack (black gutters) when geometry is unavailable
|
||
imgs = [transport.get(p["panel_uri"], f"{SHM}/cmp_{tag}_{i}.png") for i, p in enumerate(data.panels)]
|
||
rowh = H // n
|
||
still = f"{SHM}/cmp_{tag}_s.png"
|
||
subprocess.run(_stack_still_cmd(imgs, still, rowh), check=True, capture_output=True)
|
||
subprocess.run(composite_cmd(still, auds, ass, segs, rowh, out, t), check=True, capture_output=True)
|
||
for p in imgs + [still]:
|
||
os.remove(p)
|
||
manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"])
|
||
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
for p in auds + [ass, out]:
|
||
os.remove(p)
|
||
return {"clip_uri": uri, "duration": t}
|
||
|
||
|
||
# director groups: render a beat's member panels as sequential FULL-FRAME shots (each its own
|
||
# scene_cmd clip: blurred bg + ken-burns + its narration), then xfade-chain them into one clip with
|
||
# the per-member transition. replaces the vstack/traverse composite look with real transitioned crops.
|
||
class GroupInput(BaseModel):
|
||
# ordered members: [{panel_uri, audio_uri, narration_text, camera, transition}]
|
||
panels: list = []
|
||
panel_id: str = "" # leader id -> clip key
|
||
|
||
|
||
@app.post("/render/group")
|
||
async def render_group(data: GroupInput):
|
||
if len(data.panels) < 2:
|
||
raise HTTPException(400, "group needs >=2 panels")
|
||
tag = uuid.uuid4().hex[:8]
|
||
clips, durs, cleanup = [], [], []
|
||
for i, p in enumerate(data.panels):
|
||
img = transport.get(p["panel_uri"], f"{SHM}/grp_{tag}_{i}.png")
|
||
aud = transport.get(p["audio_uri"], f"{SHM}/grp_{tag}_{i}.wav")
|
||
dur = _audio_dur(aud) or 3.0
|
||
ass = f"{SHM}/grp_{tag}_{i}.ass"
|
||
_ass(p.get("narration_text", ""), dur, ass)
|
||
out = f"{SHM}/grp_{tag}_{i}.mp4"
|
||
subprocess.run(scene_cmd(img, aud, ass, out, dur, p.get("camera") or {}),
|
||
check=True, capture_output=True)
|
||
clips.append(out)
|
||
durs.append(_audio_dur(out) or (dur + PAD_S)) # full clip length incl. trailing pad
|
||
cleanup += [img, aud, ass, out]
|
||
|
||
# transition INTO member i+1 = member i's transition (transition out of the leaving shot).
|
||
trans = [(p.get("transition") or "cut") for p in data.panels[:-1]]
|
||
fg, vmap, amap = _xfade_chain(durs, trans)
|
||
final = f"{SHM}/grp_{tag}.mp4"
|
||
cmd = ["ffmpeg", "-y"]
|
||
for c in clips:
|
||
cmd += ["-i", c]
|
||
cmd += ["-filter_complex", fg, "-map", vmap, "-map", amap,
|
||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", final]
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
|
||
manga_id, chapter_id = _mc_from_uri(data.panels[0]["panel_uri"])
|
||
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(final, uri)
|
||
total = _audio_dur(final) or sum(durs)
|
||
for f in cleanup + [final]:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
return {"clip_uri": uri, "duration": total}
|
||
|
||
|
||
# scene-level narration: ONE narration+audio spans a whole beat, shown over its member panels as a
|
||
# ken-burns montage. the beat's single subtitle holds across every image; images split the narration
|
||
# duration equally. (per-panel narration is gone at this granularity — the beat is the story unit.)
|
||
class BeatInput(BaseModel):
|
||
panel_uris: list = [] # ordered member images (>=1)
|
||
audio_uri: str = "" # the beat's single narration audio
|
||
narration_text: str = ""
|
||
cameras: list = [] # per-image camera block; short/empty -> default zoom_in
|
||
weights: list = [] # 185: per-image relative screen-time; short/empty/degenerate -> equal split
|
||
panel_id: str = "" # leader id -> clip key
|
||
|
||
|
||
BEAT_MIN_PANEL_S = 1.0 # 185: no member panel flashes by faster than this
|
||
|
||
|
||
def _beat_slices(D: float, n: int, weights: list = None) -> list:
|
||
"""185: split beat duration D across n member panels by content weight, not evenly.
|
||
Each panel gets >= BEAT_MIN_PANEL_S so a low-weight panel never flashes. Degenerate input
|
||
(no/short weights, non-positive sum, or D too small for the floors) -> deterministic equal split.
|
||
Pure + total: sum(result) == D always."""
|
||
if n <= 0:
|
||
return []
|
||
if not weights or len(weights) < n or D <= n * BEAT_MIN_PANEL_S:
|
||
return [D / n] * n
|
||
w = [max(0.0, float(x)) for x in weights[:n]]
|
||
s = sum(w)
|
||
if s <= 0:
|
||
return [D / n] * n
|
||
floor = n * BEAT_MIN_PANEL_S
|
||
free = D - floor # distribute only the time above the floors by weight
|
||
return [BEAT_MIN_PANEL_S + free * (wi / s) for wi in w]
|
||
|
||
|
||
def _split_cues(text: str) -> list:
|
||
"""187: break the beat's one flowing narration into sentence/phrase cues. Split on sentence-ending
|
||
punctuation (kept) and newlines so short exclamations like 'Hold up.' stay their own cue."""
|
||
text = (text or "").strip()
|
||
if not text:
|
||
return []
|
||
parts = re.split(r"(?<=[.!?…。！？])\s+|\n+", text)
|
||
return [p.strip() for p in parts if p and p.strip()]
|
||
|
||
|
||
def cue_plan(text: str, D: float, slices: list) -> list:
|
||
"""187: a timed subtitle plan for one beat. Each cue = one sentence, timed along the SAME [0,D]
|
||
timeline as the per-panel slices, so a cue never appears before the panel on screen when it starts
|
||
(no future-panel facts leak early). Cue durations are length-weighted with a readable floor (reusing
|
||
_beat_slices); each cue is mapped to the member panel whose window contains its start. Pure/testable.
|
||
Returns [{text, start, end, panel_index}] in order."""
|
||
cues = _split_cues(text)
|
||
if not cues or D <= 0:
|
||
return []
|
||
durs = _beat_slices(D, len(cues), [len(c) for c in cues]) # length-weighted, min-hold, equal fallback
|
||
bounds, t = [], 0.0
|
||
for s in slices: # panel on-screen windows
|
||
bounds.append((t, t + s)); t += s
|
||
events, t = [], 0.0
|
||
for c, d in zip(cues, durs):
|
||
start, end = t, min(t + d, D)
|
||
pi = next((k for k, (a, b) in enumerate(bounds) if a <= start < b), max(0, len(bounds) - 1))
|
||
# clamp start to the panel's reveal so the caption is never ahead of its image
|
||
events.append({"text": c, "start": max(start, bounds[pi][0]) if bounds else start,
|
||
"end": end, "panel_index": pi})
|
||
t = end
|
||
return events
|
||
|
||
|
||
def beat_cmd(imgs: list, audio: str, ass: str, out: str, D: float, cameras: list = None,
|
||
weights: list = None) -> list:
|
||
"""ffmpeg: N images shown as a ken-burns montage under ONE narration audio; the last image holds
|
||
through the trailing pad. each image = blurred-fill bg + fitted panel + its camera move; the single
|
||
burned subtitle (in `ass`) spans the whole beat. 185: per-image screen-time is content-weighted
|
||
(see _beat_slices), equal split when weights are absent. pure -> testable without S3."""
|
||
cameras = cameras or []
|
||
n, fps = len(imgs), FPS
|
||
slices = _beat_slices(D, n, weights) # 185: content-weighted, equal-split fallback
|
||
# one frame per image (no -loop): zoompan d=frames expands that single frame to exactly `frames`
|
||
# output frames = seg seconds. looping instead would feed many frames and zoompan multiplies each.
|
||
cmd = ["ffmpeg", "-y"]
|
||
for img in imgs:
|
||
cmd += ["-i", img]
|
||
cmd += ["-i", audio]
|
||
parts = []
|
||
for i in range(n):
|
||
cam = cameras[i] if i < len(cameras) else {}
|
||
seg = slices[i] + (PAD_S if i == n - 1 else 0.0)
|
||
frames = max(1, int(seg * fps))
|
||
parts.append(
|
||
f"[{i}:v]split=2[bg{i}][fg{i}];"
|
||
f"[bg{i}]scale={W}:{H}:force_original_aspect_ratio=increase,crop={W}:{H},boxblur=20:2[bgb{i}];"
|
||
f"[fg{i}]scale={W}:{H}:force_original_aspect_ratio=decrease[fgs{i}];"
|
||
f"[bgb{i}][fgs{i}]overlay=(W-w)/2:(H-h)/2,{_motion(cam, frames)}[v{i}]"
|
||
)
|
||
concat_in = "".join(f"[v{i}]" for i in range(n))
|
||
fc = (";".join(parts) + f";{concat_in}concat=n={n}:v=1:a=0[vc];"
|
||
f"[vc]ass={ass}[v];[{n}:a]apad=pad_dur={PAD_S:.3f}[a]")
|
||
cmd += ["-filter_complex", fc, "-map", "[v]", "-map", "[a]",
|
||
"-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||
return cmd
|
||
|
||
|
||
@app.post("/render/beat")
|
||
async def render_beat(data: BeatInput):
|
||
if not data.panel_uris:
|
||
raise HTTPException(400, "beat needs >=1 panel")
|
||
tag = uuid.uuid4().hex[:8]
|
||
audio = transport.get(data.audio_uri, f"{SHM}/bt_{tag}.wav")
|
||
D = _audio_dur(audio) or 4.0
|
||
imgs = [transport.get(u, f"{SHM}/bt_{tag}_{i}.png") for i, u in enumerate(data.panel_uris)]
|
||
ass = f"{SHM}/bt_{tag}.ass"
|
||
# 187: timed sentence cues instead of one beat-long paragraph; same slice timeline as the montage,
|
||
# so each line surfaces with its panel. Empty/unsplittable text -> single caption (old behaviour).
|
||
slices = _beat_slices(D, len(imgs), data.weights)
|
||
cues = cue_plan(data.narration_text, D, slices)
|
||
cams = data.cameras or []
|
||
if cues:
|
||
cues[-1]["end"] = D + PAD_S # hold the closing line through the trailing pad
|
||
# 188: each cue dodges its panel's subject (top vs bottom) via the director's focus point.
|
||
_ass_multi([(c["text"], c["start"], c["end"],
|
||
_sub_align(cams[c["panel_index"]] if c["panel_index"] < len(cams) else {}))
|
||
for c in cues], ass)
|
||
else:
|
||
_ass_multi([(data.narration_text, 0.0, D + PAD_S)], ass)
|
||
out = f"{SHM}/bt_{tag}.mp4"
|
||
subprocess.run(beat_cmd(imgs, audio, ass, out, D, data.cameras, data.weights),
|
||
check=True, capture_output=True)
|
||
|
||
manga_id, chapter_id = _mc_from_uri(data.panel_uris[0])
|
||
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
total = _audio_dur(out) or (D + PAD_S)
|
||
for f in imgs + [audio, ass, out]:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
# 187: cue plan (text/timing/panel_index per line) returned so the orchestrator can persist it for
|
||
# the review UI to edit mapping+timing, and so #183's collage can key reveals off the same cues.
|
||
return {"clip_uri": uri, "duration": total, "cues": cues}
|
||
|
||
|
||
# --- 183: animated manga collage ------------------------------------------------------------------
|
||
# A beat's member panels laid out as a motion-comic collage instead of a full-frame ken-burns montage:
|
||
# a blurred plate of the dominant panel fills the frame, sharp aspect-fit panels rest in a planned
|
||
# template (collage.plan_layout), and non-dominant panels slide into place over a brief entrance while
|
||
# the dominant one resolves by scale. Holds stay crisp. Behind the pipeline's COLLAGE flag.
|
||
# ponytail ceilings (visual polish, QA-tuned against the reference video, no still to check here):
|
||
# - transition-only directional/radial MOTION BLUR is not applied (clean slide/scale entrance); the
|
||
# beat-to-beat "streak" transition still rides the assemble-stage xfade. Add tblend accumulation
|
||
# when a fixture shows the clean slide reads too flat.
|
||
# - per-panel slow "drift" during the hold is omitted (static hold); add a gentle zoompan when needed.
|
||
class CollageInput(BaseModel):
|
||
panel_uris: list = [] # ordered member images (reading order), 1..4
|
||
audio_uri: str = ""
|
||
narration_text: str = ""
|
||
weights: list = [] # 185 timing hints (also picks the dominant panel = argmax)
|
||
cameras: list = [] # for 188 subtitle subject-dodge
|
||
rtl: bool = True
|
||
active: int = -1 # dominant panel index; <0 -> argmax(weights) or 0
|
||
panel_id: str = ""
|
||
|
||
|
||
def collage_cmd(imgs, plate_i, rects, entrances, z_order, audio, ass, out, D, trans):
|
||
"""ffmpeg graph: blurred plate of imgs[plate_i] + each sharp panel scaled to its resting rect with a
|
||
restrained drop shadow, composited back-to-front (z_order), non-dominant panels sliding in from their
|
||
entrance offset over `trans` seconds. One narration audio; burned cues in `ass`. Pure -> testable."""
|
||
n, T = len(imgs), D + PAD_S
|
||
cmd = ["ffmpeg", "-y"]
|
||
for img in imgs:
|
||
cmd += ["-loop", "1", "-i", img] # still held for the whole clip; -t caps output
|
||
cmd += ["-i", audio]
|
||
parts = [f"[{plate_i}:v]scale={W}:{H}:force_original_aspect_ratio=increase,"
|
||
f"crop={W}:{H},boxblur=24:2,setsar=1[bg]"]
|
||
base = "bg"
|
||
for k, i in enumerate(z_order): # shadows first, at rest positions (static)
|
||
x, y, w, h = (int(round(v)) for v in rects[i])
|
||
parts.append(f"[{base}]drawbox=x={x + 7}:y={y + 7}:w={w}:h={h}:color=black@0.35:t=fill[sh{k}]")
|
||
base = f"sh{k}"
|
||
for i in range(n):
|
||
w, h = int(round(rects[i][2])), int(round(rects[i][3]))
|
||
parts.append(f"[{i}:v]scale={w}:{h},setsar=1[p{i}]")
|
||
cur = base
|
||
for k, i in enumerate(z_order):
|
||
x, y = int(round(rects[i][0])), int(round(rects[i][1]))
|
||
dx, dy = entrances[i]
|
||
# ease from (x+dx, y+dy) to (x, y) over `trans`s, then hold. commas safe inside the '...' quotes.
|
||
ease = f"max(0,1-t/{trans:.3f})"
|
||
parts.append(f"[{cur}][p{i}]overlay=x='{x}+({dx})*{ease}':y='{y}+({dy})*{ease}'[o{k}]")
|
||
cur = f"o{k}"
|
||
fc = ";".join(parts) + f";[{cur}]ass={ass}[v];[{n}:a]apad=pad_dur={PAD_S:.3f}[a]"
|
||
cmd += ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{T:.3f}",
|
||
"-r", "30", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||
return cmd
|
||
|
||
|
||
@app.post("/render/collage")
|
||
async def render_collage(data: CollageInput):
|
||
if not data.panel_uris:
|
||
raise HTTPException(400, "collage needs >=1 panel")
|
||
uris = data.panel_uris[:4] # planner templates cap at 4 readable panels
|
||
n = len(uris)
|
||
tag = uuid.uuid4().hex[:8]
|
||
audio = transport.get(data.audio_uri, f"{SHM}/cl_{tag}.wav")
|
||
D = _audio_dur(audio) or 4.0
|
||
imgs = [transport.get(u, f"{SHM}/cl_{tag}_{i}.png") for i, u in enumerate(uris)]
|
||
aspects = []
|
||
for p in imgs:
|
||
w, h = _img_size(p)
|
||
aspects.append(w / h if h else 1.0)
|
||
active = data.active if 0 <= data.active < n else (
|
||
max(range(n), key=lambda i: data.weights[i]) if len(data.weights) >= n else 0)
|
||
lay = plan_layout(aspects, data.rtl, active, (W, H))
|
||
|
||
# 185/187: content-weighted slices + timed cues, same as the montage path.
|
||
slices = _beat_slices(D, n, data.weights)
|
||
cues = cue_plan(data.narration_text, D, slices)
|
||
ass = f"{SHM}/cl_{tag}.ass"
|
||
if cues:
|
||
cues[-1]["end"] = D + PAD_S
|
||
cams = data.cameras or []
|
||
_ass_multi([(c["text"], c["start"], c["end"],
|
||
_sub_align(cams[c["panel_index"]] if c["panel_index"] < len(cams) else {}))
|
||
for c in cues], ass)
|
||
else:
|
||
_ass_multi([(data.narration_text, 0.0, D + PAD_S)], ass)
|
||
|
||
out = f"{SHM}/cl_{tag}.mp4"
|
||
subprocess.run(collage_cmd(imgs, active, lay["rects"], lay["entrances"], lay["z_order"],
|
||
audio, ass, out, D, lay["transition_s"]), check=True, capture_output=True)
|
||
manga_id, chapter_id = _mc_from_uri(uris[0])
|
||
uri = f"s3://video/{manga_id}/{chapter_id}/clips/{data.panel_id or 'p'}.mp4"
|
||
transport.put(out, uri)
|
||
total = _audio_dur(out) or (D + PAD_S)
|
||
for f in imgs + [audio, ass, out]:
|
||
if os.path.exists(f):
|
||
os.remove(f)
|
||
return {"clip_uri": uri, "duration": total, "cues": cues, "template": lay["template"]}
|
||
|
||
|
||
# #6 non-generic transitions: map the direction schema's transition.type -> (xfade name, seconds).
|
||
# "cut" stays a hard cut via the concat fast-path; the rest re-encode through an xfade chain.
|
||
XFADE = {
|
||
"cut": ("fade", 0.0),
|
||
"crossfade": ("fade", 0.5),
|
||
"dissolve": ("dissolve", 0.5),
|
||
"fade_black": ("fadeblack", 0.6),
|
||
"fade_white": ("fadewhite", 0.6),
|
||
"wipe_left": ("wipeleft", 0.4),
|
||
"wipe_right": ("wiperight", 0.4),
|
||
"push": ("slideleft", 0.4),
|
||
}
|
||
|
||
# Chapter assembly used to open every clip in one ffmpeg process. A long chapter therefore created
|
||
# one enormous xfade graph: N decoders + N-1 full-frame filter stages, enough to exhaust RAM/VRAM and
|
||
# get ffmpeg SIGKILLed by the OOM killer. Assemble a bounded tree instead. Eight inputs keeps enough
|
||
# work in each encode to be efficient without letting decoder/filter threads grow with chapter size.
|
||
ASSEMBLE_BATCH = max(2, int(os.environ.get("ASSEMBLE_BATCH", "8")))
|
||
FFMPEG_THREADS = max(1, int(os.environ.get("FFMPEG_THREADS", "2")))
|
||
|
||
|
||
def _xfade_chain(durs: list, trans: list):
|
||
"""build a filter_complex that xfades N clips with per-boundary transitions, keeping audio in
|
||
sync via matching acrossfade. trans[i] is the transition OUT of clip i (boundary i->i+1).
|
||
returns (filtergraph, video_label, audio_label). offsets accumulate as clips overlap."""
|
||
# A clip whose duration probed as 0/unreadable must not poison the chain: with dur=0 the offset
|
||
# accumulator would run BACKWARDS (cum += dur - td), swallowing every later clip into a frozen
|
||
# overlap near the middle. Floor to a small positive length so the timeline stays monotonic.
|
||
durs = [d if (d and d > 0.1) else 0.1 for d in durs]
|
||
# Normalize every input to FPS/SAR before it reaches xfade, exactly as the concat branch does. Both
|
||
# branches feed the same tree, so an un-normalized xfade input is what stretched the chapter 1.2x.
|
||
parts = [f"[{i}:v]setsar=1,fps={FPS}[n{i}]" for i in range(len(durs))]
|
||
vlast, alast, cum = "[n0]", "[0:a]", durs[0]
|
||
for i in range(1, len(durs)):
|
||
name, td = XFADE.get(trans[i - 1] if i - 1 < len(trans) else "cut", XFADE["cut"])
|
||
td = max(0.05, min(td, durs[i - 1] - 0.05, durs[i] - 0.05)) # overlap fits in both clips
|
||
off = max(cum - td, 0)
|
||
parts.append(f"{vlast}[n{i}]xfade=transition={name}:duration={td:.3f}:offset={off:.3f}[v{i}]")
|
||
parts.append(f"{alast}[{i}:a]acrossfade=d={td:.3f}[a{i}]")
|
||
vlast, alast, cum = f"[v{i}]", f"[a{i}]", cum + durs[i] - td
|
||
return ";".join(parts), vlast, alast
|
||
|
||
|
||
def _assemble_once(inputs: list[str], trans: list[str], out: str):
|
||
"""Assemble one bounded batch. `trans[i]` is the transition out of inputs[i]."""
|
||
fancy = len(inputs) >= 2 and any(t not in ("", "cut") for t in trans[:len(inputs) - 1])
|
||
if fancy:
|
||
durs = [_audio_dur(p) for p in inputs]
|
||
fg, vmap, amap = _xfade_chain(durs, trans)
|
||
cmd = ["ffmpeg", "-y", "-filter_complex_threads", str(FFMPEG_THREADS)]
|
||
# Input-side -threads limits each decoder; otherwise ffmpeg may create a decoder thread pool
|
||
# for every input in the batch in addition to the filter and libx264 pools.
|
||
for p in inputs:
|
||
cmd += ["-threads", "1", "-i", p]
|
||
cmd += ["-filter_complex", fg, "-map", vmap, "-map", amap,
|
||
"-c:v", "libx264", "-threads", str(FFMPEG_THREADS), "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", "-b:a", "192k", out]
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
return
|
||
|
||
# A cut-only batch is concatenated with the concat FILTER, not -c copy or the concat demuxer.
|
||
# The tree feeds xfade intermediates (irregular video timestamps) back in as inputs; the concat
|
||
# demuxer + -vsync cfr DROPS video frames to force CFR while audio survives -> the video ends up
|
||
# minutes short and freezes on a frame with narration playing on (the "one image, narration behind
|
||
# it" bug). The concat filter decodes and re-times every segment, so no frames are dropped. It needs
|
||
# N decoders, but the tree already bounds a batch to ASSEMBLE_BATCH inputs, so memory stays capped.
|
||
n = len(inputs)
|
||
cmd = ["ffmpeg", "-y"]
|
||
for p in inputs:
|
||
cmd += ["-i", p]
|
||
pre = "".join(f"[{i}:v]setsar=1,fps={FPS}[v{i}];" for i in range(n))
|
||
fg = pre + "".join(f"[v{i}][{i}:a]" for i in range(n)) + f"concat=n={n}:v=1:a=1[v][a]"
|
||
cmd += ["-filter_complex", fg, "-map", "[v]", "-map", "[a]",
|
||
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p",
|
||
"-threads", str(FFMPEG_THREADS), "-c:a", "aac", "-b:a", "192k", out]
|
||
subprocess.run(cmd, check=True, capture_output=True)
|
||
|
||
|
||
def _assemble_batched(inputs: list[str], transitions: list[str], out: str, tag: str,
|
||
cleanup: list[str]):
|
||
"""Bounded tree assembly preserving every original transition.
|
||
|
||
Each intermediate carries the transition OUT of its final source clip. That becomes the boundary
|
||
transition between intermediates in the next round, so batching does not change direction. A
|
||
chapter of 100 clips with batch=8 uses at most eight simultaneous inputs instead of 100.
|
||
"""
|
||
items = [{"path": p,
|
||
"out_transition": transitions[i] if i < len(transitions) else "cut",
|
||
"temporary": False}
|
||
for i, p in enumerate(inputs)]
|
||
round_no = 0
|
||
while len(items) > 1:
|
||
final_round = len(items) <= ASSEMBLE_BATCH
|
||
next_items = []
|
||
for start in range(0, len(items), ASSEMBLE_BATCH):
|
||
group = items[start:start + ASSEMBLE_BATCH]
|
||
if len(group) == 1 and not final_round:
|
||
next_items.append(group[0])
|
||
continue
|
||
target = out if final_round else f"{SHM}/asm_{tag}_r{round_no}_{start // ASSEMBLE_BATCH}.mp4"
|
||
_assemble_once([x["path"] for x in group], [x["out_transition"] for x in group], target)
|
||
if not final_round:
|
||
cleanup.append(target)
|
||
next_items.append({"path": target, "out_transition": group[-1]["out_transition"],
|
||
"temporary": not final_round})
|
||
# Intermediates from the previous round are no longer needed. Keep cleanup idempotent: remove
|
||
# them here for low /dev/shm usage and from the cleanup list so endpoint cleanup won't retry.
|
||
for item in items:
|
||
p = item["path"]
|
||
if item["temporary"] and os.path.exists(p):
|
||
os.remove(p)
|
||
if p in cleanup:
|
||
cleanup.remove(p)
|
||
items = next_items
|
||
round_no += 1
|
||
|
||
|
||
class AssembleInput(BaseModel):
|
||
clip_uris: list
|
||
transitions: list = [] # #6 per-clip transition-out type; empty/all-"cut" -> fast concat
|
||
chapter_id: str = ""
|
||
manga_id: str = ""
|
||
|
||
|
||
def _music_filter(gain: str) -> str:
|
||
"""#13 loop a music bed under the narration, ducked by sidechain compression keyed on the
|
||
narration itself, then mix. duration=first ends the mix with the video's audio."""
|
||
return (
|
||
f"[1:a]volume={gain}[bed];"
|
||
"[0:a]asplit=2[nar][key];"
|
||
"[bed][key]sidechaincompress=threshold=0.03:ratio=8:attack=20:release=400[duck];"
|
||
"[nar][duck]amix=inputs=2:duration=first:dropout_transition=0[a]"
|
||
)
|
||
|
||
|
||
def _add_music_bed(video: str, tag: str, cleanup: list) -> str:
|
||
"""mix MUSIC_BED under the assembled chapter. no-op (returns input) when unset or on failure."""
|
||
if not MUSIC_BED:
|
||
return video
|
||
music = MUSIC_BED
|
||
if MUSIC_BED.startswith("s3://"):
|
||
music = transport.get(MUSIC_BED, f"{SHM}/bed_{tag}{os.path.splitext(MUSIC_BED)[1] or '.mp3'}")
|
||
cleanup.append(music)
|
||
mixed = f"{SHM}/chapter_{tag}_mus.mp4"
|
||
try:
|
||
subprocess.run(
|
||
["ffmpeg", "-y", "-i", video, "-stream_loop", "-1", "-i", music,
|
||
"-filter_complex", _music_filter(MUSIC_GAIN), "-map", "0:v", "-map", "[a]",
|
||
"-c:v", "copy", "-c:a", "aac", "-b:a", "192k", "-shortest", mixed],
|
||
check=True, capture_output=True)
|
||
except (subprocess.CalledProcessError, FileNotFoundError) as e:
|
||
print(f"[render] music bed skipped: {e}", flush=True)
|
||
if os.path.exists(mixed):
|
||
os.remove(mixed)
|
||
return video
|
||
cleanup.append(mixed)
|
||
return mixed
|
||
|
||
|
||
@app.post("/render/assemble")
|
||
async def assemble(data: AssembleInput):
|
||
if not data.clip_uris:
|
||
raise HTTPException(400, "no clips to assemble")
|
||
tag = uuid.uuid4().hex[:8]
|
||
with ThreadPoolExecutor(max_workers=8) as pool:
|
||
locals_ = list(pool.map(
|
||
lambda iu: transport.get(iu[1], f"{SHM}/asm_{tag}_{iu[0]}.mp4"),
|
||
enumerate(data.clip_uris),
|
||
))
|
||
out = f"{SHM}/chapter_{tag}.mp4"
|
||
cleanup = list(locals_) + [out]
|
||
|
||
fancy = len(locals_) >= 2 and any(t not in ("", "cut") for t in data.transitions)
|
||
if fancy:
|
||
_assemble_batched(locals_, data.transitions, out, tag, cleanup)
|
||
else:
|
||
# all hard cuts: stream-copy concat (no re-encode) -- unchanged fast path.
|
||
listfile = f"{SHM}/asm_{tag}.txt"; cleanup.append(listfile)
|
||
with open(listfile, "w") as f:
|
||
f.write("".join(f"file '{p}'\n" for p in locals_))
|
||
subprocess.run(["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", listfile, "-c", "copy", out],
|
||
check=True, capture_output=True)
|
||
|
||
out = _add_music_bed(out, tag, cleanup)
|
||
|
||
manga_id, chapter_id = _mc_from_uri(data.clip_uris[0])
|
||
uri = f"s3://video/{manga_id}/{chapter_id}/chapter.mp4"
|
||
transport.put(out, uri)
|
||
for p in cleanup:
|
||
os.remove(p)
|
||
return {"video_uri": uri} # orchestrator save_video reads video_uri
|
||
|
||
|
||
@app.get("/health")
|
||
async def health():
|
||
return {"status": "ok"}
|
||
|
||
|
||
if __name__ == "__main__":
|
||
# self-check: ass file well-formed; ffmpeg produces a clip from a synthetic image + silence.
|
||
import shutil
|
||
_ass("Hello\nworld", 2.0, f"{SHM}/t.ass")
|
||
txt = open(f"{SHM}/t.ass").read()
|
||
assert "Hello world" in txt and "Dialogue:" in txt # short cue -> one line
|
||
|
||
# 188: subtitle redesign. wrap to <=2 lines, off-mode burns nothing, subject-dodge picks top/bottom.
|
||
assert _wrap2("one two three four five six seven eight", 12).count("\\N") == 1 # exactly two lines
|
||
long = _wrap2(" ".join(["word"] * 40), 12)
|
||
assert long.count("\\N") == 1 and long.endswith("…") # 3rd line truncated, never a paragraph
|
||
_ass_multi([("hidden", 0.0, 2.0)], f"{SHM}/off.ass", mode="off")
|
||
assert "Dialogue:" not in open(f"{SHM}/off.ass").read() # off -> nothing burned
|
||
_ass_multi([("boxed", 0.0, 2.0)], f"{SHM}/bx.ass", mode="boxed")
|
||
assert ",3," in open(f"{SHM}/bx.ass").read() # BorderStyle 3 = box
|
||
assert _sub_align({"to": [0.5, 0.8]}) == 8 and _sub_align({"to": [0.5, 0.3]}) == 2 # dodge low subject
|
||
_ass_multi([("top", 0.0, 2.0, 8)], f"{SHM}/an.ass")
|
||
assert "{\\an8}top" in open(f"{SHM}/an.ass").read() # per-cue alignment override
|
||
for f in (f"{SHM}/off.ass", f"{SHM}/bx.ass", f"{SHM}/an.ass"):
|
||
os.remove(f)
|
||
# #8 motion: each effect yields a distinct, well-formed zoompan expr; dolly aims at its target.
|
||
assert "z='1.0'" in _motion({"effect": "static"}, 50) # truly still
|
||
assert "on/49" in _motion({"effect": "pan_left"}, 50) # travels over the clip
|
||
assert "0.8*iw" in _motion({"effect": "dolly_to_subject", "to": [0.8, 0.2]}, 50)
|
||
assert _motion({}, 50) == _motion({"effect": "zoom_in"}, 50) # default == ken burns
|
||
|
||
# 185: content-weighted beat slices. always sum to D; floor honored; degenerate -> equal split.
|
||
eq = _beat_slices(12.0, 3)
|
||
assert eq == [4.0, 4.0, 4.0], eq # no weights -> equal
|
||
w = _beat_slices(12.0, 3, [3, 1, 1]) # heavier panel gets more time
|
||
assert abs(sum(w) - 12.0) < 1e-6 and w[0] > w[1] and min(w) >= BEAT_MIN_PANEL_S, w
|
||
assert all(abs(s - 0.8) < 1e-9 for s in _beat_slices(2.4, 3, [3, 1, 1])) # below floors -> equal
|
||
assert _beat_slices(12.0, 3, [0, 0, 0]) == [4.0, 4.0, 4.0] # zero-sum -> equal split
|
||
assert _beat_slices(12.0, 3, [5]) == [4.0, 4.0, 4.0] # short weights -> equal split
|
||
|
||
# 187: timed cue plan. "Hold up." must be its own cue and must not start before its panel's window.
|
||
sl = _beat_slices(9.0, 3, [1, 1, 1]) # three 3.0s panel windows
|
||
cues = cue_plan("She sees the reflection. Those are psycho eyes. Hold up.", 9.0, sl)
|
||
assert [c["text"] for c in cues] == ["She sees the reflection.", "Those are psycho eyes.", "Hold up."]
|
||
assert cues[0]["start"] == 0.0 # first cue opens the beat
|
||
for c in cues: # every cue starts within/after its panel
|
||
assert c["start"] >= c["panel_index"] * 3.0 - 1e-6, c
|
||
assert cues[-1]["panel_index"] == 2, cues[-1] # last line maps to the last panel
|
||
assert cues == cue_plan("She sees the reflection. Those are psycho eyes. Hold up.", 9.0, sl) # deterministic
|
||
assert cue_plan("", 9.0, sl) == [] # empty narration -> no cues (paragraph fallback)
|
||
# Long chapter assembly is bounded, and the transition out of a batch's last source clip is used
|
||
# to join that batch to the next one. Mock the encoder so this remains a cheap pure orchestration test.
|
||
_real_once = _assemble_once
|
||
_calls = []
|
||
try:
|
||
globals()["_assemble_once"] = lambda ins, trs, out: _calls.append((list(ins), list(trs), out))
|
||
_n = ASSEMBLE_BATCH * 2 + 1
|
||
_batch_trans = [f"t{i}" for i in range(_n)]
|
||
_assemble_batched([f"in{i}.mp4" for i in range(_n)], _batch_trans,
|
||
"/tmp/final.mp4", "batchck", [])
|
||
assert all(len(ins) <= ASSEMBLE_BATCH for ins, _, _ in _calls), _calls
|
||
assert _calls[-1][1][0] == _batch_trans[ASSEMBLE_BATCH - 1], _calls[-1]
|
||
assert _calls[-1][1][1] == _batch_trans[ASSEMBLE_BATCH * 2 - 1], _calls[-1]
|
||
finally:
|
||
globals()["_assemble_once"] = _real_once
|
||
if shutil.which("ffmpeg"):
|
||
aud = f"{SHM}/t.wav"; out = f"{SHM}/t.mp4"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1", aud],
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(aud) - 1.0) < 0.1, _audio_dur(aud) # #1: duration from audio
|
||
# both a wide-short and a tall webtoon-style panel must render (tall used to break pad)
|
||
cams = ({}, {"effect": "pan_right"}, {"effect": "dolly_to_subject", "to": [0.7, 0.3]})
|
||
for size, cam in zip(("200x60", "900x2200", "800x1200"), cams):
|
||
img = f"{SHM}/t.png"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=black:s={size}",
|
||
"-frames:v", "1", img], check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", out, 1.0, cam), check=True, capture_output=True)
|
||
assert os.path.getsize(out) > 0, size
|
||
os.remove(img)
|
||
# #13 padding: a 1.0s narration clip runs ~1.0+PAD_S with the trailing silence held.
|
||
img = f"{SHM}/tp.png"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=400x600",
|
||
"-frames:v", "1", img], check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", out, 1.0, pad=0.4), check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - 1.4) < 0.15, _audio_dur(out)
|
||
# #13 music bed: mixing a generated tone under the clip keeps duration and produces output.
|
||
bed = f"{SHM}/bed.wav"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "sine=frequency=220:duration=0.5", bed],
|
||
check=True, capture_output=True)
|
||
_saved = globals()["MUSIC_BED"]; globals()["MUSIC_BED"] = bed
|
||
cl = []
|
||
mixed = _add_music_bed(out, "selfck", cl)
|
||
assert mixed != out and os.path.exists(mixed) and abs(_audio_dur(mixed) - 1.4) < 0.2, _audio_dur(mixed)
|
||
globals()["MUSIC_BED"] = _saved
|
||
for p in (img, bed, *cl):
|
||
if os.path.exists(p): os.remove(p)
|
||
# #6 transitions: two real clips xfade into one chapter; graph offsets/labels well-formed.
|
||
fg, vmap, amap = _xfade_chain([1.0, 1.0], ["fade_white"])
|
||
assert "xfade=transition=fadewhite" in fg and vmap == "[v1]" and amap == "[a1]"
|
||
img = f"{SHM}/t.png"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=400x600",
|
||
"-frames:v", "1", img], check=True, capture_output=True)
|
||
c0, c1 = f"{SHM}/c0.mp4", f"{SHM}/c1.mp4"
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c0, 1.0), check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c1, 1.0), check=True, capture_output=True)
|
||
subprocess.run(["ffmpeg", "-y", "-i", c0, "-i", c1, "-filter_complex", fg,
|
||
"-map", vmap, "-map", amap, "-c:v", "libx264", "-pix_fmt", "yuv420p",
|
||
"-c:a", "aac", out], check=True, capture_output=True)
|
||
assert os.path.getsize(out) > 0
|
||
# A/V drift: the tree must not stretch video. The concat branch normalizes fps and the xfade
|
||
# branch used not to, so a chapter mixing both played 1.2x slow with the audio untouched and the
|
||
# narration ran ahead of the picture. Batch=2 over 4 clips forces BOTH branches plus a second
|
||
# round -- the shipped bug's exact shape. Compare the streams, not the file size.
|
||
c2, c3 = f"{SHM}/c2.mp4", f"{SHM}/c3.mp4"
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c2, 1.0), check=True, capture_output=True)
|
||
subprocess.run(scene_cmd(img, aud, f"{SHM}/t.ass", c3, 1.0), check=True, capture_output=True)
|
||
_saved_batch, cl = ASSEMBLE_BATCH, []
|
||
globals()["ASSEMBLE_BATCH"] = 2
|
||
try:
|
||
# ["crossfade","crossfade","cut","cut"] with batch=2 is the one shape that mixes branches:
|
||
# round 1 sends [c0,c1] through xfade and [c2,c3] through concat, then round 2 xfades those
|
||
# two intermediates together. A cut-only first group would keep both on the concat branch and
|
||
# the rates would agree by accident, which is why this case has to be spelled out.
|
||
_assemble_batched([c0, c1, c2, c3], ["crossfade", "crossfade", "cut", "cut"],
|
||
out, "selfck", cl)
|
||
finally:
|
||
globals()["ASSEMBLE_BATCH"] = _saved_batch
|
||
for p in cl:
|
||
if os.path.exists(p): os.remove(p)
|
||
vd, ad = _stream_dur(out, "v"), _stream_dur(out, "a")
|
||
assert abs(vd - ad) < 0.25, f"A/V drift: video {vd:.2f}s vs audio {ad:.2f}s"
|
||
for p in (img, c0, c1, c2, c3):
|
||
os.remove(p)
|
||
# #6 composite: 2 panels + 2 audios -> one stacked clip; duration = sum, subs timed per row.
|
||
a2 = f"{SHM}/a2.wav"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1.5", a2],
|
||
check=True, capture_output=True)
|
||
i0, i1 = f"{SHM}/i0.png", f"{SHM}/i1.png"
|
||
for im, sz in ((i0, "300x200"), (i1, "500x300")):
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=gray:s={sz}",
|
||
"-frames:v", "1", im], check=True, capture_output=True)
|
||
segs = [(0.0, 1.0), (1.0, 2.5)]
|
||
still = f"{SHM}/stk.png"
|
||
subprocess.run(_stack_still_cmd([i0, i1], still, H // 2), check=True, capture_output=True)
|
||
_ass_multi([("first beat", *segs[0]), ("second beat", *segs[1])], f"{SHM}/t.ass")
|
||
subprocess.run(composite_cmd(still, [aud, a2], f"{SHM}/t.ass", segs, H // 2, out, 2.5),
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - 2.5) < 0.2, _audio_dur(out) # duration = sum of both narrations
|
||
for p in (a2, i0, i1, still):
|
||
os.remove(p)
|
||
# scene-level narration: 3 images under ONE 1.5s narration -> clip = D + PAD_S, single subtitle.
|
||
b0, b1, b2 = f"{SHM}/b0.png", f"{SHM}/b1.png", f"{SHM}/b2.png"
|
||
for im, sz in ((b0, "300x200"), (b1, "800x1200"), (b2, "500x900")):
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", f"color=c=navy:s={sz}",
|
||
"-frames:v", "1", im], check=True, capture_output=True)
|
||
bnar = f"{SHM}/bnar.wav"
|
||
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1.5", bnar],
|
||
check=True, capture_output=True)
|
||
_ass("one flowing beat narration", 1.5 + PAD_S, f"{SHM}/t.ass")
|
||
cams = ({}, {"effect": "pan_right"}, {"effect": "dolly_to_subject", "to": [0.6, 0.4]})
|
||
subprocess.run(beat_cmd([b0, b1, b2], bnar, f"{SHM}/t.ass", out, 1.5, list(cams)),
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - (1.5 + PAD_S)) < 0.2, _audio_dur(out) # one narration spans N images
|
||
|
||
# 183: collage of the same 3 panels -> one clip of D+PAD, laid out by plan_layout (dominant=2).
|
||
aspects = [_img_size(p)[0] / _img_size(p)[1] for p in (b0, b1, b2)]
|
||
lay = plan_layout(aspects, rtl=True, active=2, frame=(W, H))
|
||
_ass_multi([("collage cue", 0.0, 1.5 + PAD_S)], f"{SHM}/t.ass")
|
||
subprocess.run(collage_cmd([b0, b1, b2], 2, lay["rects"], lay["entrances"], lay["z_order"],
|
||
bnar, f"{SHM}/t.ass", out, 1.5, lay["transition_s"]),
|
||
check=True, capture_output=True)
|
||
assert abs(_audio_dur(out) - (1.5 + PAD_S)) < 0.2, _audio_dur(out) # collage clip = beat length
|
||
assert os.path.getsize(out) > 0
|
||
for p in (b0, b1, b2, bnar):
|
||
os.remove(p)
|
||
os.remove(aud); os.remove(out)
|
||
print("worker_render self-check ok (ffmpeg ran)")
|
||
else:
|
||
print("worker_render self-check ok (ffmpeg absent, ass-only)")
|
||
os.remove(f"{SHM}/t.ass")
|