kami/manga-recap-pipeline
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix the 72.7s A/V gap: xfade offsets ran off the end of their input

Browse Source 
_xfade_chain positioned every transition using _audio_dur, which probes
format=duration, which is max(video, audio). A clip's audio outlasts its
video by about a frame, so the offset accumulator crept ahead of the real
picture timeline. Once the creep exceeded the transition width, xfade
emitted the transition and silently discarded the second input and every
clip downstream, exiting 0 with nothing on stderr. That is the whole of
the shipped chapter's 436.39s of video over 363.67s of audio.

Offsets now come from min(video, audio). Every input is floored to a
whole frame count and trimmed on both streams, so the accumulator tracks
the real timeline instead of estimating it. _check_assembled verifies
each encode against the predicted length and against its own audio,
because both assembly branches drop stream time without failing.

Verified over the 49 real clips of chapter 7c944dd4: the round that
turned 359s of video into 100s now loses 0.85s, and the chapter comes out
358.76s video against 358.76s audio.

The single-item passthrough was not the cause. Two round-0 groups of 8
fresh clips collapse without one, recorded void in decisions/.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
kami
2026-08-12 11:52:38 +04:00
parent a9d64fe80a
commit 1457556ce3
6 changed files with 206 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
worker_render.py
+67 -8
View File
@@ -686,31 +686,57 @@ FFMPEG_THREADS = max(1, int(os.environ.get("FFMPEG_THREADS", "2")))
def _xfade_chain(durs: list, trans: list):
"""build a filter_complex that xfades N clips with per-boundary transitions, keeping audio in
sync via matching acrossfade. trans[i] is the transition OUT of clip i (boundary i->i+1).
returns (filtergraph, video_label, audio_label). offsets accumulate as clips overlap."""
returns (filtergraph, video_label, audio_label, expected_duration). offsets accumulate as clips
overlap. `durs[i]` MUST be min(video, audio) of input i, not `format=duration`."""
# A clip whose duration probed as 0/unreadable must not poison the chain: with dur=0 the offset
# accumulator would run BACKWARDS (cum += dur - td), swallowing every later clip into a frozen
# overlap near the middle. Floor to a small positive length so the timeline stays monotonic.
durs = [d if (d and d > 0.1) else 0.1 for d in durs]
# The offset accumulator is only as good as `durs`: an offset that lands even one frame past the end
# of input i-1 makes xfade emit the transition and then SILENTLY DROP input i and the whole rest of
# the chain -- rc 0, no warning, a chapter minutes short with the audio intact. So truncate every
# input to a whole number of frames it certainly has (floor, and min(v,a) from the caller) and trim
# both streams to exactly that. Then cum is the real timeline, not an estimate of it.
durs = [max(1, int(d * FPS)) / FPS for d in durs]
# Normalize every input to FPS/SAR before it reaches xfade, exactly as the concat branch does. Both
# branches feed the same tree, so an un-normalized xfade input is what stretched the chapter 1.2x.
parts = [f"[{i}:v]setsar=1,fps={FPS}[n{i}]" for i in range(len(durs))]
vlast, alast, cum = "[n0]", "[0:a]", durs[0]
parts = []
for i, d in enumerate(durs):
parts.append(f"[{i}:v]setsar=1,fps={FPS},trim=end={d:.3f},setpts=PTS-STARTPTS[n{i}]")
parts.append(f"[{i}:a]atrim=end={d:.3f},asetpts=PTS-STARTPTS[m{i}]")
vlast, alast, cum = "[n0]", "[m0]", durs[0]
for i in range(1, len(durs)):
name, td = XFADE.get(trans[i - 1] if i - 1 < len(trans) else "cut", XFADE["cut"])
td = max(0.05, min(td, durs[i - 1] - 0.05, durs[i] - 0.05)) # overlap fits in both clips
td = max(1, int(td * FPS)) / FPS # ...on a frame boundary
off = max(cum - td, 0)
parts.append(f"{vlast}[n{i}]xfade=transition={name}:duration={td:.3f}:offset={off:.3f}[v{i}]")
parts.append(f"{alast}[{i}:a]acrossfade=d={td:.3f}[a{i}]")
parts.append(f"{alast}[m{i}]acrossfade=d={td:.3f}[a{i}]")
vlast, alast, cum = f"[v{i}]", f"[a{i}]", cum + durs[i] - td
return ";".join(parts), vlast, alast
return ";".join(parts), vlast, alast, cum
ASSEMBLE_TOL_S = 0.5 # frame-boundary + aac-padding slack; a dropped input is off by whole seconds
def _check_assembled(out: str, expect: float):
"""ffmpeg drops xfade inputs and re-times concat segments without ever failing, so verify the
result instead of trusting rc 0. Both streams, because a video-only loss is the failure mode that
shipped a 436s picture over 364s of narration."""
v, a = _stream_dur(out, "v"), _stream_dur(out, "a")
if abs(v - expect) > ASSEMBLE_TOL_S or abs(v - a) > ASSEMBLE_TOL_S:
raise RuntimeError(f"assembly lost stream time in {os.path.basename(out)}: "
f"video={v:.2f} audio={a:.2f} expected={expect:.2f}")
def _assemble_once(inputs: list[str], trans: list[str], out: str):
"""Assemble one bounded batch. `trans[i]` is the transition out of inputs[i]."""
# min(video, audio), never `format=duration`: that is max(video, audio), and feeding it to
# _xfade_chain puts the offset accumulator ahead of the real video timeline.
durs = [min(_stream_dur(p, "v"), _stream_dur(p, "a")) for p in inputs]
fancy = len(inputs) >= 2 and any(t not in ("", "cut") for t in trans[:len(inputs) - 1])
if fancy:
durs = [_audio_dur(p) for p in inputs]
fg, vmap, amap = _xfade_chain(durs, trans)
fg, vmap, amap, expect = _xfade_chain(durs, trans)
cmd = ["ffmpeg", "-y", "-filter_complex_threads", str(FFMPEG_THREADS)]
# Input-side -threads limits each decoder; otherwise ffmpeg may create a decoder thread pool
# for every input in the batch in addition to the filter and libx264 pools.
@@ -720,6 +746,7 @@ def _assemble_once(inputs: list[str], trans: list[str], out: str):
"-c:v", "libx264", "-threads", str(FFMPEG_THREADS), "-pix_fmt", "yuv420p",
"-c:a", "aac", "-b:a", "192k", out]
subprocess.run(cmd, check=True, capture_output=True)
_check_assembled(out, expect)
return
# A cut-only batch is concatenated with the concat FILTER, not -c copy or the concat demuxer.
@@ -738,6 +765,7 @@ def _assemble_once(inputs: list[str], trans: list[str], out: str):
"-c:v", "libx264", "-preset", "veryfast", "-crf", "20", "-pix_fmt", "yuv420p",
"-threads", str(FFMPEG_THREADS), "-c:a", "aac", "-b:a", "192k", out]
subprocess.run(cmd, check=True, capture_output=True)
_check_assembled(out, sum(durs))
def _assemble_batched(inputs: list[str], transitions: list[str], out: str, tag: str,
@@ -952,8 +980,14 @@ if __name__ == "__main__":
for p in (img, bed, *cl):
if os.path.exists(p): os.remove(p)
# #6 transitions: two real clips xfade into one chapter; graph offsets/labels well-formed.
fg, vmap, amap = _xfade_chain([1.0, 1.0], ["fade_white"])
fg, vmap, amap, exp = _xfade_chain([1.0, 1.0], ["fade_white"])
assert "xfade=transition=fadewhite" in fg and vmap == "[v1]" and amap == "[a1]"
# Both streams of every input trimmed to a whole frame count. That is what keeps the offset
# accumulator ON the real timeline: an offset one frame past the end of input i-1 makes xfade
# emit the transition and then silently drop input i and everything after it, rc 0, no warning.
assert fg.count(",trim=end=") == 2 and fg.count("]atrim=end=") == 2, fg
_td = max(1, int(XFADE["fade_white"][1] * FPS)) / FPS
assert abs(exp - (2.0 - _td)) < 0.001, (exp, _td)
img = f"{SHM}/t.png"
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "color=c=black:s=400x600",
"-frames:v", "1", img], check=True, capture_output=True)
@@ -986,8 +1020,33 @@ if __name__ == "__main__":
if os.path.exists(p): os.remove(p)
vd, ad = _stream_dur(out, "v"), _stream_dur(out, "a")
assert abs(vd - ad) < 0.25, f"A/V drift: video {vd:.2f}s vs audio {ad:.2f}s"
# ...and the guard that catches it in production must actually fire. `assemble` reports success
# off ffmpeg's rc, which is 0 even when a whole batch of inputs is thrown away.
try:
_check_assembled(out, vd + 5.0)
raise AssertionError("_check_assembled did not fire on a 5s loss")
except RuntimeError:
pass
for p in (img, c0, c1, c2, c3):
os.remove(p)
# The offset creep itself, reproduced small. A real clip's audio outlasts its video slightly, and
# `format=duration` reports the audio, so the accumulator walked ahead of the picture until one
# xfade window ran past the end of its first input -- at which point ffmpeg emitted the
# transition, threw away the second input and everything downstream, and exited 0. Clips with
# audio 0.4s longer than the video exaggerate one clip's worth of that creep.
cs = []
for i in range(3):
c = f"{SHM}/creep{i}.mp4"; cs.append(c)
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "testsrc2=s=160x120:r=25:d=1.0",
"-f", "lavfi", "-i", f"sine=f={300 + i * 40}:d=1.4",
"-c:v", "libx264", "-preset", "ultrafast", "-pix_fmt", "yuv420p",
"-c:a", "aac", c], check=True, capture_output=True)
_assemble_once(cs, ["fade_black", "cut", "cut"], out) # raises if an input was dropped
vd, ad = _stream_dur(out, "v"), _stream_dur(out, "a")
assert vd > 2.2, f"xfade dropped inputs: {vd:.2f}s from 3 clips of 1.0s"
assert abs(vd - ad) < 0.1, f"A/V drift: video {vd:.2f}s vs audio {ad:.2f}s"
for p in cs:
os.remove(p)
# #6 composite: 2 panels + 2 audios -> one stacked clip; duration = sum, subs timed per row.
a2 = f"{SHM}/a2.wav"
subprocess.run(["ffmpeg", "-y", "-f", "lavfi", "-i", "anullsrc=r=16000:cl=mono", "-t", "1.5", a2],