Fix the real A/V gap: a stream copy across mixed frame rates
The rebuild after 1457556 came out byte-identical to the broken file,
which proved the xfade fix never runs for this chapter. An all-cut chapter
goes down the concat demuxer with -c copy, which writes the output in the
FIRST input's time_base and reinterprets every later packet in it. 14 of
49 clips are 30/1 at 1/15360 against 35 at 25/1 at 1/12800, so those 14
play 15360/12800 = 1.2 too long with their audio untouched. collage_cmd
hardcoded -r 30 and yesterday's FPS sweep missed it.
collage_cmd now emits -r FPS, and assemble probes r_frame_rate across the
clips and routes mixed rates through the re-encoding tree. Rebuilt
chapter.mp4 is 364.120s video against 364.122s audio at 25/1, from
436.392 over 363.675.
Also settle the bbox coordinate space, measured over all 113 detections:
47 boxes have x2 past the 900px panel width, none has y2 past 1000 on
panels up to 2307px tall, and the range is exactly [0, 1000]. It is
gemma's normalized grid, not pixels, whatever the prompt asks for.
/vision converts before returning, which fixes identity's crop, the gated
face pairing that was comparing pixel face boxes against grid boxes, the
set-of-mark boxes and the review UI at once. Checked by eye on panel 7:
five of six boxes now land on their subject, including the foreground
character who had no identity.
The registry still holds boxes and embeddings enrolled from the wrong
space. vision and identity have to re-run, which is GPU work and was not
started.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
+28
-3
@@ -127,6 +127,15 @@ def _stream_dur(path: str, kind: str) -> float:
|
||||
return 0.0
|
||||
|
||||
|
||||
def _fps_of(path: str) -> str:
|
||||
"""`r_frame_rate` as ffprobe reports it. Compared as a string on purpose: two clips agree only when
|
||||
their rate AND therefore their time_base agree, and the stream-copy concat path cares about that."""
|
||||
r = subprocess.run(["ffprobe", "-v", "error", "-select_streams", "v:0",
|
||||
"-show_entries", "stream=r_frame_rate", "-of", "default=nk=1:nw=1", path],
|
||||
capture_output=True, text=True)
|
||||
return r.stdout.strip()
|
||||
|
||||
|
||||
ZMAX, ZPAN = 1.15, 1.18 # ken-burns zoom ceiling; constant zoom that gives pans room to travel
|
||||
|
||||
# Every clip and every assembly stage MUST agree on this. xfade does not resample: it reinterprets the
|
||||
@@ -614,7 +623,7 @@ def collage_cmd(imgs, plate_i, rects, entrances, z_order, audio, ass, out, D, tr
|
||||
cur = f"o{k}"
|
||||
fc = ";".join(parts) + f";[{cur}]ass={ass}[v];[{n}:a]apad=pad_dur={PAD_S:.3f}[a]"
|
||||
cmd += ["-filter_complex", fc, "-map", "[v]", "-map", "[a]", "-t", f"{T:.3f}",
|
||||
"-r", "30", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||||
"-r", str(FPS), "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", "-b:a", "192k", out]
|
||||
return cmd
|
||||
|
||||
|
||||
@@ -863,15 +872,26 @@ async def assemble(data: AssembleInput):
|
||||
cleanup = list(locals_) + [out]
|
||||
|
||||
fancy = len(locals_) >= 2 and any(t not in ("", "cut") for t in data.transitions)
|
||||
if fancy:
|
||||
# The stream-copy path writes the output with the FIRST input's time_base and reinterprets every
|
||||
# later packet in it. A clip encoded at 30fps (time_base 1/15360) copied into a 25fps container
|
||||
# (1/12800) therefore plays 15360/12800 = 1.2x too long with its audio untouched. That is the whole
|
||||
# of the shipped chapter's 436.39s of video over 363.67s of narration: 14 of 49 clips came off the
|
||||
# collage path, which hardcoded `-r 30`. Mixed rates must re-encode, so they go through the tree,
|
||||
# whose branches both normalize with `fps={FPS}`.
|
||||
rates = {_fps_of(p) for p in locals_}
|
||||
if fancy or len(rates) > 1:
|
||||
if len(rates) > 1:
|
||||
print(f"[render] mixed clip rates {sorted(rates)}, re-encoding instead of stream copy",
|
||||
flush=True)
|
||||
_assemble_batched(locals_, data.transitions, out, tag, cleanup)
|
||||
else:
|
||||
# all hard cuts: stream-copy concat (no re-encode) -- unchanged fast path.
|
||||
# all hard cuts at one shared rate: stream-copy concat, no re-encode.
|
||||
listfile = f"{SHM}/asm_{tag}.txt"; cleanup.append(listfile)
|
||||
with open(listfile, "w") as f:
|
||||
f.write("".join(f"file '{p}'\n" for p in locals_))
|
||||
subprocess.run(["ffmpeg", "-y", "-f", "concat", "-safe", "0", "-i", listfile, "-c", "copy", out],
|
||||
check=True, capture_output=True)
|
||||
_check_assembled(out, sum(min(_stream_dur(p, "v"), _stream_dur(p, "a")) for p in locals_))
|
||||
|
||||
out = _add_music_bed(out, tag, cleanup)
|
||||
|
||||
@@ -1087,6 +1107,11 @@ if __name__ == "__main__":
|
||||
check=True, capture_output=True)
|
||||
assert abs(_audio_dur(out) - (1.5 + PAD_S)) < 0.2, _audio_dur(out) # collage clip = beat length
|
||||
assert os.path.getsize(out) > 0
|
||||
# ...and it must come out at FPS like every other clip path. This one hardcoded `-r 30`, so 14 of
|
||||
# 49 clips in the shipped chapter were 30fps. `concat -c copy` writes the output with the FIRST
|
||||
# clip's time_base and reinterprets later packets in it, so those 14 played 1.2x too long with
|
||||
# their narration untouched. That, not the xfade tree, is where the 72.7s gap came from.
|
||||
assert _fps_of(out) == f"{FPS}/1", f"collage clip is {_fps_of(out)}, not {FPS}/1"
|
||||
for p in (b0, b1, b2, bnar):
|
||||
os.remove(p)
|
||||
os.remove(aud); os.remove(out)
|
||||
|
||||
Reference in New Issue
Block a user