diff --git a/CLAUDE.md b/CLAUDE.md index 17f8fd4..432b577 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -116,7 +116,7 @@ VapourBox/ | `worker/src/pixel_format.rs` | Source `pix_fmt` → pipe format (see "Source Pixel Formats") | | `worker/templates/stabilize.py` | Stabilize pass's vertical pitch correction (see "Stabilize") | | `worker/src/cpu.rs` | The x86 deps CPU tier (`v3`/`v2`) — the single decision; see "CPU tiers" | -| `worker/src/source_decode.rs` | Decoder ffmpeg args for encode + preview: full stored frame, size-guarded (see "Source Frame Size") | +| `worker/src/source_decode.rs` | Decoder ffmpeg args for encode + preview: full stored frame, size-guarded, PTS reset after a seek (see "Source Frame Size") | | `worker/src/source_timing.rs` | A variable-frame-rate source's stored frames placed on the constant-rate grid (see "Variable Frame Rate Sources") | | `worker/src/gpu_plugin.rs` | The ncnn-on-Vulkan plugins (RIFE, AI upscaling): file layout, RIFE's model add-on, the `--probe-rife` / `--probe-upscale` probe (on stand-in models, no add-on needed), and the user-facing reason/remedy when they can't run (see "GPU Plugins") | | `worker/src/animejanai.rs` | AnimeJaNai's add-on models: where they live, which one a job runs, the refusal when they're missing (see "Add-ons") | @@ -1016,10 +1016,25 @@ the **single** builder of both decoders' arguments (encode + preview): `explain_decoder_failure` ahead of the vspipe/encoder status. A job that omits `inputWidth`/`inputHeight` gets them probed by the worker (`fill_frame_size`), not the old 720x480 fallback. +- **A decoder that seeks resets PTS first (issue #157).** `rawvideo` gets + constant-rate output timing, and after an `-ss` to `(N - 0.5)/fps` that + timing repeated the first kept frame on many time bases (MP4 1/2997, MOV + 1/11988, MKV 1/1000): the preview showed N-1, a trim of 41..52 came out + `41, 41, 42 .. 51`, and the noise score read duplicated pairs as noiseless. + So `RESET_PTS_FILTER` (`setpts=PTS-STARTPTS`) goes **first in the one + `-vf`**, only when the decoder seeks; a decoder that doesn't seek passes no + `-ss` (not even `-ss 0`) and no reset, so untrimmed encodes are untouched and + preview frame 0 is the encode's frame 0. Never `-fps_mode passthrough` here: + it drops the constant-rate slots a soft-telecine source decodes to (only the + VFR decoder uses it). **Test seek alignment against an independent decode** + (`select=eq(n,N)` of the source), never preview against encode, which shift + together — `integration_seek_alignment_test.dart` (heavy). MPEG-PS/TS, MPEG-2 + in AVI and leading null frames still seek to the wrong picture for other + reasons (docs/reports/issue-157-seek-alignment). `integration_clean_aperture_test.dart` (heavy) asserts a clean-aperture MOV and MKV come through bit-exact at full size. See docs/ENGINEERING_NOTES.md, -"Clean aperture". +"Clean aperture" and "Seek alignment". ### Variable Frame Rate Sources (issue #137) diff --git a/Tests/TestResources/generate_seek_timebase_test.sh b/Tests/TestResources/generate_seek_timebase_test.sh new file mode 100755 index 0000000..4248343 --- /dev/null +++ b/Tests/TestResources/generate_seek_timebase_test.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Generate seek_timebase_2997.mov: a 29.97 fps MOV whose video track time base +# is 1/11988, the timescale a user's ProRes clip had (issue #157). +# +# Why this fixture exists: with that time base (and MP4's 1/2997, and TS's +# 1/90000), an ffmpeg that seeks and writes rawvideo used to emit the first +# kept frame twice, so the preview showed frame N-1 and trimmed encodes were a +# frame late. interlaced_test.avi (1/25) cannot show it, which is how it went +# unnoticed. integration_seek_alignment_test.dart decodes it from several start +# frames and compares against a sequential decode. +# +# Content: testsrc2 (every frame differs), 320x240, 3 s = 90 frames, H.264 with +# B-frames, plus a sine AAC track so the container has audio like a real clip. +# +# Usage: ./generate_seek_timebase_test.sh [output_dir] [ffmpeg] + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" +OUTPUT_DIR="${1:-$SCRIPT_DIR}" +FFMPEG="${2:-ffmpeg}" +FFPROBE="$(dirname "$FFMPEG")/ffprobe" +[ -x "$FFPROBE" ] || FFPROBE=ffprobe +OUT="$OUTPUT_DIR/seek_timebase_2997.mov" + +"$FFMPEG" -y -v error \ + -f lavfi -i "testsrc2=size=320x240:rate=30000/1001:duration=3.003" \ + -f lavfi -i "sine=frequency=440:sample_rate=48000:duration=3.003" \ + -map 0:v -map 1:a \ + -c:v libx264 -preset veryfast -crf 20 -g 30 -bf 2 -pix_fmt yuv420p \ + -c:a aac -b:a 64k \ + -video_track_timescale 11988 \ + "$OUT" + +tb="$("$FFPROBE" -v error -select_streams v:0 -show_entries stream=time_base -of csv=p=0 "$OUT")" +if [ "$tb" != "1/11988" ]; then + echo "expected video time base 1/11988, got $tb" >&2 + exit 1 +fi +frames="$("$FFPROBE" -v error -select_streams v:0 -count_frames -show_entries stream=nb_read_frames -of csv=p=0 "$OUT")" +echo "Wrote $OUT (time base $tb, $frames frames)" diff --git a/Tests/TestResources/seek_timebase_2997.mov b/Tests/TestResources/seek_timebase_2997.mov new file mode 100644 index 0000000..0287a8a Binary files /dev/null and b/Tests/TestResources/seek_timebase_2997.mov differ diff --git a/app/lib/services/preview_generator.dart b/app/lib/services/preview_generator.dart index 7f882ec..709129f 100644 --- a/app/lib/services/preview_generator.dart +++ b/app/lib/services/preview_generator.dart @@ -211,12 +211,43 @@ class PreviewGenerator { return thumbnails; } - /// Get the unprocessed source frame at an exact frame index. + /// The ffmpeg arguments that write source frame [frameIndex] (the "before" + /// picture) to [outputPath]. /// /// Frame-accurate: seeks to the midpoint of the interval *before* the target /// frame ((index - 0.5)/fps) so the first decoded frame (PTS >= seek time) is /// exactly `frameIndex`. This matches the worker's preview seek, so the - /// before/after comparison always shows the same source frame. + /// before/after comparison always shows the same source frame. An image + /// output is written with passthrough timing, so unlike the worker's + /// `rawvideo` decoders it needs no PTS reset after the seek (issue #157). + /// + /// `integration_seek_alignment_test.dart` runs exactly these arguments (with + /// a PNG [outputPath]) against an independent sequential decode, so keep + /// every decode option here rather than at the call site. + static List beforeFrameArgs({ + required String videoPath, + required int frameIndex, + required double frameRate, + String? toneMap, + required String outputPath, + }) { + final frame = frameIndex < 0 ? 0 : frameIndex; + final seekTime = (frame - 0.5) / frameRate; + final ss = seekTime < 0 ? 0.0 : seekTime; + return [ + '-y', + '-ss', ss.toStringAsFixed(6), + ...sourceDecodeOptions, + '-i', videoPath, + '-frames:v', '1', + if (toneMap != null) ...['-vf', toneMap], + '-q:v', '2', + outputPath, + ]; + } + + /// Get the unprocessed source frame at an exact frame index — see + /// [beforeFrameArgs]. Future getFrameAtIndex(int frameIndex) async { if (_currentVideoPath == null || _ffmpegPath == null) return null; @@ -229,22 +260,17 @@ class PreviewGenerator { // Extract frame final outputPath = '$_tempDir/frame_$frame.jpg'; - final seekTime = ((frame - 0.5) / _frameRate); - final ss = seekTime < 0 ? 0.0 : seekTime; try { final result = await Process.run( _ffmpegPath!, - [ - '-y', - '-ss', ss.toStringAsFixed(6), - ...sourceDecodeOptions, - '-i', _currentVideoPath!, - '-frames:v', '1', - if (_toneMap case final toneMap?) ...['-vf', toneMap], - '-q:v', '2', - outputPath, - ], + beforeFrameArgs( + videoPath: _currentVideoPath!, + frameIndex: frame, + frameRate: _frameRate, + toneMap: _toneMap, + outputPath: outputPath, + ), ); if (result.exitCode == 0 && await File(outputPath).exists()) { diff --git a/app/test/integration_frame_mapping_test.dart b/app/test/integration_frame_mapping_test.dart index a42c378..f208642 100644 --- a/app/test/integration_frame_mapping_test.dart +++ b/app/test/integration_frame_mapping_test.dart @@ -563,9 +563,14 @@ void main() { await check(job('subframe_sync'), [12, 13, 21]); }, timeout: const Timeout(Duration(minutes: 10))); + // The in-point is past interlaced_test.avi's leading null frames (stored + // pts 0, then 5). A seek into that gap rebases to the first stored frame + // after it (issue #157's PTS reset), so a trim starting at 1..4 is shifted + // against the untrimmed numbering — the leading-repeat-slots problem of its + // own, not what this test is about. Same offsets as 3 -> 13, 19. test('from an in-point', () async { - await check(job('subframe_sync_trim', start: 3, end: 50), [13, 19], - origin: 3); + await check(job('subframe_sync_trim', start: 6, end: 53), [16, 22], + origin: 6); }, timeout: const Timeout(Duration(minutes: 10))); }); diff --git a/app/test/integration_seek_alignment_test.dart b/app/test/integration_seek_alignment_test.dart new file mode 100644 index 0000000..ce0d2ff --- /dev/null +++ b/app/test/integration_seek_alignment_test.dart @@ -0,0 +1,222 @@ +// Seek alignment (issue #157): a preview of source frame N, and a trimmed +// encode from frame S, must deliver exactly those source pictures. +// +// The worker's decoders write `rawvideo`, which ffmpeg times at a constant +// rate. After a seek, for many time bases (MP4 1/2997, MOV 1/11988, TS +// 1/90000) that timing used to emit the first kept frame TWICE: the preview +// showed frame N-1 next to the app's correct "before" frame N, and a trim of +// 41..52 came out 41, 41, 42 .. 51. Preview and encode shifted together, so +// integration_frame_mapping_test.dart (preview vs encode, on a 1/25 AVI the +// bug cannot touch) passed throughout. +// +// So everything here is checked against an INDEPENDENT reference: a +// sequential decode of the source picked by index (`select=eq(n,N)`), never +// one pipeline path against another. The fixtures are ones whose sequential +// numbering is the app's frame numbering (no leading null frames, no +// pulldown, video starting with the container). Sources where it isn't +// (MPEG-PS/TS, MPEG-2 in AVI, soft telecine, AVI null frames) are separate +// issues; see docs/reports/issue-157-seek-alignment. +@Tags(['heavy']) +library; + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; +import 'package:vapourbox/services/preview_generator.dart'; + +import 'support/worker_harness.dart'; + +String get _outDir => '${WorkerHarness.outputDir}/seek_alignment'; + +String _resource(String name) => + p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', name); + +/// A fixture and why it is here. +class _Fixture { + final String file; + final String why; + const _Fixture(this.file, this.why); +} + +const _fixtures = [ + _Fixture('small_clip.mp4', 'MP4, time base 1/2997: affected before #157'), + _Fixture('seek_timebase_2997.mov', + 'MOV, time base 1/11988 (a user\'s ProRes timescale): affected before #157'), + _Fixture('face_restore_test.mp4', 'MP4, time base 1/30000: control'), +]; + +/// Frames checked in each fixture (all fixtures have at least 90). +const _previewFrames = [10, 41, 77]; +const _trimStart = 41; +const _trimEnd = 52; + +/// What the app learns from ffprobe, as `FieldOrderDetector` would send it. +class _Source { + final int width; + final int height; + final String pixFmt; + final double fps; + _Source(this.width, this.height, this.pixFmt, this.fps); +} + +double _rate(String? s) { + final parts = (s ?? '0/0').split('/'); + final n = double.tryParse(parts[0]) ?? 0; + final d = parts.length > 1 ? double.tryParse(parts[1]) ?? 0 : 1; + return d == 0 ? 0 : n / d; +} + +Future<_Source> _probe(String path) async { + final v = await WorkerHarness.firstStream(path, + selector: 'v:0', + entries: ['width', 'height', 'pix_fmt', 'r_frame_rate', 'avg_frame_rate']); + expect(v, isNotNull, reason: 'no video stream in $path'); + final r = _rate(v!['r_frame_rate'] as String?); + final a = _rate(v['avg_frame_rate'] as String?); + return _Source(v['width'] as int, v['height'] as int, v['pix_fmt'] as String, + r > 0 ? r : a); +} + +VideoJob _job(String input, _Source s, String name, {int? start, int? end}) { + const off = QTGMCParameters(enabled: false); + return VideoJob( + id: const Uuid().v4(), + inputPath: input, + outputPath: '$_outDir/$name.mkv', + qtgmcParameters: off, + processingPipeline: const ProcessingPipeline(deinterlace: off), + encodingSettings: EncodingSettings( + codec: VideoCodec.ffv1, + container: ContainerFormat.mkv, + audioMode: AudioMode.none, + ), + inputWidth: s.width, + inputHeight: s.height, + inputPixelFormat: s.pixFmt, + inputFrameRate: s.fps, + startFrame: start, + endFrame: end, + totalFrames: start != null && end != null ? end - start + 1 : null, + ); +} + +/// The app's own "before" picture of [frame], through the same argument +/// builder `PreviewGenerator.getFrameAtIndex` uses (PNG instead of JPEG, so +/// the comparison isn't measuring JPEG). +Future _beforeFrame(String video, int frame, double fps) async { + final out = File(p.join(Directory.systemTemp.path, + 'vb_seek_before_${DateTime.now().microsecondsSinceEpoch}.png')); + try { + final r = await Process.run( + WorkerHarness.ffmpegPath, + PreviewGenerator.beforeFrameArgs( + videoPath: video, + frameIndex: frame, + frameRate: fps, + outputPath: out.path, + ), + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: 'before frame $frame: ${r.stderr}'); + return await WorkerHarness.imageToRgb24(await out.readAsBytes(), label: 'before'); + } finally { + if (await out.exists()) await out.delete().catchError((_) => out); + } +} + +/// Asserts [img] is source frame [n] of [refs] (sequential frames keyed by +/// index): the closest of the neighbours, and near-identical to it. +/// +/// Relative, as in integration_frame_mapping_test.dart: an absolute bound +/// alone could not tell a one-frame shift from a colour conversion. The +/// absolute bound is there too, because both sides are the same decode and +/// RGB conversion of the same picture, so they agree (measured 0.0 on macOS +/// arm64). +void _expectFrame(String what, Uint8List img, int n, Map refs) { + final diffs = { + for (final e in refs.entries) e.key: WorkerHarness.meanAbsDiff(img, e.value), + }; + final best = diffs.entries.reduce((a, b) => a.value <= b.value ? a : b).key; + final table = diffs.entries + .map((e) => '${e.key}: ${e.value.toStringAsFixed(2)}') + .join(', '); + expect(best, n, reason: '$what is source frame $best, not $n ($table)'); + expect(diffs[n]!, lessThan(0.5), + reason: '$what is not the same picture as source frame $n ($table)'); +} + +Future> _neighbours(String video, int n) async => { + for (final k in [n - 1, n, n + 1]) + k: await WorkerHarness.frameRgb24(video, k, label: 'src'), + }; + +void main() { + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_outDir).create(recursive: true); + }); + + for (final f in _fixtures) { + group('${f.file} (${f.why})', () { + final video = _resource(f.file); + late _Source source; + + setUpAll(() async { + expect(await File(video).exists(), isTrue, reason: 'missing $video'); + source = await _probe(video); + }); + + for (final n in _previewFrames) { + test('the preview of frame $n, every pass off, is source frame $n, ' + 'as is the app\'s before frame', () async { + final pv = await WorkerHarness.runPreview( + _job(video, source, 'preview').toJson(), + frame: n, + label: 'seek_preview_$n'); + expect(pv.success, isTrue, reason: '${pv.error}\n${pv.errorTail}'); + final preview = + await WorkerHarness.imageToRgb24(pv.png!, label: 'preview'); + final before = await _beforeFrame(video, n, source.fps); + final refs = await _neighbours(video, n); + + _expectFrame('the preview', preview, n, refs); + _expectFrame('the before frame', before, n, refs); + }); + } + + test('a trimmed lossless encode of $_trimStart..$_trimEnd starts at ' + '$_trimStart and ends at $_trimEnd', () async { + final r = await WorkerHarness.runJob( + _job(video, source, 'trim_${p.basenameWithoutExtension(f.file)}', + start: _trimStart, end: _trimEnd) + .toJson(), + label: 'seek_trim', + ); + expect(r.success, isTrue, reason: r.error); + final out = r.outputPath!; + final count = _trimEnd - _trimStart + 1; + + final frames = await WorkerHarness.firstStream(out, + selector: 'v:0', entries: ['nb_read_frames'], countFrames: true); + expect(int.parse(frames!['nb_read_frames'].toString()), count); + + // The first two frames catch a doubled first frame (41, 41, 42 ..); + // the last catches the frame that doubling pushed off the end. + for (final i in [0, 1, count - 1]) { + final n = _trimStart + i; + _expectFrame('output frame $i of the trim', + await WorkerHarness.frameRgb24(out, i, label: 'trim'), n, + await _neighbours(video, n)); + } + }); + }); + } +} diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 936018b..4880cb6 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -3486,3 +3486,56 @@ anonymous download link for one asset returned 504 for a while (a GitHub incident; the API served the file) — re-uploading did not help, waiting did. A manifest's `url` is independent of its installed `name`, so a stuck asset could have been re-hosted under another name without touching code. + +### Seek alignment: the decoder repeated the first frame after a seek (issue #157, 2026-11-10) + +Full report with scripts: +[docs/reports/issue-157-seek-alignment/](reports/issue-157-seek-alignment/index.html). + +- Reported (videohelp forum, thread 419911 post #152): with every filter off, + the preview's Processed frame lagged the Original by one. +- Cause: the worker's decoders write `rawvideo`, which has + `AVFMT_NOTIMESTAMPS` but not `AVFMT_VARIABLE_FPS`, so the ffmpeg CLI times + it at a **constant rate**. The preview decoder (always) and a trimmed encode's + decoder seek to `(first - 0.5) / fps`; the kept frame then sits half a frame + after the output's zero, and for time bases such as MP4 1/2997, MOV 1/11988, + MKV 1/1000 and TS 1/90000 the constant-rate grid fills that gap by emitting + the frame twice. Preview: frame N-1 beside a correct Original (an image + muxer gets passthrough timing). Trim 41..52: `41, 41, 42 .. 51`. Noise + score: every sampled pair one picture twice, so sigma ~4.5 noise read 0.0 + and was never suggested. A soft-telecine trim lost 15 of 60 film frames. +- Fix: `setpts=PTS-STARTPTS` first in the seeking decoder's single `-vf` + (`source_decode::RESET_PTS_FILTER`), and no `-ss` at all when the preview + window starts at frame 0, so it decodes like the untrimmed encode (which + moved preview frame 0 onto the encode's frame 0 on two sources and left the + other 11 unchanged). Untrimmed encodes are bit-identical (framemd5, five + jobs incl. QTGMC). Not `-fps_mode passthrough`: on soft telecine it hands + over stored frames instead of the constant-rate slots an untrimmed encode + gets. +- Why no test saw it: preview and encode shifted together, and the + frame-mapping fixture (`interlaced_test.avi`, 1/25) has no doubled frame. +- Correction (review): that fixture's **leading** null frames (stored pts 0, + then 5) are the one case the reset changes. A seek into them was right + from the first real frame on (constant-rate timing filled the gap from the + seek point); the reset rebases to the frame after the gap, so a trim from + 1..4 runs up to two frames early. The frame-mapping test's "from an + in-point" used in-point 3 and failed; it now starts at 6. Mid-stream null + frames (dropped-frame captures, which report `avg == r` and so never reach + the VFR decoder) are unchanged: frames 100-104 dropped, old decoder and + reset give the same frames from every start 97-106 (both wrong for an + in-point inside the run). Subtracting the seek's half frame + instead (`PTS-0.5/fps/TB`, keeps gaps) fixes the same doubles but ties + and rounds early in a 1/25 time base and broke starts 98-99 there. Leading + null frames are a follow-up (task 16). Script: the report's + `scripts/gap_probe.py`. + `integration_seek_alignment_test.dart` compares the preview, the app's + Original (`PreviewGenerator.beforeFrameArgs`) and a trimmed FFV1 encode each + with an independent `select=eq(n,N)` decode; against the old worker it fails + 8 of 12. New fixture `seek_timebase_2997.mov` (1/11988). +- The noise report's thresholds stand: its fixtures are intra FFV1 MKVs, which + seek correctly, and its reference reads from frame 0. +- Found while measuring, **not fixed here**, all the same under passthrough + timing: MPEG-PS/TS seeks start at a later keyframe (DVD-style PS: 40 -> 48; + TS: 40 -> 50; the DVB-T capture); MPEG-2 in AVI lands 1-2 frames early; + AVI null frames and soft telecine give an untrimmed decode leading repeat + slots (79 slots for 75 frames; 122 for 120), so frame numbers disagree. diff --git a/docs/reports/README.md b/docs/reports/README.md index 533e59c..3223952 100644 --- a/docs/reports/README.md +++ b/docs/reports/README.md @@ -66,6 +66,7 @@ to it. | Date | Report | Outcome | |---|---|---| +| 2026-11-10 | [Issue #157: seek alignment, the decoder repeated the first frame after a seek](issue-157-seek-alignment/index.html) | Fixed: `setpts=PTS-STARTPTS` on every seeking `rawvideo` decoder. MP4 1/2997, MOV 1/11988, MKV 1/1000 sources previewed frame N-1 and trimmed encodes doubled their first frame (41, 41, 42 .. 51); now right from every start tried, untrimmed encodes bit-identical, and noisy sources the noise score read as 0.0 now score ~4.5. Also found, not fixed: MPEG-PS/TS seeks land on a later keyframe, MPEG-2 in AVI 1-2 frames early, AVI null frames / soft telecine add leading repeat slots | | 2026-11-10 | [Issue #148: Frame Rate combined with Noise Reduction and Sharpen](issue-148-frame-rate-combined/index.html) | Not reproduced on macOS arm64 across 5 denoisers x 4 sharpeners x FlowFPS/RIFE x 8/10-bit; no pipeline change; nightly regression test added to cover Windows/MVTools v24 and other GPUs; reporter asked for a clip | | 2026-11-09 | [Noise and halo scores for the pass list](2026-11-noise-halo-scores/index.html) | Built: `--detect-noise-halo` badges Noise Reduction at a motion-compensated noise sigma ≥ 3.5 and Dehalo at a two-sided edge overshoot ≥ 6% of the step; no false positives on 154 fixtures (clean max 2.06 / 1.48), catches white noise from sigma 4 and tape/grain noise from sigma 6, misses sharpening of already-soft edges. VCG's signalstats TOUT and halo ratio did not separate the fixtures; a spatial (Immerkær) noise score was rejected as blind to tape and grain noise | | 2026-11-08 | [Crop & Resize: BT.601 → BT.709 on SD → HD, and Detect borders](2026-11-crop-detect-709/index.html) | Built: matrix-only 601 → 709 conversion when the output is HD (default Auto; patches within 0.67 levels of the source; a gamut conversion rejected: it moved 75% green by 55 levels on players without colour management); `--detect-crop` finds blanking, bars and the head-switching band exactly on real-picture fixtures, and matches a by-eye measurement of a DVB-T broadcast | diff --git a/docs/reports/issue-157-seek-alignment/img/01-original-frame-41.png b/docs/reports/issue-157-seek-alignment/img/01-original-frame-41.png new file mode 100644 index 0000000..f61807f Binary files /dev/null and b/docs/reports/issue-157-seek-alignment/img/01-original-frame-41.png differ diff --git a/docs/reports/issue-157-seek-alignment/img/02-processed-before-fix.png b/docs/reports/issue-157-seek-alignment/img/02-processed-before-fix.png new file mode 100644 index 0000000..f1deed4 Binary files /dev/null and b/docs/reports/issue-157-seek-alignment/img/02-processed-before-fix.png differ diff --git a/docs/reports/issue-157-seek-alignment/img/03-processed-after-fix.png b/docs/reports/issue-157-seek-alignment/img/03-processed-after-fix.png new file mode 100644 index 0000000..aceed4f Binary files /dev/null and b/docs/reports/issue-157-seek-alignment/img/03-processed-after-fix.png differ diff --git a/docs/reports/issue-157-seek-alignment/img/04-difference-before-fix.png b/docs/reports/issue-157-seek-alignment/img/04-difference-before-fix.png new file mode 100644 index 0000000..0c67e7e Binary files /dev/null and b/docs/reports/issue-157-seek-alignment/img/04-difference-before-fix.png differ diff --git a/docs/reports/issue-157-seek-alignment/img/05-difference-after-fix.png b/docs/reports/issue-157-seek-alignment/img/05-difference-after-fix.png new file mode 100644 index 0000000..5d1dfec Binary files /dev/null and b/docs/reports/issue-157-seek-alignment/img/05-difference-after-fix.png differ diff --git a/docs/reports/issue-157-seek-alignment/index.html b/docs/reports/issue-157-seek-alignment/index.html new file mode 100644 index 0000000..1905b61 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/index.html @@ -0,0 +1,304 @@ + + + + + +Issue #157: seek alignment + + + +
+

Issue #157 — after a seek, the decoder repeated the first frame

+

2026-11-10 · issue #157 +(reported on the videohelp forum, thread 419911, post #152) · measured on an Apple M-series Mac +(macOS arm64), deps 1.16.0 (FFmpeg 9.0), against the worker of app 1.7.0 and the same worker with the fix

+ +
+Outcome: fixed. With every filter off, the preview's Processed frame lagged the +Original by one frame. Cause: the worker's source decoders write rawvideo, so ffmpeg +times their output at a constant rate, and after a seek, for many time bases, that timing +emitted the first kept frame twice. Every later frame landed one slot +late. The same decoder feeds trimmed encodes and the detectors, so: +
    +
  • Preview: showed frame N−1 (the app's own Original decode showed N).
    • +
  • Trimmed encodes: a trim of 41..52 came out 41, 41, 42 … 51: first +frame doubled, last frame lost, picture one frame late against the audio.
    • +
  • Noise score: every sampled pair was one picture twice, so a source with +sigma ≈ 4.5 noise scored 0.0 and Noise Reduction was never suggested.
    • +
+The fix puts setpts=PTS-STARTPTS at the front of a seeking decoder's one +-vf (worker/src/source_decode.rs), and stops the preview passing a +pointless -ss 0, so a window at frame 0 decodes exactly like an untrimmed encode. +Every affected source now decodes the right frames from every start +tried; untrimmed encodes are bit-identical to before. +
+ +

1. The mechanism

+

The ffmpeg CLI picks an output timing mode from the muxer's flags. rawvideo declares +AVFMT_NOTIMESTAMPS but not AVFMT_VARIABLE_FPS, so it gets +constant-rate timing: frames are placed on a grid at the output rate, and a frame +is repeated or dropped to keep the grid filled. An image muxer (the app's JPEG Original frame) +declares both, so it gets passthrough timing. That is why the Original side was right and the +Processed side was not.

+

Both seeking decoders seek to (first − 0.5) / fps, half a frame before the frame +wanted, so ffmpeg's accurate seek keeps exactly that frame. Its timestamp then sits half a frame +after the output's zero. For some time bases, the constant-rate grid fills that gap by emitting +the frame twice. setpts=PTS-STARTPTS rebases the first kept frame to 0, so there is no +gap to fill.

+

-fps_mode passthrough would also remove the gap, but it changes what a +soft-telecine source decodes to (§5). It is the right tool only for the VFR decoder +(timeline_decoder_args, issue #137), which places stored frames in the script itself +and is untouched here.

+ +

2. Which sources were affected

+

8 frames decoded from source frame 40, seeking as the worker does. Each picture is named by +its index in an independent sequential decode of the same file (MD5 of the luma +plane; -fps_mode passthrough, every stored frame once, no seek). The last two columns +count, over 7–8 start frames (1, 10, 40–43, 77, near the end), how often the 8 frames were exactly +start … start+7. Script: dup_probe.py.

+
+ + + + + + + + + + + + + + + + + + + + +
SourceTime baseBefore (default timing)After (setpts)Right, beforeRight, after
Affected: fixed
small_clip.mp4 (H.264, 29.97)1/299740, 40, 41 … 4640, 41 … 470/88/8
seek_timebase_2997.mov (new fixture, H.264, 29.97)1/1198840, 40, 41 … 4640, 41 … 470/88/8
ProRes 422 Proxy MOV, 29.97 (variant)1/1198840, 40, 41 … 4640, 41 … 470/88/8
H.264 MKV, 25 (variant)1/100040, 40, 41 … 4640, 41 … 470/88/8
Noisy ProRes HQ MOV / noisy H.264 MP4 (variants)1/11988, 1/299740, 40, 41 … 4640, 41 … 470/88/8
Unaffected: unchanged
colorize_test.mp4 (24)1/1228840 … 4740 … 478/88/8
face_restore_test.mp4 (29.97)1/3000040 … 4740 … 478/88/8
prores422_10bit_telecine.mov (30 frames)1/30000rightright2/22/2
pal-sd-25.mov1/25static colour bars: every frame identical, so this fixture can't show a shift either way
Seek lands on the wrong picture for other reasons (§6): the duplicate run is gone, the position is not fixed
pal-dvbt-fieldcoded-25i.ts1/9000028 ×828, 29 … 350/80/8
MPEG-2 PAL DVD program stream, open GOP (variant)1/9000048 ×848, 49 … 550/81/8
H.264 MPEG-TS, 25 (variant)1/9000050 ×850, 51 … 570/80/8
hard_telecine_test.avi (MPEG-2 in AVI)1001/3000038, 39 … 4538, 39 … 450/80/8
interlaced_test.avi (Lagarith, 75 stored frames)1/2536 … 4336 … 430/71/7
soft_telecine_test.mkv (MPEG-2, 23.976)1/100040, 40, 40, 42, 42, 44, 45, 4540, 40, 40, 41, 42 … 450/80/8
+
+

The "variant" rows are built by make_variants.py from ffmpeg's +testsrc2 (every frame differs) and are not committed. The committed +seek_timebase_2997.mov is made by +Tests/TestResources/generate_seek_timebase_test.sh; its 1/11988 timescale is the one +from a user's ProRes clip in the same forum thread. Limit: MD5 identity says +which stored picture came out, which is exact for this question, but only for decodes of the +same file; it says nothing about colour.

+

Not fully confirmed: the plan asked for a TS "without an audio-first gap". ffmpeg's +MPEG-TS muxer adds its own 1.4 s mux delay, so the variant's video starts at 1.48 s anyway, and like +the PS variant its seek lands on a later keyframe. Neither TS file isolates the duplicate from the +position bug, and both are recorded as follow-up work (§6).

+ +

3. What the user saw, and what they see now

+

Script: worker_probe.py runs the worker itself, with every pass off. The preview of +frame N, and the app's Original (the exact arguments of PreviewGenerator.beforeFrameArgs, +to PNG), are each compared with the sequential decode's frames N−2 … N+2 by mean absolute RGB +difference; the closest is reported. A trimmed FFV1 encode of 41..52 is named frame by frame by MD5.

+ + + + + + +
SourcePreview of 10 / 41 / 60, beforeafterOriginalTrim 41..52, beforeafter
small_clip.mp4N−1, N−1, N−1N, N, NN (both)41, 41, 42 … 5141, 42 … 52
seek_timebase_2997.movN−1, N−1, N−1N, N, NN (both)41, 41, 42 … 5141, 42 … 52
colorize_test.mp4unchanged; nearly static black-and-white, so the nearest-frame test is not decisive (neighbours within 2 levels of each other)same as preview41 … 5241 … 52
interlaced_test.aviunchanged; against the sequential decode it reads N−2 at 41 and 60 because of its 4 leading null frames (§6)same as preview37 … 48 both (same reason)
+

Preview against Original, before the fix: mean difference 2.7–11.5 on small_clip.mp4, +4.0–5.5 on the MOV. After: 0.00 on both.

+ +
+small_clip.mp4 frame 41, the Original +
1. The Original side for small_clip.mp4 +frame 41: the app's own decode, correct before and after. Every image here is 2× nearest-neighbour.
+
+
+Processed preview of frame 41 before the fix +
2. The Processed preview of frame 41 before the fix, every pass +off. It is frame 40: compare the man's chin and collar with image 1.
+
+
+Difference between preview and Original before the fix, amplified 8x +
3. |image 2 − image 1|, amplified 8×. Every moving edge lights up: +with nothing processed, this is the whole of what the "one frame behind" report saw.
+
+
+Processed preview of frame 41 after the fix +
4. The Processed preview of frame 41 after the fix.
+
+
+Difference after the fix: black +
5. |image 4 − image 1|, amplified 8×: black, i.e. the same picture.
+
+ +

4. What must not change

+

Untrimmed encodes

+

An untrimmed encode does not seek, so its decoder arguments are unchanged by construction (unit +test decoders_that_do_not_seek_are_unchanged). Measured anyway: every decoded frame of an +FFV1 encode by the worker before and after (framemd5) is +identical for small_clip.mp4 (3151 frames), +seek_timebase_2997.mov (90), interlaced_test.avi with no passes (75) and with +QTGMC (150), and soft_telecine_test.mkv (120). Script: identity_probe.py.

+

The preview of frame 0

+

The preview used to pass -ss 0.000000 for a window starting at frame 0. That +is a seek: ffmpeg re-bases timestamps on the seek point and trims anything before it. +The fix passes no -ss there, like an untrimmed encode. Measured on all 13 sources +(frame0_probe.py): preview frame 0 is unchanged on 11 and changed on two, +hard_telecine_test.avi and the DVB-T .ts. On both it now +equals frame 0 of the untrimmed encode (mean difference 0.00; before 9.24 and +2.13). With setpts alone, preview frame 0 on late-starting video would have depended on the +seek; that is why the decision is "no -ss, no reset" rather than "always reset".

+

Telecine

+

telecine_probe.py, trimmed encodes of frames 10..69:

+
    +
  • IVTC on hard_telecine_test.avi: 48 frames before and after, +bit-identical (this source has no duplicate to remove).
    • +
  • No passes on soft_telecine_test.mkv: 60 frames before and after. Before, +15 of the 60 stored film frames were missing from the output (11, 13, 16, 18, +23, …), replaced by repeats of their neighbours. After, none are missing. +The first frame still comes out three times, exactly as it does at the start of an untrimmed decode +(the untrimmed decode of this file is 122 slots for 120 stored frames).
    • +
  • Soft Telecine method, same file: 48 frames before and after; the pictures differ. +That method drops 1 frame in 5 even untrimmed when the job's rate is 23.976 (25 of 120 missing). That +is its own behaviour, recorded in §6, not something this fix changes.
    • +
+

So setpts keeps what a constant-rate decode does with pulldown. The extra repeat runs and +drops in the default column came from the seek gap, not the source.

+ +

5. The detectors

+

--detect-crop, --detect-noise-halo and --detect-native decode +through preview_decoder_args, so they get the fix. detector_probe.py runs the +first two with the worker before and after, and counts how many of the noise detector's 24 sampled +pairs the old decoder delivered as the same picture twice:

+ + + + + + + +
SourcePairs that were one picture twiceNoise before → afterSuggested
Noisy H.264 MP4, 1/2997 (temporal noise, ffmpeg noise=alls=10)24/240.0 → 4.62no → yes
Noisy ProRes HQ MOV, 1/11988 (same noise)24/240.0 → 4.18no → yes
small_clip.mp4, seek_timebase_2997.mov, clean variants24/240.0 → 0.0no → no
pal-dvbt-fieldcoded-25i.ts15/240.0 → 0.0no → no
interlaced_test.avi, colorize_test.mp4, face_restore_test.mp40/24unchangedunchanged
+

Halo scores moved by under 0.7 (none near the 6.0 threshold), and the crop report (crop, crop with +bars, confidence) was identical on every source. On the affected clean sources the noise score stays +0.0 because they are compressed clean video, whose skipped blocks read noiseless anyway (as the noise +report already says). Limit: the noisy sources are synthetic; the point is only that +a measurable noise was read as none.

+

The noise report's calibration is not affected. Its fixtures +(2026-11-noise-halo-scores) are intra-frame FFV1 +MKVs, which decode correctly after a seek (checked: an FFV1 MKV at 25 fps matched from every start +frame). Its reference measurement reads from frame 0 without seeking. Only its informational full-clip +worker runs on small_clip.mp4 and the DVB-T .ts read duplicated pairs, and both +score 0.0 before and after. Its thresholds stand and its scripts are unchanged.

+ +

6. Not fixed here: the seek lands on the wrong picture

+

Measuring against an independent decode turned up sources where the seek itself, not the timing, +lands somewhere else. Every row is the same with -fps_mode passthrough, so it is not this +bug. They are separate follow-ups:

+
    +
  • MPEG-PS and MPEG-TS (DVD program stream, H.264 TS, the DVB-T capture): the decode +starts at a later keyframe (start 40 → 48 with a 12-frame GOP, → 50 with a 25-frame GOP). The +earlier hypothesis for the DVB-T file, "video starts 1.32 s after its audio", may be part of it. This is +what DVD/VOB users get, so it is the most important of the three.
    • +
  • MPEG-2 in AVI (hard_telecine_test.avi): lands 1–2 frames early (start 10 +→ 8, 40 → 38), and start 1 → 45.
    • +
  • Leading repeats in an untrimmed decode: interlaced_test.avi decodes to 79 +constant-rate slots for 75 stored frames (frame 0 four times, frame 1 twice: AVI null frames), and +soft_telecine_test.mkv to 122 slots for 120. The app's frame numbers and the encode's then +disagree by those slots, and the Soft Telecine method's 1-in-5 drop depends on which rate the job +carries.
    • +
  • Correction (review): one case this fix does change. A seek into a run of +leading null frames (interlaced_test.avi, stored pts 0 then 5, in-points 1–4) used to be +right from the first real frame on, because constant-rate timing filled the gap from the seek point; the +reset rebases to the first stored frame after the gap, so the whole trim now runs up to two frames early +(an in-point of 3: 4, 6, 7 … instead of 4, 4, 4, 6, 7 …; the first slot is wrong either way). This broke +integration_frame_mapping_test.dart's "from an in-point" (in-point 3), which now starts at 6, +past the gap. A seek into a mid-stream null-frame run (a dropped-frame capture) is unchanged: +measured on an FFV1 AVI with frames 100–104 dropped, the old decoder and the reset give the same frames from +every start 97–106 (both wrong for an in-point inside the run, 100–104). Leading null frames are task 16's. +scripts/gap_probe.py.
    • +
+

integration_seek_alignment_test.dart covers only sources whose sequential numbering +is the app's numbering, so it does not claim these.

+ +

7. Decisions

+
    +
  • setpts=PTS-STARTPTS, first in the one -vf: it rebases to the +first frame the accurate seek kept (the seek's own trim runs before the user filter chain), and +ffmpeg honours only the last -vf, so it can't be a second one. Measured against the alternative of subtracting the seek's fixed half +frame (setpts=PTS-0.5/fps/TB, which would keep a gap after the seek point): it fixes the same +49 doubled starts, but in a 1/25 time base the half frame ties and rounds a frame early, and it broke seeks +just before a mid-stream null-frame run that the reset handles like the old decoder.
    • +
  • Not -fps_mode passthrough: same result on ordinary sources, but on +soft telecine it hands over stored frames (40 … 47) instead of the constant-rate slots an untrimmed +encode gets. That would make trimmed and untrimmed encodes of one source disagree.
    • +
  • Only when the decoder seeks: untrimmed encodes stay byte-identical, and the preview +of frame 0 decodes like the encode of frame 0 (§4).
    • +
  • Tested against an independent decode: preview-vs-encode tests could never see this, +because both paths shifted together. The heavy test compares the preview, the app's Original and a +trimmed encode each with select=eq(n,N) of the source. It fails 8 of 12 cases against the +worker before the fix.
    • +
+ +

8. What wasn't tested

+
    +
  • Windows and Linux. The duplicate comes from the ffmpeg CLI's timing logic, not from a decoder or +the CPU, and every platform runs FFmpeg 9.0, so the same behaviour is expected. The heavy test runs +nightly on all four.
    • +
  • VFR sources: their decoder (timeline_decoder_args) already uses passthrough timing and +is unchanged.
    • +
  • Descale's --detect-native wasn't re-run: it decodes single frames, so the fix only +changes which frame it measures.
    • +
+ +

9. Reproduce

+
cd worker && cargo build                      # the "after" worker
+# build a "before" worker from the commit before the fix, next to a deps/ directory
+BEFORE_WORKER=/path/to/before/vapourbox-worker \
+  docs/reports/issue-157-seek-alignment/scripts/run.sh
+

scripts/: seeklib.py (shared decode helpers), make_variants.py, +dup_probe.py (§2), worker_probe.py (§3), images.py (images 1–5), +identity_probe.py, frame0_probe.py, telecine_probe.py (§4), +detector_probe.py (§5), run.sh (all of them). Tests: +app/test/integration_seek_alignment_test.dart (heavy) and the unit tests in +worker/src/source_decode.rs.

+
+ + diff --git a/docs/reports/issue-157-seek-alignment/scripts/detector_probe.py b/docs/reports/issue-157-seek-alignment/scripts/detector_probe.py new file mode 100644 index 0000000..ed59013 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/detector_probe.py @@ -0,0 +1,49 @@ +"""The detectors decode with the preview's decoder, so the fix reaches them. +Runs `--detect-noise-halo` and `--detect-crop` with the worker before and after +the fix and prints what changed. For the noise score, also counts how many of +the sampled pairs the old decoder delivered as the same picture twice (the +motion-compensated difference of such a pair is zero: no noise). + + python3 detector_probe.py --before WORKER --after WORKER [source ...]""" +import json +import os +import subprocess +import sys +from seeklib import FF, RES, WORK, probe, sequential, seek_decode +import make_variants +from worker_probe import job_for + +FIXTURES = ['small_clip.mp4', 'seek_timebase_2997.mov', 'face_restore_test.mp4', + 'colorize_test.mp4', 'interlaced_test.avi', 'pal-dvbt-fieldcoded-25i.ts'] + + +def detect(worker, flag, src, s, total): + r = subprocess.run([worker, flag, '--config', + job_for(src, s, os.path.join(WORK, 'x.mkv'), totalFrames=total)], + capture_output=True, text=True) + return json.loads(r.stdout.strip().splitlines()[-1]) + + +def main(): + a = sys.argv[1:] + before, after = a[a.index('--before') + 1], a[a.index('--after') + 1] + rest = [x for x in a if x not in ('--before', '--after', before, after)] + paths = rest or [os.path.join(RES, f) for f in FIXTURES] + make_variants.build() + for src in paths: + s = probe(src) + total = len(sequential(src, 100000)) + nb, na = (detect(w, '--detect-noise-halo', src, s, total) for w in (before, after)) + same = sum(1 for f in nb.get('frames', []) + if len(set(seek_decode(src, f, 2, 'default'))) == 1) + cb, ca = (detect(w, '--detect-crop', src, s, total) for w in (before, after)) + keys = ('crop', 'cropWithBars', 'confidence') + print(f"{os.path.basename(src):28s} noise {nb.get('noise')} -> {na.get('noise')} " + f"(high {nb.get('noiseHigh')} -> {na.get('noiseHigh')}; {same}/{len(nb.get('frames', []))} " + f"old pairs were one picture twice) halo {nb.get('halo')} -> {na.get('halo')} " + f"crop {'unchanged' if all(cb.get(k) == ca.get(k) for k in keys) else 'CHANGED'}" + + ('' if all(cb.get(k) == ca.get(k) for k in keys) else + f" {[cb.get(k) for k in keys]} -> {[ca.get(k) for k in keys]}")) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/issue-157-seek-alignment/scripts/dup_probe.py b/docs/reports/issue-157-seek-alignment/scripts/dup_probe.py new file mode 100644 index 0000000..a49eb2c --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/dup_probe.py @@ -0,0 +1,74 @@ +"""Decode 8 frames from several start frames, the way the worker's decoders +seek, under three output timings, and name each picture by its index in an +independent sequential decode (MD5 of the luma plane). + + python3 dup_probe.py [--json out.json] [source ...] + +With no sources: every committed fixture plus make_variants.py's variants. +Reference: the sequential `-fps_mode passthrough` decode (every stored frame +once). `slots` adds the comparison against the untrimmed constant-rate decode, +which is what an untrimmed encode feeds the script (the right reference for +soft telecine, whose pulldown repeats are slots of their own).""" +import json +import os +import sys +from seeklib import RES, probe, sequential, seek_decode, indices +import make_variants + +FIXTURES = ['small_clip.mp4', 'seek_timebase_2997.mov', 'interlaced_test.avi', + 'colorize_test.mp4', 'face_restore_test.mp4', 'pal-sd-25.mov', + 'prores422_10bit_telecine.mov', 'hard_telecine_test.avi', + 'soft_telecine_test.mkv', 'pal-dvbt-fieldcoded-25i.ts'] +COUNT = 8 + + +def starts_for(n): + s = [1, 10, 40, 41, 42, 43, 77] + return [x for x in s if x + COUNT < n] + ([n - COUNT - 2] if n > 30 else []) + + +def run(path): + s = probe(path) + seq = sequential(path, 100000) + slots = sequential(path, 100000, timing='default') + rows = [] + for start in starts_for(len(seq)): + row = {'start': start} + for mode in ('default', 'passthrough', 'setpts'): + got = seek_decode(path, start, COUNT, mode) + row[mode] = indices(got, seq) + row[mode + '_slots'] = indices(got, slots) + row[mode + '_ok'] = got == seq[start:start + COUNT] + row[mode + '_slots_ok'] = got == slots[start:start + COUNT] + rows.append(row) + return {'source': os.path.basename(path), 'time_base': s['time_base'], + 'codec': s['codec_name'], 'fps': round(s['fps'], 3), + 'start_time': s.get('start_time'), 'frames': len(seq), + 'slots': len(slots), 'rows': rows} + + +def main(): + args = sys.argv[1:] + out = None + if args[:1] == ['--json']: + out, args = args[1], args[2:] + paths = args or [os.path.join(RES, f) for f in FIXTURES] + make_variants.build() + results = [] + for p in paths: + r = run(p) + results.append(r) + print(f"== {r['source']} tb {r['time_base']} {r['codec']} {r['fps']} fps " + f"start_time {r['start_time']} {r['frames']} frames ({r['slots']} slots)") + for row in r['rows']: + for mode in ('default', 'passthrough', 'setpts'): + flag = 'ok ' if row[mode + '_ok'] else 'BAD' + sflag = 'ok ' if row[mode + '_slots_ok'] else 'BAD' + print(f" start {row['start']:4d} {mode:11s} vs seq {flag} {row[mode]}" + f" vs slots {sflag} {row[mode + '_slots']}") + if out: + with open(out, 'w') as f: + json.dump(results, f, indent=1) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/issue-157-seek-alignment/scripts/frame0_probe.py b/docs/reports/issue-157-seek-alignment/scripts/frame0_probe.py new file mode 100644 index 0000000..8ee0c7a --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/frame0_probe.py @@ -0,0 +1,50 @@ +"""Preview of frame 0 before and after the fix, compared with each other and +with frame 0 of an untrimmed lossless encode (the fix stops passing `-ss 0` +when the window starts at frame 0, matching the untrimmed encode, which never +seeks). + + python3 frame0_probe.py --before WORKER --after WORKER [source ...]""" +import os +import subprocess +import sys +from seeklib import FF, RES, WORK, probe +import make_variants +from worker_probe import job_for, mad + +FIXTURES = ['small_clip.mp4', 'seek_timebase_2997.mov', 'interlaced_test.avi', 'colorize_test.mp4', + 'face_restore_test.mp4', 'prores422_10bit_telecine.mov', 'hard_telecine_test.avi', + 'soft_telecine_test.mkv', 'pal-dvbt-fieldcoded-25i.ts'] + + +def preview_rgb(worker, src, s): + r = subprocess.run([worker, '--config', job_for(src, s, os.path.join(WORK, 'x.mkv')), + '--preview', '--frame', '0'], capture_output=True, check=True) + return rgb(['-f', 'png_pipe', '-i', '-'], r.stdout) + + +def rgb(args, stdin=None): + return subprocess.run([FF, '-v', 'error', *args, '-frames:v', '1', '-f', 'rawvideo', + '-pix_fmt', 'rgb24', '-'], input=stdin, capture_output=True, + check=True).stdout + + +def main(): + a = sys.argv[1:] + before, after = a[a.index('--before') + 1], a[a.index('--after') + 1] + rest = [x for x in a if x not in ('--before', '--after', before, after)] + paths = rest or [os.path.join(RES, f) for f in FIXTURES] + make_variants.build() + for src in paths: + s = probe(src) + enc = os.path.join(WORK, 'full.mkv') + if os.path.exists(enc): + os.remove(enc) + subprocess.run([after, '--config', job_for(src, s, enc, endFrame=4, totalFrames=5)], + capture_output=True, check=True) + e0 = rgb(['-i', enc]) + b, f = preview_rgb(before, src, s), preview_rgb(after, src, s) + print(f"{os.path.basename(src):30s} preview of frame 0 unchanged: {b == f} " + f"vs encode frame 0, mean abs diff: before {mad(b, e0):.2f}, after {mad(f, e0):.2f}") + + +if __name__ == '__main__': + main() diff --git a/docs/reports/issue-157-seek-alignment/scripts/gap_probe.py b/docs/reports/issue-157-seek-alignment/scripts/gap_probe.py new file mode 100644 index 0000000..6541002 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/gap_probe.py @@ -0,0 +1,58 @@ +"""Review correction (§6, §7): how the PTS reset, the old decoder and a fixed +half-frame shift handle null frames, measured against the untrimmed +constant-rate decode (the app's frame numbering). + + python3 gap_probe.py + +Two sources: interlaced_test.avi (leading null frames: stored pts 0, then 5) +and an FFV1 AVI built here from testsrc2 with stored frames 100-104 dropped +(the avi muxer writes them as null frames, like a dropped-frame capture). +Prints each start's first 8 pictures as slot indices, and whether all 14 +decoded frames match the untrimmed slots ('ALL'), only the last 6 ('tail'), +or neither.""" +import os +import subprocess +from seeklib import FF, RES, WORK, probe, sequential, indices, _raw, _hashes + +MODES = { + 'old': lambda s: None, + 'reset': lambda s: 'setpts=PTS-STARTPTS', + 'half': lambda s: f'setpts=PTS-{0.5 / s["fps"]:.9f}/TB', +} +COUNT = 14 + + +def decode(path, start, vf): + s = probe(path) + ss = max(0.0, (start - 0.5) / s['fps']) + v = vf(s) + return _hashes(_raw(['-ss', f'{ss:.6f}', '-apply_cropping', 'codec', '-i', path, + '-map', '0:v:0', '-frames:v', str(COUNT)] + (['-vf', v] if v else [])), + s['width'], s['height']) + + +def mid_gap_avi(): + out = os.path.join(WORK, 'mid_gap.avi') + subprocess.run([FF, '-v', 'error', '-y', '-f', 'lavfi', + '-i', 'testsrc2=size=320x240:rate=25:duration=12', + '-vf', "select='not(between(n,100,104))'", '-fps_mode', 'passthrough', + '-c:v', 'ffv1', out], check=True) + return out + + +def run(path, starts): + slots = sequential(path, 100000, timing='default') + print(f'== {os.path.basename(path)} {len(slots)} slots') + for st in starts: + want = indices(slots[st:st + COUNT], slots) + cells = [] + for name, vf in MODES.items(): + got = indices(decode(path, st, vf), slots) + verdict = 'ALL ' if got == want else ('tail' if got[-6:] == want[-6:] else 'BAD ') + cells.append(f'{name}:{verdict} {got[:8]}') + print(f' {st:4d} want {want[:8]}\n ' + '\n '.join(cells)) + + +if __name__ == '__main__': + run(os.path.join(RES, 'interlaced_test.avi'), range(1, 8)) + run(mid_gap_avi(), [50, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 150]) diff --git a/docs/reports/issue-157-seek-alignment/scripts/identity_probe.py b/docs/reports/issue-157-seek-alignment/scripts/identity_probe.py new file mode 100644 index 0000000..2776f1f --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/identity_probe.py @@ -0,0 +1,49 @@ +"""Untrimmed encodes must not change: the fix only touches decoders that seek. +Encodes each source untrimmed with the worker before and after the fix, FFV1, +and compares every decoded frame (ffmpeg's framemd5 of the video stream). + + python3 identity_probe.py --before WORKER --after WORKER""" +import os +import subprocess +import sys +from seeklib import FF, RES, WORK, probe +from worker_probe import job_for + +CASES = [('small_clip.mp4', False), ('seek_timebase_2997.mov', False), + ('interlaced_test.avi', False), ('interlaced_test.avi', True), + ('soft_telecine_test.mkv', False)] + + +def framemd5(path): + out = subprocess.run([FF, '-v', 'error', '-i', path, '-map', '0:v:0', '-f', 'framemd5', '-'], + capture_output=True, text=True, check=True).stdout + return [l.split(',')[-1].strip() for l in out.splitlines() if l and not l.startswith('#')] + + +def encode(worker, src, qtgmc, tag): + s = probe(src) + out = os.path.join(WORK, f'ident_{tag}.mkv') + if os.path.exists(out): + os.remove(out) + extra = {} + if qtgmc: + q = {'enabled': True, 'preset': 'Super Fast', 'tff': True} + extra = {'qtgmcParameters': q, 'processingPipeline': {'deinterlace': q}} + r = subprocess.run([worker, '--config', job_for(src, s, out, **extra)], capture_output=True) + if r.returncode: + raise SystemExit(r.stderr.decode(errors='replace')[-800:]) + return framemd5(out) + + +def main(): + a = sys.argv[1:] + before, after = a[a.index('--before') + 1], a[a.index('--after') + 1] + for name, qtgmc in CASES: + src = os.path.join(RES, name) + b, f = encode(before, src, qtgmc, 'b'), encode(after, src, qtgmc, 'a') + print(f"{name:26s} {'QTGMC' if qtgmc else 'no passes':9s} {len(b)} frames " + f"identical: {b == f}") + + +if __name__ == '__main__': + main() diff --git a/docs/reports/issue-157-seek-alignment/scripts/images.py b/docs/reports/issue-157-seek-alignment/scripts/images.py new file mode 100644 index 0000000..bc99ed7 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/images.py @@ -0,0 +1,59 @@ +"""Screenshots for the report: small_clip.mp4 frame 41 as the app shows it. + + python3 images.py --before WORKER --after WORKER + +Writes ../img/: the Original (the app's before frame), the Processed preview +from the worker before and after the fix (every pass off), and each preview's +difference from the Original, amplified 8x (black = identical).""" +import os +import subprocess +import sys +from seeklib import FF, RES, WORK, probe +from worker_probe import job_for, before_frame_args + +IMG = os.path.join(os.path.dirname(os.path.abspath(__file__)), '..', 'img') +SRC = os.path.join(RES, 'small_clip.mp4') +FRAME = 41 + + +def preview(worker, out): + s = probe(SRC) + r = subprocess.run([worker, '--config', job_for(SRC, s, os.path.join(WORK, 'x.mkv')), + '--preview', '--frame', str(FRAME)], capture_output=True, check=True) + with open(out, 'wb') as f: + f.write(r.stdout) + + +def scaled(src, out): + """2x nearest-neighbour, so the pixels stay real.""" + subprocess.run([FF, '-y', '-v', 'error', '-i', src, '-vf', 'scale=iw*2:ih*2:flags=neighbor', + '-frames:v', '1', out], check=True) + + +def diff(a, b, out): + subprocess.run([FF, '-y', '-v', 'error', '-i', a, '-i', b, '-filter_complex', + '[0:v]format=rgb24[x];[1:v]format=rgb24[y];[x][y]blend=all_mode=difference,' + 'lutrgb=r=val*8:g=val*8:b=val*8,scale=iw*2:ih*2:flags=neighbor', + '-frames:v', '1', out], check=True) + + +def main(): + a = sys.argv[1:] + before, after = a[a.index('--before') + 1], a[a.index('--after') + 1] + os.makedirs(IMG, exist_ok=True) + s = probe(SRC) + orig = os.path.join(WORK, 'orig.png') + subprocess.run([FF, '-v', 'error', *before_frame_args(SRC, FRAME, s['fps'], orig)], check=True) + old, new = os.path.join(WORK, 'pv_old.png'), os.path.join(WORK, 'pv_new.png') + preview(before, old) + preview(after, new) + scaled(orig, os.path.join(IMG, '01-original-frame-41.png')) + scaled(old, os.path.join(IMG, '02-processed-before-fix.png')) + scaled(new, os.path.join(IMG, '03-processed-after-fix.png')) + diff(old, orig, os.path.join(IMG, '04-difference-before-fix.png')) + diff(new, orig, os.path.join(IMG, '05-difference-after-fix.png')) + print('wrote', sorted(os.listdir(IMG))) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/issue-157-seek-alignment/scripts/make_variants.py b/docs/reports/issue-157-seek-alignment/scripts/make_variants.py new file mode 100644 index 0000000..02e1e36 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/make_variants.py @@ -0,0 +1,58 @@ +"""Build the source variants the probe runs on top of the committed fixtures, +into $SEEK_WORK/variants. Each is testsrc2 (every frame differs), so a picture +identifies its frame unambiguously. Not committed: rebuild with this script.""" +import os +import subprocess +from seeklib import FF, WORK + +OUT = os.path.join(WORK, 'variants') +os.makedirs(OUT, exist_ok=True) + +SRC = ['-f', 'lavfi', '-i'] +AUDIO = ['-f', 'lavfi', '-i', 'sine=frequency=440:sample_rate=48000:duration=6'] + +VARIANTS = { + # Matroska, time base 1/1000, 25 fps H.264 with B-frames. + 'mkv_25.mkv': [*SRC, 'testsrc2=size=320x240:rate=25:duration=6', *AUDIO, + '-map', '0:v', '-map', '1:a', '-c:v', 'libx264', '-preset', 'veryfast', + '-g', '25', '-bf', '2', '-pix_fmt', 'yuv420p', '-c:a', 'aac'], + # A PAL DVD-style MPEG-2 program stream: open GOPs, B-frames, MP2 audio. + 'dvd_pal.mpg': [*SRC, 'testsrc2=size=720x576:rate=25:duration=6', *AUDIO, + '-map', '0:v', '-map', '1:a', '-c:v', 'mpeg2video', '-b:v', '6M', + '-g', '12', '-bf', '2', '-flags', '-cgop', '-pix_fmt', 'yuv420p', + '-c:a', 'mp2', '-f', 'vob'], + # MPEG-TS, time base 1/90000, video and audio starting together. + 'ts_25_video_first.ts': [*SRC, 'testsrc2=size=720x576:rate=25:duration=6', *AUDIO, + '-map', '0:v', '-map', '1:a', '-c:v', 'libx264', + '-preset', 'veryfast', '-g', '25', '-bf', '2', + '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-f', 'mpegts'], + # 29.97 ProRes in MOV with a 1/11988 video time base, like the user's clip. + 'prores_11988.mov': [*SRC, 'testsrc2=size=720x480:rate=30000/1001:duration=6', *AUDIO, + '-map', '0:v', '-map', '1:a', '-c:v', 'prores_ks', '-profile:v', '0', + '-c:a', 'pcm_s16le', '-video_track_timescale', '11988'], + # Noisy sources for the noise score (temporal noise, sigma ~6 levels), in + # two affected time bases. Lightly compressed so the noise survives. + 'noisy_prores_11988.mov': [*SRC, 'testsrc2=size=720x480:rate=30000/1001:duration=6,' + 'noise=alls=10:allf=t', *AUDIO, '-map', '0:v', '-map', '1:a', + '-c:v', 'prores_ks', '-profile:v', '3', '-c:a', 'pcm_s16le', + '-video_track_timescale', '11988'], + 'noisy_h264_2997.mp4': [*SRC, 'testsrc2=size=720x480:rate=30000/1001:duration=6,' + 'noise=alls=10:allf=t', *AUDIO, '-map', '0:v', '-map', '1:a', + '-c:v', 'libx264', '-preset', 'veryfast', '-crf', '8', + '-pix_fmt', 'yuv420p', '-c:a', 'aac', '-video_track_timescale', '2997'], +} + + +def build(): + paths = [] + for name, args in VARIANTS.items(): + path = os.path.join(OUT, name) + if not os.path.exists(path): + subprocess.run([FF, '-y', '-v', 'error', *args, path], check=True) + paths.append(path) + return paths + + +if __name__ == '__main__': + for p in build(): + print(p) diff --git a/docs/reports/issue-157-seek-alignment/scripts/run.sh b/docs/reports/issue-157-seek-alignment/scripts/run.sh new file mode 100755 index 0000000..7207059 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/run.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# Reproduce every number and image in the report. +# +# BEFORE_WORKER= ./run.sh +# +# The "after" worker is this checkout's debug build (cd worker && cargo build). +# Build the "before" one from the parent of the fix commit, in a checkout that +# also has deps/ (debug workers find deps/ by searching upward from themselves). +# Scratch files go to $SEEK_WORK (default: a temporary directory); results are +# written there as *.txt / *.json. Needs deps/ (or $VAPOURBOX_DEPS_DIR). +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +AFTER="${AFTER_WORKER:-$REPO/worker/target/debug/vapourbox-worker}" +BEFORE="${BEFORE_WORKER:?set BEFORE_WORKER to a worker built without the fix}" +export SEEK_WORK="${SEEK_WORK:-$(mktemp -d)}" +cd "$HERE" +python3 make_variants.py +python3 dup_probe.py --json "$SEEK_WORK/dup.json" | tee "$SEEK_WORK/dup.txt" +for v in before after; do + w=$BEFORE; [ $v = after ] && w=$AFTER + python3 worker_probe.py --worker "$w" --json "$SEEK_WORK/worker_$v.json" | tee "$SEEK_WORK/worker_$v.txt" +done +python3 frame0_probe.py --before "$BEFORE" --after "$AFTER" | tee "$SEEK_WORK/frame0.txt" +python3 identity_probe.py --before "$BEFORE" --after "$AFTER" | tee "$SEEK_WORK/identity.txt" +python3 telecine_probe.py --before "$BEFORE" --after "$AFTER" | tee "$SEEK_WORK/telecine.txt" +python3 detector_probe.py --before "$BEFORE" --after "$AFTER" | tee "$SEEK_WORK/detectors.txt" +python3 gap_probe.py | tee "$SEEK_WORK/gap.txt" +python3 images.py --before "$BEFORE" --after "$AFTER" +echo "Results in $SEEK_WORK" diff --git a/docs/reports/issue-157-seek-alignment/scripts/seeklib.py b/docs/reports/issue-157-seek-alignment/scripts/seeklib.py new file mode 100644 index 0000000..9bfcd38 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/seeklib.py @@ -0,0 +1,108 @@ +"""Shared helpers for the issue #157 seek-alignment probes. + +Paths come from the repo (found from this file's location) or the environment, +never from a personal path: + + VAPOURBOX_DEPS_DIR deps/ to take ffmpeg/ffprobe from + (default: /deps/) + SEEK_WORK scratch directory for variants, raw frames and encodes + (default: a fresh temporary directory) +""" +import hashlib +import json +import os +import platform +import subprocess +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..')) +RES = os.path.join(REPO, 'Tests', 'TestResources') + + +def _platform_id(): + system = platform.system() + machine = platform.machine().lower() + arch = 'arm64' if machine in ('arm64', 'aarch64') else 'x64' + if system == 'Darwin': + return f'macos-{arch}' + if system == 'Windows': + return 'windows-x64' + return f'linux-{arch}' + + +DEPS = os.environ.get('VAPOURBOX_DEPS_DIR') or os.path.join(REPO, 'deps', _platform_id()) +EXE = '.exe' if platform.system() == 'Windows' else '' +FF = os.path.join(DEPS, 'ffmpeg', 'ffmpeg' + EXE) +FP = os.path.join(DEPS, 'ffmpeg', 'ffprobe' + EXE) +WORK = os.environ.get('SEEK_WORK') or tempfile.mkdtemp(prefix='seek157_') +os.makedirs(WORK, exist_ok=True) + +# The PTS reset the worker's seeking decoders now carry (source_decode.rs). +RESET_PTS = 'setpts=PTS-STARTPTS' + + +def _rate(s): + n, d = (int(x) for x in s.split('/')) + return n / d if d else 0.0 + + +def probe(path): + """Width, height, pix_fmt, time base, field order and the frame rate the + app would send (FieldOrderDetector.selectFrameRate: avg when r is ~2x avg, + the field-coded case, else r).""" + out = subprocess.run( + [FP, '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=width,height,pix_fmt,r_frame_rate,avg_frame_rate,time_base,' + 'field_order,start_time,codec_name', '-of', 'json', path], + capture_output=True, text=True, check=True).stdout + s = json.loads(out)['streams'][0] + r, a = _rate(s['r_frame_rate']), _rate(s['avg_frame_rate']) + fps = r if r > 0 else a + if r > 0 and a > 0 and 1.8 < r / a < 2.2: + fps = a + s['fps'] = fps + return s + + +def _raw(args): + return subprocess.run([FF, '-v', 'quiet', *args, '-f', 'rawvideo', '-pix_fmt', 'gray', '-'], + capture_output=True).stdout + + +def _hashes(data, w, h): + n = w * h + return [hashlib.md5(data[i:i + n]).hexdigest() for i in range(0, len(data) - n + 1, n)] + + +def sequential(path, count, timing='passthrough'): + """The first `count` pictures, decoded from the start with no seek. + `passthrough`: every stored frame once (the independent reference). + `default`: rawvideo's own constant-rate timing, which is exactly what an + untrimmed encode feeds the script (pulldown repeats included).""" + s = probe(path) + extra = ['-fps_mode', 'passthrough'] if timing == 'passthrough' else [] + return _hashes(_raw(['-apply_cropping', 'codec', '-i', path, '-map', '0:v:0', + '-frames:v', str(count), *extra]), s['width'], s['height']) + + +def seek_decode(path, start, count, mode): + """`count` frames from source frame `start`, seeking the way the worker + does ((start - 0.5) / fps before -i). `mode`: 'default' (the worker before + #157), 'passthrough', or 'setpts' (the worker after #157).""" + s = probe(path) + ss = max(0.0, (start - 0.5) / s['fps']) + extra = {'default': [], 'passthrough': ['-fps_mode', 'passthrough'], + 'setpts': ['-vf', RESET_PTS]}[mode] + return _hashes(_raw(['-ss', f'{ss:.6f}', '-apply_cropping', 'codec', '-i', path, + '-map', '0:v:0', '-frames:v', str(count), *extra]), + s['width'], s['height']) + + +def indices(got, reference): + """Each decoded picture as its index in the reference decode ('?' if absent). + The first match wins, so a repeated reference picture reads as its first slot.""" + first = {} + for i, h in enumerate(reference): + first.setdefault(h, i) + return [first.get(h, '?') for h in got] diff --git a/docs/reports/issue-157-seek-alignment/scripts/telecine_probe.py b/docs/reports/issue-157-seek-alignment/scripts/telecine_probe.py new file mode 100644 index 0000000..d834607 --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/telecine_probe.py @@ -0,0 +1,46 @@ +"""Telecine paths through a trimmed encode, before and after the fix: the +output frame count (IVTC's decimation and Soft Telecine's relabel depend on +the decoder handing over every slot) and which pictures come out. + + python3 telecine_probe.py --before WORKER --after WORKER""" +import os +import subprocess +import sys +from seeklib import RES, WORK, probe +from identity_probe import framemd5 +from worker_probe import job_for + +CASES = [('hard_telecine_test.avi', 'ivtc', 10, 69), ('soft_telecine_test.mkv', 'softTelecine', 10, 69), + ('soft_telecine_test.mkv', None, 10, 69)] + + +def encode(worker, src, method, start, end, tag): + s = probe(src) + out = os.path.join(WORK, f'tc_{tag}.mkv') + if os.path.exists(out): + os.remove(out) + extra = {'startFrame': start, 'endFrame': end, 'totalFrames': end - start + 1} + if method: + q = {'enabled': True, 'method': method, 'tff': True, 'ivtcCycle': 5} + extra.update({'qtgmcParameters': q, 'processingPipeline': {'deinterlace': q}}) + r = subprocess.run([worker, '--config', job_for(src, s, out, **extra)], capture_output=True) + if r.returncode: + raise SystemExit(r.stderr.decode(errors='replace')[-800:]) + return framemd5(out) + + +def main(): + a = sys.argv[1:] + before, after = a[a.index('--before') + 1], a[a.index('--after') + 1] + for name, method, start, end in CASES: + src = os.path.join(RES, name) + b = encode(before, src, method, start, end, 'b') + f = encode(after, src, method, start, end, 'a') + lead_b = sum(1 for x in b[1:4] if x == b[0]) + lead_a = sum(1 for x in f[1:4] if x == f[0]) + print(f"{name:24s} {method or 'no passes':13s} trim {start}..{end}: frames {len(b)} -> {len(f)}; " + f"identical: {b == f}; first frame repeated {lead_b} -> {lead_a} times") + + +if __name__ == '__main__': + main() diff --git a/docs/reports/issue-157-seek-alignment/scripts/worker_probe.py b/docs/reports/issue-157-seek-alignment/scripts/worker_probe.py new file mode 100644 index 0000000..224183d --- /dev/null +++ b/docs/reports/issue-157-seek-alignment/scripts/worker_probe.py @@ -0,0 +1,141 @@ +"""Run the worker itself, before and after the fix, and check which source +picture each path delivers. + + python3 worker_probe.py --worker PATH [--json out.json] [source ...] + +For each source and frame N, with every pass off: + * preview: `--preview --frame N` (the Processed side) + * before: the app's own before-frame command (PreviewGenerator.beforeFrameArgs, + mirrored here; PNG output instead of JPEG so nothing is lossy) + * each is compared (mean absolute RGB difference) with the sequential decode's + frames N-2..N+2 (`select=eq(n,N)`, image2 = passthrough timing); the + closest offset is reported. 0 = the right frame. +And one trimmed lossless encode (FFV1/MKV, frames START..END): each output +frame is named by its index in the sequential decode (MD5 of the luma plane). + +The worker must be a debug build so it finds deps/ by searching upward.""" +import json +import os +import subprocess +import sys +import uuid +from seeklib import FF, RES, WORK, probe, sequential, indices + +OFFSETS = (-2, -1, 0, 1, 2) + + +def rgb(args, tag, pre=None): + # One -vf only: ffmpeg honours just the last, so a selecting filter goes + # in front of the scale in the same chain. + dst = os.path.join(WORK, f'{tag}.rgb') + chain = ','.join(([pre] if pre else []) + ['scale=320:-2:flags=neighbor', 'format=rgb24']) + subprocess.run([FF, '-y', '-v', 'error', *args, '-vf', chain, + '-frames:v', '1', '-f', 'rawvideo', dst], check=True) + with open(dst, 'rb') as f: + return f.read() + + +def mad(a, b): + n = min(len(a), len(b)) + step = 5 + return sum(abs(a[i] - b[i]) for i in range(0, n, step)) / len(range(0, n, step)) + + +def job_for(src, s, out, **extra): + fo = s.get('field_order') or '' + order = 'tff' if fo in ('tt', 'tb') else 'bff' if fo in ('bb', 'bt') else 'progressive' + j = {'id': str(uuid.uuid4()), 'inputPath': src, 'outputPath': out, + 'qtgmcParameters': {'enabled': False}, + 'processingPipeline': {'deinterlace': {'enabled': False}}, + 'encodingSettings': {'codec': 'ffv1', 'container': 'mkv', 'audioMode': 'none'}, + 'detectedFieldOrder': order, 'inputFrameRate': s['fps'], + 'inputWidth': s['width'], 'inputHeight': s['height'], 'inputPixelFormat': s['pix_fmt']} + j.update(extra) + path = os.path.join(WORK, f'job_{j["id"]}.json') + with open(path, 'w') as f: + json.dump(j, f) + return path + + +def before_frame_args(src, n, fps, out): + """PreviewGenerator.beforeFrameArgs, verbatim but for the output name.""" + ss = max(0.0, (n - 0.5) / fps) + return ['-y', '-ss', f'{ss:.6f}', '-apply_cropping', 'codec', '-i', src, + '-frames:v', '1', '-q:v', '2', out] + + +def check_frame(worker, src, s, n): + pv = os.path.join(WORK, 'pv.png') + r = subprocess.run([worker, '--config', job_for(src, s, os.path.join(WORK, 'x.mkv')), + '--preview', '--frame', str(n)], capture_output=True) + if r.returncode or not r.stdout: + return {'frame': n, 'error': r.stderr.decode(errors='replace')[-400:]} + with open(pv, 'wb') as f: + f.write(r.stdout) + bf = os.path.join(WORK, 'before.png') + subprocess.run([FF, '-v', 'error', *before_frame_args(src, n, s['fps'], bf)], check=True) + got = {'preview': rgb(['-i', pv], 'pv'), 'before': rgb(['-i', bf], 'bf')} + refs = {k: rgb(['-i', src, '-fps_mode', 'passthrough'], f'g{k}', + pre=f'select=eq(n\\,{n + k})') + for k in OFFSETS if n + k >= 0} + row = {'frame': n} + for name, img in got.items(): + d = {k: round(mad(img, ref), 2) for k, ref in refs.items()} + row[name] = d + row[name + '_best'] = min(d, key=d.get) + row['preview_vs_before'] = round(mad(got['preview'], got['before']), 2) + return row + + +def check_trim(worker, src, s, start, end): + out = os.path.join(WORK, f'trim_{uuid.uuid4().hex}.mkv') + r = subprocess.run([worker, '--config', job_for(src, s, out, startFrame=start, endFrame=end, + totalFrames=end - start + 1)], + capture_output=True) + if r.returncode or not os.path.exists(out): + return {'start': start, 'end': end, 'error': r.stderr.decode(errors='replace')[-400:]} + got = sequential(out, 100000) + seq = sequential(src, end + 20) + idx = indices(got, seq) + os.remove(out) + return {'start': start, 'end': end, 'frames': idx, 'ok': idx == list(range(start, end + 1))} + + +DEFAULT = ['small_clip.mp4', 'seek_timebase_2997.mov', 'interlaced_test.avi', 'colorize_test.mp4'] + + +def main(): + args = sys.argv[1:] + worker, out = None, None + while args and args[0].startswith('--'): + if args[0] == '--worker': + worker, args = args[1], args[2:] + elif args[0] == '--json': + out, args = args[1], args[2:] + paths = args or [os.path.join(RES, f) for f in DEFAULT] + results = [] + for src in paths: + s = probe(src) + r = {'source': os.path.basename(src), 'time_base': s['time_base'], 'frames': [], + 'trim': check_trim(worker, src, s, 41, 52)} + for n in (10, 41, 60): + r['frames'].append(check_frame(worker, src, s, n)) + results.append(r) + print(f"== {r['source']} ({s['time_base']})") + for row in r['frames']: + if 'error' in row: + print(f" frame {row['frame']}: ERROR {row['error']}") + continue + print(f" frame {row['frame']:3d}: preview is source frame N{row['preview_best']:+d} " + f"({row['preview']}), before is N{row['before_best']:+d}, " + f"preview vs before {row['preview_vs_before']}") + t = r['trim'] + print(f" trim {t['start']}..{t['end']}: {t.get('frames', t.get('error'))} " + f"{'OK' if t.get('ok') else 'WRONG'}") + if out: + with open(out, 'w') as f: + json.dump(results, f, indent=1) + + +if __name__ == '__main__': + main() diff --git a/worker/src/source_decode.rs b/worker/src/source_decode.rs index 6cfece0..bb46f18 100644 --- a/worker/src/source_decode.rs +++ b/worker/src/source_decode.rs @@ -26,14 +26,40 @@ //! the expected size through untouched (a full-frame `crop` is a pointer //! no-op) and fails the decode on any other size, including a mid-stream //! resolution change, which with no guard at all would desync into garbage. +//! 3. **A seek must not shift the frames** (issue #157). Writing `rawvideo`, +//! ffmpeg times its output at a constant rate (the muxer has +//! `AVFMT_NOTIMESTAMPS` but not `AVFMT_VARIABLE_FPS`). After an input `-ss` +//! to `(first - 0.5) / fps`, the first kept frame arrives half a frame +//! after the seek point, and for many time bases (1/2997 MP4, 1/11988 MOV, +//! 1/90000 TS) constant-rate timing fills that gap by emitting the frame +//! **twice**: the preview showed frame N-1 and a trim of 100..109 came out +//! `100, 100, 101 .. 108`. [`RESET_PTS_FILTER`] rebases the first kept frame +//! to 0, on every decoder that seeks. Not `-fps_mode passthrough`: that +//! also drops a soft-telecine source's pulldown repeats, which IVTC expects. //! -//! See docs/ENGINEERING_NOTES.md, "Clean aperture: decode the stored frame". +//! See docs/ENGINEERING_NOTES.md, "Clean aperture: decode the stored frame", +//! and docs/reports/issue-157-seek-alignment. /// Input option placed before `-i` on every ffmpeg that decodes the source's /// pictures for the pipeline (encode decoder, preview decoder). See the module /// docs for why `codec` and not `none`/`all`. pub const APPLY_CROPPING: [&str; 2] = ["-apply_cropping", "codec"]; +/// First filter of a decoder that seeks: the first kept frame becomes PTS 0, so +/// constant-rate output timing has no gap to fill with a repeated frame. See +/// rule 3 in the module docs. +pub const RESET_PTS_FILTER: &str = "setpts=PTS-STARTPTS"; + +/// The decoder's single `-vf` (ffmpeg honours only the last one): the PTS +/// reset when the decoder seeks, then the size guard. +fn decoder_filter(width: i32, height: i32, seeks: bool) -> String { + if seeks { + format!("{},{}", RESET_PTS_FILTER, size_guard_filter(width, height)) + } else { + size_guard_filter(width, height) + } +} + /// The message ffmpeg's `crop` filter logs when [`size_guard_filter`] rejects a /// frame. Used to recognise a guard failure in the decoder's stderr and replace /// ffmpeg's cryptic wording with an explanation. @@ -87,13 +113,17 @@ pub fn encode_decoder_args( total_frames: Option, ) -> Vec { let mut args: Vec = Vec::new(); + let mut seeks = false; // Seek to the start frame if specified (before -i for an accurate, // fast decode-seek). Target the midpoint of the interval *before* // `start` so the first kept frame (PTS >= seek time) is exactly `start`, // not start±1 — the boundary `start/fps` is ambiguous under PTS rounding. + // The seek then needs the PTS reset (module docs, rule 3); an untrimmed + // decode is left exactly as it was. if let Some(start) = start_frame { if start > 0 { + seeks = true; let fps = input_frame_rate.unwrap_or(29.97); let seek_time = ((start as f64 - 0.5) / fps).max(0.0); args.push("-ss".to_string()); @@ -105,7 +135,7 @@ pub fn encode_decoder_args( args.extend([ "-i".to_string(), input_path.to_string(), "-map".to_string(), "0:v:0".to_string(), // only first video stream - "-vf".to_string(), size_guard_filter(width, height), + "-vf".to_string(), decoder_filter(width, height, seeks), "-f".to_string(), "rawvideo".to_string(), "-pix_fmt".to_string(), pix_fmt.to_string(), "-v".to_string(), "error".to_string(), @@ -135,8 +165,10 @@ pub fn encode_decoder_args( args } -/// The preview path's decoder: `num_frames` raw frames from `seek_time` into -/// `output_path`. Same cropping and size rules as [`encode_decoder_args`]. +/// The preview path's decoder (also the detectors'): `num_frames` raw frames +/// from `seek_time` into `output_path`. Same cropping, size and seek rules as +/// [`encode_decoder_args`]: a window starting at frame 0 (`seek_time` 0) does +/// not seek, exactly like an untrimmed encode, so the two agree on frame 0. pub fn preview_decoder_args( input_path: &str, seek_time: f64, @@ -146,13 +178,18 @@ pub fn preview_decoder_args( pix_fmt: &str, output_path: &str, ) -> Vec { - let mut args: Vec = vec!["-ss".to_string(), format!("{:.6}", seek_time)]; + let seeks = seek_time > 0.0; + let mut args: Vec = Vec::new(); + if seeks { + args.push("-ss".to_string()); + args.push(format!("{:.6}", seek_time)); + } args.extend(APPLY_CROPPING.iter().map(|s| s.to_string())); args.extend([ "-i".to_string(), input_path.to_string(), "-map".to_string(), "0:v:0".to_string(), "-frames:v".to_string(), num_frames.to_string(), - "-vf".to_string(), size_guard_filter(width, height), + "-vf".to_string(), decoder_filter(width, height, seeks), "-f".to_string(), "rawvideo".to_string(), "-pix_fmt".to_string(), pix_fmt.to_string(), "-v".to_string(), "error".to_string(), @@ -262,13 +299,58 @@ mod tests { fn decoders_guard_the_probed_size() { for (path, args) in both() { let vf = pos(&args, "-vf").unwrap_or_else(|| panic!("{path}: no -vf")); - assert_eq!(args[vf + 1], size_guard_filter(720, 576), "{path}"); + // The guard is the chain's last filter (a seeking decoder resets + // PTS ahead of it). + assert!(args[vf + 1].ends_with(&size_guard_filter(720, 576)), "{path}: {args:?}"); // Exactly one -vf: ffmpeg honours only the last. assert_eq!(args.iter().filter(|a| *a == "-vf").count(), 1, "{path}"); assert!(vf > pos(&args, "-i").unwrap(), "{path}: -vf must be an output option"); } } + fn vf(args: &[String]) -> &str { + &args[pos(args, "-vf").expect("a -vf") + 1] + } + + #[test] + fn seeking_decoders_reset_pts_before_the_guard() { + // Issue #157: after a seek, rawvideo's constant-rate timing repeated + // the first frame. The reset must come first in the ONE -vf chain. + let trimmed = + encode_decoder_args("in.mov", 720, 576, "yuv420p", Some(25.0), Some(10), Some(19), None); + for (path, args) in [("trimmed encode", trimmed), ("preview", preview_args())] { + assert!(pos(&args, "-ss").is_some(), "{path} seeks"); + assert_eq!( + vf(&args), + format!("{},{}", RESET_PTS_FILTER, size_guard_filter(720, 576)), + "{path}" + ); + assert_eq!(args.iter().filter(|a| *a == "-vf").count(), 1, "{path}"); + assert_eq!(RESET_PTS_FILTER, "setpts=PTS-STARTPTS"); + // Never passthrough timing: it drops soft-telecine pulldown repeats. + assert!(pos(&args, "-fps_mode").is_none(), "{path}"); + } + } + + #[test] + fn decoders_that_do_not_seek_are_unchanged() { + // An untrimmed encode (and a trim from frame 0) keeps exactly its old + // arguments: no seek, so nothing to reset. + let untrimmed = encode_args(); + let from_zero = + encode_decoder_args("in.mov", 720, 576, "yuv422p10le", Some(25.0), Some(0), Some(19), None); + let preview_at_zero = + preview_decoder_args("in.mov", 0.0, 4, 720, 576, "yuv422p10le", "frames.raw"); + for (path, args) in + [("untrimmed", untrimmed), ("from zero", from_zero), ("preview at 0", preview_at_zero)] + { + assert!(pos(&args, "-ss").is_none(), "{path}: {args:?}"); + assert_eq!(vf(&args), size_guard_filter(720, 576), "{path}"); + } + // The VFR decoder places stored frames itself and is left alone. + assert_eq!(vf(&timeline_args()), size_guard_filter(720, 576)); + } + #[test] fn size_guard_is_a_full_frame_crop_that_rejects_other_sizes() { let g = size_guard_filter(720, 576);