diff --git a/app/test/integration_frame_rate_combined_test.dart b/app/test/integration_frame_rate_combined_test.dart new file mode 100644 index 0000000..0ef7584 --- /dev/null +++ b/app/test/integration_frame_rate_combined_test.dart @@ -0,0 +1,170 @@ +/// Issue #148: Frame Rate combined with Noise Reduction and Sharpen. +/// +/// The report was "black shadow-like patches" in the output once Denoise and +/// Sharpen were added to a frame rate conversion that was clean on its own. +/// It did not reproduce on macOS arm64 (docs/reports/issue-148-frame-rate- +/// combined), so this pins the measured behaviour rather than a fix: adding +/// the default Noise Reduction (SMDegrain) and Sharpen (LSFmod) must not make +/// the synthesized frames any more prone to dark patches than frame rate +/// conversion alone. +/// +/// The metric is the share of luma pixels in a synthesized frame that are +/// darker than BOTH real frames either side of it by more than 24/255. An +/// interpolator has nothing to build such a pixel from, so it is the signature +/// of the reported defect. On the fixture it is ~0.001-0.005% whatever the +/// passes; a single black 32x32 block in this 320x240 clip would be 1.3% of +/// the frame it appears in. It is checked two ways: the mean over every +/// synthesized frame (within 0.05 percentage points of frame rate alone), and +/// the worst single frame (within 0.25), because a patch confined to one frame +/// adds only ~0.03 points to a mean over ~40 frames and would pass that alone. +/// +/// Heavy (full encodes) — nightly, not the push gate. +@Tags(['heavy']) +library; + +// ignore_for_file: avoid_print — these tests print diagnostics to the test log. + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/frame_rate_parameters.dart'; +import 'package:vapourbox/models/noise_reduction_parameters.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/sharpen_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/gpu_expectations.dart'; +import 'support/worker_harness.dart'; + +String get _outDir => '${WorkerHarness.outputDir}/frame_rate_combined'; + +// small_clip.mp4: 320x240 yuv420p, 29.97 fps, real (film) motion. +const _w = 320, _h = 240, _frames = 40; + +/// Percentage points the combined pipeline may exceed frame-rate-only by: +/// the mean over synthesized frames, and the worst single one. Measured +/// worst frames are ~0.02-0.05%, so 0.25 is ~5x above that spread and ~5x +/// below one black 32x32 block. +const _meanTolerance = 0.05, _worstTolerance = 0.25; + +typedef _DarkShare = ({double mean, double worst}); + +VideoJob _job(String label, FrameRateMethod method, {required bool filters}) => + VideoJob( + id: const Uuid().v4(), + inputPath: p.join( + WorkerHarness.repoRoot, 'Tests', 'TestResources', 'small_clip.mp4'), + outputPath: '$_outDir/$label.mkv', + processingPipeline: ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + noiseReduction: NoiseReductionParameters(enabled: filters), + sharpen: SharpenParameters(enabled: filters), + frameRate: FrameRateParameters( + enabled: true, target: FrameRateTarget.ntsc5994, method: method), + ), + // Lossless, so the measurement sees the pipeline, not the encoder. + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, + container: ContainerFormat.mkv, + audioMode: AudioMode.none, + ), + inputWidth: _w, + inputHeight: _h, + inputPixelFormat: 'yuv420p', + inputFrameRate: 29.97, + totalFrames: _frames, + endFrame: _frames - 1, + ); + +/// Every frame of [path] as 8-bit luma. +Future> _lumaFrames(String path) async { + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-v', 'error', '-i', path, '-f', 'rawvideo', '-pix_fmt', 'gray', '-'], + stdoutEncoding: null, + environment: WorkerHarness.ffmpegEnv); + expect(r.exitCode, 0, reason: r.stderr.toString()); + final bytes = r.stdout as List; + const size = _w * _h; + return [ + for (var o = 0; o + size <= bytes.length; o += size) + Uint8List.fromList(bytes.sublist(o, o + size)), + ]; +} + +/// Over the synthesized frames (odd indices at 2x), the mean and the worst +/// percentage of pixels darker than both neighbouring real frames by more +/// than [t]. +_DarkShare _darkPatchPercent(List f, {int t = 24}) { + var sum = 0.0, worst = 0.0, n = 0; + for (var m = 1; m + 1 < f.length; m += 2) { + final a = f[m - 1], y = f[m], b = f[m + 1]; + var dark = 0; + for (var i = 0; i < y.length; i++) { + final lo = a[i] < b[i] ? a[i] : b[i]; + if (y[i] < lo - t) dark++; + } + final share = dark / y.length; + sum += share; + if (share > worst) worst = share; + n++; + } + expect(n, greaterThan(10), reason: 'too few synthesized frames to measure'); + return (mean: 100 * sum / n, worst: 100 * worst); +} + +Future<_DarkShare> _measure(String label, FrameRateMethod method, + {required bool filters}) async { + final r = await WorkerHarness.runJob( + _job(label, method, filters: filters).toJson(), + label: label, + timeout: const Duration(minutes: 20), + extraEnv: const {'VAPOURBOX_STALL_TIMEOUT_SECS': '1200'}); + expect(r.success, isTrue, reason: r.error); + final frames = await _lumaFrames(r.outputPath!); + // 29.97 -> 59.94 doubles the count exactly. + expect(frames.length, _frames * 2); + final d = _darkPatchPercent(frames); + print(' $label: dark-patch mean ${d.mean.toStringAsFixed(4)}% ' + 'worst ${d.worst.toStringAsFixed(4)}%'); + return d; +} + +void main() { + setUpAll(() async { + await WorkerHarness.ensureReady(); + Directory(_outDir).createSync(recursive: true); + }); + + for (final method in [FrameRateMethod.flowFps, FrameRateMethod.rife]) { + test('${method.name}: Noise Reduction + Sharpen add no dark patches', + () async { + if (method == FrameRateMethod.rife && + !await requireGpuPlugin('--probe-rife')) { + return; + } + final alone = + await _measure('${method.name}_alone', method, filters: false); + final combined = + await _measure('${method.name}_nr_sharpen', method, filters: true); + expect(combined.mean, lessThan(alone.mean + _meanTolerance), + reason: 'synthesized frames gained dark patches once Noise ' + 'Reduction and Sharpen were added (issue #148): mean ' + '${combined.mean.toStringAsFixed(4)}% vs ' + '${alone.mean.toStringAsFixed(4)}% with frame rate conversion ' + 'alone'); + expect(combined.worst, lessThan(alone.worst + _worstTolerance), + reason: 'a synthesized frame gained dark patches once Noise ' + 'Reduction and Sharpen were added (issue #148): worst frame ' + '${combined.worst.toStringAsFixed(4)}% vs ' + '${alone.worst.toStringAsFixed(4)}% with frame rate conversion ' + 'alone'); + }, timeout: const Timeout(Duration(minutes: 45))); + } +} diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index 858dc50..936018b 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -3230,6 +3230,32 @@ Untested: a real capture with a visible head-switching band (none in one), and Windows/Linux (the analysis is plain Rust; only the decode is platform-specific). +### Frame Rate + Noise Reduction + Sharpen: dark patches not reproduced (issue #148, 2026-11-10) + +Full report with scripts: +[docs/reports/issue-148-frame-rate-combined/](reports/issue-148-frame-rate-combined/index.html). + +- Reported: "black shadow-like patches" once Denoise and Sharpen were added to + a frame rate conversion that was clean alone. No clip, method or platform. +- Measured on macOS arm64 by exec'ing the template's own `{{#FRAME_RATE}}` + block after the filters: the share of synthesized-frame pixels darker than + **both** real neighbours by >24/255. 5 denoisers x 4 sharpeners x + FlowFPS/RIFE x 8/10-bit sit at 0.001-0.005% with or without the filters (a + single black 32x32 block in the 320x240 fixture is 1.3%). Fast pan and hard + cuts: same with and without. +- Suspects ruled out here: FlowFPS at 10/16-bit matches 8-bit to 0.14/255 + mean (MVTools master); RIFE's RGBS input is already out of range from the + *source* (1.2% of samples < -0.05), sharpening barely moves it. +- No pass changed. A clamp before RIFE or a prefiltered FlowFPS search would + change every frame-rate job's output for no measured defect. +- Untested and most likely: MVTools **v24** (Windows, macOS x64) and RIFE on + non-Apple Vulkan drivers. `integration_frame_rate_combined_test.dart` + (heavy, nightly, all four platforms) asserts NR + Sharpen stay within 0.05 + percentage points of frame rate alone on the mean over synthesized frames + **and** within 0.25 on the worst single frame, so it covers exactly those. + The mean alone would miss a one-frame patch (~0.03 points over 40 frames). +- `pal-sd-25.mov` is static colour bars: useless for anything motion-related. + ### Testing a deps change before publishing A PR that changes `deps/` has a chicken-and-egg problem: `ci-test.yml` and diff --git a/docs/reports/README.md b/docs/reports/README.md index 71d790e..533e59c 100644 --- a/docs/reports/README.md +++ b/docs/reports/README.md @@ -66,6 +66,7 @@ to it. | Date | Report | Outcome | |---|---|---| +| 2026-11-10 | [Issue #148: Frame Rate combined with Noise Reduction and Sharpen](issue-148-frame-rate-combined/index.html) | Not reproduced on macOS arm64 across 5 denoisers x 4 sharpeners x FlowFPS/RIFE x 8/10-bit; no pipeline change; nightly regression test added to cover Windows/MVTools v24 and other GPUs; reporter asked for a clip | | 2026-11-09 | [Noise and halo scores for the pass list](2026-11-noise-halo-scores/index.html) | Built: `--detect-noise-halo` badges Noise Reduction at a motion-compensated noise sigma ≥ 3.5 and Dehalo at a two-sided edge overshoot ≥ 6% of the step; no false positives on 154 fixtures (clean max 2.06 / 1.48), catches white noise from sigma 4 and tape/grain noise from sigma 6, misses sharpening of already-soft edges. VCG's signalstats TOUT and halo ratio did not separate the fixtures; a spatial (Immerkær) noise score was rejected as blind to tape and grain noise | | 2026-11-08 | [Crop & Resize: BT.601 → BT.709 on SD → HD, and Detect borders](2026-11-crop-detect-709/index.html) | Built: matrix-only 601 → 709 conversion when the output is HD (default Auto; patches within 0.67 levels of the source; a gamut conversion rejected: it moved 75% green by 55 levels on players without colour management); `--detect-crop` finds blanking, bars and the head-switching band exactly on real-picture fixtures, and matches a by-eye measurement of a DVB-T broadcast | | 2026-11-08 | [Classical filters for deps 1.16.0: SangNom, vszip NLMeans/WNNM, vs-jetpack ports](2026-11-classical-filters/index.html) | Built: SangNom AA (sharpest; 8/10/16-bit parity; slowest of the three at 1080p), NLMeans (CPU) = KNLMeansCL's output without OpenCL, the IVTC fade fix (field gap 4.12 → 0.06), vine dehalo and mpeg2stinx (both advanced; mpeg2stinx lost PSNR on every fixture). Not built: WNNM (luma ≈ BM3D, chroma 2.6–7.2 dB worse; fast, but NLMeans (CPU) covers that with better chroma); havsfunc's santiag `sangnom` type stays off (calls a double-rate SangNom) | diff --git a/docs/reports/issue-148-frame-rate-combined/img/01-film-real-synth-real.png b/docs/reports/issue-148-frame-rate-combined/img/01-film-real-synth-real.png new file mode 100644 index 0000000..8348490 Binary files /dev/null and b/docs/reports/issue-148-frame-rate-combined/img/01-film-real-synth-real.png differ diff --git a/docs/reports/issue-148-frame-rate-combined/img/02-pan-real-synth-real.png b/docs/reports/issue-148-frame-rate-combined/img/02-pan-real-synth-real.png new file mode 100644 index 0000000..9fc29fb Binary files /dev/null and b/docs/reports/issue-148-frame-rate-combined/img/02-pan-real-synth-real.png differ diff --git a/docs/reports/issue-148-frame-rate-combined/index.html b/docs/reports/issue-148-frame-rate-combined/index.html new file mode 100644 index 0000000..bf3eeab --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/index.html @@ -0,0 +1,250 @@ + + + + + +Issue #148 — Frame Rate combined with Noise Reduction and Sharpen + + + +
+

Issue #148 — Frame Rate combined with Noise Reduction and Sharpen

+

2026-11-10 · issue #148 · +measured on an Apple M4 Pro (macOS arm64), deps 1.14.0 bundle, against the templates of app 1.6.0

+ +
+Outcome: not reproduced; no pipeline change made. The reporter saw +"glitches" and "black shadow-like patches" once Noise Reduction and Sharpen were +added to a frame rate conversion that was clean on its own. On macOS arm64, across +5 denoisers × 4 sharpeners × FlowFPS/RIFE × 8/10-bit, plus a fast pan and hard +cuts, adding Noise Reduction and Sharpen never measurably +increased dark patches in the synthesized frames. Every cell sits at the same +~0.001–0.005% noise floor; a single black 32×32 block would be 1.3%. The three +code-level suspects were each tested and none holds up here (below). +The likeliest remaining explanations are platform-specific (Windows MVTools v24, +or RIFE on a particular GPU/driver) or content-specific, and both need the +reporter's clip, method and GPU. A nightly regression test now pins the measured +behaviour (app/test/integration_frame_rate_combined_test.dart). +
+ +

1. What runs, and in what order

+

In the generated script: Noise Reduction → … → Resize → Sharpen +(after resize, issue #109) → Film Grain → Frame Rate (last video pass). +Denoise, then sharpen, then interpolate is the conventional order, so a pass-order +bug was not the lead. What adding the two passes changes is the input the +interpolator sees.

+

The probes do not copy the frame-rate code. They extract the +{{#FRAME_RATE}} block from worker/templates/pipeline_template.vpy, +fill its placeholders as the worker does (block 16, overlap 8, 29.97 → 59.94), and +exec it after haf.SMDegrain / haf.LSFmod etc. at the app's +defaults. RIFE runs through MoltenVK on the M4 Pro GPU.

+ +

2. The metric

+

For each synthesized frame (every odd frame at 2×), the share of luma +pixels darker than both real frames either side of it by more than +24/255. An interpolator has nothing to build such a pixel from, so "darker than +both" is the signature of the reported black patches. A brighter-than-both share +is recorded alongside it as a control.

+

What it can't tell you: motion alone produces such +pixels at moving edges (see the pan, §4), so it compares configurations; it is not +an absolute defect count. It sees luma only. One 40-frame film clip is not a +population, and Apple Silicon running MVTools master is not Windows running v24.

+ +

3. Hypotheses tested

+ +

H1 — RIFE fed out-of-range RGB by the sharpener

+

The RIFE block converts YUV to RGBS and passes it on unclamped; a +sharpener's overshoot could push samples below 0, outside what the network was +trained on. Measured (scripts/range.py): the share of RGB samples below +−0.05 in the worst frame.

+ + + + + + +
Input to RIFEminmaxworst-frame share < −0.05
source−0.0820.9041.242%
SMDegrain−0.0820.9041.241%
LSFmod−0.1070.9091.439%
SMDegrain + LSFmod−0.1030.9041.436%
+

Weak. The unfiltered source already sends +1.2% of samples below −0.05 into RIFE; sharpening adds about a sixth to that, and the +matrix (§4) shows RIFE's output unaffected either way. Clamping before RIFE was the +planned fix for this hypothesis and was not made: with no measurable +effect it would only change synthesized frames for every user on speculation.

+ +

H2 — FlowFPS's motion search misled by denoised/sharpened detail

+

Both mv.Analyse calls search the same unfiltered super clip. Flattened +texture or sharpening halos could mislead block matching, and FlowFPS warps along +whatever vectors it gets. Not supported: FlowFPS's dark +share with any denoiser or sharpener is within the noise of FlowFPS alone (§4), and +the frames look the same (§5). A prefiltered search clip was the planned fix; it +was not made, because it would change every FlowFPS job's output with no +measured defect to justify it.

+ +

H3 — FlowFPS wrong above 8 bits

+

A >8-bit source (or 4:2:0 at >8-bit, converted to YUV422P16 by the +block) runs FlowFPS at high depth. Measured (scripts/depth.py): FlowFPS on +the same clip at 10 and 16 bits differs from 8-bit by a mean 0.13/255 +(worst frame 0.29) and 0.14/255 (worst 0.32). Ruled +out on arm64 (MVTools master). Windows and macOS x64 ship the v24 release, +which was not tested.

+ +

4. The matrix

+

Dark share of synthesized frames, mean over frames (worst frame in brackets), +29.97 → 59.94, from scripts/matrix.py. The 10-bit source is the same clip +converted to YUV420P10. Sharpeners: LSFmod at strength 200 (twice the +default), CAS at 1.0 (its maximum), aWarpSharp2 at defaults.

+
+ + + + + + + + + + + + + + + + +
Noise Reduction + SharpenFlowFPS 8-bitFlowFPS 10-bitRIFE 8-bitRIFE 10-bit
none + none (FPS only)0.001% (0.033)0.001% (0.033)0.001% (0.026)0.001% (0.026)
none + LSFmod 2000.001% (0.014)0.001% (0.013)0.002% (0.034)0.002% (0.034)
none + CAS 1.00.005% (0.042)0.005% (0.043)0.005% (0.035)0.005% (0.033)
none + aWarpSharp20.001% (0.016)0.001% (0.016)0.001% (0.021)0.001% (0.021)
SMDegrain + none0.001% (0.030)0.001% (0.031)0.001% (0.033)0.001% (0.031)
SMDegrain + LSFmod 2000.001% (0.017)0.001% (0.017)0.002% (0.038)0.002% (0.038)
SMDegrain + CAS 1.00.005% (0.042)0.004% (0.038)0.005% (0.048)0.005% (0.044)
SMDegrain + aWarpSharp20.001% (0.017)0.001% (0.013)0.001% (0.027)0.001% (0.027)
MCTemporalDenoise + none0.001% (0.029)0.001% (0.030)0.001% (0.033)0.001% (0.031)
MCTemporalDenoise + LSFmod 2000.001% (0.017)0.001% (0.013)0.002% (0.036)0.002% (0.038)
DFTTest + none0.001% (0.031)0.001% (0.031)0.001% (0.031)0.001% (0.033)
DFTTest + LSFmod 2000.001% (0.010)0.001% (0.010)0.002% (0.035)0.002% (0.035)
FFT3DFilter (sharpen 1.0) + none0.001% (0.018)0.001% (0.017)0.002% (0.029)0.002% (0.025)
FFT3DFilter (sharpen 1.0) + LSFmod 2000.003% (0.031)0.003% (0.023)0.003% (0.034)0.003% (0.035)
+
+

The highest cell, CAS at maximum, is 0.005%: about 4 pixels in a 320×240 +frame, and the same with or without a denoiser. For scale, one black 32×32 block +is 1.3%, some 260× more.

+ +

Stress: fast pan and hard cuts

+

scripts/stress.py. The pan crops a moving window (8 px/frame) from a 2× +enlargement; the cuts splice two parts of the clip every 8 frames.

+ + + + +
CaseFlowFPS aloneFlowFPS + SMDegrain + LSFmodRIFE aloneRIFE + SMDegrain + LSFmod
fast pan0.658% (0.871)0.767% (1.014)0.610% (0.779)0.714% (0.938)
hard cuts0.001% (0.033)0.001% (0.039)0.051% (0.547)0.051% (0.562)
+

On the pan the metric rises for every configuration because motion itself puts +pixels darker than both neighbours at moving edges; the filtered pipeline is ~15% +higher, consistent with sharpening adding edge contrast, and the frames show no +patches (§5). At cuts RIFE's worst frame is the one spanning the cut, and it is the +same with and without the filters.

+ +

5. What it looks like

+

Grid of film frames: rows FlowFPS alone, FlowFPS with filters, RIFE alone, RIFE with filters; columns real, synthesized, real

+

Figure 1 — film clip, output frames 10/11/12. Columns: real · synthesized · real. +Rows top to bottom: FlowFPS alone · FlowFPS + SMDegrain + LSFmod · RIFE alone · RIFE + SMDegrain + LSFmod. +Look in the middle column for dark blotches near the moving leg and headlamp that +are absent from both neighbours: there are none in any row.

+

The same grid for a fast synthetic pan

+

Figure 2 — fast pan, output frames 28/29/30, same layout. The +synthesized column moves smoothly between its neighbours in all four rows; the +filtered rows are visibly crisper and no darker.

+ +

6. Corrections made during the work

+
+
    +
  • The plan assumed the 10-bit fixture pal-sd-25.mov would test real +motion at depth. It is static colour bars, so every metric on it was +exactly 0, which proves nothing. The 10-bit column above uses the film clip +converted to 10-bit instead.
    • +
  • The plan named H1 (RIFE input range) the most likely cause. The range +measurement made it the least supported: the source alone is already out of range +by nearly as much.
    • +
+
+ +

Correction (review, same day). +The regression test as first committed compared only the mean over the ~40 +synthesized frames, with the 0.05-point tolerance. A black block confined to one +frame adds just 1.3/40 ≈ 0.03 points to that mean, so it would have passed. The test +now also bounds the worst single frame (§8).

+ +

7. What wasn't tested

+
    +
  • Windows x64 and macOS x64, which ship the MVTools v24 +release rather than master. FlowFPS's frame count already differs between the two +(issue #129); its pixels may too. This is the likeliest place for the bug to be.
    • +
  • RIFE on NVIDIA/AMD/Intel Vulkan drivers. RIFE ran on Apple GPU via MoltenVK only.
    • +
  • KNLMeansCL — the machine measured had no usable OpenCL device in this +session. BM3D, mClean, TemporalDegrain2 and the rest of the denoisers weren't measured.
    • +
  • Combinations with QTGMC (double-rate deinterlace before the retime), +resize, or other passes; the reporter listed only "FPS", "Denoise" and "Sharpness".
    • +
  • The preview path, and the encoder: everything here is the script's +output, upstream of ffmpeg. The regression test encodes losslessly (FFV1) for the +same reason.
    • +
  • The reporter's own footage. None was supplied.
    • +
  • The full app pipeline, end to end. No Rust toolchain was available to build a +1.6.0 worker on the machine measured, so the probes exec the template block +directly. The new regression test drives the real worker and will run in +nightly.yml; it was type-checked (flutter analyze) but not +executed before commit.
    • +
+ +

8. Decisions

+
    +
  • No change to the Frame Rate, Noise Reduction or Sharpen passes. +Both candidate fixes (clamp before RIFE; prefiltered FlowFPS search) would alter +output for every frame-rate job, and neither addresses anything measured.
    • +
  • A nightly regression test, +app/test/integration_frame_rate_combined_test.dart: for FlowFPS and RIFE +(RIFE where the GPU probe says it runs), the dark share with default Noise +Reduction + Sharpen must stay within 0.05 percentage points of frame rate alone +on the mean over synthesized frames, and within 0.25 on the worst single frame. +The mean tolerance is ~10× the noise floor; the worst-frame one is ~5× the +worst frames measured here (0.02–0.05%) and ~5× below one black 32×32 block (1.3%), +so it fails on the reported defect without flaking on sharpening contrast. It runs +on all four nightly platforms, which includes MVTools v24 on Windows and macOS x64 +and RIFE on the Windows/Linux runners: the coverage this report lacks.
    • +
  • Ask the reporter for a short clip, the Frame Rate method and target, +the Noise Reduction and Sharpen methods, the OS and the GPU, linking this report.
    • +
+ +

9. Reproducing

+
VAPOURBOX_DEPS_DIR=<deps/macos-arm64> ISSUE148_SCRATCH=<scratch dir> \
+  docs/reports/issue-148-frame-rate-combined/scripts/run.sh
+

run.sh decodes 40 frames of Tests/TestResources/small_clip.mp4 +(from 0 s and from 30 s) into the scratch directory and runs range.py, +depth.py, matrix.py, stress.py and images.py +with the bundle's Python. harness.py holds the loader, the template +extraction and the metric. The RIFE paths it fills in are the macOS bundle layout.

+
+ + diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/depth.py b/docs/reports/issue-148-frame-rate-combined/scripts/depth.py new file mode 100644 index 0000000..2586c6d --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/depth.py @@ -0,0 +1,15 @@ +"""FlowFPS at 10/16-bit against 8-bit on the same clip (hypothesis 3).""" +from harness import * +src = load("small.yuv", 320, 240, vs.YUV420P8, 40, 30000, 1001) +def flow(c): + sup=core.mv.Super(c,pel=2) + bw=core.mv.Analyse(sup,isb=True,blksize=16,overlap=4); fw=core.mv.Analyse(sup,isb=False,blksize=16,overlap=4) + return core.mv.FlowFPS(c,sup,bw,fw,num=60000,den=1001) +ref=flow(src) +for bits in (10,16): + hi=core.resize.Point(src,format=src.format.replace(bits_per_sample=bits).id) + out=core.resize.Point(flow(hi),format=vs.YUV420P8) + d=core.std.PlaneStats(ref,out) + worst=max(range(1,70,2),key=lambda i:d.get_frame(i).props.PlaneStatsDiff) + vals=[d.get_frame(i).props.PlaneStatsDiff*255 for i in range(1,70,2)] + print(f"{bits}-bit vs 8-bit FlowFPS: synthesized mean diff {sum(vals)/len(vals):.2f}/255, worst {max(vals):.2f} (frame {worst})") diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/harness.py b/docs/reports/issue-148-frame-rate-combined/scripts/harness.py new file mode 100644 index 0000000..db7691f --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/harness.py @@ -0,0 +1,63 @@ +"""Shared by the issue #148 probes: load raw video, exec the template's FRAME_RATE block, measure.""" +import vapoursynth as vs, havsfunc as haf, sys, ctypes, re, os +core = vs.core +DEPS = os.environ["DEPS"]; TEMPLATE = os.environ["TEMPLATE"] + +def load(path, W, H, fmt, N, fpsnum, fpsden): + raw = open(path, 'rb').read() + c = core.std.BlankClip(width=W, height=H, format=fmt, length=N, fpsnum=fpsnum, fpsden=fpsden) + f0 = c.get_frame(0); bps = c.format.bytes_per_sample + geo = [(f0[p].shape[1] * bps, f0[p].shape[0]) for p in range(3)] + fs = sum(w * h for w, h in geo) + def put(n, f): + f = f.copy(); o = n * fs + for p, (w, h) in enumerate(geo): + ptr = f.get_write_ptr(p).value; st = f.get_stride(p) + for y in range(h): + ctypes.memmove(ptr + y * st, raw[o + y * w:o + (y + 1) * w], w) + o += w * h + return f + return core.std.ModifyFrame(c, c, put) + +def section(text, name): + m = re.search(r"\{\{#%s\}\}\n(.*?)\{\{/%s\}\}" % (name, name), text, re.S) + return m.group(1) + +def frame_rate_block(method, num, den, override=None): + t = override or open(TEMPLATE).read() + blk = section(t, "FRAME_RATE") + for m in ("FLOWFPS", "DUPLICATE", "RIFE"): + tag = "FRAME_RATE_" + m + keep = (m == method) + blk = re.sub(r"\{\{#%s\}\}\n(.*?)\{\{/%s\}\}\n" % (tag, tag), + (lambda mm: mm.group(1)) if keep else "", blk, flags=re.S) + subs = {"FPS_TARGET_NUM": num, "FPS_TARGET_DEN": den, "FPS_BLOCK_SIZE": 16, "FPS_OVERLAP": 8, + "RIFE_PRELOAD": repr(DEPS + "/vapoursynth/moltenvk/libMoltenVK.dylib"), + "RIFE_PLUGIN": repr(DEPS + "/vapoursynth/rife/librife.dylib"), + "RIFE_MODEL_DIR": repr(DEPS + "/vapoursynth/rife/models/rife-v4.26_ensembleFalse"), + "RIFE_MATRIX": "470bg", "RIFE_RANGE": "limited"} + for k, v in subs.items(): + blk = blk.replace("{{%s}}" % k, str(v)) + assert "{{" not in blk, blk + return blk + +def run(clip, method, nr, sh, num, den, override=None): + if nr: clip = haf.SMDegrain(clip, tr=2, thSAD=300) + if sh: clip = haf.LSFmod(clip, strength=100, overshoot=1, undershoot=1, soft=0) + g = {"clip": clip, "core": core, "vs": vs, "haf": haf} + exec(frame_rate_block(method, num, den, override), g) + return g["clip"] + +def luma8(c): + return core.resize.Point(core.std.ShufflePlanes(c, 0, vs.GRAY), format=vs.GRAY8) + +def dark_metric(out, t=24): + """Share of pixels in each synthesized frame darker than both real neighbours by > t.""" + y = luma8(out) + prev = y[0] + y[:-1]; nxt = y[1:] + y[-1] + dark = core.std.Expr([prev, y, nxt], f"y x z min {t} - < 255 0 ?") + brt = core.std.Expr([prev, y, nxt], f"y x z max {t} + > 255 0 ?") + ds = core.std.PlaneStats(dark); bs = core.std.PlaneStats(brt) + vals = [(ds.get_frame(m).props.PlaneStatsAverage, bs.get_frame(m).props.PlaneStatsAverage, m) + for m in range(1, len(y) - 1, 2)] + return vals diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/images.py b/docs/reports/issue-148-frame-rate-combined/scripts/images.py new file mode 100644 index 0000000..60ae9be --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/images.py @@ -0,0 +1,22 @@ +"""The two comparison grids in the report: rows FlowFPS alone, FlowFPS+NR+Sharpen, +RIFE alone, RIFE+NR+Sharpen; columns real / synthesized / real.""" +from harness import * + +def grid(src, frames, out): + rows = [] + for m in ("FLOWFPS", "RIFE"): + for nr, sh in [(0, 0), (1, 1)]: + o = run(src, m, nr, sh, 60000, 1001) + rows.append(core.std.StackHorizontal([o[i] for i in frames])) + img = core.resize.Bicubic(core.std.StackVertical(rows), format=vs.RGB24, matrix_in_s="170m") + f = img.get_frame(0) + pl = [bytes(f[p]) for p in range(3)] + with open(out, "wb") as fh: + fh.write(b"P6 %d %d 255\n" % (f.width, f.height)) + fh.write(bytes(v for t in zip(*pl) for v in t)) + +a = load("small.yuv", 320, 240, vs.YUV420P8, 40, 30000, 1001) +grid(a, (10, 11, 12), "film.ppm") +big = core.resize.Bicubic(a, 640, 480) +pan = core.std.FrameEval(a, lambda n: core.std.CropAbs(big, 320, 240, left=min(n * 8, 318), top=120)) +grid(pan, (28, 29, 30), "pan.ppm") diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/matrix.py b/docs/reports/issue-148-frame-rate-combined/scripts/matrix.py new file mode 100644 index 0000000..3097778 --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/matrix.py @@ -0,0 +1,28 @@ +"""Denoisers x sharpeners x FlowFPS/RIFE at 8 and 10 bits: the dark-patch matrix.""" +import os, sys +from harness import * +s8 = load("small.yuv", 320, 240, vs.YUV420P8, 40, 30000, 1001) +s10 = core.resize.Point(s8, format=vs.YUV420P10) +NR = {"none": lambda c: c, + "smdegrain": lambda c: haf.SMDegrain(c, tr=2, thSAD=300), + "mctd": lambda c: haf.MCTemporalDenoise(c, settings="medium"), + # KNLMeansCL omitted: needs an OpenCL device (none on the machine measured). + #"knlm": lambda c: core.knlm.KNLMeansCL(core.knlm.KNLMeansCL(c, d=1, a=2, s=4, h=1.2, channels="Y"), d=1,a=2,s=4,h=1.2,channels="UV"), + "dfttest": lambda c: core.dfttest.DFTTest(c, sigma=8), + "fft3d_sharp": lambda c: core.fft3dfilter.FFT3DFilter(c, sigma=2, sharpen=1.0)} +SH = {"none": lambda c: c, + "lsfmod200": lambda c: haf.LSFmod(c, strength=200), + "cas1": lambda c: core.cas.CAS(c, sharpness=1.0), + "awarp": lambda c: core.warp.AWarpSharp2(c, thresh=128, blur=2, type=0, depth=16)} +for which, (sname, src) in [(m, s) for m in ("FLOWFPS", "RIFE") for s in (("8bit", s8), ("10bit", s10))]: + for nn, nf in NR.items(): + for sn, sf in SH.items(): + if nn != "none" and sn not in ("none","lsfmod200") and nn != "smdegrain": continue + g = {"clip": sf(nf(src)), "core": core, "vs": vs, "haf": haf} + try: + exec(frame_rate_block(which, 60000, 1001), g) + v = dark_metric(g["clip"]) + except Exception as e: + print(f"{sname} {which} {nn}+{sn}: ERROR {e}", flush=True); continue + dm = sum(a for a,_,_ in v)/len(v); dx = max(v); bx = max(b for _,b,_ in v) + print(f"{sname:5s} {which:7s} {nn:11s}+{sn:9s}: dark mean {100*dm:.3f}% max {100*dx[0]:.3f}% (f{dx[2]}) bright max {100*bx:.3f}%", flush=True) diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/range.py b/docs/reports/issue-148-frame-rate-combined/scripts/range.py new file mode 100644 index 0000000..9312497 --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/range.py @@ -0,0 +1,30 @@ +"""How far outside [0,1] the RGBS fed to RIFE goes, with and without NR/sharpen.""" +import vapoursynth as vs, havsfunc as haf, sys, ctypes +core = vs.core +W,H,N=320,240,40 +raw=open("small.yuv",'rb').read() +fs=W*H*3//2 +c=core.std.BlankClip(width=W,height=H,format=vs.YUV420P8,length=N,fpsnum=30000,fpsden=1001) +def put(n,f): + f=f.copy(); o=n*fs + for p,(w,h) in enumerate([(W,H),(W//2,H//2),(W//2,H//2)]): + ptr=f.get_write_ptr(p).value; st=f.get_stride(p) + for y in range(h): + ctypes.memmove(ptr+y*st, raw[o+y*w:o+(y+1)*w], w) + o+=w*h + return f +src=core.std.ModifyFrame(c,c,put) +def stats(name,clip): + rgb=core.resize.Bicubic(clip,format=vs.RGBS,matrix_in_s="170m",range_in_s="limited") + mn,mx,neg=9,-9,0.0 + for p in range(3): + pl=core.std.ShufflePlanes(rgb,p,vs.GRAY) + s=core.std.PlaneStats(pl); ng=core.std.PlaneStats(core.std.Expr(pl,"x -0.05 < 1 0 ?")) + for i in range(N): + fp=s.get_frame(i).props; mn=min(mn,fp.PlaneStatsMin); mx=max(mx,fp.PlaneStatsMax) + neg=max(neg,ng.get_frame(i).props.PlaneStatsAverage) + print(f"{name:20s} min={mn:+.3f} max={mx:.3f} worst-frame share < -0.05: {100*neg:.3f}%") +stats("source",src) +stats("smdegrain",haf.SMDegrain(src,tr=2,thSAD=300)) +stats("lsfmod",haf.LSFmod(src,strength=100)) +stats("smdegrain+lsfmod",haf.LSFmod(haf.SMDegrain(src,tr=2,thSAD=300),strength=100)) diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/run.sh b/docs/reports/issue-148-frame-rate-combined/scripts/run.sh new file mode 100755 index 0000000..0bc19da --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/run.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Reproduces every number in the issue #148 report. +# +# VAPOURBOX_DEPS_DIR= ISSUE148_SCRATCH= ./run.sh +# +# Decodes the fixture into $ISSUE148_SCRATCH (raw video, not committed), then +# runs each probe with the bundled Python/VapourSynth. The probes exec the +# repo's own {{#FRAME_RATE}} block from pipeline_template.vpy, so they measure +# the code as it is, not a copy of it. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${VAPOURBOX_DEPS_DIR:?set to a deps/ directory}" +: "${ISSUE148_SCRATCH:?set to a scratch directory}" +mkdir -p "$ISSUE148_SCRATCH" +D="$VAPOURBOX_DEPS_DIR" +CLIP="$REPO/Tests/TestResources/small_clip.mp4" +"$D/ffmpeg/ffmpeg" -v error -y -i "$CLIP" -frames:v 40 -f rawvideo -pix_fmt yuv420p "$ISSUE148_SCRATCH/small.yuv" +"$D/ffmpeg/ffmpeg" -v error -y -ss 30 -i "$CLIP" -frames:v 40 -f rawvideo -pix_fmt yuv420p "$ISSUE148_SCRATCH/small30.yuv" + +export TEMPLATE="$REPO/worker/templates/pipeline_template.vpy" DEPS="$D" +export PYTHONHOME="$D/python" PYTHONPATH="$HERE:$D:$D/python-packages" +export VAPOURSYNTH_PLUGIN_PATH="$D/vapoursynth/plugins" +export DYLD_LIBRARY_PATH="$D/vapoursynth:$D/python/lib" LD_LIBRARY_PATH="$D/vapoursynth:$D/python/lib" +export XDG_CONFIG_HOME="$D/config" +PY="$D/python/bin/python3" +cd "$ISSUE148_SCRATCH" +for probe in range depth matrix stress; do + echo "== $probe" + "$PY" "$HERE/$probe.py" 2>&1 | grep -E '^(source|smdegrain|lsfmod|[0-9]+-bit|8bit|10bit|pan|cuts)' +done +"$PY" "$HERE/images.py" >/dev/null 2>&1 +echo "images: $ISSUE148_SCRATCH/*.ppm" diff --git a/docs/reports/issue-148-frame-rate-combined/scripts/stress.py b/docs/reports/issue-148-frame-rate-combined/scripts/stress.py new file mode 100644 index 0000000..ecbf7bd --- /dev/null +++ b/docs/reports/issue-148-frame-rate-combined/scripts/stress.py @@ -0,0 +1,15 @@ +"""Fast pan and hard cuts: where interpolators fail hardest.""" +import os, sys +from harness import * +a = load("small.yuv", 320, 240, vs.YUV420P8, 40, 30000, 1001) +b = load("small30.yuv", 320, 240, vs.YUV420P8, 40, 30000, 1001) +# fast pan: 8 px/frame across a 2x frame +big = core.resize.Bicubic(a, 640, 480) +pan = core.std.FrameEval(a, lambda n: core.std.CropAbs(big, 320, 240, left=min(n*8, 318), top=120)) +cuts = a[0:8] + b[0:8] + a[8:16] + b[8:16] + a[16:24] +for name, src in (("pan", pan), ("cuts", cuts)): + for m in ("FLOWFPS", "RIFE"): + for nr, sh in [(0,0),(1,1)]: + v = dark_metric(run(src, m, nr, sh, 60000, 1001)) + dm = sum(x for x,_,_ in v)/len(v); dx = max(v) + print(f"{name:4s} {m:7s} nr={nr} sh={sh}: dark mean {100*dm:.3f}% max {100*dx[0]:.3f}% (f{dx[2]})", flush=True)