diff --git a/.github/scripts/provision-vulkan.sh b/.github/scripts/provision-vulkan.sh deleted file mode 100755 index d2df0265..00000000 --- a/.github/scripts/provision-vulkan.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# Give a CI runner a Vulkan device, so the ncnn-on-Vulkan features (RIFE, and -# Real-ESRGAN / Real-CUGAN upscaling) are actually exercised rather than skipped. -# -# The hosted runners have no GPU, so Linux and Windows get Mesa's lavapipe, a -# Vulkan driver that runs on the CPU — slow, but the same plugin, model and -# script path a user's GPU takes, and ncnn accepts a CPU device. macOS arm64 -# gets nothing: neither MoltenVK nor lavapipe can run there (see its case). -# -# Wherever a device is provisioned this sets VAPOURBOX_EXPECT_VULKAN=1, and the -# GPU tests then FAIL rather than skip if a feature reports itself unavailable -# (app/test/support/gpu_expectations.dart). On a platform that does not ship -# the features (macos-x64, linux-arm64) nothing is provisioned and the tests -# assert the "not available for this platform" answer instead. -# -# Usage: provision-vulkan.sh (macos-arm64, windows-x64, linux-x64, …) -set -euo pipefail - -platform="${1:?usage: provision-vulkan.sh }" - -# Windows: the Khronos loader from LunarG's runtime components and lavapipe from -# mesa-dist-win (MIT), each pinned by hash. Keep these with the README's note. -VKRT_VERSION="1.4.328.1" -VKRT_SHA256="293efbb2c11e60429df7770f40e3c5c811a7fa42af81de0cc41e8718ddb2b373" -MESA_VERSION="26.2.2" -MESA_SHA256="0f21394d44036019bb5c07a3788dd278fd7b7b3469077a4686f0d639e8e874a1" - -case "$platform" in - linux-x64) - sudo apt-get update -qq - sudo apt-get install -y -qq --no-install-recommends mesa-vulkan-drivers libvulkan1 vulkan-tools - vulkaninfo --summary || true - echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" - ;; - windows-x64) - dir="$RUNNER_TEMP/vulkan" - rm -rf "$dir" && mkdir -p "$dir" - curl -fsSL -o "$dir/vkrt.zip" \ - "https://sdk.lunarg.com/sdk/download/$VKRT_VERSION/windows/VulkanRT-X64-$VKRT_VERSION-Components.zip" - echo "$VKRT_SHA256 $dir/vkrt.zip" | sha256sum -c - - unzip -q -j "$dir/vkrt.zip" \ - "VulkanRT-X64-$VKRT_VERSION-Components/x64/vulkan-1.dll" \ - "VulkanRT-X64-$VKRT_VERSION-Components/x64/vulkaninfo.exe" -d "$dir" - curl -fsSL -o "$dir/mesa.7z" \ - "https://github.com/pal1000/mesa-dist-win/releases/download/$MESA_VERSION/mesa3d-$MESA_VERSION-release-msvc.7z" - echo "$MESA_SHA256 $dir/mesa.7z" | sha256sum -c - - "/c/Program Files/7-Zip/7z.exe" x -y -o"$dir/mesa" "$dir/mesa.7z" \ - x64/vulkan_lvp.dll x64/lvp_icd.x86_64.json >/dev/null - icd="$(cygpath -w "$dir/mesa/x64/lvp_icd.x86_64.json")" - # The loader reads VK_DRIVER_FILES; older loaders read VK_ICD_FILENAMES. - echo "VK_DRIVER_FILES=$icd" >> "$GITHUB_ENV" - echo "VK_ICD_FILENAMES=$icd" >> "$GITHUB_ENV" - # vulkan-1.dll on PATH, where the plugins' LoadLibrary finds it. - cygpath -w "$dir" >> "$GITHUB_PATH" - VK_DRIVER_FILES="$icd" "$dir/vulkaninfo.exe" --summary || true - echo "VAPOURBOX_EXPECT_VULKAN=1" >> "$GITHUB_ENV" - ;; - macos-arm64) - # No device here, and none can be provided. The runners are VMs whose Metal - # device MoltenVK cannot open (it segfaults creating the instance), and - # lavapipe crashes in its shader compiler: the bundle's akarin links LLVM - # statically and exports its symbols, ~2,300 of them weak, and dyld binds - # lavapipe's own LLVM to akarin's copy of those (crash report: libLLVM.23 - # in PassBuilder::registerCGSCCAnalyses; CI run 40160839608). So the GPU - # tests skip here, the probe still has to answer with a real reason, and - # MoltenVK is tested on real Macs only. - echo "macOS arm64 runner: no usable Vulkan device; GPU tests skip (see the comment above)." - ;; - *) - echo "No GPU features ship on $platform; the tests assert that answer instead." - ;; -esac diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index 756842b8..f4bfe932 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -3,7 +3,8 @@ name: CI Tests # Pulls the published dependency bundle + the whisper add-on (binary + small # model) and runs the Rust test suite plus the Flutter tests, EXCLUDING the # heavy full-encode integration tests (those run in nightly.yml via -# `--tags heavy`). macOS is tested natively on both arm64 (macos-15) and x64 +# `--tags heavy`) and the `gpu` tests, which need a Vulkan device these +# runners lack (app/dart_test.yaml). macOS is tested natively on both arm64 (macos-15) and x64 # (macos-15-intel); Windows on x64; Linux on x64 (ubuntu-24.04). on: @@ -104,11 +105,6 @@ jobs: ln -sfn "$WP/lib" addons/whisper/lib test -x addons/whisper/bin/whisper-cli || { echo "::error::whisper-cli not found under $WP/bin"; exit 1; } - - name: Provision Vulkan (RIFE, AI upscaling) - # Lavapipe on the GPU-less Linux/Windows runners; sets - # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. - run: bash .github/scripts/provision-vulkan.sh macos-${{ matrix.arch }} - - name: Rust tests # Debug build: DependencyLocator's upward search then finds the repo-root # deps/ and addons/ (release mode looks under ~/Library). @@ -123,7 +119,9 @@ jobs: - name: Flutter tests # Excludes @Tags(['heavy']) — the full-encode integration tests, which # run in nightly.yml. The script-only integration tests + unit tests run here. - run: cd app && flutter test --exclude-tags heavy + # Also excludes @Tags(['gpu']): the hosted runners have no GPU, and the + # tests that run RIFE / AI upscaling need a Vulkan device (dart_test.yaml). + run: cd app && flutter test --exclude-tags heavy,gpu - name: Crash reports (on failure) # A plugin that segfaults in vspipe shows only "signal 11" in the test @@ -187,12 +185,6 @@ jobs: cp "$(dirname "$CLI")"/*.dll addons/whisper/ 2>/dev/null || true ls addons/whisper/ - - name: Provision Vulkan (RIFE, AI upscaling) - # Lavapipe on the GPU-less Linux/Windows runners; sets - # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. - shell: bash - run: bash .github/scripts/provision-vulkan.sh windows-x64 - - name: Rust tests run: cd worker && cargo test -- --nocapture @@ -205,7 +197,9 @@ jobs: - name: Flutter tests # Excludes @Tags(['heavy']) — the full-encode integration tests, which # run in nightly.yml. The script-only integration tests + unit tests run here. - run: cd app && flutter test --exclude-tags heavy + # Also excludes @Tags(['gpu']): the hosted runners have no GPU, and the + # tests that run RIFE / AI upscaling need a Vulkan device (dart_test.yaml). + run: cd app && flutter test --exclude-tags heavy,gpu linux: runs-on: ubuntu-24.04 # must match the deps-build runner: those binaries @@ -281,11 +275,6 @@ jobs: ln -sfn "${{ github.workspace }}/deps" "$HOME/.local/share/VapourBox/deps" ln -sfn "${{ github.workspace }}/addons" "$HOME/.local/share/VapourBox/addons" - - name: Provision Vulkan (RIFE, AI upscaling) - # Lavapipe on the GPU-less Linux/Windows runners; sets - # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. - run: bash .github/scripts/provision-vulkan.sh linux-x64 - - name: Rust tests run: cd worker && cargo test -- --nocapture @@ -298,4 +287,6 @@ jobs: - name: Flutter tests # Excludes @Tags(['heavy']) — the full-encode integration tests, which # run in nightly.yml. The script-only integration tests + unit tests run here. - run: cd app && flutter test --exclude-tags heavy + # Also excludes @Tags(['gpu']): the hosted runners have no GPU, and the + # tests that run RIFE / AI upscaling need a Vulkan device (dart_test.yaml). + run: cd app && flutter test --exclude-tags heavy,gpu diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index ed9c5fbe..eef70dde 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -113,11 +113,6 @@ jobs: ln -sfn "$WP/lib" addons/whisper/lib test -x addons/whisper/bin/whisper-cli || { echo "::error::whisper-cli not found under $WP/bin"; exit 1; } - - name: Provision Vulkan (RIFE, AI upscaling) - # Lavapipe on the GPU-less Linux/Windows runners; sets - # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. - run: bash .github/scripts/provision-vulkan.sh macos-${{ matrix.arch }} - - name: Build worker (debug) run: cd worker && cargo build @@ -135,7 +130,8 @@ jobs: # found" and prove nothing. env: VAPOURBOX_WORKER: ${{ github.workspace }}/worker/target/debug/vapourbox-worker - run: cd app && flutter test --tags heavy + # Not @Tags(['gpu']): the hosted runners have no GPU (dart_test.yaml). + run: cd app && flutter test --tags heavy --exclude-tags gpu - name: Crash reports (on failure) # A plugin that segfaults in vspipe shows only "signal 11" in the test @@ -204,12 +200,6 @@ jobs: cp "$(dirname "$CLI")"/*.exe addons/whisper/ 2>/dev/null || true cp "$(dirname "$CLI")"/*.dll addons/whisper/ 2>/dev/null || true - - name: Provision Vulkan (RIFE, AI upscaling) - # Lavapipe on the GPU-less Linux/Windows runners; sets - # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. - shell: bash - run: bash .github/scripts/provision-vulkan.sh windows-x64 - - name: Build worker (debug) run: cd worker && cargo build @@ -224,7 +214,8 @@ jobs: # `flutter test`, where the running executable is the test runner. env: VAPOURBOX_WORKER: ${{ github.workspace }}\worker\target\debug\vapourbox-worker.exe - run: cd app && flutter test --tags heavy + # Not @Tags(['gpu']): the hosted runners have no GPU (dart_test.yaml). + run: cd app && flutter test --tags heavy --exclude-tags gpu linux: if: ${{ github.event_name == 'schedule' || inputs.platform == 'all' || inputs.platform == 'linux' }} @@ -294,11 +285,6 @@ jobs: ln -sfn "${{ github.workspace }}/deps" "$HOME/.local/share/VapourBox/deps" ln -sfn "${{ github.workspace }}/addons" "$HOME/.local/share/VapourBox/addons" - - name: Provision Vulkan (RIFE, AI upscaling) - # Lavapipe on the GPU-less Linux/Windows runners; sets - # VAPOURBOX_EXPECT_VULKAN=1 so the GPU tests must run, not skip. - run: bash .github/scripts/provision-vulkan.sh linux-x64 - - name: Build worker (debug) run: cd worker && cargo build @@ -316,4 +302,5 @@ jobs: # found" and prove nothing. env: VAPOURBOX_WORKER: ${{ github.workspace }}/worker/target/debug/vapourbox-worker - run: cd app && flutter test --tags heavy + # Not @Tags(['gpu']): the hosted runners have no GPU (dart_test.yaml). + run: cd app && flutter test --tags heavy --exclude-tags gpu diff --git a/.gitignore b/.gitignore index c37ace7f..f37d339e 100644 --- a/.gitignore +++ b/.gitignore @@ -117,7 +117,7 @@ BundledDependencies/Scripts/*.py !BundledDependencies/Scripts/.gitkeep # Add-ons (whisper binary + models, downloaded at runtime) -addons/ +/addons/ # Claude Code .claude/ @@ -127,3 +127,6 @@ tmpclaude-* licenseplan.md *.mkv !Tests/TestResources/soft_telecine_test.mkv + +# Locally converted add-on model files awaiting upload (see docs/ADDONS.md) +addon-staging/ diff --git a/CLAUDE.md b/CLAUDE.md index 03721cb9..fac29752 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -71,6 +71,7 @@ VapourBox/ │ │ └── widgets/ # Reusable UI widgets │ ├── assets/filters/ # Built-in filter schemas (JSON) │ │ └── core/ # Core filters (deinterlace, denoise, etc.) +│ ├── assets/addons/ # Add-on manifests, one JSON per add-on (docs/ADDONS.md) │ ├── macos/ # macOS platform config │ ├── windows/ # Windows platform config │ └── linux/ # Linux platform config @@ -95,9 +96,10 @@ VapourBox/ │ └── linux-arm64/ │ ├── licenses/ # GPL, LGPL, NOTICES -├── docs/ # BUILDING, FILTER_SCHEMA, ENGINEERING_NOTES, … +├── docs/ # BUILDING, FILTER_SCHEMA, ADDONS, ENGINEERING_NOTES, … │ └── reports/ # Investigation reports (evidence behind decisions) ├── Scripts/ # Build, package, and release scripts +│ └── models/ # Model conversion scripts for self-hosted add-on files └── .github/workflows/ # CI build workflows ``` @@ -115,11 +117,22 @@ VapourBox/ | `worker/src/cpu.rs` | The x86 deps CPU tier (`v3`/`v2`) — the single decision; see "CPU tiers" | | `worker/src/source_decode.rs` | Decoder ffmpeg args for encode + preview: full stored frame, size-guarded (see "Source Frame Size") | | `worker/src/source_timing.rs` | A variable-frame-rate source's stored frames placed on the constant-rate grid (see "Variable Frame Rate Sources") | -| `worker/src/gpu_plugin.rs` | The ncnn-on-Vulkan plugins (RIFE, AI upscaling): file layout, the `--probe-rife` / `--probe-upscale` probe, and the user-facing reason/remedy when they can't run (see "GPU Plugins") | +| `worker/src/gpu_plugin.rs` | The ncnn-on-Vulkan plugins (RIFE, AI upscaling): file layout, RIFE's model add-on, the `--probe-rife` / `--probe-upscale` probe (on stand-in models, no add-on needed), and the user-facing reason/remedy when they can't run (see "GPU Plugins") | | `worker/src/animejanai.rs` | AnimeJaNai's add-on models: where they live, which one a job runs, the refusal when they're missing (see "Add-ons") | +| `worker/src/upscale_addons.rs` | The manifest add-on upscalers (Real-ESRGAN anime, Real-CUGAN, Real-ESRGAN General, Live Action SPAN, Real-ESRGAN x4plus/x2plus): which add-on holds which model, each network's own factor (see "GPU Plugins") | +| `worker/src/addons.rs` | Manifest add-ons (compiled in from `app/assets/addons/` by `worker/build.rs`): install state, `require`, `addon_dir` (see "Add-ons", docs/ADDONS.md) | +| `worker/src/ai_runtime.rs` | The AI runtime add-on's `--probe-ai-runtime`: runs the self-test model through `templates/ai_runtime.py`, classifies failures | +| `worker/templates/ai_runtime.py` | ONNX Runtime helpers for passes: `setup`, `session` (provider choice), `map_frames`, NCHW frame conversion, `to_rgbs`/`from_rgbs`, `shifted`, `FrameMemo`, `Runner` | +| `worker/templates/face_restore.py` | The Face Restoration pass: RetinaFace decode, FFHQ alignment, restore, BiSeNet mask, temporal smoothing, bit-exact merge (see "Face Restoration") | +| `worker/src/models/face_restore_parameters.rs` | Face Restoration's parameters, its add-ons per model, and its temporal reach | +| `worker/templates/colorize.py` | The Colourisation pass: model run at a fixed square size, temporal smoothing, Lab/RGB to chroma, luma kept bit-exact (see "Colourisation") | +| `worker/src/models/colorize_parameters.rs` | Colourisation's parameters, its add-on per model, render size and temporal reach | | `worker/src/comment_metadata.rs` | The source's comment tag, prefixed with the version and passed to ffmpeg as an FFMETADATA file (issue #138) | | `worker/src/hdr_metadata.rs` | HDR10 mastering display / MaxCLL read from the source for x265, and the Dolby Vision profile (see "HDR Sources and Dolby Vision") | | `worker/src/native_res.rs` | `--detect-native`: the resolution and kernel a source was upscaled from, for the Descale pass (see "Descale") | +| `worker/src/crop_detect.rs` | `--detect-crop`: black borders, blanking, letterbox bars and the head-switching band, for Crop & Resize's Detect (see "Crop Detection"); also the detectors' shared sampling and decode (`decode_samples`) | +| `worker/src/noise_halo.rs` | `--detect-noise-halo`: a source's noise sigma and halo score, for the pass list's suggestions (see "Noise and Halo Scores") | +| `worker/src/audio_restoration.rs` | Audio restoration: probes, the one `AudioPlan` (every `-af` filter, in order), the render before the encode, and the encoder's audio map/codec args (see "Audio Restoration") | | `worker/templates/pipeline_template.vpy` | VapourSynth script template | | `worker/tests/filter_integration_test.rs` | Filter integration tests | @@ -135,7 +148,15 @@ VapourBox/ | `app/lib/services/temp_directory_service.dart` | Configurable scratch-file directory (see "Temporary Files Directory") | | `app/lib/services/overwrite_behavior_service.dart` | Default action for existing output files (see "Existing Output Files") | | `app/lib/models/native_res_report.dart` | What `--detect-native` found, as the Descale panel's Detect shows and applies it | +| `app/lib/models/crop_detect_report.dart` | What `--detect-crop` found, and which suggestion (`CropApply`) the panel applies | +| `app/lib/models/noise_halo_report.dart` | What `--detect-noise-halo` measured; stored on `QueueItem.noiseHalo`, read by `relevanceFor` | +| `app/lib/views/pass_settings/crop_detect_panel.dart` | Crop & Resize's Detect borders panel: findings, Apply crop / Crop the bars too / Repair the band instead | | `app/lib/services/addon_service.dart` | Every optional third-party download (Whisper, AnimeJaNai): catalogue, install with requirements, remove with dependants (see "Add-ons") | +| `app/lib/services/manifest_addon.dart` | `AddonManifest` + `ManifestAddonInstaller`: the generic installer for `app/assets/addons/*.json` (docs/ADDONS.md) | +| `app/lib/models/audio_restoration_parameters.dart` | The Audio Restoration pass (`ProcessingPipeline.audio`, `PassType.audioRestoration`) and the DeepFilterNet add-on prompt | +| `app/lib/models/ai_runtime_status.dart` | The worker's `--probe-ai-runtime` result | +| `app/lib/models/face_restore_parameters.dart` | Face Restoration's parameters and the add-ons each model needs (`faceRestoreAddonIds`) | +| `app/lib/models/colorize_parameters.dart` | Colourisation's parameters and the add-on each model needs (`colorizeAddonSelectedBy`) | | `app/lib/models/hdr_display.dart` | The display tone-map for the app's own source decodes (twin of `ColorMetadata::display_tonemap_filter`) and Dolby Vision detection | | `app/lib/models/gpu_plugin_status.dart` | The worker's GPU-plugin probe results, and the panels' notice decision | | `app/assets/filters/core/*.json` | Built-in filter schema definitions | @@ -474,8 +495,8 @@ tried again — is in the renderer's header comment. **Name presets after the source, not the technique.** The user knows they captured a DV tape; they don't know it wants SMDegrain with a chroma-bleed fix. That is why the set includes `DV Camcorder Tape`, `PAL DVD / Broadcast`, -`Anime DVD`, `8mm / Super 8 Film Scan` and `Low-Light Phone / Camera Video` -alongside the three quality tiers. +`Anime DVD`, `8mm / Super 8 Film Scan`, `Low-Light Phone / Camera Video` and +`SD Tape → HD (YouTube / TV)` alongside the three quality tiers. **Set `category`.** `PresetCategory.quality` for a speed/quality tier, `PresetCategory.source` for one shaped around a kind of source; the menu in @@ -683,19 +704,27 @@ default pipeline must be completely silent — both are asserted in `pass_advice_test.dart` / `pass_relevance_test.dart`. **Subtitles: transcribe first, mux last.** The pipeline order is -`transcribe the source -> encode (burning in if asked) -> mux as a post-pass`, -and it is load-bearing: burn-in needs the transcript before the encoder runs; -muxing needs the encoded file to exist first. Transcription must run against the -same trimmed audio window the encode uses (`extract_audio_range`), or cues drift -by exactly the trimmed-off head — this is safe only because **nothing in the -pipeline retimes audio**; a future audio-retiming pass would need to account for -it. `SubtitleOutput::{burns_in, muxes, keeps_srt_file}` are asserted to cover -every mode, so a mode that does none of the three can't silently produce nothing. +`render restored audio (if on) -> transcribe -> encode (burning in if asked) -> +mux as a post-pass`, and it is load-bearing: burn-in needs the transcript before +the encoder runs; muxing needs the encoded file to exist first. **Transcription +must hear exactly the output's soundtrack.** With Audio Restoration on, that is +the rendered track (`AudioRestorer::render`), which is the output's audio +sample for sample — trimmed, offset, drift-corrected — so cues follow any +retiming by construction (`integration_audio_restoration_test.dart` checks a +2 s offset moves the cues ~2 s). Without it, transcription reads the same +trimmed window the encode uses (`extract_audio_range`), or cues drift by exactly +the trimmed-off head; that is safe because nothing *else* retimes audio (IVTC +and frame-rate conversion change the video timeline only). Any new step that +moves audio in time belongs in `audio_restoration`, never in the encode. +`SubtitleOutput::{burns_in, muxes, keeps_srt_file}` are asserted to cover every +mode, so a mode that does none of the three can't silently produce nothing. **FFmpeg's `-vf` chain accumulates into one `Vec` joined with commas — never push a second bare `-vf`.** ffmpeg only honours the *last* `-vf`, so two separate pushes (e.g. aspect stamping and subtitle burn-in) would silently drop -the first one's effect. +the first one's effect. The same goes for `-af`: every audio filter is chosen in +`AudioPlan::new` and each ffmpeg run gets exactly one `-af`; the encode itself +never filters audio (`test_real_ffmpeg_builder_delegates_audio_to_audio_restoration`). **Nothing of unbounded length goes on a child process's command line.** Windows caps it at 32,767 characters and refuses to start the process (os @@ -857,6 +886,34 @@ The preview path needs the same source tags fed to swscale (`swscale_input_opts()`) or it guesses the matrix and shows a colour shift the real encode doesn't have. +**BT.601 → BT.709 on SD → HD (app 1.7.0) is the one place the script +converts colour.** Crop & Resize's `convertToBt709` (`Bt709Conversion`: +`auto` default, `on`, `off`) converts a BT.601 source (tagged +`bt470bg`/`smpte170m`, or untagged below 720 lines) when the output is HD +(≥ 720 lines or ≥ 1280 wide, from `CropResizeParameters::output_size`, which +mirrors the script's fitting). Rules: + +- **One decision: `VideoJob::converts_to_bt709`.** Everything that describes + what the script DELIVERS uses `VideoJob::output_color_metadata()`, never + `color_metadata()`: the encoder's input and output tags, the preview's + swscale step, and the passes after Crop & Resize (Face Restoration, RIFE, + the border fill). Only the AI upscale, which runs inside the resize before + the conversion, keeps the source's. A missed site relabels the converted + samples back to 601 — a silent colour shift. +- **The matrix only.** Output tags say `bt709` for matrix and transfer; the + primaries keep the source's (tagged, or EBU/SMPTE C by height), because the + samples were not gamut-converted. A gamut conversion moved a 75% green by + 55 levels on a player that does not colour-manage + (docs/reports/2026-11-crop-detect-709). +- **R78's resizers let frame properties override `*_in_s` arguments**, so the + `{{#COLOR_709}}` block (both templates, right after `{{/RESIZE}}`) removes + `_Matrix`/`_Range` first. Any new block that converts colour with explicit + `*_in_s` arguments needs the same. +- `integration_bt709_conversion_test.dart` (heavy) checks the pixels: every + patch within 3 levels (measured ≤ 0.67) of the source decoded as 601, + through the tags and when read as 709 regardless, preview = encode, and Off + stays 601 and bit-identical in YUV. + ### HDR Sources and Dolby Vision (issue #133) An HDR source is processed and encoded as HDR; only what the user *sees* is @@ -981,7 +1038,7 @@ A >8-bit source is read natively off the pipe. Two ways this breaks after that: computed **in the script from `clip.format`**, since a preceding pass may have changed the depth. 3. **A limit counted per histogram bin scales with the CONTENT's depth, not - the container's.** `vszip.CLAHE` (8/16-bit only, so deeper clips run at + the container's.** `vszip.CLAHE` (8/16-bit only before vszip 23; the template still runs deeper clips at 16-bit) needs `limit << (16 - content_depth)`: 8-bit content in a 16-bit histogram fills every 256th bin, genuine 10-bit every 64th, and ProRes 4444 decodes as 12-bit whatever it holds. The template measures the content depth @@ -1202,8 +1259,20 @@ usable Vulkan implementation ncnn can segfault** (RIFE does, in `ncnn::create_gpu_instance`; VSNCNN raises "get_gpu_device failed") rather than fail cleanly. -- **Never autoloaded.** RIFE and its model live in `vapoursynth/rife/`, VSNCNN - and its models in `vapoursynth/ncnn/`, and on macOS the MoltenVK both use in +- **Plugins in the bundle, models in add-ons (deps 1.16.0).** RIFE's model + is the `frame-rate-rife` add-on (upstream's ncnn conversion, fetched from + the plugin's repo at a pinned commit, installed under + `rife-v4.26_ensembleFalse/` because the plugin reads the model version — and + its padding — from substrings of that path); Real-ESRGAN anime and + Real-CUGAN are `upscale-anime` (vs-mlrt's ONNX files, byte for byte, which + we host because vs-mlrt publishes them only in an ~810 MB archive). Both are + declared in `ProcessingPipeline::required_addons()` and refused through + `addons::require` like every add-on; the panels prompt via + `modelAddonSelectedBy`, on choosing the method and on switching the pass + on. `attribution_test.dart` fails if the bundle lists a model or a + `download-deps-*` script fetches one. +- **Never autoloaded.** RIFE lives in `vapoursynth/rife/`, VSNCNN in + `vapoursynth/ncnn/`, and on macOS the MoltenVK both use in `vapoursynth/moltenvk/` — all outside the plugin directory. The templates `LoadPlugin` them inside `{{#FRAME_RATE_RIFE}}` / `{{#UPSCALE_AI}}` only, so a missing runtime DLL, a broken driver or AVX in a static initializer can @@ -1213,7 +1282,13 @@ than fail cleanly. first call: both dylibs leave `vkGetInstanceProcAddr` to whatever is already in the process. - **One probe, one wording.** `gpu_plugin.rs` prechecks the platform and files, - then runs the plugin once in a throwaway vspipe, printing stage markers so a + then runs the plugin once in a throwaway vspipe on a **stand-in network of + its own** — never an add-on, so "can't run here" is known before anything + is downloaded: VSNCNN gets `NCNN_SELFTEST_MODEL` (264 bytes, a 1x1 conv, + `Scripts/models/make_ncnn_selftest.py`) as bytes + (`path_is_serialization=True`), RIFE a two-layer weightless `flownet` with + its v4 blobs written beside the script — and checks the answer (a wrong one + is `WrongResult`). It prints stage markers so a silent crash still says where it died, and classifies the result into a `GpuPluginReason` with a user-facing remedy (`GpuPluginStatus::message`, naming the feature). The app shows that text the moment the feature is @@ -1248,8 +1323,9 @@ than fail cleanly. mistakes on one clip but missed cuts between dark shots; switching needs more footage first. See docs/ENGINEERING_NOTES.md (2026-11-03). - **Upscaling: models, factors, tiles.** Five of vs-mlrt's ONNX conversions - ship (`gpu_plugin::UPSCALE_MODELS`): Real-ESRGAN animevideo 2x/4x and - Real-CUGAN 2x at three strengths. Factors follow the edge-directed path's + are the `upscale-anime` add-on (`upscale_addons::MODELS`, files prefixed + `upscale-`): Real-ESRGAN animevideo 2x/4x and Real-CUGAN 2x at three + strengths. Factors follow the edge-directed path's rule (3x behaves as 2x); CUGAN 4x is its 2x model twice, denoising on the first pass only (`CropResizeParameters::ai_upscale_models`). Above SD the frame runs in 512-px tiles with 16 px overlap — whole-frame CUGAN at 1080p @@ -1266,32 +1342,64 @@ than fail cleanly. `ensure_methods_ready` also refuses it when `animejanai::runtime` finds the models missing (`MISSING_MESSAGE`). It keeps flat colour within 0.5 levels (Real-ESRGAN: up to 11). +- **Live-action models are manifest add-ons on the same block** (app 1.7.0, + `upscale_addons.rs`): `upscale-general` — Real-ESRGAN General + (`realesr-general-x4v3`, optional denoise variant `-wdn-`) and jcj83429's + 2xLiveActionV1 SPAN (Apache-2.0, upstream's ONNX pinned at a commit) — and + `upscale-esrgan-plus` — Real-ESRGAN x4plus/x2plus. Real-ESRGAN publishes + `.pth` only: `Scripts/models/convert_realesrgan.py` converts (opset 13, + x2plus's `pixel_unshuffle` as a one-hot stride-2 conv, which ncnn imports) + and we host the ONNX on `addon-models-v1`. They run on VSNCNN, so they need + no `ai-runtime` and exist exactly where VSNCNN ships (asserted). Declared in + `required_addons()` via `UpscaleMethod::addon`; the panel prompts with + `upscaleAddonSelectedBy`. **Factor rules:** General is 4x only, so 2x runs + 4x and resamples down (`{{#AI_RESAMPLE}}`, Spline36 in float RGB — as + upstream's `--outscale`); SPAN is 2x only, 4x runs it twice; x2plus/x4plus + by factor. **x4plus runs in 384-px tiles above 384x384** (`ai_tiling`): + whole-frame at 720x576 lost the Vulkan device on a 16 GB Mac and 512 tiles + thrashed (29 s a frame); ~7 s a DVD frame at 4x. **A tile never exceeds + the frame** (`min(tile, width/height)`, every method): a tile larger than + the frame in either direction segfaults VSNCNN, and a wide, short picture + passes the area test. Flat-colour drift vs + Spline36: General 2.9, SPAN 0.6 (2x) / 1.3 (4x), x2plus 4.2, x4plus 12.2. + 2xVHS2HD (RealPLKSR) was probed and **rejected**: ncnn's importer has no + ScatterND/Shape/Range/Where in its ONNX (see + docs/reports/2026-11-ai-upscale-addons). +- **AI strength** (`aiStrength`, 0-1, default 1) applies to every AI method: + one `std.Merge` of a Spline36 upscale of the same frame with the network's + result, at the clip's format, before the props copy (`{{#AI_STRENGTH}}`). + At 1.0 the block is removed and the script is byte-identical to one without + the field (`test_188`); 0.5 halves a model's tint (11.2 → 5.8 levels). - **VSNCNN is built from source on macOS and Linux** (only Windows has a published build): static protobuf v3.21.12 and onnx at vs-mlrt's pinned commit, Tencent's prebuilt ncnn 20260526, VapourSynth's API 3 headers from - the R78 tag. Models come from vs-mlrt's ~810 MB `models.vX.7z` (the only - place they are published), extracted in one pass and each checked by hash. - Every pin and hash is in all three `download-deps-*` scripts. -- **Tested for real in CI.** The hosted runners have no GPU, so - `.github/scripts/provision-vulkan.sh` gives Linux and Windows Mesa's - lavapipe (a CPU Vulkan driver ncnn accepts) and sets - `VAPOURBOX_EXPECT_VULKAN=1`. **macOS arm64 gets nothing and its GPU tests - skip**: the runners are VMs whose Metal device MoltenVK cannot open, and - lavapipe crashes there because the bundle's akarin exports its static - LLVM's symbols, which dyld then binds lavapipe's own LLVM to (see - docs/ENGINEERING_NOTES.md). MoltenVK is tested on real Macs only — run - the GPU suites locally with `VAPOURBOX_EXPECT_VULKAN=1` after touching it. - Under that variable every GPU test - **fails** rather than skips if a feature reports itself unavailable + the R78 tag. Every pin is in all three `download-deps-*` scripts, which + install no model. +- **Tests that need a Vulkan device are tagged `gpu`, and CI leaves them + out.** The hosted runners have no GPU, so the push gate runs + `--exclude-tags heavy,gpu` and nightly `--tags heavy --exclude-tags + gpu` (`app/dart_test.yaml`). Without a device CI still checks that each + probe answers with a real reason (not `probeFailed`) and that a job that + cannot run is refused with the probe's wording. **Run the `gpu` tests on a + machine with a GPU after touching RIFE, VSNCNN, MoltenVK, the upscale + block or a model add-on**, with `VAPOURBOX_EXPECT_VULKAN=1` + (`flutter test --tags gpu`, plus `--tags heavy` for the encodes): under + that variable an unavailable feature **fails** rather than skips (`app/test/support/gpu_expectations.dart`), so a broken bundle cannot hide - behind "no GPU". `integration_gpu_plugins_test.dart` (push gate) probes, - checks the refusal and the script, and renders a real frame per feature; + behind "no GPU". Nothing else proves the Windows and Linux bundles' GPU + plugins actually run, so do it on those platforms before a deps or app + release too. Any test that runs a network through ncnn gets the tag (a + `group` cannot take tags in flutter_test, so tag each `test`). + `integration_gpu_plugins_test.dart` probes, checks the refusal and the + script, and renders a real frame per feature; `integration_ai_upscale_test.dart` and the RIFE group of `integration_frame_mapping_test.dart` (heavy) encode and check pixels — the upscalers against Spline36 on a synthetic pattern (colour within 2 levels for - CUGAN 2x, bar centre within 0.75 px for all). + CUGAN 2x, bar centre within 0.75 px for all). The ONNX Runtime passes (Face + Restoration, Colourisation) are not `gpu`: they run on the CPU provider + where there is no GPU, which is a path users take. -See docs/ENGINEERING_NOTES.md (2026-10-24, 2026-10-25) for the measurements. +See docs/ENGINEERING_NOTES.md (2026-10-24, 2026-10-25, 2026-11-09) for the measurements. ### Descale @@ -1327,6 +1435,76 @@ keeps chroma and the source size. Rules: `integration_descale_test.dart` (heavy) checks detection on known masters, protection (grid exact vs. destroyed without the mask) and preview = encode. +### Crop Detection (`--detect-crop`, app 1.7.0) + +Crop & Resize's **Detect borders** (`CropDetectPanel`) runs +`crop_detect::detect`: 24 frames across the source or its trim, decoded as +8-bit 4:4:4 and measured line by line in plain Rust (no VapourSynth). It is +advisory exactly like Descale's Detect: nothing changes until the user picks +Apply crop, Crop the bars too, or Repair the band instead. Rules: + +- **Brightness is a high percentile over every usable frame**, so a line that + carries picture in any frame is picture; a border line must also be flat + (detail floor 6 levels, so noisy blanking still counts). Frames too dark to + judge (central p90 < 48) are left out of everything. One line counts only if + it is true black in every frame (a capture card's first line). +- **Bars wider than 5% of the dimension are framing** (letterbox / pillarbox): + reported, in `cropWithBars`, never in the recommended `crop`. +- **Head switching**: a bottom line torn against the line above *in the same + field* (d = 2 when the job's field order is TFF/BFF), in luma or chroma + across ≥ 6 of 8 width segments, or shifted sideways by ≥ 3 px; in ≥ 50% of + usable frames; touching the bottom (≤ 2 lines skipped). Breadth and + persistence are what keep a burnt-in subtitle out. +- **Rounded outward to steps the Crop pass accepts**, in the Crop pass's + orientation (`orient` maps through Rotate / Flip): even everywhere, fours + vertically when a 4:2:0 picture is still interlaced at the crop. Edge + Repair's alternative stays in source orientation and is offered only up to + its panel's 16-line maximum. +- `confidence` is the share of usable frames whose picture reaches the edge; + low means thin evidence (a dark film), not a wrong edge. The panel warns + below 0.25. +- `integration_crop_detect_test.dart` (heavy) builds fixtures from real VHS + picture with known borders, a band, a letterbox, subtitles and fades, and + asserts every edge within 2 lines; the unit tests in `crop_detect.rs` cover + the same traps on synthetic frames. + +### Noise and Halo Scores (`--detect-noise-halo`, app 1.7.0) + +The pass list badges Noise Reduction and Dehalo as Suggested only from a +measurement of the picture (`noise_halo::detect`), run by +`MainViewModel._measureNoiseHalo` in the background once a source is +analysed, one source at a time, and stored on `QueueItem.noiseHalo`. Rules, +from docs/reports/2026-11-noise-halo-scores: + +- **The worker decides.** `NOISE_HIGH` (3.5, sigma in 8-bit levels) and + `HALO_HIGH` (6.0, % of the step) live in `noise_halo.rs`; the report's + `noiseHigh` / `haloHigh` are the verdicts, and `relevanceFor` reads only + those, never the numbers. Change a threshold only with the report's scripts + re-run (`probelib.final_*` mirrors the Rust; `worker_parity.py` checks them + against each other; no verdict may differ). +- **Suggest only; a low or missing score is silent.** Both measures + under-report (codec-skipped blocks read noiseless, sharpening of soft edges + rings past the plateau zone), so a low score is not evidence and nothing is + ever `notApplicable` from it. A failed measurement suggests nothing and + logs nothing. +- **Noise is temporal, not spatial.** Motion-compensated frame difference + (16x16 blocks, +-10 px, match chosen on smoothed pictures), 10th percentile + of blocks, 25th percentile of 24 sampled pairs (12 above 1080p). A spatial + high-pass (Immerkaer, signalstats) is blind to tape-like and grain noise and + reads dense texture as noise; VCG's TOUT does not respond to noise at all. +- **Halo is per edge and two-sided.** Step edges followed over 7 lines of one + field, plateaus 6-10 px out; each edge scores min(overshoot, undershoot), so + a cartoon outline scores 0. Fewer than 200 edges is no score. +- **Per field unless known progressive** (`ScanType.progressive` / + `softTelecine`), so combing never reads as noise; **luma in the source's own + format and depth** (a yuv -> gray decode expands the range), in 8-bit units. +- **The whole source is sampled, not the trim**, so the badge doesn't change + as in / out points move. Uses `crop_detect::decode_samples`, so all + detectors sample and decode alike. + +`integration_noise_halo_test.dart` (heavy) checks real clips: the VHS capture +clean / with sigma 6 noise (8- and 10-bit agree), DVB-T clean / sharpened. + ### Add-ons: optional third-party downloads Anything VapourBox can use but does not ship — because of its size (Whisper) @@ -1357,6 +1535,215 @@ by `AddonService.addonsDir`), only when the user asks. Rules: - **Tested against the real download**: `WorkerHarness.ensureAnimeJanai()` installs from the pinned URL with the app's own extraction, so CI fails if the link or hashes stop holding. +- **Every AI model is an add-on, never in the deps bundle** — no exceptions + since deps 1.16.0, when RIFE's and the anime upscalers' models moved out + (`attribution_test.dart` guards it) — and every new + add-on is a **manifest add-on**: one `app/assets/addons/.json` + (format, checklist and hosting: **[docs/ADDONS.md](docs/ADDONS.md)**), + installed by the generic `ManifestAddonInstaller` + (`app/lib/services/manifest_addon.dart`) and compiled into the worker by + `worker/build.rs` (`worker/src/addons.rs`), so both sides read the same + file. Kinds: `files` (model pack, archive members hash-checked one by one) + and `python` (wheels unpacked into `addons//site-packages/`). An install + is staged in `.new` and ends with the `.vapourbox-addon.json` marker + (id, version, platform); a manifest `version` bump makes the old install + count as missing. Whisper and AnimeJaNai keep their hand-written installers. + A plain file marked `"executable": true` is a program the worker runs + (DeepFilterNet): the installer sets its executable bit and, on macOS, signs + it ad hoc if `codesign --verify` fails (Apple Silicon will not run an + unsigned arm64 binary); the worker finds it with `Manifest::executable_for`. +- **A pass declares its add-ons in `ProcessingPipeline::required_addons()`** + (`AddonNeed { feature, ids }`); `ScriptGenerator::ensure_methods_ready` + refuses the job via `addons::require` (naming them and Settings → Add-ons), + and emits the `{{#AI_RUNTIME}}` block (both templates) when one needs + `ai-runtime`. +- **The AI runtime (`ai-runtime`, python kind)** is ONNX Runtime + NumPy for + the deps Python 3.12: onnxruntime 1.31.1 (macOS arm64, Linux), + onnxruntime-directml 1.24.4 (Windows), 1.23.2 on Intel Macs (`minOs` 13.0 — + the last Intel wheel). Passes use `templates/ai_runtime.py` (`setup`, + `session`, `map_frames`, NCHW helpers): it puts `site-packages` on + `sys.path` itself because **Windows' embeddable Python ignores + PYTHONPATH**. `--probe-ai-runtime` runs a committed self-test model the way + a job would (`ai_runtime.rs`, positive results cached per boot); + `VAPOURBOX_AI_FORCE_CPU=1` forces the CPU provider. +- **`VAPOURBOX_ADDON_MIRROR=`** makes the installer (app and + `WorkerHarness.ensureAddon`) take files from a local directory, still size- + and hash-checked — how an add-on is tested before its files are uploaded. + Files we convert ourselves are staged in `addon-staging/` (gitignored, with + the conversion venv in `addon-staging/convert-venv`) and hosted on the + `addon-models-v1` release (not Latest), which only the owner uploads to, + each beside its upstream licence text (`LICENSE-.txt`). While a + release is in development they may sit on a deps release candidate instead; + `release.sh` refuses to cut an app release while any manifest names an + `-rc` tag (docs/ADDONS.md, "Hosting"). +- **Tested for real**: `WorkerHarness.ensureAddon(id)` installs an add-on and + its requirements with the app's own installer; `integration_ai_runtime_test.dart` + (heavy) installs `ai-runtime` and runs the probe. `manifest_addon_test.dart` + asserts the worker (`--list-addons`) compiled in the same manifests. + +### Face Restoration + +The Face Restoration pass (`FaceRestoreParameters`, `PassType::FaceRestore`, +app 1.7.0) finds faces with RetinaFace, aligns each to the FFHQ 512 template, +restores it with RestoreFormer++ (default), GFPGAN v1.4 or CodeFormer, masks +it to the face itself with BiSeNet and blends it back +(`worker/templates/face_restore.py`, on `ai_runtime`). Rules: + +- **Models are add-ons, converted by committed scripts.** `face-restoration` + (RetinaFace R50 + BiSeNet + RestoreFormer++, the base), `face-gfpgan` and + `face-codeformer` (each `requires` the base; both non-commercial, with a + `usageNote`). `Scripts/models/make_face_*.py` reproduce the ONNX files from + upstream checkpoints pinned by hash; files are hosted on `addon-models-v1`. + Static shapes (detector 640x640 padded, 512x512 faces) so Core ML takes the + whole graph. GFPGAN is exported with its **fixed noise buffers** + (upstream draws new noise per call: flicker by construction); CodeFormer's + `w` is a graph **input** (exact at w = 0). Never ParseNet (CC BY-NC-SA, from + PSFRGAN) or InsightFace's detectors (non-commercial), whatever upstreams + default to. +- **After Crop & Resize, before Sharpen** — restored at the delivered + resolution; first row of the Finishing stage. +- **A pure function of neighbouring frames, never stateful.** Neighbours come + through `ai_runtime.shifted` clips into `std.ModifyFrame`; shared work + (detections, each frame's restored faces) through `ai_runtime.FrameMemo`. + Temporal smoothing (radius 1): landmarks averaged with hysteresis and + one-frame gap filling, and each restored face averaged in aligned space with + its neighbours', weighted down where the *input* moved. The preview's + `FrameMap` radius is **twice** that (`frames_read_either_side`): the blend + reads a neighbour, whose landmarks read one more. +- **Bit-exact outside the faces**: the restored RGB frame is converted back + to the source format and composited with `std.MaskedMerge` (mask luma-sized, + `first_plane=True`), so a zero mask leaves every plane untouched. Never blend + in RGB and convert the whole frame back. +- **Model runs are serialised per pass** (`ai_runtime.Runner` with one shared + lock), with a CPU retry if a GPU run fails: concurrent Core ML predictions + from VapourSynth's threads ran a 16 GB Mac out of GPU memory. +- **NumPy only** — no OpenCV (it is in neither the deps bundle nor the + ai-runtime add-on); `pipeline_parity.py` in the report checks the NumPy + detection, alignment and paste-back against facexlib/OpenCV. +- The panel prompts for the model's add-ons (`AddonPrompt.ensure`) when the + pass is turned on or the model changes, and shows the non-commercial note + for GFPGAN/CodeFormer; the worker refuses through `required_addons()`. + +`integration_face_restore_test.dart` (heavy) checks the face changes and every +sample outside it is bit-exact (8- and 10-bit), each model runs, and preview = +encode. See docs/ENGINEERING_NOTES.md (2026-11-08) and +docs/reports/2026-11-face-restoration. + +### Colourisation + +The Colourisation pass (`ColorizeParameters`, `PassType::Colorize`, app 1.7.0) +colours black-and-white footage with DDColor-T (default) or DeOldify's Video +model (`worker/templates/colorize.py`, on `ai_runtime`). Rules: + +- **The luma is never touched.** The model's colour (DDColor: CIELAB a/b, + recombined with the frame's own lightness by a chain of `std.Expr` stages; + DeOldify: RGB) becomes YUV with the source's matrix, is converted to the + source format, and only its U and V are taken (`ShufflePlanes` with the + input as plane 0). Y is bit-exact at every depth; strength 0 is the input. + A source that already has colour is coloured from its luma all the same + (its own chroma replaced) — said in the panel's text, never blocked. +- **Between Chroma Fixes and Color Correction** (Detail & Color stage): the + model judges the cleaned-up picture, and the result can be graded. It runs + before Crop & Resize, so it uses the source's colour tags, not the BT.709 + conversion's. +- **Models are add-ons, one per model** (`colorize-ddcolor`, 220 MB, Apache-2.0; + `colorize-deoldify`, 870 MB, MIT; each `requires` `ai-runtime`), converted by + `Scripts/models/make_colorize_{ddcolor,deoldify}.py` byte for byte. DeOldify + is rebuilt in plain PyTorch (fastai v1 does not install with a current + torch) and verified against an independent export. Both exports have + **dynamic** height/width; each session fixes them with ONNX Runtime's + free-dimension overrides (`ai_runtime.session(..., dims=)`), so one file + serves every render size and Core ML still gets static shapes. Core ML takes + each whole graph only because the exports rewrite `ReplicationPad2d`, + `Tensor.repeat` and DDColor's einsum (`colorize_models_common.py`) — without + that DDColor split into six partitions and ran 2.6x slower. +- **Core ML compiles DDColor for 50-80 s per session** (every vspipe process; + DeOldify ~5 s). So the preview template passes `preview=True` and DDColor + previews on the CPU (`colorize.PREVIEW_ON_CPU`); outputs agree to <0.001 + CIELAB units. Don't add `ModelCacheDirectory` casually: its cache is ~900 MB + per render size and the key is the model path, not the overridden shape. +- **Temporal smoothing is a pure function of frames n-r..n+r**: each frame's + render-size prediction is averaged with its neighbours' (`shifted` + + `FrameMemo`), weighted to zero where the grey picture differs + (`MOTION_LIMIT`), so colour follows motion and a cut takes nothing from the + last shot. The `FrameMap` radius is `temporal_radius`. +- No built-in preset turns it on (not even the film-scan one): a download of + 220-870 MB and an invented colour are the user's choice. + +`integration_colorize_test.dart` (heavy) checks luma bit-exact (8- and 10-bit, +and on a colour source), strength 0 = input, both models, and preview = encode. +See docs/ENGINEERING_NOTES.md (2026-11-08) and docs/reports/2026-11-colourisation. + +### Audio Restoration + +The one pass that works on the soundtrack (`AudioRestorationParameters`, +`ProcessingPipeline.audio`, app 1.7.0), in the pass list's "Audio & Subtitles" +stage. **Not a VapourSynth pass**: it is not in `PassType` (Rust) or +`enabled_passes`, never touches either `.vpy` (`test_185`), and the preview is +picture only (the panel says so; its edits don't regenerate the preview). +Rules, from docs/reports/2026-11-audio-restoration: + +- **Rendered before the encode, to a WAV, then mapped by the encoder** — never + an `-af` on the encode. That is what makes DeepFilterNet (a separate + program), two-pass loudness, a negative offset (reading audio from before the + trim point) and transcribing the output's own soundtrack possible, and it + keeps audio independent of how the video encode is run. `RenderedAudio` owns + the scratch directory and is held to the end of the job (after the mux). +- **One decision site, fixed order.** `AudioPlan::new` chooses every filter: + `aresample=first_pts=0` (a stream that starts after the video keeps its gap — + a WAV has no timestamps) → `pan` → `adeclip` → `adeclick` → rumble `highpass` + → hum `bandreject` ×3 → `afftdn` + `atrim` of its delay → drift resample → + lead `adelay` → [DeepFilterNet] → `loudnorm` + `aresample` → `alimiter` → + `apad`+`atrim` to the picture's length. Each ffmpeg run gets one `-af` + (`every_combination_keeps_the_order_and_one_af_per_run`, 512 × 3 cases). +- **Measured traps, each now handled in the plan:** `afftdn` outputs 25 ms + late at every rate (`atrim=start_sample=rate/40`); its default `nf=-50` took + 0.7 dB off −39 dB hiss — it gets the source's measured floor (quietest tenth + of 50 ms windows, clamped −80…−30), and never `tn=1`, which undid it; + `alimiter` needs `latency=1` (else it delays by its attack) and `level=0` + (else it normalises up to 0 dBFS); `loudnorm` resamples to 192 kHz, so an + `aresample` follows it; DeepFilterNet delays 30 ms — `-D` compensates and + shortens the tail, which `apad` restores. +- **Drift is corrected by resampling, not `atempo`.** atempo moved every click + ~20 ms early; drift is a sample-clock error, so resampling restores pitch as + well as length. `asetrate` takes only an integer, so the plan upsamples to an + `r1` in 4–8× the rate where `s / r1` is within ~1e-11 of the factor + (`resample_ratio`). The factor is (whole source's audio length) / (whole + encoded picture's length, `ProcessingPipeline::output_duration_secs` — + output frames over output rate from the `FrameMap`s); applied only when the + difference is > 0.05 s and ≤ the user's cap (0.5% default), skipped within + 1e-6 of 1. +- **Offset and drift compose on the source's timeline**: source time for output + time t is `(start + t − offset) × factor`; a positive offset from the start + is silence (`adelay`), never a negative seek. +- **The window is the encoded picture's** (`OutputWindow::for_job`): from the + trim's in-point, `output_duration_secs` long. The rendered track is exactly + that many samples, so with restoration the output's audio is the picture's + length (passthrough could leave it longer). +- **One stream is restored** (`streamIndex`, default the first). The encoder + maps the rendered track first, then the source's other audio streams + untouched, each with its own tags (`encode_map_args`; naming one stream's + `-map_metadata` switches off ffmpeg's copying for the rest). Passthrough + copies the others but must encode the restored one: FLAC in MKV, AAC at the + chosen quality in MP4/MOV, PCM in AVI (`restored_passthrough_codec`). +- **Automatic choices are measured, never guessed**: a dead channel (≥30 dB + below the other, which is above −60 dBFS) and 50/60 Hz hum (Goertzel over + 2 s blocks, a line ≥12 dB above its neighbours) are read from a minute in + the middle of the window; finding nothing leaves the audio alone and says so. +- **DeepFilterNet is the `deepfilternet` manifest add-on** (upstream + deep-filter 0.5.6 binaries pinned by hash, MIT OR Apache-2.0, `minOs` 13.0 + on macOS since upstream builds target it). The worker checks it in + `AudioRestorer::plan` (`DEEP_FILTER_FEATURE`), **not** in + `required_addons()`: that list gates the script generator, and a preview + must never be refused for an audio add-on. +- **No built-in preset turns it on** (audited when it shipped): every step + depends on the recording, and switching the pass on re-encodes audio that + the presets otherwise copy. A source preset may later enable only the + self-checking steps (automatic channel fix, automatic hum), which do nothing + when they find nothing. +- `vapourbox-worker --audio-plan --config job.json` prints every argument list + without rendering (`integration_audio_plan_test.dart`, push gate); + `integration_audio_restoration_test.dart` (heavy) measures the encoded audio. ### BM3D and KNLMeans Noise Reduction @@ -1397,6 +1784,70 @@ the simple slots are full at four). (`NoiseReductionParameters::temporal_radius`): BM3D's Final step re-reads its basic estimate's neighbours, so it reaches 2r; a fixed 3 was too few. +### SangNom, NLMeans (CPU) and the vs-jetpack ports (deps 1.16.0, app 1.7.0) + +Five classical additions, all in the deps bundle (no add-ons). The evidence is +[docs/reports/2026-11-classical-filters](docs/reports/2026-11-classical-filters/index.html). + +- **SangNom is its own Anti-aliasing method, never santiag's `type`.** + havsfunc's `santiag(type="sangnom")` calls `SangNom(order=0)` at strength + 0, which SangNom r45 treats as double-rate: it refuses a clip without + `_FieldBased` and would double the frame count with it. So + `effective_santiag_type` stays pinned to nnedi3 and `AntiAliasMethod::SangNom` + runs its own block (2x Spline36, `SangNom(order=1)` per axis, back down, + field order restored — SangNom flags its output progressive). `aa` is in + 8-bit units; the plugin scales it (8 vs 10-bit: 0.05 levels). +- **SangNom's licence is split**: the source header and LICENSE grant MIT + (james1201, Victor Efimov) and the same header says "Binaries are GPL v2"; + the PyPI wheels we ship declare GPL-2.0. NOTICES, About and the README say + both. One file per platform from the r45 wheels; x64 is SSE2 only, so no + `_tierFiles` entry. +- **vszip must be >= 24.0.0** — the first release with the CPU `NLMeans` + (WNNM has been there since 22.0.0). `vapoursynth_integration_test` checks + `vszip.NLMeans` itself, not just the namespace, so a stale 22.1.0 bundle + fails. macOS x64 builds the sdist with **Zig 0.17.0** (`-Doptimize=fast`; + 0.17 renamed the modes), and the headerpad patch now matches the + 8-space `lib.use_llvm` inside `if (build_vs)`. vszip 23 made CLAHE accept + every depth; the template still feeds it 16-bit and its parity test holds + (1.34 levels against a 3.0 tolerance). +- **NLMeans (CPU)** (`NoiseReductionMethod::NlMeans`, advanced) shares the + `knlm_*` parameters with KNLMeansCL: same algorithm, same arguments, same + picture (0.000 levels apart on the probe, 0.0003 in the heavy test). It + needs no OpenCL, so `KNLM_NO_DEVICE_MESSAGE` (and the app's fallback text + in `worker_manager.dart`) names it as the way out. +- **WNNM is not a method.** Measured against the shipped BM3D on three Kodak + pans at sigma 4/8/12, each at its best sigma: luma +0.7 to −2.3 dB (ahead + only at sigma 4 with a temporal radius), chroma 2.6–7.2 dB worse. It is + fast (10 fps at 1080p with a temporal radius on an M1 Pro, where the shipped + BM3D path runs 0.2 fps), but NLMeans (CPU) fills that niche with better + chroma (23 fps). + Revisit only with a reason BM3D's quality and NLMeans' speed don't cover. +- **The vs-jetpack ports are template code, not a dependency**: vine_dehalo + (`DehaloMethod::Vine`, advanced), mpeg2stinx (`DeblockMethod::Mpeg2Stinx`, + advanced) and FixInterlacedFades (`QTGMCParameters::ivtc_fix_fades`, an IVTC + option, mode `average`/`match`). Ported from release 3.0.0 (MIT, + Jaded-Encoding-Thaumaturgy) through `_expr()` / `_nnedi3()` (test_92/93), + without akarin's frame-property Expr (macOS x64 has no akarin), and checked + against vs-jetpack itself: mpeg2stinx 0.04 levels, the fade fix exact + (vine could not run there: its scalers need resize2, which we don't ship). +- **mpeg2stinx is a cross-field filter and always takes the whole frame** — + the Deblock pass's per-field path skips it — and reads one frame each side + (`FrameMap::Identity { radius: 1 }`). On every synthetic fixture tried it + *lost* PSNR (it softens vertical detail); the schema says so. It is for the + specific comb-like noise of film DVDs encoded as interlaced MPEG-2. +- **The fade fix runs right after VFM, before VDecimate**, in float. Its + per-frame gains are stamped as frame properties by `ModifyFrame` and applied + by **one** akarin expression (`x x.FadeGain0 *`); only where akarin is + absent (macOS x64) does a FrameEval make a `std.Expr` per frame. **Never + create an akarin node per frame inside concurrent FrameEvals**: the first + version did, and vspipe segfaulted, aborted or spun one core for 97 minutes + on 30 frames — fine single-threaded, and the small heavy test missed it. The + heavy suite now times a full 720x576 IVTC job. `std.DoubleWeave` flags its output + interlaced, so the block sets `_FieldBased` back to 0 — without it a later + resize treats progressive frames as fields. `average` leaves the last frames + of a fade to full black half-lit (one field is already black); `match` + cleans them. + ### Preset System Presets save complete filter pipeline + encoding settings. The built-in presets @@ -1585,7 +2036,7 @@ cd worker && cargo test --test subtitle_integration_test -- --nocapture ### 2. Flutter `test/` suite — `app/test/`, run by `flutter test` ← the CI gate ```bash -cd app && flutter test --exclude-tags heavy # what the push gate runs +cd app && flutter test --exclude-tags heavy,gpu # what the push gate runs cd app && flutter test # everything, incl. heavy (local) ``` @@ -1603,9 +2054,11 @@ Headless Dart-VM tests. Three groups: `addons/`. - **Integration tests** — `integration_*_test.dart`. They spawn the worker end-to-end via `app/test/support/worker_harness.dart`. The encode-heavy ones - are `@Tags(['heavy'])`: the **push gate** runs `--exclude-tags heavy` + are `@Tags(['heavy'])`: the **push gate** runs `--exclude-tags heavy,gpu` (script-only: `integration_filter_parameters_test`, - `integration_qtgmc_parameters_test`); **nightly.yml** runs `--tags heavy`. + `integration_qtgmc_parameters_test`); **nightly.yml** runs `--tags heavy + --exclude-tags gpu`. Tests that need a Vulkan device are tagged `gpu` and + run on no CI runner (see "GPU Plugins: RIFE and AI Upscaling"). `grep -l "Tags(\['heavy'\])" app/test/*.dart` is the authority for which files those are. @@ -1652,12 +2105,12 @@ cd worker && cargo run --release -- --config test_job.json Runs on push to `main` + PRs (skipped for doc-only changes via `paths-ignore`). Each job pulls the published deps bundle + the whisper add-on, then runs -**suite #1 (`cargo test`) and suite #2 (`flutter test --exclude-tags heavy`)** — +**suite #1 (`cargo test`) and suite #2 (`flutter test --exclude-tags heavy,gpu`)** — including the subtitle integration test, excluding the heavy full-encode integration tests. Matrix: macOS **arm64** (`macos-15`), macOS **x64** (`macos-15-intel`), **Windows x64**, **Linux x64** (`ubuntu-24.04`). The heavy full-encode integration tests run separately in `.github/workflows/nightly.yml` -(cron + `workflow_dispatch`) via `flutter test --tags heavy` on the same +(cron + `workflow_dispatch`) via `flutter test --tags heavy --exclude-tags gpu` on the same 4-platform matrix, running each x64 platform against **both** CPU-tier bundles. Fixtures (`small_clip.mp4`, telecine/interlaced clips) are committed under `Tests/TestResources/`. @@ -2190,6 +2643,7 @@ Notes: | `Scripts/package-deps-macos.sh` | Package macOS deps | | `Scripts/package-deps-windows.ps1` | Package Windows deps | | `Scripts/package-deps-linux.sh` | Package Linux deps | +| `Scripts/promote-deps-rc.py` | Re-stamp a tested deps release candidate's assets as the final version: names, `version.json`, delta `baseSha256`, sidecars; verifies every other entry unchanged | | `Scripts/make-deps-delta.py` | Cut an x64 platform's v2 delta from a full v2 build and the v3 bundle (run by the `delta-x64` job of each `build-deps-*` workflow) | | `Scripts/download-deps-linux.sh` | Build Linux deps from source | | `Scripts/run-debug-linux.sh` | Dev build + run for Linux | @@ -2304,4 +2758,5 @@ Full write-ups (root causes, measurements) for each entry are in | 1.12.0 | 2026-10-04 | **Issue #101**: akarin 1.4.1 → **1.5.0** on every platform. The 1.4.1 Linux wheel put JIT code on the `brk` heap, where SELinux denies `PROT_EXEC`, so every job segfaulted on Fedora/RHEL from deps 1.8.0 on. `download-deps-linux.sh` now fails if `libakarin.so` lacks the mmap allocator | | 1.13.0 | 2026-10 | One new plugin; every other binary is built from the same sources as 1.12.0. Adds **vszip** 22.1.0 (MIT) for `vszip.CLAHE`, Color Correction's local contrast (issue #90). Baseline build only, flat in the plugin directory on every platform (the wheels' AVX2/AVX-512 variants are left out); macOS x64 builds the sdist with Zig 0.16.0 for the 12.0 floor, with header padding so it can be signed. The v2 CPU tier is no longer a second full bundle: each x64 platform publishes its v3 bundle plus `…-x64-v2-delta.zip`, the files in `_tierFiles` (zsmooth; on macOS also MVTools), extracted over it (`"tierFormat": "delta"`). Every x64 machine downloads the same bundle; only an older CPU fetches the few MB more. This is also what lets the Linux AppImage carry its deps for both tiers without doubling in size | | 1.14.0 | 2026-10-25 | Adds **RIFE** 4.26 (`r9_mod_v33`, MIT) + its model in `vapoursynth/rife/`, and **VSNCNN** (vs-mlrt v15.16, GPL-3.0) + Real-ESRGAN / Real-CUGAN models in `vapoursynth/ncnn/`, on macos-arm64, windows-x64 and linux-x64, with **MoltenVK** 1.4.2 (Apache-2.0) in `vapoursynth/moltenvk/` on macOS. Not autoloaded; see "GPU Plugins". **BM3D** r12 (MIT) on every platform; it had been macOS arm64 only (see "BM3D and KNLMeans Noise Reduction"). **descale** r12 (MIT) on every platform, for the Descale pass. (AnimeJaNai's models, used by the same VSNCNN, are an app add-on, not in the bundle). Published as the tested `deps-v1.14.0-rc1` bundles with only the version changed; every other file is byte-identical | +| 1.16.0 | 2026-11-09 | **vszip 22.1.0 → 24.0.0** on every platform (CPU `NLMeans` for Noise Reduction's NLMeans (CPU) and the vine dehalo; macOS x64 sdist now built with Zig 0.17.0). Adds **SangNom** r45 (PyPI wheels; MIT source, GPL-2.0 binaries) on every platform for Anti-aliasing's SangNom method. Both single-file on x64 (no AVX), so `_tierFiles` is unchanged. **rc2: the AI models leave the bundle** — RIFE 4.26's (now the `frame-rate-rife` add-on) and the Real-ESRGAN anime / Real-CUGAN ones (now `upscale-anime`); the plugins stay, and their probes run stand-in models. Published as the tested `deps-v1.16.0-rc2` bundles with only the version changed (`Scripts/promote-deps-rc.py`) | | 1.15.0 | 2026-11-03 | **Issue #144**: KNLMeansCL moves to Jaded Encoding Thaumaturgy's fork **v1.3** on every platform (Windows: PyPI wheel, replacing Khanattila's 2018 v1.1.1 zip, which refused 10-bit 4:2:0; macOS/Linux: the `v1.3` tag, replacing unpinned master, which gave unrepeatable results on Apple's OpenCL). Both meson patchers learn v1.3's assigned `run_command` include probe. Published as the tested `deps-v1.15.0-rc1` bundles with only the version changed | diff --git a/README.md b/README.md index 69da48e2..715835ac 100644 --- a/README.md +++ b/README.md @@ -17,14 +17,18 @@ VapourBox runs [VapourSynth](https://www.vapoursynth.com/), QTGMC and FFmpeg for - **Converts and archives.** Reads camcorder DV and AVCHD, MPEG-2 from discs and recorders, MXF, and ordinary MP4/MKV/AVI/MOV files, including formats consumer converters mishandle (NTSC DV's 4:1:1 chroma, 10–16-bit film scans). Writes H.264, H.265, ProRes or lossless FFV1, with hardware encoding where available. DVDs and `VIDEO_TS` folders are read directly, and aspect ratio is carried through correctly. - **Fixes how the picture is stored.** Detects interlaced and telecined sources, then deinterlaces with QTGMC or recovers the original film frames with inverse telecine (or by stripping soft-telecine flags). - **Cleans up old footage.** Optional filters for tape noise, smeared colour, dust and scratches, flicker, ghosting, compression blocking, banding, camera shake and dark, flat pictures. They're off until you turn them on, and the presets turn on what a given kind of source needs. +- **Colours black-and-white footage.** **Colourisation** predicts colour for film scans, old broadcasts and home movies with AI (DDColor or DeOldify Video, each an optional download), steadied across frames, and keeps the brightness of every pixel exactly as it was, so the result can be graded in Color Correction. +- **Restores faces.** **Face Restoration** finds the faces in a soft VHS or low-resolution picture and redraws them with AI (RestoreFormer++, or GFPGAN or CodeFormer for non-commercial use) at the output resolution, steadied across frames to keep flicker down, leaving every other pixel untouched. - **Restores anime.** **Descale** undoes the upscale most HD anime went through in production, **BM3D** cleans flat colour without smearing lines, and **AI upscaling** redraws line art at 2× or 4×: Real-ESRGAN (sharpest, fastest), Real-CUGAN (gentler, with denoising) or AnimeJaNai (trained on HD anime). +- **Upscales live action** with AI models trained on real-world footage, each an optional download: Real-ESRGAN General (fast, with a noise-removing variant), Live Action SPAN (repairs compression, blur and halos on DVD and broadcast sources while keeping colour), or Real-ESRGAN x4plus (most detailed, slow). An **AI Strength** control blends any AI upscale back towards a plain one. - **Converts frame rates** — the broadcast standards or any rate you type in (24 → 48 or 60, say) — with motion interpolation or the **RIFE** AI interpolator. +- **Restores the sound too.** Replaces a dead channel, removes hum, rumble, clicks, clipping and hiss, cleans up speech with the **DeepFilterNet** AI model, puts sound that is out of step or drifting back in sync, and evens out loudness. - **Writes subtitles** from speech with Whisper AI, as an `.srt` file, an embedded track, or burnt into the picture. ## How it works 1. **Add a source.** Drop in a file, a folder, or a `VIDEO_TS` folder, or open a DVD. VapourBox reports what it found (interlaced, telecined or progressive), so you don't have to guess. -2. **Choose the output and any processing.** Pick a format, then start from a preset named after your source (VHS Cleanup, DV Camcorder Tape, Anime DVD, …) or switch filters on yourself. Filters that suit the source are marked **Suggested**. +2. **Choose the output and any processing.** Pick a format, then start from a preset named after your source (VHS Cleanup, DV Camcorder Tape, Anime DVD, …) or switch filters on yourself. Filters that suit the source are marked **Suggested** — Noise Reduction and Dehalo too, when a quick measurement of the picture (run in the background as the video loads) finds it clearly noisy or ringing around its edges. 3. **Check the preview, then run.** A before/after preview updates as you change settings. Queue as many files as you like and leave them to run. ## Download @@ -80,9 +84,9 @@ Where an AppImage can't be mounted, use the `.tar.gz` of the same build: extract ## Hardware and add-ons -**GPU features.** RIFE frame interpolation and AI upscaling run on the graphics card through Vulkan: on any Apple Silicon Mac, and on Windows and Linux x64 PCs with a current graphics driver (RIFE on Windows also needs the Microsoft Visual C++ Redistributable). They aren't available on Intel Macs or Linux on ARM yet. GPU deinterlacing and KNLMeans denoising use OpenCL instead; without it, deinterlacing falls back to the CPU. Whenever you choose a GPU feature your machine can't run, VapourBox says what's missing and how to fix it. +**GPU features.** RIFE frame interpolation and AI upscaling run on the graphics card through Vulkan: on any Apple Silicon Mac, and on Windows and Linux x64 PCs with a current graphics driver (RIFE on Windows also needs the Microsoft Visual C++ Redistributable). They aren't available on Intel Macs or Linux on ARM yet. GPU deinterlacing and KNLMeans denoising use OpenCL instead; without it, deinterlacing falls back to the CPU, and NLMeans gives KNLMeans' result on the CPU. Whenever you choose a GPU feature your machine can't run, VapourBox says what's missing and how to fix it. Face restoration runs through the AI runtime instead: on the graphics card and Neural Engine on a Mac (Core ML), on any DirectX 12 graphics card on Windows (DirectML), and on the CPU on Linux, which is several times slower. -**Add-ons.** A few features use third-party downloads that aren't part of VapourBox: the Whisper engine and language models for subtitles, and the AnimeJaNai models for AI upscaling. VapourBox offers to download one the first time you need it, stating its source, size and licence, and **Settings → Add-ons** installs or removes them at any time. AnimeJaNai is free for **non-commercial use only** (CC BY-NC-SA 4.0); for paid work, use Real-ESRGAN or Real-CUGAN instead. +**Add-ons.** A few features use third-party downloads that aren't part of VapourBox: the Whisper engine and language models for subtitles, the models for AI upscaling (anime and live action) and for RIFE frame interpolation, DeepFilterNet for speech enhancement in Audio Restoration, the face restoration and colourisation models, and the AI runtime (Microsoft's ONNX Runtime) that AI model add-ons run on. VapourBox offers to download one the first time you need it, stating its source, size and licence, and **Settings → Add-ons** installs or removes them at any time. AnimeJaNai is free for **non-commercial use only** (CC BY-NC-SA 4.0); for paid work, use Real-ESRGAN or Real-CUGAN instead. Of the face restoration models, RestoreFormer++ (the default) is free for any use; GFPGAN and CodeFormer are for **non-commercial use only**. **Older processors.** Intel and AMD CPUs from before about 2013 (no AVX2) are supported on every platform. VapourBox checks the processor on first launch and installs a matching set of components, which runs some filters more slowly but never differently. @@ -117,16 +121,19 @@ Most sources need only a few of these, and the presets pick them for you. Filter | 10 | **Dehalo** | Bright outlines and ringing around edges | | 11 | **Deblock** | Blocking from heavy compression | | 12 | **Deband** | Visible steps in gradients and skies | -| 13 | **Anti-Aliasing** | Stair-stepping on diagonal edges | +| 13 | **Anti-Aliasing** | Stair-stepping on diagonal edges: daa, santiag, or SangNom (sharpest) | | 14 | **Chroma Fixes** | Colour shifted off the picture, bleeding, rainbowing, dot crawl and leftover combing | -| 15 | **Color Correction** | Brightness, contrast, saturation, levels, white balance, shadow detail and local contrast, set by hand or measured | -| 16 | **Stabilize** | Shake and weave from telecine, film scans or a handheld camera | -| 17 | **Rotate / Flip** | Sideways, upside-down or mirrored footage | -| 18 | **Crop & Resize** | Overscan, scaling and upscaling (edge-directed or AI), and borders to reach an exact frame size | -| 19 | **Sharpen** | Soft pictures, sharpened after the resize so the result isn't softened again | -| 20 | **Film Grain** | Grain added back after denoising, so the picture doesn't look plastic | -| 21 | **Frame Rate** | PAL ↔ NTSC and other rate changes, or any custom rate, by motion interpolation, RIFE, or frame repetition | -| 22 | **Subtitles** | Speech-to-text with Whisper AI | +| 15 | **Colourisation** | Black-and-white footage, coloured by AI (DDColor or DeOldify Video, optional downloads); the brightness of every pixel is kept exactly | +| 16 | **Color Correction** | Brightness, contrast, saturation, levels, white balance, shadow detail and local contrast, set by hand or measured | +| 17 | **Stabilize** | Shake and weave from telecine, film scans or a handheld camera | +| 18 | **Rotate / Flip** | Sideways, upside-down or mirrored footage | +| 19 | **Crop & Resize** | Black borders, tape blanking and head-switching noise (measured for you by **Detect borders**), scaling and upscaling (edge-directed or AI) with SD colour converted to HD's BT.709, and borders to reach an exact frame size | +| 20 | **Face Restoration** | Blurry, low-resolution faces, redrawn by AI at the output resolution (optional download) | +| 21 | **Sharpen** | Soft pictures, sharpened after the resize so the result isn't softened again | +| 22 | **Film Grain** | Grain added back after denoising, so the picture doesn't look plastic | +| 23 | **Frame Rate** | PAL ↔ NTSC and other rate changes, or any custom rate, by motion interpolation, RIFE, or frame repetition | +| 24 | **Audio Restoration** | A dead channel, hum, rumble, clicks, clipping, hiss, sound out of sync or drifting, uneven loudness; speech enhancement with DeepFilterNet | +| 25 | **Subtitles** | Speech-to-text with Whisper AI, from the restored sound | The order is designed so each step works on the right picture: damage and noise are removed first, the picture is framed and resized next, and only then sharpened and given grain. The frame rate is converted last, so every other filter works on real frames. Borders are added after everything else, so nothing touches them. @@ -188,7 +195,7 @@ Every button with a shortcut shows it in its tooltip, and **?** lists them all.
Presets -**For Your Source:** VHS Cleanup, DV Camcorder Tape, PAL DVD / Broadcast, DVD IVTC, Anime DVD, 8mm / Super 8 Film Scan and Low-Light Phone / Camera Video. Pick the one that matches what you have. +**For Your Source:** VHS Cleanup, SD Tape → HD (YouTube / TV), DV Camcorder Tape, PAL DVD / Broadcast, DVD IVTC, Anime DVD, 8mm / Super 8 Film Scan and Low-Light Phone / Camera Video. Pick the one that matches what you have. **Quality Only:** Fast, Balanced and High Quality just deinterlace, with increasing effort, for a picture that's already clean. @@ -202,7 +209,8 @@ Your own presets are saved alongside these, and can be **exported and imported** By default each filter shows a short, curated set of choices. **Settings → General → Show advanced options** (or the switch beside any filter's settings) reveals every method and parameter, including: - all 70+ QTGMC parameters, and opt-in higher-precision deinterlacing (4:2:2 chroma, 16-bit processing); -- the less common denoisers: DFTTest, FFT3DFilter, TTempSmooth, FluxSmooth, STPresso, TemporalDegrain2, BM3D, KNLMeans and a large-window median; +- the less common denoisers: DFTTest, FFT3DFilter, TTempSmooth, FluxSmooth, STPresso, TemporalDegrain2, BM3D, KNLMeans (GPU), NLMeans (the same filter on the CPU) and a large-window median; +- specialist repairs: the vine dehalo, mpeg2stinx for the comb-like noise of film DVDs, and a fade-fix mode for IVTC (fades added after telecine); - fine control of edge-directed upscaling (NNEDI3's neurons, quality and prescreener; EEDI3's weights); - Descale's explicit native width and edge handling. @@ -241,7 +249,7 @@ The shutdown is never forced, so an app with unsaved work can stop it. VapourBox ## Building from source -See [docs/BUILDING.md](docs/BUILDING.md) for build instructions, project structure and the development workflow. +See [docs/BUILDING.md](docs/BUILDING.md) for build instructions, project structure and the development workflow, and [docs/ADDONS.md](docs/ADDONS.md) for how optional downloads (AI models and the AI runtime) are described, installed and tested. The investigations behind feature decisions (measurements, screenshots and the reasoning, including ideas that weren't built) are written up in [docs/reports/](docs/reports/README.md). @@ -271,11 +279,19 @@ VapourBox is a front end to other people's work. Thanks in particular to: - **mawen1250** — Retinex, BM3D and mvsfunc - **Edoardo Brunetti** (maintained by Jaded Encoding Thaumaturgy) — KNLMeansCL - **Frechdachs** (maintained by Jaded Encoding Thaumaturgy) — descale -- **vszip** by Julek — CLAHE, the local-contrast lift in Color Correction -- **RIFE** by hzwer (with Megvii) — the frame interpolation network, run through HolyWu's VapourSynth plugin (built on nihui's rife-ncnn-vulkan and Tencent's ncnn), with **MoltenVK** for Vulkan on macOS -- **Real-ESRGAN** by Xintao Wang and **Real-CUGAN** by bilibili — the AI upscaling networks, run through AmusementClub's **vs-mlrt** (VSNCNN) +- **vszip** by Julek — CLAHE, the local-contrast lift in Color Correction, and NLMeans, the CPU non-local-means denoiser +- **SangNom** by james1201 and Victor Efimov — the SangNom anti-aliasing method +- **vs-jetpack** by the Jaded Encoding Thaumaturgy project — FixInterlacedFades, vine_dehalo and mpeg2stinx, ported into VapourBox's filter scripts +- **RIFE** by hzwer (with Megvii) — the frame interpolation network, an optional download, run through HolyWu's VapourSynth plugin (built on nihui's rife-ncnn-vulkan and Tencent's ncnn), with **MoltenVK** for Vulkan on macOS +- **Real-ESRGAN** by Xintao Wang and **Real-CUGAN** by bilibili — the anime upscaling networks, an optional download, run through AmusementClub's **vs-mlrt** (VSNCNN) - **AnimeJaNai** by the-database — anime upscaling models, an optional download (CC BY-NC-SA 4.0, non-commercial use only) +- **RestoreFormer++** by Zhouxia Wang, Jiawei Zhang, Tianshui Chen, Wenping Wang and Ping Luo — the default face restoration model, with the **RetinaFace** detector by biubug6 (Pytorch_Retinaface), the **BiSeNet** face parser by zll (face-parsing.PyTorch) and **facexlib** by Xintao Wang, an optional download +- **GFPGAN** by THL A29 Limited, a Tencent company, and **CodeFormer** by S-Lab — alternative face restoration models, optional downloads for non-commercial use only +- **DDColor** by Xiaoyang Kang, Tao Yang, Wenqi Ouyang, Peiran Ren, Lingzhi Li and Xuansong Xie, and **DeOldify** by Jason Antic — the colourisation models, optional downloads +- **Real-ESRGAN** general and x4plus/x2plus models by Xintao Wang, and **2xLiveActionV1_SPAN** by jcj83429 — upscaling models for real-world footage, optional downloads (BSD-3-Clause; Apache-2.0) - **whisper.cpp** by the ggml authors and OpenAI's **Whisper** models — speech recognition for subtitles, an optional download +- **ONNX Runtime** by Microsoft Corporation, with **NumPy** by the NumPy Developers and, on Windows, Microsoft's **DirectML** — the AI runtime add-on that runs the optional AI models, an optional download +- **DeepFilterNet** by Hendrik Schröter — speech enhancement for Audio Restoration, an optional download (MIT or Apache-2.0) - **libdvdread** by VideoLAN — DVD reading - **FFmpeg** — encoding and decoding - **Hybrid** by Selur — the inspiration for this project diff --git a/Scripts/deps-expected-plugins.json b/Scripts/deps-expected-plugins.json index 2763a287..d8edfec1 100644 --- a/Scripts/deps-expected-plugins.json +++ b/Scripts/deps-expected-plugins.json @@ -13,63 +13,28 @@ "vapoursynth/plugins/libzsmooth.so" ] }, - "_extraFilesComment": "Files a COMPLETE bundle must hold outside the plugin directory, as paths from the bundle root. Today those are the ncnn-on-Vulkan plugins (worker/src/gpu_plugin.rs): RIFE in vapoursynth/rife/, VSNCNN and its upscaling models in vapoursynth/ncnn/, and on macOS the MoltenVK both preload from vapoursynth/moltenvk/ — deliberately outside the autoload directory, so the plugin-directory lists above cannot cover them. The package-deps-* scripts assert these too. A platform with no entry ships neither (macos-x64, linux-arm64), and the worker's gpu_plugin::platform_ships_rife / platform_ships_upscale must agree — app/test/attribution_test.dart checks both directions.", + "_extraFilesComment": "Files a COMPLETE bundle must hold outside the plugin directory, as paths from the bundle root. Today those are the ncnn-on-Vulkan plugins (worker/src/gpu_plugin.rs): RIFE in vapoursynth/rife/, VSNCNN in vapoursynth/ncnn/, and on macOS the MoltenVK both preload from vapoursynth/moltenvk/. Their models are NOT here: every AI model is an add-on (docs/ADDONS.md), and app/test/attribution_test.dart fails if one is listed — deliberately outside the autoload directory, so the plugin-directory lists above cannot cover them. The package-deps-* scripts assert these too. A platform with no entry ships neither (macos-x64, linux-arm64), and the worker's gpu_plugin::platform_ships_rife / platform_ships_upscale must agree — app/test/attribution_test.dart checks both directions.", "_extraFiles": { "macos-arm64": [ "vapoursynth/moltenvk/libMoltenVK.dylib", "vapoursynth/rife/librife.dylib", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", - "vapoursynth/ncnn/libvsncnn.dylib", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" + "vapoursynth/ncnn/libvsncnn.dylib" ], "windows-x64": [ "vapoursynth/rife/librife.dll", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", - "vapoursynth/ncnn/vsncnn.dll", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" + "vapoursynth/ncnn/vsncnn.dll" ], "windows-x64-v2": [ "vapoursynth/rife/librife.dll", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", - "vapoursynth/ncnn/vsncnn.dll", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" + "vapoursynth/ncnn/vsncnn.dll" ], "linux-x64": [ "vapoursynth/rife/librife.so", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", - "vapoursynth/ncnn/libvsncnn.so", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" + "vapoursynth/ncnn/libvsncnn.so" ], "linux-x64-v2": [ "vapoursynth/rife/librife.so", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.bin", - "vapoursynth/rife/models/rife-v4.26_ensembleFalse/flownet.param", - "vapoursynth/ncnn/libvsncnn.so", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", - "vapoursynth/ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-no-denoise.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-conservative.onnx", - "vapoursynth/ncnn/models/cugan/up2x-latest-denoise3x.onnx" + "vapoursynth/ncnn/libvsncnn.so" ] }, "windows-x64": [ @@ -108,7 +73,8 @@ "vsznedi3.dll", "zsmooth.dll", "bm3d.dll", - "libdescale.dll" + "libdescale.dll", + "sangnom.dll" ], "windows-x64-v2": [ "AddGrain.dll", @@ -146,7 +112,8 @@ "vsznedi3.dll", "zsmooth.dll", "bm3d.dll", - "libdescale.dll" + "libdescale.dll", + "sangnom.dll" ], "macos-arm64": [ "libaddgrain.dylib", @@ -184,7 +151,8 @@ "libvszip.dylib", "libznedi3.dylib", "libzsmooth.dylib", - "libdescale.dylib" + "libdescale.dylib", + "libsangnom.dylib" ], "macos-x64": [ "libaddgrain.dylib", @@ -220,7 +188,8 @@ "libznedi3.dylib", "libzsmooth.dylib", "libbm3d.dylib", - "libdescale.dylib" + "libdescale.dylib", + "libsangnom.dylib" ], "macos-x64-v2": [ "libaddgrain.dylib", @@ -256,7 +225,8 @@ "libznedi3.dylib", "libzsmooth.dylib", "libbm3d.dylib", - "libdescale.dylib" + "libdescale.dylib", + "libsangnom.dylib" ], "linux-x64": [ "libaddgrain.so", @@ -293,7 +263,8 @@ "libznedi3.so", "libzsmooth.so", "libbm3d.so", - "libdescale.so" + "libdescale.so", + "libsangnom.so" ], "linux-x64-v2": [ "libaddgrain.so", @@ -330,7 +301,8 @@ "libznedi3.so", "libzsmooth.so", "libbm3d.so", - "libdescale.so" + "libdescale.so", + "libsangnom.so" ], "linux-arm64": [ "libaddgrain.so", @@ -368,6 +340,7 @@ "libznedi3.so", "libzsmooth.so", "libbm3d.so", - "libdescale.so" + "libdescale.so", + "libsangnom.so" ] } diff --git a/Scripts/download-deps-linux.sh b/Scripts/download-deps-linux.sh index 2ee017f9..01383734 100755 --- a/Scripts/download-deps-linux.sh +++ b/Scripts/download-deps-linux.sh @@ -13,6 +13,14 @@ set -e +# Every download goes through this. Upstream hosts (GitHub's release CDN above +# all) sometimes answer with a 5xx error page or cut a transfer short: plain +# `curl -L` saves that page as the archive and the build dies later on a +# baffling "not in gzip format". --fail turns an HTTP error into a failed +# command, and the retries (any error, transfer cut short included) ride out a +# transient outage. Output is rewritten from the start on each attempt. +curl() { command curl --fail --retry 6 --retry-delay 15 --retry-all-errors "$@"; } + FORCE=false TIER=v3 while [[ $# -gt 0 ]]; do @@ -1567,10 +1575,12 @@ else fi # ============================================================================ -# vszip — CLAHE (Colour Correction's local contrast, issue #90) +# vszip — CLAHE, NLMeans (CPU) and the vine dehalo method # ============================================================================ -# Only core.vszip.CLAHE is used. Keep VSZIP_VERSION in step across -# download-deps-{macos,linux}.sh and download-deps-windows.ps1. +# core.vszip.CLAHE (Colour Correction's local contrast, issue #90) and, from +# 24.0.0 (the first release with it), core.vszip.NLMeans: Noise Reduction's +# CPU non-local means and the Dehalo pass's vine method. Keep VSZIP_VERSION in +# step across download-deps-{macos,linux}.sh and download-deps-windows.ps1. # # The x86_64 wheel carries three builds — libvszip.so (baseline x86-64), # libvszip.avx2.so and libvszip.zn4.so (AVX-512) — that VapourSynth picks @@ -1581,14 +1591,18 @@ fi # by path and fault); and the AVX-512 build is never chosen on a machine no # runner can test (the CTMF lesson). CLAHE is cheap next to anything else in a # job, so the AVX2 speedup is not worth that machinery. -# manylinux_2_17 is far under our glibc 2.39 floor. -VSZIP_VERSION="22.1.0" +# manylinux_2_17 is far under our glibc 2.39 floor. 24.0.0's baseline build no +# longer calls libm's fmaf per lane, and contains no AVX at all (checked with +# objdump), so it still needs no _tierFiles entry. Pinned by hash. +VSZIP_VERSION="24.0.0" echo "" -echo "=== Downloading vszip $VSZIP_VERSION (CLAHE) ===" +echo "=== Downloading vszip $VSZIP_VERSION (CLAHE, NLMeans) ===" if [ "$FORCE" = true ] || [ ! -f "$PLUGINS_DIR/libvszip.so" ]; then case "$ARCH" in - x86_64) VSZIP_TAG="manylinux_2_17_x86_64" ;; - aarch64) VSZIP_TAG="manylinux_2_17_aarch64" ;; + x86_64) VSZIP_TAG="manylinux_2_17_x86_64" + VSZIP_SHA256="abf1be51f3b4733500eca04f7179236f8383e09b298ac4ca2c8e8959a93a92da" ;; + aarch64) VSZIP_TAG="manylinux_2_17_aarch64" + VSZIP_SHA256="8f2a87cd1cbb0a6b5392b6f43c9dd22bb531c7d2021c4c2b66fe6856c8dc7f92" ;; *) VSZIP_TAG="" ;; esac VSZIP_URL="" @@ -1605,6 +1619,7 @@ PYEOF mkdir -p "$BUILD_DIR/vszip" if [ -n "$VSZIP_URL" ] \ && curl -fL -o "$BUILD_DIR/vszip.whl" "$VSZIP_URL" \ + && echo "$VSZIP_SHA256 $BUILD_DIR/vszip.whl" | sha256sum -c - \ && unzip -q "$BUILD_DIR/vszip.whl" -d "$BUILD_DIR/vszip"; then # The baseline build only, by exact name — see above. cp "$BUILD_DIR/vszip/vapoursynth/plugins/vszip/libvszip.so" "$PLUGINS_DIR/" 2>/dev/null || true @@ -1622,6 +1637,64 @@ else echo " vszip already exists, skipping" fi +# ============================================================================ +# SangNom — Anti-aliasing's SangNom method (core.sangnom.SangNom) +# ============================================================================ +# The PyPI wheels (r45), pinned by hash. Source is MIT; upstream states its +# binaries are GPL v2 (see licenses/NOTICES.txt). x86_64 has SSE2 code only +# (no AVX anywhere, checked with objdump), so one file serves both CPU tiers. +# manylinux_2_28 (glibc 2.28) is under our 2.39 floor; it needs the system +# libstdc++, as every C++ plugin here does. Keep SANGNOM_VERSION in step with +# download-deps-{macos.sh,windows.ps1}. +SANGNOM_VERSION="45.0" +case "$ARCH" in + x86_64) SANGNOM_TAG="manylinux_2_28_x86_64" + SANGNOM_SHA256="dd71e28fa7d0383101b6ab5e84307be80984f69a43838238686de1480793525d" ;; + aarch64) SANGNOM_TAG="manylinux_2_28_aarch64" + SANGNOM_SHA256="4fd6e39bc32aec755853f974936324081223d6b7e082cc38e717c9b8dad0cf6a" ;; + *) SANGNOM_TAG="" ;; +esac +SANGNOM_OUT="$PLUGINS_DIR/libsangnom.so" +echo "" +echo "=== Installing SangNom $SANGNOM_VERSION ===" +if [ "$FORCE" = false ] && [ -f "$SANGNOM_OUT" ]; then + echo " SangNom already exists, skipping" +else + rm -f "$SANGNOM_OUT" + # Run on its own, not as `if ( set -e ... ); then`: bash ignores set -e in + # anything that is an if/&&/|| condition, subshells included. + set +e + ( + set -e + [ -n "$SANGNOM_TAG" ] + cd "$BUILD_DIR" + rm -rf sangnom-whl sangnom.whl + url=$("$PYTHON_BIN" - "$SANGNOM_VERSION" "$SANGNOM_TAG" <<'PYEOF2' +import json, sys, urllib.request +ver, tag = sys.argv[1], sys.argv[2] +d = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/vapoursynth-sangnom/{ver}/json")) +print(next(f["url"] for f in d["urls"] if f["filename"].endswith(tag + ".whl"))) +PYEOF2 +) + curl -fsSL -o sangnom.whl "$url" + echo "$SANGNOM_SHA256 sangnom.whl" | sha256sum -c - + mkdir sangnom-whl && unzip -q sangnom.whl -d sangnom-whl + cp sangnom-whl/vapoursynth/plugins/sangnom.so "$SANGNOM_OUT" + chmod 755 "$SANGNOM_OUT" + rm -rf sangnom-whl sangnom.whl + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then + echo " Installed SangNom -> libsangnom.so" + BUILT_PLUGINS+=("sangnom") + else + rm -f "$SANGNOM_OUT" + echo " Failed to install SangNom" + FAILED_PLUGINS+=("sangnom") + fi +fi + # ============================================================================ # descale — the Descale pass (core.descale.De*) # ============================================================================ @@ -1676,8 +1749,8 @@ fi # ============================================================================ # Lives in vapoursynth/rife/, NOT the autoload directory: the template loads it # only for a RIFE job (see worker/src/gpu_plugin.rs), so a machine whose GPU stack or -# CPU cannot run it fails that method alone. Keep RIFE_TAG, the model commit and -# the model hashes in step with download-deps-{macos.sh,windows.ps1}. +# CPU cannot run it fails that method alone. Keep RIFE_TAG in step with +# download-deps-{macos.sh,windows.ps1}. # # x86_64 only: upstream publishes no aarch64 build, and the app reports RIFE as # unavailable on linux-arm64. The prebuilt needs glibc 2.35 (under our 2.39 @@ -1686,21 +1759,18 @@ fi # The same file serves both CPU tiers; if it ever proves to need AVX at load, # the app's probe reports that as an unsupported CPU rather than crashing jobs. # -# The model is the ncnn conversion committed to the plugin's repo (there is no -# release asset for it), fetched at a pinned commit and checked by hash. +# The plugin only. Its model is the `frame-rate-rife` add-on (deps 1.16.0 on; +# app/assets/addons/frame-rate-rife.json): every AI model is an add-on, never +# in this bundle (docs/ADDONS.md). RIFE_TAG="r9_mod_v33" RIFE_LINUX_X64_SHA256="18a00a5e3ac90a5dfcfdf85fdb81d977381a5a2560a4c45dbe9a469d007b1886" -RIFE_MODEL_COMMIT="96fdcecdf982ef3237e16e2b105eea849fa022ab" -RIFE_MODEL_NAME="rife-v4.26_ensembleFalse" -RIFE_MODEL_BIN_SHA256="94d58e30b75d7c7609cfa6f3bdad524deddd14f5f75e85c36d2f827ef5c64731" -RIFE_MODEL_PARAM_SHA256="79f16c28903f93f8308f0c4c947f8c7e0c17d99a57b85473e5f298dd578d137b" RIFE_DIR="$DEPS_DIR/vapoursynth/rife" echo "" -echo "=== Installing RIFE ($RIFE_TAG, model $RIFE_MODEL_NAME) ===" +echo "=== Installing RIFE ($RIFE_TAG) ===" if [ "$ARCH" != "x86_64" ]; then echo " Skipped on $ARCH: upstream publishes no build" elif [ "$FORCE" = false ] && [ -f "$RIFE_DIR/librife.so" ] \ - && [ -f "$RIFE_DIR/models/$RIFE_MODEL_NAME/flownet.bin" ]; then + && [ ! -d "$RIFE_DIR/models" ]; then echo " RIFE already exists, skipping" else rm -rf "$RIFE_DIR" @@ -1711,20 +1781,13 @@ else set +e ( set -e - mkdir -p "$RIFE_DIR/models/$RIFE_MODEL_NAME" + mkdir -p "$RIFE_DIR" cd "$BUILD_DIR" curl -fsSL -o librife.so \ "https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/releases/download/$RIFE_TAG/librife_linux_x86-64.so" echo "$RIFE_LINUX_X64_SHA256 librife.so" | sha256sum -c - mv librife.so "$RIFE_DIR/librife.so" chmod 755 "$RIFE_DIR/librife.so" - for f in flownet.bin flownet.param; do - curl -fsSL -o "$f" \ - "https://raw.githubusercontent.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/$RIFE_MODEL_COMMIT/models/$RIFE_MODEL_NAME/$f" - done - echo "$RIFE_MODEL_BIN_SHA256 flownet.bin" | sha256sum -c - - echo "$RIFE_MODEL_PARAM_SHA256 flownet.param" | sha256sum -c - - mv flownet.bin flownet.param "$RIFE_DIR/models/$RIFE_MODEL_NAME/" ) _block_status=$? set -e @@ -1739,7 +1802,7 @@ else fi # ============================================================================ -# VSNCNN (vs-mlrt) + Real-ESRGAN / Real-CUGAN — Crop & Resize's AI upscaling +# VSNCNN (vs-mlrt) — runs Crop & Resize's AI upscalers # ============================================================================ # vs-mlrt publishes VSNCNN for Windows only; this is upstream's own # linux-ncnn.yml recipe at the pinned tag: static protobuf and onnx from @@ -1750,24 +1813,15 @@ fi # dlopens the system's libvulkan.so.1; one build serves both CPU tiers (ncnn # dispatches its SIMD at run time). # -# The models are vs-mlrt's ONNX conversions (2.4-5.2 MB each), shipped only -# inside one ~810 MB archive per release: fetched pinned and hash-checked, the -# five VapourBox uses extracted and each checked by its own hash. Keep every pin -# and hash in step with download-deps-{macos.sh,windows.ps1} and -# gpu_plugin::UPSCALE_MODELS. +# The plugin only. Every model it runs is an add-on (deps 1.16.0 on; +# app/assets/addons/upscale-*.json, docs/ADDONS.md) — the Real-ESRGAN anime and +# Real-CUGAN models included, which were bundled here until then. Keep every +# pin in step with download-deps-{macos.sh,windows.ps1}. VSMLRT_TAG="v15.16" -VSMLRT_MODELS_SHA256="d4eaa8c1c4459082185a9e1a2c54d7e2c776abeea6aef6aee0cb8f6c151dfb02" PROTOBUF_TAG="v3.21.12" ONNX_COMMIT="b86cc54efce19530fb953e4b21f57e6b3888534c" NCNN_TAG="20260526" NCNN_UBUNTU_SHA256="c6daf0c8fb2546de420c7b320e2ce62ba764163ace0ba63a2ea0cdfca70d684d" -UPSCALE_MODEL_HASHES=( - "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx 6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" - "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx 414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" - "cugan/up2x-latest-no-denoise.onnx 3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" - "cugan/up2x-latest-conservative.onnx 555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" - "cugan/up2x-latest-denoise3x.onnx 7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" -) # VSNCNN is an API 3 plugin (VapourSynth.h / VSHelper.h). R78 still loads # those, but the build's own include dir need not carry the old headers, so # they are fetched from the same VapourSynth tag, by hash. @@ -1778,11 +1832,11 @@ VSAPI3_HEADER_HASHES=( ) NCNN_DIR="$DEPS_DIR/vapoursynth/ncnn" echo "" -echo "=== Installing VSNCNN ($VSMLRT_TAG) + upscaling models ===" +echo "=== Installing VSNCNN ($VSMLRT_TAG) ===" if [ "$ARCH" != "x86_64" ]; then echo " Skipped on $ARCH: Tencent publishes no aarch64 Linux ncnn" elif [ "$FORCE" = false ] && [ -f "$NCNN_DIR/libvsncnn.so" ] \ - && [ -f "$NCNN_DIR/models/cugan/up2x-latest-denoise3x.onnx" ]; then + && [ ! -d "$NCNN_DIR/models" ]; then echo " VSNCNN already exists, skipping" else rm -rf "$NCNN_DIR" @@ -1794,7 +1848,7 @@ else ( set -e W="$BUILD_DIR/vsncnn" - rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR/models" + rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR" cd "$W" curl -fsSL "https://github.com/protocolbuffers/protobuf/archive/refs/tags/$PROTOBUF_TAG.tar.gz" | tar xz cmake -S "protobuf-${PROTOBUF_TAG#v}/cmake" -B pb-build -G Ninja -DCMAKE_BUILD_TYPE=Release \ @@ -1829,24 +1883,6 @@ else cmake --build vsncnn-build >/dev/null cp vsncnn-build/libvsncnn.so "$NCNN_DIR/libvsncnn.so" chmod 755 "$NCNN_DIR/libvsncnn.so" - curl -fsSL -o models.7z \ - "https://github.com/AmusementClub/vs-mlrt/releases/download/$VSMLRT_TAG/models.$VSMLRT_TAG.7z" - echo "$VSMLRT_MODELS_SHA256 models.7z" | sha256sum -c - - wanted=() - for entry in "${UPSCALE_MODEL_HASHES[@]}"; do wanted+=("models/${entry%% *}"); done - # One pass: the archive is solid, so each separate extraction would - # decompress it from the start again. - if command -v 7z >/dev/null; then - 7z x -y -bd models.7z "${wanted[@]}" >/dev/null - else - bsdtar -xf models.7z "${wanted[@]}" - fi - for entry in "${UPSCALE_MODEL_HASHES[@]}"; do - rel="${entry%% *}"; sum="${entry##* }" - echo "$sum models/$rel" | sha256sum -c - - mkdir -p "$NCNN_DIR/models/$(dirname "$rel")" - mv "models/$rel" "$NCNN_DIR/models/$rel" - done cd "$BUILD_DIR" && rm -rf "$W" ) _block_status=$? diff --git a/Scripts/download-deps-macos.sh b/Scripts/download-deps-macos.sh index 36e2536c..59267d8d 100755 --- a/Scripts/download-deps-macos.sh +++ b/Scripts/download-deps-macos.sh @@ -17,6 +17,14 @@ set -e +# Every download goes through this. Upstream hosts (GitHub's release CDN above +# all) sometimes answer with a 5xx error page or cut a transfer short: plain +# `curl -L` saves that page as the archive and the build dies later on a +# baffling "not in gzip format". --fail turns an HTTP error into a failed +# command, and the retries (any error, transfer cut short included) ride out a +# transient outage. Output is rewritten from the start on each attempt. +curl() { command curl --fail --retry 6 --retry-delay 15 --retry-all-errors "$@"; } + FORCE=false TIER=v3 while [[ $# -gt 0 ]]; do @@ -2074,19 +2082,20 @@ else fi # ============================================================================ -# vszip — CLAHE (Colour Correction's local contrast, issue #90) +# vszip — CLAHE, NLMeans (CPU) and the vine dehalo method # ============================================================================ -# Only core.vszip.CLAHE is used. Keep VSZIP_VERSION in step across -# download-deps-{macos,linux}.sh and download-deps-windows.ps1. +# core.vszip.CLAHE (Colour Correction's local contrast, issue #90) and, from +# 24.0.0 (the first release with it), core.vszip.NLMeans: Noise Reduction's +# CPU non-local means and the Dehalo pass's vine method. Keep VSZIP_VERSION in +# step across download-deps-{macos,linux}.sh and download-deps-windows.ps1. # -# arm64 takes the PyPI wheel's dylib: minos 13, under this arch's 15.0 target. -# It has zero bytes between its load commands and __text, so install_name_tool -# cannot fit a longer id (the repoint loop below fails quietly and leaves -# @rpath, which is harmless for a dlopen'd plugin). Re-signing is safe: it is -# already linker-signed, and codesign replaces an existing signature in place -# rather than adding a load command. +# arm64 takes the PyPI wheel's dylib: minos 15.0 from 24.0.0 (13 before), which +# is exactly this arch's 15.0 target. 24.0.0's has 4 KB free after its load +# commands (22.1.0's had none); re-signing is safe either way: it is already +# linker-signed, and codesign replaces an existing signature in place rather +# than adding a load command. # -# x64 builds the same release's sdist, for two reasons. The wheel is minos 13.0 +# x64 builds the same release's sdist, for two reasons. The wheel is minos 15.0 # against this bundle's 12.0 floor (issue #39). And Zig's x86_64 macOS output # carries NO code signature and leaves 8 bytes after its load commands, so # `codesign` — which must add a 16-byte LC_CODE_SIGNATURE — silently writes the @@ -2095,15 +2104,18 @@ fi # gives the header room; the guard below fails the build if it ever doesn't. # Built with upstream's default x86_64-macos CPU (same as its wheel), which has # no AVX — so one build serves both CPU tiers and needs no _tierFiles entry. -VSZIP_VERSION="22.1.0" -VSZIP_SDIST_SHA256="93c1aaf5867ad43e1f21b77418fa89b0b1ed2191d0c4bfb03a65c958826ed8c9" -# The sdist's pyproject pins ziglang==0.16.0; its build.zig does not compile -# under 0.17 (Optimize enum rename). -VSZIP_ZIG_VERSION="0.16.0" -VSZIP_ZIG_SHA256="0387557ed1877bc6a2e1802c8391953baddba76081876301c522f52977b52ba7" +VSZIP_VERSION="24.0.0" +VSZIP_SDIST_SHA256="b2d6157228d0e786a19ac9055814e6141841ffc2c7b42ac1858b0600fc4a69cb" +# The sdist's pyproject pins ziglang==0.17.0 (from vszip 23.0.0; its +# build.zig.zon refuses anything older). 0.17 renamed the optimize modes, hence +# -Doptimize=fast below rather than ReleaseFast. Cross-built from arm64 with +# the same recipe on 2026-11-08: minos 12.0, 4 KB header room, no AVX, no +# static initializers (check-load-time-simd.py). +VSZIP_ZIG_VERSION="0.17.0" +VSZIP_ZIG_SHA256="4f9a1c5269aa17ebda5e6d3c2b89d6cbf36f7d2b22a0306e9ab98f25f95529c6" VSZIP_OUT="$PLUGINS_DIR/libvszip.dylib" echo "" -echo "=== Installing vszip $VSZIP_VERSION (CLAHE) ===" +echo "=== Installing vszip $VSZIP_VERSION (CLAHE, NLMeans) ===" if [ "$FORCE" = false ] && [ -f "$VSZIP_OUT" ]; then echo " vszip already exists, skipping" else @@ -2145,10 +2157,12 @@ else mkdir -p vszip-src tar -xzf vszip.tar.gz -C vszip-src --strip-components=1 cd vszip-src - grep -q '^ lib.use_llvm = true;$' build.zig || { + # The VapourSynth library's `lib.use_llvm` sits inside `if (build_vs)` + # from 23.0.0 (8 spaces); the AviSynth+ one (12 spaces) is never built. + grep -q '^ lib.use_llvm = true;$' build.zig || { echo " ERROR: vszip build.zig changed; re-check the headerpad patch" >&2; exit 1; } - sed -i.bak 's/^ lib.use_llvm = true;$/ lib.use_llvm = true;\ - lib.headerpad_max_install_names = true;/' build.zig + sed -i.bak 's/^ lib.use_llvm = true;$/ lib.use_llvm = true;\ + lib.headerpad_max_install_names = true;/' build.zig grep -q 'lib.headerpad_max_install_names = true;' build.zig # Also fetches vszip's own Zig dependencies (pinned by hash in # build.zig.zon), so this step needs network access. @@ -2157,7 +2171,7 @@ else *.*) vszip_min="$MACOS_MIN_VERSION" ;; *) vszip_min="${MACOS_MIN_VERSION}.0" ;; esac - ../vszip-zig/zig build -Doptimize=ReleaseFast -Dtarget="x86_64-macos.${vszip_min}" + ../vszip-zig/zig build -Doptimize=fast -Dtarget="x86_64-macos.${vszip_min}" cp zig-out/lib/libvszip.dylib "$VSZIP_OUT" ) || true rm -rf "$BUILD_DIR/vszip-zig" "$BUILD_DIR/vszip-zig.tar.xz" \ @@ -2196,6 +2210,64 @@ else fi fi # end vszip install +# ============================================================================ +# SangNom — Anti-aliasing's SangNom method (core.sangnom.SangNom) +# ============================================================================ +# The PyPI wheels (r45), pinned by hash: arm64 is minos 11.0 and x86_64 10.15, +# both under their bundle's floor. Source is MIT; upstream states its binaries +# are GPL v2 (see licenses/NOTICES.txt). The x86_64 dylib has SSE2 code only +# (no AVX anywhere, checked with objdump; no static initializers), so one file +# serves both CPU tiers. It links only libc++ and libSystem. Keep +# SANGNOM_VERSION in step with download-deps-{linux.sh,windows.ps1}. +SANGNOM_VERSION="45.0" +if [ "$ARCH" = "arm64" ]; then + SANGNOM_TAG="macosx_11_0_arm64" + SANGNOM_SHA256="e558c2ffd7d5f6657878e97504c536cdb5e199c558537767b229bd37f8764810" +else + SANGNOM_TAG="macosx_10_15_x86_64" + SANGNOM_SHA256="cbcb274aa86732d01d4f39ab4058c32d580901163ac372c0219cdb3fa468840d" +fi +SANGNOM_OUT="$PLUGINS_DIR/libsangnom.dylib" +echo "" +echo "=== Installing SangNom $SANGNOM_VERSION ===" +if [ "$FORCE" = false ] && [ -f "$SANGNOM_OUT" ]; then + echo " SangNom already exists, skipping" +else + rm -f "$SANGNOM_OUT" + # Run on its own, not as `if ( set -e ... ); then` (see descale below). + set +e + ( + set -e + cd "$BUILD_DIR" + rm -rf sangnom-whl sangnom.whl + url=$("$PYTHON_BIN" - "$SANGNOM_VERSION" "$SANGNOM_TAG" <<'PYEOF2' +import json, sys, urllib.request +ver, tag = sys.argv[1], sys.argv[2] +d = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/vapoursynth-sangnom/{ver}/json")) +print(next(f["url"] for f in d["urls"] if f["filename"].endswith(tag + ".whl"))) +PYEOF2 +) + curl -fsSL -o sangnom.whl "$url" + echo "$SANGNOM_SHA256 sangnom.whl" | shasum -a 256 -c - + mkdir sangnom-whl && unzip -q sangnom.whl -d sangnom-whl + cp sangnom-whl/vapoursynth/plugins/sangnom.dylib "$SANGNOM_OUT" + chmod u+w "$SANGNOM_OUT" + install_name_tool -id "@loader_path/libsangnom.dylib" "$SANGNOM_OUT" + codesign -s - -f "$SANGNOM_OUT" + rm -rf sangnom-whl sangnom.whl + ) + _block_status=$? + set -e + if [ "$_block_status" -eq 0 ]; then + echo " Installed SangNom -> libsangnom.dylib" + BUILT_PLUGINS+=("sangnom") + else + rm -f "$SANGNOM_OUT" + echo " Failed to install SangNom" + FAILED_PLUGINS+=("sangnom") + fi +fi + # ============================================================================ # descale — the Descale pass (core.descale.De*) # ============================================================================ @@ -2311,8 +2383,8 @@ fi # RIFE 4.26 — the Frame Rate pass's neural interpolator (ncnn on Vulkan) # ============================================================================ # Lives in vapoursynth/rife/, NOT the autoload directory: the template loads it -# only for a RIFE job (see worker/src/gpu_plugin.rs). Keep RIFE_TAG, the model -# commit and the model hashes in step with download-deps-{linux.sh,windows.ps1}. +# only for a RIFE job (see worker/src/gpu_plugin.rs). Keep RIFE_TAG in step with +# download-deps-{linux.sh,windows.ps1}. # # arm64 only. The upstream x86_64 dylib targets macOS 13 against this arch's # 12.0 floor (issue #39) and runs AVX in a static initializer @@ -2320,21 +2392,18 @@ fi # the app reports RIFE as unavailable there. It resolves Vulkan from the # MoltenVK installed above. # -# The model is the ncnn conversion committed to the plugin's repo (there is no -# release asset for it), fetched at a pinned commit and checked by hash. +# The plugin only. Its model is the `frame-rate-rife` add-on (deps 1.16.0 on; +# app/assets/addons/frame-rate-rife.json): every AI model is an add-on, never +# in this bundle (docs/ADDONS.md). RIFE_TAG="r9_mod_v33" RIFE_MACOS_ARM64_SHA256="3041811d2b71afd2fa12d64484f64396c6fa5259ab63715316694f4110fc0e67" -RIFE_MODEL_COMMIT="96fdcecdf982ef3237e16e2b105eea849fa022ab" -RIFE_MODEL_NAME="rife-v4.26_ensembleFalse" -RIFE_MODEL_BIN_SHA256="94d58e30b75d7c7609cfa6f3bdad524deddd14f5f75e85c36d2f827ef5c64731" -RIFE_MODEL_PARAM_SHA256="79f16c28903f93f8308f0c4c947f8c7e0c17d99a57b85473e5f298dd578d137b" RIFE_DIR="$DEPS_DIR/vapoursynth/rife" echo "" -echo "=== Installing RIFE ($RIFE_TAG, model $RIFE_MODEL_NAME) ===" +echo "=== Installing RIFE ($RIFE_TAG) ===" if [ "$ARCH" != "arm64" ]; then echo " Skipped on $ARCH: no compatible prebuilt (see above)" elif [ "$FORCE" = false ] && [ -f "$RIFE_DIR/librife.dylib" ] \ - && [ -f "$RIFE_DIR/models/$RIFE_MODEL_NAME/flownet.bin" ]; then + && [ ! -d "$RIFE_DIR/models" ]; then echo " RIFE already exists, skipping" else rm -rf "$RIFE_DIR" @@ -2345,19 +2414,12 @@ else set +e ( set -e - mkdir -p "$RIFE_DIR/models/$RIFE_MODEL_NAME" + mkdir -p "$RIFE_DIR" cd "$BUILD_DIR" curl -fsSL -o librife.dylib \ "https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/releases/download/$RIFE_TAG/librife_macos_arm64.dylib" echo "$RIFE_MACOS_ARM64_SHA256 librife.dylib" | shasum -a 256 -c - mv librife.dylib "$RIFE_DIR/librife.dylib" - for f in flownet.bin flownet.param; do - curl -fsSL -o "$f" \ - "https://raw.githubusercontent.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/$RIFE_MODEL_COMMIT/models/$RIFE_MODEL_NAME/$f" - done - echo "$RIFE_MODEL_BIN_SHA256 flownet.bin" | shasum -a 256 -c - - echo "$RIFE_MODEL_PARAM_SHA256 flownet.param" | shasum -a 256 -c - - mv flownet.bin flownet.param "$RIFE_DIR/models/$RIFE_MODEL_NAME/" chmod u+w "$RIFE_DIR/librife.dylib" codesign -s - -f "$RIFE_DIR/librife.dylib" ) @@ -2374,7 +2436,7 @@ else fi # ============================================================================ -# VSNCNN (vs-mlrt) + Real-ESRGAN / Real-CUGAN — Crop & Resize's AI upscaling +# VSNCNN (vs-mlrt) — runs Crop & Resize's AI upscalers # ============================================================================ # vs-mlrt publishes VSNCNN for Windows only, so it is built here from the # pinned tag the way upstream's linux-ncnn.yml builds it: static protobuf and @@ -2384,24 +2446,15 @@ fi # (-undefined dynamic_lookup), exactly as for RIFE. Lives in vapoursynth/ncnn/, # never autoloaded (see worker/src/gpu_plugin.rs). arm64 only, as for RIFE. # -# The models are vs-mlrt's ONNX conversions, 2.4-5.2 MB each. vs-mlrt ships -# them only inside one ~810 MB archive per release, so that is fetched (pinned -# and hash-checked), the five files VapourBox uses are extracted, and each is -# checked by its own hash. Keep every pin and hash in step with -# download-deps-{linux.sh,windows.ps1} and gpu_plugin::UPSCALE_MODELS. +# The plugin only. Every model it runs is an add-on (deps 1.16.0 on; +# app/assets/addons/upscale-*.json, docs/ADDONS.md) — the Real-ESRGAN anime and +# Real-CUGAN models included, which were bundled here until then. Keep every +# pin in step with download-deps-{linux.sh,windows.ps1}. VSMLRT_TAG="v15.16" -VSMLRT_MODELS_SHA256="d4eaa8c1c4459082185a9e1a2c54d7e2c776abeea6aef6aee0cb8f6c151dfb02" PROTOBUF_TAG="v3.21.12" ONNX_COMMIT="b86cc54efce19530fb953e4b21f57e6b3888534c" NCNN_TAG="20260526" NCNN_MACOS_VULKAN_SHA256="e3ec1d473390a52c288968ecf1321fd2fdd25d8eaa2031a6b7d5c97f3e6961b4" -UPSCALE_MODEL_HASHES=( - "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx 6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" - "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx 414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" - "cugan/up2x-latest-no-denoise.onnx 3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" - "cugan/up2x-latest-conservative.onnx 555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" - "cugan/up2x-latest-denoise3x.onnx 7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" -) # VSNCNN is an API 3 plugin (VapourSynth.h / VSHelper.h). R78 still loads # those, but the build's own include dir need not carry the old headers, so # they are fetched from the same VapourSynth tag, by hash. @@ -2412,11 +2465,11 @@ VSAPI3_HEADER_HASHES=( ) NCNN_DIR="$DEPS_DIR/vapoursynth/ncnn" echo "" -echo "=== Installing VSNCNN ($VSMLRT_TAG) + upscaling models ===" +echo "=== Installing VSNCNN ($VSMLRT_TAG) ===" if [ "$ARCH" != "arm64" ]; then echo " Skipped on $ARCH: no MoltenVK here (see above)" elif [ "$FORCE" = false ] && [ -f "$NCNN_DIR/libvsncnn.dylib" ] \ - && [ -f "$NCNN_DIR/models/cugan/up2x-latest-denoise3x.onnx" ]; then + && [ ! -d "$NCNN_DIR/models" ]; then echo " VSNCNN already exists, skipping" else rm -rf "$NCNN_DIR" @@ -2428,7 +2481,7 @@ else ( set -e W="$BUILD_DIR/vsncnn" - rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR/models" + rm -rf "$W" && mkdir -p "$W" "$NCNN_DIR" cd "$W" # arm64 pins no target for the rest of the script; this one compiles # directly with clang, so it needs one: the arm64 bundle's 15.0. @@ -2480,22 +2533,6 @@ else -Wl,-undefined,dynamic_lookup -Wl,-install_name,@loader_path/libvsncnn.dylib \ -o "$NCNN_DIR/libvsncnn.dylib" codesign -s - -f "$NCNN_DIR/libvsncnn.dylib" - # Models. - curl -fsSL -o models.7z \ - "https://github.com/AmusementClub/vs-mlrt/releases/download/$VSMLRT_TAG/models.$VSMLRT_TAG.7z" - echo "$VSMLRT_MODELS_SHA256 models.7z" | shasum -a 256 -c - - # One pass: the archive is solid, so each separate extraction would - # decompress it from the start again. - wanted=() - for entry in "${UPSCALE_MODEL_HASHES[@]}"; do wanted+=("models/${entry%% *}"); done - tar -xf models.7z "${wanted[@]}" - for entry in "${UPSCALE_MODEL_HASHES[@]}"; do - rel="${entry%% *}"; sum="${entry##* }" - echo "$sum models/$rel" | shasum -a 256 -c - - mkdir -p "$NCNN_DIR/models/$(dirname "$rel")" - mv "models/$rel" "$NCNN_DIR/models/$rel" - done - rm -f models.7z cd "$BUILD_DIR" && rm -rf "$W" ) _block_status=$? diff --git a/Scripts/download-deps-windows.ps1 b/Scripts/download-deps-windows.ps1 index 3af20d17..28331058 100644 --- a/Scripts/download-deps-windows.ps1 +++ b/Scripts/download-deps-windows.ps1 @@ -95,10 +95,26 @@ $VSRelease = "R78" $PythonVersion = "3.12.10" $PythonTag = "312" +# Every download goes through Invoke-WithRetry. Upstream hosts (GitHub's release +# CDN above all) sometimes time out with a 5xx or cut a transfer short; a few +# spaced retries ride that out instead of failing the whole build. Written as +# a loop rather than -MaximumRetryCount, which Windows PowerShell 5.1 lacks. +function Invoke-WithRetry { + param([scriptblock]$Action, [string]$What, [int]$Attempts = 6, [int]$DelaySec = 15) + for ($i = 1; ; $i++) { + try { return & $Action } + catch { + if ($i -ge $Attempts) { throw } + Write-Host " $What failed (attempt $i of $Attempts): $($_.Exception.Message); retrying in $DelaySec s" -ForegroundColor Yellow + Start-Sleep -Seconds $DelaySec + } + } +} + function Download-File { param([string]$Url, [string]$OutFile) Write-Host " Downloading: $Url" -ForegroundColor Gray - Invoke-WebRequest -Uri $Url -OutFile $OutFile -UseBasicParsing + Invoke-WithRetry -What $Url { Invoke-WebRequest -Uri $Url -OutFile $OutFile -UseBasicParsing } } # ============================================================================= @@ -729,7 +745,7 @@ Write-Host "" Write-Host "Downloading akarin $AkarinVersion (LLVM JIT for std.Expr)..." -ForegroundColor Yellow if (-not (Test-Path "$PluginsDir\libakarin.dll")) { try { - $AkMeta = Invoke-RestMethod -Uri "https://pypi.org/pypi/vapoursynth-akarin/$AkarinVersion/json" + $AkMeta = Invoke-WithRetry -What "PyPI vapoursynth-akarin" { Invoke-RestMethod -Uri "https://pypi.org/pypi/vapoursynth-akarin/$AkarinVersion/json" } $AkUrl = ($AkMeta.urls | Where-Object { $_.filename -like "*win_amd64.whl" } | Select-Object -First 1).url if (-not $AkUrl) { throw "no win_amd64 wheel for akarin $AkarinVersion" } @@ -737,7 +753,7 @@ if (-not (Test-Path "$PluginsDir\libakarin.dll")) { $AkWhl = Join-Path $TempDir "akarin.whl" $AkZip = Join-Path $TempDir "akarin-wheel.zip" $AkOut = Join-Path $TempDir "akarin-extract" - Invoke-WebRequest -Uri $AkUrl -OutFile $AkWhl -UseBasicParsing + Download-File -Url $AkUrl -OutFile $AkWhl # A wheel is a zip, but Expand-Archive validates the *extension* and # refuses .whl outright -- same trap as the VapourSynth wheel above. Copy-Item $AkWhl $AkZip -Force @@ -766,7 +782,7 @@ if (-not (Test-Path "$PluginsDir\libakarin.dll")) { } # ============================================================================= -# Bwdif + DeDot + vszip + BM3D + descale + KNLMeansCL - PyPI wheels +# Bwdif + DeDot + vszip + BM3D + descale + KNLMeansCL + SangNom - PyPI wheels # ============================================================================= # None of these upstreams publishes a Windows release asset, so each comes from # its PyPI wheel, resolved through the JSON API the same way akarin is (never @@ -777,7 +793,9 @@ if (-not (Test-Path "$PluginsDir\libakarin.dll")) { # only the system runtime, so there is nothing to place beside it. Bwdif and # DeDot hold exactly vapoursynth\plugins\.dll. # -# vszip (core.vszip.CLAHE, Colour Correction's local contrast, issue #90) holds +# vszip (core.vszip.CLAHE, Colour Correction's local contrast, issue #90; and +# from 24.0.0, the first release with it, core.vszip.NLMeans for Noise +# Reduction's CPU non-local means and the Dehalo pass's vine method) holds # three builds under vapoursynth\plugins\vszip\ - vszip.dll (baseline x86-64), # vszip.avx2.dll and vszip.zn4.dll (AVX-512) - that VapourSynth picks between # through the wheel's manifest.vs. Only the baseline is taken, flat in @@ -793,7 +811,7 @@ if (-not (Test-Path "$PluginsDir\libakarin.dll")) { $WheelPlugins = @( @{ Name = "bwdif"; Package = "vapoursynth-bwdif"; Version = "5.1"; Dll = "bwdif.dll" }, @{ Name = "dedot"; Package = "vapoursynth-dedot"; Version = "3.0"; Dll = "dedot.dll" }, - @{ Name = "vszip"; Package = "vapoursynth-vszip"; Version = "22.1.0"; Dll = "vszip.dll"; Src = "vszip\vszip.dll" }, + @{ Name = "vszip"; Package = "vapoursynth-vszip"; Version = "24.0.0"; Dll = "vszip.dll"; Src = "vszip\vszip.dll" }, # core.bm3d (Noise Reduction's BM3D method). Wheel 12.0 is git tag r12, as # built from source by download-deps-{macos,linux}.sh. fftw is linked in # statically: the DLL imports only the UCRT. @@ -808,7 +826,12 @@ $WheelPlugins = @( # zip (2018), which refuses every 9-15-bit clip except packed YUV444P10 / # RGB30 - so any 10-bit 4:2:0 source failed (issue #144). OpenCL is linked # statically; the DLL imports only Windows system libraries. - @{ Name = "knlmeanscl"; Package = "vapoursynth-knlmeanscl"; Version = "1.3"; Dll = "KNLMeansCL.dll" } + @{ Name = "knlmeanscl"; Package = "vapoursynth-knlmeanscl"; Version = "1.3"; Dll = "KNLMeansCL.dll" }, + # core.sangnom.SangNom (Anti-aliasing's SangNom method). Wheel 45.0 is git + # tag r45, the same release the macOS/Linux wheels give. SSE2 only (no AVX + # in the DLL), so one file serves both tiers; imports only the UCRT. Source + # is MIT, upstream states its binaries are GPL v2 (licenses\NOTICES.txt). + @{ Name = "sangnom"; Package = "vapoursynth-sangnom"; Version = "45.0"; Dll = "sangnom.dll" } ) foreach ($W in $WheelPlugins) { @@ -816,7 +839,7 @@ foreach ($W in $WheelPlugins) { Write-Host "Downloading $($W.Name) $($W.Version) (PyPI wheel)..." -ForegroundColor Yellow if (-not (Test-Path "$PluginsDir\$($W.Dll)")) { try { - $Meta = Invoke-RestMethod -Uri "https://pypi.org/pypi/$($W.Package)/$($W.Version)/json" + $Meta = Invoke-WithRetry -What "PyPI $($W.Package)" { Invoke-RestMethod -Uri "https://pypi.org/pypi/$($W.Package)/$($W.Version)/json" } $Url = ($Meta.urls | Where-Object { $_.filename -like "*win_amd64.whl" } | Select-Object -First 1).url if (-not $Url) { throw "no win_amd64 wheel for $($W.Package) $($W.Version)" } @@ -824,7 +847,7 @@ foreach ($W in $WheelPlugins) { $Whl = Join-Path $TempDir "$($W.Name).whl" $Zip = Join-Path $TempDir "$($W.Name)-wheel.zip" $Out = Join-Path $TempDir "$($W.Name)-extract" - Invoke-WebRequest -Uri $Url -OutFile $Whl -UseBasicParsing + Download-File -Url $Url -OutFile $Whl # Expand-Archive validates the *extension* and refuses .whl outright. Copy-Item $Whl $Zip -Force Remove-Item $Out -Recurse -Force -ErrorAction SilentlyContinue @@ -855,40 +878,27 @@ foreach ($W in $WheelPlugins) { # RIFE job (see worker\src\gpu_plugin.rs), so a machine without a Vulkan driver, or # without the MSVC OpenMP runtime (VCOMP140.DLL) the DLL imports, fails that # method alone - and the app's probe names which one is missing. vulkan-1.dll -# comes with the GPU driver and is not ours to ship. Keep $RifeTag, the model -# commit and the model hashes in step with download-deps-{macos,linux}.sh. +# comes with the GPU driver and is not ours to ship. Keep $RifeTag in step +# with download-deps-{macos,linux}.sh. # -# The model is the ncnn conversion committed to the plugin's repo (there is no -# release asset for it), fetched at a pinned commit and checked by hash. +# The plugin only. Its model is the `frame-rate-rife` add-on (deps 1.16.0 on; +# app\assets\addons\frame-rate-rife.json): every AI model is an add-on, never +# in this bundle (docs\ADDONS.md). $RifeTag = "r9_mod_v33" $RifeDllSha256 = "36a25b471be88e6f915320c818022dc8657dd9beac22a8c3158bd7f4260cc410" -$RifeModelCommit = "96fdcecdf982ef3237e16e2b105eea849fa022ab" -$RifeModelName = "rife-v4.26_ensembleFalse" -$RifeModelHashes = @{ - "flownet.bin" = "94d58e30b75d7c7609cfa6f3bdad524deddd14f5f75e85c36d2f827ef5c64731" - "flownet.param" = "79f16c28903f93f8308f0c4c947f8c7e0c17d99a57b85473e5f298dd578d137b" -} $RifeDir = "$FullTargetDir\vapoursynth\rife" -$RifeModelDir = "$RifeDir\models\$RifeModelName" Write-Host "" -Write-Host "Downloading RIFE $RifeTag (model $RifeModelName)..." -ForegroundColor Yellow -if (-not ((Test-Path "$RifeDir\librife.dll") -and (Test-Path "$RifeModelDir\flownet.bin"))) { +Write-Host "Downloading RIFE $RifeTag..." -ForegroundColor Yellow +if (-not ((Test-Path "$RifeDir\librife.dll") -and -not (Test-Path "$RifeDir\models"))) { try { Remove-Item $RifeDir -Recurse -Force -ErrorAction SilentlyContinue - New-Item -ItemType Directory -Force -Path $RifeModelDir | Out-Null + New-Item -ItemType Directory -Force -Path $RifeDir | Out-Null $RifeDll = Join-Path $TempDir "librife.dll" Download-File -Url "https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/releases/download/$RifeTag/librife_windows_x86-64.dll" -OutFile $RifeDll $Got = (Get-FileHash -Path $RifeDll -Algorithm SHA256).Hash.ToLower() if ($Got -ne $RifeDllSha256) { throw "librife.dll sha256 $Got, expected $RifeDllSha256" } Move-Item $RifeDll "$RifeDir\librife.dll" -Force - foreach ($F in $RifeModelHashes.Keys) { - $Out = Join-Path $TempDir $F - Download-File -Url "https://raw.githubusercontent.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/$RifeModelCommit/models/$RifeModelName/$F" -OutFile $Out - $Got = (Get-FileHash -Path $Out -Algorithm SHA256).Hash.ToLower() - if ($Got -ne $RifeModelHashes[$F]) { throw "$F sha256 $Got, expected $($RifeModelHashes[$F])" } - Move-Item $Out "$RifeModelDir\$F" -Force - } Write-Host " RIFE installed -> vapoursynth\rife\" -ForegroundColor Green } catch { Remove-Item $RifeDir -Recurse -Force -ErrorAction SilentlyContinue @@ -899,7 +909,7 @@ if (-not ((Test-Path "$RifeDir\librife.dll") -and (Test-Path "$RifeModelDir\flow } # ============================================================================= -# VSNCNN (vs-mlrt) + Real-ESRGAN / Real-CUGAN - Crop & Resize's AI upscaling +# VSNCNN (vs-mlrt) - runs Crop & Resize's AI upscalers # ============================================================================= # vs-mlrt's own Windows build, pinned and hash-checked. Lives in # vapoursynth\ncnn\, never in vs-plugins (see worker\src\gpu_plugin.rs). It links @@ -907,31 +917,23 @@ if (-not ((Test-Path "$RifeDir\librife.dll") -and (Test-Path "$RifeModelDir\flow # GPU driver's, at run time. One file serves both CPU tiers: ncnn dispatches # its SIMD at run time. # -# The models are vs-mlrt's ONNX conversions (2.4-5.2 MB each), shipped only -# inside one ~810 MB archive per release: fetched pinned and hash-checked, the -# five VapourBox uses extracted and each checked by its own hash. Keep every pin -# and hash in step with download-deps-{macos,linux}.sh and -# gpu_plugin::UPSCALE_MODELS. Needs 7-Zip, like the other .7z downloads. +# The plugin only. Every model it runs is an add-on (deps 1.16.0 on; +# app\assets\addons\upscale-*.json, docs\ADDONS.md) - the Real-ESRGAN anime and +# Real-CUGAN models included, which were bundled here until then. Keep every +# pin in step with download-deps-{macos,linux}.sh. Needs 7-Zip, like the other +# .7z downloads. $VsmlrtTag = "v15.16" $VsncnnArchiveSha256 = "2ccf0ec1f84cd60f4a744c04d2a40d988da32f6970542f4f81669d44e9ea2ff1" -$VsmlrtModelsSha256 = "d4eaa8c1c4459082185a9e1a2c54d7e2c776abeea6aef6aee0cb8f6c151dfb02" -$UpscaleModelHashes = [ordered]@{ - "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx" = "6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" - "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx" = "414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" - "cugan/up2x-latest-no-denoise.onnx" = "3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" - "cugan/up2x-latest-conservative.onnx" = "555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" - "cugan/up2x-latest-denoise3x.onnx" = "7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" -} $NcnnDir = "$FullTargetDir\vapoursynth\ncnn" Write-Host "" -Write-Host "Downloading VSNCNN $VsmlrtTag + upscaling models..." -ForegroundColor Yellow +Write-Host "Downloading VSNCNN $VsmlrtTag..." -ForegroundColor Yellow if (-not $Has7z) { Write-Host " Skipping (7-Zip not installed)" -ForegroundColor Yellow -} elseif (-not ((Test-Path "$NcnnDir\vsncnn.dll") -and (Test-Path "$NcnnDir\models\cugan\up2x-latest-denoise3x.onnx"))) { +} elseif (-not ((Test-Path "$NcnnDir\vsncnn.dll") -and -not (Test-Path "$NcnnDir\models"))) { try { Remove-Item $NcnnDir -Recurse -Force -ErrorAction SilentlyContinue - New-Item -ItemType Directory -Force -Path "$NcnnDir\models" | Out-Null + New-Item -ItemType Directory -Force -Path $NcnnDir | Out-Null $Work = Join-Path $TempDir "vsncnn" Remove-Item $Work -Recurse -Force -ErrorAction SilentlyContinue New-Item -ItemType Directory -Force -Path $Work | Out-Null @@ -945,24 +947,6 @@ if (-not $Has7z) { $Dll = Get-ChildItem -Path $Work -Recurse -Filter "vsncnn.dll" | Select-Object -First 1 if (-not $Dll) { throw "vsncnn.dll missing from the VSNCNN archive" } Copy-Item $Dll.FullName "$NcnnDir\vsncnn.dll" -Force - - $Models = Join-Path $Work "models.7z" - Download-File -Url "https://github.com/AmusementClub/vs-mlrt/releases/download/$VsmlrtTag/models.$VsmlrtTag.7z" -OutFile $Models - $Got = (Get-FileHash -Path $Models -Algorithm SHA256).Hash.ToLower() - if ($Got -ne $VsmlrtModelsSha256) { throw "models archive sha256 $Got, expected $VsmlrtModelsSha256" } - # One pass: the archive is solid, so separate extractions would each - # decompress it from the start again. - $Wanted = $UpscaleModelHashes.Keys | ForEach-Object { "models/$_" } - & $7zPath x $Models -o"$Work" -y @Wanted | Out-Null - if ($LASTEXITCODE -ne 0) { throw "7-Zip failed on the models archive (exit $LASTEXITCODE)" } - foreach ($Rel in $UpscaleModelHashes.Keys) { - $Src = Join-Path $Work ("models\" + $Rel.Replace('/', '\')) - $Got = (Get-FileHash -Path $Src -Algorithm SHA256).Hash.ToLower() - if ($Got -ne $UpscaleModelHashes[$Rel]) { throw "$Rel sha256 $Got, expected $($UpscaleModelHashes[$Rel])" } - $Dest = Join-Path "$NcnnDir\models" $Rel.Replace('/', '\') - New-Item -ItemType Directory -Force -Path (Split-Path $Dest) | Out-Null - Move-Item $Src $Dest -Force - } Remove-Item $Work -Recurse -Force -ErrorAction SilentlyContinue Write-Host " VSNCNN installed -> vapoursynth\ncnn\" -ForegroundColor Green } catch { diff --git a/Scripts/models/colorize_models_common.py b/Scripts/models/colorize_models_common.py new file mode 100644 index 00000000..c6b7f84f --- /dev/null +++ b/Scripts/models/colorize_models_common.py @@ -0,0 +1,96 @@ +"""Shared pieces of the colourisation model conversions +(make_colorize_ddcolor.py, make_colorize_deoldify.py). The download, export +and record plumbing is face_models_common's; downloads are cached in +addon-staging/colorize-work (FACE_MODELS_WORK overrides it).""" + +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import face_models_common as fm # noqa: E402 + +os.environ.setdefault("FACE_MODELS_WORK", os.path.join(fm.STAGING, "colorize-work")) + + +def bake_norms(net): + """Replace every spectral- or weight-normalised weight with its value. + + In eval mode torch's spectral_norm divides weight_orig by the sigma its + stored u and v give, on every call (and weight_norm recombines g and v), + so a traced graph keeps a second, reshaped copy of each weight to compute + the normalisation from (50 MB of DDColor-T's export). The remove_* + functions compute the same weight once, without a power iteration, so + the network's output is unchanged.""" + import torch + from torch.nn.utils.spectral_norm import SpectralNorm + from torch.nn.utils.weight_norm import WeightNorm + + for module in net.modules(): + for hook in list(module._forward_pre_hooks.values()): + if isinstance(hook, SpectralNorm): + torch.nn.utils.remove_spectral_norm(module, hook.name) + elif isinstance(hook, WeightNorm): + torch.nn.utils.remove_weight_norm(module, hook.name) + + +def edge_pad_top_left(): + """A module equal to nn.ReplicationPad2d((1, 0, 1, 0)) — repeat the first + row and column — written as slices and concatenations. Core ML's ML + Program has no edge-mode Pad, so the original splits the graph into a + partition per pad (DDColor's export: six partitions, 80 s to compile); + this exports as Slice + Concat, which it takes, and gives the same + numbers exactly.""" + import torch + from torch import nn + + class EdgePadTopLeft(nn.Module): + def forward(self, x): + x = torch.cat([x[:, :, :1], x], dim=2) + return torch.cat([x[:, :, :, :1], x], dim=3) + + return EdgePadTopLeft() + + +def replace_replication_pads(net): + """Swap every ReplicationPad2d((1, 0, 1, 0)) in `net` for edge_pad_top_left().""" + from torch import nn + + count = 0 + for module in list(net.modules()): + for name, child in list(module.named_children()): + if isinstance(child, nn.ReplicationPad2d) and tuple(child.padding) == (1, 0, 1, 0): + setattr(module, name, edge_pad_top_left()) + count += 1 + return count + + +def grey_to_rgb(luma): + """(1, 1, H, W) -> (1, 3, H, W) by concatenation, which Core ML takes + (Tensor.repeat exports as Expand/Tile, which it does not).""" + import torch + + return torch.cat([luma, luma, luma], dim=1) + + +def test_input(size, seed=0): + """A deterministic grey (1, 1, size, size) picture in [0, 1]: a soft + blob, two hard edges and noise, so the networks see structure.""" + rng = np.random.default_rng(seed) + y, x = np.mgrid[0:size, 0:size].astype(np.float32) / size + img = 0.35 + 0.3 * np.exp(-((x - 0.4) ** 2 + (y - 0.35) ** 2) / 0.05) + img += 0.25 * (x > 0.7) - 0.15 * (y > 0.8) + img += rng.normal(0, 0.03, img.shape).astype(np.float32) + return np.clip(img, 0, 1)[None, None].astype(np.float32) + + +def parity_at_sizes(sess, refs, input_name, unit): + """ONNX Runtime's output against PyTorch references {size: (x, ref)}.""" + out = {} + for size, (x, ref) in refs.items(): + got = sess.run(None, {input_name: x})[0] + d = np.abs(ref.astype(np.float64) - got) + out[str(size)] = {"max": float(d.max()), "mean": float(d.mean()), "unit": unit, + "refRange": [float(ref.min()), float(ref.max())]} + return out diff --git a/Scripts/models/colorize_provider_bench.py b/Scripts/models/colorize_provider_bench.py new file mode 100644 index 00000000..7f4ca36e --- /dev/null +++ b/Scripts/models/colorize_provider_bench.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Time the colourisation ONNX models at each render size on every execution +provider this machine's onnxruntime offers, and compare each provider's +output with the CPU's. Used for the provider table in +docs/reports/2026-11-colourisation. + + python Scripts/models/colorize_provider_bench.py [runs] + +Sessions are made the way worker/templates/ai_runtime.py makes them (Core ML +as an MLProgram on all compute units, the render size fixed with +free-dimension overrides); "dynamic" rows leave the size free, for +comparison. Session creation is timed separately: it includes Core ML's +compilation of the graph. +""" + +import json +import os +import sys +import time + +import numpy as np +import onnxruntime as ort + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from colorize_models_common import test_input # noqa: E402 + +SIZES = { + "colorize-ddcolor-tiny.onnx": (256, 384, 512), + "colorize-deoldify-video.onnx": (256, 336, 448), +} +OPTIONS = {"CoreMLExecutionProvider": {"ModelFormat": "MLProgram", "MLComputeUnits": "ALL"}} + + +def session(path, provider, size=None): + opts = ort.SessionOptions() + opts.log_severity_level = 3 + if size: + opts.add_free_dimension_override_by_name("height", size) + opts.add_free_dimension_override_by_name("width", size) + providers = [(provider, OPTIONS[provider]) if provider in OPTIONS else provider] + if provider != "CPUExecutionProvider": + providers.append("CPUExecutionProvider") + return ort.InferenceSession(path, sess_options=opts, providers=providers) + + +def main(): + staging = sys.argv[1] + runs = int(sys.argv[2]) if len(sys.argv) > 2 else 5 + only = sys.argv[3:] # optional: model files, providers or "dynamic" + providers = [p for p in ort.get_available_providers() if p in ("CoreMLExecutionProvider", "CPUExecutionProvider")] + for name, sizes in SIZES.items(): + path = os.path.join(staging, name) + if not os.path.exists(path) or (only and name not in only and not any(o.endswith("Provider") or o == "dynamic" for o in only)): + continue + for size in sizes: + x = test_input(size, seed=size) + cpu_out = None + variants = [(p, True) for p in sorted(providers, key=lambda p: p != "CPUExecutionProvider")] + if "dynamic" in only: + variants.append(("CoreMLExecutionProvider", False)) + for p, fixed in variants: + t0 = time.perf_counter() + try: + s = session(path, p, size if fixed else None) + except Exception as e: # Core ML refuses some dynamic shapes outright + print(json.dumps({"model": name, "size": size, "provider": p, "fixedShape": fixed, + "error": str(e)[:200]}), flush=True) + continue + created = time.perf_counter() - t0 + feeds = {s.get_inputs()[0].name: x} + out = s.run(None, feeds) + t0 = time.perf_counter() + for _ in range(runs): + out = s.run(None, feeds) + per = (time.perf_counter() - t0) / runs + row = {"model": name, "size": size, "provider": p, "fixedShape": fixed, + "sessionSeconds": round(created, 2), "msPerRun": round(per * 1000, 1), + "onnxruntime": ort.__version__} + if p == "CPUExecutionProvider": + cpu_out = out + elif cpu_out is not None: + row["maxDiffVsCpu"] = float(max(np.abs(a - b).max() for a, b in zip(out, cpu_out))) + row["meanDiffVsCpu"] = float(max(np.abs(a - b).mean() for a, b in zip(out, cpu_out))) + print(json.dumps(row), flush=True) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/convert_realesrgan.py b/Scripts/models/convert_realesrgan.py new file mode 100644 index 00000000..d4d2a16e --- /dev/null +++ b/Scripts/models/convert_realesrgan.py @@ -0,0 +1,288 @@ +#!/usr/bin/env python3 +"""Convert xinntao's Real-ESRGAN PyTorch checkpoints to ONNX for VSNCNN. + +Upstream (https://github.com/xinntao/Real-ESRGAN, BSD-3-Clause) publishes +these models only as `.pth` checkpoints. The Crop & Resize AI upscaler runs +networks through vs-mlrt's VSNCNN (`ncnn.Model`), which reads ONNX, so we +convert them and host the result on the `addon-models-v1` release (see +docs/ADDONS.md, "Hosting files we convert ourselves"). + +Two architectures, each defined here so no basicsr/realesrgan install is +needed (both transcribed from basicsr 1.4.2 / realesrgan 0.3.0, the code the +checkpoints were trained with): + +- SRVGGNetCompact (`realesr-general-x4v3`, `realesr-general-wdn-x4v3`): the + same architecture as vs-mlrt's `RealESRGANv2-animevideo-*` ONNX the deps + bundle ships, with 32 body convs instead of 16. +- RRDBNet (`RealESRGAN_x4plus`, `RealESRGAN_x2plus`): 23 RRDB blocks. + +The ONNX match the conventions of the models VSNCNN already runs: one input +`input` and one output `output`, float32 NCHW RGB in 0..1, batch, height and +width dynamic, opset 13, IR version 7 (vs-mlrt's own conversions). Two +deliberate choices keep them inside what ncnn's ONNX importer handles: + +- x2plus starts with `pixel_unshuffle(2)`, which the TorchScript exporter + writes as a 6-D Reshape/Transpose pair ncnn cannot import. It is replaced by + the equivalent stride-2 convolution with one-hot weights (exact: every output + is one input sample times 1.0). +- The legacy (TorchScript) exporter is used, not dynamo: it writes plain + Conv/LeakyRelu/PRelu/Resize/Add graphs at opset 13. + +Every checkpoint is pinned by URL and SHA-256; a different download fails. +With torch 2.15.0 and onnx 1.23.2 (the conversion venv of 2026-11) the output +is byte-for-byte reproducible: two runs give the hashes pinned in +`app/assets/addons/upscale-general.json` / `upscale-esrgan-plus.json`. +After export the script checks PyTorch against ONNX Runtime on a seeded random +image and a smooth synthetic one, and prints the maximum and mean absolute +difference (in 0..1 units) — the parity numbers in +docs/reports/2026-11-ai-upscale-addons/. + +Usage (in the conversion venv, see docs/ADDONS.md): + + python -I Scripts/models/convert_realesrgan.py [model ...] + +`` holds the downloaded `.pth` files (or they are fetched +into it); each model is written as `/upscale-.onnx`. +""" +import hashlib +import os +import sys +import urllib.request + +import numpy as np +import onnx +import onnxruntime as ort +import torch +from torch import nn +from torch.nn import functional as F + +OPSET = 13 +IR_VERSION = 7 + +REL = "https://github.com/xinntao/Real-ESRGAN/releases/download" + +# name -> (checkpoint url, sha256, architecture, kwargs) +MODELS = { + "realesr-general-x4v3": ( + f"{REL}/v0.2.5.0/realesr-general-x4v3.pth", + "8dc7edb9ac80ccdc30c3a5dca6616509367f05fbc184ad95b731f05bece96292", + "srvgg", dict(num_feat=64, num_conv=32, upscale=4), + ), + "realesr-general-wdn-x4v3": ( + f"{REL}/v0.2.5.0/realesr-general-wdn-x4v3.pth", + "1641f8c4464b9f097c9fdda5589273713f67cf59f3d909e0bd688f0cee269dca", + "srvgg", dict(num_feat=64, num_conv=32, upscale=4), + ), + "RealESRGAN_x4plus": ( + f"{REL}/v0.1.0/RealESRGAN_x4plus.pth", + "4fa0d38905f75ac06eb49a7951b426670021be3018265fd191d2125df9d682f1", + "rrdb", dict(scale=4), + ), + "RealESRGAN_x2plus": ( + f"{REL}/v0.2.1/RealESRGAN_x2plus.pth", + "49fafd45f8fd7aa8d31ab2a22d14d91b536c34494a5cfe31eb5d89c2fa266abb", + "rrdb", dict(scale=2), + ), +} + + +# --- SRVGGNetCompact (realesrgan/archs/srvgg_arch.py) ----------------------- + +class SRVGGNetCompact(nn.Module): + def __init__(self, num_in_ch=3, num_out_ch=3, num_feat=64, num_conv=16, upscale=4): + super().__init__() + self.upscale = upscale + self.body = nn.ModuleList() + self.body.append(nn.Conv2d(num_in_ch, num_feat, 3, 1, 1)) + self.body.append(nn.PReLU(num_parameters=num_feat)) + for _ in range(num_conv): + self.body.append(nn.Conv2d(num_feat, num_feat, 3, 1, 1)) + self.body.append(nn.PReLU(num_parameters=num_feat)) + self.body.append(nn.Conv2d(num_feat, num_out_ch * upscale * upscale, 3, 1, 1)) + self.upsampler = nn.PixelShuffle(upscale) + + def forward(self, x): + out = x + for layer in self.body: + out = layer(out) + out = self.upsampler(out) + return out + F.interpolate(x, scale_factor=self.upscale, mode="nearest") + + +# --- RRDBNet (basicsr/archs/rrdbnet_arch.py) -------------------------------- + +class ResidualDenseBlock(nn.Module): + def __init__(self, num_feat=64, num_grow_ch=32): + super().__init__() + self.conv1 = nn.Conv2d(num_feat, num_grow_ch, 3, 1, 1) + self.conv2 = nn.Conv2d(num_feat + num_grow_ch, num_grow_ch, 3, 1, 1) + self.conv3 = nn.Conv2d(num_feat + 2 * num_grow_ch, num_grow_ch, 3, 1, 1) + self.conv4 = nn.Conv2d(num_feat + 3 * num_grow_ch, num_grow_ch, 3, 1, 1) + self.conv5 = nn.Conv2d(num_feat + 4 * num_grow_ch, num_feat, 3, 1, 1) + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + def forward(self, x): + x1 = self.lrelu(self.conv1(x)) + x2 = self.lrelu(self.conv2(torch.cat((x, x1), 1))) + x3 = self.lrelu(self.conv3(torch.cat((x, x1, x2), 1))) + x4 = self.lrelu(self.conv4(torch.cat((x, x1, x2, x3), 1))) + x5 = self.conv5(torch.cat((x, x1, x2, x3, x4), 1)) + return x5 * 0.2 + x + + +class RRDB(nn.Module): + def __init__(self, num_feat, num_grow_ch=32): + super().__init__() + self.rdb1 = ResidualDenseBlock(num_feat, num_grow_ch) + self.rdb2 = ResidualDenseBlock(num_feat, num_grow_ch) + self.rdb3 = ResidualDenseBlock(num_feat, num_grow_ch) + + def forward(self, x): + out = self.rdb3(self.rdb2(self.rdb1(x))) + return out * 0.2 + x + + +class PixelUnshuffleConv(nn.Module): + """`F.pixel_unshuffle(x, s)` as a stride-s convolution with one-hot + weights: output channel c*s*s + i*s + j takes input channel c at offset + (i, j) of each s x s cell — pixel_unshuffle's channel order.""" + + def __init__(self, channels, s): + super().__init__() + w = torch.zeros(channels * s * s, channels, s, s) + for c in range(channels): + for i in range(s): + for j in range(s): + w[c * s * s + i * s + j, c, i, j] = 1.0 + self.register_buffer("weight", w) + self.s = s + + def forward(self, x): + return F.conv2d(x, self.weight, stride=self.s) + + +class RRDBNet(nn.Module): + def __init__(self, num_in_ch=3, num_out_ch=3, scale=4, num_feat=64, num_block=23, num_grow_ch=32): + super().__init__() + self.scale = scale + if scale == 2: + num_in_ch = num_in_ch * 4 + self.unshuffle = PixelUnshuffleConv(3, 2) + elif scale == 1: + num_in_ch = num_in_ch * 16 + self.unshuffle = PixelUnshuffleConv(3, 4) + else: + self.unshuffle = None + self.conv_first = nn.Conv2d(num_in_ch, num_feat, 3, 1, 1) + self.body = nn.Sequential(*[RRDB(num_feat, num_grow_ch) for _ in range(num_block)]) + self.conv_body = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_up1 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_up2 = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_hr = nn.Conv2d(num_feat, num_feat, 3, 1, 1) + self.conv_last = nn.Conv2d(num_feat, num_out_ch, 3, 1, 1) + self.lrelu = nn.LeakyReLU(negative_slope=0.2, inplace=True) + + def forward(self, x): + feat = self.unshuffle(x) if self.unshuffle is not None else x + feat = self.conv_first(feat) + feat = feat + self.conv_body(self.body(feat)) + feat = self.lrelu(self.conv_up1(F.interpolate(feat, scale_factor=2, mode="nearest"))) + feat = self.lrelu(self.conv_up2(F.interpolate(feat, scale_factor=2, mode="nearest"))) + return self.conv_last(self.lrelu(self.conv_hr(feat))) + + +# --- conversion ------------------------------------------------------------- + +def sha256(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fetch(url, digest, ckpt_dir): + path = os.path.join(ckpt_dir, url.rsplit("/", 1)[1]) + if not os.path.exists(path): + print(f"downloading {url}") + urllib.request.urlretrieve(url, path) + got = sha256(path) + if got != digest: + sys.exit(f"{path}: sha256 {got}, expected {digest}") + return path + + +def build(arch, kwargs, ckpt): + net = SRVGGNetCompact(**kwargs) if arch == "srvgg" else RRDBNet(**kwargs) + state = torch.load(ckpt, map_location="cpu", weights_only=True) + state = state.get("params_ema", state.get("params", state)) + missing, unexpected = net.load_state_dict(state, strict=False) + # The only parameter not in the checkpoint is our unshuffle conv's buffer. + if unexpected or any(not k.startswith("unshuffle.") for k in missing): + sys.exit(f"{ckpt}: missing {missing}, unexpected {unexpected}") + if getattr(net, "unshuffle", None) is not None: + # The replacement must be exactly upstream's pixel_unshuffle. + x = torch.rand(1, 3, 12, 16) + if not torch.equal(net.unshuffle(x), F.pixel_unshuffle(x, net.unshuffle.s)): + sys.exit("PixelUnshuffleConv differs from F.pixel_unshuffle") + return net.eval() + + +def test_images(h=96, w=128): + rng = np.random.default_rng(1) + noise = rng.random((1, 3, h, w), dtype=np.float32) + yy, xx = np.mgrid[0:h, 0:w].astype(np.float32) + smooth = np.stack([ + 0.5 + 0.4 * np.sin(xx / 9.0), + 0.5 + 0.4 * np.cos(yy / 7.0), + (xx + yy) / (h + w), + ])[None].astype(np.float32) + return {"random": noise, "smooth": smooth} + + +def convert(name, ckpt_dir, out_dir): + url, digest, arch, kwargs = MODELS[name] + net = build(arch, kwargs, fetch(url, digest, ckpt_dir)) + out = os.path.join(out_dir, f"upscale-{name}.onnx") + dummy = torch.rand(1, 3, 64, 64) + torch.onnx.export( + net, (dummy,), out, dynamo=False, opset_version=OPSET, + input_names=["input"], output_names=["output"], + dynamic_axes={"input": {0: "batch", 2: "height", 3: "width"}, + "output": {0: "batch", 2: "height", 3: "width"}}, + do_constant_folding=True, + ) + model = onnx.load(out) + model.ir_version = IR_VERSION + model.producer_name = "vapourbox" + onnx.checker.check_model(model) + onnx.save(model, out) + + sess = ort.InferenceSession(out, providers=["CPUExecutionProvider"]) + ops = sorted({n.op_type for n in model.graph.node}) + print(f"{name}: {os.path.getsize(out)} bytes, sha256 {sha256(out)}, ops {ops}") + for label, img in test_images().items(): + with torch.no_grad(): + ref = net(torch.from_numpy(img)).numpy() + got = sess.run(None, {"input": img})[0] + diff = np.abs(ref - got) + print(f" parity {label}: shape {got.shape}, max {diff.max():.3e}, mean {diff.mean():.3e}" + f" (8-bit levels: max {diff.max() * 255:.4f})") + return out + + +def main(): + if len(sys.argv) < 3: + sys.exit(__doc__) + ckpt_dir, out_dir = sys.argv[1], sys.argv[2] + names = sys.argv[3:] or list(MODELS) + for n in names: + if n not in MODELS: + sys.exit(f"unknown model {n}; known: {', '.join(MODELS)}") + torch.manual_seed(0) + for n in names: + convert(n, ckpt_dir, out_dir) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/face_models_common.py b/Scripts/models/face_models_common.py new file mode 100644 index 00000000..50936202 --- /dev/null +++ b/Scripts/models/face_models_common.py @@ -0,0 +1,193 @@ +"""Shared plumbing for the face-restoration model conversions +(make_face_restoration.py, make_face_gfpgan.py, make_face_codeformer.py). + +Each conversion fetches an upstream PyTorch checkpoint (pinned by URL and +SHA-256) and the upstream source that defines its network (pinned by commit), +exports it to ONNX with the TorchScript exporter at a fixed opset and IR +version, and checks the ONNX model against PyTorch on the same inputs. + +Opset 17 / IR 8 keeps every model loadable by the oldest ONNX Runtime the +ai-runtime add-on pins (1.23.2, the last with an Intel macOS wheel). + +Run in the conversion venv (docs/ADDONS.md). The work directory caches the +downloads; it defaults to addon-staging/face-work at the repo root +(FACE_MODELS_WORK overrides it, VAPOURBOX_ADDON_STAGING the staging directory). +""" + +import hashlib +import io +import json +import os +import sys +import tarfile +import time +import types +import urllib.request + +import numpy as np + +OPSET = 17 +IR_VERSION = 8 + +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +STAGING = os.environ.get("VAPOURBOX_ADDON_STAGING") or os.path.join(REPO_ROOT, "addon-staging") + + +def work_dir(): + d = os.environ.get("FACE_MODELS_WORK") or os.path.join(STAGING, "face-work") + os.makedirs(d, exist_ok=True) + return d + + +def sha256_of(path): + h = hashlib.sha256() + with open(path, "rb") as f: + for chunk in iter(lambda: f.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +def fetch(url, sha256): + """Download `url` into the work directory's ckpt/ (once) and check its hash.""" + dest_dir = os.path.join(work_dir(), "ckpt") + os.makedirs(dest_dir, exist_ok=True) + dest = os.path.join(dest_dir, url.rsplit("/", 1)[1]) + if not os.path.exists(dest): + print(f"downloading {url}") + tmp = dest + ".partial" + urllib.request.urlretrieve(url, tmp) + os.replace(tmp, dest) + got = sha256_of(dest) + if got != sha256: + raise SystemExit(f"{dest}: SHA-256 {got}, expected {sha256}") + return dest + + +def upstream_source(repo, commit): + """Unpack github.com/ at `commit` into the work directory's src/ and + return its path. Only the Python files are needed (the network classes).""" + name = repo.split("/")[1] + dest = os.path.join(work_dir(), "src", f"{name}-{commit[:12]}") + if not os.path.isdir(dest): + url = f"https://codeload.github.com/{repo}/tar.gz/{commit}" + print(f"downloading {url}") + data = urllib.request.urlopen(url).read() + os.makedirs(dest + ".partial", exist_ok=True) + with tarfile.open(fileobj=io.BytesIO(data), mode="r:gz") as tar: + for m in tar.getmembers(): + parts = m.name.split("/", 1) + if len(parts) < 2 or not m.isfile() or not m.name.endswith(".py"): + continue + m.name = parts[1] + tar.extract(m, dest + ".partial", filter="data") + os.replace(dest + ".partial", dest) + return dest + + +def stub_module(name, **attrs): + """Insert an empty module (and its parents) into sys.modules, so upstream + code that imports training-only helpers can be imported for its network + classes alone.""" + parts = name.split(".") + for i in range(1, len(parts) + 1): + sub = ".".join(parts[:i]) + if sub not in sys.modules: + mod = types.ModuleType(sub) + mod.__path__ = [] + sys.modules[sub] = mod + if i > 1: + setattr(sys.modules[".".join(parts[: i - 1])], parts[i - 1], mod) + for k, v in attrs.items(): + setattr(sys.modules[name], k, v) + return sys.modules[name] + + +class Registry: + """Stand-in for basicsr's ARCH_REGISTRY: register() is a no-op decorator.""" + + def register(self, *args, **kwargs): + if args and callable(args[0]): + return args[0] + return lambda cls: cls + + +def load_checkpoint(path): + import torch + + return torch.load(path, map_location="cpu", weights_only=True) + + +def export(model, inputs, input_names, output_names, out_path, dynamic_axes=None): + """Export `model` (in eval mode) with the TorchScript exporter at OPSET, + stamp IR_VERSION, check it, and return the file's size and SHA-256.""" + import onnx + import torch + + model.eval() + tmp = out_path + ".tmp" + with torch.no_grad(): + torch.onnx.export( + model, + tuple(inputs), + tmp, + input_names=input_names, + output_names=output_names, + opset_version=OPSET, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + dynamo=False, + ) + m = onnx.load(tmp) + m.ir_version = IR_VERSION + m.producer_name = "vapourbox" + onnx.checker.check_model(m) + onnx.save(m, out_path) + os.remove(tmp) + return {"file": os.path.basename(out_path), "size": os.path.getsize(out_path), "sha256": sha256_of(out_path)} + + +def parity(model, onnx_path, feeds, providers=("CPUExecutionProvider",)): + """Max and mean absolute difference between PyTorch and ONNX Runtime on + the same inputs, per output, and the time ONNX Runtime took.""" + import onnxruntime as ort + import torch + + with torch.no_grad(): + ref = model(*[torch.from_numpy(v) for v in feeds.values()]) + if not isinstance(ref, (tuple, list)): + ref = [ref] + ref = [r.numpy() for r in ref if isinstance(r, torch.Tensor)] + sess = ort.InferenceSession(onnx_path, providers=list(providers)) + t0 = time.perf_counter() + got = sess.run(None, feeds) + dt = time.perf_counter() - t0 + out = [] + for name, a, b in zip([o.name for o in sess.get_outputs()], ref, got): + d = np.abs(a.astype(np.float64) - b.astype(np.float64)) + out.append({"output": name, "max": float(d.max()), "mean": float(d.mean())}) + return {"providers": list(providers), "seconds": dt, "outputs": out} + + +def write_record(name, record): + """Write the conversion record (sizes, hashes, parity) beside the models.""" + path = os.path.join(STAGING, f"{name}.conversion.json") + with open(path, "w") as f: + json.dump(record, f, indent=2) + print(json.dumps(record, indent=2)) + return path + + +def face_test_input(seed=0): + """A deterministic face-like 512x512 test input in [-1, 1]: smooth blobs + plus noise, so the networks see structure rather than pure noise.""" + rng = np.random.default_rng(seed) + y, x = np.mgrid[0:512, 0:512].astype(np.float32) / 512.0 + img = np.stack( + [ + 0.6 * np.exp(-((x - 0.5) ** 2 + (y - 0.5) ** 2) / 0.08) - 0.2, + 0.5 * np.exp(-((x - 0.5) ** 2 + (y - 0.45) ** 2) / 0.06) - 0.3, + 0.4 * np.exp(-((x - 0.5) ** 2 + (y - 0.55) ** 2) / 0.1) - 0.35, + ] + ) + img += rng.normal(0, 0.05, img.shape).astype(np.float32) + return np.clip(img, -1, 1)[None].astype(np.float32) diff --git a/Scripts/models/face_provider_bench.py b/Scripts/models/face_provider_bench.py new file mode 100644 index 00000000..d3d8d405 --- /dev/null +++ b/Scripts/models/face_provider_bench.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Time each face-restoration ONNX model on every execution provider this +machine's onnxruntime offers, and compare each provider's output with the +CPU's. Used for the provider table in docs/reports/issue-face-restoration. + + python Scripts/models/face_provider_bench.py [runs] + +Session options mirror worker/templates/ai_runtime.py (Core ML as an +MLProgram on all compute units). +""" + +import json +import os +import sys +import time + +import numpy as np +import onnxruntime as ort + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from face_models_common import face_test_input # noqa: E402 + +MODELS = { + "face-restoration-retinaface-r50.onnx": lambda: {"input": det_input()}, + "face-restoration-retinaface-mnet.onnx": lambda: {"input": det_input()}, + "face-restoration-bisenet.onnx": lambda: {"input": face_test_input()}, + "face-restoration-restoreformerpp.onnx": lambda: {"input": face_test_input()}, + "face-gfpgan-v1.4.onnx": lambda: {"input": face_test_input()}, + "face-codeformer-v0.1.0.onnx": lambda: {"input": face_test_input(), "w": np.array([0.7], np.float32)}, +} +OPTIONS = {"CoreMLExecutionProvider": {"ModelFormat": "MLProgram", "MLComputeUnits": "ALL"}} + + +def det_input(): + rng = np.random.default_rng(1) + return (rng.uniform(0, 255, (1, 3, 640, 640)) - 117.0).astype(np.float32) + + +def session(path, provider): + opts = ort.SessionOptions() + opts.log_severity_level = 3 + providers = [(provider, OPTIONS[provider]) if provider in OPTIONS else provider] + if provider != "CPUExecutionProvider": + providers.append("CPUExecutionProvider") + return ort.InferenceSession(path, sess_options=opts, providers=providers) + + +def main(): + staging = sys.argv[1] + runs = int(sys.argv[2]) if len(sys.argv) > 2 else 5 + providers = [p for p in ort.get_available_providers() if p in ("CoreMLExecutionProvider", "CPUExecutionProvider")] + results = [] + for name, feeds in MODELS.items(): + path = os.path.join(staging, name) + if not os.path.exists(path): + continue + f = feeds() + cpu_out = None + for p in sorted(providers, key=lambda p: p != "CPUExecutionProvider"): + t0 = time.perf_counter() + s = session(path, p) + created = time.perf_counter() - t0 + out = s.run(None, f) # warm-up + t0 = time.perf_counter() + for _ in range(runs): + out = s.run(None, f) + per = (time.perf_counter() - t0) / runs + row = {"model": name, "provider": p, "sessionSeconds": round(created, 2), "msPerRun": round(per * 1000, 1)} + if p == "CPUExecutionProvider": + cpu_out = out + else: + row["maxDiffVsCpu"] = float(max(np.abs(a - b).max() for a, b in zip(out, cpu_out))) + print(json.dumps(row), flush=True) + results.append(row) + return results + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_ai_runtime_selftest.py b/Scripts/models/make_ai_runtime_selftest.py new file mode 100644 index 00000000..50c75c1a --- /dev/null +++ b/Scripts/models/make_ai_runtime_selftest.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python3 +"""Build the AI runtime's self-test model: y = x * 2 + 1 on a float32 NCHW +tensor with three channels and any height and width. + +`vapourbox-worker --probe-ai-runtime` runs it through ONNX Runtime from the +ai-runtime add-on to prove that the add-on imports, that a session can be +created on the chosen execution provider, and that it computes the right +answer. It is deliberately tiny (two nodes, no weights) so it can live inside +`worker/templates/ai_runtime.py` as base64 and ship wherever the templates do. + +Written with an explicit IR version and opset rather than the onnx package's +defaults, so the oldest ONNX Runtime the add-on pins (1.23.2, the last with an +Intel macOS wheel) can load it. + +Usage (in the conversion venv, see docs/ADDONS.md): + + python Scripts/models/make_ai_runtime_selftest.py Tests/TestResources/ai_runtime_selftest.onnx + +then paste the printed base64 into `_SELFTEST_MODEL_B64` in +`worker/templates/ai_runtime.py`. `ai_runtime_selftest_test.dart` asserts the +two agree. +""" +import base64 +import sys + +import onnx +from onnx import TensorProto, helper + +IR_VERSION = 8 +OPSET = 13 + + +def build() -> onnx.ModelProto: + x = helper.make_tensor_value_info("x", TensorProto.FLOAT, [1, 3, "h", "w"]) + y = helper.make_tensor_value_info("y", TensorProto.FLOAT, [1, 3, "h", "w"]) + two = helper.make_tensor("two", TensorProto.FLOAT, [], [2.0]) + one = helper.make_tensor("one", TensorProto.FLOAT, [], [1.0]) + graph = helper.make_graph( + [ + helper.make_node("Mul", ["x", "two"], ["x2"]), + helper.make_node("Add", ["x2", "one"], ["y"]), + ], + "vapourbox_selftest", + [x], + [y], + initializer=[two, one], + ) + model = helper.make_model( + graph, + producer_name="vapourbox", + opset_imports=[helper.make_opsetid("", OPSET)], + ) + model.ir_version = IR_VERSION + onnx.checker.check_model(model) + return model + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit(__doc__) + data = build().SerializeToString() + with open(sys.argv[1], "wb") as f: + f.write(data) + print(f"{len(data)} bytes") + print(base64.b64encode(data).decode()) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_colorize_ddcolor.py b/Scripts/models/make_colorize_ddcolor.py new file mode 100644 index 00000000..3c8117fa --- /dev/null +++ b/Scripts/models/make_colorize_ddcolor.py @@ -0,0 +1,149 @@ +#!/usr/bin/env python3 +"""Convert DDColor-T (piddnad/DDColor, the ddcolor_paper_tiny checkpoint) to +colorize-ddcolor-tiny.onnx for the colorize-ddcolor add-on. + +DDColor predicts the a and b channels of CIELAB from a grey picture; the +Colourisation pass (worker/templates/colorize.py) puts them back together +with the source's own lightness, as upstream's ColorizationPipeline does. + +The exported graph wraps the network so its interface is the pass's: + + input "luma" (1, 1, H, W) float32, the grey picture in [0, 1] + output "ab" (1, 2, H, W) float32, CIELAB a and b (about -128..127) + +H and W are dynamic (named "height" and "width"), multiples of 32: DDColor's +decoder concatenates skip connections without resampling, so a size the +encoder's four stride-2 stages don't divide fails. The pass fixes them per +session (ai_runtime's free-dimension overrides), which gives Core ML the +static shapes it needs. Upstream feeds the grey picture as RGB with three +equal channels (the L of CIELAB with a = b = 0, converted back to RGB), so +the wrapper repeats the one channel; the network normalises with ImageNet's +mean and deviation itself. + +Licence: Apache-2.0 for the code (LICENSE) and the weights (the Hugging +Face model card's licence field). + +Usage (conversion venv, see docs/ADDONS.md): + + VAPOURBOX_ADDON_STAGING=/path/to/addon-staging python Scripts/models/make_colorize_ddcolor.py +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import colorize_models_common as cm # noqa: E402 +import face_models_common as fm # noqa: E402 + +DDCOLOR = ("piddnad/DDColor", "2adb63f2656ac41cbdf7b894cddd94121a3faf13") +# ddcolor_paper_tiny at Hugging Face revision cf9fd99c (the LFS hash the +# repository lists for pytorch_model.bin). +CKPT = ( + "https://huggingface.co/piddnad/ddcolor_paper_tiny/resolve/" + "cf9fd99c1d7472689ec7413441c1b799a51866a3/pytorch_model.bin", + "8a1277bc90a1bfbb6d2d83933a9a6bc821931879ca93e26e4fcec12165d41fce", +) +OUT = "colorize-ddcolor-tiny.onnx" +EXPORT_SIZE = 512 # upstream's input_size +PARITY_SIZES = (512, 256, 384, 640) + + +def ddcolor(): + import torch + + src = fm.upstream_source(*DDCOLOR) + # Import the network modules without running basicsr/__init__.py (which + # pulls in training code and its dependencies). + for pkg in ("basicsr", "basicsr.archs", "basicsr.archs.ddcolor_arch_utils", "ddcolor"): + fm.stub_module(pkg).__path__ = [os.path.join(src, *pkg.split("."))] + from ddcolor.model import DDColor + + net = DDColor( + encoder_name="convnext-t", + decoder_name="MultiScaleColorDecoder", + input_size=[EXPORT_SIZE, EXPORT_SIZE], + num_output_channels=2, + last_norm="Spectral", + do_normalize=False, + num_queries=100, + num_scales=3, + dec_layers=9, + ) + state = fm.load_checkpoint(fm.fetch(*CKPT)) + state = state.get("params", state) + # Upstream loads with strict=False; check that nothing is actually missing. + missing, unexpected = net.load_state_dict(state, strict=False) + if missing or unexpected: + raise SystemExit(f"checkpoint mismatch: missing {missing}, unexpected {unexpected}") + net.eval() + + class Colorize(torch.nn.Module): + def __init__(self, inner): + super().__init__() + self.inner = inner + + def forward(self, luma): + return self.inner(cm.grey_to_rgb(luma)) + + return Colorize(net).eval() + + +def core_ml_friendly(net): + """Rewrite the two kinds of operation Core ML's ML Program cannot take, + each into an exact equivalent: the decoder's ReplicationPad2d (four of + them) and the colour decoder's einsum("bqc,bchw->bqhw"), which becomes a + matrix product over the flattened image. Without these, Core ML splits + the graph into six partitions and compiles for about 80 s at every + session (docs/reports/2026-11-colourisation).""" + import types + + import torch + + import ddcolor.model as upstream + + pads = cm.replace_replication_pads(net) + + def einsum(equation, *operands): + if equation == "bqc,bchw->bqhw": + embed, feats = operands + return torch.matmul(embed, feats.flatten(2)).unflatten(2, feats.shape[2:]) + return torch.einsum(equation, *operands) + + proxy = types.ModuleType("torch_with_matmul_einsum") + proxy.__getattr__ = lambda name: getattr(torch, name) + proxy.einsum = einsum + upstream.torch = proxy + return {"replicationPadsRewritten": pads, "einsumRewritten": True} + + +def main(): + import onnxruntime as ort + import torch + + torch.manual_seed(0) + net = ddcolor() + # References from the network exactly as upstream loads it, before the + # spectral norms are baked in and the Core ML rewrites made for export. + refs = {} + for size in PARITY_SIZES: + xs = cm.test_input(size, seed=size) + with torch.no_grad(): + refs[size] = (xs, net(torch.from_numpy(xs)).numpy()) + cm.bake_norms(net) + rewrites = core_ml_friendly(net) + out = os.path.join(fm.STAGING, OUT) + x = cm.test_input(EXPORT_SIZE) + info = fm.export(net, [torch.from_numpy(x)], ["luma"], ["ab"], out, + dynamic_axes={"luma": {2: "height", 3: "width"}, "ab": {2: "height", 3: "width"}}) + info["checkpoint"] = CKPT[0] + info["checkpointSha256"] = CKPT[1] + info["source"] = f"https://github.com/{DDCOLOR[0]}/tree/{DDCOLOR[1]}" + info["rewrites"] = rewrites + sess = ort.InferenceSession(out, providers=["CPUExecutionProvider"]) + info["parity"] = cm.parity_at_sizes(sess, refs, "luma", "CIELAB a/b") + fm.write_record("colorize-ddcolor", {"opset": fm.OPSET, "irVersion": fm.IR_VERSION, + "torch": torch.__version__, "models": [info]}) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_colorize_deoldify.py b/Scripts/models/make_colorize_deoldify.py new file mode 100644 index 00000000..a73b2ab0 --- /dev/null +++ b/Scripts/models/make_colorize_deoldify.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +"""Convert DeOldify's Video generator (jantic/DeOldify, ColorizeVideo_gen.pth) +to colorize-deoldify-video.onnx for the colorize-deoldify add-on. + +DeOldify is built on fastai v1 (pinned to 1.0.60, Python 3.6-3.8 era), which +does not install alongside a current PyTorch. The generator is therefore +rebuilt here in plain PyTorch, module for module, from DeOldify's +deoldify/unet.py and layers.py at the pinned commit and the fastai 1.0.60 +layers they use (conv_layer, res_block, PixelShuffle_ICNR, SelfAttention, +MergeLayer, SigmoidRange), on torchvision's ResNet-101. The checkpoint loads +into it with strict=True (every one of its 754 tensors, no missing and no +extra key), and the result is checked against an independent export: +FaceFusion's deoldify.onnx (models-3.0.0), a conversion of the same weights +made with fastai itself (VERIFY_REFERENCE below; downloaded only to verify, +never shipped). + +What is DeOldify's video colouriser (get_video_colorizer, render_factor 21): +DynamicUnetWide(resnet101, nf_factor=2, blur, self-attention, spectral norm, +y_range=(-3, 3)) over a grey picture stretched to a render_factor x 16 pixel +square, ImageNet-normalised, denormalised after. The exported graph wraps it +so its interface is the pass's: + + input "luma" (1, 1, H, W) float32, the grey picture in [0, 1] + output "rgb" (1, 3, H, W) float32, the colourised picture in [0, 1] + +H and W are dynamic (named "height" and "width"), multiples of 16. The +colouriser keeps only the output's chroma (upstream: cv2's YUV, Y from the +source); the pass does the same in the source's own YUV +(worker/templates/colorize.py). Two departures, both at sizes upstream never +sees or in values it would mangle: + +- fastai's U-Net resamples a skip connection (nearest) only when its size + differs from the upsampled path's; the rebuilt block always resamples to the + skip's size (the identity when they agree), so one traced graph serves every + size rather than only the size it was traced at. +- Upstream converts the denormalised output to uint8 with no clamp, so a value + outside [0, 1] wraps around; the export clamps. + +The export also writes the blur's ReplicationPad2d as slices and +concatenations (colorize_models_common.edge_pad_top_left, exact), since Core +ML has no edge-mode Pad and would otherwise split the graph around each one. + +Licence: MIT for the code and, per DeOldify's README ("are trained by +ourselves and are released under the MIT license"), the weights. Copyright +(c) 2018 Jason Antic (LICENSE). + +Usage (conversion venv, see docs/ADDONS.md): + + VAPOURBOX_ADDON_STAGING=/path/to/addon-staging python Scripts/models/make_colorize_deoldify.py +""" + +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import colorize_models_common as cm # noqa: E402 +import face_models_common as fm # noqa: E402 + +DEOLDIFY_COMMIT = "jantic/DeOldify@5f86c28923799036ad4b1bf7af8c629ac65efd75" +# The README's "Video" generator link. The same file (same SHA-256) is +# mirrored at huggingface.co/spensercai/DeOldify (revision 448956a5), should +# DeepAI's host go away. +CKPT = ( + "https://data.deepai.org/deoldify/ColorizeVideo_gen.pth", + "e3d98bb6a222354c79f95485c2f078a89dc724e9183662506d9e0f5aafac83ad", +) +# Independent conversion of the same weights, for verification only. +VERIFY_REFERENCE = ( + "https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/deoldify.onnx", + "a00de09afacbed28b30ea63d174e1ade855b497c0fc61372217953c429696a9a", +) +OUT = "colorize-deoldify-video.onnx" +EXPORT_SIZE = 336 # render_factor 21, upstream's default for video +PARITY_SIZES = (336, 256, 512) +IMAGENET_MEAN = (0.485, 0.456, 0.406) +IMAGENET_STD = (0.229, 0.224, 0.225) + + +def build(): + import torch + import torch.nn.functional as F + import torchvision + from torch import nn + from torch.nn.utils import spectral_norm, weight_norm + + def conv(ni, nf, ks=3, norm="spectral", use_activ=True, bn=False, self_attention=False): + """DeOldify's custom_conv_layer / fastai's conv_layer: conv (spectral + or weight normalised), ReLU, BatchNorm, self-attention, in that order. + Bias only without BatchNorm.""" + c = nn.Conv2d(ni, nf, kernel_size=ks, padding=(ks - 1) // 2, bias=not bn) + c = spectral_norm(c) if norm == "spectral" else weight_norm(c) + layers = [c] + if use_activ: + layers.append(nn.ReLU(True)) + if bn: + layers.append(nn.BatchNorm2d(nf)) + if self_attention: + layers.append(SelfAttention(nf)) + return nn.Sequential(*layers) + + class SelfAttention(nn.Module): + """fastai 1.0.60's SelfAttention (SAGAN), spectral-normalised 1x1 conv1d.""" + + def __init__(self, n): + super().__init__() + self.query = spectral_norm(nn.Conv1d(n, n // 8, 1, bias=False)) + self.key = spectral_norm(nn.Conv1d(n, n // 8, 1, bias=False)) + self.value = spectral_norm(nn.Conv1d(n, n, 1, bias=False)) + self.gamma = nn.Parameter(torch.tensor([0.0])) + + def forward(self, x): + size = x.size() + x = x.view(*size[:2], -1) + f, g, h = self.query(x), self.key(x), self.value(x) + beta = F.softmax(torch.bmm(f.transpose(1, 2), g), dim=1) + o = self.gamma * torch.bmm(h, beta) + x + return o.view(*size).contiguous() + + class CustomPixelShuffle(nn.Module): + """DeOldify's CustomPixelShuffle_ICNR with blur (spectral norm + BN).""" + + def __init__(self, ni, nf): + super().__init__() + self.conv = conv(ni, nf * 4, ks=1, use_activ=False, bn=True) + self.shuf = nn.PixelShuffle(2) + self.pad = nn.ReplicationPad2d((1, 0, 1, 0)) + self.blur = nn.AvgPool2d(2, stride=1) + self.relu = nn.ReLU(True) + + def forward(self, x): + return self.blur(self.pad(self.shuf(self.relu(self.conv(x))))) + + class UnetBlockWide(nn.Module): + def __init__(self, up_in_c, x_in_c, n_out, self_attention): + super().__init__() + up_out = x_out = n_out // 2 + self.shuf = CustomPixelShuffle(up_in_c, up_out) + self.bn = nn.BatchNorm2d(x_in_c) + self.conv = conv(up_out + x_in_c, x_out, bn=True, self_attention=self_attention) + self.relu = nn.ReLU() + + def forward(self, up_in, s): + up_out = self.shuf(up_in) + # Always resampled to the skip's size (see the module docstring). + up_out = F.interpolate(up_out, s.shape[-2:], mode="nearest") + return self.conv(self.relu(torch.cat([up_out, self.bn(s)], dim=1))) + + class PixelShuffleICNR(nn.Module): + """fastai's PixelShuffle_ICNR with its defaults (weight norm). Its + blur=False does not turn the blur off: fastai 1.0.60 always creates + the AvgPool2d and tests the module (always true), not the flag, so + the blur runs. Leaving it out was the difference the verification + against FaceFusion's export caught (mean 0.016, max 0.12).""" + + def __init__(self, ni): + super().__init__() + self.conv = conv(ni, ni * 4, ks=1, norm="weight", use_activ=False) + self.shuf = nn.PixelShuffle(2) + self.pad = nn.ReplicationPad2d((1, 0, 1, 0)) + self.blur = nn.AvgPool2d(2, stride=1) + self.relu = nn.ReLU(True) + + def forward(self, x): + return self.blur(self.pad(self.shuf(self.relu(self.conv(x))))) + + class ResBlock(nn.Module): + """fastai's res_block(ni, norm_type=Spectral): two convs, then x + input.""" + + def __init__(self, ni): + super().__init__() + self.layers = nn.ModuleList([conv(ni, ni), conv(ni, ni)]) + + def forward(self, x): + return x + self.layers[1](self.layers[0](x)) + + class DynamicUnetWide(nn.Module): + """DeOldify's DynamicUnetWide for resnet101, nf_factor 2, as a module + list numbered as fastai's SequentialEx numbers it (so the + checkpoint's keys load unchanged).""" + + def __init__(self): + super().__init__() + r = torchvision.models.resnet101(weights=None) + body = nn.Sequential(r.conv1, r.bn1, r.relu, r.maxpool, r.layer1, r.layer2, r.layer3, r.layer4) + nf = 1024 + self.layers = nn.ModuleList([ + body, # 0 + nn.BatchNorm2d(2048), # 1 + nn.ReLU(), # 2 + nn.Sequential(conv(2048, 4096, bn=True), conv(4096, 2048, bn=True)), # 3 middle + UnetBlockWide(2048, 1024, nf, False), # 4 skip: layer3 + UnetBlockWide(512, 512, nf, True), # 5 skip: layer2, self-attention + UnetBlockWide(512, 256, nf, False), # 6 skip: layer1 + UnetBlockWide(512, 64, nf // 2, False), # 7 skip: the stem's ReLU + PixelShuffleICNR(256), # 8 + nn.Identity(), # 9 MergeLayer(dense): concat the input + ResBlock(259), # 10 + conv(259, 3, ks=1, use_activ=False), # 11 + ]) + + def forward(self, x): + skips = {} + h = x + for i, layer in enumerate(self.layers[0]): + h = layer(h) + if i in (2, 4, 5, 6): + skips[i] = h + h = self.layers[3](self.layers[2](self.layers[1](h))) + h = self.layers[4](h, skips[6]) + h = self.layers[5](h, skips[5]) + h = self.layers[6](h, skips[4]) + h = self.layers[7](h, skips[2]) + h = self.layers[8](h) + h = torch.cat([h, x], dim=1) + h = self.layers[10](h) + h = self.layers[11](h) + return torch.sigmoid(h) * 6.0 - 3.0 # SigmoidRange(-3, 3) + + return DynamicUnetWide() + + +def deoldify(): + import torch + + net = build() + torch.serialization.add_safe_globals([slice]) # fastai's checkpoint pickles a slice + ckpt = torch.load(fm.fetch(*CKPT), map_location="cpu", weights_only=True) + state = ckpt["model"] + net.load_state_dict(state, strict=True) + net.eval() + + class Colorize(torch.nn.Module): + """Grey [0, 1] in, RGB [0, 1] out, as ColorizerFilter._model_process.""" + + def __init__(self, inner): + super().__init__() + self.inner = inner + self.register_buffer("mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1)) + self.register_buffer("std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1)) + + def forward(self, luma): + x = (cm.grey_to_rgb(luma) - self.mean) / self.std + return torch.clamp(self.inner(x) * self.std + self.mean, 0.0, 1.0) + + return Colorize(net).eval(), len(state) + + +def verify_against_reference(net): + """Compare the rebuilt network with FaceFusion's independent export of + the same checkpoint, on that export's own interface (the normalised + three-channel grey picture in, the normalised picture out).""" + import onnxruntime as ort + import torch + + path = fm.fetch(*VERIFY_REFERENCE) + sess = ort.InferenceSession(path, providers=["CPUExecutionProvider"]) + inp = sess.get_inputs()[0] + shape = [d if isinstance(d, int) else 256 for d in inp.shape] + size = shape[2] + g = cm.test_input(size, seed=7) + # FaceFusion's graph takes the grey picture as 0-255 RGB and returns + # 0-255 RGB, normalising and denormalising inside; ours takes [0, 1] luma. + ref = sess.run(None, {inp.name: np.repeat(g, 3, axis=1) * 255.0})[0] / 255.0 + with torch.no_grad(): + got = net(torch.from_numpy(g)).numpy() + d = np.abs(ref.astype(np.float64) - got) + return {"reference": VERIFY_REFERENCE[0], "referenceSha256": VERIFY_REFERENCE[1], + "inputShape": shape, "max": float(d.max()), "mean": float(d.mean()), "unit": "[0, 1] RGB"} + + +def main(): + import onnxruntime as ort + import torch + + torch.manual_seed(0) + net, tensors = deoldify() + record = {"checkpointTensors": tensors, "strictLoad": True} + record["verification"] = verify_against_reference(net) + refs = {} + for size in PARITY_SIZES: + xs = cm.test_input(size, seed=size) + with torch.no_grad(): + refs[size] = (xs, net(torch.from_numpy(xs)).numpy()) + cm.bake_norms(net) + # Core ML has no edge-mode Pad: the five blur pads, rewritten exactly. + record["replicationPadsRewritten"] = cm.replace_replication_pads(net) + out = os.path.join(fm.STAGING, OUT) + x = cm.test_input(EXPORT_SIZE) + info = fm.export(net, [torch.from_numpy(x)], ["luma"], ["rgb"], out, + dynamic_axes={"luma": {2: "height", 3: "width"}, "rgb": {2: "height", 3: "width"}}) + info["checkpoint"] = CKPT[0] + info["checkpointSha256"] = CKPT[1] + info["source"] = "https://github.com/" + DEOLDIFY_COMMIT.replace("@", "/tree/") + sess = ort.InferenceSession(out, providers=["CPUExecutionProvider"]) + info["parity"] = cm.parity_at_sizes(sess, refs, "luma", "[0, 1] RGB") + record.update({"opset": fm.OPSET, "irVersion": fm.IR_VERSION, "torch": torch.__version__, "models": [info]}) + fm.write_record("colorize-deoldify", record) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_face_codeformer.py b/Scripts/models/make_face_codeformer.py new file mode 100644 index 00000000..efb45e5f --- /dev/null +++ b/Scripts/models/make_face_codeformer.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Convert CodeFormer (sczhou/CodeFormer, release v0.1.0's codeformer.pth) to +face-codeformer-v0.1.0.onnx for the face-codeformer add-on. + +Inputs: `input`, (1, 3, 512, 512) RGB in [-1, 1]; `w`, shape (1,), the +fidelity weight (0 = the codebook's best guess, 1 = closest to the input). +Output: `output`, (1, 3, 512, 512) RGB in [-1, 1]. float32, static shapes. +Upstream's inference runs with adain=True, as this does. + +Why `w` is an input rather than a few fixed exports: CodeFormer uses w in one +place, each fusion block's `out = dec + w * (dec * scale + shift)`, behind an +`if w > 0`. The export traces the w > 0 branch, and at w = 0 that branch adds +exactly zero, so one graph is the network at every w — where fixed exports +would mean a ~377 MB file per offered value. The parity record checks w = 0, +0.5, 0.7 and 1 against PyTorch (which skips the fusion at w = 0). + +Licence: S-Lab License 1.0 — redistribution for non-commercial purposes only, +with the copyright notice. The add-on carries that as its usageNote. + +Usage (conversion venv, see docs/ADDONS.md): + + VAPOURBOX_ADDON_STAGING=/path/to/addon-staging python Scripts/models/make_face_codeformer.py +""" + +import logging +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import face_models_common as fm # noqa: E402 + +CODEFORMER = ("sczhou/CodeFormer", "b33cc7d639d6545bfcccc7e0bc6ae51f24e79c2b") +CKPT = ( + "https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth", + "1009e537e0c2a07d4cabce6355f53cb66767cd4b4297ec7a4a64ca4b8a5684b7", +) +W_CHECKS = [0.0, 0.5, 0.7, 1.0] + + +def codeformer(): + import torch + + src = fm.upstream_source(*CODEFORMER) + fm.stub_module("basicsr.utils", get_root_logger=lambda *a, **k: logging.getLogger("codeformer")) + fm.stub_module("basicsr.utils.registry", ARCH_REGISTRY=fm.Registry()) + fm.stub_module("basicsr.archs").__path__ = [os.path.join(src, "basicsr", "archs")] + from basicsr.archs.codeformer_arch import CodeFormer + + net = CodeFormer(dim_embd=512, codebook_size=1024, n_head=8, n_layers=9, connect_list=["32", "64", "128", "256"]) + ckpt = fm.load_checkpoint(fm.fetch(*CKPT)) + net.load_state_dict(ckpt["params_ema"], strict=True) + net.eval() + + import torch.nn.functional as F + from basicsr.archs.codeformer_arch import adaptive_instance_normalization + + class Restore(torch.nn.Module): + """CodeFormer.forward(x, w, adain=True)[0], line for line, except that + feature sizes are taken as Python ints (`str(x.shape[-1])` is a + tensor's repr under the tracer, which breaks the dict lookups) and the + fusion runs for every w (see the module docstring).""" + + def __init__(self, inner): + super().__init__() + self.inner = inner + + def forward(self, x, w): + m = self.inner + enc_feat_dict = {} + out_list = [m.fuse_encoder_block[f] for f in m.connect_list] + for i, block in enumerate(m.encoder.blocks): + x = block(x) + if i in out_list: + enc_feat_dict[str(int(x.shape[-1]))] = x.clone() + lq_feat = x + pos_emb = m.position_emb.unsqueeze(1).repeat(1, x.shape[0], 1) + query_emb = m.feat_emb(lq_feat.flatten(2).permute(2, 0, 1)) + for layer in m.ft_layers: + query_emb = layer(query_emb, query_pos=pos_emb) + logits = m.idx_pred_layer(query_emb).permute(1, 0, 2) + soft_one_hot = F.softmax(logits, dim=2) + _, top_idx = torch.topk(soft_one_hot, 1, dim=2) + quant_feat = m.quantize.get_codebook_feat(top_idx, shape=[x.shape[0], 16, 16, 256]) + quant_feat = adaptive_instance_normalization(quant_feat, lq_feat) + x = quant_feat + fuse_list = [m.fuse_generator_block[f] for f in m.connect_list] + for i, block in enumerate(m.generator.blocks): + x = block(x) + if i in fuse_list: + f_size = str(int(x.shape[-1])) + x = m.fuse_convs_dict[f_size](enc_feat_dict[f_size], x, w) + return x + + class Reference(torch.nn.Module): + """Upstream's own call, with w a Python float (so w = 0 skips the + fusion blocks, as it does in inference_codeformer.py).""" + + def __init__(self, inner): + super().__init__() + self.inner = inner + + def forward(self, x, w): + return self.inner(x, w=float(w.reshape(-1)[0]), adain=True)[0] + + return Restore(net).eval(), Reference(net).eval() + + +def main(): + import torch + + torch.manual_seed(0) + net, reference = codeformer() + out = os.path.join(fm.STAGING, "face-codeformer-v0.1.0.onnx") + x = fm.face_test_input() + w = np.array([0.5], np.float32) + info = fm.export(net, [torch.from_numpy(x), torch.from_numpy(w)], ["input", "w"], ["output"], out) + info["checkpoint"] = CKPT[0] + info["parity"] = {} + for wv in W_CHECKS: + info["parity"][f"w={wv}"] = fm.parity(reference, out, {"input": x, "w": np.array([wv], np.float32)}) + fm.write_record("face-codeformer", {"opset": fm.OPSET, "irVersion": fm.IR_VERSION, + "torch": torch.__version__, "models": [info]}) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_face_gfpgan.py b/Scripts/models/make_face_gfpgan.py new file mode 100644 index 00000000..3fa22e86 --- /dev/null +++ b/Scripts/models/make_face_gfpgan.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 +"""Convert GFPGAN v1.4 (TencentARC/GFPGAN, release v1.3.0's GFPGANv1.4.pth) to +face-gfpgan-v1.4.onnx for the face-gfpgan add-on. + +The network is GFPGANv1Clean (the "clean" architecture GFPGANer uses for +v1.4, with no custom CUDA ops). Input and output are (1, 3, 512, 512) RGB in +[-1, 1], float32, static. + +One deliberate difference from upstream inference: GFPGANer runs the +StyleGAN2 decoder with randomize_noise=True, drawing fresh noise for every +call. On video that is new random texture on every frame — flicker by +construction — so the export uses randomize_noise=False, the generator's +fixed noise buffers (part of the checkpoint), which makes the model a pure +function of its input. The parity record compares against PyTorch with the +same setting, and also states how far upstream's random-noise output sits +from it, for scale. + +Licence: GFPGAN is Apache-2.0 except its StyleGAN2-derived code, which is +under NVIDIA's licence (non-commercial: research or evaluation only). The +add-on carries that as its usageNote. + +Usage (conversion venv, see docs/ADDONS.md): + + VAPOURBOX_ADDON_STAGING=/path/to/addon-staging python Scripts/models/make_face_gfpgan.py +""" + +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import face_models_common as fm # noqa: E402 + +GFPGAN = ("TencentARC/GFPGAN", "7552a7791caad982045a7bbe5634bbf1cd5c8679") +CKPT = ( + "https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth", + "e2cd4703ab14f4d01fd1383a8a8b266f9a5833dacee8e6a79d3bf21a1b6be5ad", +) + + +def gfpgan(): + import torch + + src = fm.upstream_source(*GFPGAN) + fm.stub_module("basicsr.utils.registry", ARCH_REGISTRY=fm.Registry()) + fm.stub_module("basicsr.archs.arch_util", default_init_weights=lambda *a, **k: None) + for pkg in ("gfpgan", "gfpgan.archs"): + fm.stub_module(pkg).__path__ = [os.path.join(src, *pkg.split("."))] + from gfpgan.archs.gfpganv1_clean_arch import GFPGANv1Clean + + net = GFPGANv1Clean( + out_size=512, + num_style_feat=512, + channel_multiplier=2, + decoder_load_path=None, + fix_decoder=False, + num_mlp=8, + input_is_latent=True, + different_w=True, + narrow=1, + sft_half=True, + ) + ckpt = fm.load_checkpoint(fm.fetch(*CKPT)) + net.load_state_dict(ckpt["params_ema"] if "params_ema" in ckpt else ckpt["params"], strict=True) + net.eval() + + class Restore(torch.nn.Module): + def __init__(self, inner, randomize_noise=False): + super().__init__() + self.inner = inner + self.randomize_noise = randomize_noise + + def forward(self, x): + return self.inner(x, return_rgb=False, randomize_noise=self.randomize_noise)[0] + + return Restore(net).eval(), Restore(net, randomize_noise=True).eval() + + +def main(): + import torch + + torch.manual_seed(0) + net, upstream = gfpgan() + out = os.path.join(fm.STAGING, "face-gfpgan-v1.4.onnx") + x = fm.face_test_input() + info = fm.export(net, [torch.from_numpy(x)], ["input"], ["output"], out) + info["checkpoint"] = CKPT[0] + info["parity"] = fm.parity(net, out, {"input": x}) + with torch.no_grad(): + fixed = net(torch.from_numpy(x)).numpy() + rand = upstream(torch.from_numpy(x)).numpy() + d = np.abs(fixed - rand) + info["fixedVsRandomNoise"] = {"max": float(d.max()), "mean": float(d.mean()), "unit": "[-1, 1] output"} + fm.write_record("face-gfpgan", {"opset": fm.OPSET, "irVersion": fm.IR_VERSION, + "torch": torch.__version__, "models": [info]}) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_face_restoration.py b/Scripts/models/make_face_restoration.py new file mode 100644 index 00000000..773d07e8 --- /dev/null +++ b/Scripts/models/make_face_restoration.py @@ -0,0 +1,201 @@ +#!/usr/bin/env python3 +"""Convert the face-restoration add-on's three models to ONNX: + + face-restoration-retinaface-r50.onnx RetinaFace ResNet-50 face detector + (facexlib's detection_Resnet50_Final.pth, + network from biubug6/Pytorch_Retinaface, MIT) + face-restoration-bisenet.onnx BiSeNet face parser (facexlib's + parsing_bisenet.pth, network from + zllrunning/face-parsing.PyTorch, MIT) + face-restoration-restoreformerpp.onnx RestoreFormer++ (wzhouxiff/RestoreFormerPlusPlus + v1.0.0, Apache-2.0) + +`--detector mobilenet` also writes face-restoration-retinaface-mnet.onnx +(detection_mobilenet0.25_Final.pth), used only for the detector comparison in +docs/reports/issue-face-restoration (the add-on ships ResNet-50). + +Shapes are static, which Core ML needs to take a whole graph: the detector +takes (1, 3, 640, 640) BGR minus the (104, 117, 123) mean, as upstream does, +padded by the caller; BiSeNet takes (1, 3, 512, 512) RGB normalised with +ImageNet's mean and std; RestoreFormer++ takes and returns (1, 3, 512, 512) +RGB in [-1, 1]. All three are float32. + +Usage (conversion venv, see docs/ADDONS.md): + + VAPOURBOX_ADDON_STAGING=/path/to/addon-staging \\ + python Scripts/models/make_face_restoration.py [--detector mobilenet] + +prints, and writes beside the models, a record of each file's size, SHA-256 +and its parity with PyTorch (max / mean absolute difference, CPU provider). +""" + +import argparse +import os +import sys + +import numpy as np + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import face_models_common as fm # noqa: E402 + +FACEXLIB = ("xinntao/facexlib", "260620ae93990a300f4b16448df9bb459f1caba9") +RESTOREFORMER = ("wzhouxiff/RestoreFormerPlusPlus", "59d250f45d6b11a3e620d62e6e067ca1288f93a9") + +CKPT = { + "r50": ( + "https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth", + "6d1de9c2944f2ccddca5f5e010ea5ae64a39845a86311af6fdf30841b0a5a16d", + ), + "mnet": ( + "https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_mobilenet0.25_Final.pth", + "2979b33ffafda5d74b6948cd7a5b9a7a62f62b949cef24e95fd15d2883a65220", + ), + "bisenet": ( + "https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_bisenet.pth", + "468e13ca13a9b43cc0881a9f99083a430e9c0a38abd935431d1c28ee94b26567", + ), + "restoreformerpp": ( + "https://github.com/wzhouxiff/RestoreFormerPlusPlus/releases/download/v1.0.0/RestoreFormer++.ckpt", + "613fe52805f86bf8c2bffff08ae9f7a0b99f408be1bf221767af6183038be3a2", + ), +} + +DET_SIZE = 640 + + +def facexlib_packages(): + """Make facexlib's detection and parsing modules importable without + running its package __init__ files (which pull in every other model).""" + src = fm.upstream_source(*FACEXLIB) + for pkg in ("facexlib", "facexlib.detection", "facexlib.parsing"): + mod = fm.stub_module(pkg) + mod.__path__ = [os.path.join(src, *pkg.split("."))] + return src + + +def detector(kind): + import torch + + facexlib_packages() + from facexlib.detection.retinaface import RetinaFace + + net = RetinaFace(network_name="resnet50" if kind == "r50" else "mobile0.25", device=torch.device("cpu")) + state = fm.load_checkpoint(fm.fetch(*CKPT[kind])) + state = {k[7:] if k.startswith("module.") else k: v for k, v in state.items()} + net.load_state_dict(state, strict=True) + return net.eval() + + +def bisenet(): + import torch + + facexlib_packages() + from facexlib.parsing.bisenet import BiSeNet + + net = BiSeNet(num_class=19) + net.load_state_dict(fm.load_checkpoint(fm.fetch(*CKPT["bisenet"])), strict=True) + + class First(torch.nn.Module): + """BiSeNet returns (out, out16, out32); inference uses out only.""" + + def __init__(self, inner): + super().__init__() + self.inner = inner + + def forward(self, x): + return self.inner(x)[0] + + return First(net.eval()).eval() + + +def restoreformerpp(): + import torch + + src = fm.upstream_source(*RESTOREFORMER) + sys.path.insert(0, src) + from RestoreFormer.modules.vqvae.vqvae_arch import VQVAEGANMultiHeadTransformer + + net = VQVAEGANMultiHeadTransformer(head_size=4, ex_multi_scale_num=1) + ckpt = fm.load_checkpoint(fm.fetch(*CKPT["restoreformerpp"])) + state = {k[len("vqvae."):] if k.startswith("vqvae.") else k: v for k, v in ckpt["state_dict"].items()} + # Upstream loads with strict=False: the checkpoint also holds the + # training-only discriminator and perceptual-loss weights. Insist that + # every weight the network itself has is present. + missing, unexpected = net.load_state_dict(state, strict=False) + if missing: + raise SystemExit(f"RestoreFormer++ checkpoint lacks {missing[:5]}…") + + class Restore(torch.nn.Module): + """Upstream calls self.RF(x)[0]: the restored image.""" + + def __init__(self, inner): + super().__init__() + self.inner = inner + + def forward(self, x): + return self.inner(x)[0] + + print(f"RestoreFormer++: {len(unexpected)} checkpoint tensors unused (training-only)") + return Restore(net.eval()).eval() + + +def det_input(seed=1): + rng = np.random.default_rng(seed) + img = rng.uniform(0, 255, (1, 3, DET_SIZE, DET_SIZE)).astype(np.float32) + return img - np.array([104.0, 117.0, 123.0], np.float32)[None, :, None, None] + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--detector", choices=["r50", "mobilenet", "both"], default="r50") + ap.add_argument("--only", choices=["detector", "bisenet", "restoreformerpp"], action="append") + args = ap.parse_args() + import torch + + torch.manual_seed(0) + os.makedirs(fm.STAGING, exist_ok=True) + record = {"opset": fm.OPSET, "irVersion": fm.IR_VERSION, "torch": torch.__version__, "models": []} + only = set(args.only or ["detector", "bisenet", "restoreformerpp"]) + + if "detector" in only: + kinds = {"r50": ["r50"], "mobilenet": ["mnet"], "both": ["r50", "mnet"]}[args.detector] + for kind in kinds: + net = detector(kind) + out = os.path.join(fm.STAGING, f"face-restoration-retinaface-{kind}.onnx") + x = det_input() + info = fm.export(net, [torch.from_numpy(x)], ["input"], ["loc", "conf", "landmarks"], out) + info["checkpoint"] = CKPT[kind][0] + info["parity"] = fm.parity(net, out, {"input": x}) + record["models"].append(info) + + if "bisenet" in only: + net = bisenet() + out = os.path.join(fm.STAGING, "face-restoration-bisenet.onnx") + x = fm.face_test_input() + info = fm.export(net, [torch.from_numpy(x)], ["input"], ["parsing"], out) + info["checkpoint"] = CKPT["bisenet"][0] + p = fm.parity(net, out, {"input": x}) + # What the pass uses is the argmax class per pixel. + import onnxruntime as ort + + with torch.no_grad(): + ref = net(torch.from_numpy(x)).numpy().argmax(1) + got = ort.InferenceSession(out, providers=["CPUExecutionProvider"]).run(None, {"input": x})[0].argmax(1) + p["argmaxAgreement"] = float((ref == got).mean()) + info["parity"] = p + record["models"].append(info) + + if "restoreformerpp" in only: + net = restoreformerpp() + out = os.path.join(fm.STAGING, "face-restoration-restoreformerpp.onnx") + x = fm.face_test_input() + info = fm.export(net, [torch.from_numpy(x)], ["input"], ["output"], out) + info["checkpoint"] = CKPT["restoreformerpp"][0] + info["parity"] = fm.parity(net, out, {"input": x}) + record["models"].append(info) + + fm.write_record("face-restoration", record) + + +if __name__ == "__main__": + main() diff --git a/Scripts/models/make_ncnn_selftest.py b/Scripts/models/make_ncnn_selftest.py new file mode 100644 index 00000000..51d7af1b --- /dev/null +++ b/Scripts/models/make_ncnn_selftest.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Build the AI-upscaling probe's self-test model for VSNCNN (vs-mlrt's +`ncnn.Model`): one 1x1 convolution on a float32 NCHW tensor with three +channels and any height and width, which swaps the first and third channels +and adds 0.125 to every one: + + y[0] = x[2] + 0.125, y[1] = x[1] + 0.125, y[2] = x[0] + 0.125 + +`vapourbox-worker --probe-upscale` runs it through the bundled VSNCNN plugin +on the GPU and checks that answer. The upscaling models themselves are an +add-on (docs/ADDONS.md), so the probe must not need one: with this model it +tells "no Vulkan device / broken plugin" apart from "models not installed" +before the user has downloaded anything. It is compiled into the worker +(`include_bytes!` in `worker/src/gpu_plugin.rs`) and handed to VSNCNN as bytes +(`path_is_serialization=True`), so nothing is written to disk. + +A convolution, rather than the elementwise arithmetic of the AI runtime's +self-test, because that is what the real upscalers are made of and what +vs-mlrt's onnx2ncnn and ncnn's Vulkan shaders must handle. Written with an +explicit IR version and opset that vs-mlrt's pinned onnx accepts. + +Usage (in the conversion venv, see docs/ADDONS.md): + + python Scripts/models/make_ncnn_selftest.py Tests/TestResources/ncnn_selftest.onnx + +`gpu_plugin::tests::ncnn_selftest_is_the_committed_model` and the Dart +`gpu_plugin_selftest_test.dart` check the committed file is what the worker +embeds. +""" +import sys + +import onnx +from onnx import TensorProto, helper + +IR_VERSION = 7 +OPSET = 11 +BIAS = 0.125 + + +def build() -> onnx.ModelProto: + x = helper.make_tensor_value_info("input", TensorProto.FLOAT, ["n", 3, "h", "w"]) + y = helper.make_tensor_value_info("output", TensorProto.FLOAT, ["n", 3, "h", "w"]) + # out channel o reads in channel 2 - o. + weight = [1.0 if i == 2 - o else 0.0 for o in range(3) for i in range(3)] + w = helper.make_tensor("weight", TensorProto.FLOAT, [3, 3, 1, 1], weight) + b = helper.make_tensor("bias", TensorProto.FLOAT, [3], [BIAS] * 3) + graph = helper.make_graph( + [helper.make_node("Conv", ["input", "weight", "bias"], ["output"], kernel_shape=[1, 1])], + "vapourbox_ncnn_selftest", + [x], + [y], + initializer=[w, b], + ) + model = helper.make_model( + graph, + producer_name="vapourbox", + opset_imports=[helper.make_opsetid("", OPSET)], + ) + model.ir_version = IR_VERSION + onnx.checker.check_model(model) + return model + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit(__doc__) + data = build().SerializeToString() + with open(sys.argv[1], "wb") as f: + f.write(data) + print(f"{len(data)} bytes") + + +if __name__ == "__main__": + main() diff --git a/Scripts/promote-deps-rc.py b/Scripts/promote-deps-rc.py new file mode 100755 index 00000000..680f7d77 --- /dev/null +++ b/Scripts/promote-deps-rc.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Promote a tested deps release candidate to its final version. + +A deps release is published as the exact bundles CI and the nightly tested +on the release candidate, with only the version changed: each file name, each +zip's version.json, each v2 delta's baseSha256 (re-pointed at the re-stamped +bundle, since the app checks it against the bundle zip it downloaded) and the +.sha256.json sidecars. Every other entry is copied with the same name, +contents (CRC), size, permissions, timestamp and compression method, and that +is verified afterwards, along with each candidate sidecar against its zip and +each candidate delta against its candidate bundle. + + gh release download deps-vX.Y.Z-rcN -D rc + python3 -I Scripts/promote-deps-rc.py rc final X.Y.Z-rcN X.Y.Z + gh release create deps-vX.Y.Z --latest=false ... # never Latest + gh release upload deps-vX.Y.Z final/* + +The zips come out a little smaller than the candidate's (Python recompresses +each entry); the files inside are identical. +""" +import hashlib +import json +import os +import sys +import zipfile + +src, out, old, new = sys.argv[1:5] +os.makedirs(out, exist_ok=True) + + +def sha256(path): + h = hashlib.sha256() + with open(path, 'rb') as f: + for b in iter(lambda: f.read(1 << 22), b''): + h.update(b) + return h.hexdigest() + + +def restamp(name, base_sha=None): + old_zip = os.path.join(src, name) + new_name = name.replace(f'-{old}-', f'-{new}-') + new_zip = os.path.join(out, new_name) + with zipfile.ZipFile(old_zip) as zin, zipfile.ZipFile(new_zip, 'w', allowZip64=True) as zout: + seen = 0 + for info in zin.infolist(): + data = zin.read(info) + if info.filename == 'version.json': + seen += 1 + v = json.loads(data) + assert v['version'] == old, v + v['version'] = new + if base_sha is not None: + assert 'baseSha256' in v, v + v['baseSha256'] = base_sha + else: + assert 'baseSha256' not in v, v + data = (json.dumps(v, indent=2) + '\n').encode() + out_info = zipfile.ZipInfo(info.filename, info.date_time) + for attr in ('compress_type', 'external_attr', 'create_system', 'create_version', + 'extract_version', 'internal_attr', 'comment', 'extra'): + setattr(out_info, attr, getattr(info, attr)) + with zout.open(out_info, 'w', force_zip64=len(data) > 0x7fffffff) as w: + w.write(data) + # zipfile gives a 0 attribute 0600 when writing the local header; + # the central directory (written at close) is what readers use, so + # put the original back there. + out_info.external_attr = info.external_attr + assert seen == 1, f'{name}: {seen} version.json entries' + + # Verify: same entries, same contents/attributes, except version.json. + with zipfile.ZipFile(old_zip) as a, zipfile.ZipFile(new_zip) as b: + ia, ib = a.infolist(), b.infolist() + assert [i.filename for i in ia] == [i.filename for i in ib], name + for x, y in zip(ia, ib): + if x.filename == 'version.json': + continue + for attr in ('CRC', 'file_size', 'external_attr', 'date_time', 'compress_type', 'create_system'): + assert getattr(x, attr) == getattr(y, attr), (name, x.filename, attr) + assert b.testzip() is None, name + + digest = sha256(new_zip) + side = json.load(open(old_zip + '.sha256.json')) + assert side['filename'] == name and side['version'] == old, side + assert side['sha256'] == sha256(old_zip), f'{name}: rc sidecar does not match its zip' + side.update(filename=new_name, sha256=digest, size=os.path.getsize(new_zip), version=new) + if base_sha is not None: + side['baseSha256'] = base_sha + with open(new_zip + '.sha256.json', 'w') as f: + f.write(json.dumps(side, indent=2) + '\n') + print(f'{new_name} {digest} {side["size"]}') + return digest + + +zips = sorted(n for n in os.listdir(src) if n.endswith('.zip')) +bundles = [n for n in zips if not n.endswith('-v2-delta.zip')] +deltas = [n for n in zips if n.endswith('-v2-delta.zip')] +assert len(bundles) == 5 and len(deltas) == 3, zips + +new_sha = {} +for n in bundles: + new_sha[n] = restamp(n) +for d in deltas: + base = d.replace('-v2-delta.zip', '.zip') + old_side = json.load(open(os.path.join(src, d + '.sha256.json'))) + assert old_side['baseSha256'] == sha256(os.path.join(src, base)), f'{d}: rc delta was not cut against {base}' + restamp(d, base_sha=new_sha[base]) +print('ok: 5 bundles, 3 deltas') diff --git a/Scripts/release.sh b/Scripts/release.sh index d71043cc..2a876329 100755 --- a/Scripts/release.sh +++ b/Scripts/release.sh @@ -125,6 +125,19 @@ if [[ "$DEPS_TAG_IN_TREE" == *-rc* ]]; then echo "before cutting an app release. RC tags must never ship." exit 1 fi +# The same hazard for add-on files: while a release is in development, converted +# models may be hosted on the deps release candidate (docs/ADDONS.md, "Hosting"). +# They must move to their permanent release, and the manifests be repointed, +# before an app release can name them. +RC_ADDON_URLS=$(grep -l 'releases/download/[^/"]*-rc[0-9]*/' "$PROJECT_ROOT"/app/assets/addons/*.json 2>/dev/null || true) +if [[ -n "$RC_ADDON_URLS" ]]; then + echo -e "${RED}ERROR: add-on manifests download from a release candidate:${NC}" + echo "$RC_ADDON_URLS" | sed 's/^/ /' + echo "" + echo "Upload their files to addon-models-v1 and repoint the manifests" + echo "before cutting an app release. RC tags must never ship." + exit 1 +fi # Get current versions from GitHub echo -e "${YELLOW}Fetching current versions from GitHub...${NC}" diff --git a/Scripts/run-debug-linux.sh b/Scripts/run-debug-linux.sh index d01b5c18..03dcee90 100755 --- a/Scripts/run-debug-linux.sh +++ b/Scripts/run-debug-linux.sh @@ -83,8 +83,9 @@ cp "$WORKER_DIR/target/debug/vapourbox-worker" "$BUNDLE_DIR/" # Copy templates mkdir -p "$BUNDLE_DIR/templates" cp "$WORKER_DIR/templates/"*.vpy "$BUNDLE_DIR/templates/" -cp "$WORKER_DIR/templates/pipe_source.py" "$BUNDLE_DIR/templates/" -cp "$WORKER_DIR/templates/spotless.py" "$BUNDLE_DIR/templates/" +# Every helper module (pipe_source, ai_runtime, the vendored filters), as the +# packaging scripts do: one left out works from worker/templates/ and fails here. +cp "$WORKER_DIR/templates/"*.py "$BUNDLE_DIR/templates/" echo "==> Launching VapourBox (debug)..." "$BUNDLE_DIR/vapourbox" & diff --git a/Scripts/run-debug-macos.sh b/Scripts/run-debug-macos.sh index 1d90394a..3c989830 100755 --- a/Scripts/run-debug-macos.sh +++ b/Scripts/run-debug-macos.sh @@ -98,8 +98,9 @@ cp "$WORKER_DIR/target/debug/vapourbox-worker" "$DEBUG_APP/Contents/MacOS/" # Copy templates (includes pipe_source.py used by VapourSynth scripts) mkdir -p "$DEBUG_APP/Contents/MacOS/templates" cp "$WORKER_DIR/templates/"*.vpy "$DEBUG_APP/Contents/MacOS/templates/" -cp "$WORKER_DIR/templates/pipe_source.py" "$DEBUG_APP/Contents/MacOS/templates/" -cp "$WORKER_DIR/templates/spotless.py" "$DEBUG_APP/Contents/MacOS/templates/" +# Every helper module (pipe_source, ai_runtime, the vendored filters), as the +# packaging scripts do: one left out works from worker/templates/ and fails here. +cp "$WORKER_DIR/templates/"*.py "$DEBUG_APP/Contents/MacOS/templates/" # Always strip quarantine from deps. macOS SIGKILLs quarantined binaries # (ffmpeg/ffprobe/vspipe) on exec, which surfaces as opaque "Failed to run diff --git a/Scripts/update-version.sh b/Scripts/update-version.sh index 3faa0865..d27598db 100755 --- a/Scripts/update-version.sh +++ b/Scripts/update-version.sh @@ -80,28 +80,22 @@ if [ -n "$APP_VERSION" ]; then # Update macOS Info.plist. # - # PlistBuddy exists only on macOS, and this used to be the ONLY path — so - # running this script from Windows or Linux skipped the plist without a word, - # and the macOS DMG built by CI came out labelled with the previous version - # while every other file said the new one. The plist is XML, so sed can do it - # anywhere; PlistBuddy stays the preferred tool where it exists. + # Always with sed, never PlistBuddy. PlistBuddy existed only on macOS, so + # running this from Windows or Linux once skipped the plist without a word + # (the DMG came out labelled with the previous version); and PlistBuddy + # rewrites the whole file, dropping its XML comments — including the one + # explaining why Impeller is forced off (issue #91). sed changes only the + # on the line after each version key. MAC_PLIST="$PROJECT_ROOT/app/macos/Runner/Info.plist" if [ -f "$MAC_PLIST" ]; then - if command -v /usr/libexec/PlistBuddy &> /dev/null; then - /usr/libexec/PlistBuddy -c "Set :CFBundleShortVersionString $APP_VERSION" "$MAC_PLIST" - /usr/libexec/PlistBuddy -c "Set :CFBundleVersion $APP_VERSION" "$MAC_PLIST" - echo " Updated macOS Info.plist (PlistBuddy)" - else - # Replace the on the line following each version key. - for KEY in CFBundleShortVersionString CFBundleVersion; do - if [[ "$OSTYPE" == "darwin"* ]]; then - sed -i '' "/${KEY}<\/key>/{n;s|.*|${APP_VERSION}|;}" "$MAC_PLIST" - else - sed -i "/${KEY}<\/key>/{n;s|.*|${APP_VERSION}|;}" "$MAC_PLIST" - fi - done - echo " Updated macOS Info.plist (sed)" - fi + for KEY in CFBundleShortVersionString CFBundleVersion; do + if [[ "$OSTYPE" == "darwin"* ]]; then + sed -i '' "/${KEY}<\/key>/{n;s|.*|${APP_VERSION}|;}" "$MAC_PLIST" + else + sed -i "/${KEY}<\/key>/{n;s|.*|${APP_VERSION}|;}" "$MAC_PLIST" + fi + done + echo " Updated macOS Info.plist" # Never leave the plist behind again: fail loudly instead. if ! grep -q "${APP_VERSION}" "$MAC_PLIST"; then diff --git a/Tests/TestResources/ai_runtime_selftest.onnx b/Tests/TestResources/ai_runtime_selftest.onnx new file mode 100644 index 00000000..b386ce13 Binary files /dev/null and b/Tests/TestResources/ai_runtime_selftest.onnx differ diff --git a/Tests/TestResources/colorize_test.mp4 b/Tests/TestResources/colorize_test.mp4 new file mode 100644 index 00000000..74e9696e Binary files /dev/null and b/Tests/TestResources/colorize_test.mp4 differ diff --git a/Tests/TestResources/face_restore_test.mp4 b/Tests/TestResources/face_restore_test.mp4 new file mode 100644 index 00000000..85cbe462 Binary files /dev/null and b/Tests/TestResources/face_restore_test.mp4 differ diff --git a/Tests/TestResources/generate_colorize_test.sh b/Tests/TestResources/generate_colorize_test.sh new file mode 100755 index 00000000..0718d98a --- /dev/null +++ b/Tests/TestResources/generate_colorize_test.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Generate colorize_test.mp4: a 5-second black-and-white SD clip of real film, +# for the Colourisation pass's tests (integration_colorize_test.dart) and its +# report (docs/reports/2026-11-colourisation). +# +# Provenance: "Diary of a Sergeant" (1945), a US War Department film (Army +# Pictorial Service, MISC. 1129; National Archives 111-M-1129), from +# Wikimedia Commons, where it is marked public domain as a work of the US +# federal government: +# https://commons.wikimedia.org/wiki/File:Diary_of_a_Sergeant_(1945).webm +# (1920x1080 VP9, the 4:3 picture pillarboxed, 24 fps). +# +# The clip is seconds 156-161: a close-up of a face, a dissolve, and a two-shot +# of two men at a bedside (skin, hair, white cotton, bedding), with the +# pillarbox cropped off, scaled to 640x480, and its chroma made exactly +# neutral (the scan carries a faint tint; a black-and-white source should +# carry none). +# +# Usage: ./generate_colorize_test.sh [ffmpeg] +# (the first ~80 MB of the original is enough: `curl -r 0-80000000 -o src.webm `) +set -euo pipefail +SRC="${1:?usage: $0 [ffmpeg]}" +FFMPEG="${2:-ffmpeg}" +OUT="$(cd "$(dirname "$0")" && pwd)/colorize_test.mp4" + +"$FFMPEG" -y -v error -ss 156 -t 5 -i "$SRC" -an \ + -vf "crop=1440:1080:240:0,scale=640:480:flags=lanczos,format=gray,format=yuv420p,setsar=1" \ + -c:v libx264 -preset slow -crf 20 -pix_fmt yuv420p \ + -color_primaries bt470bg -color_trc bt709 -colorspace smpte170m -color_range tv \ + -movflags +faststart "$OUT" +ls -l "$OUT" diff --git a/Tests/TestResources/generate_face_test.sh b/Tests/TestResources/generate_face_test.sh new file mode 100755 index 00000000..2e75e29b --- /dev/null +++ b/Tests/TestResources/generate_face_test.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Generate face_restore_test.mp4: a 5-second, VHS-like SD clip with a real face, +# for the Face Restoration pass's tests (integration_face_restore_test.dart). +# +# Provenance: "NASA Astronaut Anne McClain Talks with Fox13 News, Seattle – +# March 31, 2025" (NASA Johnson Space Center, catalogue iss072m260901634), +# from Wikimedia Commons, where it is marked public domain as a work of NASA, a +# US government agency: +# https://commons.wikimedia.org/wiki/File:NASA_Astronaut_Anne_McClain_Talks_with_Fox13_News,_Seattle_%E2%80%93_March_31,_2025_(iss072m260901634).webm +# NASA's media guidelines ask that NASA material not be used to imply +# endorsement; a test fixture does not. +# +# The clip is seconds 50-55 of the 1080p original, cropped to the astronaut's +# upper body (so the face is ~90 px tall at SD, a size face restoration is +# meant for), scaled to 640x480 at 29.97 fps, then degraded the way a VHS dub +# is: luma resolution cut to 256x360, chroma to a quarter of the width, +# temporal noise, and a soft H.264 encode. +# +# Usage: ./generate_face_test.sh [ffmpeg] +# (the first ~80 MB of the original is enough: `curl -r 0-80000000 -o src.webm `) +set -euo pipefail +SRC="${1:?usage: $0 [ffmpeg]}" +FFMPEG="${2:-ffmpeg}" +OUT="$(cd "$(dirname "$0")" && pwd)/face_restore_test.mp4" + +"$FFMPEG" -y -v error -ss 50 -t 5 -i "$SRC" -an \ + -vf "fps=30000/1001,crop=720:540:728:140,scale=640:480:flags=lanczos,\ +scale=256:360:flags=area,scale=640:480:flags=bicubic,format=yuv420p,\ +split[y][c];[c]scale=160:480:flags=bicubic,scale=640:480:flags=bicubic[cb];\ +[y][cb]mergeplanes=0x001112:yuv420p,noise=alls=8:allf=t,setsar=1" \ + -c:v libx264 -preset slow -crf 27 -pix_fmt yuv420p \ + -color_primaries bt470bg -color_trc bt709 -colorspace smpte170m -color_range tv \ + -movflags +faststart "$OUT" +ls -l "$OUT" diff --git a/Tests/TestResources/ncnn_selftest.onnx b/Tests/TestResources/ncnn_selftest.onnx new file mode 100644 index 00000000..e769e2e2 Binary files /dev/null and b/Tests/TestResources/ncnn_selftest.onnx differ diff --git a/app/assets/addons/ai-runtime.json b/app/assets/addons/ai-runtime.json new file mode 100644 index 00000000..3c7210c4 --- /dev/null +++ b/app/assets/addons/ai-runtime.json @@ -0,0 +1,111 @@ +{ + "id": "ai-runtime", + "version": "1.0.0", + "name": "AI runtime (ONNX Runtime)", + "feature": "AI models", + "description": "Runs AI models for the features that need it, on the graphics card where it can (DirectML on Windows, Core ML on macOS) and otherwise on the CPU: Microsoft's ONNX Runtime and NumPy, installed for VapourBox's own Python. The AI model add-ons need it.", + "licence": "MIT", + "licenceUrl": "https://github.com/microsoft/onnxruntime/blob/main/LICENSE", + "credit": "Microsoft Corporation", + "noticeName": "ONNX Runtime", + "homepage": "https://onnxruntime.ai", + "components": [ + { + "noticeName": "NumPy", + "licence": "BSD-3-Clause", + "licenceUrl": "https://github.com/numpy/numpy/blob/main/LICENSE.txt", + "credit": "NumPy Developers", + "homepage": "https://numpy.org" + }, + { + "noticeName": "DirectML", + "licence": "Microsoft Software License Terms", + "licenceUrl": "https://www.nuget.org/packages/Microsoft.AI.DirectML/1.15.4/License", + "credit": "Microsoft Corporation", + "homepage": "https://github.com/microsoft/DirectML", + "platforms": ["windows-x64"] + } + ], + "requires": [], + "kind": "python", + "python": "cp312", + "platforms": { + "macos-arm64": { "minOs": "14.0" }, + "macos-x64": { "minOs": "13.0" }, + "windows-x64": {}, + "linux-x64": {}, + "linux-arm64": {} + }, + "files": [ + { + "name": "onnxruntime-1.31.1-cp312-cp312-macosx_14_0_arm64.whl", + "url": "https://files.pythonhosted.org/packages/0e/f7/cecc0efeeea83bff2f7789ae1b80f27fa0f0210d6d9d66d1f0031a6b5e47/onnxruntime-1.31.1-cp312-cp312-macosx_14_0_arm64.whl", + "size": 20882125, + "sha256": "96dd2564b10ea7153ad9c207e7cd8e5917cb3ba521e0e0a62ac03486fdc8d56c", + "platforms": ["macos-arm64"] + }, + { + "name": "numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl", + "url": "https://files.pythonhosted.org/packages/31/2f/5ea3570fcb8ccd0882bea99436a513b2c85dad8f774a2057849130a8fb99/numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl", + "size": 5451717, + "sha256": "9cb18a327b49c5c337f972b03682f6a49855525faaf3c0d3e9c96cd0fd8880a8", + "platforms": ["macos-arm64"] + }, + { + "name": "onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/91/9d/a81aafd899b900101988ead7fb14974c8a58695338ab6a0f3d6b0100f30b/onnxruntime-1.23.2-cp312-cp312-macosx_13_0_x86_64.whl", + "size": 19157705, + "sha256": "218295a8acae83905f6f1aed8cacb8e3eb3bd7513a13fe4ba3b2664a19fc4a6b", + "platforms": ["macos-x64"] + }, + { + "name": "numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/d0/97/ba2074e92b7befea137e77ea8471e768bbd87c339b7e8c9f5a931949f977/numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl", + "size": 17001609, + "sha256": "c6342f54c67093cae5c0227eb0eb772fdb79f2a2c37a6eb278b9909ee06aa356", + "platforms": ["macos-x64"] + }, + { + "name": "onnxruntime_directml-1.24.4-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/88/ea/33814eb0ec96775eda4c1d30b0d86e91d7d2cd0d84c66d3915aef0e06fa3/onnxruntime_directml-1.24.4-cp312-cp312-win_amd64.whl", + "size": 25111930, + "sha256": "f2ecb68b7b7b259d2ef3112ae760149f9b5a1e7c0fbb73d539da6250a648a614", + "platforms": ["windows-x64"] + }, + { + "name": "numpy-2.5.4-cp312-cp312-win_amd64.whl", + "url": "https://files.pythonhosted.org/packages/12/33/fefe62073dc8acfd0f2b9ed7c003af2f50aa61555e113e6db02b8f79f145/numpy-2.5.4-cp312-cp312-win_amd64.whl", + "size": 12573091, + "sha256": "a7b1b6353e36a7e50de2973a38d705c88ee93adcf120673cee7f45a4a3fa223a", + "platforms": ["windows-x64"] + }, + { + "name": "onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/eb/27/c6a823bac55b2977bea2027c3ec9393a2c1002cb9a09eb40354563e02730/onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_x86_64.whl", + "size": 23760988, + "sha256": "1f3dea47f2f0e5117c2404cea7d21fd253ae7836a3bf167a501cdfbb2600d536", + "platforms": ["linux-x64"] + }, + { + "name": "numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "url": "https://files.pythonhosted.org/packages/43/a3/c799c62e19c337e6d3770b08e475887fb30ce8477d3c09efca6b2f0228a6/numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", + "size": 16727283, + "sha256": "fbde6962867ee75b48b0ee29b2b9372ec5d617799dbaf38e82dc0596f2f7738a", + "platforms": ["linux-x64"] + }, + { + "name": "onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_aarch64.whl", + "url": "https://files.pythonhosted.org/packages/ba/e4/1ed1a7a5f4702d899091fa2b976d377995250c307326ec8dc9ae6378aa13/onnxruntime-1.31.1-cp312-cp312-manylinux_2_28_aarch64.whl", + "size": 21420803, + "sha256": "0b9a492176886992e0e875f06bc0367e333f5fff11711a2d642e18b02f3009cc", + "platforms": ["linux-arm64"] + }, + { + "name": "numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", + "url": "https://files.pythonhosted.org/packages/dc/96/8319e2457ae4333c62c815c7006b869a4f60985c1e01024c2f8c6c040fe5/numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", + "size": 15695312, + "sha256": "fe4d21ab149f15e4e6043dfb0de87e6e5f34ac176cde83060e9802981fca2ac2", + "platforms": ["linux-arm64"] + } + ] +} diff --git a/app/assets/addons/colorize-ddcolor.json b/app/assets/addons/colorize-ddcolor.json new file mode 100644 index 00000000..c416e893 --- /dev/null +++ b/app/assets/addons/colorize-ddcolor.json @@ -0,0 +1,25 @@ +{ + "id": "colorize-ddcolor", + "version": "1.0.0", + "name": "DDColor colourisation model", + "feature": "Colourisation", + "description": "DDColor-T, the default model for the Colourisation pass: vivid, natural colour for black-and-white footage, predicted per frame and steadied over neighbouring frames. Converted to ONNX from the project's own published weights.", + "licence": "Apache-2.0", + "licenceUrl": "https://github.com/piddnad/DDColor/blob/master/LICENSE", + "credit": "Xiaoyang Kang, Tao Yang, Wenqi Ouyang, Peiran Ren, Lingzhi Li and Xuansong Xie", + "noticeName": "DDColor model", + "homepage": "https://github.com/piddnad/DDColor", + "requires": [ + "ai-runtime" + ], + "kind": "files", + "platforms": "all", + "files": [ + { + "name": "colorize-ddcolor-tiny.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/colorize-ddcolor-tiny.onnx", + "size": 220687346, + "sha256": "9300e48e7612fec4a0fffc38eb005fdd65bcc7a859390d6de0db3abf44e46d1c" + } + ] +} diff --git a/app/assets/addons/colorize-deoldify.json b/app/assets/addons/colorize-deoldify.json new file mode 100644 index 00000000..c53179cb --- /dev/null +++ b/app/assets/addons/colorize-deoldify.json @@ -0,0 +1,25 @@ +{ + "id": "colorize-deoldify", + "version": "1.0.0", + "name": "DeOldify Video colourisation model", + "feature": "Colourisation", + "description": "DeOldify's Video model, an alternative for the Colourisation pass: more muted colour than DDColor, trained to stay steady from frame to frame. Converted to ONNX from the project's own published weights.", + "licence": "MIT", + "licenceUrl": "https://github.com/jantic/DeOldify/blob/master/LICENSE", + "credit": "Jason Antic", + "noticeName": "DeOldify Video model", + "homepage": "https://github.com/jantic/DeOldify", + "requires": [ + "ai-runtime" + ], + "kind": "files", + "platforms": "all", + "files": [ + { + "name": "colorize-deoldify-video.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/colorize-deoldify-video.onnx", + "size": 872841884, + "sha256": "aadb46c7d44af94ba0d01b1d3069f90ec83d08c14c60f0d203b3962094835e5a" + } + ] +} diff --git a/app/assets/addons/deepfilternet.json b/app/assets/addons/deepfilternet.json new file mode 100644 index 00000000..381a3ea3 --- /dev/null +++ b/app/assets/addons/deepfilternet.json @@ -0,0 +1,63 @@ +{ + "id": "deepfilternet", + "version": "1.0.0", + "name": "DeepFilterNet speech enhancement", + "feature": "Audio restoration", + "description": "Turns down the noise behind speech — hiss, hum, traffic, a noisy room — with a small neural network that runs on the CPU. Audio Restoration's Speech enhancement uses it. One self-contained program (DeepFilterNet 3, deep-filter 0.5.6), downloaded from the DeepFilterNet project's own release.", + "licence": "MIT OR Apache-2.0", + "licenceUrl": "https://github.com/Rikorose/DeepFilterNet/blob/v0.5.6/LICENSE-MIT", + "credit": "Hendrik Schröter", + "noticeName": "DeepFilterNet", + "homepage": "https://github.com/Rikorose/DeepFilterNet", + "requires": [], + "kind": "files", + "platforms": { + "macos-arm64": { "minOs": "13.0" }, + "macos-x64": { "minOs": "13.0" }, + "windows-x64": {}, + "linux-x64": {}, + "linux-arm64": {} + }, + "files": [ + { + "name": "deep-filter-0.5.6-aarch64-apple-darwin", + "url": "https://github.com/Rikorose/DeepFilterNet/releases/download/v0.5.6/deep-filter-0.5.6-aarch64-apple-darwin", + "size": 27877081, + "sha256": "4601e7f4e4c03e59a4c5b5000216ef3add3e808799cfccd95e14e83ea4611081", + "platforms": ["macos-arm64"], + "executable": true + }, + { + "name": "deep-filter-0.5.6-x86_64-apple-darwin", + "url": "https://github.com/Rikorose/DeepFilterNet/releases/download/v0.5.6/deep-filter-0.5.6-x86_64-apple-darwin", + "size": 29933512, + "sha256": "d3be84003acb7c23e738ad7f70a158ec779a8d233a82e7fa3e717d112eb5b50f", + "platforms": ["macos-x64"], + "executable": true + }, + { + "name": "deep-filter-0.5.6-x86_64-pc-windows-msvc.exe", + "url": "https://github.com/Rikorose/DeepFilterNet/releases/download/v0.5.6/deep-filter-0.5.6-x86_64-pc-windows-msvc.exe", + "size": 26912256, + "sha256": "75e11fa16445f560cb6b021521ddb89e89270d13b83089705d98776f58fd7915", + "platforms": ["windows-x64"], + "executable": true + }, + { + "name": "deep-filter-0.5.6-x86_64-unknown-linux-musl", + "url": "https://github.com/Rikorose/DeepFilterNet/releases/download/v0.5.6/deep-filter-0.5.6-x86_64-unknown-linux-musl", + "size": 36417296, + "sha256": "70775e251eee44c0f2451a1e833326cf8bcbbe304d3e7cd12851e6fce72ef7da", + "platforms": ["linux-x64"], + "executable": true + }, + { + "name": "deep-filter-0.5.6-aarch64-unknown-linux-gnu", + "url": "https://github.com/Rikorose/DeepFilterNet/releases/download/v0.5.6/deep-filter-0.5.6-aarch64-unknown-linux-gnu", + "size": 39238496, + "sha256": "14e02a1c0028f3ca0bdf83b62b3336e56ba0556894ef295a95e8573f06557166", + "platforms": ["linux-arm64"], + "executable": true + } + ] +} diff --git a/app/assets/addons/face-codeformer.json b/app/assets/addons/face-codeformer.json new file mode 100644 index 00000000..098abead --- /dev/null +++ b/app/assets/addons/face-codeformer.json @@ -0,0 +1,26 @@ +{ + "id": "face-codeformer", + "version": "1.0.0", + "name": "CodeFormer face restoration model", + "feature": "Face restoration", + "description": "CodeFormer, an alternative model for the Face Restoration pass with a fidelity control: lower trusts its learned faces more, higher stays closer to the picture. Converted to ONNX from S-Lab's published weights.", + "licence": "S-Lab License 1.0 (NC)", + "licenceUrl": "https://github.com/sczhou/CodeFormer/blob/master/LICENSE", + "usageNote": "Non-commercial use only (S-Lab License 1.0).", + "credit": "S-Lab", + "noticeName": "CodeFormer model", + "homepage": "https://github.com/sczhou/CodeFormer", + "requires": [ + "face-restoration" + ], + "kind": "files", + "platforms": "all", + "files": [ + { + "name": "face-codeformer-v0.1.0.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/face-codeformer-v0.1.0.onnx", + "size": 376827102, + "sha256": "536db9b0d73400d09ca19ba1ea4978ba3b084e3bc0c3ab951597e5536c031e18" + } + ] +} diff --git a/app/assets/addons/face-gfpgan.json b/app/assets/addons/face-gfpgan.json new file mode 100644 index 00000000..17dc6f6d --- /dev/null +++ b/app/assets/addons/face-gfpgan.json @@ -0,0 +1,26 @@ +{ + "id": "face-gfpgan", + "version": "1.0.0", + "name": "GFPGAN v1.4 face restoration model", + "feature": "Face restoration", + "description": "GFPGAN v1.4, an alternative model for the Face Restoration pass: stronger, more stylised detail than RestoreFormer++. Converted to ONNX from Tencent ARC's published weights.", + "licence": "Apache-2.0 + NVIDIA Source Code License (NC)", + "licenceUrl": "https://github.com/TencentARC/GFPGAN/blob/master/LICENSE", + "usageNote": "Non-commercial use only (research or evaluation): GFPGAN's StyleGAN2-derived network is under NVIDIA's Source Code License.", + "credit": "THL A29 Limited, a Tencent company", + "noticeName": "GFPGAN v1.4 model", + "homepage": "https://github.com/TencentARC/GFPGAN", + "requires": [ + "face-restoration" + ], + "kind": "files", + "platforms": "all", + "files": [ + { + "name": "face-gfpgan-v1.4.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/face-gfpgan-v1.4.onnx", + "size": 340357340, + "sha256": "c84d5c2bf1ddc5e61cfc71524b1a155b297c1c3e884255e6a8c0ce5230a91d94" + } + ] +} diff --git a/app/assets/addons/face-restoration.json b/app/assets/addons/face-restoration.json new file mode 100644 index 00000000..bb642382 --- /dev/null +++ b/app/assets/addons/face-restoration.json @@ -0,0 +1,60 @@ +{ + "id": "face-restoration", + "version": "1.0.0", + "name": "Face restoration (RestoreFormer++)", + "feature": "Face restoration", + "description": "Restores blurry, low-resolution faces in the Face Restoration pass: the RetinaFace face detector, the BiSeNet face parser that keeps the blend to the face itself, and RestoreFormer++, the default restoration model. Converted to ONNX from the projects' own published weights.", + "licence": "Apache-2.0", + "licenceUrl": "https://github.com/wzhouxiff/RestoreFormerPlusPlus/blob/main/LICENSE", + "credit": "Zhouxia Wang, Jiawei Zhang, Tianshui Chen, Wenping Wang and Ping Luo", + "noticeName": "RestoreFormer++ model", + "homepage": "https://github.com/wzhouxiff/RestoreFormerPlusPlus", + "components": [ + { + "noticeName": "RetinaFace face detector", + "licence": "MIT", + "licenceUrl": "https://github.com/biubug6/Pytorch_Retinaface/blob/master/LICENSE.MIT", + "credit": "biubug6 (Pytorch_Retinaface)", + "homepage": "https://github.com/biubug6/Pytorch_Retinaface" + }, + { + "noticeName": "BiSeNet face parser", + "licence": "MIT", + "licenceUrl": "https://github.com/zllrunning/face-parsing.PyTorch/blob/master/LICENSE", + "credit": "zll (face-parsing.PyTorch)", + "homepage": "https://github.com/zllrunning/face-parsing.PyTorch" + }, + { + "noticeName": "facexlib", + "licence": "MIT", + "licenceUrl": "https://github.com/xinntao/facexlib/blob/master/LICENSE", + "credit": "Xintao Wang", + "homepage": "https://github.com/xinntao/facexlib" + } + ], + "requires": [ + "ai-runtime" + ], + "kind": "files", + "platforms": "all", + "files": [ + { + "name": "face-restoration-retinaface-r50.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/face-restoration-retinaface-r50.onnx", + "size": 109106945, + "sha256": "da4d5fa7473e3043b5b15df3aa3586d25c5cd94d5cbb53cb8d6309f525381098" + }, + { + "name": "face-restoration-bisenet.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/face-restoration-bisenet.onnx", + "size": 52598302, + "sha256": "2e8a24164b64477200bf7d0c7022858a736f36409e184e4bf3c8cb39844bf048" + }, + { + "name": "face-restoration-restoreformerpp.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/face-restoration-restoreformerpp.onnx", + "size": 295272527, + "sha256": "640da24f9eaae52fe4e840b4394c65583b69e0091947a0d659ccd2176c8cd86b" + } + ] +} diff --git a/app/assets/addons/frame-rate-rife.json b/app/assets/addons/frame-rate-rife.json new file mode 100644 index 00000000..66834f42 --- /dev/null +++ b/app/assets/addons/frame-rate-rife.json @@ -0,0 +1,33 @@ +{ + "id": "frame-rate-rife", + "version": "1.0.0", + "name": "RIFE 4.26 interpolation model", + "feature": "Frame rate conversion", + "description": "The neural network behind the Frame Rate pass's AI interpolation (RIFE): it invents the in-between frames of a frame rate conversion on the graphics card, with fewer warping artefacts than motion interpolation.", + "licence": "MIT", + "licenceUrl": "https://github.com/hzwer/Practical-RIFE/blob/main/LICENSE", + "credit": "hzwer", + "noticeName": "RIFE 4.26 model", + "homepage": "https://github.com/hzwer/Practical-RIFE", + "requires": [], + "kind": "files", + "platforms": { + "macos-arm64": {}, + "windows-x64": {}, + "linux-x64": {} + }, + "files": [ + { + "name": "rife-v4.26_ensembleFalse/flownet.param", + "url": "https://raw.githubusercontent.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/96fdcecdf982ef3237e16e2b105eea849fa022ab/models/rife-v4.26_ensembleFalse/flownet.param", + "size": 67885, + "sha256": "79f16c28903f93f8308f0c4c947f8c7e0c17d99a57b85473e5f298dd578d137b" + }, + { + "name": "rife-v4.26_ensembleFalse/flownet.bin", + "url": "https://raw.githubusercontent.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan/96fdcecdf982ef3237e16e2b105eea849fa022ab/models/rife-v4.26_ensembleFalse/flownet.bin", + "size": 11360684, + "sha256": "94d58e30b75d7c7609cfa6f3bdad524deddd14f5f75e85c36d2f827ef5c64731" + } + ] +} diff --git a/app/assets/addons/upscale-anime.json b/app/assets/addons/upscale-anime.json new file mode 100644 index 00000000..2610c242 --- /dev/null +++ b/app/assets/addons/upscale-anime.json @@ -0,0 +1,60 @@ +{ + "id": "upscale-anime", + "version": "1.0.0", + "name": "Real-ESRGAN anime & Real-CUGAN models", + "feature": "AI upscaling", + "description": "The anime upscaling models for Crop & Resize, run on the graphics card: Real-ESRGAN's anime video model (2x and 4x) and Real-CUGAN (2x, at three noise-removal strengths).", + "licence": "BSD-3-Clause", + "licenceUrl": "https://github.com/xinntao/Real-ESRGAN/blob/master/LICENSE", + "credit": "Xintao Wang", + "noticeName": "Real-ESRGAN anime video model", + "homepage": "https://github.com/xinntao/Real-ESRGAN", + "components": [ + { + "noticeName": "Real-CUGAN models", + "licence": "MIT", + "licenceUrl": "https://github.com/bilibili/ailab/blob/main/Real-CUGAN/LICENSE", + "credit": "bilibili", + "homepage": "https://github.com/bilibili/ailab/tree/main/Real-CUGAN" + } + ], + "requires": [], + "kind": "files", + "platforms": { + "macos-arm64": {}, + "windows-x64": {}, + "linux-x64": {} + }, + "files": [ + { + "name": "upscale-RealESRGANv2-animevideo-xsx2.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-RealESRGANv2-animevideo-xsx2.onnx", + "size": 2407691, + "sha256": "6fad590f96df553a5cc348a9c5129e7874504a7dcd414e4a15fb0b281fca7e0c" + }, + { + "name": "upscale-RealESRGANv2-animevideo-xsx4.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-RealESRGANv2-animevideo-xsx4.onnx", + "size": 2490781, + "sha256": "414de3a5da4c56652c99f3bc0b960910e3561835a6c7a9cca546475bc4792158" + }, + { + "name": "upscale-cugan-up2x-latest-no-denoise.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-cugan-up2x-latest-no-denoise.onnx", + "size": 5148559, + "sha256": "3587278c3a7e62bbcb00a23b4de291ff906f62a88856ee457cbe4211cc593574" + }, + { + "name": "upscale-cugan-up2x-latest-conservative.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-cugan-up2x-latest-conservative.onnx", + "size": 5148559, + "sha256": "555bca3eb90caa37c7e1a02aec9cda3ca8479f252746c1885d382f87312f651c" + }, + { + "name": "upscale-cugan-up2x-latest-denoise3x.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-cugan-up2x-latest-denoise3x.onnx", + "size": 5148559, + "sha256": "7bcb0510f9f989e681a51d39b7170883d3d5e75f1ec23ef5146e8ea50c3cc845" + } + ] +} diff --git a/app/assets/addons/upscale-esrgan-plus.json b/app/assets/addons/upscale-esrgan-plus.json new file mode 100644 index 00000000..7a0fb6c8 --- /dev/null +++ b/app/assets/addons/upscale-esrgan-plus.json @@ -0,0 +1,33 @@ +{ + "id": "upscale-esrgan-plus", + "version": "1.0.0", + "name": "Real-ESRGAN x4plus / x2plus models", + "feature": "AI upscaling", + "description": "Real-ESRGAN's original large models (x4plus and x2plus) for Crop & Resize: the most detailed Real-ESRGAN upscale of real-world footage, run on the graphics card, and very slow — about a second per DVD frame at 2x and ten at 4x on an M1 Pro.", + "licence": "BSD-3-Clause", + "licenceUrl": "https://github.com/xinntao/Real-ESRGAN/blob/master/LICENSE", + "credit": "Xintao Wang", + "noticeName": "Real-ESRGAN x4plus / x2plus models", + "homepage": "https://github.com/xinntao/Real-ESRGAN", + "requires": [], + "kind": "files", + "platforms": { + "macos-arm64": {}, + "windows-x64": {}, + "linux-x64": {} + }, + "files": [ + { + "name": "upscale-RealESRGAN_x4plus.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-RealESRGAN_x4plus.onnx", + "size": 67051646, + "sha256": "86df1f74dbb7ee7521f2e342fb610eb7c147e161cbd377faafa706d4880716ca" + }, + { + "name": "upscale-RealESRGAN_x2plus.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-RealESRGAN_x2plus.onnx", + "size": 67073184, + "sha256": "0fb8264f02ead334932e50fd5a23d8ca4884fcbaa747943acf5ae6278a112d49" + } + ] +} diff --git a/app/assets/addons/upscale-general.json b/app/assets/addons/upscale-general.json new file mode 100644 index 00000000..5283914a --- /dev/null +++ b/app/assets/addons/upscale-general.json @@ -0,0 +1,48 @@ +{ + "id": "upscale-general", + "version": "1.0.0", + "name": "General & live-action upscaling models", + "feature": "AI upscaling", + "description": "Two upscaling models for real-world footage rather than anime, run on the graphics card by Crop & Resize: Real-ESRGAN's general model (4x, with a noise-removing variant) and jcj83429's 2xLiveActionV1 SPAN model (2x), which repairs compression, blur and halos while keeping detail.", + "licence": "BSD-3-Clause", + "licenceUrl": "https://github.com/xinntao/Real-ESRGAN/blob/master/LICENSE", + "credit": "Xintao Wang", + "noticeName": "Real-ESRGAN general model", + "homepage": "https://github.com/xinntao/Real-ESRGAN", + "components": [ + { + "noticeName": "2xLiveActionV1_SPAN model", + "licence": "Apache-2.0", + "licenceUrl": "https://github.com/jcj83429/upscaling/blob/5d8cdd2e17750b64be39ccab3a8763d91128fe15/2xLiveActionV1_SPAN/LICENSE", + "credit": "jcj83429", + "homepage": "https://github.com/jcj83429/upscaling/tree/main/2xLiveActionV1_SPAN" + } + ], + "requires": [], + "kind": "files", + "platforms": { + "macos-arm64": {}, + "windows-x64": {}, + "linux-x64": {} + }, + "files": [ + { + "name": "upscale-realesr-general-x4v3.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-realesr-general-x4v3.onnx", + "size": 4866424, + "sha256": "f3fe05b8a48076ec0fe58143cf39f34fbc93b42197f897e329613cf8521d79d8" + }, + { + "name": "upscale-realesr-general-wdn-x4v3.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/upscale-realesr-general-wdn-x4v3.onnx", + "size": 4866424, + "sha256": "d0b7c5d84193670b94db88585fe2120cd48816c12f3cedfd8c803167db6b451a" + }, + { + "name": "2xLiveActionV1_SPAN_490000.onnx", + "url": "https://raw.githubusercontent.com/jcj83429/upscaling/5d8cdd2e17750b64be39ccab3a8763d91128fe15/2xLiveActionV1_SPAN/2xLiveActionV1_SPAN_490000.onnx", + "size": 1654748, + "sha256": "bfa72f3c6347076aed140d0836cee30c27ea434c047beeaf9466469483836ecc" + } + ] +} diff --git a/app/assets/deps-version.json b/app/assets/deps-version.json index 6537b71f..3208d14c 100644 --- a/app/assets/deps-version.json +++ b/app/assets/deps-version.json @@ -1,7 +1,7 @@ { - "version": "1.15.0", - "releaseTag": "deps-v1.15.0", - "releaseDate": "2026-11-03", + "version": "1.16.0", + "releaseTag": "deps-v1.16.0", + "releaseDate": "2026-11-09", "tierFormat": "delta", "githubRepo": "StuartCameronCode/VapourBox" } diff --git a/app/assets/filters/core/anti_alias.json b/app/assets/filters/core/anti_alias.json index 6ecd2118..558459a1 100644 --- a/app/assets/filters/core/anti_alias.json +++ b/app/assets/filters/core/anti_alias.json @@ -4,7 +4,7 @@ "version": "1.0.0", "name": "Anti-Aliasing", "description": "Smooth stair-stepping on diagonal edges", - "longDescription": "Removes the stair-stepping — \"jaggies\" — that appears along diagonal edges. Both methods work by re-interpolating the frame with an edge-directed kernel and keeping the smoother result, so the edge is rebuilt rather than blurred.\n\nUse it after deinterlacing and after upscaling, which are the two things that create the stepping. It runs before sharpening on purpose: sharpening a stair-stepped edge makes the steps more visible, not less. On a source with no diagonal detail it does nothing but cost time.", + "longDescription": "Removes the stair-stepping — \"jaggies\" — that appears along diagonal edges. Every method works by re-interpolating the frame with an edge-directed kernel and keeping the smoother result, so the edge is rebuilt rather than blurred.\n\nUse it after deinterlacing and after upscaling, which are the two things that create the stepping. It runs before sharpening on purpose: sharpening a stair-stepped edge makes the steps more visible, not less. On a source with no diagonal detail it does nothing but cost time.", "category": "enhancement", "icon": "gesture", "order": 7, @@ -33,6 +33,16 @@ "santiagStrh", "santiagStrv" ] + }, + { + "id": "sangnom", + "sinceAppVersion": "1.7.0", + "name": "SangNom", + "description": "The sharpest: rebuilds edges at twice the size with SangNom, then scales back down. Slower than daa on HD sources, since it works at twice the size. Strong enough to thin line art slightly, which suits anime more than live action", + "function": "core.sangnom.SangNom", + "parameters": [ + "sangnomAa" + ] } ], "parameters": { @@ -48,7 +58,8 @@ "default": "daa", "options": [ "daa", - "santiag" + "santiag", + "sangnom" ], "ui": { "hidden": true @@ -71,6 +82,23 @@ } } }, + "sangnomAa": { + "type": "integer", + "default": 48, + "min": 0, + "max": 128, + "step": 4, + "ui": { + "label": "Strength", + "description": "How hard SangNom smooths along the edges it finds. 48 is the plugin's default; 0 still rebuilds the edges but without smoothing them", + "widget": "slider", + "visibleWhen": { + "method": [ + "sangnom" + ] + } + } + }, "santiagStrv": { "type": "integer", "default": 1, @@ -95,7 +123,8 @@ "title": "Settings", "parameters": [ "santiagStrh", - "santiagStrv" + "santiagStrv", + "sangnomAa" ], "expanded": true } diff --git a/app/assets/filters/core/audio_restoration.json b/app/assets/filters/core/audio_restoration.json new file mode 100644 index 00000000..9390c5df --- /dev/null +++ b/app/assets/filters/core/audio_restoration.json @@ -0,0 +1,343 @@ +{ + "$schema": "https://vapourbox.app/schemas/filter-v1.json", + "id": "audio_restoration", + "version": "1.0.0", + "sinceAppVersion": "1.7.0", + "name": "Audio Restoration", + "description": "Repair, clean and re-sync the soundtrack. Applied during the encode — the preview shows the picture only.", + "longDescription": "Fixes the faults old recordings bring with their sound: a capture with one dead channel, mains hum, rumble, clicks, clipped peaks, tape hiss, speech buried in noise, sound that is out of step with the picture or drifts out of step over a long tape, and loudness that is far too quiet or uneven. Switch on only the steps the recording needs; with none ticked the sound is left exactly as it was.\n\nThe sound is processed once, before the video is encoded, to exactly the length of the picture (after any trim, deinterlacing or frame-rate change). Subtitles are transcribed from the restored sound, so they follow any sync offset or drift correction. The preview is picture only — listen to a short trimmed export to judge the result.\n\nOnly one audio stream is restored (the first, unless chosen otherwise); any others are kept as they are. When Output → Audio is set to copy, the restored stream is encoded anyway, because it is new audio: FLAC in MKV, AAC in MP4 and MOV, PCM in AVI.", + "category": "enhancement", + "icon": "graphic_eq", + "order": 95, + "methods": [ + { + "id": "restore", + "name": "Restore", + "description": "FFmpeg's audio filters, and DeepFilterNet for speech", + "function": "custom", + "parameters": [ + "channelFix", + "offsetMs", + "driftCorrection", + "driftMaxPercent", + "declip", + "declick", + "declickThreshold", + "rumbleFilter", + "rumbleCutoffHz", + "humRemoval", + "humFrequency", + "denoise", + "denoiseStrength", + "deepFilter", + "deepFilterAttenuation", + "loudnessNormalize", + "loudnessTarget", + "limiter", + "limiterCeilingDb", + "streamIndex" + ] + } + ], + "implementation": [ + { + "function": "pan", + "role": "channel fix: copy the live channel over a dead one, or mix the two", + "activeWhen": { "channelFix": ["auto", "leftOnly", "rightOnly", "mix"] } + }, + { "function": "adeclip", "role": "rebuild clipped peaks", "activeWhen": { "declip": true } }, + { "function": "adeclick", "role": "remove clicks and crackle", "activeWhen": { "declick": true } }, + { "function": "highpass", "role": "remove rumble", "activeWhen": { "rumbleFilter": true } }, + { + "function": "bandreject", + "role": "notch out mains hum and its first two harmonics", + "activeWhen": { "humRemoval": true } + }, + { "function": "afftdn", "role": "broadband noise reduction", "activeWhen": { "denoise": true } }, + { "function": "atempo", "role": "drift correction", "activeWhen": { "driftCorrection": true } }, + { "function": "-ss / adelay", "role": "where the sound starts: the trim and the sync offset" }, + { + "function": "deep-filter (DeepFilterNet)", + "role": "speech enhancement", + "activeWhen": { "deepFilter": true } + }, + { + "function": "loudnorm", + "role": "loudness normalisation, measured first, then applied as one gain", + "activeWhen": { "loudnessNormalize": true } + }, + { "function": "alimiter", "role": "peak limiter", "activeWhen": { "limiter": true } }, + { "function": "apad / atrim", "role": "the sound cut or padded to the picture's length" } + ], + "parameters": { + "enabled": { + "type": "boolean", + "default": false, + "ui": { "hidden": true } + }, + "method": { + "type": "enum", + "default": "restore", + "options": ["restore"], + "ui": { "hidden": true } + }, + "channelFix": { + "type": "enum", + "default": "off", + "options": ["off", "auto", "leftOnly", "rightOnly", "mix"], + "ui": { + "label": "Channels", + "description": "For a stereo capture where one channel is silent or broken. Automatic measures both and copies the live one over a silent one; otherwise it leaves them alone.", + "widget": "dropdown", + "optionLabels": { + "off": "Leave as is", + "auto": "Automatic (fix a silent channel)", + "leftOnly": "Left channel only", + "rightOnly": "Right channel only", + "mix": "Mix both channels" + } + } + }, + "offsetMs": { + "type": "integer", + "default": 0, + "min": -10000, + "max": 10000, + "step": 10, + "ui": { + "label": "Sync offset (ms)", + "description": "Moves the sound against the picture. Positive delays the sound (use it when lips move before you hear them); negative plays it earlier.", + "widget": "number" + } + }, + "driftCorrection": { + "type": "boolean", + "default": false, + "ui": { + "label": "Correct drift", + "description": "For sound that starts in sync and slowly drifts out over a long recording: stretches or squeezes it to the picture's length. Only applied when the two differ by more than 0.05 s and by no more than the limit below.", + "widget": "checkbox" + } + }, + "driftMaxPercent": { + "type": "number", + "default": 0.5, + "min": 0.05, + "max": 2.0, + "step": 0.05, + "ui": { + "label": "Largest drift corrected (%)", + "description": "A bigger difference is not clock drift — usually a missing stretch of audio or the wrong track — and is left alone rather than disguised.", + "widget": "slider", + "precision": 2, + "visibleWhen": { "driftCorrection": true } + } + }, + "declip": { + "type": "boolean", + "default": false, + "ui": { + "label": "Repair clipping", + "description": "Rebuilds peaks that were flattened by recording too loud.", + "widget": "checkbox" + } + }, + "declick": { + "type": "boolean", + "default": false, + "ui": { + "label": "Remove clicks", + "description": "Clicks, crackle and dropouts, as from vinyl or a damaged tape.", + "widget": "checkbox" + } + }, + "declickThreshold": { + "type": "number", + "default": 2.0, + "min": 1.0, + "max": 10.0, + "step": 0.5, + "ui": { + "label": "Click threshold", + "description": "Lower removes more, but can soften sharp sounds like drum hits.", + "widget": "slider", + "precision": 1, + "visibleWhen": { "declick": true } + } + }, + "rumbleFilter": { + "type": "boolean", + "default": false, + "ui": { + "label": "Remove rumble", + "description": "Low-frequency rumble from handling, wind or a turntable motor.", + "widget": "checkbox" + } + }, + "rumbleCutoffHz": { + "type": "integer", + "default": 40, + "min": 20, + "max": 120, + "step": 5, + "ui": { + "label": "Rumble cutoff (Hz)", + "description": "Everything below this is turned down.", + "widget": "slider", + "visibleWhen": { "rumbleFilter": true } + } + }, + "humRemoval": { + "type": "boolean", + "default": false, + "ui": { + "label": "Remove mains hum", + "description": "The steady 50 or 60 Hz hum of a ground loop, with its first two harmonics.", + "widget": "checkbox" + } + }, + "humFrequency": { + "type": "enum", + "default": "auto", + "options": ["auto", "50", "60"], + "ui": { + "label": "Hum frequency", + "description": "Automatic measures the sound and removes whichever hum is there — nothing, if neither is. 50 Hz is the mains in Europe, Africa, most of Asia and Australia; 60 Hz in the Americas.", + "widget": "dropdown", + "optionLabels": { "auto": "Automatic", "50": "50 Hz", "60": "60 Hz" }, + "visibleWhen": { "humRemoval": true } + } + }, + "denoise": { + "type": "boolean", + "default": false, + "ui": { + "label": "Reduce hiss and noise", + "description": "Broadband noise such as tape hiss or fan noise.", + "widget": "checkbox" + } + }, + "denoiseStrength": { + "type": "number", + "default": 12.0, + "min": 3.0, + "max": 40.0, + "step": 1.0, + "ui": { + "label": "Noise reduction (dB)", + "description": "How far the noise is turned down. High values make speech sound watery.", + "widget": "slider", + "precision": 0, + "visibleWhen": { "denoise": true } + } + }, + "deepFilter": { + "type": "boolean", + "default": false, + "ui": { + "label": "Speech enhancement (DeepFilterNet)", + "description": "A neural network that separates speech from the noise behind it — very effective on dialogue, not meant for music. A separate download (about 30 MB).", + "widget": "checkbox" + } + }, + "deepFilterAttenuation": { + "type": "integer", + "default": 100, + "min": 6, + "max": 100, + "step": 1, + "ui": { + "label": "Noise removed (dB)", + "description": "How far DeepFilterNet may turn the noise down. 100 removes all it can; 15–30 keeps a little of the room, which often sounds more natural.", + "widget": "slider", + "visibleWhen": { "deepFilter": true } + } + }, + "loudnessNormalize": { + "type": "boolean", + "default": false, + "ui": { + "label": "Normalise loudness", + "description": "Measures the whole soundtrack and sets one gain so it plays at a standard loudness (EBU R128), peaks kept under −1.5 dBTP.", + "widget": "checkbox" + } + }, + "loudnessTarget": { + "type": "number", + "default": -16.0, + "min": -31.0, + "max": -10.0, + "step": 1.0, + "ui": { + "label": "Loudness target (LUFS)", + "description": "−16 suits online video and phones; −23 is the broadcast standard (EBU R128); −24 for US broadcast (ATSC A/85).", + "widget": "slider", + "precision": 0, + "visibleWhen": { "loudnessNormalize": true } + } + }, + "limiter": { + "type": "boolean", + "default": false, + "ui": { + "label": "Limit peaks", + "description": "Catches any peak above the ceiling, after everything else.", + "widget": "checkbox" + } + }, + "limiterCeilingDb": { + "type": "number", + "default": -1.0, + "min": -12.0, + "max": 0.0, + "step": 0.5, + "ui": { + "label": "Ceiling (dBFS)", + "widget": "slider", + "precision": 1, + "visibleWhen": { "limiter": true } + } + }, + "streamIndex": { + "type": "integer", + "default": 0, + "min": 0, + "max": 15, + "step": 1, + "ui": { + "label": "Audio stream to restore", + "description": "0 is the first audio stream. The others are kept as they are, after the restored one.", + "widget": "number" + } + } + }, + "ui": { + "sections": [ + { + "title": "Channels and Sync", + "parameters": ["channelFix", "offsetMs", "driftCorrection", "driftMaxPercent"], + "expanded": true + }, + { + "title": "Repair", + "parameters": ["declip", "declick", "declickThreshold", "rumbleFilter", "rumbleCutoffHz", "humRemoval", "humFrequency"], + "expanded": true + }, + { + "title": "Noise", + "parameters": ["denoise", "denoiseStrength", "deepFilter", "deepFilterAttenuation"], + "expanded": true + }, + { + "title": "Loudness", + "parameters": ["loudnessNormalize", "loudnessTarget", "limiter", "limiterCeilingDb"], + "expanded": true + }, + { + "title": "Source Stream", + "parameters": ["streamIndex"], + "advancedOnly": true + } + ] + } +} diff --git a/app/assets/filters/core/colorize.json b/app/assets/filters/core/colorize.json new file mode 100644 index 00000000..b464a597 --- /dev/null +++ b/app/assets/filters/core/colorize.json @@ -0,0 +1,155 @@ +{ + "$schema": "https://vapourbox.app/schemas/filter-v1.json", + "id": "colorize", + "version": "1.0.0", + "sinceAppVersion": "1.7.0", + "name": "Colourisation", + "description": "Colour black-and-white footage with AI", + "longDescription": "Turns black-and-white film scans, old broadcasts and home movies into colour. A neural network trained on over a million colour photographs judges what colour each thing in the picture probably was — skin, sky, foliage, clothing — from its shapes and shading alone.\n\nOnly the colour is added: the brightness of every pixel stays exactly as it was, so no detail is lost or invented. The colours are the network's best guess, not a record of the original, and it can be wrong about anything whose colour could have been anything, such as clothes or cars. Each frame's colour is steadied against its neighbours so it doesn't flicker, and it runs before Color Correction, so you can grade the result there.\n\nIf the video already has colour, that colour is replaced by the network's.\n\nThe models are separate downloads, free for any use. DDColor (220 MB) gives vivid, natural colour. DeOldify Video (870 MB) is more muted and was trained to stay steady on video. On a Mac they run on the graphics card and Neural Engine, on Windows on any DirectX 12 graphics card, and otherwise on the CPU, which is much slower.", + "category": "color", + "icon": "colorize", + "methods": [ + { + "id": "ddcolor", + "name": "DDColor", + "description": "Vivid, natural colour (220 MB download). The default", + "function": "custom", + "parameters": [ + "ddcolorRenderSize", + "strength", + "temporalRadius" + ] + }, + { + "id": "deoldify", + "name": "DeOldify Video", + "description": "More muted colour, trained to stay steady on video (870 MB download)", + "function": "custom", + "parameters": [ + "deoldifyRenderFactor", + "strength", + "temporalRadius" + ] + } + ], + "parameters": { + "enabled": { + "type": "boolean", + "default": false, + "ui": { + "hidden": true + } + }, + "method": { + "type": "enum", + "default": "ddcolor", + "options": [ + "ddcolor", + "deoldify" + ], + "ui": { + "hidden": true + } + }, + "ddcolorRenderSize": { + "type": "integer", + "default": 512, + "min": 256, + "max": 768, + "step": 32, + "ui": { + "label": "Render Size", + "description": "The size (in pixels, square) the network sees the picture at. 512 is what DDColor was made for; smaller is faster and can give more uniform colour, larger is slower. The colour is scaled to the video's own size either way", + "widget": "slider", + "precision": 0, + "visibleWhen": { + "method": [ + "ddcolor" + ] + } + } + }, + "deoldifyRenderFactor": { + "type": "integer", + "default": 21, + "min": 10, + "max": 40, + "step": 1, + "ui": { + "label": "Render Factor", + "description": "DeOldify's render resolution: the network sees a square 16 times this many pixels across (21 = 336 pixels, its default for video). Higher picks up smaller things and tends to steadier colour, but is slower and can wash colour out", + "widget": "slider", + "precision": 0, + "visibleWhen": { + "method": [ + "deoldify" + ] + } + } + }, + "strength": { + "type": "number", + "default": 1.0, + "min": 0.0, + "max": 1.5, + "step": 0.05, + "ui": { + "label": "Colour Strength", + "description": "How much of the predicted colour is applied. 1 is the network's colour; lower fades it toward black and white, higher makes it more saturated", + "widget": "slider", + "precision": 2 + } + }, + "temporalRadius": { + "type": "integer", + "default": 2, + "min": 0, + "max": 4, + "step": 1, + "ui": { + "label": "Steady Colour", + "description": "How many frames either side each frame's colour is averaged with, wherever the picture did not move, so colour doesn't flicker from frame to frame. 0 colours every frame on its own; more is steadier on still shots and slower to start a preview", + "widget": "slider", + "precision": 0 + } + } + }, + "implementation": [ + { + "function": "colorize.colorize", + "role": "predicts the colour, steadies it over neighbouring frames and puts it with the source's own luma (ONNX Runtime)" + }, + { + "function": "DDColor-T", + "role": "predicts CIELAB colour, recombined with the frame's lightness", + "activeWhen": { + "method": "ddcolor" + } + }, + { + "function": "DeOldify Video", + "role": "predicts a colour picture, of which only the colour is kept", + "activeWhen": { + "method": "deoldify" + } + }, + { + "function": "core.std.ShufflePlanes", + "role": "keeps the source's Y plane bit-exact, replacing only U and V" + } + ], + "ui": { + "sections": [ + { + "title": "Colourisation", + "parameters": [ + "ddcolorRenderSize", + "deoldifyRenderFactor", + "strength", + "temporalRadius" + ], + "expanded": true + } + ] + } +} diff --git a/app/assets/filters/core/crop_resize.json b/app/assets/filters/core/crop_resize.json index 8a75b57d..f5c60468 100644 --- a/app/assets/filters/core/crop_resize.json +++ b/app/assets/filters/core/crop_resize.json @@ -4,7 +4,7 @@ "version": "1.3.0", "name": "Crop & Resize", "description": "Crop borders, resize or upscale, and add bars to a target size", - "longDescription": "Trims unwanted borders, changes the output resolution, and adds bars out to a target frame size. Cropping happens first, then resizing, then the bars.\n\nUse crop to cut the head-switching noise along the bottom of VHS captures and the black overscan edges of broadcast material — otherwise the encoder spends bitrate on them. Use resize for a target resolution, or Upscale for a much better 2x/4x enlargement than a plain kernel: NNEDI3 or EEDI3 on any machine, or an AI upscaler trained on anime (Real-ESRGAN, Real-CUGAN, AnimeJaNai) on the graphics card. Keep crop values even so they stay aligned with chroma subsampling.\n\nUse Add Borders to bring a cropped picture back to an exact frame size for authoring (720x576 for PAL DVD, 720x480 for NTSC) without rescaling it: the picture is centred and the rest filled with bars.", + "longDescription": "Trims unwanted borders, changes the output resolution, and adds bars out to a target frame size. Cropping happens first, then resizing, then the bars.\n\nUse crop to cut the head-switching noise along the bottom of VHS captures and the black overscan edges of broadcast material — otherwise the encoder spends bitrate on them. Detect borders measures them on frames from your video and offers a crop to apply; a letterbox bar is offered separately, since it is part of the picture's framing. Use resize for a target resolution, or Upscale for a much better 2x/4x enlargement than a plain kernel: NNEDI3 or EEDI3 on any machine, or an AI upscaler on the graphics card: trained on anime (Real-ESRGAN, Real-CUGAN, AnimeJaNai) or on real-world footage (Real-ESRGAN General, Live Action SPAN, Real-ESRGAN x4plus, each a separate download). Keep crop values even so they stay aligned with chroma subsampling. An SD video taken to HD is converted from BT.601 to BT.709 colour, the HD standard, so players that assume 709 for an HD picture show its colours correctly.\n\nUse Add Borders to bring a cropped picture back to an exact frame size for authoring (720x576 for PAL DVD, 720x480 for NTSC) without rescaling it: the picture is centred and the rest filled with bars.", "category": "transform", "icon": "crop", "order": 9, @@ -33,6 +33,7 @@ "bicubicC", "lanczosTaps", "maintainAspect", + "convertToBt709", "pixelAspect", "customSar", "displayAspect", @@ -78,7 +79,7 @@ "type": "integer", "default": 8, "min": 0, - "max": 200, + "max": 480, "step": 2, "optional": true, "vapoursynth": { @@ -97,7 +98,7 @@ "type": "integer", "default": 8, "min": 0, - "max": 200, + "max": 480, "step": 2, "optional": true, "vapoursynth": { @@ -116,7 +117,7 @@ "type": "integer", "default": 4, "min": 0, - "max": 200, + "max": 480, "step": 2, "optional": true, "vapoursynth": { @@ -135,7 +136,7 @@ "type": "integer", "default": 4, "min": 0, - "max": 200, + "max": 480, "step": 2, "optional": true, "vapoursynth": { @@ -232,6 +233,26 @@ } } }, + "convertToBt709": { + "type": "enum", + "default": "auto", + "options": [ + "auto", + "on", + "off" + ], + "sinceAppVersion": "1.7.0", + "ui": { + "label": "Convert Colour to BT.709 (HD standard)", + "widget": "dropdown", + "optionLabels": { + "auto": "Auto: when the output is HD", + "on": "Always", + "off": "Never" + }, + "description": "An SD video's colour is BT.601. Upscaled to HD it is still tagged 601, which is correct, but many players and TVs ignore the tag on an HD picture and decode it as BT.709, which shifts greens and reds. Auto converts the colour to BT.709 when the output is at least 720 lines (or 1280 wide), so every player shows it the same; Always converts at any size. The source's primaries are kept and tagged. Never leaves the colour and its tags as they are." + } + }, "useIntegerUpscale": { "type": "boolean", "default": false, @@ -249,7 +270,10 @@ "spline36", "realEsrgan", "realCugan", - "animeJanai" + "animeJanai", + "realEsrganGeneral", + "liveAction", + "realEsrganPlus" ], "optional": true, "ui": { @@ -258,11 +282,14 @@ "nnedi3Rpow2": "NNEDI3", "eedi3Rpow2": "EEDI3", "spline36": "Spline36", - "realEsrgan": "Real-ESRGAN", - "realCugan": "Real-CUGAN", - "animeJanai": "AnimeJaNai (download, non-commercial)" + "realEsrgan": "Real-ESRGAN (download)", + "realCugan": "Real-CUGAN (download)", + "animeJanai": "AnimeJaNai (download, non-commercial)", + "realEsrganGeneral": "Real-ESRGAN General (download)", + "liveAction": "Live Action SPAN (download)", + "realEsrganPlus": "Real-ESRGAN x4plus (download, slow)" }, - "description": "How the picture is enlarged. NNEDI3 is the usual choice, an edge-directed doubler that runs anywhere; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. AnimeJaNai, trained on HD anime, keeps colour almost exactly and comes in three speeds; it is a separate download, free for non-commercial use only. Use them after deinterlacing: on a combed picture they sharpen the combing too", + "description": "How the picture is enlarged. NNEDI3 is the usual choice, an edge-directed doubler that runs anywhere; EEDI3 is slower and follows near-horizontal edges better (it uses NNEDI3 to guide its search); Spline36 is plain resampling with no edge direction. Real-ESRGAN and Real-CUGAN are neural upscalers trained on anime: they redraw clean line art rather than interpolating it, and run on the graphics card (Vulkan). Real-ESRGAN is sharper and faster; Real-CUGAN is gentler and can remove noise as it goes. Their models are one small separate download (20 MB), offered when you choose either. AnimeJaNai, trained on HD anime, keeps colour almost exactly and comes in three speeds; it is a separate download, free for non-commercial use only. For live action and other real-world footage there are three more, each a separate download: Real-ESRGAN General, fast and sharp, which redraws texture and can remove noise too; Live Action SPAN, trained on film and video to repair compression, blur and halos while keeping detail and colour (it cannot handle VHS damage — use it on DVD, broadcast or digital sources); and Real-ESRGAN x4plus, the most detailed and very slow. Use them after deinterlacing: on a combed picture they sharpen the combing too", "widget": "dropdown", "visibleWhen": { "useIntegerUpscale": true @@ -271,7 +298,10 @@ "optionSinceAppVersion": { "realEsrgan": "1.5.0", "realCugan": "1.5.0", - "animeJanai": "1.5.0" + "animeJanai": "1.5.0", + "realEsrganGeneral": "1.7.0", + "liveAction": "1.7.0", + "realEsrganPlus": "1.7.0" } }, "upscaleFactor": { @@ -352,6 +382,46 @@ } } }, + "generalDenoise": { + "type": "boolean", + "default": false, + "sinceAppVersion": "1.7.0", + "ui": { + "label": "Real-ESRGAN General Denoise", + "description": "Use the variant trained to remove noise as it upscales (realesr-general-wdn-x4v3). Leave off for clean sources: it also smooths fine texture", + "visibleWhen": { + "useIntegerUpscale": true, + "upscaleMethod": [ + "realEsrganGeneral" + ] + } + } + }, + "aiStrength": { + "type": "number", + "default": 1.0, + "min": 0.0, + "max": 1.0, + "step": 0.05, + "sinceAppVersion": "1.7.0", + "ui": { + "label": "AI Strength", + "description": "How much of the AI upscale to keep. Below 1 the result is blended with a plain Spline36 upscale of the same frame, which tones down a model that redraws too much or shifts flat colours (Real-ESRGAN tints flat areas by a few levels). 1 is the model alone", + "widget": "slider", + "precision": 2, + "visibleWhen": { + "useIntegerUpscale": true, + "upscaleMethod": [ + "realEsrgan", + "realCugan", + "animeJanai", + "realEsrganGeneral", + "liveAction", + "realEsrganPlus" + ] + } + } + }, "bicubicB": { "type": "number", "default": 0.0, @@ -891,7 +961,8 @@ "bicubicB", "bicubicC", "lanczosTaps", - "maintainAspect" + "maintainAspect", + "convertToBt709" ], "expanded": true }, @@ -904,7 +975,9 @@ "upscaleFactor", "cuganNoise", "animejanaiModel", - "animejanaiSharp" + "animejanaiSharp", + "generalDenoise", + "aiStrength" ], "expanded": false }, @@ -1015,6 +1088,40 @@ "upscaleMethod": "animeJanai" } }, + { + "function": "core.ncnn.Model", + "role": "Real-ESRGAN General (add-on), 4x, on the GPU (vs-mlrt's VSNCNN); core.resize.Spline36 to 2x after it", + "activeWhen": { + "useIntegerUpscale": true, + "upscaleMethod": "realEsrganGeneral" + } + }, + { + "function": "core.ncnn.Model", + "role": "2xLiveActionV1 SPAN (add-on), 2x, twice for 4x, on the GPU (vs-mlrt's VSNCNN)", + "activeWhen": { + "useIntegerUpscale": true, + "upscaleMethod": "liveAction" + } + }, + { + "function": "core.ncnn.Model", + "role": "Real-ESRGAN x2plus / x4plus (add-on), by factor, on the GPU (vs-mlrt's VSNCNN)", + "activeWhen": { + "useIntegerUpscale": true, + "upscaleMethod": "realEsrganPlus" + } + }, + { + "function": "core.resize.Spline36", + "role": "BT.601 to BT.709 colour (the matrix only), after the resize, when the output is HD", + "activeWhen": { + "convertToBt709": [ + "auto", + "on" + ] + } + }, { "function": "core.std.AddBorders", "role": "Pad to Fill's letterbox/pillarbox bars, in the border colour", diff --git a/app/assets/filters/core/deblock.json b/app/assets/filters/core/deblock.json index ff2a6eef..22e97b36 100644 --- a/app/assets/filters/core/deblock.json +++ b/app/assets/filters/core/deblock.json @@ -49,6 +49,15 @@ "dctStrength", "dctPlanes" ] + }, + { + "id": "mpeg2stinx", + "sinceAppVersion": "1.7.0", + "name": "mpeg2stinx", + "description": "For film DVDs after inverse telecine: removes the comb-like noise left where progressive frames were encoded as interlaced MPEG-2. Not a general deblocker; on clean or detailed sources it softens vertical detail", + "function": "mpeg2stinx (vs-jetpack, ported): core.bwdif.Bwdif + _nnedi3", + "parameters": [], + "advancedOnly": true } ], "parameters": { @@ -65,7 +74,8 @@ "options": [ "deblock_qed", "deblock", - "dctfilter" + "dctfilter", + "mpeg2stinx" ], "ui": { "hidden": true diff --git a/app/assets/filters/core/dehalo.json b/app/assets/filters/core/dehalo.json index 12ac5ef8..4921116b 100644 --- a/app/assets/filters/core/dehalo.json +++ b/app/assets/filters/core/dehalo.json @@ -123,6 +123,19 @@ "deringThr", "deringDarkthr" ] + }, + { + "id": "vine", + "sinceAppVersion": "1.7.0", + "name": "vine (NLMeans)", + "description": "Removes halos with non-local means and then puts back the edge detail the halo was riding on, so edges stay crisp where DeHalo_alpha softens them. Slower; best on line art and clean sources", + "function": "vine_dehalo (vs-jetpack, ported): core.vszip.NLMeans + _nnedi3", + "parameters": [ + "vineStrength", + "vineSharp", + "vineSigma" + ], + "advancedOnly": true } ], "parameters": { @@ -144,7 +157,8 @@ "edge_cleaner", "vinverse", "vinverse2", - "hq_deringmod" + "hq_deringmod", + "vine" ], "ui": { "hidden": true @@ -748,6 +762,60 @@ ] } } + }, + "vineStrength": { + "type": "number", + "default": 16.0, + "min": 1.0, + "max": 64.0, + "step": 1.0, + "ui": { + "label": "Strength", + "description": "How hard halos are smoothed away. Higher removes wider, brighter halos and more fine texture with them", + "widget": "slider", + "precision": 0, + "visibleWhen": { + "method": [ + "vine" + ] + } + } + }, + "vineSharp": { + "type": "number", + "default": 0.5, + "min": 0.05, + "max": 1.0, + "step": 0.05, + "ui": { + "label": "Sharpness", + "description": "How much of the twice-size pass is blended in, which keeps edges finer", + "widget": "slider", + "precision": 2, + "visibleWhen": { + "method": [ + "vine" + ] + } + } + }, + "vineSigma": { + "type": "number", + "default": 1.0, + "min": 0.3, + "max": 4.0, + "step": 0.1, + "ui": { + "label": "Detail split", + "description": "Where fine detail ends and halo begins (a blur radius). Larger treats wider structures as halo", + "widget": "slider", + "precision": 1, + "visibleWhen": { + "method": [ + "vine" + ] + } + } } }, "ui": { @@ -770,7 +838,9 @@ "vinverseChroma", "deringMrad", "deringMsmooth", - "deringThr" + "deringThr", + "vineStrength", + "vineSharp" ], "expanded": true }, @@ -789,7 +859,8 @@ "edgeSmallMode", "edgeHotPixels", "deringMthr", - "deringDarkthr" + "deringDarkthr", + "vineSigma" ], "expanded": false, "advancedOnly": true diff --git a/app/assets/filters/core/deinterlace.json b/app/assets/filters/core/deinterlace.json index 5dbc463c..407859c0 100644 --- a/app/assets/filters/core/deinterlace.json +++ b/app/assets/filters/core/deinterlace.json @@ -1961,6 +1961,45 @@ ] } } + }, + "ivtcFixFades": { + "type": "boolean", + "default": false, + "sinceAppVersion": "1.7.0", + "ui": { + "label": "Fix fades added after telecine", + "description": "For fades that still comb after inverse telecine (the fade was applied to the telecined video). Rescales each field so both sit at the same point in the fade, without blurring. Leaves other frames practically unchanged", + "widget": "checkbox", + "visibleWhen": { + "method": [ + "ivtc" + ] + } + } + }, + "ivtcFadeMode": { + "type": "enum", + "default": "average", + "options": [ + "average", + "match" + ], + "sinceAppVersion": "1.7.0", + "ui": { + "label": "Fade fix mode", + "description": "Average puts both fields at their mean level; Match darkens the brighter field to the darker one, which also cleans the last frames of a fade to full black", + "widget": "dropdown", + "optionLabels": { + "average": "Average", + "match": "Match" + }, + "visibleWhen": { + "method": [ + "ivtc" + ], + "ivtcFixFades": true + } + } } }, "parameterPresets": { @@ -2095,7 +2134,8 @@ "title": "IVTC Settings", "parameters": [ "ivtcMode", - "ivtcCycle" + "ivtcCycle", + "ivtcFixFades" ], "expanded": true }, @@ -2107,7 +2147,8 @@ "ivtcBlockX", "ivtcBlockY", "ivtcDupthresh", - "ivtcScthresh" + "ivtcScthresh", + "ivtcFadeMode" ], "expanded": false, "advancedOnly": true diff --git a/app/assets/filters/core/face_restore.json b/app/assets/filters/core/face_restore.json new file mode 100644 index 00000000..9894f70d --- /dev/null +++ b/app/assets/filters/core/face_restore.json @@ -0,0 +1,196 @@ +{ + "$schema": "https://vapourbox.app/schemas/filter-v1.json", + "id": "face_restore", + "version": "1.0.0", + "sinceAppVersion": "1.7.0", + "name": "Face Restoration", + "description": "Restore detail in blurry, low-resolution faces with AI", + "longDescription": "Finds the faces in every frame and redraws them with a neural network trained on hundreds of thousands of portraits, so eyes, teeth and skin come back from a soft VHS or low-resolution picture. Only the face itself is replaced (not hair, neck or background), blended in at the video's own resolution, and every pixel outside the faces is left exactly as it was.\n\nIt runs after Crop & Resize, so upscaling first gives the network more to work with. A restored face is the network's best reconstruction, not the original: it can look smoother or slightly different from the person, which is what Strength is for. Faces smaller than the minimum size are left alone, since there is too little in them to restore.\n\nThe models are separate downloads. RestoreFormer++ is the default and free for any use. GFPGAN and CodeFormer are alternatives for non-commercial use only; CodeFormer adds a fidelity control. On a Mac they run on the graphics card and Neural Engine, on Windows on any DirectX 12 graphics card, and otherwise on the CPU, which is much slower.", + "category": "enhancement", + "icon": "face_retouching_natural", + "methods": [ + { + "id": "restore", + "name": "Face Restoration", + "description": "Detect each face, restore it with the chosen model, blend it back", + "function": "custom", + "parameters": [ + "model", + "strength", + "codeFormerFidelity", + "temporalSmoothing", + "detectionThreshold", + "minFaceSize", + "maxFaces" + ] + } + ], + "parameters": { + "enabled": { + "type": "boolean", + "default": false, + "ui": { + "hidden": true + } + }, + "model": { + "type": "enum", + "default": "restoreFormer", + "options": [ + "restoreFormer", + "gfpgan", + "codeFormer" + ], + "ui": { + "label": "Model", + "optionLabels": { + "restoreFormer": "RestoreFormer++", + "gfpgan": "GFPGAN v1.4 (download, non-commercial)", + "codeFormer": "CodeFormer (download, non-commercial)" + }, + "description": "The network that redraws the faces. RestoreFormer++ is natural-looking and free for any use. GFPGAN gives stronger, more stylised detail; CodeFormer is good on very damaged faces and has a fidelity control. Both are separate downloads for non-commercial use only", + "widget": "dropdown" + } + }, + "strength": { + "type": "number", + "default": 0.8, + "min": 0.0, + "max": 1.0, + "step": 0.05, + "ui": { + "label": "Strength", + "description": "How much of the restored face replaces the original. Lower keeps more of the real face and its texture, and flickers less", + "widget": "slider", + "precision": 2 + } + }, + "codeFormerFidelity": { + "type": "number", + "default": 0.7, + "min": 0.0, + "max": 1.0, + "step": 0.05, + "ui": { + "label": "Fidelity", + "description": "Lower trusts CodeFormer's learned faces more (cleaner, less like the person); higher stays closer to the picture. 0.5 to 0.7 suits most video", + "widget": "slider", + "precision": 2, + "visibleWhen": { + "model": [ + "codeFormer" + ] + } + } + }, + "temporalSmoothing": { + "type": "boolean", + "default": true, + "ui": { + "label": "Steady faces between frames", + "description": "Steady each face over the frames either side: its position is smoothed, one-frame gaps in detection are bridged, and the restored face is averaged with its neighbours wherever the picture is still, so detail the network invents doesn't shimmer. Turn off to restore every frame on its own", + "widget": "checkbox" + } + }, + "detectionThreshold": { + "type": "number", + "default": 0.8, + "min": 0.3, + "max": 0.99, + "step": 0.01, + "ui": { + "label": "Detection Confidence", + "description": "How sure the face detector must be. Lower finds more faces in poor pictures, and more things that are not faces", + "widget": "slider", + "precision": 2 + } + }, + "minFaceSize": { + "type": "integer", + "default": 32, + "min": 8, + "max": 1024, + "step": 1, + "ui": { + "label": "Minimum Face Size", + "description": "Faces smaller than this many pixels are left alone: there is too little in them to restore, and the network would invent a face rather than recover one", + "widget": "number" + } + }, + "maxFaces": { + "type": "integer", + "default": 8, + "min": 1, + "max": 64, + "step": 1, + "ui": { + "label": "Maximum Faces", + "description": "Restore at most this many faces per frame, the largest first. Each face costs as much time as the next, so a crowd is slow", + "widget": "number" + } + } + }, + "implementation": [ + { + "function": "face_restore.restore_faces", + "role": "finds, aligns, restores and blends back each face (ONNX Runtime)" + }, + { + "function": "RetinaFace (ResNet-50)", + "role": "finds the faces and their eyes, nose and mouth" + }, + { + "function": "RestoreFormer++", + "role": "restores each face", + "activeWhen": { + "model": "restoreFormer" + } + }, + { + "function": "GFPGAN v1.4", + "role": "restores each face", + "activeWhen": { + "model": "gfpgan" + } + }, + { + "function": "CodeFormer", + "role": "restores each face, as close to the picture as Fidelity asks", + "activeWhen": { + "model": "codeFormer" + } + }, + { + "function": "BiSeNet", + "role": "masks the blend to the face itself" + }, + { + "function": "core.std.MaskedMerge", + "role": "blends the faces in, leaving every other pixel untouched" + } + ], + "ui": { + "sections": [ + { + "title": "Restoration", + "parameters": [ + "model", + "strength", + "codeFormerFidelity", + "temporalSmoothing" + ], + "expanded": true + }, + { + "title": "Face detection", + "parameters": [ + "detectionThreshold", + "minFaceSize", + "maxFaces" + ], + "expanded": false, + "advancedOnly": true + } + ] + } +} diff --git a/app/assets/filters/core/frame_rate.json b/app/assets/filters/core/frame_rate.json index 3692440f..4a7f6c92 100644 --- a/app/assets/filters/core/frame_rate.json +++ b/app/assets/filters/core/frame_rate.json @@ -21,7 +21,7 @@ "id": "rife", "sinceAppVersion": "1.5.0", "name": "AI interpolation (RIFE)", - "description": "Builds the new frames with the RIFE 4.26 neural network on the GPU. Somewhat more accurate than motion interpolation, several times slower; needs Vulkan.", + "description": "Builds the new frames with the RIFE 4.26 neural network on the GPU. Somewhat more accurate than motion interpolation, several times slower; needs Vulkan. The network is a separate 11 MB download, offered when you choose it.", "function": "core.rife.RIFE", "parameters": [] }, diff --git a/app/assets/filters/core/noise_reduction.json b/app/assets/filters/core/noise_reduction.json index 2e7e0585..29b4a6b3 100644 --- a/app/assets/filters/core/noise_reduction.json +++ b/app/assets/filters/core/noise_reduction.json @@ -204,6 +204,21 @@ "knlmChroma" ], "advancedOnly": true + }, + { + "id": "nlmeans", + "sinceAppVersion": "1.7.0", + "name": "NLMeans (CPU)", + "description": "The same non-local means filter as KNLMeansCL, with the same settings and the same result, run on the processor instead of the graphics card. Works on every computer; slower than KNLMeansCL where a GPU is available", + "function": "core.vszip.NLMeans", + "parameters": [ + "knlmH", + "knlmD", + "knlmA", + "knlmS", + "knlmChroma" + ], + "advancedOnly": true } ], "parameters": { @@ -232,7 +247,8 @@ "mclean", "temporal_degrain2", "bm3d", - "knlmeans" + "knlmeans", + "nlmeans" ], "ui": { "hidden": true @@ -1184,7 +1200,8 @@ "precision": 1, "visibleWhen": { "method": [ - "knlmeans" + "knlmeans", + "nlmeans" ] } } @@ -1202,7 +1219,8 @@ "widget": "slider", "visibleWhen": { "method": [ - "knlmeans" + "knlmeans", + "nlmeans" ] } } @@ -1220,7 +1238,8 @@ "widget": "slider", "visibleWhen": { "method": [ - "knlmeans" + "knlmeans", + "nlmeans" ] } } @@ -1238,7 +1257,8 @@ "widget": "slider", "visibleWhen": { "method": [ - "knlmeans" + "knlmeans", + "nlmeans" ] } } @@ -1253,7 +1273,8 @@ "widget": "checkbox", "visibleWhen": { "method": [ - "knlmeans" + "knlmeans", + "nlmeans" ] } } @@ -1412,7 +1433,7 @@ "expanded": true }, { - "title": "KNLMeansCL", + "title": "Non-local means (KNLMeansCL / NLMeans)", "parameters": [ "knlmH", "knlmD", diff --git a/app/assets/filters/manifest.json b/app/assets/filters/manifest.json index 00ac3319..c2e13e76 100644 --- a/app/assets/filters/manifest.json +++ b/app/assets/filters/manifest.json @@ -20,5 +20,8 @@ "deflicker.json", "edge_repair.json", "ghost_removal.json", - "descale.json" + "descale.json", + "face_restore.json", + "colorize.json", + "audio_restoration.json" ] diff --git a/app/assets/whisper-addon.json b/app/assets/whisper-addon.json index 0efd468a..a650ae93 100644 --- a/app/assets/whisper-addon.json +++ b/app/assets/whisper-addon.json @@ -43,7 +43,7 @@ "small": { "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin", "filename": "ggml-small.bin", - "size": 466616853 + "size": 487601967 }, "medium": { "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin", @@ -53,7 +53,7 @@ "large-v3-turbo": { "url": "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin", "filename": "ggml-large-v3-turbo.bin", - "size": 1623277662 + "size": 1624555275 } } } diff --git a/app/dart_test.yaml b/app/dart_test.yaml index ed2888c8..a2d1e12f 100644 --- a/app/dart_test.yaml +++ b/app/dart_test.yaml @@ -4,6 +4,15 @@ # that run the worker end-to-end through vspipe | ffmpeg). They are slow, so the # per-push CI gate runs `flutter test --exclude-tags heavy` and the nightly # workflow runs `flutter test --tags heavy`. Run them locally with either filter. +# +# The `gpu` tag marks tests that run RIFE or AI upscaling (ncnn on Vulkan), so +# need a Vulkan device. The hosted CI runners have no GPU, so both workflows +# exclude it (`--exclude-tags heavy,gpu`, `--tags heavy --exclude-tags gpu`). +# A plain local `flutter test` includes them; they skip, saying why, where the +# probe finds no device, and fail instead under VAPOURBOX_EXPECT_VULKAN=1. Run +# them alone with `flutter test --tags gpu`. tags: heavy: # Full-encode integration tests — run nightly, excluded from the push gate. + gpu: + # Needs a Vulkan device (RIFE, AI upscaling) — excluded on the GPU-less CI runners. diff --git a/app/lib/models/ai_runtime_status.dart b/app/lib/models/ai_runtime_status.dart new file mode 100644 index 00000000..df55d885 --- /dev/null +++ b/app/lib/models/ai_runtime_status.dart @@ -0,0 +1,82 @@ +/// Whether the AI runtime add-on (ONNX Runtime) runs on this computer, and on +/// which execution provider, as reported by the worker's `--probe-ai-runtime`. +/// +/// The worker owns the decision and the wording (`worker/src/ai_runtime.rs`): +/// it checks the add-on is installed, imports ONNX Runtime the way a job does +/// (`worker/templates/ai_runtime.py`) and runs a tiny test model, directly and +/// through a VapourSynth frame. [message] is its explanation with a remedy, +/// shown verbatim. +library; + +class AiRuntimeStatus { + final bool available; + + /// The worker's reason id (`notInstalled`, `importFailed`, …), for logging. + final String? reason; + + /// What to tell the user, remedy included. Null when [available]. + final String? message; + + /// onnxruntime's version, when it loaded. + final String? version; + + /// Every execution provider the installed onnxruntime offers + /// (`CoreMLExecutionProvider`, `DmlExecutionProvider`, `CPUExecutionProvider`…). + final List providers; + + /// The provider the test model ran on. + final String? provider; + + /// Why the preferred GPU provider was not used, if it fell back to the CPU. + final String? fallback; + + const AiRuntimeStatus({ + required this.available, + this.reason, + this.message, + this.version, + this.providers = const [], + this.provider, + this.fallback, + }); + + /// The status when the probe itself could not be run or read. + const AiRuntimeStatus.unknown() + : available = false, + reason = 'probeFailed', + message = "VapourBox couldn't check whether the AI runtime add-on runs on " + 'this computer. Restart VapourBox to try again.', + version = null, + providers = const [], + provider = null, + fallback = null; + + factory AiRuntimeStatus.fromJson(Map json) { + final available = json['available'] == true; + final message = json['message']; + if (!available && message is! String) return const AiRuntimeStatus.unknown(); + return AiRuntimeStatus( + available: available, + reason: json['reason'] as String?, + message: available ? null : message as String, + version: json['version'] as String?, + providers: (json['providers'] as List? ?? const []).cast(), + provider: json['provider'] as String?, + fallback: json['fallback'] as String?, + ); + } + + /// Whether the test model ran on a GPU (or Neural Engine) provider rather + /// than the CPU. + bool get onGpu => available && provider != null && provider != 'CPUExecutionProvider'; + + /// The provider in words, for the UI ("Core ML", "DirectML", "CPU"). + static String providerLabel(String provider) => switch (provider) { + 'CoreMLExecutionProvider' => 'Core ML', + 'DmlExecutionProvider' => 'DirectML', + 'CUDAExecutionProvider' => 'CUDA', + 'TensorrtExecutionProvider' => 'TensorRT', + 'CPUExecutionProvider' => 'CPU', + _ => provider.replaceAll('ExecutionProvider', ''), + }; +} diff --git a/app/lib/models/anti_alias_parameters.dart b/app/lib/models/anti_alias_parameters.dart index 26977c7d..18513f83 100644 --- a/app/lib/models/anti_alias_parameters.dart +++ b/app/lib/models/anti_alias_parameters.dart @@ -11,7 +11,9 @@ enum AntiAliasMethod { @JsonValue('daa') daa('daa', 'daa'), @JsonValue('santiag') - santiag('santiag', 'santiag'); + santiag('santiag', 'santiag'), + @JsonValue('sangnom') + sangnom('sangnom', 'SangNom'); const AntiAliasMethod(this.value, this.displayName); final String value; @@ -37,17 +39,21 @@ class AntiAliasParameters { /// Horizontal strength for santiag (0 disables the horizontal pass). final int santiagStrh; - /// Interpolator for santiag. Only `nnedi3` is bundled, so the worker pins it - /// there regardless — havsfunc's other options would fail at script - /// evaluation. + /// Interpolator for santiag. The worker pins it to `nnedi3` regardless: + /// eedi2 is not bundled, and havsfunc's `sangnom` type is broken against + /// SangNom r45 (SangNom is its own method instead). final String santiagType; + /// SangNom's anti-aliasing strength, 0-128 in 8-bit units. + final int sangnomAa; + const AntiAliasParameters({ this.enabled = false, this.method = AntiAliasMethod.daa, this.santiagStrv = 1, this.santiagStrh = 1, this.santiagType = 'nnedi3', + this.sangnomAa = 48, }); AntiAliasParameters copyWith({ @@ -56,6 +62,7 @@ class AntiAliasParameters { int? santiagStrv, int? santiagStrh, String? santiagType, + int? sangnomAa, }) => AntiAliasParameters( enabled: enabled ?? this.enabled, @@ -63,6 +70,7 @@ class AntiAliasParameters { santiagStrv: santiagStrv ?? this.santiagStrv, santiagStrh: santiagStrh ?? this.santiagStrh, santiagType: santiagType ?? this.santiagType, + sangnomAa: sangnomAa ?? this.sangnomAa, ); /// Short summary for the pass list row. @@ -71,6 +79,7 @@ class AntiAliasParameters { if (method == AntiAliasMethod.santiag) { return 'santiag (H$santiagStrh / V$santiagStrv)'; } + if (method == AntiAliasMethod.sangnom) return 'SangNom ($sangnomAa)'; return method.displayName; } diff --git a/app/lib/models/audio_restoration_parameters.dart b/app/lib/models/audio_restoration_parameters.dart new file mode 100644 index 00000000..a7bd7db9 --- /dev/null +++ b/app/lib/models/audio_restoration_parameters.dart @@ -0,0 +1,219 @@ +import 'package:json_annotation/json_annotation.dart'; + +part 'audio_restoration_parameters.g.dart'; + +/// The add-on speech enhancement runs (`assets/addons/deepfilternet.json`). +const deepFilterNetAddonId = 'deepfilternet'; + +/// Whether an edit to [filterId]'s values switches speech enhancement on — +/// the moment the DeepFilterNet add-on is offered. +bool deepFilterChosen( + String filterId, + Map oldValues, + Map newValues, +) => + filterId == 'audio_restoration' && + newValues['deepFilter'] == true && + oldValues['deepFilter'] != true; + +/// What to do about the two channels of a stereo source. +enum ChannelFix { + @JsonValue('off') + off('Leave as is'), + @JsonValue('auto') + auto('Automatic (fix a silent channel)'), + @JsonValue('leftOnly') + leftOnly('Left channel only'), + @JsonValue('rightOnly') + rightOnly('Right channel only'), + @JsonValue('mix') + mix('Mix both channels'); + + const ChannelFix(this.displayName); + final String displayName; +} + +/// Which mains frequency to notch out. +enum HumFrequency { + @JsonValue('auto') + auto('Automatic'), + @JsonValue('50') + hz50('50 Hz'), + @JsonValue('60') + hz60('60 Hz'); + + const HumFrequency(this.displayName); + final String displayName; +} + +/// Audio restoration: the one pass that works on the soundtrack. +/// +/// Rendered by the worker before the encode (`worker/src/audio_restoration.rs`); +/// nothing here reaches the VapourSynth script, and the preview is picture +/// only. Every step is off by default, so switching the pass on changes +/// nothing until a step is chosen. +@JsonSerializable() +class AudioRestorationParameters { + final bool enabled; + + /// Which of the source's audio streams to restore (0 = the first). + final int streamIndex; + + final ChannelFix channelFix; + + /// Positive delays the sound, negative advances it (milliseconds). + final int offsetMs; + + final bool driftCorrection; + + /// The largest drift corrected, as a percentage of the length. + final double driftMaxPercent; + + final bool declip; + final bool declick; + + /// adeclick threshold: lower removes more. + final double declickThreshold; + + final bool humRemoval; + final HumFrequency humFrequency; + + final bool rumbleFilter; + final int rumbleCutoffHz; + + final bool denoise; + + /// afftdn noise reduction in dB. + final double denoiseStrength; + + /// DeepFilterNet speech enhancement (an add-on). + final bool deepFilter; + + /// DeepFilterNet's attenuation limit in dB (100 = no limit). + final int deepFilterAttenuation; + + final bool loudnessNormalize; + + /// Integrated loudness target, LUFS. + final double loudnessTarget; + + final bool limiter; + + /// Limiter ceiling, dBFS. + final double limiterCeilingDb; + + const AudioRestorationParameters({ + this.enabled = false, + this.streamIndex = 0, + this.channelFix = ChannelFix.off, + this.offsetMs = 0, + this.driftCorrection = false, + this.driftMaxPercent = 0.5, + this.declip = false, + this.declick = false, + this.declickThreshold = 2.0, + this.humRemoval = false, + this.humFrequency = HumFrequency.auto, + this.rumbleFilter = false, + this.rumbleCutoffHz = 40, + this.denoise = false, + this.denoiseStrength = 12.0, + this.deepFilter = false, + this.deepFilterAttenuation = 100, + this.loudnessNormalize = false, + this.loudnessTarget = -16.0, + this.limiter = false, + this.limiterCeilingDb = -1.0, + }); + + AudioRestorationParameters copyWith({ + bool? enabled, + int? streamIndex, + ChannelFix? channelFix, + int? offsetMs, + bool? driftCorrection, + double? driftMaxPercent, + bool? declip, + bool? declick, + double? declickThreshold, + bool? humRemoval, + HumFrequency? humFrequency, + bool? rumbleFilter, + int? rumbleCutoffHz, + bool? denoise, + double? denoiseStrength, + bool? deepFilter, + int? deepFilterAttenuation, + bool? loudnessNormalize, + double? loudnessTarget, + bool? limiter, + double? limiterCeilingDb, + }) { + return AudioRestorationParameters( + enabled: enabled ?? this.enabled, + streamIndex: streamIndex ?? this.streamIndex, + channelFix: channelFix ?? this.channelFix, + offsetMs: offsetMs ?? this.offsetMs, + driftCorrection: driftCorrection ?? this.driftCorrection, + driftMaxPercent: driftMaxPercent ?? this.driftMaxPercent, + declip: declip ?? this.declip, + declick: declick ?? this.declick, + declickThreshold: declickThreshold ?? this.declickThreshold, + humRemoval: humRemoval ?? this.humRemoval, + humFrequency: humFrequency ?? this.humFrequency, + rumbleFilter: rumbleFilter ?? this.rumbleFilter, + rumbleCutoffHz: rumbleCutoffHz ?? this.rumbleCutoffHz, + denoise: denoise ?? this.denoise, + denoiseStrength: denoiseStrength ?? this.denoiseStrength, + deepFilter: deepFilter ?? this.deepFilter, + deepFilterAttenuation: + deepFilterAttenuation ?? this.deepFilterAttenuation, + loudnessNormalize: loudnessNormalize ?? this.loudnessNormalize, + loudnessTarget: loudnessTarget ?? this.loudnessTarget, + limiter: limiter ?? this.limiter, + limiterCeilingDb: limiterCeilingDb ?? this.limiterCeilingDb, + ); + } + + /// On, with at least one step chosen (twin of the worker's `has_effect`). + bool get hasEffect => + enabled && + (channelFix != ChannelFix.off || + offsetMs != 0 || + driftCorrection || + declip || + declick || + humRemoval || + rumbleFilter || + denoise || + deepFilter || + loudnessNormalize || + limiter); + + /// The steps switched on, in the order they run, for the pass row. + List get activeSteps => [ + if (channelFix != ChannelFix.off) 'Channels', + if (declip) 'Declip', + if (declick) 'Declick', + if (rumbleFilter) 'Rumble', + if (humRemoval) 'Hum', + if (denoise) 'Denoise', + if (offsetMs != 0) 'Sync ${offsetMs > 0 ? '+' : ''}$offsetMs ms', + if (driftCorrection) 'Drift', + if (deepFilter) 'DeepFilterNet', + if (loudnessNormalize) 'Loudness', + if (limiter) 'Limiter', + ]; + + String get summary { + if (!enabled) return 'Off'; + final steps = activeSteps; + if (steps.isEmpty) return 'Nothing selected'; + if (steps.length <= 3) return steps.join(', '); + return '${steps.take(2).join(', ')} +${steps.length - 2}'; + } + + factory AudioRestorationParameters.fromJson(Map json) => + _$AudioRestorationParametersFromJson(json); + Map toJson() => _$AudioRestorationParametersToJson(this); +} diff --git a/app/lib/models/colorize_parameters.dart b/app/lib/models/colorize_parameters.dart new file mode 100644 index 00000000..3b45eb78 --- /dev/null +++ b/app/lib/models/colorize_parameters.dart @@ -0,0 +1,99 @@ +import 'package:json_annotation/json_annotation.dart'; + +part 'colorize_parameters.g.dart'; + +/// Which network predicts the colour. Each lives in its own add-on; the wire +/// names match the worker's `ColorizeModel` +/// (`worker/src/models/colorize_parameters.rs`) and the schema's method ids. +enum ColorizeModel { + /// DDColor-T (Apache-2.0): vivid, predicted per frame. + @JsonValue('ddcolor') + ddcolor('DDColor', 'colorize-ddcolor'), + + /// DeOldify's Video model (MIT): muted, steady from frame to frame. + @JsonValue('deoldify') + deoldify('DeOldify Video', 'colorize-deoldify'); + + const ColorizeModel(this.displayName, this.addonId); + final String displayName; + + /// The add-on holding this model's file. + final String addonId; + + /// The wire value, which is also the schema's method id. + String get value => switch (this) { + ColorizeModel.ddcolor => 'ddcolor', + ColorizeModel.deoldify => 'deoldify', + }; + + static ColorizeModel fromValue(String? value) => + ColorizeModel.values.firstWhere((m) => m.value == value, orElse: () => ColorizeModel.ddcolor); +} + +/// What the add-on prompt says the download is for. +const colorizeAddonReason = 'Colourisation needs its AI model.'; + +/// The add-on the Colourisation panel's current [values] need, or null for +/// any other panel, or while the pass is off. The worker refuses a job +/// without it (`ColorizeParameters::required_addons`); the manifests' +/// `requires` bring in the AI runtime. +String? colorizeAddonSelectedBy(String filterId, bool enabled, Map values) => + filterId == 'colorize' && enabled ? ColorizeModel.fromValue(values['method'] as String?).addonId : null; + +/// Colourisation: colour black-and-white footage with a neural network, +/// keeping the source's luma exactly and replacing only its chroma. +@JsonSerializable() +class ColorizeParameters { + final bool enabled; + final ColorizeModel model; + + /// DDColor's input size in pixels (a square; a multiple of 32). + final int ddcolorRenderSize; + + /// DeOldify's render factor: the model sees a square of 16x this. + final int deoldifyRenderFactor; + + /// How much of the predicted colour is applied: 0 grey, 1 as predicted, + /// above 1 more saturated. + final double strength; + + /// Frames either side each frame's colour is averaged with, where the + /// picture did not move, so it does not flicker. 0 colours every frame on + /// its own. + final int temporalRadius; + + const ColorizeParameters({ + this.enabled = false, + this.model = ColorizeModel.ddcolor, + this.ddcolorRenderSize = 512, + this.deoldifyRenderFactor = 21, + this.strength = 1.0, + this.temporalRadius = 2, + }); + + ColorizeParameters copyWith({ + bool? enabled, + ColorizeModel? model, + int? ddcolorRenderSize, + int? deoldifyRenderFactor, + double? strength, + int? temporalRadius, + }) { + return ColorizeParameters( + enabled: enabled ?? this.enabled, + model: model ?? this.model, + ddcolorRenderSize: ddcolorRenderSize ?? this.ddcolorRenderSize, + deoldifyRenderFactor: deoldifyRenderFactor ?? this.deoldifyRenderFactor, + strength: strength ?? this.strength, + temporalRadius: temporalRadius ?? this.temporalRadius, + ); + } + + /// The add-on this configuration needs. + String get addonId => model.addonId; + + String get summary => enabled ? model.displayName : 'Off'; + + factory ColorizeParameters.fromJson(Map json) => _$ColorizeParametersFromJson(json); + Map toJson() => _$ColorizeParametersToJson(this); +} diff --git a/app/lib/models/crop_detect_report.dart b/app/lib/models/crop_detect_report.dart new file mode 100644 index 00000000..e2914dd8 --- /dev/null +++ b/app/lib/models/crop_detect_report.dart @@ -0,0 +1,231 @@ +/// What `vapourbox-worker --detect-crop` found at the edges of a video: black +/// borders and analog blanking, letterbox / pillarbox bars, and the +/// head-switching band at the bottom of a tape capture, for Crop & Resize +/// (`worker/src/crop_detect.rs`, app 1.7.0). +/// +/// Advisory: the panel shows it, and only an Apply puts it into the controls. +library; + +/// Four sides of a crop (or of Edge Repair's widths). +class CropSides { + final int left; + final int right; + final int top; + final int bottom; + + const CropSides({this.left = 0, this.right = 0, this.top = 0, this.bottom = 0}); + + static CropSides? fromJson(Object? json) { + if (json is! Map) return null; + int v(String k) => (json[k] as num?)?.toInt() ?? 0; + return CropSides(left: v('left'), right: v('right'), top: v('top'), bottom: v('bottom')); + } + + bool get isZero => left == 0 && right == 0 && top == 0 && bottom == 0; + + @override + String toString() => 'L $left, R $right, T $top, B $bottom'; + + @override + bool operator ==(Object other) => + other is CropSides && + other.left == left && + other.right == right && + other.top == top && + other.bottom == bottom; + + @override + int get hashCode => Object.hash(left, right, top, bottom); +} + +/// What one edge was found to be. +enum CropEdgeKind { + none, + + /// A narrow black border or analog blanking: cropped. + blanking, + + /// A letterbox / pillarbox bar: the picture's framing, cropped only on + /// request. + bar, + + /// The torn band at the bottom of a tape capture. + headSwitching; + + static CropEdgeKind parse(Object? value) => + CropEdgeKind.values.firstWhere((k) => k.name == value, orElse: () => CropEdgeKind.none); +} + +/// One edge as measured, in the orientation the Crop pass sees. +class CropEdge { + final CropEdgeKind kind; + + /// Dark, flat lines from the frame edge. + final int blank; + + /// The dim ramp beside them. + final int ramp; + + /// Head-switching lines (bottom of the source). + final int band; + + /// The width of a bar. + final int bar; + + /// Fraction of usable frames that agree with this edge. + final double confidence; + + const CropEdge({ + this.kind = CropEdgeKind.none, + this.blank = 0, + this.ramp = 0, + this.band = 0, + this.bar = 0, + this.confidence = 1, + }); + + factory CropEdge.fromJson(Object? json) { + if (json is! Map) return const CropEdge(); + int v(String k) => (json[k] as num?)?.toInt() ?? 0; + return CropEdge( + kind: CropEdgeKind.parse(json['kind']), + blank: v('blank'), + ramp: v('ramp'), + band: v('band'), + bar: v('bar'), + confidence: (json['confidence'] as num?)?.toDouble() ?? 1, + ); + } + + /// "12 px of blanking and a 3-px soft edge", for the panel. + String describe({required bool horizontal}) { + final unit = horizontal ? 'px' : 'lines'; + switch (kind) { + case CropEdgeKind.none: + return 'nothing to crop'; + case CropEdgeKind.blanking: + final soft = ramp > 0 ? ' and a $ramp-${horizontal ? 'px' : 'line'} soft edge' : ''; + return '$blank $unit of black / blanking$soft'; + case CropEdgeKind.bar: + return '$bar-${horizontal ? 'px' : 'line'} ${horizontal ? 'pillarbox' : 'letterbox'} bar'; + case CropEdgeKind.headSwitching: + final barNote = bar > 0 ? ', under a $bar-line letterbox bar' : ''; + return '$band lines of head-switching noise$barNote'; + } + } +} + +class CropDetectReport { + /// Whether anything worth cropping was found. + final bool found; + + /// The recommended crop: blanking, soft edges and the head-switching band, + /// not bars. Rounded to steps the Crop pass accepts. + final CropSides crop; + + /// The same with the letterbox / pillarbox bars included; null without bars. + final CropSides? cropWithBars; + + /// Edge Repair's widths for repairing the head-switching band instead of + /// cropping it, and the crop that goes with that; null without a band. + final CropSides? edgeRepair; + final CropSides? cropWithRepair; + + final CropEdge left; + final CropEdge right; + final CropEdge top; + final CropEdge bottom; + + /// The lowest confidence among the edges that found something. + final double confidence; + + final int usableFrames; + final int sampledFrames; + + /// Why nothing was found, or why the analysis failed. + final String? message; + final String? error; + + const CropDetectReport({ + required this.found, + this.crop = const CropSides(), + this.cropWithBars, + this.edgeRepair, + this.cropWithRepair, + this.left = const CropEdge(), + this.right = const CropEdge(), + this.top = const CropEdge(), + this.bottom = const CropEdge(), + this.confidence = 1, + this.usableFrames = 0, + this.sampledFrames = 0, + this.message, + this.error, + }); + + const CropDetectReport.failed(String this.error) + : found = false, + crop = const CropSides(), + cropWithBars = null, + edgeRepair = null, + cropWithRepair = null, + left = const CropEdge(), + right = const CropEdge(), + top = const CropEdge(), + bottom = const CropEdge(), + confidence = 0, + usableFrames = 0, + sampledFrames = 0, + message = null; + + factory CropDetectReport.fromJson(Map json) { + if (json['error'] is String) return CropDetectReport.failed(json['error'] as String); + final edges = json['edges'] is Map ? json['edges'] as Map : const {}; + return CropDetectReport( + found: json['found'] == true, + crop: CropSides.fromJson(json['crop']) ?? const CropSides(), + cropWithBars: CropSides.fromJson(json['cropWithBars']), + edgeRepair: CropSides.fromJson(json['edgeRepair']), + cropWithRepair: CropSides.fromJson(json['cropWithRepair']), + left: CropEdge.fromJson(edges['left']), + right: CropEdge.fromJson(edges['right']), + top: CropEdge.fromJson(edges['top']), + bottom: CropEdge.fromJson(edges['bottom']), + confidence: (json['confidence'] as num?)?.toDouble() ?? 1, + usableFrames: (json['usableFrames'] as num?)?.toInt() ?? 0, + sampledFrames: (json['frames'] as List?)?.length ?? 0, + message: json['message'] as String?, + ); + } + + /// Below this the panel says the edges were hard to judge: fewer than one + /// sampled frame in four shows picture right up to an edge it found. The + /// rest are not contradicting it — their picture is dark there too — but + /// the evidence is thin (a dark film, or dark scenery beside a letterbox). + static const lowConfidence = 0.25; + + bool get isLowConfidence => confidence < lowConfidence; + + /// Each side that found something, described. + List<(String, String)> get findings => [ + for (final (name, edge, horizontal) in [ + ('Left', left, true), + ('Right', right, true), + ('Top', top, false), + ('Bottom', bottom, false), + ]) + if (edge.kind != CropEdgeKind.none) (name, edge.describe(horizontal: horizontal)), + ]; +} + +/// Which of the report's suggestions to apply. +enum CropApply { + /// The recommended crop. + crop, + + /// The crop with the letterbox / pillarbox bars too. + cropWithBars, + + /// Repair the head-switching band with Edge Repair; crop the rest. + repairBand, +} diff --git a/app/lib/models/crop_resize_parameters.dart b/app/lib/models/crop_resize_parameters.dart index e4cfb250..3a97bd8e 100644 --- a/app/lib/models/crop_resize_parameters.dart +++ b/app/lib/models/crop_resize_parameters.dart @@ -42,6 +42,19 @@ enum UpscaleMethod { /// non-commercial use only, downloaded when it is chosen. @JsonValue('animeJanai') animeJanai, + /// Real-ESRGAN's general model, for real-world footage, on the GPU (needs + /// Vulkan). 4x only: a 2x upscale resamples its result down. Its model is + /// the `upscale-general` add-on. + @JsonValue('realEsrganGeneral') + realEsrganGeneral, + /// jcj83429's 2xLiveActionV1 SPAN model, for live action, on the GPU. 2x + /// only: 4x runs it twice. Also in the `upscale-general` add-on. + @JsonValue('liveAction') + liveAction, + /// Real-ESRGAN x4plus / x2plus, the large, slow models, on the GPU. The + /// `upscale-esrgan-plus` add-on. + @JsonValue('realEsrganPlus') + realEsrganPlus, } /// What to do with the pixel aspect ratio (SAR) on output. @@ -75,6 +88,25 @@ enum BorderColor { custom, } +/// Whether an SD (BT.601) source's colour is converted to BT.709 (app 1.7.0). +/// +/// The worker makes the decision (`VideoJob::converts_to_bt709`): it converts +/// the samples in the script and declares BT.709 to the encoder to match. +enum Bt709Conversion { + /// Convert a BT.601 source when the output is HD-sized (720 lines, or 1280 + /// columns). + @JsonValue('auto') + auto, + + /// Convert a BT.601 source whatever the output size. + @JsonValue('on') + on, + + /// Never convert. + @JsonValue('off') + off, +} + /// Crop/resize preset options. enum CropResizePreset { @JsonValue('off') @@ -196,6 +228,13 @@ class CropResizeParameters { /// AnimeJaNai's Sharp variant of the HD model. final bool animejanaiSharp; + /// Real-ESRGAN General's denoise variant (realesr-general-wdn-x4v3). + final bool generalDenoise; + + /// How much of an AI upscale to keep, 0.0-1.0: the rest is a Spline36 + /// upscale of the same frame. 1.0 (the default) is the network alone. + final double aiStrength; + // --- nnedi3 controls for the edge-directed upscale paths --- // With EEDI3 selected these shape the nnedi3 sclip that guides it. @@ -231,6 +270,13 @@ class CropResizeParameters { /// Maximum search distance (EEDI3 `mdis`). final int? upscaleMdis; + // --- Colour --- + + /// BT.601 -> BT.709 conversion. Missing from a preset saved before 1.7.0, + /// which therefore gets [Bt709Conversion.auto], as the worker does. + @JsonKey(defaultValue: Bt709Conversion.auto, unknownEnumValue: Bt709Conversion.auto) + final Bt709Conversion convertToBt709; + const CropResizeParameters({ this.enabled = false, this.preset = CropResizePreset.off, @@ -267,6 +313,8 @@ class CropResizeParameters { this.cuganNoise = 'conservative', this.animejanaiModel = 'compact', this.animejanaiSharp = false, + this.generalDenoise = false, + this.aiStrength = 1.0, this.upscaleNsize, this.upscaleNeurons, this.upscaleQual, @@ -277,6 +325,7 @@ class CropResizeParameters { this.upscaleGamma, this.upscaleNrad, this.upscaleMdis, + this.convertToBt709 = Bt709Conversion.auto, }); /// Create parameters from a preset. @@ -365,6 +414,8 @@ class CropResizeParameters { String? cuganNoise, String? animejanaiModel, bool? animejanaiSharp, + bool? generalDenoise, + double? aiStrength, int? upscaleNsize, int? upscaleNeurons, int? upscaleQual, @@ -375,6 +426,7 @@ class CropResizeParameters { double? upscaleGamma, int? upscaleNrad, int? upscaleMdis, + Bt709Conversion? convertToBt709, }) { return CropResizeParameters( enabled: enabled ?? this.enabled, @@ -407,6 +459,8 @@ class CropResizeParameters { cuganNoise: cuganNoise ?? this.cuganNoise, animejanaiModel: animejanaiModel ?? this.animejanaiModel, animejanaiSharp: animejanaiSharp ?? this.animejanaiSharp, + generalDenoise: generalDenoise ?? this.generalDenoise, + aiStrength: aiStrength ?? this.aiStrength, upscaleNsize: upscaleNsize ?? this.upscaleNsize, upscaleNeurons: upscaleNeurons ?? this.upscaleNeurons, upscaleQual: upscaleQual ?? this.upscaleQual, @@ -417,6 +471,7 @@ class CropResizeParameters { upscaleGamma: upscaleGamma ?? this.upscaleGamma, upscaleNrad: upscaleNrad ?? this.upscaleNrad, upscaleMdis: upscaleMdis ?? this.upscaleMdis, + convertToBt709: convertToBt709 ?? this.convertToBt709, ); } diff --git a/app/lib/models/deblock_parameters.dart b/app/lib/models/deblock_parameters.dart index fb490f3b..b7a2e387 100644 --- a/app/lib/models/deblock_parameters.dart +++ b/app/lib/models/deblock_parameters.dart @@ -10,7 +10,9 @@ enum DeblockMethod { @JsonValue('Deblock') deblock('Deblock', 'Deblock'), @JsonValue('DCTFilter') - dctFilter('DCTFilter', 'DCTFilter'); + dctFilter('DCTFilter', 'DCTFilter'), + @JsonValue('mpeg2stinx') + mpeg2stinx('mpeg2stinx', 'mpeg2stinx'); const DeblockMethod(this.value, this.displayName); final String value; @@ -25,6 +27,9 @@ enum DeblockMethod { case DeblockMethod.dctFilter: return 'Attenuates high DCT frequency bands — targets ringing and ' 'mosquito noise rather than block edges'; + case DeblockMethod.mpeg2stinx: + return 'Comb-like noise in film DVDs encoded as interlaced MPEG-2 ' + '(vs-jetpack\'s mpeg2stinx)'; } } } diff --git a/app/lib/models/dehalo_parameters.dart b/app/lib/models/dehalo_parameters.dart index cdeb0af9..f01d9677 100644 --- a/app/lib/models/dehalo_parameters.dart +++ b/app/lib/models/dehalo_parameters.dart @@ -20,7 +20,9 @@ enum DehaloMethod { @JsonValue('Vinverse2') vinverse2('Vinverse2', 'Vinverse 2 (de-ghost)'), @JsonValue('HQDeringmod') - hqDeringmod('HQDeringmod', 'HQDeringmod (de-ring)'); + hqDeringmod('HQDeringmod', 'HQDeringmod (de-ring)'), + @JsonValue('vine') + vine('vine', 'vine (NLMeans)'); const DehaloMethod(this.value, this.displayName); final String value; @@ -45,6 +47,9 @@ enum DehaloMethod { case DehaloMethod.hqDeringmod: return 'Masked ring removal — treats the overshoot beside an edge ' 'while protecting the edge itself'; + case DehaloMethod.vine: + return 'Non-local-means dehalo that keeps the edge detail the halo ' + 'rides on (vs-jetpack\'s vine_dehalo)'; } } } @@ -165,6 +170,17 @@ class DehaloParameters { /// `thr / 4`, which is usually what you want. final double? deringDarkthr; + // --- vine parameters --- + + /// NLMeans strength (vs-jetpack's `strength`). + final double vineStrength; + + /// How much of the supersampled pass is blended in (`sharp`, 0-1]. + final double vineSharp; + + /// Gaussian sigma of the low/high frequency split. + final double vineSigma; + const DehaloParameters({ this.enabled = false, this.method = DehaloMethod.dehaloAlpha, @@ -197,6 +213,9 @@ class DehaloParameters { this.deringMthr, this.deringThr, this.deringDarkthr, + this.vineStrength = 16.0, + this.vineSharp = 0.5, + this.vineSigma = 1.0, }); DehaloParameters copyWith({ @@ -231,6 +250,9 @@ class DehaloParameters { int? deringMthr, double? deringThr, double? deringDarkthr, + double? vineStrength, + double? vineSharp, + double? vineSigma, }) { return DehaloParameters( enabled: enabled ?? this.enabled, @@ -264,6 +286,9 @@ class DehaloParameters { deringMthr: deringMthr ?? this.deringMthr, deringThr: deringThr ?? this.deringThr, deringDarkthr: deringDarkthr ?? this.deringDarkthr, + vineStrength: vineStrength ?? this.vineStrength, + vineSharp: vineSharp ?? this.vineSharp, + vineSigma: vineSigma ?? this.vineSigma, ); } diff --git a/app/lib/models/face_restore_parameters.dart b/app/lib/models/face_restore_parameters.dart new file mode 100644 index 00000000..72e37ee3 --- /dev/null +++ b/app/lib/models/face_restore_parameters.dart @@ -0,0 +1,131 @@ +import 'package:json_annotation/json_annotation.dart'; + +part 'face_restore_parameters.g.dart'; + +/// Which network restores the faces. Each lives in an add-on (see +/// [FaceRestoreModel.addonIds]); the wire names match the worker's +/// `FaceRestoreModel` (`worker/src/models/face_restore_parameters.rs`). +enum FaceRestoreModel { + /// RestoreFormer++ (Apache-2.0), in the base `face-restoration` add-on. + @JsonValue('restoreFormer') + restoreFormer('RestoreFormer++', 'face-restoration'), + + /// GFPGAN v1.4: non-commercial (NVIDIA's licence on its StyleGAN2 code). + @JsonValue('gfpgan') + gfpgan('GFPGAN v1.4', 'face-gfpgan'), + + /// CodeFormer: non-commercial (S-Lab License 1.0), with a fidelity control. + @JsonValue('codeFormer') + codeFormer('CodeFormer', 'face-codeformer'); + + const FaceRestoreModel(this.displayName, this.addonId); + final String displayName; + + /// The add-on holding this model's file. + final String addonId; + + /// The wire value (`restoreFormer`, `gfpgan`, `codeFormer`). + String get value => switch (this) { + FaceRestoreModel.restoreFormer => 'restoreFormer', + FaceRestoreModel.gfpgan => 'gfpgan', + FaceRestoreModel.codeFormer => 'codeFormer', + }; + + static FaceRestoreModel fromValue(String? value) => FaceRestoreModel.values + .firstWhere((m) => m.value == value, orElse: () => FaceRestoreModel.restoreFormer); +} + +/// The base add-on: the face detector, the face parser and RestoreFormer++. +const faceRestorationAddonId = 'face-restoration'; + +/// The add-ons a Face Restoration job with [model] needs: the base always +/// (detector and parser), plus the model's own. The worker refuses a job +/// without them (`FaceRestoreParameters::required_addons`); the manifests' +/// `requires` bring in the AI runtime. +List faceRestoreAddonIds(FaceRestoreModel model) => [ + faceRestorationAddonId, + if (model.addonId != faceRestorationAddonId) model.addonId, + ]; + +/// What the add-on prompt says the download is for. +const faceRestoreAddonReason = 'Face restoration needs its AI models.'; + +/// The add-ons the Face Restoration panel's current [values] need, or null +/// for any other panel, or while the pass is off. +List? faceRestoreAddonsSelectedBy(String filterId, bool enabled, Map values) => + filterId == 'face_restore' && enabled + ? faceRestoreAddonIds(FaceRestoreModel.fromValue(values['model'] as String?)) + : null; + +/// Face Restoration: find the faces in each frame and restore them with a +/// neural network, blended back at the clip's own resolution. +@JsonSerializable() +class FaceRestoreParameters { + final bool enabled; + final FaceRestoreModel model; + + /// How much of the restored face replaces the original (0-1). + final double strength; + + /// CodeFormer's fidelity weight: 0 trusts its learned faces, 1 stays + /// closest to the picture. Ignored by the other models. + final double codeFormerFidelity; + + /// The detector score (0-1) a face needs. + final double detectionThreshold; + + /// Faces smaller than this (pixels, the larger side of the face's box) are + /// left alone. + final int minFaceSize; + + /// At most this many faces (the largest) are restored per frame. + final int maxFaces; + + /// Steady each face over the frames either side: smooth its landmarks, + /// bridge one-frame detection gaps, and average the restored face with its + /// neighbours' where the picture is still, so it doesn't jitter, blink or + /// shimmer. + final bool temporalSmoothing; + + const FaceRestoreParameters({ + this.enabled = false, + this.model = FaceRestoreModel.restoreFormer, + this.strength = 0.8, + this.codeFormerFidelity = 0.7, + this.detectionThreshold = 0.8, + this.minFaceSize = 32, + this.maxFaces = 8, + this.temporalSmoothing = true, + }); + + FaceRestoreParameters copyWith({ + bool? enabled, + FaceRestoreModel? model, + double? strength, + double? codeFormerFidelity, + double? detectionThreshold, + int? minFaceSize, + int? maxFaces, + bool? temporalSmoothing, + }) { + return FaceRestoreParameters( + enabled: enabled ?? this.enabled, + model: model ?? this.model, + strength: strength ?? this.strength, + codeFormerFidelity: codeFormerFidelity ?? this.codeFormerFidelity, + detectionThreshold: detectionThreshold ?? this.detectionThreshold, + minFaceSize: minFaceSize ?? this.minFaceSize, + maxFaces: maxFaces ?? this.maxFaces, + temporalSmoothing: temporalSmoothing ?? this.temporalSmoothing, + ); + } + + /// The add-ons this configuration needs. + List get addonIds => faceRestoreAddonIds(model); + + String get summary => enabled ? model.displayName : 'Off'; + + factory FaceRestoreParameters.fromJson(Map json) => + _$FaceRestoreParametersFromJson(json); + Map toJson() => _$FaceRestoreParametersToJson(this); +} diff --git a/app/lib/models/gpu_plugin_status.dart b/app/lib/models/gpu_plugin_status.dart index 0e7e8a90..daf26725 100644 --- a/app/lib/models/gpu_plugin_status.dart +++ b/app/lib/models/gpu_plugin_status.dart @@ -2,11 +2,15 @@ /// worker's `--probe-rife` / `--probe-upscale`. /// /// The worker owns the decision and the wording (`worker/src/gpu_plugin.rs`): -/// it checks the platform and the bundled files, then runs the plugin once and -/// classifies how it failed — no Vulkan device, a missing system library, an -/// older deps bundle. [GpuPluginStatus.message] is that explanation with its -/// remedy, shown to the user verbatim; the worker fails such a job with the -/// same text. +/// it checks the platform and the bundled plugin files, then runs the plugin +/// once on a stand-in network of its own and classifies how it failed — no +/// Vulkan device, a missing system library, an older deps bundle, a wrong +/// answer. [GpuPluginStatus.message] is that explanation with its remedy, +/// shown to the user verbatim; the worker fails such a job with the same text. +/// +/// The models the plugins run are add-ons, checked separately +/// ([modelAddonSelectedBy]): the probe needs none, so "this computer can't run +/// it" is known before anything is downloaded. library; /// The two features that run a network through ncnn on the GPU. @@ -88,6 +92,47 @@ bool animeJanaiSelectedBy(String filterId, Map values) => values['useIntegerUpscale'] == true && values['upscaleMethod'] == 'animeJanai'; +/// Crop & Resize's AI upscale methods, all run by VSNCNN on the GPU. +const aiUpscaleMethods = { + 'realEsrgan', + 'realCugan', + 'animeJanai', + 'realEsrganGeneral', + 'liveAction', + 'realEsrganPlus', +}; + +/// The add-on holding each AI upscale method's models — every one of them: +/// no model is in the deps bundle. The worker's `UpscaleMethod::addon` (plus +/// AnimeJaNai's hand-written add-on). +const upscaleMethodAddons = { + 'realEsrgan': 'upscale-anime', + 'realCugan': 'upscale-anime', + 'animeJanai': 'animejanai', + 'realEsrganGeneral': 'upscale-general', + 'liveAction': 'upscale-general', + 'realEsrganPlus': 'upscale-esrgan-plus', +}; + +/// The add-on a panel's current values need for their AI upscale, if any. +String? upscaleAddonSelectedBy(String filterId, Map values) => + filterId == 'crop_resize' && values['useIntegerUpscale'] == true + ? upscaleMethodAddons[values['upscaleMethod']] + : null; + +/// The add-on holding RIFE's model (the worker's `gpu_plugin::RIFE_ADDON`). +const rifeAddonId = 'frame-rate-rife'; + +/// The add-on a Frame Rate panel's values need: RIFE's model, when RIFE is +/// the method. +String? frameRateAddonSelectedBy(String filterId, Map values) => + filterId == 'frame_rate' && values['method'] == 'rife' ? rifeAddonId : null; + +/// The model add-on a panel's current values need for a GPU feature (an AI +/// upscaler or RIFE), if any. +String? modelAddonSelectedBy(String filterId, Map values) => + upscaleAddonSelectedBy(filterId, values) ?? frameRateAddonSelectedBy(filterId, values); + /// Which GPU feature, if any, a panel's current values select. GpuPlugin? gpuPluginSelectedBy(String filterId, Map values) { switch (filterId) { @@ -95,8 +140,7 @@ GpuPlugin? gpuPluginSelectedBy(String filterId, Map values) { return values['method'] == 'rife' ? GpuPlugin.rife : null; case 'crop_resize': final method = values['upscaleMethod']; - return values['useIntegerUpscale'] == true && - (method == 'realEsrgan' || method == 'realCugan' || method == 'animeJanai') + return values['useIntegerUpscale'] == true && aiUpscaleMethods.contains(method) ? GpuPlugin.upscale : null; } diff --git a/app/lib/models/noise_halo_report.dart b/app/lib/models/noise_halo_report.dart new file mode 100644 index 00000000..65a500e0 --- /dev/null +++ b/app/lib/models/noise_halo_report.dart @@ -0,0 +1,61 @@ +/// What `vapourbox-worker --detect-noise-halo` measured in a source: how noisy +/// it is and how strongly its edges ring (`worker/src/noise_halo.rs`, app +/// 1.7.0). Measured in the background when a video is loaded; the pass list +/// uses it only to badge Noise Reduction or Dehalo as recommended +/// ([relevanceFor]). Nothing is ever changed because of it. +/// +/// The thresholds are decided by the worker, once: [noiseHigh] and [haloHigh] +/// are its verdicts, and the app never re-derives them from the numbers. +/// Calibration: docs/reports/2026-11-noise-halo-scores/. +library; + +class NoiseHaloReport { + /// Noise sigma in 8-bit levels (motion-compensated frame difference), null + /// when it could not be measured. + final double? noise; + + /// Halo score: ringing beside step edges, % of the step. Null with too few + /// edges to judge. + final double? halo; + + final int haloEdges; + final int usablePairs; + + /// Whether each is high enough to recommend its pass. + final bool noiseHigh; + final bool haloHigh; + + /// Why the measurement failed; every verdict is false then. + final String? error; + + const NoiseHaloReport({ + this.noise, + this.halo, + this.haloEdges = 0, + this.usablePairs = 0, + this.noiseHigh = false, + this.haloHigh = false, + this.error, + }); + + const NoiseHaloReport.failed(String this.error) + : noise = null, + halo = null, + haloEdges = 0, + usablePairs = 0, + noiseHigh = false, + haloHigh = false; + + factory NoiseHaloReport.fromJson(Map json) { + final error = json['error']; + if (error != null) return NoiseHaloReport.failed('$error'); + return NoiseHaloReport( + noise: (json['noise'] as num?)?.toDouble(), + halo: (json['halo'] as num?)?.toDouble(), + haloEdges: (json['haloEdges'] as num?)?.toInt() ?? 0, + usablePairs: (json['usablePairs'] as num?)?.toInt() ?? 0, + noiseHigh: json['noiseHigh'] == true, + haloHigh: json['haloHigh'] == true, + ); + } +} diff --git a/app/lib/models/noise_reduction_parameters.dart b/app/lib/models/noise_reduction_parameters.dart index 13d077ac..8024e851 100644 --- a/app/lib/models/noise_reduction_parameters.dart +++ b/app/lib/models/noise_reduction_parameters.dart @@ -38,7 +38,9 @@ enum NoiseReductionMethod { @JsonValue('bm3d') bm3d('BM3D'), @JsonValue('knlMeans') - knlMeans('KNLMeansCL'); + knlMeans('KNLMeansCL'), + @JsonValue('nlMeans') + nlMeans('NLMeans (CPU)'); const NoiseReductionMethod(this.displayName); final String displayName; diff --git a/app/lib/models/parameter_converter.dart b/app/lib/models/parameter_converter.dart index 936375d1..e98f4328 100644 --- a/app/lib/models/parameter_converter.dart +++ b/app/lib/models/parameter_converter.dart @@ -1,8 +1,11 @@ import 'chroma_denoise_parameters.dart'; import 'anti_alias_parameters.dart'; import 'geometry_parameters.dart'; +import 'audio_restoration_parameters.dart'; import 'deflicker_parameters.dart'; import 'descale_parameters.dart'; +import 'colorize_parameters.dart'; +import 'face_restore_parameters.dart'; import 'edge_repair_parameters.dart'; import 'frame_rate_parameters.dart'; import 'ghost_removal_parameters.dart'; @@ -139,6 +142,8 @@ class ParameterConverter { 'ivtcCycle': params.ivtcCycle, 'ivtcDupthresh': params.ivtcDupthresh, 'ivtcScthresh': params.ivtcScthresh, + 'ivtcFixFades': params.ivtcFixFades, + 'ivtcFadeMode': params.ivtcFadeMode, }, ); } @@ -192,6 +197,9 @@ class ParameterConverter { case NoiseReductionMethod.knlMeans: method = 'knlmeans'; break; + case NoiseReductionMethod.nlMeans: + method = 'nlmeans'; + break; } return DynamicParameters( @@ -281,6 +289,7 @@ class ParameterConverter { DehaloMethod.vinverse => 'vinverse', DehaloMethod.vinverse2 => 'vinverse2', DehaloMethod.hqDeringmod => 'hq_deringmod', + DehaloMethod.vine => 'vine', }; /// Convert chroma denoise (CCD) parameters to dynamic format. @@ -326,6 +335,9 @@ class ParameterConverter { 'highThreshold': params.highThreshold, 'yahrBlur': params.yahrBlur, 'yahrDepth': params.yahrDepth, + 'vineStrength': params.vineStrength, + 'vineSharp': params.vineSharp, + 'vineSigma': params.vineSigma, }; final lastOptional = {}; @@ -377,9 +389,10 @@ class ParameterConverter { filterId: 'anti_alias', enabled: params.enabled, values: { - 'method': params.method == AntiAliasMethod.santiag ? 'santiag' : 'daa', + 'method': params.method.value, 'santiagStrh': params.santiagStrh, 'santiagStrv': params.santiagStrv, + 'sangnomAa': params.sangnomAa, }, ); } @@ -389,12 +402,14 @@ class ParameterConverter { final v = params.values; return AntiAliasParameters( enabled: params.enabled, - method: v['method'] == 'santiag' - ? AntiAliasMethod.santiag - : AntiAliasMethod.daa, + method: AntiAliasMethod.values.firstWhere( + (m) => m.value == v['method'], + orElse: () => AntiAliasMethod.daa, + ), santiagStrh: _asInt(v['santiagStrh']) ?? 1, santiagStrv: _asInt(v['santiagStrv']) ?? 1, - // Not exposed: only nnedi3 is bundled, and the worker pins it there. + sangnomAa: _asInt(v['sangnomAa']) ?? 48, + // santiagType is not exposed: the worker pins it to nnedi3. ); } @@ -492,6 +507,67 @@ class ParameterConverter { ); } + /// Convert colourisation parameters to dynamic format. The model is the + /// schema's method. + static DynamicParameters fromColorize(ColorizeParameters params) { + return DynamicParameters( + filterId: 'colorize', + enabled: params.enabled, + values: { + 'method': params.model.value, + 'ddcolorRenderSize': params.ddcolorRenderSize, + 'deoldifyRenderFactor': params.deoldifyRenderFactor, + 'strength': params.strength, + 'temporalRadius': params.temporalRadius, + }, + ); + } + + /// Convert dynamic parameters to colourisation parameters. + static ColorizeParameters toColorize(DynamicParameters params) { + final v = params.values; + return ColorizeParameters( + enabled: params.enabled, + model: ColorizeModel.fromValue(v['method'] as String?), + ddcolorRenderSize: _asInt(v['ddcolorRenderSize']) ?? 512, + deoldifyRenderFactor: _asInt(v['deoldifyRenderFactor']) ?? 21, + strength: (v['strength'] as num?)?.toDouble() ?? 1.0, + temporalRadius: _asInt(v['temporalRadius']) ?? 2, + ); + } + + /// Convert face restoration parameters to dynamic format. + static DynamicParameters fromFaceRestore(FaceRestoreParameters params) { + return DynamicParameters( + filterId: 'face_restore', + enabled: params.enabled, + values: { + 'model': params.model.value, + 'strength': params.strength, + 'codeFormerFidelity': params.codeFormerFidelity, + 'detectionThreshold': params.detectionThreshold, + 'minFaceSize': params.minFaceSize, + 'maxFaces': params.maxFaces, + 'temporalSmoothing': params.temporalSmoothing, + }, + ); + } + + /// Convert dynamic parameters to face restoration parameters. + static FaceRestoreParameters toFaceRestore(DynamicParameters params) { + final v = params.values; + return FaceRestoreParameters( + enabled: params.enabled, + model: FaceRestoreModel.fromValue(v['model'] as String?), + strength: (v['strength'] as num?)?.toDouble() ?? 0.8, + codeFormerFidelity: (v['codeFormerFidelity'] as num?)?.toDouble() ?? 0.7, + detectionThreshold: (v['detectionThreshold'] as num?)?.toDouble() ?? 0.8, + minFaceSize: _asInt(v['minFaceSize']) ?? 32, + maxFaces: _asInt(v['maxFaces']) ?? 8, + temporalSmoothing: v['temporalSmoothing'] as bool? ?? true, + ); + } + /// Convert deflicker parameters to dynamic format. static DynamicParameters fromDeflicker(DeflickerParameters params) { return DynamicParameters( @@ -693,6 +769,9 @@ class ParameterConverter { case DeblockMethod.dctFilter: method = 'dctfilter'; break; + case DeblockMethod.mpeg2stinx: + method = 'mpeg2stinx'; + break; } return DynamicParameters( @@ -920,6 +999,9 @@ class ParameterConverter { 'cuganNoise': params.cuganNoise, 'animejanaiModel': params.animejanaiModel, 'animejanaiSharp': params.animejanaiSharp, + 'generalDenoise': params.generalDenoise, + 'aiStrength': params.aiStrength, + 'convertToBt709': params.convertToBt709.name, }; // Kernel tuning and the nnedi3/EEDI3 controls start unticked, so the // plugin's own default applies until the user opts in. Fallbacks are those @@ -972,6 +1054,83 @@ class ParameterConverter { ); } + /// The schema's option values for the audio restoration enums (the same + /// strings the worker reads). + static const channelFixValues = { + ChannelFix.off: 'off', + ChannelFix.auto: 'auto', + ChannelFix.leftOnly: 'leftOnly', + ChannelFix.rightOnly: 'rightOnly', + ChannelFix.mix: 'mix', + }; + static const humFrequencyValues = { + HumFrequency.auto: 'auto', + HumFrequency.hz50: '50', + HumFrequency.hz60: '60', + }; + + /// Convert audio restoration parameters to dynamic format. + static DynamicParameters fromAudioRestoration(AudioRestorationParameters params) { + return DynamicParameters( + filterId: 'audio_restoration', + enabled: params.enabled, + values: { + 'streamIndex': params.streamIndex, + 'channelFix': channelFixValues[params.channelFix], + 'offsetMs': params.offsetMs, + 'driftCorrection': params.driftCorrection, + 'driftMaxPercent': params.driftMaxPercent, + 'declip': params.declip, + 'declick': params.declick, + 'declickThreshold': params.declickThreshold, + 'humRemoval': params.humRemoval, + 'humFrequency': humFrequencyValues[params.humFrequency], + 'rumbleFilter': params.rumbleFilter, + 'rumbleCutoffHz': params.rumbleCutoffHz, + 'denoise': params.denoise, + 'denoiseStrength': params.denoiseStrength, + 'deepFilter': params.deepFilter, + 'deepFilterAttenuation': params.deepFilterAttenuation, + 'loudnessNormalize': params.loudnessNormalize, + 'loudnessTarget': params.loudnessTarget, + 'limiter': params.limiter, + 'limiterCeilingDb': params.limiterCeilingDb, + }, + ); + } + + /// Convert dynamic parameters to audio restoration parameters. + static AudioRestorationParameters toAudioRestoration(DynamicParameters params) { + final v = params.values; + const d = AudioRestorationParameters(); + T pick(Map table, Object? value, T fallback) => table.entries + .firstWhere((e) => e.value == value, orElse: () => MapEntry(fallback, '')) + .key; + return AudioRestorationParameters( + enabled: params.enabled, + streamIndex: _asInt(v['streamIndex']) ?? d.streamIndex, + channelFix: pick(channelFixValues, v['channelFix'], d.channelFix), + offsetMs: _asInt(v['offsetMs']) ?? d.offsetMs, + driftCorrection: v['driftCorrection'] as bool? ?? d.driftCorrection, + driftMaxPercent: (v['driftMaxPercent'] as num?)?.toDouble() ?? d.driftMaxPercent, + declip: v['declip'] as bool? ?? d.declip, + declick: v['declick'] as bool? ?? d.declick, + declickThreshold: (v['declickThreshold'] as num?)?.toDouble() ?? d.declickThreshold, + humRemoval: v['humRemoval'] as bool? ?? d.humRemoval, + humFrequency: pick(humFrequencyValues, v['humFrequency'], d.humFrequency), + rumbleFilter: v['rumbleFilter'] as bool? ?? d.rumbleFilter, + rumbleCutoffHz: _asInt(v['rumbleCutoffHz']) ?? d.rumbleCutoffHz, + denoise: v['denoise'] as bool? ?? d.denoise, + denoiseStrength: (v['denoiseStrength'] as num?)?.toDouble() ?? d.denoiseStrength, + deepFilter: v['deepFilter'] as bool? ?? d.deepFilter, + deepFilterAttenuation: _asInt(v['deepFilterAttenuation']) ?? d.deepFilterAttenuation, + loudnessNormalize: v['loudnessNormalize'] as bool? ?? d.loudnessNormalize, + loudnessTarget: (v['loudnessTarget'] as num?)?.toDouble() ?? d.loudnessTarget, + limiter: v['limiter'] as bool? ?? d.limiter, + limiterCeilingDb: (v['limiterCeilingDb'] as num?)?.toDouble() ?? d.limiterCeilingDb, + ); + } + /// Convert a full processing pipeline to a dynamic pipeline. static DynamicPipeline fromPipeline(ProcessingPipeline pipeline) { return DynamicPipeline( @@ -997,6 +1156,9 @@ class ParameterConverter { 'color_correction': fromColorCorrection(pipeline.colorCorrection), 'chroma_fixes': fromChromaFixes(pipeline.chromaFixes), 'crop_resize': fromCropResize(pipeline.cropResize), + 'face_restore': fromFaceRestore(pipeline.faceRestore), + 'colorize': fromColorize(pipeline.colorize), + 'audio_restoration': fromAudioRestoration(pipeline.audio), 'subtitles': fromSubtitles(pipeline.subtitles), }, ); @@ -1121,6 +1283,8 @@ class ParameterConverter { ivtcCycle: v['ivtcCycle'] as int?, ivtcDupthresh: (v['ivtcDupthresh'] as num?)?.toDouble(), ivtcScthresh: (v['ivtcScthresh'] as num?)?.toDouble(), + ivtcFixFades: v['ivtcFixFades'] as bool? ?? false, + ivtcFadeMode: v['ivtcFadeMode'] as String? ?? 'average', ); } @@ -1172,6 +1336,9 @@ class ParameterConverter { case 'knlmeans': method = NoiseReductionMethod.knlMeans; break; + case 'nlmeans': + method = NoiseReductionMethod.nlMeans; + break; default: method = NoiseReductionMethod.smDegrain; } @@ -1274,6 +1441,7 @@ class ParameterConverter { 'vinverse' => DehaloMethod.vinverse, 'vinverse2' => DehaloMethod.vinverse2, 'hq_deringmod' => DehaloMethod.hqDeringmod, + 'vine' => DehaloMethod.vine, _ => DehaloMethod.dehaloAlpha, }; @@ -1310,6 +1478,9 @@ class ParameterConverter { deringMthr: _asInt(v['deringMthr']), deringThr: (v['deringThr'] as num?)?.toDouble(), deringDarkthr: (v['deringDarkthr'] as num?)?.toDouble(), + vineStrength: (v['vineStrength'] as num?)?.toDouble() ?? 16.0, + vineSharp: (v['vineSharp'] as num?)?.toDouble() ?? 0.5, + vineSigma: (v['vineSigma'] as num?)?.toDouble() ?? 1.0, ); } @@ -1325,6 +1496,9 @@ class ParameterConverter { case 'dctfilter': method = DeblockMethod.dctFilter; break; + case 'mpeg2stinx': + method = DeblockMethod.mpeg2stinx; + break; default: method = DeblockMethod.deblockQed; } @@ -1551,6 +1725,12 @@ class ParameterConverter { cuganNoise: v['cuganNoise'] as String? ?? 'conservative', animejanaiModel: v['animejanaiModel'] as String? ?? 'compact', animejanaiSharp: v['animejanaiSharp'] as bool? ?? false, + generalDenoise: v['generalDenoise'] as bool? ?? false, + aiStrength: (v['aiStrength'] as num?)?.toDouble() ?? 1.0, + convertToBt709: Bt709Conversion.values.firstWhere( + (m) => m.name == (v['convertToBt709'] as String? ?? 'auto'), + orElse: () => Bt709Conversion.auto, + ), // Absent (checkbox off) stays null so the script omits it entirely. bicubicB: (v['bicubicB'] as num?)?.toDouble(), bicubicC: (v['bicubicC'] as num?)?.toDouble(), @@ -1614,6 +1794,12 @@ class ParameterConverter { descale: dynamic.get('descale') != null ? toDescale(dynamic.get('descale')!) : const DescaleParameters(), + faceRestore: dynamic.get('face_restore') != null + ? toFaceRestore(dynamic.get('face_restore')!) + : const FaceRestoreParameters(), + colorize: dynamic.get('colorize') != null + ? toColorize(dynamic.get('colorize')!) + : const ColorizeParameters(), edgeRepair: dynamic.get('edge_repair') != null ? toEdgeRepair(dynamic.get('edge_repair')!) : const EdgeRepairParameters(), @@ -1656,6 +1842,9 @@ class ParameterConverter { cropResize: dynamic.get('crop_resize') != null ? toCropResize(dynamic.get('crop_resize')!) : const CropResizeParameters(), + audio: dynamic.get('audio_restoration') != null + ? toAudioRestoration(dynamic.get('audio_restoration')!) + : const AudioRestorationParameters(), subtitles: dynamic.get('subtitles') != null ? toSubtitles(dynamic.get('subtitles')!) : const SubtitleParameters(), diff --git a/app/lib/models/pass_advice.dart b/app/lib/models/pass_advice.dart index bb94c4ee..1f1a25b4 100644 --- a/app/lib/models/pass_advice.dart +++ b/app/lib/models/pass_advice.dart @@ -161,6 +161,18 @@ List adviseOn(ProcessingPipeline pipeline) { )); } + // --- Chroma Denoise before Colourisation --- + // Colourisation keeps only the luma and predicts the colour from it, so + // the colour Chroma Denoise cleaned (which runs first) is thrown away. + if (on(PassType.colorize) && on(PassType.chromaDenoise)) { + advice.add(const PassAdvice( + PassType.chromaDenoise, + 'Colourisation runs after this and replaces the colour with its own, ' + 'predicted from the brightness alone, so the colour cleaned here does ' + 'not reach the result. Turn Chroma Denoise off to save time.', + )); + } + // --- Rotating interlaced material destroys it --- // Fields are stored as alternating horizontal lines. A quarter turn puts them // in alternating COLUMNS, where no deinterlacer can find them: measured, diff --git a/app/lib/models/pass_relevance.dart b/app/lib/models/pass_relevance.dart index 33e4b380..b2beed30 100644 --- a/app/lib/models/pass_relevance.dart +++ b/app/lib/models/pass_relevance.dart @@ -1,4 +1,5 @@ import '../services/field_order_detector.dart'; +import 'noise_halo_report.dart'; import 'processing_pipeline.dart'; /// How much a pass has to do with the file that is actually loaded. @@ -26,8 +27,12 @@ enum PassRelevance { /// - **Recommend sparingly.** A badge on nine rows out of thirteen is not a /// recommendation, it is decoration. Only claims backed by strong evidence /// from detection are made, which is why most passes come back [neutral] — -/// film dirt, scratches, grain and over-sharpening are all invisible to +/// film dirt, scratches, banding and colour casts are all invisible to /// ffprobe, so the app has nothing to say about them and says nothing. +/// Noise and over-sharpening are the exception: the worker measures them in +/// the picture itself (`--detect-noise-halo`, [NoiseHaloReport]), and only a +/// clearly high measurement badges its pass — a low or missing one says +/// nothing, never "not needed". /// - **Never reorder, never disable.** The pass list's order is the order the /// passes run in, and a [notApplicable] pass stays enabled and editable — /// detection is a hint, not an authority, and it is wrong often enough @@ -53,11 +58,13 @@ class PassRelevanceResult { /// only appear at or below SD. const int _sdMaxHeight = 576; -/// Decide how relevant [pass] is to [info]. +/// Decide how relevant [pass] is to [info], and, for Noise Reduction and +/// Dehalo, to what was [measured] in the picture. /// /// Returns [PassRelevanceResult.neutral] for a null [info] (nothing loaded yet) /// and for anything detection can't speak to. -PassRelevanceResult relevanceFor(PassType pass, VideoInfo? info) { +PassRelevanceResult relevanceFor(PassType pass, VideoInfo? info, + {NoiseHaloReport? measured}) { if (info == null) return PassRelevanceResult.neutral; final isProgressive = info.scanType == ScanType.progressive; @@ -123,18 +130,46 @@ PassRelevanceResult relevanceFor(PassType pass, VideoInfo? info) { } return PassRelevanceResult.neutral; - // Everything below is invisible to detection. Film dirt, scratches, grain, - // halos, banding and colour casts need eyes on the picture, so the app has - // no opinion and offers none. + // Measured in the picture, so only the worker's verdict counts: it decides + // the threshold (calibrated in docs/reports/2026-11-noise-halo-scores), and + // it under-reports rather than over-reports, so a low score is not + // evidence the pass isn't needed and says nothing. + case PassType.noiseReduction: + if (measured != null && measured.error == null && measured.noiseHigh) { + return const PassRelevanceResult( + PassRelevance.recommended, + 'noise measured high', + ); + } + return PassRelevanceResult.neutral; + + case PassType.dehalo: + if (measured != null && measured.error == null && measured.haloHigh) { + return const PassRelevanceResult( + PassRelevance.recommended, + 'halos measured around edges', + ); + } + return PassRelevanceResult.neutral; + + // Everything below is invisible to detection. Film dirt, scratches, + // banding and colour casts need eyes on the picture, so the app has no + // opinion and offers none. (Chroma noise is not measured: the noise score + // is luma only.) case PassType.descratch: case PassType.spotless: - case PassType.noiseReduction: case PassType.chromaDenoise: - case PassType.dehalo: case PassType.deband: case PassType.sharpen: case PassType.colorCorrection: + // Whether a video has faces worth restoring is not in its metadata. + case PassType.faceRestore: + // Black-and-white footage is usually stored as colour video with neutral + // chroma, so its metadata does not say it is black and white. + case PassType.colorize: case PassType.subtitles: + // A dead channel, hum or drift is not in the metadata either. + case PassType.audioRestoration: // Anti-aliasing and stabilisation are tempting to tie to detection — // jaggies follow deinterlacing, shake follows handheld capture — but // neither is actually visible in the metadata, only plausible from it. diff --git a/app/lib/models/processing_pipeline.dart b/app/lib/models/processing_pipeline.dart index 3ea43f04..e738e0d4 100644 --- a/app/lib/models/processing_pipeline.dart +++ b/app/lib/models/processing_pipeline.dart @@ -1,10 +1,13 @@ import 'package:json_annotation/json_annotation.dart'; import 'anti_alias_parameters.dart'; +import 'audio_restoration_parameters.dart'; import 'geometry_parameters.dart'; import 'deflicker_parameters.dart'; import 'descale_parameters.dart'; +import 'colorize_parameters.dart'; import 'edge_repair_parameters.dart'; +import 'face_restore_parameters.dart'; import 'frame_rate_parameters.dart'; import 'ghost_removal_parameters.dart'; import 'grain_parameters.dart'; @@ -50,6 +53,9 @@ enum PassType { colorCorrection, chromaFixes, cropResize, + faceRestore, + colorize, + audioRestoration, subtitles, } @@ -99,6 +105,12 @@ extension PassTypeExtension on PassType { return 'Chroma Fixes'; case PassType.cropResize: return 'Crop / Resize'; + case PassType.faceRestore: + return 'Face Restoration'; + case PassType.colorize: + return 'Colourisation'; + case PassType.audioRestoration: + return 'Audio Restoration'; case PassType.subtitles: return 'Subtitles'; } @@ -148,6 +160,12 @@ extension PassTypeExtension on PassType { return 'Fix chroma bleeding and crawl artifacts'; case PassType.cropResize: return 'Crop borders and resize output'; + case PassType.faceRestore: + return 'Restore detail in blurry, low-resolution faces with AI'; + case PassType.colorize: + return 'Colour black-and-white footage with AI'; + case PassType.audioRestoration: + return 'Repair, clean and re-sync the soundtrack'; case PassType.subtitles: return 'Generate subtitles using Whisper AI'; } @@ -203,6 +221,15 @@ class ProcessingPipeline { /// Crop and resize pass parameters. final CropResizeParameters cropResize; + /// Face restoration pass parameters (an add-on model). + final FaceRestoreParameters faceRestore; + + /// Colourisation pass parameters (an add-on model). + final ColorizeParameters colorize; + /// Audio restoration (rendered by ffmpeg before the encode; not a + /// VapourSynth pass, so not in [enabledPasses]). + final AudioRestorationParameters audio; + /// Subtitle generation pass parameters (Whisper AI). final SubtitleParameters subtitles; @@ -228,6 +255,9 @@ class ProcessingPipeline { this.colorCorrection = const ColorCorrectionParameters(), this.chromaFixes = const ChromaFixParameters(), this.cropResize = const CropResizeParameters(), + this.faceRestore = const FaceRestoreParameters(), + this.colorize = const ColorizeParameters(), + this.audio = const AudioRestorationParameters(), this.subtitles = const SubtitleParameters(), }); @@ -256,6 +286,9 @@ class ProcessingPipeline { colorCorrection: const ColorCorrectionParameters(enabled: false), chromaFixes: const ChromaFixParameters(enabled: false), cropResize: const CropResizeParameters(enabled: false), + faceRestore: const FaceRestoreParameters(enabled: false), + colorize: const ColorizeParameters(enabled: false), + audio: const AudioRestorationParameters(enabled: false), subtitles: const SubtitleParameters(enabled: false), ); } @@ -320,6 +353,12 @@ class ProcessingPipeline { if (chromaFixes.enabled) { passes.add(PassType.chromaFixes); } + // Colourisation follows the cleanup and Chroma Fixes (whose repairs are + // to the chroma it replaces) and precedes Colour Correction, so the result + // can be graded. + if (colorize.enabled) { + passes.add(PassType.colorize); + } if (colorCorrection.enabled) { passes.add(PassType.colorCorrection); } @@ -341,6 +380,12 @@ class ProcessingPipeline { (cropResize.cropEnabled || cropResize.resizeEnabled)) { passes.add(PassType.cropResize); } + // Faces are restored at the delivered resolution, after the resize, and + // before the sharpening and grain that would otherwise act on the face + // the restorer is about to replace. + if (faceRestore.enabled) { + passes.add(PassType.faceRestore); + } // Sharpening follows the resize (issue #109), so it works on the delivered // pixels: sharpened earlier, a downscale softens the edges again and an // upscale enlarges the halos. It still follows anti-aliasing, and precedes @@ -362,7 +407,7 @@ class ProcessingPipeline { return passes; } - /// Get count of enabled passes (includes subtitles). + /// Get count of enabled passes (includes audio restoration and subtitles). int get enabledPassCount { var count = 0; if (deinterlace.enabled) count++; @@ -386,11 +431,15 @@ class ProcessingPipeline { if (colorCorrection.enabled) count++; if (chromaFixes.enabled) count++; if (cropResize.enabled) count++; + if (faceRestore.enabled) count++; + if (colorize.enabled) count++; + if (audio.enabled) count++; if (subtitles.enabled) count++; return count; } - /// Get count of enabled video processing passes (excludes subtitles). + /// Get count of enabled video processing passes (excludes audio + /// restoration and subtitles). int get videoPassCount { var count = 0; if (deinterlace.enabled) count++; @@ -414,6 +463,8 @@ class ProcessingPipeline { if (colorCorrection.enabled) count++; if (chromaFixes.enabled) count++; if (cropResize.enabled) count++; + if (faceRestore.enabled) count++; + if (colorize.enabled) count++; return count; } @@ -462,6 +513,12 @@ class ProcessingPipeline { return chromaFixes.enabled; case PassType.cropResize: return cropResize.enabled; + case PassType.faceRestore: + return faceRestore.enabled; + case PassType.colorize: + return colorize.enabled; + case PassType.audioRestoration: + return audio.enabled; case PassType.subtitles: return subtitles.enabled; } @@ -514,6 +571,12 @@ class ProcessingPipeline { return chromaFixes.summary; case PassType.cropResize: return cropResize.summary; + case PassType.faceRestore: + return faceRestore.summary; + case PassType.colorize: + return colorize.summary; + case PassType.audioRestoration: + return audio.summary; case PassType.subtitles: return subtitles.summary; } @@ -541,6 +604,9 @@ class ProcessingPipeline { ColorCorrectionParameters? colorCorrection, ChromaFixParameters? chromaFixes, CropResizeParameters? cropResize, + FaceRestoreParameters? faceRestore, + ColorizeParameters? colorize, + AudioRestorationParameters? audio, SubtitleParameters? subtitles, }) { return ProcessingPipeline( @@ -565,6 +631,9 @@ class ProcessingPipeline { colorCorrection: colorCorrection ?? this.colorCorrection, chromaFixes: chromaFixes ?? this.chromaFixes, cropResize: cropResize ?? this.cropResize, + faceRestore: faceRestore ?? this.faceRestore, + colorize: colorize ?? this.colorize, + audio: audio ?? this.audio, subtitles: subtitles ?? this.subtitles, ); } @@ -650,6 +719,12 @@ class ProcessingPipeline { return copyWith( cropResize: cropResize.copyWith(enabled: enabled), ); + case PassType.faceRestore: + return copyWith(faceRestore: faceRestore.copyWith(enabled: enabled)); + case PassType.colorize: + return copyWith(colorize: colorize.copyWith(enabled: enabled)); + case PassType.audioRestoration: + return copyWith(audio: audio.copyWith(enabled: enabled)); case PassType.subtitles: return copyWith( subtitles: subtitles.copyWith(enabled: enabled), diff --git a/app/lib/models/processing_preset.dart b/app/lib/models/processing_preset.dart index 64f4b71a..e6ea370d 100644 --- a/app/lib/models/processing_preset.dart +++ b/app/lib/models/processing_preset.dart @@ -4,6 +4,7 @@ import 'package:uuid/uuid.dart'; import 'chroma_denoise_parameters.dart'; import 'chroma_fix_parameters.dart'; import 'color_correction_parameters.dart'; +import 'crop_resize_parameters.dart'; import 'deband_parameters.dart'; import 'deblock_parameters.dart'; import 'dehalo_parameters.dart'; @@ -480,12 +481,87 @@ class ProcessingPreset { ); } + /// Built-in preset: an SD tape capture made into an HD file for YouTube or + /// a TV (app 1.7.0). + /// + /// The tape cleanup of [builtInVhsCleanup], kept lighter, then the three + /// things an SD capture needs before an HD screen shows it properly: square + /// pixels (a TV or YouTube stretches anamorphic SD unpredictably), an NNEDI3 + /// doubling resized to 1080 lines (a plain resize from 576 lines is soft), + /// and BT.709 colour — an HD-sized picture left in BT.601 is decoded as 709 + /// by many players, which shifts greens and reds. Light CAS after the + /// upscale, on the delivered pixels. H.264 at CRF 17 in 8-bit 4:2:0 MP4 with + /// AAC: what every TV, browser and upload service plays, which a 4:2:2 + /// capture-card source would otherwise not be. + static ProcessingPreset builtInSdTapeToHd() { + return ProcessingPreset( + id: 'builtin-sd-tape-to-hd', + name: 'SD Tape \u2192 HD (YouTube / TV)', + description: + 'VHS / Hi8 / DV capture: deinterlace, clean up, upscale to 1080 lines in BT.709', + pipeline: ProcessingPipeline( + deinterlace: QTGMCParameters( + enabled: true, + preset: QTGMCPreset.slow, + ), + // fromPreset: the enum alone is only a label. + noiseReduction: + NoiseReductionParameters.fromPreset(NoiseReductionPreset.light), + chromaDenoise: ChromaDenoiseParameters( + enabled: true, + threshold: 4.0, + ), + cropResize: CropResizeParameters( + enabled: true, + preset: CropResizePreset.custom, + // Cropping stays off: how much a tape needs is measured per source + // (Crop & Resize's Detect). These are the panel's own starting + // values, so ticking Enable Cropping shows what it always shows. + cropLeft: 8, + cropRight: 8, + cropTop: 4, + cropBottom: 4, + pixelAspect: PixelAspectMode.square, + useIntegerUpscale: true, + upscaleMethod: UpscaleMethod.nnedi3Rpow2, + upscaleFactor: 2, + resizeEnabled: true, + // A box: 4:3 lands at 1440x1080, 16:9 at 1920x1080. + targetWidth: 1920, + targetHeight: 1080, + kernel: ResizeKernel.spline36, + maintainAspect: true, + // Auto would convert here too (the output is 1080 lines); stated, so + // the preset says what its name promises whatever the default. + convertToBt709: Bt709Conversion.on, + ), + sharpen: SharpenParameters( + enabled: true, + method: SharpenMethod.cas, + casSharpness: 0.3, + ), + ), + encodingSettings: EncodingSettings( + codec: VideoCodec.h264, + container: ContainerFormat.mp4, + encoderPreset: 'slow', + quality: 17, + chromaSubsampling: ChromaSubsampling.yuv420, + audioMode: AudioMode.convert, + audioCodec: AudioCodec.aac, + ), + isBuiltIn: true, + category: PresetCategory.source, + ); + } + static List builtInPresets() { return [ builtInFast(), builtInBalanced(), builtInHighQuality(), builtInVhsCleanup(), + builtInSdTapeToHd(), builtInDvCamcorder(), builtInPalDvd(), builtInDvdIvtc(), diff --git a/app/lib/models/progress_info.dart b/app/lib/models/progress_info.dart index 9a7b7f64..6c7cf8fc 100644 --- a/app/lib/models/progress_info.dart +++ b/app/lib/models/progress_info.dart @@ -44,6 +44,9 @@ class ProgressInfo { /// True during subtitle embedding phase. bool get isEmbeddingPhase => phase == 'embedding'; + /// True while audio restoration renders the soundtrack (before the encode). + bool get isAudioPhase => phase == 'audio'; + /// Formatted ETA string (e.g., "1h 23m 45s"). String get etaFormatted { if (eta <= 0 || !eta.isFinite) return '--'; diff --git a/app/lib/models/qtgmc_parameters.dart b/app/lib/models/qtgmc_parameters.dart index 9d63d0ef..7c4634e2 100644 --- a/app/lib/models/qtgmc_parameters.dart +++ b/app/lib/models/qtgmc_parameters.dart @@ -271,6 +271,14 @@ class QTGMCParameters { /// VDecimate scene change threshold final double? ivtcScthresh; + /// Fix fades added after telecine (vs-jetpack's FixInterlacedFades), right + /// after field matching. + final bool ivtcFixFades; + + /// `average` (both fields to their mean) or `match` (the brighter field to + /// the darker). + final String ivtcFadeMode; + const QTGMCParameters({ this.enabled = true, this.method = DeinterlaceMethod.qtgmc, @@ -363,6 +371,8 @@ class QTGMCParameters { this.ivtcCycle, this.ivtcDupthresh, this.ivtcScthresh, + this.ivtcFixFades = false, + this.ivtcFadeMode = 'average', }); factory QTGMCParameters.fromJson(Map json) => @@ -461,6 +471,8 @@ class QTGMCParameters { double? ivtcDupthresh, double? ivtcScthresh, bool? bwdifEdeint, + bool? ivtcFixFades, + String? ivtcFadeMode, }) { return QTGMCParameters( enabled: enabled ?? this.enabled, @@ -554,6 +566,8 @@ class QTGMCParameters { ivtcDupthresh: ivtcDupthresh ?? this.ivtcDupthresh, ivtcScthresh: ivtcScthresh ?? this.ivtcScthresh, bwdifEdeint: bwdifEdeint ?? this.bwdifEdeint, + ivtcFixFades: ivtcFixFades ?? this.ivtcFixFades, + ivtcFadeMode: ivtcFadeMode ?? this.ivtcFadeMode, ); } } diff --git a/app/lib/models/queue_item.dart b/app/lib/models/queue_item.dart index 6c98ef33..dcfddb09 100644 --- a/app/lib/models/queue_item.dart +++ b/app/lib/models/queue_item.dart @@ -3,6 +3,7 @@ import 'dart:typed_data'; import 'package:uuid/uuid.dart'; import '../services/field_order_detector.dart'; +import 'noise_halo_report.dart'; /// Status of a queue item. enum QueueItemStatus { @@ -48,6 +49,11 @@ class QueueItem { String outputPath; VideoInfo? videoInfo; + /// How noisy the source is and how strongly its edges ring, measured in the + /// background after [videoInfo] (app 1.7.0); null until it comes back. + /// Transient: only the pass list's suggestions read it. + NoiseHaloReport? noiseHalo; + /// Optional DVD source info (for display and temp file cleanup). DvdSourceInfo? dvdSourceInfo; diff --git a/app/lib/services/addon_service.dart b/app/lib/services/addon_service.dart index f6c01fa5..ae662750 100644 --- a/app/lib/services/addon_service.dart +++ b/app/lib/services/addon_service.dart @@ -10,8 +10,29 @@ import 'package:path/path.dart' as path; import 'addon_manager.dart'; import 'dependency_manager.dart'; +import 'manifest_addon.dart'; import 'whisper_addon_manager.dart'; +/// A further third-party component inside an add-on, credited on its own: +/// its own NOTICES entry, About tile and README credit, like the add-on's +/// main one. The AI runtime is ONNX Runtime plus NumPy (and on Windows +/// Microsoft's DirectML), each under its own licence. +class AddonCredit { + final String noticeName; + final String licence; + final String? licenceUrl; + final String credit; + final String sourceUrl; + + const AddonCredit({ + required this.noticeName, + required this.licence, + this.licenceUrl, + required this.credit, + required this.sourceUrl, + }); +} + /// One optional third-party download: something VapourBox can use but does not /// ship, because of its size or its licence. Shown on Settings → Add-ons and /// in the prompt a feature raises when it needs one. @@ -55,6 +76,10 @@ class AddonInfo { /// Why it cannot be used on this platform, or null if it can. final String? unavailableReason; + /// Further components it includes on this platform, each credited + /// separately (see [AddonCredit]); the main one is the fields above. + final List additionalCredits; + const AddonInfo({ required this.id, required this.name, @@ -69,7 +94,20 @@ class AddonInfo { required this.downloadBytes, this.requires = const [], this.unavailableReason, + this.additionalCredits = const [], }); + + /// Every component to credit: the main one, then [additionalCredits]. + List get credits => [ + AddonCredit( + noticeName: noticeName, + licence: licence, + licenceUrl: licenceUrl, + credit: credit, + sourceUrl: sourceUrl, + ), + ...additionalCredits, + ]; } /// How one add-on is installed, recognised and removed under the add-ons @@ -139,6 +177,9 @@ class AddonService extends ChangeNotifier { as Map, platform, ), + // Every add-on described by a manifest in assets/addons/. + for (final manifest in await AddonManifest.loadBundled()) + ManifestAddonInstaller(manifest, platform), ]); } await refresh(); diff --git a/app/lib/services/manifest_addon.dart b/app/lib/services/manifest_addon.dart new file mode 100644 index 00000000..5c1a2aba --- /dev/null +++ b/app/lib/services/manifest_addon.dart @@ -0,0 +1,572 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive_io.dart'; +import 'package:crypto/crypto.dart'; +import 'package:flutter/services.dart'; +import 'package:path/path.dart' as path; + +import 'addon_manager.dart'; +import 'addon_service.dart'; + +/// Add-ons described entirely by a manifest, one JSON file per add-on in +/// `assets/addons/` (format: docs/ADDONS.md). The worker compiles in the same +/// files (`worker/src/addons.rs`), so the app and the worker agree on what +/// each add-on is, where it installs and what "installed" means. +/// +/// Two kinds: +/// * `files` — a model pack: each file is downloaded into `addons//`, +/// or, for an archive entry, only its listed members are kept, each checked +/// against its own hash. +/// * `python` — Python packages for VapourBox's own Python: each file is a +/// wheel, unpacked into `addons//site-packages/`. +/// +/// Everything is pinned by SHA-256; an install is built in `addons/.new` +/// and renamed into place, and ends by writing [markerName], so neither the +/// app nor the worker ever sees a partial install. A manifest whose `version` +/// changes makes the installed copy count as missing, so the feature asks for +/// it again and the install replaces it. + +/// The platform ids the add-ons are described for (the deps bundle's). +const addonPlatformIds = [ + 'macos-arm64', + 'macos-x64', + 'windows-x64', + 'linux-x64', + 'linux-arm64', +]; + +/// `VAPOURBOX_ADDON_MIRROR=DIR`: take each file from `DIR/` + the last path +/// segment of its url instead of downloading it, still checked against its +/// size and hash — for testing an add-on whose files are not uploaded yet. +/// A file the mirror doesn't hold is downloaded as usual, so a mirror of our +/// own converted files still installs add-ons whose files come from upstream +/// (the AI runtime's wheels). +const addonMirrorEnv = 'VAPOURBOX_ADDON_MIRROR'; + +/// One file a manifest add-on installs. +class AddonManifestFile { + /// Its name in `addons//` (a `files` add-on), or the wheel's file name. + final String name; + final String url; + final int size; + final String sha256; + + /// The platforms it is for; null means every platform the add-on supports. + final List? platforms; + + /// For an archive: the members to keep (nothing else is extracted). Null + /// for a file kept as downloaded, or a wheel. + final List? extract; + + /// A program the worker runs (DeepFilterNet's `deep-filter`): installed + /// with its executable bit set and, on macOS, signed if it is not. + final bool executable; + + const AddonManifestFile({ + required this.name, + required this.url, + required this.size, + required this.sha256, + this.platforms, + this.extract, + this.executable = false, + }); + + factory AddonManifestFile.fromJson(Map json) => AddonManifestFile( + name: json['name'] as String, + url: json['url'] as String, + size: json['size'] as int, + sha256: json['sha256'] as String, + platforms: (json['platforms'] as List?)?.cast(), + extract: (json['extract'] as List?) + ?.map((m) => AddonArchiveMember.fromJson((m as Map).cast())) + .toList(), + executable: json['executable'] as bool? ?? false, + ); + + bool isFor(String platformId) => platforms == null || platforms!.contains(platformId); +} + +/// A member of an archive entry: its path inside the archive, the name it is +/// installed under, and its own hash. +class AddonArchiveMember { + final String path; + final String name; + final String sha256; + + const AddonArchiveMember({required this.path, required this.name, required this.sha256}); + + factory AddonArchiveMember.fromJson(Map json) => AddonArchiveMember( + path: json['path'] as String, + name: json['name'] as String? ?? (json['path'] as String).split('/').last, + sha256: json['sha256'] as String, + ); +} + +/// One add-on's manifest (`assets/addons/.json`). +class AddonManifest { + final String id; + final String version; + final String name; + final String feature; + final String description; + final String licence; + final String? licenceUrl; + final String? usageNote; + final String credit; + final String noticeName; + final String homepage; + final List requires; + + /// `files` or `python`. + final String kind; + + /// Platform id → its requirements (`minOs`, macOS only), for the platforms + /// it supports. `"platforms": "all"` in the file is every [addonPlatformIds]. + final Map> platforms; + final List files; + + /// Further components credited separately, each with optional `platforms`. + final List> components; + + const AddonManifest({ + required this.id, + required this.version, + required this.name, + required this.feature, + required this.description, + required this.licence, + this.licenceUrl, + this.usageNote, + required this.credit, + required this.noticeName, + required this.homepage, + this.requires = const [], + required this.kind, + required this.platforms, + required this.files, + this.components = const [], + }); + + static const kinds = {'files', 'python'}; + + /// Parse and validate: a malformed manifest fails here, in the test suite, + /// rather than as a half-working add-on. + factory AddonManifest.fromJson(Map json) { + final rawPlatforms = json['platforms']; + final platforms = rawPlatforms == 'all' + ? {for (final p in addonPlatformIds) p: {}} + : { + for (final MapEntry(:key, :value) in (rawPlatforms as Map).entries) + key as String: (value as Map?)?.cast() ?? {}, + }; + final m = AddonManifest( + id: json['id'] as String, + version: json['version'] as String, + name: json['name'] as String, + feature: json['feature'] as String, + description: json['description'] as String, + licence: json['licence'] as String, + licenceUrl: json['licenceUrl'] as String?, + usageNote: json['usageNote'] as String?, + credit: json['credit'] as String, + noticeName: json['noticeName'] as String, + homepage: json['homepage'] as String, + requires: (json['requires'] as List? ?? []).cast(), + kind: json['kind'] as String, + platforms: platforms, + files: (json['files'] as List) + .map((f) => AddonManifestFile.fromJson((f as Map).cast())) + .toList(), + components: [ + for (final c in json['components'] as List? ?? []) (c as Map).cast(), + ], + ); + m._validate(); + return m; + } + + void _validate() { + void fail(String why) => throw FormatException('Add-on manifest "$id": $why'); + if (!RegExp(r'^[a-z0-9][a-z0-9-]*$').hasMatch(id)) fail('id must be lower-case-with-dashes'); + if (!kinds.contains(kind)) fail('kind must be one of $kinds'); + for (final p in platforms.keys) { + if (!addonPlatformIds.contains(p)) fail('unknown platform $p'); + } + final hash = RegExp(r'^[0-9a-f]{64}$'); + final names = {}; + for (final f in files) { + if (!hash.hasMatch(f.sha256)) fail('${f.name} needs a lower-case hex sha256'); + if (f.size <= 0) fail('${f.name} needs its size'); + if (!f.url.startsWith('https://')) fail('${f.name} must download over https'); + if (!_safePath(f.name)) fail('${f.name} is not a plain file name or relative path'); + if (!names.add(f.name)) fail('${f.name} is listed twice'); + for (final p in f.platforms ?? const []) { + if (!platforms.containsKey(p)) fail('${f.name} names $p, which the add-on does not support'); + } + if (kind == 'python' && (!f.name.endsWith('.whl') || f.extract != null)) { + fail('a python add-on lists only wheels (${f.name})'); + } + if (f.executable && (kind != 'files' || f.extract != null)) { + fail('only a plain file of a files add-on can be executable (${f.name})'); + } + for (final member in f.extract ?? const []) { + if (!hash.hasMatch(member.sha256)) fail('${member.path} needs a sha256'); + if (!_safePath(member.name)) fail('${member.name} is not a plain file name or relative path'); + } + } + for (final p in platforms.keys) { + if (!files.any((f) => f.isFor(p))) fail('no files for $p'); + } + } + + static bool _safeName(String name) => + name.isNotEmpty && !name.contains('/') && !name.contains('\\') && name != '.' && name != '..'; + + /// A plain file name, or a relative path of plain names joined by `/` — + /// for a plugin that reads meaning from its model's directory name (RIFE + /// takes the model version, and with it the padding, from the path). + static bool _safePath(String name) => name.split('/').every(_safeName); + + /// The files this platform installs. + List filesFor(String platformId) => + files.where((f) => f.isFor(platformId)).toList(); + + /// Every manifest shipped in `assets/addons/`, found through the asset + /// manifest so a new file needs no index to be picked up. + static Future> loadBundled({AssetBundle? bundle}) async { + final b = bundle ?? rootBundle; + final assets = (await AssetManifest.loadFromAssetBundle(b)) + .listAssets() + .where((a) => a.startsWith('assets/addons/') && a.endsWith('.json')) + .toList() + ..sort(); + return [ + for (final a in assets) + AddonManifest.fromJson(jsonDecode(await b.loadString(a)) as Map), + ]; + } + + /// Every manifest in a directory on disk (tests, and the harness). + static List loadDirectory(Directory dir) { + final files = dir.listSync().whereType().where((f) => f.path.endsWith('.json')).toList() + ..sort((a, b) => a.path.compareTo(b.path)); + return [ + for (final f in files) + AddonManifest.fromJson(jsonDecode(f.readAsStringSync()) as Map), + ]; + } +} + +/// Installs, recognises and removes one manifest add-on (see the library +/// comment). Every file is verified before anything is unpacked. +class ManifestAddonInstaller implements AddonInstaller { + final AddonManifest manifest; + final String platformId; + + /// The running macOS version ("15.1"), for `minOs`; read from the system + /// unless given (tests). + final String? _osVersion; + + /// Where to take files from instead of the network; defaults to + /// [addonMirrorEnv]. An empty string means no mirror whatever the + /// environment says (the installer's own unit tests use it). + final String? _mirror; + + ManifestAddonInstaller(this.manifest, this.platformId, {String? osVersion, String? mirror}) + : _osVersion = osVersion, + _mirror = mirror; + + /// Written last into an install, as its commit marker: the add-on's id, + /// version and platform. The worker reads the same file. + static const markerName = '.vapourbox-addon.json'; + + String? get mirror { + if (_mirror != null) return _mirror.isEmpty ? null : _mirror; + final env = Platform.environment[addonMirrorEnv]; + return env == null || env.isEmpty ? null : env; + } + + List get _files => manifest.filesFor(platformId); + + @override + AddonInfo get info => AddonInfo( + id: manifest.id, + name: manifest.name, + feature: manifest.feature, + description: manifest.description, + licence: manifest.licence, + licenceUrl: manifest.licenceUrl, + usageNote: manifest.usageNote, + credit: manifest.credit, + noticeName: manifest.noticeName, + sourceUrl: manifest.homepage, + downloadBytes: _files.fold(0, (sum, f) => sum + f.size), + requires: manifest.requires, + unavailableReason: unavailableReason, + additionalCredits: [ + for (final c in manifest.components) + if ((c['platforms'] as List?)?.contains(platformId) ?? true) + AddonCredit( + noticeName: c['noticeName'] as String, + licence: c['licence'] as String, + licenceUrl: c['licenceUrl'] as String?, + credit: c['credit'] as String, + sourceUrl: c['homepage'] as String, + ), + ], + ); + + static const _platformNames = { + 'macos-arm64': 'Apple Silicon Macs', + 'macos-x64': 'Intel Macs', + 'windows-x64': 'Windows', + 'linux-x64': 'Linux (x64)', + 'linux-arm64': 'Linux on ARM', + }; + + /// Why it cannot be installed here, or null: the platform is not in the + /// manifest's table, or (macOS) the system is older than its `minOs`. + String? get unavailableReason { + final requirements = manifest.platforms[platformId]; + if (requirements == null) { + return '${manifest.name} isn\'t available for ' + '${_platformNames[platformId] ?? 'this platform'}.'; + } + final minOs = requirements['minOs'] as String?; + if (minOs != null && platformId.startsWith('macos')) { + final running = _osVersion ?? macOsVersion(); + if (running != null && compareOsVersions(running, minOs) < 0) { + return '${manifest.name} needs macOS $minOs or later; this Mac has $running.'; + } + } + return null; + } + + /// The macOS version this is running on, or null elsewhere / unreadable. + static String? macOsVersion() { + if (!Platform.isMacOS) return null; + return RegExp(r'\d+(\.\d+)*').firstMatch(Platform.operatingSystemVersion)?.group(0); + } + + /// Compare dotted versions numerically ("13.0" < "13.10"). + static int compareOsVersions(String a, String b) { + final pa = a.split('.').map((s) => int.tryParse(s) ?? 0).toList(); + final pb = b.split('.').map((s) => int.tryParse(s) ?? 0).toList(); + for (var i = 0; i < pa.length || i < pb.length; i++) { + final x = i < pa.length ? pa[i] : 0, y = i < pb.length ? pb[i] : 0; + if (x != y) return x.compareTo(y); + } + return 0; + } + + Directory target(Directory addonsDir) => Directory(path.join(addonsDir.path, manifest.id)); + + /// The files a complete install of this platform's files holds, relative to + /// the add-on's directory (a python add-on: its site-packages directory). + List get expectedPaths => manifest.kind == 'python' + ? ['site-packages'] + : [ + for (final f in _files) + if (f.extract == null) f.name else ...f.extract!.map((m) => m.name), + ]; + + @override + bool isInstalled(Directory addonsDir) { + final dir = target(addonsDir); + final marker = File(path.join(dir.path, markerName)); + if (!marker.existsSync()) return false; + try { + final json = jsonDecode(marker.readAsStringSync()) as Map; + if (json['id'] != manifest.id || + json['version'] != manifest.version || + json['platform'] != platformId) { + return false; + } + } catch (_) { + return false; + } + return expectedPaths.every((p) => FileSystemEntity.typeSync(path.join(dir.path, p)) != + FileSystemEntityType.notFound); + } + + @override + Future install(Directory addonsDir, AddonManager downloader) async { + final reason = unavailableReason; + if (reason != null) throw StateError(reason); + final dest = target(addonsDir); + final staging = Directory('${dest.path}.new'); + final old = Directory('${dest.path}.old'); + // Downloads are kept beside the add-ons, not in temp, so a cancelled one + // resumes (AddonManager keeps the .partial), and are removed once installed. + final downloads = + Directory(path.join(addonsDir.path, '.downloads', '${manifest.id}-${manifest.version}')); + try { + await downloads.create(recursive: true); + final fetched = {}; + for (final f in _files) { + final file = File(path.join(downloads.path, f.name)); + await file.parent.create(recursive: true); + await _fetch(f, file, downloader); + if (downloader.isCancelled) return; + fetched[f] = file; + } + + if (await staging.exists()) await staging.delete(recursive: true); + await staging.create(recursive: true); + for (final MapEntry(key: f, value: file) in fetched.entries) { + if (manifest.kind == 'python') { + await extractWheel(file, Directory(path.join(staging.path, 'site-packages'))); + } else if (f.extract != null) { + await extractMembers(file, f.extract!, staging); + } else { + final out = File(path.join(staging.path, f.name)); + await out.parent.create(recursive: true); + await file.copy(out.path); + } + } + if (Platform.isMacOS) { + // Downloads made by the app are not quarantined today, but a mirror + // copy may be; a quarantined dylib is killed on load. + await Process.run('xattr', ['-cr', staging.path]); + } + for (final f in fetched.keys.where((f) => f.executable)) { + await prepareExecutable(File(path.join(staging.path, f.name))); + } + await File(path.join(staging.path, markerName)).writeAsString(jsonEncode({ + 'id': manifest.id, + 'version': manifest.version, + 'platform': platformId, + })); + + if (await old.exists()) await old.delete(recursive: true); + if (await dest.exists()) await dest.rename(old.path); + await staging.rename(dest.path); + if (await old.exists()) await old.delete(recursive: true); + await downloads.delete(recursive: true); + } finally { + if (await staging.exists()) await staging.delete(recursive: true); + } + } + + /// Make a downloaded program runnable: the executable bit (a download + /// has none), and on macOS a signature. Apple Silicon refuses to run an + /// unsigned arm64 binary at all, and upstream's Intel build is unsigned; + /// one that verifies (DeepFilterNet's arm64 build is linker-signed) is left + /// as it is. Quarantine was cleared with the rest of the install above. + static Future prepareExecutable(File file) async { + if (Platform.isWindows) return; + final chmod = await Process.run('chmod', ['755', file.path]); + if (chmod.exitCode != 0) { + throw StateError('Could not make ${path.basename(file.path)} executable: ${chmod.stderr}'); + } + if (!Platform.isMacOS) return; + final verify = await Process.run('codesign', ['--verify', file.path]); + if (verify.exitCode == 0) return; + final sign = await Process.run('codesign', ['--sign', '-', '--force', file.path]); + if (sign.exitCode != 0) { + throw StateError('Could not sign ${path.basename(file.path)}: ${sign.stderr}'); + } + } + + /// Get [f] into [file], from the mirror if one is set and holds it, else + /// the network; either way it must match its size and hash. + Future _fetch(AddonManifestFile f, File file, AddonManager downloader) async { + if (await file.exists() && await _matches(file, f)) return; + final mirrorDir = mirror; + final source = mirrorDir == null + ? null + : File(path.join(mirrorDir, Uri.parse(f.url).pathSegments.last)); + if (source != null && await source.exists()) { + await source.copy(file.path); + } else { + await downloader.downloadFile(f.url, file, expectedSize: f.size, expectedSha256: f.sha256); + if (downloader.isCancelled) return; + } + if (!await _matches(file, f)) { + final actual = await sha256OfFile(file); + final length = await file.length(); + await file.delete(); + throw StateError('${f.name} does not match the pinned size and hash ' + '(got $length bytes, sha256 $actual; expected ${f.size} bytes, sha256 ${f.sha256})'); + } + } + + static Future _matches(File file, AddonManifestFile f) async => + await file.length() == f.size && await sha256OfFile(file) == f.sha256; + + static Future sha256OfFile(File file) async => + (await sha256.bind(file.openRead()).first).toString(); + + /// A wheel's contents into [siteDir], as pip lays them out: `*.data/purelib` + /// and `*.data/platlib` go to the top, other `.data` directories (scripts, + /// headers) are left out. + static Future extractWheel(File wheel, Directory siteDir) async { + final input = InputFileStream(wheel.path); + try { + final archive = ZipDecoder().decodeBuffer(input); + for (final entry in archive.files) { + if (!entry.isFile) continue; + var rel = entry.name.replaceAll('\\', '/'); + final data = RegExp(r'^[^/]+\.data/([^/]+)/(.*)$').firstMatch(rel); + if (data != null) { + if (data.group(1) != 'purelib' && data.group(1) != 'platlib') continue; + rel = data.group(2)!; + } + final out = File(_safeJoin(siteDir.path, rel)); + await out.parent.create(recursive: true); + final stream = OutputFileStream(out.path); + entry.writeContent(stream); + await stream.close(); + } + } finally { + await input.close(); + } + } + + /// The listed [members] of an archive into [into], each checked against its + /// own hash; nothing else in the archive is written, and nothing at all if + /// one is missing or wrong. + static Future extractMembers( + File zip, List members, Directory into) async { + final input = InputFileStream(zip.path); + try { + final archive = ZipDecoder().decodeBuffer(input); + final found = >{}; + for (final m in members) { + final entry = archive.files.where((f) => f.isFile && f.name.replaceAll('\\', '/') == m.path); + if (entry.isEmpty) throw StateError('${m.path} is missing from ${path.basename(zip.path)}'); + final bytes = entry.first.content as List; + final actual = sha256.convert(bytes).toString(); + if (actual != m.sha256) { + throw StateError('${m.path} does not match its expected hash ($actual)'); + } + found[m] = bytes; + } + await into.create(recursive: true); + for (final MapEntry(key: m, value: bytes) in found.entries) { + final out = File(path.join(into.path, m.name)); + await out.parent.create(recursive: true); + await out.writeAsBytes(bytes); + } + } finally { + await input.close(); + } + } + + /// [rel] under [root], refusing a path that would land outside it. + static String _safeJoin(String root, String rel) { + final joined = path.normalize(path.join(root, rel)); + if (path.isAbsolute(rel) || !path.isWithin(root, joined)) { + throw StateError('Archive entry escapes its directory: $rel'); + } + return joined; + } + + @override + Future remove(Directory addonsDir) async { + final dir = target(addonsDir); + if (await dir.exists()) await dir.delete(recursive: true); + } +} diff --git a/app/lib/services/worker_manager.dart b/app/lib/services/worker_manager.dart index ecde1c15..fc456894 100644 --- a/app/lib/services/worker_manager.dart +++ b/app/lib/services/worker_manager.dart @@ -3,9 +3,12 @@ import 'dart:convert'; import 'dart:io'; +import '../models/ai_runtime_status.dart'; import '../models/progress_info.dart'; import '../models/gpu_plugin_status.dart'; +import '../models/crop_detect_report.dart'; import '../models/native_res_report.dart'; +import '../models/noise_halo_report.dart'; import '../models/video_job.dart'; import 'job_runner.dart'; import 'process_tree.dart'; @@ -189,7 +192,8 @@ class WorkerManager implements JobRunner { opencl: false, knlm: false, knlmMessage: 'VapourBox could not check for an OpenCL device, which ' - 'KNLMeans needs. BM3D does the same job on the CPU.', + 'KNLMeans needs. NLMeans (CPU) is the same filter run on the ' + 'processor.', ); final toolLocator = ToolLocator.instance; final workerPath = toolLocator.workerPath; @@ -267,6 +271,39 @@ class WorkerManager implements JobRunner { } } + /// Ask the worker whether the AI runtime add-on runs here, and on which + /// execution provider (`--probe-ai-runtime`). The first call after a boot + /// (or after installing the add-on) imports ONNX Runtime and runs a test + /// model, so it takes a few seconds; a positive answer is cached by the + /// worker. Never throws: an unreadable answer is [AiRuntimeStatus.unknown]. + static Future probeAiRuntime() async { + final toolLocator = ToolLocator.instance; + final workerPath = toolLocator.workerPath; + if (workerPath == null) return const AiRuntimeStatus.unknown(); + try { + final result = await Process.run( + workerPath, + ['--probe-ai-runtime'], + environment: toolLocator.workerEnvironment, + workingDirectory: File(workerPath).parent.path, + ); + if (result.exitCode != 0) return const AiRuntimeStatus.unknown(); + for (final line in const LineSplitter().convert(result.stdout.toString())) { + try { + final decoded = jsonDecode(line.trim()); + if (decoded is Map && decoded['available'] is bool) { + return AiRuntimeStatus.fromJson(decoded); + } + } catch (_) { + // Not the JSON line; keep scanning. + } + } + return const AiRuntimeStatus.unknown(); + } catch (_) { + return const AiRuntimeStatus.unknown(); + } + } + /// Find the resolution and kernel [job]'s video was upscaled from /// (`--detect-native`). Decodes a few frames and tries every height and the /// common kernels, so it takes tens of seconds on HD. Never throws: a failure @@ -304,6 +341,81 @@ class WorkerManager implements JobRunner { } } + /// Measure the black borders, blanking and head-switching band at the + /// edges of [job]'s video (`--detect-crop`, app 1.7.0). Decodes a couple of + /// dozen frames, so it takes a second or two. Never throws: a failure comes + /// back as [CropDetectReport.failed]. + static Future detectCrop(VideoJob job) async { + final toolLocator = ToolLocator.instance; + final workerPath = toolLocator.workerPath; + if (workerPath == null) { + return const CropDetectReport.failed('The VapourBox worker could not be found.'); + } + final config = File(await TempDirectoryService.instance + .filePath('detect_crop_${job.id}.json')); + try { + await config.writeAsString(jsonEncode(job.toJson())); + final result = await Process.run( + workerPath, + ['--detect-crop', '--config', config.path], + environment: toolLocator.workerEnvironment, + workingDirectory: File(workerPath).parent.path, + ); + for (final line in const LineSplitter().convert(result.stdout.toString()).reversed) { + final trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + return CropDetectReport.fromJson(jsonDecode(trimmed) as Map); + } catch (_) { + // Not the JSON line; keep scanning. + } + } + return CropDetectReport.failed('The analysis did not report a result: ${result.stderr}'); + } catch (e) { + return CropDetectReport.failed('$e'); + } finally { + if (await config.exists()) await config.delete(); + } + } + + /// Measure how noisy [job]'s video is and how strongly its edges ring + /// (`--detect-noise-halo`, app 1.7.0), for the pass list's suggestions. + /// Decodes a couple of dozen frame pairs, so it takes a few seconds; run in + /// the background. Never throws: a failure comes back as + /// [NoiseHaloReport.failed], which suggests nothing. + static Future detectNoiseHalo(VideoJob job) async { + final toolLocator = ToolLocator.instance; + final workerPath = toolLocator.workerPath; + if (workerPath == null) { + return const NoiseHaloReport.failed('The VapourBox worker could not be found.'); + } + final config = File(await TempDirectoryService.instance + .filePath('detect_noise_halo_${job.id}.json')); + try { + await config.writeAsString(jsonEncode(job.toJson())); + final result = await Process.run( + workerPath, + ['--detect-noise-halo', '--config', config.path], + environment: toolLocator.workerEnvironment, + workingDirectory: File(workerPath).parent.path, + ); + for (final line in const LineSplitter().convert(result.stdout.toString()).reversed) { + final trimmed = line.trim(); + if (!trimmed.startsWith('{')) continue; + try { + return NoiseHaloReport.fromJson(jsonDecode(trimmed) as Map); + } catch (_) { + // Not the JSON line; keep scanning. + } + } + return NoiseHaloReport.failed('The analysis did not report a result: ${result.stderr}'); + } catch (e) { + return NoiseHaloReport.failed('$e'); + } finally { + if (await config.exists()) await config.delete(); + } + } + /// How long to let the worker shut itself down before forcing it. /// /// This MUST comfortably exceed the worker's cancellation poll interval diff --git a/app/lib/viewmodels/main_viewmodel.dart b/app/lib/viewmodels/main_viewmodel.dart index 43dbac94..a308f160 100644 --- a/app/lib/viewmodels/main_viewmodel.dart +++ b/app/lib/viewmodels/main_viewmodel.dart @@ -6,7 +6,10 @@ import 'package:path/path.dart' as path; import '../models/dvd_info.dart'; import '../models/dynamic_parameters.dart'; import '../models/gpu_plugin_status.dart'; +import '../models/crop_detect_report.dart'; +import '../models/filter_registry.dart'; import '../models/native_res_report.dart'; +import '../models/noise_halo_report.dart'; import '../models/encoding_settings.dart'; import '../models/parameter_converter.dart'; import '../models/preview_frame_info.dart'; @@ -101,6 +104,17 @@ class MainViewModel extends ChangeNotifier { ({String itemId, NativeResReport report})? _nativeRes; String? _detectingNativeResFor; + // Crop & Resize's Detect (app 1.7.0), keyed the same way. + final Future Function(VideoJob job) _detectCrop; + ({String itemId, CropDetectReport report})? _cropReport; + String? _detectingCropFor; + + // How noisy each source is and how strongly its edges ring (app 1.7.0), + // measured in the background after a source is analysed, one source at a + // time in the order they were added; the result lives on its QueueItem. + final Future Function(VideoJob job) _detectNoiseHalo; + Future _noiseHaloQueue = Future.value(); + // Set in dispose() so the async OpenCL probe doesn't notifyListeners() after // teardown (the top-level view model normally lives for the app's lifetime). bool _disposed = false; @@ -304,6 +318,113 @@ class MainViewModel extends ChangeNotifier { ); } + /// What Crop & Resize's Detect found for the selected video, or null. + CropDetectReport? get cropReport => + _cropReport != null && _cropReport!.itemId == _selectedItemId ? _cropReport!.report : null; + + /// Whether Detect is measuring the selected video's edges. + bool get detectingCrop => _detectingCropFor != null && _detectingCropFor == _selectedItemId; + + /// Measure the black borders, blanking and head-switching band of the + /// selected video (its trimmed range if there is one). The job carries the + /// pipeline and field order, because where the crop lands depends on them: + /// after Rotate / Flip, and on a still-interlaced 4:2:0 picture in fours. + Future detectCrop() async { + final item = selectedItem; + if (item == null || _detectingCropFor != null) return; + _detectingCropFor = item.id; + notifyListeners(); + final info = item.videoInfo; + final frameCount = info?.frameCount ?? 0; + final fieldOrder = _getEffectiveFieldOrder(item); + final job = VideoJob( + inputPath: item.inputPath, + outputPath: item.outputPath, + processingPipeline: _processingPipeline.copyWith( + deinterlace: _processingPipeline.deinterlace.copyWith( + tff: fieldOrder == FieldOrder.topFieldFirst, + ), + ), + encodingSettings: _encodingSettings, + detectedFieldOrder: fieldOrder, + totalFrames: info?.frameCount, + inputFrameRate: info?.frameRate, + inputWidth: info?.width, + inputHeight: info?.height, + inputPixelFormat: info?.pixelFormat, + inputSar: info?.sar, + inputColorRange: info?.colorRange, + startFrame: item.inPoint == null ? null : (item.inPoint! * frameCount).round(), + endFrame: item.outPoint == null ? null : (item.outPoint! * frameCount).round(), + ); + final report = await _detectCrop(job); + if (_disposed) return; + _cropReport = (itemId: item.id, report: report); + _detectingCropFor = null; + notifyListeners(); + } + + /// The Crop sliders' maximum, from the schema, so an applied value is one + /// the control can show. + int _cropMax() { + final def = FilterRegistry.instance.get('crop_resize')?.parameters['cropLeft']; + return def?.max?.toInt() ?? 480; + } + + /// Put one of Detect's suggestions into Crop & Resize (and, for + /// [CropApply.repairBand], Edge Repair), switching the passes on. Only ever + /// called from the panel's buttons: nothing is applied on its own. + void applyCropDetection(CropApply mode) { + final report = cropReport; + if (report == null || report.error != null || !report.found) return; + final sides = switch (mode) { + CropApply.crop => report.crop, + CropApply.cropWithBars => report.cropWithBars, + CropApply.repairBand => report.cropWithRepair, + }; + if (sides == null) return; + final max = _cropMax(); + int clamp(int v) => v.clamp(0, max); + final current = getDynamicParams('crop_resize'); + updateDynamicParams( + 'crop_resize', + DynamicParameters( + filterId: 'crop_resize', + enabled: current.enabled || !sides.isZero, + values: { + ...current.values, + 'cropEnabled': !sides.isZero, + 'cropLeft': clamp(sides.left), + 'cropRight': clamp(sides.right), + 'cropTop': clamp(sides.top), + 'cropBottom': clamp(sides.bottom), + }, + enabledOptionalParams: current.enabledOptionalParams, + lastOptionalValues: current.lastOptionalValues, + ), + ); + final repair = report.edgeRepair; + if (mode == CropApply.repairBand && repair != null) { + final er = getDynamicParams('edge_repair'); + updateDynamicParams( + 'edge_repair', + DynamicParameters( + filterId: 'edge_repair', + enabled: true, + values: { + ...er.values, + if (repair.left > 0) 'left': repair.left, + if (repair.right > 0) 'right': repair.right, + if (repair.top > 0) 'top': repair.top, + if (repair.bottom > 0) 'bottom': repair.bottom, + }, + enabledOptionalParams: er.enabledOptionalParams, + lastOptionalValues: er.lastOptionalValues, + ), + ); + } + } + bool get canShutDown => _systemShutdown.isSupported; bool get shutdownWhenDone => _shutdownWhenDone; @@ -344,7 +465,8 @@ class MainViewModel extends ChangeNotifier { _regenerateOutputPath(); } notifyListeners(); - _requestPreviewUpdate(); + // The preview is picture only: nothing in audio restoration changes it. + if (filterId != 'audio_restoration') _requestPreviewUpdate(); } DynamicParameters _convertToParams(String filterId) { @@ -391,6 +513,12 @@ class MainViewModel extends ChangeNotifier { return ParameterConverter.fromChromaFixes(_processingPipeline.chromaFixes); case 'crop_resize': return ParameterConverter.fromCropResize(_processingPipeline.cropResize); + case 'face_restore': + return ParameterConverter.fromFaceRestore(_processingPipeline.faceRestore); + case 'colorize': + return ParameterConverter.fromColorize(_processingPipeline.colorize); + case 'audio_restoration': + return ParameterConverter.fromAudioRestoration(_processingPipeline.audio); case 'subtitles': return ParameterConverter.fromSubtitles(_processingPipeline.subtitles); default: @@ -476,6 +604,16 @@ class MainViewModel extends ChangeNotifier { descale: ParameterConverter.toDescale(params), ); break; + case 'face_restore': + _processingPipeline = _processingPipeline.copyWith( + faceRestore: ParameterConverter.toFaceRestore(params), + ); + break; + case 'colorize': + _processingPipeline = _processingPipeline.copyWith( + colorize: ParameterConverter.toColorize(params), + ); + break; case 'edge_repair': _processingPipeline = _processingPipeline.copyWith( edgeRepair: ParameterConverter.toEdgeRepair(params), @@ -517,6 +655,11 @@ class MainViewModel extends ChangeNotifier { cropResize: ParameterConverter.toCropResize(params), ); break; + case 'audio_restoration': + _processingPipeline = _processingPipeline.copyWith( + audio: ParameterConverter.toAudioRestoration(params), + ); + break; case 'subtitles': _processingPipeline = _processingPipeline.copyWith( subtitles: ParameterConverter.toSubtitles(params), @@ -635,9 +778,13 @@ class MainViewModel extends ChangeNotifier { JobRunner? runner, SystemShutdown? systemShutdown, Future Function(VideoJob job)? nativeResDetector, + Future Function(VideoJob job)? cropDetector, + Future Function(VideoJob job)? noiseHaloDetector, }) : _workerManager = runner ?? WorkerManager(), _systemShutdown = systemShutdown ?? PlatformSystemShutdown(), - _detectNative = nativeResDetector ?? WorkerManager.detectNativeResolution { + _detectNative = nativeResDetector ?? WorkerManager.detectNativeResolution, + _detectCrop = cropDetector ?? WorkerManager.detectCrop, + _detectNoiseHalo = noiseHaloDetector ?? WorkerManager.detectNoiseHalo { _setupSubscriptions(); _initializePreviewGenerator(); _probeGpuCapabilities(); @@ -848,6 +995,52 @@ class MainViewModel extends ChangeNotifier { if (_selectedItemId == item.id) { _requestPreviewUpdate(); } + + _measureNoiseHalo(_queue[index]); + } + + /// The selected source's noise / halo measurement, or null until it is in + /// (or when nothing is selected). Only the pass list's suggestions read it. + NoiseHaloReport? get noiseHaloReport => selectedItem?.noiseHalo; + + /// Measure [item]'s noise and halos in the background (app 1.7.0). Queued + /// behind any earlier measurement so a dropped folder doesn't start one + /// worker per file at once. Never blocks anything and never fails loudly: a + /// failed measurement is stored as such, and suggests nothing. + /// + /// The whole source is sampled, not its trim, so the result stays valid as + /// the in / out points move. Anything not known to be progressive is + /// measured field by field, so combing can't be read as noise. + void _measureNoiseHalo(QueueItem item) { + final info = item.videoInfo; + if (info == null || item.noiseHalo != null) return; + _noiseHaloQueue = _noiseHaloQueue.then((_) async { + if (_disposed || !_queue.any((q) => q.id == item.id)) return; + final progressive = info.scanType == ScanType.progressive || + info.scanType == ScanType.softTelecine; + final job = VideoJob( + inputPath: item.inputPath, + outputPath: item.outputPath, + encodingSettings: _encodingSettings, + detectedFieldOrder: + progressive ? FieldOrder.progressive : FieldOrder.topFieldFirst, + totalFrames: info.frameCount, + inputFrameRate: info.frameRate, + inputWidth: info.width, + inputHeight: info.height, + inputPixelFormat: info.pixelFormat, + inputColorRange: info.colorRange, + ); + NoiseHaloReport report; + try { + report = await _detectNoiseHalo(job); + } catch (e) { + report = NoiseHaloReport.failed('$e'); + } + if (_disposed) return; + item.noiseHalo = report; + notifyListeners(); + }); } /// Removes a video from the queue. @@ -1380,6 +1573,12 @@ class MainViewModel extends ChangeNotifier { return 'chroma_fixes'; case PassType.cropResize: return 'crop_resize'; + case PassType.faceRestore: + return 'face_restore'; + case PassType.colorize: + return 'colorize'; + case PassType.audioRestoration: + return 'audio_restoration'; case PassType.subtitles: return 'subtitles'; } @@ -1482,8 +1681,11 @@ class MainViewModel extends ChangeNotifier { } // Subtitle-only mode: no video processing passes enabled, only subtitles + // Audio restoration needs the full job (it renders the output's + // soundtrack), so it rules out subtitle-only mode. final isSubtitleOnly = _processingPipeline.subtitles.enabled && - _processingPipeline.enabledPasses.isEmpty; + _processingPipeline.enabledPasses.isEmpty && + !_processingPipeline.audio.hasEffect; _lastJobSubtitleOnly = isSubtitleOnly && _processingPipeline.subtitles.output == SubtitleOutput.srtFile; @@ -1855,7 +2057,8 @@ class MainViewModel extends ChangeNotifier { // Subtitle-only with SRT output: use .srt extension final isSubtitleOnly = _processingPipeline.subtitles.enabled && - _processingPipeline.videoPassCount == 0; + _processingPipeline.videoPassCount == 0 && + !_processingPipeline.audio.hasEffect; if (isSubtitleOnly && _processingPipeline.subtitles.output == SubtitleOutput.srtFile) { return path.join(outputDir, '$outputFilename.srt'); } diff --git a/app/lib/views/about_dialog.dart b/app/lib/views/about_dialog.dart index 70796274..f94fe40a 100644 --- a/app/lib/views/about_dialog.dart +++ b/app/lib/views/about_dialog.dart @@ -182,6 +182,14 @@ class _AboutDialogState extends State { copyright: 'mawen1250', url: 'https://github.com/HomeOfVapourSynthEvolution/mvsfunc', ), + _ComponentTile( + name: 'vs-jetpack (ported code)', + license: 'MIT', + copyright: 'Jaded-Encoding-Thaumaturgy; ' + 'FixInterlacedFades, vine_dehalo and mpeg2stinx ' + 'ported into VapourBox\'s templates', + url: 'https://github.com/Jaded-Encoding-Thaumaturgy/vs-jetpack', + ), _ComponentTile( name: 'adjust', license: 'WTFPL', @@ -266,6 +274,12 @@ class _AboutDialogState extends State { 'vapoursynth-zig (LGPL 2.1)', url: 'https://github.com/dnjulek/vapoursynth-zip', ), + _ComponentTile( + name: 'SangNom', + license: 'MIT source; binaries GPL 2.0', + copyright: 'james1201; Victor Efimov', + url: 'https://github.com/dubhatervapoursynth/vapoursynth-sangnom', + ), _ComponentTile( name: 'RIFE (VapourSynth-RIFE-ncnn-Vulkan)', license: 'MIT', @@ -275,7 +289,7 @@ class _AboutDialogState extends State { url: 'https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan', ), _ComponentTile( - name: 'RIFE 4.26 model', + name: 'RIFE 4.26 model (optional download, frame rate add-on)', license: 'MIT', copyright: 'hzwer; architecture by Megvii Inc.', url: 'https://github.com/hzwer/Practical-RIFE', @@ -296,8 +310,8 @@ class _AboutDialogState extends State { url: 'https://github.com/AmusementClub/vs-mlrt', ), _ComponentTile( - name: 'Real-ESRGAN anime video model', - license: 'BSD 3-Clause', + name: 'Real-ESRGAN anime video model (optional download, upscaling add-on)', + license: 'BSD-3-Clause', copyright: 'Xintao Wang', url: 'https://github.com/xinntao/Real-ESRGAN', ), @@ -308,7 +322,73 @@ class _AboutDialogState extends State { url: 'https://github.com/the-database/mpv-upscale-2x_animejanai', ), _ComponentTile( - name: 'Real-CUGAN models', + name: 'RestoreFormer++ model (optional download, face restoration add-on)', + license: 'Apache-2.0', + copyright: 'Zhouxia Wang, Jiawei Zhang, Tianshui Chen, Wenping Wang and Ping Luo', + url: 'https://github.com/wzhouxiff/RestoreFormerPlusPlus', + ), + _ComponentTile( + name: 'RetinaFace face detector (optional download, face restoration add-on)', + license: 'MIT', + copyright: 'biubug6 (Pytorch_Retinaface)', + url: 'https://github.com/biubug6/Pytorch_Retinaface', + ), + _ComponentTile( + name: 'BiSeNet face parser (optional download, face restoration add-on)', + license: 'MIT', + copyright: 'zll (face-parsing.PyTorch)', + url: 'https://github.com/zllrunning/face-parsing.PyTorch', + ), + _ComponentTile( + name: 'facexlib (optional download, face restoration add-on)', + license: 'MIT', + copyright: 'Xintao Wang', + url: 'https://github.com/xinntao/facexlib', + ), + _ComponentTile( + name: 'GFPGAN v1.4 model (optional download, non-commercial use only)', + license: 'Apache-2.0 + NVIDIA Source Code License (NC)', + copyright: 'THL A29 Limited, a Tencent company', + url: 'https://github.com/TencentARC/GFPGAN', + ), + _ComponentTile( + name: 'CodeFormer model (optional download, non-commercial use only)', + license: 'S-Lab License 1.0 (NC)', + copyright: 'S-Lab', + url: 'https://github.com/sczhou/CodeFormer', + ), + _ComponentTile( + name: 'DDColor model (optional download, colourisation add-on)', + license: 'Apache-2.0', + copyright: 'Xiaoyang Kang, Tao Yang, Wenqi Ouyang, Peiran Ren, Lingzhi Li and Xuansong Xie', + url: 'https://github.com/piddnad/DDColor', + ), + _ComponentTile( + name: 'DeOldify Video model (optional download, colourisation add-on)', + license: 'MIT', + copyright: 'Jason Antic', + url: 'https://github.com/jantic/DeOldify', + ), + _ComponentTile( + name: 'Real-ESRGAN general model (optional download, upscaling add-on)', + license: 'BSD-3-Clause', + copyright: 'Xintao Wang', + url: 'https://github.com/xinntao/Real-ESRGAN', + ), + _ComponentTile( + name: '2xLiveActionV1_SPAN model (optional download, upscaling add-on)', + license: 'Apache-2.0', + copyright: 'Copyright 2025 jcj83429', + url: 'https://github.com/jcj83429/upscaling/tree/main/2xLiveActionV1_SPAN', + ), + _ComponentTile( + name: 'Real-ESRGAN x4plus / x2plus models (optional download, upscaling add-on)', + license: 'BSD-3-Clause', + copyright: 'Xintao Wang', + url: 'https://github.com/xinntao/Real-ESRGAN', + ), + _ComponentTile( + name: 'Real-CUGAN models (optional download, upscaling add-on)', license: 'MIT', copyright: 'bilibili', url: 'https://github.com/bilibili/ailab/tree/main/Real-CUGAN', @@ -493,6 +573,30 @@ class _AboutDialogState extends State { copyright: 'OpenAI', url: 'https://github.com/openai/whisper', ), + _ComponentTile( + name: 'ONNX Runtime (optional download, AI runtime add-on)', + license: 'MIT', + copyright: 'Microsoft Corporation', + url: 'https://github.com/microsoft/onnxruntime', + ), + _ComponentTile( + name: 'NumPy (optional download, AI runtime add-on)', + license: 'BSD-3-Clause', + copyright: 'NumPy Developers', + url: 'https://github.com/numpy/numpy', + ), + _ComponentTile( + name: 'DirectML (optional download, AI runtime add-on, Windows)', + license: 'Microsoft Software License Terms', + copyright: 'Microsoft Corporation', + url: 'https://github.com/microsoft/DirectML', + ), + _ComponentTile( + name: 'DeepFilterNet (optional download, speech enhancement add-on)', + license: 'MIT OR Apache-2.0', + copyright: 'Hendrik Schröter', + url: 'https://github.com/Rikorose/DeepFilterNet', + ), _ComponentTile( name: 'libdvdread', license: 'GPL 2.0+', diff --git a/app/lib/views/addon_dialogs.dart b/app/lib/views/addon_dialogs.dart index 7d6ae214..aafbc3e3 100644 --- a/app/lib/views/addon_dialogs.dart +++ b/app/lib/views/addon_dialogs.dart @@ -69,6 +69,19 @@ class AddonTerms extends StatelessWidget { _Link(text: 'Source', url: info.sourceUrl, style: small), ], ), + // Components under their own licence (the AI runtime's NumPy, and + // on Windows DirectML), each with its own credit and links. + for (final c in info.additionalCredits) + Wrap( + crossAxisAlignment: WrapCrossAlignment.center, + spacing: 4, + children: [ + Text('Includes ${c.noticeName}: ${c.credit} · ', style: small), + _Link(text: c.licence, url: c.licenceUrl, style: small), + Text('·', style: small), + _Link(text: 'Source', url: c.sourceUrl, style: small), + ], + ), ], ); } diff --git a/app/lib/views/pass_list/pass_list_item.dart b/app/lib/views/pass_list/pass_list_item.dart index 7c1f4412..1332dad6 100644 --- a/app/lib/views/pass_list/pass_list_item.dart +++ b/app/lib/views/pass_list/pass_list_item.dart @@ -263,6 +263,12 @@ class PassListItem extends StatelessWidget { return Icons.tune; case PassType.cropResize: return Icons.crop; + case PassType.faceRestore: + return Icons.face_retouching_natural; + case PassType.colorize: + return Icons.colorize; + case PassType.audioRestoration: + return Icons.graphic_eq; case PassType.subtitles: return Icons.subtitles; } diff --git a/app/lib/views/pass_list/pass_list_panel.dart b/app/lib/views/pass_list/pass_list_panel.dart index d1aaf80b..61c75d74 100644 --- a/app/lib/views/pass_list/pass_list_panel.dart +++ b/app/lib/views/pass_list/pass_list_panel.dart @@ -1,7 +1,10 @@ import 'package:flutter/material.dart'; import 'package:provider/provider.dart'; +import '../../models/colorize_parameters.dart'; +import '../../models/face_restore_parameters.dart'; import '../../models/filter_registry.dart'; +import '../../models/gpu_plugin_status.dart'; import '../../models/pass_relevance.dart'; import '../../models/processing_pipeline.dart'; import '../../services/whats_new_service.dart'; @@ -60,6 +63,7 @@ class PassListPanel extends StatelessWidget { passes: [ PassType.antiAlias, PassType.chromaFixes, + PassType.colorize, PassType.colorCorrection, ], ), @@ -69,11 +73,19 @@ class PassListPanel extends StatelessWidget { ), ( title: 'Finishing', - passes: [PassType.sharpen, PassType.grain, PassType.frameRate], + passes: [ + PassType.faceRestore, + PassType.sharpen, + PassType.grain, + PassType.frameRate, + ], ), + // Neither is a picture pass: the soundtrack is rendered, then (when + // subtitles are on) transcribed, both before the encode. Restoration + // comes first because the transcript is of the restored sound. ( - title: 'Post-Processing', - passes: [PassType.subtitles], + title: 'Audio & Subtitles', + passes: [PassType.audioRestoration, PassType.subtitles], ), ]; @@ -100,7 +112,8 @@ class PassListPanel extends StatelessWidget { subtitle: subtitle, isEnabled: isEnabled, isExpanded: isExpanded, - relevance: relevanceFor(passType, viewModel.videoInfo), + relevance: relevanceFor(passType, viewModel.videoInfo, + measured: viewModel.noiseHaloReport), isNew: WhatsNewService.instance.isSchemaNew(schema), onToggle: onToggle ?? (enabled) => viewModel.togglePass(passType, enabled), onTap: () => viewModel.selectPass(passType), @@ -153,7 +166,9 @@ class PassListPanel extends StatelessWidget { pipeline.grain.enabled); case PassType.frameRate: return item(passType, 'Frame Rate', pipeline.frameRate.summary, - pipeline.frameRate.enabled); + pipeline.frameRate.enabled, + onToggle: (enabled) => + _handleModelAddonToggle(context, viewModel, passType, enabled)); case PassType.deflicker: return item(passType, 'Deflicker', pipeline.deflicker.summary, pipeline.deflicker.enabled); @@ -175,7 +190,30 @@ class PassListPanel extends StatelessWidget { pipeline.colorCorrection.enabled); case PassType.cropResize: return item(passType, 'Crop / Resize', pipeline.cropResize.summary, - pipeline.cropResize.enabled); + pipeline.cropResize.enabled, + onToggle: (enabled) => + _handleModelAddonToggle(context, viewModel, passType, enabled)); + case PassType.faceRestore: + return item( + passType, + 'Face Restoration', + pipeline.faceRestore.summary, + pipeline.faceRestore.enabled, + onToggle: (enabled) => + _handleFaceRestoreToggle(context, viewModel, enabled), + ); + case PassType.colorize: + return item( + passType, + 'Colourisation', + pipeline.colorize.summary, + pipeline.colorize.enabled, + onToggle: (enabled) => + _handleColorizeToggle(context, viewModel, enabled), + ); + case PassType.audioRestoration: + return item(passType, 'Audio Restoration', pipeline.audio.summary, + pipeline.audio.enabled); case PassType.subtitles: return item( passType, @@ -284,6 +322,70 @@ class PassListPanel extends StatelessWidget { viewModel.togglePass(PassType.subtitles, enabled); } + /// Frame Rate with RIFE, and Crop & Resize with an AI upscaler, run models + /// that are add-ons: switching the pass on with such a method chosen offers + /// the download, as choosing the method does. That is also how a pipeline + /// saved before the models became add-ons (deps 1.16.0) gets them. Declining + /// leaves the pass off. + Future _handleModelAddonToggle( + BuildContext context, + MainViewModel viewModel, + PassType passType, + bool enabled, + ) async { + if (enabled) { + final filterId = PassSettingsInline.filterIdFor(passType); + final addon = modelAddonSelectedBy(filterId, viewModel.getDynamicParams(filterId).values); + if (addon != null) { + final ready = await AddonPrompt.ensure( + context, + [addon], + reason: upscaleAddonReason(addon), + ); + if (!ready) return; + } + } + viewModel.togglePass(passType, enabled); + } + + /// Face Restoration's models are add-ons: switching the pass on offers to + /// download the ones its chosen model needs (with their licence terms; the + /// GFPGAN and CodeFormer ones are non-commercial). Declining leaves it off. + Future _handleFaceRestoreToggle( + BuildContext context, + MainViewModel viewModel, + bool enabled, + ) async { + if (enabled) { + final ready = await AddonPrompt.ensure( + context, + viewModel.processingPipeline.faceRestore.addonIds, + reason: faceRestoreAddonReason, + ); + if (!ready) return; + } + viewModel.togglePass(PassType.faceRestore, enabled); + } + + /// Colourisation's models are add-ons: switching the pass on offers to + /// download the chosen model (and the AI runtime it runs on). Declining + /// leaves it off. + Future _handleColorizeToggle( + BuildContext context, + MainViewModel viewModel, + bool enabled, + ) async { + if (enabled) { + final ready = await AddonPrompt.ensure( + context, + [viewModel.processingPipeline.colorize.addonId], + reason: colorizeAddonReason, + ); + if (!ready) return; + } + viewModel.togglePass(PassType.colorize, enabled); + } + String _getPassSummary(ProcessingPipeline pipeline) { final videoCount = pipeline.videoPassCount; final hasSubtitles = pipeline.subtitles.enabled; diff --git a/app/lib/views/pass_settings/crop_detect_panel.dart b/app/lib/views/pass_settings/crop_detect_panel.dart new file mode 100644 index 00000000..684571a9 --- /dev/null +++ b/app/lib/views/pass_settings/crop_detect_panel.dart @@ -0,0 +1,145 @@ +import 'package:flutter/material.dart'; + +import '../../models/crop_detect_report.dart'; +import '../../services/whats_new_service.dart'; +import '../../viewmodels/main_viewmodel.dart'; +import '../../widgets/warning_banner.dart'; + +/// Crop & Resize's Detect (app 1.7.0): measures the selected video's black +/// borders, blanking and head-switching band and offers what it found. Like +/// Descale's Detect it is advisory: the controls change only when the user +/// clicks one of the Apply buttons, and a letterbox bar is offered separately +/// because cropping the picture's own framing is the user's choice. +class CropDetectPanel extends StatelessWidget { + final MainViewModel viewModel; + + const CropDetectPanel({super.key, required this.viewModel}); + + /// The release Detect shipped in, for its NEW badge. + static const sinceAppVersion = '1.7.0'; + + @override + Widget build(BuildContext context) { + final theme = Theme.of(context); + final report = viewModel.cropReport; + final running = viewModel.detectingCrop; + final canRun = viewModel.selectedItem != null && !running; + final scheme = theme.colorScheme; + final children = [ + Row( + children: [ + Badge( + isLabelVisible: WhatsNewService.instance.isNew(sinceAppVersion), + backgroundColor: scheme.tertiaryContainer, + textColor: scheme.onTertiaryContainer, + textStyle: const TextStyle(fontSize: 9, fontWeight: FontWeight.w700, letterSpacing: 0.4), + offset: const Offset(10, -8), + label: const Text('NEW'), + child: OutlinedButton.icon( + onPressed: canRun ? viewModel.detectCrop : null, + icon: running + ? const SizedBox.square( + dimension: 14, child: CircularProgressIndicator(strokeWidth: 2)) + : const Icon(Icons.crop_free, size: 16), + label: Text(running ? 'Detecting…' : 'Detect borders'), + ), + ), + const SizedBox(width: 12), + Expanded( + child: Text( + running + ? 'Measuring the edges of frames from across this video.' + : viewModel.selectedItem == null + ? 'Select a video to analyse.' + : 'Finds black borders, tape blanking and head-switching noise.', + style: theme.textTheme.bodySmall, + ), + ), + ], + ), + ]; + if (!running && report != null) { + children.add(_result(context, report)); + } + return Padding( + padding: const EdgeInsets.only(bottom: 12), + child: Column(crossAxisAlignment: CrossAxisAlignment.stretch, children: children), + ); + } + + Widget _result(BuildContext context, CropDetectReport report) { + final theme = Theme.of(context); + if (report.error != null) { + return WarningBanner(message: 'Detection failed: ${report.error}'); + } + if (!report.found) { + return Padding( + padding: const EdgeInsets.only(top: 8), + child: Text( + report.message ?? 'Nothing to crop at the edges.', + style: theme.textTheme.bodySmall, + ), + ); + } + final c = report.crop; + final lines = [ + for (final (side, what) in report.findings) + Text('$side: $what', style: theme.textTheme.bodySmall), + const SizedBox(height: 4), + SelectableText( + c.isZero + ? 'Nothing to crop apart from the bars.' + : 'Crop left ${c.left}, right ${c.right}, top ${c.top}, bottom ${c.bottom}.', + ), + ]; + if (report.isLowConfidence) { + lines.add(WarningBanner( + message: 'Hard to judge: in only ${(report.confidence * 100).round()}% of the ' + 'sampled frames does the picture reach an edge that was found; in the ' + 'rest it is dark there too. Check the preview before applying.', + )); + } + final buttons = [ + if (!c.isZero) + FilledButton( + onPressed: () => viewModel.applyCropDetection(CropApply.crop), + child: const Text('Apply crop'), + ), + if (report.cropWithBars != null) + OutlinedButton( + onPressed: () => viewModel.applyCropDetection(CropApply.cropWithBars), + child: const Text('Crop the bars too'), + ), + if (report.edgeRepair != null) + OutlinedButton( + onPressed: () => viewModel.applyCropDetection(CropApply.repairBand), + child: const Text('Repair the band instead'), + ), + ]; + if (report.cropWithBars != null) { + lines.add(Text( + 'The bars are part of the picture\'s framing (a widescreen film in a ' + '4:3 frame, or 4:3 in HD): they are only cropped if you choose to.', + style: theme.textTheme.bodySmall, + )); + } + if (report.edgeRepair != null) { + lines.add(Text( + 'Repairing the head-switching band with Edge Repair keeps the frame ' + 'height (for DVD authoring, say) instead of cropping it.', + style: theme.textTheme.bodySmall, + )); + } + return Padding( + padding: const EdgeInsets.only(top: 8), + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + ...lines, + const SizedBox(height: 8), + Wrap(spacing: 8, runSpacing: 8, children: buttons), + ], + ), + ); + } +} diff --git a/app/lib/views/pass_settings/pass_settings_inline.dart b/app/lib/views/pass_settings/pass_settings_inline.dart index d9d963ca..cc76cd54 100644 --- a/app/lib/views/pass_settings/pass_settings_inline.dart +++ b/app/lib/views/pass_settings/pass_settings_inline.dart @@ -2,6 +2,9 @@ import 'package:flutter/material.dart'; import 'package:provider/provider.dart'; import '../../models/dynamic_parameters.dart'; +import '../../models/colorize_parameters.dart'; +import '../../models/face_restore_parameters.dart'; +import '../../models/audio_restoration_parameters.dart'; import '../../models/filter_registry.dart'; import '../../models/filter_schema.dart'; import '../../models/pass_advice.dart'; @@ -13,6 +16,7 @@ import '../../utils/pixel_format.dart'; import '../../viewmodels/main_viewmodel.dart'; import '../../widgets/warning_banner.dart'; import '../settings/dynamic_filter_panel.dart'; +import 'crop_detect_panel.dart'; import '../addon_dialogs.dart'; /// The settings for one processing pass, rendered inline underneath that pass's @@ -74,6 +78,12 @@ class PassSettingsInline extends StatelessWidget { return 'chroma_fixes'; case PassType.cropResize: return 'crop_resize'; + case PassType.faceRestore: + return 'face_restore'; + case PassType.colorize: + return 'colorize'; + case PassType.audioRestoration: + return 'audio_restoration'; case PassType.subtitles: return 'subtitles'; } @@ -103,8 +113,20 @@ class PassSettingsInline extends StatelessWidget { _buildGpuPluginNotice(context, viewModel, filterId, params), _buildKnlmNotice(viewModel, filterId, params), if (animeJanaiSelectedBy(filterId, params.values)) - const AnimeJanaiNotice(), + const AnimeJanaiNotice() + else if (modelAddonSelectedBy(filterId, params.values) case final id?) + UpscaleAddonNotice(addonId: id, reason: upscaleAddonReason(id)), + if (filterId == 'face_restore') + FaceRestoreNotice( + model: FaceRestoreModel.fromValue(params.values['model'] as String?), + ), + if (filterId == 'colorize') + UpscaleAddonNotice( + addonId: ColorizeModel.fromValue(params.values['method'] as String?).addonId, + reason: colorizeAddonReason, + ), if (filterId == 'descale') _buildDescaleDetect(context, viewModel), + if (filterId == 'crop_resize') CropDetectPanel(viewModel: viewModel), _buildBitDepthWarning(context, viewModel, schema, params), DynamicFilterPanelCompact( schema: schema, @@ -364,28 +386,62 @@ class PassSettingsInline extends StatelessWidget { if (!ready) return; } - // AnimeJaNai's models are an add-on: choosing it downloads them, after - // saying what they are and that they are for non-commercial use only. - if (_choosesAnimeJanai(filterId, oldParams, newParams)) { + // Every AI upscaler's and RIFE's models are add-ons: choosing one + // downloads its models, after saying what they are, their licence and + // size (AnimeJaNai's: non-commercial use only). Moving between two + // methods of one add-on asks once. + final addon = modelAddonSelectedBy(filterId, newParams.values); + if (addon != null && addon != modelAddonSelectedBy(filterId, oldParams.values)) { if (!context.mounted) return; final ready = await AddonPrompt.ensure( context, - [animeJanaiAddonId], - reason: 'AnimeJaNai upscaling needs its models.', + [addon], + reason: upscaleAddonReason(addon), + ); + if (!ready) return; + } + // Face Restoration's models are add-ons: switching it on, or choosing + // another model while it is on, offers to download what that model needs + // (with its terms: GFPGAN and CodeFormer are non-commercial). + final faceAddons = + faceRestoreAddonsSelectedBy(filterId, newParams.enabled, newParams.values); + if (faceAddons != null && + (!oldParams.enabled || oldParams.values['model'] != newParams.values['model'])) { + if (!context.mounted) return; + final ready = await AddonPrompt.ensure( + context, + faceAddons, + reason: faceRestoreAddonReason, + ); + if (!ready) return; + } + // Colourisation's models are add-ons: switching it on, or choosing the + // other model while it is on, offers that model's download. + final colorizeAddon = colorizeAddonSelectedBy(filterId, newParams.enabled, newParams.values); + if (colorizeAddon != null && + (!oldParams.enabled || oldParams.values['method'] != newParams.values['method'])) { + if (!context.mounted) return; + final ready = await AddonPrompt.ensure( + context, + [colorizeAddon], + reason: colorizeAddonReason, + ); + if (!ready) return; + } + // DeepFilterNet is an add-on too: choosing speech enhancement offers it. + if (deepFilterChosen(filterId, oldParams.values, newParams.values) && + newParams.enabled) { + if (!context.mounted) return; + final ready = await AddonPrompt.ensure( + context, + [deepFilterNetAddonId], + reason: 'Speech enhancement needs DeepFilterNet.', ); if (!ready) return; } viewModel.updateDynamicParams(filterId, newParams); } - bool _choosesAnimeJanai( - String filterId, - DynamicParameters oldParams, - DynamicParameters newParams, - ) => - animeJanaiSelectedBy(filterId, newParams.values) && - !animeJanaiSelectedBy(filterId, oldParams.values); - Widget _buildFallbackPanel(BuildContext context, PassType passType) { return Padding( padding: const EdgeInsets.symmetric(vertical: 16), @@ -543,19 +599,53 @@ class _FilterDescriptionState extends State<_FilterDescription> { } } +/// Why an AI upscale method (or RIFE) asks for its add-on, in +/// [AddonPrompt.ensure]. +String upscaleAddonReason(String addonId) => switch (addonId) { + animeJanaiAddonId => 'AnimeJaNai upscaling needs its models.', + rifeAddonId => 'AI interpolation (RIFE) needs its model.', + 'upscale-anime' => 'Real-ESRGAN and Real-CUGAN upscaling need their models.', + 'upscale-esrgan-plus' => 'Real-ESRGAN x4plus upscaling needs its models.', + _ => 'Real-ESRGAN General and Live Action SPAN upscaling need their models.', + }; + /// Under Crop & Resize while AnimeJaNai is chosen: its terms, always — the /// non-commercial restriction is the user's to keep in mind on every job — /// and, if its models are not installed (deleted since, or chosen by a /// preset), a button to download them. -class AnimeJanaiNotice extends StatefulWidget { +class AnimeJanaiNotice extends UpscaleAddonNotice { + const AnimeJanaiNotice({super.key, super.service}) + : super( + addonId: animeJanaiAddonId, + reason: 'AnimeJaNai upscaling needs its models.', + label: 'AnimeJaNai', + ); +} + +/// Under Crop & Resize while an add-on upscaler is chosen: the add-on's terms +/// and, if its models are not installed (deleted since, or chosen by a +/// preset), a button to download them. +class UpscaleAddonNotice extends StatefulWidget { + final String addonId; + final String reason; + + /// Names the models in the status line ("AnimeJaNai models installed."); + /// without it the add-on's own name is used. + final String? label; final AddonService? service; - const AnimeJanaiNotice({super.key, this.service}); + const UpscaleAddonNotice({ + super.key, + required this.addonId, + required this.reason, + this.label, + this.service, + }); @override - State createState() => _AnimeJanaiNoticeState(); + State createState() => _UpscaleAddonNoticeState(); } -class _AnimeJanaiNoticeState extends State { +class _UpscaleAddonNoticeState extends State { AddonService get _service => widget.service ?? AddonService.instance; @override @@ -569,9 +659,11 @@ class _AnimeJanaiNoticeState extends State { return ListenableBuilder( listenable: _service, builder: (context, _) { - final info = _service.info(animeJanaiAddonId); + final id = widget.addonId; + final info = _service.info(id); if (info == null) return const SizedBox.shrink(); - final installed = _service.isInstalled(animeJanaiAddonId); + final installed = _service.isInstalled(id); + final label = widget.label; return Padding( padding: const EdgeInsets.only(bottom: 12), child: Row( @@ -581,9 +673,13 @@ class _AnimeJanaiNoticeState extends State { crossAxisAlignment: CrossAxisAlignment.start, children: [ Text( - installed - ? 'AnimeJaNai models installed.' - : 'AnimeJaNai\'s models are not installed.', + label != null + ? (installed + ? '$label models installed.' + : '$label\'s models are not installed.') + : (installed + ? '${info.name}: installed.' + : '${info.name}: not installed.'), style: Theme.of(context).textTheme.bodySmall, ), AddonTerms(info: info), @@ -594,9 +690,8 @@ class _AnimeJanaiNoticeState extends State { OutlinedButton.icon( onPressed: _service.installingId != null ? null - : () => AddonPrompt.ensure(context, [animeJanaiAddonId], - reason: 'AnimeJaNai upscaling needs its models.', - service: _service), + : () => AddonPrompt.ensure(context, [id], + reason: widget.reason, service: _service), icon: const Icon(Icons.download, size: 16), label: Text('Download (${formatAddonSize(info.downloadBytes)})'), ), @@ -607,3 +702,76 @@ class _AnimeJanaiNoticeState extends State { ); } } + +/// Under Face Restoration: whether the chosen model's add-ons are installed +/// (with a button to download them if not — deleted since, or chosen by a +/// preset), and the model add-on's terms. GFPGAN's and CodeFormer's +/// non-commercial restriction is shown in bold whenever either is chosen, +/// since it is the user's to keep in mind on every job. Advisory only: the +/// worker refuses a job whose add-ons are missing. +class FaceRestoreNotice extends StatefulWidget { + final FaceRestoreModel model; + final AddonService? service; + const FaceRestoreNotice({super.key, required this.model, this.service}); + + @override + State createState() => _FaceRestoreNoticeState(); +} + +class _FaceRestoreNoticeState extends State { + AddonService get _service => widget.service ?? AddonService.instance; + + @override + void initState() { + super.initState(); + _service.load(); + } + + @override + Widget build(BuildContext context) { + return ListenableBuilder( + listenable: _service, + builder: (context, _) { + final ids = faceRestoreAddonIds(widget.model); + final info = _service.info(widget.model.addonId); + if (info == null) return const SizedBox.shrink(); + final missing = _service.missing(ids); + final installed = missing.isEmpty; + final bytes = missing + .map(_service.info) + .whereType() + .fold(0, (sum, i) => sum + i.downloadBytes); + return Padding( + padding: const EdgeInsets.only(bottom: 12), + child: Row( + children: [ + Expanded( + child: Column( + crossAxisAlignment: CrossAxisAlignment.start, + children: [ + Text( + installed + ? '${widget.model.displayName} is installed.' + : 'The ${widget.model.displayName} add-on is not installed.', + style: Theme.of(context).textTheme.bodySmall, + ), + AddonTerms(info: info), + ], + ), + ), + if (!installed) + OutlinedButton.icon( + onPressed: _service.installingId != null + ? null + : () => AddonPrompt.ensure(context, ids, + reason: faceRestoreAddonReason, service: _service), + icon: const Icon(Icons.download, size: 16), + label: Text('Download (${formatAddonSize(bytes)})'), + ), + ], + ), + ); + }, + ); + } +} diff --git a/app/lib/views/progress_panel.dart b/app/lib/views/progress_panel.dart index 020d3d5d..3104ec2a 100644 --- a/app/lib/views/progress_panel.dart +++ b/app/lib/views/progress_panel.dart @@ -242,6 +242,7 @@ class ProgressPanel extends StatelessWidget { if (progress.isExtractingPhase) return 'Extracting DVD Title'; if (progress.isSubtitlePhase) return 'Generating Subtitles'; if (progress.isEmbeddingPhase) return 'Embedding Subtitles'; + if (progress.isAudioPhase) return 'Restoring Audio'; } return 'Processing'; case ProcessingState.cancelling: @@ -284,6 +285,23 @@ class ProgressPanel extends StatelessWidget { ); } + // Audio restoration: a percentage, or just the label while DeepFilterNet + // (which reports nothing) runs. + if (progress.isAudioPhase) { + return Row( + mainAxisAlignment: MainAxisAlignment.center, + children: [ + _buildDetailChip( + context, + icon: Icons.graphic_eq, + value: progress.isIndeterminate + ? 'Restoring audio...' + : 'Restoring audio... ${progress.percentComplete}%', + ), + ], + ); + } + // Embedding phase: show text only (indeterminate) if (progress.isEmbeddingPhase) { return Row( diff --git a/app/macos/Runner/Info.plist b/app/macos/Runner/Info.plist index b19806b8..3b2fa70a 100644 --- a/app/macos/Runner/Info.plist +++ b/app/macos/Runner/Info.plist @@ -17,9 +17,9 @@ CFBundlePackageType APPL CFBundleShortVersionString - 1.6.0 + 1.7.0 CFBundleVersion - 1.6.0 + 1.7.0 [^\n]*\n([\s\S]*)').firstMatch(block); + if (m == null) continue; + final start = int.parse(m.group(1)!) * 3600 + int.parse(m.group(2)!) * 60 + + int.parse(m.group(3)!) + int.parse(m.group(4)!) / 1000; + final text = m.group(5)!.toLowerCase().replaceAll(RegExp(r'[^a-z]'), ''); + if (text.length >= 8) byText.putIfAbsent(text, () => []).add(start); + } + return {for (final e in byText.entries) if (e.value.length == 1) e.key: e.value.single}; + } + + final plain = await cues(0); + final delayed = await cues(2000); + final shifts = [ + for (final text in plain.keys) + if (delayed.containsKey(text)) delayed[text]! - plain[text]!, + ]..sort(); + // Whisper segments the two runs differently and its timestamps are + // coarse, so the cues it transcribed identically are compared, by their + // median shift, to the 2 s the sound was moved. + expect(shifts.length, greaterThanOrEqualTo(2), + reason: 'too few cues transcribed alike to compare: $plain / $delayed'); + expect(shifts[shifts.length ~/ 2], closeTo(2.0, 0.6), + reason: 'cue shifts $shifts (cues $plain, then $delayed)'); + }, timeout: const Timeout(Duration(minutes: 35))); + + test('DeepFilterNet lowers stationary noise and keeps the timing', () async { + await WorkerHarness.ensureAddon('deepfilternet'); + // A voiced, syllable-modulated harmonic complex in bursts, over pink noise. + final src = await AudioFixtures.source('speechlike_noise', + seconds: 8, + filterComplex: "aevalsrc='(0.12*sin(2*PI*140*t)+0.08*sin(2*PI*280*t)+0.05*sin(2*PI*420*t)" + "+0.03*sin(2*PI*840*t))*(0.5+0.5*sin(2*PI*5*t))*lt(mod(t\\,1.6)\\,0.9)':s=$_rate:d=8[v];" + 'anoisesrc=c=pink:a=0.03:r=$_rate:d=8:seed=3[n];' + '[v][n]amix=inputs=2:normalize=0,pan=stereo|c0=c0|c1=c0[a]'); + final out = await _encode(AudioFixtures.job(src, {'deepFilter': true}, label: 'deepfilternet')); + final before = (await AudioFixtures.decode(src, mono: true)).first; + final after = (await AudioFixtures.decode(out, mono: true)).first; + expect(after.length, before.length, reason: 'the track keeps the picture\'s length'); + // Noise alone: the gaps (0.95–1.55 s of every 1.6 s cycle). + double gaps(Float32List x) { + var sum = 0.0; + var n = 0; + for (var cycle = 1; cycle < 4; cycle++) { + final from = ((cycle * 1.6 + 0.95) * _rate).round(), to = ((cycle * 1.6 + 1.55) * _rate).round(); + for (var i = from; i < to; i++) { + sum += x[i] * x[i]; + } + n += to - from; + } + return 10 * math.log(sum / n + 1e-20) / math.ln10; + } + final drop = gaps(before) - gaps(after); + expect(drop, greaterThan(10), reason: 'noise down by only ${drop.toStringAsFixed(1)} dB'); + // Timing: the bursts' envelopes line up to the millisecond. + final lag = AudioFixtures.lag(_envelope(before), _envelope(after), 60); + expect(lag.abs(), lessThanOrEqualTo(1), reason: 'DeepFilterNet moved the sound by $lag ms'); + }, timeout: const Timeout(Duration(minutes: 10))); +} diff --git a/app/test/integration_bt709_conversion_test.dart b/app/test/integration_bt709_conversion_test.dart new file mode 100644 index 00000000..a6435c43 --- /dev/null +++ b/app/test/integration_bt709_conversion_test.dart @@ -0,0 +1,295 @@ +/// BT.601 -> BT.709 when an SD source is upscaled to HD (Crop & Resize, app +/// 1.7.0): the picture must look the same, and only the matrix may change. +/// +/// An SD capture is BT.601. Resized to 1080 lines it used to keep its 601 tag, +/// which is correct, but a player that ignores the tag on an HD picture decodes +/// it as BT.709 and shifts greens and reds. With the conversion on, the worker +/// converts the samples in the script and declares 709 on both sides of the +/// encoder; this file checks the PICTURE, not just the tags, in the way +/// `integration_colour_tag_pixels_test.dart` does: +/// +/// * the converted encode, decoded through its own tags, shows each saturated +/// patch at the RGB the source shows decoded as 601 (within 3 levels); +/// * so does the same encode decoded as 709 regardless of the tags, which is +/// what the tag-ignoring player does and the whole point of converting; +/// * with the conversion off, the tags stay 601 and the samples are the +/// source's own (within resampling of a flat patch), exactly as before; +/// * the preview of a frame shows what the encode stores. +/// +/// Heavy (full-encode) — runs in the nightly workflow, not the push gate. +@Tags(['heavy']) +library; + +// ignore_for_file: avoid_print — these tests print diagnostics to the test log. + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/crop_resize_parameters.dart'; +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/worker_harness.dart'; + +String get _outDir => '${WorkerHarness.outputDir}/bt709_conversion'; + +const _frames = 6; +const _srcW = 720; +const _srcH = 576; + +/// The patches: 75% bars, two 100% primaries, a skin tone and grey. +const _patches = >{ + 'red75': [191, 0, 0], + 'green75': [0, 191, 0], + 'blue75': [0, 0, 191], + 'yellow75': [191, 191, 0], + 'cyan75': [0, 191, 191], + 'magenta75': [191, 0, 191], + 'skin': [224, 172, 140], + 'grey': [128, 128, 128], +}; + +/// Tolerance for a converted patch against the source decoded as 601, in +/// 8-bit RGB levels: the conversion is computed in float and dithered back to +/// 8-bit 4:2:0, and both sides are rounded again on the way to RGB. +const _tolerance = 3; + +Future _ffmpeg(List args) async { + final r = await Process.run(WorkerHarness.ffmpegPath, ['-y', '-v', 'error', ...args], + environment: WorkerHarness.ffmpegEnv); + if (r.exitCode != 0) throw Exception('ffmpeg ${args.join(' ')}: ${r.stderr}'); +} + +/// An SD BT.601 clip of eight flat patches (4 x 2), made in RGB and converted +/// to limited-range 4:2:0 with the 601 matrix, then tagged as such. +Future _makeFixture(String name, {required bool tagged}) async { + final path = '$_outDir/$name'; + final colours = _patches.values.toList(); + final pw = _srcW ~/ 4, ph = _srcH ~/ 2; + final inputs = []; + for (final c in colours) { + final hex = c.map((v) => v.toRadixString(16).padLeft(2, '0')).join(); + inputs.addAll(['-f', 'lavfi', '-i', 'color=c=0x$hex:s=${pw}x$ph:r=25:d=${_frames / 25},format=rgb24']); + } + final graph = '[0][1][2][3]hstack=4[top];[4][5][6][7]hstack=4[bot];' + '[top][bot]vstack,scale=out_color_matrix=bt601:out_range=tv,format=yuv420p[v]'; + await _ffmpeg([ + ...inputs, + '-filter_complex', graph, + '-map', '[v]', + '-frames:v', '$_frames', + '-c:v', 'ffv1', + if (tagged) ...[ + '-colorspace', 'smpte170m', '-color_primaries', 'smpte170m', + '-color_trc', 'smpte170m', '-color_range', 'tv', + ], + path, + ]); + return path; +} + +/// Frame [index] of [path] as rgb24, converted with [matrix] (an swscale +/// `in_color_matrix` name) and limited range. Explicit, so "decoded through +/// its tags" and "decoded as 709 whatever the tags" are both stated rather +/// than left to ffmpeg's guess. +Future<({Uint8List rgb, int w, int h})> _decode(String path, String matrix, + {int index = 2}) async { + final info = (await WorkerHarness.firstStream(path, + selector: 'v:0', entries: ['width', 'height']))!; + final w = info['width'] as int, h = info['height'] as int; + final out = '$_outDir/${const Uuid().v4()}.rgb'; + await _ffmpeg([ + '-i', path, + '-vf', 'select=eq(n\\,$index),scale=in_color_matrix=$matrix:in_range=tv:out_range=pc:flags=accurate_rnd+full_chroma_int+bitexact,format=rgb24', + '-frames:v', '1', '-f', 'rawvideo', out, + ]); + final bytes = await File(out).readAsBytes(); + await File(out).delete(); + return (rgb: bytes, w: w, h: h); +} + +/// The mean RGB of a 9x9 block at each patch's centre. +Map> _patchRgb(({Uint8List rgb, int w, int h}) frame) { + final result = >{}; + final names = _patches.keys.toList(); + for (var i = 0; i < names.length; i++) { + final cx = ((i % 4) + 0.5) * frame.w ~/ 4; + final cy = ((i ~/ 4) + 0.5) * frame.h ~/ 2; + final sum = [0.0, 0.0, 0.0]; + for (var dy = -4; dy <= 4; dy++) { + for (var dx = -4; dx <= 4; dx++) { + final o = ((cy + dy) * frame.w + (cx + dx)) * 3; + for (var c = 0; c < 3; c++) { + sum[c] += frame.rgb[o + c]; + } + } + } + result[names[i]] = sum.map((s) => s / 81).toList(); + } + return result; +} + +double _maxDelta(List a, List b) => + [for (var c = 0; c < 3; c++) (a[c] - b[c]).abs()].reduce((x, y) => x > y ? x : y); + +/// swscale's name for an ffprobe `color_space`. +String _swsMatrix(String? tag) => switch (tag) { + 'bt709' => 'bt709', + 'bt470bg' => 'bt470bg', + _ => 'smpte170m', + }; + +void main() { + late String tagged; + late String untagged480; + late Map> source601; + + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_outDir).create(recursive: true); + tagged = await _makeFixture('sd601_patches.mkv', tagged: true); + source601 = _patchRgb(await _decode(tagged, 'smpte170m')); + print(' source decoded as 601: ' + '${source601.map((k, v) => MapEntry(k, v.map((x) => x.round()).toList()))}'); + }); + + VideoJob job(String name, String input, Bt709Conversion mode, + {int width = _srcW, int height = _srcH, bool sourceTagged = true, int targetH = 1080}) { + return VideoJob( + id: const Uuid().v4(), + inputPath: input, + outputPath: '$_outDir/$name', + processingPipeline: ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + cropResize: CropResizeParameters( + enabled: true, + resizeEnabled: true, + targetWidth: targetH * 16 ~/ 9, + targetHeight: targetH, + maintainAspect: true, + convertToBt709: mode, + ), + ), + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, + container: ContainerFormat.mkv, + audioMode: AudioMode.none, + ), + totalFrames: _frames, + inputFrameRate: 25, + inputWidth: width, + inputHeight: height, + inputPixelFormat: 'yuv420p', + inputColorMatrix: sourceTagged ? 'smpte170m' : null, + inputColorPrimaries: sourceTagged ? 'smpte170m' : null, + inputColorTransfer: sourceTagged ? 'smpte170m' : null, + inputColorRange: sourceTagged ? 'tv' : null, + ); + } + + Future encode(VideoJob j) async { + final result = await WorkerHarness.runJob(j.toJson(), label: j.outputPath); + expect(result.success, isTrue, reason: '${result.error}\n${result.logs.join('\n')}'); + return result.outputPath ?? j.outputPath; + } + + Future> tags(String path) async => + (await WorkerHarness.firstStream(path, selector: 'v:0', entries: [ + 'width', 'height', 'pix_fmt', + 'color_space', 'color_primaries', 'color_transfer', 'color_range', + ]))!; + + test('converted: tagged 709, and every patch shows the source\'s colour', () async { + final out = await encode(job('on_1080.mkv', tagged, Bt709Conversion.auto)); + final t = await tags(out); + print(' converted output: $t'); + expect(t['height'], 1080); + expect(t['color_space'], 'bt709'); + expect(t['color_transfer'], 'bt709'); + expect(t['color_range'], 'tv'); + expect(t['color_primaries'], 'smpte170m', + reason: 'only the matrix is converted, so the primaries stay the source\'s'); + + final viaTags = _patchRgb(await _decode(out, _swsMatrix(t['color_space'] as String?))); + final as709 = _patchRgb(await _decode(out, 'bt709')); + for (final name in _patches.keys) { + final d = _maxDelta(viaTags[name]!, source601[name]!); + print(' $name: source ${source601[name]!.map((x) => x.round()).toList()}' + ' output ${viaTags[name]!.map((x) => x.round()).toList()} max delta ${d.toStringAsFixed(2)}'); + expect(d, lessThanOrEqualTo(_tolerance), reason: '$name through the output\'s tags'); + expect(_maxDelta(as709[name]!, source601[name]!), lessThanOrEqualTo(_tolerance), + reason: '$name decoded as 709 by a player that ignores the tag'); + } + }, timeout: const Timeout(Duration(minutes: 5))); + + test('off: tags stay 601 and the samples are the source\'s', () async { + final out = await encode(job('off_1080.mkv', tagged, Bt709Conversion.off)); + final t = await tags(out); + print(' unconverted output: $t'); + expect(t['height'], 1080); + expect(t['color_space'], 'smpte170m'); + expect(t['color_primaries'], 'smpte170m'); + // Decoded through its (601) tags: the source's colours, as before 1.7.0. + final viaTags = _patchRgb(await _decode(out, 'smpte170m')); + // The samples themselves are the source's: a flat patch survives a resize + // unchanged, so decoding both as 601 must agree to rounding. + for (final name in _patches.keys) { + expect(_maxDelta(viaTags[name]!, source601[name]!), lessThanOrEqualTo(1.0), reason: name); + } + // The problem the conversion exists for: the same file read as 709. + final misread = _patchRgb(await _decode(out, 'bt709')); + final worst = _patches.keys + .map((n) => (n, _maxDelta(misread[n]!, source601[n]!))) + .reduce((a, b) => a.$2 > b.$2 ? a : b); + print(' unconverted, read as 709: worst ${worst.$1} off by ${worst.$2.toStringAsFixed(1)}'); + expect(worst.$2, greaterThan(15), + reason: 'control: reading 601 samples as 709 must visibly shift colour'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('an SD-sized output under Auto is left alone', () async { + // A 700-line box: 875x700, under both HD thresholds. + final sd = job('auto_sd.mkv', tagged, Bt709Conversion.auto, targetH: 700); + final out = await encode(sd); + final t = await tags(out); + expect(t['height'], lessThan(720)); + expect(t['color_space'], 'smpte170m'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('untagged NTSC-sized source: read as 601, tagged with NTSC primaries', () async { + untagged480 = '$_outDir/sd601_untagged_480.mkv'; + await _ffmpeg(['-i', tagged, '-vf', 'scale=720:480:flags=neighbor', '-c:v', 'ffv1', untagged480]); + final out = await encode(job('untagged_480.mkv', untagged480, Bt709Conversion.auto, + height: 480, sourceTagged: false)); + final t = await tags(out); + print(' untagged 480 output: $t'); + expect(t['color_space'], 'bt709'); + expect(t['color_primaries'], 'smpte170m'); + final viaTags = _patchRgb(await _decode(out, 'bt709')); + final src = _patchRgb(await _decode(untagged480, 'smpte170m')); + for (final name in _patches.keys) { + expect(_maxDelta(viaTags[name]!, src[name]!), lessThanOrEqualTo(_tolerance), reason: name); + } + }, timeout: const Timeout(Duration(minutes: 5))); + + test('the preview shows what the converted encode stores', () async { + final j = job('on_preview.mkv', tagged, Bt709Conversion.auto); + final out = await encode(j); + const frame = 2; + final preview = await WorkerHarness.runPreview(j.toJson(), frame: frame); + expect(preview.success, isTrue, reason: '$preview'); + final previewRgb = await WorkerHarness.imageToRgb24(preview.png!, label: 'pv709'); + // Decoded the way ffmpeg decodes it by default, through the file's tags: + // the preview's own YUV-to-RGB step uses swscale's default path too, so + // the two are compared like for like (the accurate path above differs + // from the default one by ~1.5/255 on this content, in both directions). + final encoded = await WorkerHarness.frameRgb24(out, frame, label: 'enc709'); + final diff = WorkerHarness.meanAbsDiff(previewRgb, encoded); + print(' preview vs encode mean abs diff: ${diff.toStringAsFixed(3)}/255'); + expect(diff, lessThan(1.0), reason: 'the preview shows different colours from the encode'); + }, timeout: const Timeout(Duration(minutes: 5))); +} diff --git a/app/test/integration_classical_filters_test.dart b/app/test/integration_classical_filters_test.dart new file mode 100644 index 00000000..13b7bb61 --- /dev/null +++ b/app/test/integration_classical_filters_test.dart @@ -0,0 +1,248 @@ +/// End-to-end checks for the classical filters added with deps 1.16.0 +/// (app 1.7.0): SangNom anti-aliasing, NLMeans on the CPU (vszip 24), and the +/// three vs-jetpack ports — vine dehalo, mpeg2stinx, and the IVTC fade fix. +/// +/// Each runs the worker for real (vspipe | ffmpeg) and checks a valid output; +/// the fade fix and NLMeans are also checked by what they do to the picture. +/// The measurements behind these methods are in +/// docs/reports/2026-11-classical-filters. +/// +/// Heavy (full-encode) — tagged so it runs in the nightly workflow. +@Tags(['heavy']) +library; + +// ignore_for_file: avoid_print — these tests print diagnostics to the test log. + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/anti_alias_parameters.dart'; +import 'package:vapourbox/models/deblock_parameters.dart'; +import 'package:vapourbox/models/dehalo_parameters.dart'; +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/noise_reduction_parameters.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/worker_harness.dart'; + +String get _outDir => '${WorkerHarness.outputDir}/classical_filters'; + +VideoJob _job(String name, ProcessingPipeline pipeline, + {String? input, int? frames, double? fps, int? width, int? height}) => + VideoJob( + id: const Uuid().v4(), + inputPath: input ?? WorkerHarness.inputFile, + outputPath: '$_outDir/$name.mkv', + processingPipeline: pipeline, + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, + container: ContainerFormat.mkv, + audioMode: AudioMode.none, + ), + totalFrames: frames, + inputFrameRate: fps, + inputWidth: width, + inputHeight: height, + inputPixelFormat: input == null ? null : 'yuv420p', + ); + +Future _expectValidVideo(JobResult result) async { + final tail = result.logs.length > 25 + ? result.logs.sublist(result.logs.length - 25) + : result.logs; + expect(result.success, isTrue, + reason: '${result.error}\n--- worker log (tail) ---\n${tail.join('\n')}'); + final v = await WorkerHarness.firstStream(result.outputPath!, + selector: 'v:0', entries: ['codec_name', 'width', 'height']); + expect(v, isNotNull, reason: 'output should contain a video stream'); +} + +/// Mean |even-row mean - odd-row mean| of an rgb24 frame, 0-255: how far the +/// two fields sit apart in brightness. +double _fieldGap(Uint8List rgb, int width) { + final rows = rgb.length ~/ (width * 3); + var even = 0.0, odd = 0.0; + for (var y = 0; y < rows; y++) { + var sum = 0; + final start = y * width * 3; + for (var i = start; i < start + width * 3; i++) { + sum += rgb[i]; + } + if (y.isEven) { + even += sum; + } else { + odd += sum; + } + } + final perRow = width * 3.0; + return ((even / (rows / 2)) - (odd / (rows / 2))).abs() / perRow; +} + +void main() { + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_outDir).create(recursive: true); + }); + + const progressive = QTGMCParameters(enabled: false); + + group('deps 1.16.0 classical filters', () { + test('anti-alias: SangNom runs end-to-end', () async { + final result = await WorkerHarness.runJob( + _job('aa_sangnom', const ProcessingPipeline( + deinterlace: progressive, + antiAlias: AntiAliasParameters(enabled: true, method: AntiAliasMethod.sangnom), + )).toJson(), + label: 'aa_sangnom'); + await _expectValidVideo(result); + }, timeout: const Timeout(Duration(minutes: 6))); + + test('noise reduction: NLMeans (CPU) runs end-to-end without OpenCL', + () async { + // OpenCL switched off for the worker: the CPU method must not care. + final result = await WorkerHarness.runJob( + _job('nr_nlmeans_cpu', const ProcessingPipeline( + deinterlace: progressive, + noiseReduction: NoiseReductionParameters( + enabled: true, method: NoiseReductionMethod.nlMeans, knlmH: 2.0), + )).toJson(), + label: 'nr_nlmeans_cpu', + extraEnv: const {'VAPOURBOX_DISABLE_OPENCL': '1'}); + await _expectValidVideo(result); + }, timeout: const Timeout(Duration(minutes: 6))); + + test('noise reduction: NLMeans (CPU) gives KNLMeansCL\'s picture', () async { + final probe = await Process.run(WorkerHarness.workerPath, ['--probe-opencl'], + environment: WorkerHarness.workerEnv, + workingDirectory: File(WorkerHarness.workerPath).parent.path); + if (probe.stdout.toString().contains('"knlmDevice":null')) { + print(' skipped: no OpenCL device for the KNLMeansCL reference'); + return; + } + Future frame(NoiseReductionMethod method) async { + final r = await WorkerHarness.runPreview( + _job('nlm_vs_knlm_${method.name}', ProcessingPipeline( + deinterlace: progressive, + noiseReduction: NoiseReductionParameters( + enabled: true, method: method, knlmH: 2.5, knlmD: 1), + )).toJson(), + frame: 10, + label: 'nlm_${method.name}'); + expect(r.success, isTrue, reason: r.errorTail); + return WorkerHarness.imageToRgb24(r.png!, label: 'nlm_${method.name}'); + } + + final cpu = await frame(NoiseReductionMethod.nlMeans); + final gpu = await frame(NoiseReductionMethod.knlMeans); + final diff = WorkerHarness.meanAbsDiff(cpu, gpu); + print(' NLMeans (CPU) vs KNLMeansCL: mean |diff| $diff'); + // Same algorithm and arguments: identical to the level on the probe + // (docs/reports/2026-11-classical-filters); the slack is GPU float order. + expect(diff, lessThan(0.5)); + }, timeout: const Timeout(Duration(minutes: 6))); + + test('dehalo: vine runs end-to-end', () async { + final result = await WorkerHarness.runJob( + _job('dehalo_vine', const ProcessingPipeline( + deinterlace: progressive, + dehalo: DehaloParameters(enabled: true, method: DehaloMethod.vine), + )).toJson(), + label: 'dehalo_vine'); + await _expectValidVideo(result); + }, timeout: const Timeout(Duration(minutes: 8))); + + test('deblock: mpeg2stinx runs end-to-end after IVTC', () async { + final result = await WorkerHarness.runJob( + _job('deblock_mpeg2stinx', const ProcessingPipeline( + deinterlace: QTGMCParameters( + enabled: true, method: DeinterlaceMethod.ivtc, tff: true), + deblock: DeblockParameters(enabled: true, method: DeblockMethod.mpeg2stinx), + )).toJson(), + label: 'deblock_mpeg2stinx'); + await _expectValidVideo(result); + }, timeout: const Timeout(Duration(minutes: 8))); + + test('ivtc: the fade fix finishes a full-size job in bounded time', () async { + // The first version created an akarin node per frame inside two + // concurrent FrameEvals: vspipe aborted, or spun one core for 97 minutes + // on 30 frames, while the same graph single-threaded took a second. The + // whole 720x576 fixture must encode well inside the timeout. + final sw = Stopwatch()..start(); + final result = await WorkerHarness.runJob( + _job('ivtc_fades_bounded', const ProcessingPipeline( + deinterlace: QTGMCParameters( + enabled: true, + method: DeinterlaceMethod.ivtc, + tff: true, + ivtcFixFades: true), + )).toJson(), + label: 'ivtc_fades_bounded', + timeout: const Duration(minutes: 3)); + print(' IVTC + fade fix, 720x576: ${sw.elapsed.inMilliseconds} ms'); + await _expectValidVideo(result); + expect(sw.elapsed, lessThan(const Duration(minutes: 2))); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('ivtc: the fade fix closes the gap between a fading frame\'s fields', + () async { + // A fade applied AFTER the fields were split (as one applied to + // telecined video is): each field fades at its own time, so every frame + // of the fade has a brighter and a darker field — the comb no field + // match can remove. 320x240, 30 frames (brightened so the fade has + // somewhere to go), fading to black over half a second. + final fixture = p.join(_outDir, 'fade_after_split.mkv'); + final make = await Process.run( + WorkerHarness.ffmpegPath, + [ + '-y', '-v', 'error', '-i', + p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', 'small_clip.mp4'), + '-vf', 'select=between(n\\,1400\\,1429),setpts=N/30/TB,eq=brightness=0.35,separatefields,' + 'fade=t=out:st=0:d=0.5,weave=first_field=top,format=yuv420p', + '-r', '30', '-c:v', 'ffv1', fixture, + ], + environment: WorkerHarness.ffmpegEnv, + ); + expect(make.exitCode, 0, reason: make.stderr.toString()); + + Future gap(bool fix) async { + final result = await WorkerHarness.runJob( + _job( + 'ivtc_fades_$fix', + ProcessingPipeline( + deinterlace: QTGMCParameters( + enabled: true, + method: DeinterlaceMethod.ivtc, + tff: true, + ivtcFixFades: fix, + ), + ), + input: fixture, + frames: 30, + fps: 30, + width: 320, + height: 240, + ).toJson(), + label: 'ivtc_fades_$fix'); + await _expectValidVideo(result); + // Frames well inside the fade (before it reaches black). + var total = 0.0; + for (final i in [2, 4, 6, 8]) { + total += _fieldGap(await WorkerHarness.frameRgb24(result.outputPath!, i), 320); + } + return total / 4; + } + + final before = await gap(false); + final after = await gap(true); + print(' field brightness gap: without fix $before, with fix $after'); + expect(before, greaterThan(1.0), reason: 'the fixture has no fade comb'); + expect(after, lessThan(before * 0.25)); + }, timeout: const Timeout(Duration(minutes: 8))); + }); +} diff --git a/app/test/integration_colorize_test.dart b/app/test/integration_colorize_test.dart new file mode 100644 index 00000000..95e69c81 --- /dev/null +++ b/app/test/integration_colorize_test.dart @@ -0,0 +1,217 @@ +// Colourisation end to end, on real black-and-white film +// (Tests/TestResources/colorize_test.mp4: "Diary of a Sergeant", 1945, a US +// War Department film in the public domain; see generate_colorize_test.sh), +// with the add-ons installed by the app's own installer +// (WorkerHarness.ensureAddon; set VAPOURBOX_ADDON_MIRROR to test files not +// uploaded yet): +// +// * DDColor colours the picture, and the Y plane of every frame is +// bit-exact to the same job with the pass off — the pass replaces only +// the chroma; +// * strength 0 is the input, bit-exact in every plane (the fixture's chroma +// is exactly neutral); +// * DeOldify Video runs the same way; +// * a 10-bit source comes out 10-bit with its luma bit-exact; +// * a source that already has colour keeps its luma and gets the model's +// colour instead of its own; +// * the preview is the encode's frame. +@Tags(['heavy']) +@Timeout(Duration(minutes: 60)) +library; + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/colorize_parameters.dart'; +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/worker_harness.dart'; + +const _w = 640, _h = 480; +// Frames 40-55 of the fixture (the close-up): the encode keeps 16 frames. +const _start = 40, _end = 55; +const _count = _end - _start + 1; + +String get _fixture => p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', 'colorize_test.mp4'); +String get _colourFixture => p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', 'face_restore_test.mp4'); +String get _dir => p.join(WorkerHarness.outputDir, 'colorize'); + +Map _job(String name, ColorizeParameters? cz, + {String? input, String pixFmt = 'yuv420p', bool trimmed = true, double fps = 24, int total = 120}) { + final job = VideoJob( + id: const Uuid().v4(), + inputPath: input ?? _fixture, + outputPath: p.join(_dir, '$name.mkv'), + processingPipeline: ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + colorize: cz ?? const ColorizeParameters(enabled: false), + ), + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, container: ContainerFormat.mkv, audioMode: AudioMode.none), + inputWidth: _w, + inputHeight: _h, + inputPixelFormat: pixFmt, + inputFrameRate: fps, + totalFrames: trimmed ? _count : total, + startFrame: trimmed ? _start : null, + endFrame: trimmed ? _end : null, + ); + return job.toJson(); +} + +Future _encode(String name, ColorizeParameters? cz, + {String? input, String pixFmt = 'yuv420p', double fps = 24}) async { + final r = await WorkerHarness.runJob(_job(name, cz, input: input, pixFmt: pixFmt, fps: fps), + label: name, timeout: const Duration(minutes: 25), + // On the Intel-Mac CI VMs Core ML's virtual GPU fails and the models run + // on a slow CPU: CodeFormer's first frame (five frames of faces, for the + // smoothing) took ~10 min, past the worker's 600 s stall limit. + extraEnv: const {'VAPOURBOX_STALL_TIMEOUT_SECS': '1200'}); + expect(r.success, isTrue, reason: '${r.error}\n${r.logs.reversed.take(30).toList().reversed.join('\n')}'); + // ignore: avoid_print + print(' $name: ${r.duration.inMilliseconds / 1000}s for $_count frames'); + return r.outputPath ?? p.join(_dir, '$name.mkv'); +} + +Future _raw(String path, String pixFmt) async { + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-v', 'error', '-i', path, '-f', 'rawvideo', '-pix_fmt', pixFmt, '-'], + environment: WorkerHarness.ffmpegEnv, + stdoutEncoding: null, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + return Uint8List.fromList(r.stdout as List); +} + +/// Compare two decodes of 4:2:0 video ([bytes] per sample): the number of +/// luma samples that differ, the number of chroma samples that differ, and +/// the mean distance of [b]'s chroma from neutral (in 8-bit levels). +({int lumaChanged, int chromaChanged, double chroma, int frames}) _compare(Uint8List a, Uint8List b, + {int bytes = 1}) { + expect(a.length, b.length, reason: 'the two encodes differ in size'); + final luma = _w * _h, chroma = luma ~/ 2; + final frameSamples = luma + chroma; + final frames = a.length ~/ (frameSamples * bytes); + int sample(Uint8List d, int i) => bytes == 1 ? d[i] : d[i * 2] | (d[i * 2 + 1] << 8); + final neutral = bytes == 1 ? 128 : 512; + final scale = bytes == 1 ? 1.0 : 0.25; + var lumaChanged = 0, chromaChanged = 0; + var dist = 0.0; + for (var f = 0; f < frames; f++) { + final base = f * frameSamples; + for (var i = base; i < base + luma; i++) { + if (sample(a, i) != sample(b, i)) lumaChanged++; + } + for (var i = base + luma; i < base + frameSamples; i++) { + final v = sample(b, i); + if (sample(a, i) != v) chromaChanged++; + dist += (v - neutral).abs() * scale; + } + } + return (lumaChanged: lumaChanged, chromaChanged: chromaChanged, chroma: dist / (frames * chroma), frames: frames); +} + +void main() { + late String baseline; + late Uint8List baselineRaw; + + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_dir).create(recursive: true); + // Each model's add-on, with its requirement (the AI runtime). + await WorkerHarness.ensureAddon('colorize-ddcolor'); + await WorkerHarness.ensureAddon('colorize-deoldify'); + baseline = await _encode('baseline', null); + baselineRaw = await _raw(baseline, 'yuv420p'); + }); + + test('DDColor colours the picture and keeps every luma sample', () async { + final out = await _encode('ddcolor', const ColorizeParameters(enabled: true)); + final c = _compare(baselineRaw, await _raw(out, 'yuv420p')); + // ignore: avoid_print + print(' ddcolor: mean chroma ${c.chroma.toStringAsFixed(2)} levels from neutral, ' + '${c.lumaChanged} luma samples changed'); + expect(c.frames, _count); + expect(c.lumaChanged, 0, reason: 'the pass must keep the source\'s luma bit-exact'); + expect(c.chroma, greaterThan(2.0), reason: 'the picture was not coloured'); + }); + + test('strength 0 is the input, bit-exact', () async { + final out = await _encode('strength0', const ColorizeParameters(enabled: true, strength: 0)); + final c = _compare(baselineRaw, await _raw(out, 'yuv420p')); + expect(c.lumaChanged, 0); + expect(c.chromaChanged, 0); + }); + + test('DeOldify Video colours the picture and keeps every luma sample', () async { + final out = await _encode('deoldify', + const ColorizeParameters(enabled: true, model: ColorizeModel.deoldify, temporalRadius: 1)); + final c = _compare(baselineRaw, await _raw(out, 'yuv420p')); + // ignore: avoid_print + print(' deoldify: mean chroma ${c.chroma.toStringAsFixed(2)} levels from neutral'); + expect(c.lumaChanged, 0); + expect(c.chroma, greaterThan(1.0)); + }); + + test('a 10-bit source stays 10-bit, its luma bit-exact', () async { + final src10 = p.join(_dir, 'colorize_10bit.mkv'); + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-i', _fixture, '-vf', 'format=yuv420p10le', '-c:v', 'ffv1', src10], + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + final off = await _encode('baseline_10bit', null, input: src10, pixFmt: 'yuv420p10le'); + final on = await _encode('ddcolor_10bit', const ColorizeParameters(enabled: true, temporalRadius: 0), + input: src10, pixFmt: 'yuv420p10le'); + final stream = await WorkerHarness.firstStream(on, selector: 'v:0', entries: ['pix_fmt']); + expect(stream?['pix_fmt'], 'yuv420p10le'); + final c = _compare(await _raw(off, 'yuv420p10le'), await _raw(on, 'yuv420p10le'), bytes: 2); + // ignore: avoid_print + print(' 10-bit: mean chroma ${c.chroma.toStringAsFixed(2)} (8-bit levels), ${c.lumaChanged} luma changed'); + expect(c.lumaChanged, 0); + expect(c.chroma, greaterThan(2.0)); + }); + + test('a colour source keeps its luma and gets the model\'s colour', () async { + // The face fixture is colour video (640x480, 29.97 fps). + final off = await _encode('colour_baseline', null, input: _colourFixture, fps: 29.97); + final on = await _encode('colour_ddcolor', const ColorizeParameters(enabled: true, temporalRadius: 0), + input: _colourFixture, fps: 29.97); + final c = _compare(await _raw(off, 'yuv420p'), await _raw(on, 'yuv420p')); + expect(c.lumaChanged, 0); + expect(c.chromaChanged, greaterThan(0), reason: 'the source\'s own colour should be replaced'); + }); + + test('the preview is the encode\'s frame', () async { + const cz = ColorizeParameters(enabled: true); + final out = await _encode('preview_ref', cz); + // Two frames or more from the trim's ends, so the encode's temporal + // neighbours (two either side) are the preview's. + for (final frame in [_start + 2, _start + 10]) { + final pv = await WorkerHarness.runPreview(_job('preview', cz, trimmed: false), frame: frame, + // Same budget as _encode: on the Intel-Mac VMs Core ML falls back to + // the CPU, and the v2 runner's first frame alone can take ~10 min. + timeout: const Duration(minutes: 20), + extraEnv: const {'VAPOURBOX_STALL_TIMEOUT_SECS': '1200'}); + expect(pv.success, isTrue, reason: '${pv.error}\n${pv.logs}'); + final preview = await WorkerHarness.imageToRgb24(pv.png!, label: 'cz_pv'); + final encoded = await WorkerHarness.frameRgb24(out, frame - _start, label: 'cz_enc'); + final input = await WorkerHarness.frameRgb24(baseline, frame - _start, label: 'cz_in'); + final diff = WorkerHarness.meanAbsDiff(preview, encoded); + final coloured = WorkerHarness.meanAbsDiff(input, encoded); + // ignore: avoid_print + print(' frame $frame: preview vs encode $diff, input vs encode $coloured'); + expect(diff, lessThan(0.5), reason: 'the preview shows a different picture'); + expect(coloured, greaterThan(diff * 4), reason: 'the comparison cannot tell coloured from not'); + } + }, timeout: const Timeout(Duration(minutes: 75))); +} diff --git a/app/test/integration_crop_detect_test.dart b/app/test/integration_crop_detect_test.dart new file mode 100644 index 00000000..c12cc239 --- /dev/null +++ b/app/test/integration_crop_detect_test.dart @@ -0,0 +1,332 @@ +/// Crop & Resize's Detect (`vapourbox-worker --detect-crop`, app 1.7.0), +/// run through the real worker on videos whose borders are known. +/// +/// The fixtures are built here from real picture: frames of the VHS capture +/// `interlaced_test.avi`, cut to its inner region (so they carry no borders of +/// their own) and placed on a canvas with black borders, noisy blanking, a +/// letterbox, burnt-in subtitles, fades to black and a simulated +/// head-switching band (the bottom lines shifted sideways by 5-13 px, with +/// noisy luma and wrecked chroma, different in every frame). Each is encoded +/// losslessly and measured; every edge must come back within 2 lines of what +/// was drawn. Two real files are measured as well: a DVB-T broadcast (black +/// side bars, black bottom lines; checked by eye, see the report) and the VHS +/// capture itself. +/// +/// The unit tests in `worker/src/crop_detect.rs` cover the same traps on +/// synthetic frames; this file proves the decode, the sampling and the JSON +/// as the app runs them. +/// +/// Heavy (decodes and encodes video) — runs in the nightly workflow. +@Tags(['heavy']) +library; + +// ignore_for_file: avoid_print — these tests print diagnostics to the test log. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:uuid/uuid.dart'; + +import 'support/worker_harness.dart'; + +String get _outDir => '${WorkerHarness.outputDir}/crop_detect'; +String get _vhs => '${WorkerHarness.repoRoot}/Tests/TestResources/interlaced_test.avi'; +String get _dvbt => '${WorkerHarness.repoRoot}/Tests/TestResources/pal-dvbt-fieldcoded-25i.ts'; + +const _w = 720; +const _h = 576; +const _frames = 30; + +/// What to draw. +class _Spec { + final String name; + final int left, right, top, bottom; + final double blankNoise; + final int band; + final bool subtitles; + final bool fades; + + const _Spec(this.name, + {this.left = 0, + this.right = 0, + this.top = 0, + this.bottom = 0, + this.blankNoise = 0, + this.band = 0, + this.subtitles = false, + this.fades = false}); +} + +/// [_frames] frames of the VHS capture's inner region, resized to [w]x[h], +/// as yuv444p. +Future> _picture(int w, int h) async { + final r = await Process.run( + WorkerHarness.ffmpegPath, + [ + '-v', 'error', '-i', _vhs, + '-vf', 'crop=664:536:28:12,scale=$w:$h:flags=bicubic', + '-frames:v', '$_frames', '-f', 'rawvideo', '-pix_fmt', 'yuv444p', '-', + ], + environment: WorkerHarness.ffmpegEnv, + stdoutEncoding: null, + ); + if (r.exitCode != 0) throw Exception('decoding the picture: ${r.stderr}'); + final bytes = r.stdout as List; + final size = w * h * 3; + return [ + for (var i = 0; i + size <= bytes.length; i += size) + Uint8List.fromList(bytes.sublist(i, i + size)), + ]; +} + +/// Build and encode one fixture; returns its path. +Future _fixture(_Spec s) async { + final pw = _w - s.left - s.right; + final ph = _h - s.top - s.bottom; + final picture = await _picture(pw, ph); + final rnd = Random(7); + const n = _w * _h; + final out = BytesBuilder(copy: false); + for (var t = 0; t < picture.length; t++) { + final f = Uint8List(n * 3); + f.fillRange(0, n, 16); + f.fillRange(n, 3 * n, 128); + final p = picture[t]; + final fade = s.fades && t % 3 == 0; + for (var y = 0; y < ph; y++) { + for (var x = 0; x < pw; x++) { + final si = y * pw + x; + final di = (y + s.top) * _w + x + s.left; + var luma = p[si]; + var u = p[pw * ph + si], v = p[2 * pw * ph + si]; + if (fade) { + luma = 16 + (luma - 16) ~/ 12; + u = 128 + (u - 128) ~/ 12; + v = 128 + (v - 128) ~/ 12; + } + f[di] = luma; + f[n + di] = u; + f[2 * n + di] = v; + } + } + if (s.blankNoise > 0) { + for (var y = 0; y < _h; y++) { + for (var x = 0; x < s.left; x++) { + f[y * _w + x] = 16 + (rnd.nextDouble() * s.blankNoise).round(); + } + } + } + // A subtitle in two frames out of three: white blocks across the middle + // third, its bottom 40 lines above the frame's. + if (s.subtitles && t % 3 != 0) { + for (var y = _h - 62; y < _h - 40; y++) { + for (var x = _w ~/ 3; x < 2 * _w ~/ 3; x++) { + if ((x ~/ 9 + y ~/ 6 + t) % 3 != 0) f[y * _w + x] = 232; + } + } + } + // Head switching: each line torn sideways from a picture line above it, + // with noisy luma and wrecked chroma. + for (var o = 0; o < s.band; o++) { + final y = _h - 1 - o; + final shift = 5 + (o * 3 + t) % 9; + final src = y - 10; + for (var x = 0; x < _w; x++) { + final i = y * _w + x; + final from = f[src * _w + (x - shift + _w) % _w]; + f[i] = (from * 0.7 + 40 + (rnd.nextDouble() - 0.5) * 50).round().clamp(16, 235); + f[n + i] = (128 + (rnd.nextDouble() - 0.5) * 80).round(); + f[2 * n + i] = (128 + (rnd.nextDouble() - 0.5) * 80).round(); + } + } + out.add(f); + } + final path = '$_outDir/${s.name}.mkv'; + final p = await Process.start( + WorkerHarness.ffmpegPath, + [ + '-y', '-v', 'error', + '-f', 'rawvideo', '-pix_fmt', 'yuv444p', '-s', '${_w}x$_h', '-r', '25', '-i', '-', + '-c:v', 'ffv1', '-pix_fmt', 'yuv422p', path, + ], + environment: WorkerHarness.ffmpegEnv, + ); + final err = p.stderr.transform(utf8.decoder).join(); + p.stdout.drain(); + p.stdin.add(out.takeBytes()); + await p.stdin.close(); + if (await p.exitCode != 0) throw Exception('encoding ${s.name}: ${await err}'); + return path; +} + +/// Run the worker's `--detect-crop` on [path]; returns the report and the time. +Future<(Map, Duration)> _detect( + String path, { + required int width, + required int height, + required int frames, + required double fps, + required String pixFmt, + String fieldOrder = 'progressive', +}) async { + final job = { + 'id': const Uuid().v4(), + 'inputPath': path, + 'outputPath': '$_outDir/unused.mkv', + 'qtgmcParameters': {'enabled': false}, + 'processingPipeline': { + 'deinterlace': {'enabled': false}, + }, + 'encodingSettings': {}, + 'detectedFieldOrder': fieldOrder, + 'totalFrames': frames, + 'inputFrameRate': fps, + 'inputWidth': width, + 'inputHeight': height, + 'inputPixelFormat': pixFmt, + }; + final config = File('$_outDir/${job['id']}.json'); + await config.writeAsString(jsonEncode(job)); + final sw = Stopwatch()..start(); + final r = await Process.run( + WorkerHarness.workerPath, + ['--detect-crop', '--config', config.path], + environment: WorkerHarness.workerEnv, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + sw.stop(); + await config.delete(); + final line = const LineSplitter() + .convert(r.stdout as String) + .lastWhere((l) => l.trim().startsWith('{'), orElse: () => '{}'); + final report = jsonDecode(line) as Map; + expect(report['error'], isNull, reason: '${r.stderr}'); + return (report, sw.elapsed); +} + +/// The measured (unrounded) value for one edge. +int _edge(Map r, String side, [String key = 'crop']) => + ((r['edges'] as Map)[side] as Map)[key] as int; + +String _kind(Map r, String side) => ((r['edges'] as Map)[side] as Map)['kind'] as String; + +void _near(Map r, String side, int expected, {String key = 'crop'}) { + final got = _edge(r, side, key); + expect((got - expected).abs(), lessThanOrEqualTo(2), + reason: '$side ($key): measured $got, drawn $expected'); +} + +void _summary(String name, Map r, Duration t) { + final e = r['edges'] as Map; + String side(String s) { + final m = e[s] as Map; + return '$s ${m['kind']} ${m['crop']}/${m['cropWithBars']} (conf ${(m['confidence'] as num).toStringAsFixed(2)})'; + } + + print(' $name: ${t.inMilliseconds} ms, usable ${r['usableFrames']}/${(r['frames'] as List).length}, ' + 'crop ${r['crop']} ${side('left')}; ${side('right')}; ${side('top')}; ${side('bottom')}'); +} + +void main() { + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_outDir).create(recursive: true); + }); + + Future> run(_Spec s, {String fieldOrder = 'progressive'}) async { + final path = await _fixture(s); + final (r, t) = await _detect(path, + width: _w, height: _h, frames: _frames, fps: 25, pixFmt: 'yuv422p', fieldOrder: fieldOrder); + _summary(s.name, r, t); + return r; + } + + test('VHS: blanking, noisy blanking and a head-switching band', () async { + final r = await run(const _Spec('vhs', left: 14, right: 10, top: 2, band: 8, blankNoise: 12)); + expect(r['found'], isTrue); + _near(r, 'left', 14); + _near(r, 'right', 10); + _near(r, 'top', 2); + expect(_kind(r, 'bottom'), 'headSwitching'); + _near(r, 'bottom', 8); + // Rounded outward to even values the Crop pass accepts. + final crop = r['crop'] as Map; + for (final side in ['left', 'right', 'top', 'bottom']) { + expect((crop[side] as int) % 2, 0, reason: side); + expect(crop[side] as int, greaterThanOrEqualTo(_edge(r, side)), reason: side); + } + expect(r['edgeRepair'], {'left': 0, 'right': 0, 'top': 0, 'bottom': 8}); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('interlaced: the band is found per field, the crop in fours', () async { + final r = await run(const _Spec('vhs_interlaced', left: 14, right: 10, band: 12), + fieldOrder: 'tff'); + expect(r['interlaced'], isTrue); + _near(r, 'left', 14); + _near(r, 'right', 10); + expect(_kind(r, 'bottom'), 'headSwitching'); + _near(r, 'bottom', 12); + // 4:2:2 keeps its chroma per line, so twos are enough here. + expect(r['stepY'], 2); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('letterbox: the bars are reported but not in the recommended crop', () async { + final r = await run(const _Spec('letterbox', left: 8, right: 8, top: 72, bottom: 72)); + expect(_kind(r, 'top'), 'bar'); + expect(_kind(r, 'bottom'), 'bar'); + expect((r['crop'] as Map)['top'], 0); + expect((r['crop'] as Map)['bottom'], 0); + _near(r, 'top', 72, key: 'cropWithBars'); + _near(r, 'bottom', 72, key: 'cropWithBars'); + _near(r, 'left', 8); + _near(r, 'right', 8); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('burnt-in subtitles near the bottom are not head switching', () async { + final r = await run(const _Spec('subtitles', left: 8, right: 8, subtitles: true)); + expect(_kind(r, 'bottom'), isNot('headSwitching')); + expect(_edge(r, 'bottom'), lessThanOrEqualTo(2)); + _near(r, 'left', 8); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('fades to black do not move the edges', () async { + final r = await run(const _Spec('fades', left: 12, right: 12, top: 4, fades: true)); + expect(r['usableFrames'] as int, lessThan((r['frames'] as List).length), + reason: 'the faded frames are left out'); + _near(r, 'left', 12); + _near(r, 'right', 12); + _near(r, 'top', 4); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('a real DVB-T broadcast: black side bars and bottom lines', () async { + // Measured by eye on the decoded frames (docs/reports/2026-11-crop-detect-709): + // 7 black columns left, 8 right, 4 black lines and one dim line at the + // bottom, nothing at the top. + final (r, t) = await _detect(_dvbt, + width: 720, height: 576, frames: 250, fps: 25, pixFmt: 'yuv420p', fieldOrder: 'tff'); + _summary('pal-dvbt', r, t); + _near(r, 'left', 7); + _near(r, 'right', 8); + _near(r, 'top', 0); + _near(r, 'bottom', 5); + // Still interlaced 4:2:0 at the crop (deinterlacing is off here): fours. + expect(r['stepY'], 4); + expect((r['crop'] as Map)['bottom'], 8); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('the real VHS capture: its blanking is found', () async { + // 10 dark columns then a ramp at the left, a narrower one at the right, + // and one black first line (see the report). + final (r, t) = await _detect(_vhs, + width: 720, height: 576, frames: 79, fps: 25, pixFmt: 'yuv422p', fieldOrder: 'tff'); + _summary('interlaced_test', r, t); + expect(r['found'], isTrue); + expect(_edge(r, 'left', 'blank'), inInclusiveRange(8, 12)); + expect(_edge(r, 'left'), inInclusiveRange(10, 18)); + expect(_edge(r, 'right'), inInclusiveRange(3, 9)); + expect(_edge(r, 'top'), 1); + }, timeout: const Timeout(Duration(minutes: 5))); +} diff --git a/app/test/integration_face_restore_test.dart b/app/test/integration_face_restore_test.dart new file mode 100644 index 00000000..a686ec29 --- /dev/null +++ b/app/test/integration_face_restore_test.dart @@ -0,0 +1,235 @@ +// Face Restoration end to end, on a VHS-like clip of a real face +// (Tests/TestResources/face_restore_test.mp4: NASA public-domain footage, +// see generate_face_test.sh), with the add-ons installed by the app's own +// installer (WorkerHarness.ensureAddon; set VAPOURBOX_ADDON_MIRROR to test +// files not uploaded yet): +// +// * RestoreFormer++ finds the face and changes it, and every pixel outside a +// generous box around the face is bit-exact to the same job with the pass +// off (the pass composites in the source format with a mask that is zero +// there) — in every plane, every frame; +// * the GFPGAN and CodeFormer add-ons run, with and without smoothing; +// * the preview is the encode's frame; +// * a 10-bit source comes out 10-bit, changed in the face and bit-exact +// outside it. +@Tags(['heavy']) +@Timeout(Duration(minutes: 40)) +library; + +import 'dart:io'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/face_restore_parameters.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/models/qtgmc_parameters.dart'; +import 'package:vapourbox/models/video_job.dart'; + +import 'support/worker_harness.dart'; + +const _w = 640, _h = 480; +// Frames 30-45 of the fixture: the encode keeps 16 frames. +const _start = 30, _end = 45; +const _count = _end - _start + 1; + +/// The face's detected box in the fixture is about x 255-365, y 149-286 +/// (docs/reports/2026-11-face-restoration). Changes must stay inside this box, +/// which adds a little over half the face's size on every side — the +/// feathered mask reaches past the face itself, and the head moves. +const _outer = (x0: 190, y0: 70, x1: 430, y1: 370); + +/// The middle of the face, which restoration must visibly change. +const _inner = (x0: 280, y0: 180, x1: 340, y1: 255); + +String get _fixture => p.join(WorkerHarness.repoRoot, 'Tests', 'TestResources', 'face_restore_test.mp4'); +String get _dir => p.join(WorkerHarness.outputDir, 'face_restore'); + +/// A job over the trimmed range ([trimmed]), or the whole fixture (the +/// preview's job: its `--frame` is a source frame, and the worker takes +/// `totalFrames` as the post-trim count). +Map _job(String name, FaceRestoreParameters? fr, + {String? input, String pixFmt = 'yuv420p', bool trimmed = true}) { + final job = VideoJob( + id: const Uuid().v4(), + inputPath: input ?? _fixture, + outputPath: p.join(_dir, '$name.mkv'), + processingPipeline: ProcessingPipeline( + deinterlace: const QTGMCParameters(enabled: false), + faceRestore: fr ?? const FaceRestoreParameters(enabled: false), + ), + encodingSettings: const EncodingSettings( + codec: VideoCodec.ffv1, container: ContainerFormat.mkv, audioMode: AudioMode.none), + inputWidth: _w, + inputHeight: _h, + inputPixelFormat: pixFmt, + inputFrameRate: 29.97, + totalFrames: trimmed ? _count : 150, + startFrame: trimmed ? _start : null, + endFrame: trimmed ? _end : null, + ); + return job.toJson(); +} + +Future _encode(String name, FaceRestoreParameters? fr, + {String? input, String pixFmt = 'yuv420p'}) async { + final r = await WorkerHarness.runJob(_job(name, fr, input: input, pixFmt: pixFmt), + label: name, timeout: const Duration(minutes: 25), + // On the Intel-Mac CI VMs Core ML's virtual GPU fails and the models run + // on a slow CPU: CodeFormer's first frame (five frames of faces, for the + // smoothing) took ~10 min, past the worker's 600 s stall limit. + extraEnv: const {'VAPOURBOX_STALL_TIMEOUT_SECS': '1200'}); + expect(r.success, isTrue, reason: '${r.error}\n${r.logs.reversed.take(30).toList().reversed.join('\n')}'); + // ignore: avoid_print + print(' $name: ${r.duration.inMilliseconds / 1000}s for $_count frames'); + return r.outputPath ?? p.join(_dir, '$name.mkv'); +} + +/// Every frame of [path] as raw planar samples in [pixFmt]. +Future _raw(String path, String pixFmt) async { + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-v', 'error', '-i', path, '-f', 'rawvideo', '-pix_fmt', pixFmt, '-'], + environment: WorkerHarness.ffmpegEnv, + stdoutEncoding: null, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + return Uint8List.fromList(r.stdout as List); +} + +/// Compare two decodes of 4:2:0 video sample by sample ([bytes] per sample). +/// Returns the mean absolute luma difference inside [_inner] (in 8-bit +/// levels) and the number of samples that differ outside [_outer]. +({double innerDiff, int outsideChanged, int frames}) _compare(Uint8List a, Uint8List b, {int bytes = 1}) { + expect(a.length, b.length, reason: 'the two encodes differ in size'); + final cw = _w ~/ 2, ch = _h ~/ 2; + final frameSamples = _w * _h + 2 * cw * ch; + final frames = a.length ~/ (frameSamples * bytes); + int sample(Uint8List d, int i) => bytes == 1 ? d[i] : d[i * 2] | (d[i * 2 + 1] << 8); + final scale = bytes == 1 ? 1.0 : 1 / 4.0; + var inner = 0.0, innerN = 0, outside = 0; + for (var f = 0; f < frames; f++) { + final base = f * frameSamples; + for (var y = 0; y < _h; y++) { + for (var x = 0; x < _w; x++) { + final i = base + y * _w + x; + final d = (sample(a, i) - sample(b, i)).abs(); + final inOuter = x >= _outer.x0 && x < _outer.x1 && y >= _outer.y0 && y < _outer.y1; + if (!inOuter && d != 0) outside++; + if (x >= _inner.x0 && x < _inner.x1 && y >= _inner.y0 && y < _inner.y1) { + inner += d * scale; + innerN++; + } + } + } + for (var plane = 0; plane < 2; plane++) { + final pbase = base + _w * _h + plane * cw * ch; + for (var y = 0; y < ch; y++) { + for (var x = 0; x < cw; x++) { + final inOuter = x * 2 >= _outer.x0 && x * 2 < _outer.x1 && y * 2 >= _outer.y0 && y * 2 < _outer.y1; + if (!inOuter && sample(a, pbase + y * cw + x) != sample(b, pbase + y * cw + x)) outside++; + } + } + } + } + return (innerDiff: inner / innerN, outsideChanged: outside, frames: frames); +} + +void main() { + late String baseline; + late Uint8List baselineRaw; + + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_dir).create(recursive: true); + // The base add-on (detector, parser, RestoreFormer++) and the two + // non-commercial models, each with its requirements (the AI runtime). + await WorkerHarness.ensureAddon('face-restoration'); + await WorkerHarness.ensureAddon('face-gfpgan'); + await WorkerHarness.ensureAddon('face-codeformer'); + baseline = await _encode('baseline', null); + baselineRaw = await _raw(baseline, 'yuv420p'); + }); + + test('RestoreFormer++ changes the face and nothing outside it', () async { + final out = await _encode('restoreformer', const FaceRestoreParameters(enabled: true)); + final c = _compare(baselineRaw, await _raw(out, 'yuv420p')); + // ignore: avoid_print + print(' restoreformer: face mean |diff| ${c.innerDiff.toStringAsFixed(2)}, ' + '${c.outsideChanged} samples changed outside the face box'); + expect(c.frames, _count); + expect(c.innerDiff, greaterThan(2.0), reason: 'the face was not restored'); + expect(c.outsideChanged, 0, reason: 'pixels outside the face changed'); + }); + + test('strength 0 is the input, bit-exact', () async { + final out = await _encode('strength0', const FaceRestoreParameters(enabled: true, strength: 0)); + final c = _compare(baselineRaw, await _raw(out, 'yuv420p')); + expect(c.innerDiff, 0); + expect(c.outsideChanged, 0); + }); + + for (final (model, smoothing) in [ + (FaceRestoreModel.gfpgan, true), + (FaceRestoreModel.codeFormer, true), + (FaceRestoreModel.codeFormer, false), + ]) { + test('${model.displayName} runs (smoothing ${smoothing ? 'on' : 'off'})', () async { + final out = await _encode('${model.value}_${smoothing ? 'smooth' : 'plain'}', + FaceRestoreParameters(enabled: true, model: model, temporalSmoothing: smoothing)); + final c = _compare(baselineRaw, await _raw(out, 'yuv420p')); + // ignore: avoid_print + print(' ${model.value}: face mean |diff| ${c.innerDiff.toStringAsFixed(2)}'); + expect(c.innerDiff, greaterThan(2.0), reason: 'the face was not restored'); + expect(c.outsideChanged, 0, reason: 'pixels outside the face changed'); + }); + } + + test('the preview is the encode\'s frame', () async { + const fr = FaceRestoreParameters(enabled: true); + final out = await _encode('preview_ref', fr); + // Two frames or more from the trim's ends, so the encode's temporal + // neighbours (two either side) are the preview's. + for (final frame in [_start + 2, _start + 10]) { + final pv = await WorkerHarness.runPreview(_job('preview', fr, trimmed: false), frame: frame, + // Same budget as _encode: on the Intel-Mac VMs Core ML falls back to + // the CPU, and the v2 runner's first frame alone can take ~10 min. + timeout: const Duration(minutes: 20), + extraEnv: const {'VAPOURBOX_STALL_TIMEOUT_SECS': '1200'}); + expect(pv.success, isTrue, reason: '${pv.error}\n${pv.logs}'); + final preview = await WorkerHarness.imageToRgb24(pv.png!, label: 'fr_pv'); + final encoded = await WorkerHarness.frameRgb24(out, frame - _start, label: 'fr_enc'); + final input = await WorkerHarness.frameRgb24(baseline, frame - _start, label: 'fr_in'); + final diff = WorkerHarness.meanAbsDiff(preview, encoded); + final restored = WorkerHarness.meanAbsDiff(input, encoded); + // ignore: avoid_print + print(' frame $frame: preview vs encode $diff, input vs encode $restored'); + expect(diff, lessThan(0.5), reason: 'the preview shows a different picture'); + expect(restored, greaterThan(diff * 4), reason: 'the comparison cannot tell restored from not'); + } + }, timeout: const Timeout(Duration(minutes: 75))); + + test('a 10-bit source stays 10-bit, restored in the face and exact outside it', () async { + final src10 = p.join(_dir, 'face_10bit.mkv'); + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-y', '-v', 'error', '-i', _fixture, '-vf', 'format=yuv420p10le', '-c:v', 'ffv1', src10], + environment: WorkerHarness.ffmpegEnv, + ); + expect(r.exitCode, 0, reason: '${r.stderr}'); + final off = await _encode('baseline_10bit', null, input: src10, pixFmt: 'yuv420p10le'); + final on = await _encode('restoreformer_10bit', const FaceRestoreParameters(enabled: true), + input: src10, pixFmt: 'yuv420p10le'); + final stream = await WorkerHarness.firstStream(on, selector: 'v:0', entries: ['pix_fmt']); + expect(stream?['pix_fmt'], 'yuv420p10le'); + final c = _compare(await _raw(off, 'yuv420p10le'), await _raw(on, 'yuv420p10le'), bytes: 2); + // ignore: avoid_print + print(' 10-bit: face mean |diff| ${c.innerDiff.toStringAsFixed(2)} (8-bit levels), ' + '${c.outsideChanged} samples changed outside'); + expect(c.innerDiff, greaterThan(2.0)); + expect(c.outsideChanged, 0); + }); +} diff --git a/app/test/integration_filter_parameters_test.dart b/app/test/integration_filter_parameters_test.dart index d961387d..14799647 100644 --- a/app/test/integration_filter_parameters_test.dart +++ b/app/test/integration_filter_parameters_test.dart @@ -24,6 +24,8 @@ import 'package:vapourbox/models/descale_parameters.dart'; import 'package:vapourbox/models/descratch_parameters.dart'; import 'package:vapourbox/models/dynamic_parameters.dart'; import 'package:vapourbox/models/encoding_settings.dart'; +import 'package:vapourbox/models/colorize_parameters.dart'; +import 'package:vapourbox/models/face_restore_parameters.dart'; import 'package:vapourbox/models/frame_rate_parameters.dart'; import 'package:vapourbox/models/filter_schema.dart'; import 'package:vapourbox/models/noise_reduction_parameters.dart'; @@ -35,6 +37,9 @@ import 'package:vapourbox/models/spotless_parameters.dart'; import 'package:vapourbox/models/stabilize_parameters.dart'; import 'package:vapourbox/models/video_job.dart'; +import 'package:vapourbox/services/manifest_addon.dart'; + +import 'support/gpu_expectations.dart'; import 'support/vfr_fixture.dart'; import 'support/worker_harness.dart'; @@ -177,6 +182,8 @@ VideoJob buildJob({ GeometryParameters? geometry, GrainParameters? grain, DescaleParameters? descale, + FaceRestoreParameters? faceRestore, + ColorizeParameters? colorize, }) => VideoJob( id: const Uuid().v4(), inputPath: TestConfig.inputFile, @@ -199,6 +206,8 @@ VideoJob buildJob({ geometry: geometry ?? const GeometryParameters(), grain: grain ?? const GrainParameters(), descale: descale ?? const DescaleParameters(), + faceRestore: faceRestore ?? const FaceRestoreParameters(), + colorize: colorize ?? const ColorizeParameters(), ), encodingSettings: const EncodingSettings( codec: VideoCodec.h264, container: ContainerFormat.mkv, audioMode: AudioMode.none, @@ -451,6 +460,43 @@ void main() { }, timeout: const Timeout(Duration(minutes: 2))); + // App 1.7.0: BT.601 -> BT.709 when an SD source is resized to HD. From the + // panel's own values, so the schema key, the converter and the Rust field + // are all on the trip. The fixture is an untagged 720x576 capture, which + // the worker reads as BT.601 (170m). + test('crop_resize: Convert Colour to BT.709 converts SD resized to HD', () async { + final schema = loadSchema('crop_resize'); + expect(schema.parameters['convertToBt709']?.options, ['auto', 'on', 'off']); + Future scriptFor(String mode, {int targetHeight = 1080}) async { + final dyn = ParameterConverter.fromCropResize(const CropResizeParameters(enabled: true)) + .withValues({ + 'resizeEnabled': true, + 'targetWidth': targetHeight * 16 ~/ 9, + 'targetHeight': targetHeight, + 'convertToBt709': mode, + }); + final job = buildJob( + testName: 'bt709_$mode$targetHeight', + cropResize: ParameterConverter.toCropResize(dyn), + ); + return generateScriptViaWorker(job); + } + + const block = 'RemoveFrameProps(clip, props=["_Matrix", "_Range", "_ColorRange"])'; + final auto = await scriptFor('auto'); + expect(auto, contains(block)); + expect(auto, contains('matrix_in_s="170m"')); + expect(auto, contains('matrix_s="709"')); + expect(auto, contains('range_in_s="limited"')); + // After the resize, so the conversion sees the HD picture. + expect(auto.indexOf(block), greaterThan(auto.indexOf('core.resize.Spline36(\n clip,\n width=target_w'))); + expect(await scriptFor('off'), isNot(contains(block))); + expect(await scriptFor('auto', targetHeight: 576), isNot(contains(block)), + reason: 'an SD-sized output keeps BT.601 under Auto'); + expect(await scriptFor('on', targetHeight: 576), contains(block)); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + // Issue #50: temperature/tint white balance. U carries blue-yellow and V // carries red-cyan, so warm is -U/+V; a sign error here is invisible until // you look at the picture. @@ -1189,6 +1235,124 @@ void main() { print(' PASS'); }, timeout: const Timeout(Duration(minutes: 2))); + test('face restoration: every setting reaches the script, or the job is refused', + () async { + // The models are add-ons of ~700 MB (base + CodeFormer), far too much + // for a push-gate test to download. So: with + // VAPOURBOX_ADDON_MIRROR set (the converted files in addon-staging), + // install them the app's way and check the script; if they are already + // installed, check the script; otherwise check the refusal, which is + // what CI sees, and which names the add-ons and Settings → Add-ons. + loadSchema('face_restore'); + const typed = FaceRestoreParameters( + enabled: true, + model: FaceRestoreModel.codeFormer, + strength: 0.6, + codeFormerFidelity: 0.5, + detectionThreshold: 0.7, + minFaceSize: 40, + maxFaces: 3, + temporalSmoothing: false, + ); + // Through the converter, as the panel would hand it to the worker. + final roundTripped = + ParameterConverter.toFaceRestore(ParameterConverter.fromFaceRestore(typed)); + final job = buildJob(testName: 'face_restore', faceRestore: roundTripped); + + if (Platform.environment['VAPOURBOX_ADDON_MIRROR']?.isNotEmpty ?? false) { + await WorkerHarness.ensureAddon('face-codeformer'); + } + final manifests = { + for (final m in AddonManifest.loadDirectory( + Directory('${TestConfig.projectRoot}/app/assets/addons'))) + m.id: m, + }; + final installed = ['ai-runtime', ...roundTripped.addonIds].every((id) => + ManifestAddonInstaller(manifests[id]!, WorkerHarness.platform) + .isInstalled(WorkerHarness.addonsDirectory)); + + if (installed) { + print(' Generating Face Restoration script...'); + final script = await generateScriptViaWorker(job); + expect(script, contains('import ai_runtime\n')); + expect(script, contains('_face_restore.restore_faces(')); + expect(script, contains('face-codeformer-v0.1.0.onnx')); + expect(script, contains('face-restoration-retinaface-r50.onnx')); + expect(script, contains('face-restoration-bisenet.onnx')); + expect(script, contains('model="codeformer"')); + expect(script, contains('strength=0.600')); + expect(script, contains('fidelity=0.500')); + expect(script, contains('threshold=0.700')); + expect(script, contains('min_face=40')); + expect(script, contains('max_faces=3')); + expect(script, contains('temporal_radius=0')); + } else { + print(' Face restoration add-ons not installed: checking the refusal'); + final result = await WorkerHarness.runJob(job.toJson(), label: 'face_restore'); + expect(result.success, isFalse); + final text = '${result.error}\n${result.logs.join('\n')}'; + expect(text, contains('Face restoration needs')); + expect(text, contains('Settings → Add-ons')); + } + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 5))); + + test('colourisation: every setting reaches the script, or the job is refused', + () async { + // The models are add-ons of 220 MB (DDColor) and 870 MB (DeOldify), + // too much for a push-gate test to download. So, as for Face + // Restoration: with VAPOURBOX_ADDON_MIRROR set, install the add-on the + // app's way and check the script; if it is already installed, check the + // script; otherwise check the refusal (what CI sees), which names the + // add-on and Settings → Add-ons. + loadSchema('colorize'); + const typed = ColorizeParameters( + enabled: true, + model: ColorizeModel.deoldify, + ddcolorRenderSize: 384, + deoldifyRenderFactor: 25, + strength: 0.75, + temporalRadius: 3, + ); + // Through the converter, as the panel would hand it to the worker. + final roundTripped = + ParameterConverter.toColorize(ParameterConverter.fromColorize(typed)); + expect(roundTripped.toJson(), typed.toJson()); + final job = buildJob(testName: 'colorize', colorize: roundTripped); + + if (Platform.environment['VAPOURBOX_ADDON_MIRROR']?.isNotEmpty ?? false) { + await WorkerHarness.ensureAddon('colorize-deoldify'); + } + final manifests = { + for (final m in AddonManifest.loadDirectory( + Directory('${TestConfig.projectRoot}/app/assets/addons'))) + m.id: m, + }; + final installed = ['ai-runtime', roundTripped.addonId].every((id) => + ManifestAddonInstaller(manifests[id]!, WorkerHarness.platform) + .isInstalled(WorkerHarness.addonsDirectory)); + + if (installed) { + print(' Generating Colourisation script...'); + final script = await generateScriptViaWorker(job); + expect(script, contains('import ai_runtime\n')); + expect(script, contains('_colorize.colorize(')); + expect(script, contains('colorize-deoldify-video.onnx')); + expect(script, contains('model="deoldify"')); + expect(script, contains('render_size=400')); + expect(script, contains('strength=0.750')); + expect(script, contains('temporal_radius=3')); + } else { + print(' Colourisation add-on not installed: checking the refusal'); + final result = await WorkerHarness.runJob(job.toJson(), label: 'colorize'); + expect(result.success, isFalse); + final text = '${result.error}\n${result.logs.join('\n')}'; + expect(text, contains('Colourisation needs')); + expect(text, contains('Settings → Add-ons')); + } + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 10))); + test('sharpen: aWarpSharp2 params', () async { loadSchema('sharpen'); final typed = const SharpenParameters( @@ -1487,7 +1651,10 @@ void main() { // Deinterlacing off + a detected field order: every method runs between // SeparateFields and DoubleWeave (docs/reports/2026-11-deblock-order). loadSchema('deblock'); - for (final method in DeblockMethod.values) { + // mpeg2stinx is a cross-field filter and always takes the whole frame + // (asserted in its own test below). + for (final method + in DeblockMethod.values.where((m) => m != DeblockMethod.mpeg2stinx)) { final job = buildJob( testName: 'deblock_per_field_${method.name}', deblock: DeblockParameters(enabled: true, method: method), @@ -1501,6 +1668,7 @@ void main() { DeblockMethod.deblockQed => 'haf.Deblock_QED(', DeblockMethod.deblock => 'core.deblock.Deblock(', DeblockMethod.dctFilter => 'core.dctf.DCTFilter(', + DeblockMethod.mpeg2stinx => throw StateError('excluded above'), }; final at = script.indexOf(call); expect(at > split && at < weave, isTrue, @@ -1516,6 +1684,115 @@ void main() { print(' PASS'); }, timeout: const Timeout(Duration(minutes: 2))); + // --- deps 1.16.0: SangNom, vszip NLMeans, vs-jetpack ports (app 1.7.0) --- + + test('anti-alias: SangNom AA, strength reaching both axes', () async { + loadSchema('anti_alias'); + final job = buildJob( + testName: 'aa_sangnom', + antiAlias: const AntiAliasParameters( + enabled: true, + method: AntiAliasMethod.sangnom, + sangnomAa: 72, + ), + ); + final script = await generateScriptViaWorker(job); + expect(script, contains('core.sangnom.SangNom(_aa, order=1, aa=72)')); + expect(script, + contains('core.sangnom.SangNom(core.std.Transpose(_aa), order=1, aa=72)')); + expect(script, isNot(contains('haf.santiag('))); + expect(script, isNot(contains('haf.daa('))); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + + test('noise reduction: NLMeans (CPU) uses the KNLMeans settings, no OpenCL', + () async { + loadSchema('noise_reduction'); + final job = buildJob( + testName: 'nr_nlm_cpu', + noiseReduction: const NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.nlMeans, + knlmH: 2.5, + knlmD: 2, + knlmChroma: false, + ), + ); + // No OpenCL involved (test_201 checks it runs where KNLMeans is refused). + final script = await generateScriptViaWorker(job); + expect(script, + contains('core.vszip.NLMeans(clip, d=2, a=2, s=4, h=2.50, channels="Y")')); + expect(script, contains(' if False:')); + expect(script, isNot(contains('core.knlm.KNLMeansCL('))); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + + test('dehalo: vine params reach the ported block', () async { + loadSchema('dehalo'); + final job = buildJob( + testName: 'dehalo_vine', + dehalo: const DehaloParameters( + enabled: true, + method: DehaloMethod.vine, + vineStrength: 20.0, + vineSharp: 0.75, + vineSigma: 1.5, + ), + ); + final script = await generateScriptViaWorker(job); + expect(script, contains('_vine_strength = 20.000')); + expect(script, contains('_vine_sharp = 0.750')); + expect(script, contains('_vine_sigma = 1.500')); + expect(script, contains('core.vszip.NLMeans(_vine_y, d=0, a=2, s=0, h=_vine_strength)')); + expect(script, isNot(contains('haf.DeHalo_alpha('))); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + + test('deblock: mpeg2stinx takes the whole frame with the detected order', + () async { + loadSchema('deblock'); + final job = buildJob( + testName: 'deblock_mpeg2stinx', + deblock: const DeblockParameters( + enabled: true, method: DeblockMethod.mpeg2stinx), + ).copyWith(detectedFieldOrder: FieldOrder.bottomFieldFirst); + final script = await generateScriptViaWorker(job); + expect(script, contains('_stx_tff = False')); + expect(script, + contains('clip = core.std.SetFieldBased(core.std.Merge(_stx_1, _stx_2), 0)')); + expect(script, isNot(contains('_deblock_per_field')), + reason: 'mpeg2stinx is a cross-field filter: never per field'); + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + + test('ivtc: fade fix follows field matching, in the chosen mode', () async { + loadSchema('deinterlace'); + for (final (fix, mode) in [(true, 'match'), (false, 'average')]) { + final job = buildJob( + testName: 'ivtc_fades_$fix', + deinterlace: QTGMCParameters( + enabled: true, + method: DeinterlaceMethod.ivtc, + tff: true, + ivtcFixFades: fix, + ivtcFadeMode: mode, + ), + ); + final script = await generateScriptViaWorker(job); + if (fix) { + final vfm = script.indexOf('core.vivtc.VFM('); + final at = script.indexOf('_fade_fields = core.std.SeparateFields('); + final decimate = script.indexOf('core.vivtc.VDecimate('); + expect(at, greaterThan(vfm)); + expect(at, lessThan(decimate)); + expect(script, contains("if 'match' == 'average' else min(a, b)")); + } else { + expect(script, isNot(contains('_fade_fields'))); + } + } + print(' PASS'); + }, timeout: const Timeout(Duration(minutes: 2))); + test('grain: AddGrain params are not depth-scaled', () async { loadSchema('grain'); final job = buildJob( @@ -1705,5 +1982,74 @@ void main() { expect(script, isNot(contains('{{#DEINT'))); print(' PASS'); }, timeout: const Timeout(Duration(minutes: 2))); + + // --- AI upscale strength (1.7.0) --- + // Below 1 the network's result is blended with a Spline36 upscale of the + // same frame in one std.Merge; at 1 nothing is added. Needs VSNCNN to run + // here (the worker refuses an AI job it cannot run), so it follows the + // GPU rule: tagged `gpu` (left out on the GPU-less CI runners), skipped + // without Vulkan unless VAPOURBOX_EXPECT_VULKAN=1. + test('crop_resize: AI strength blends a Spline36 upscale into the AI result', () async { + if (!await requireGpuPlugin('--probe-upscale')) return; + // Real-ESRGAN's anime model is an add-on (deps 1.16.0 on). + await WorkerHarness.ensureAddon('upscale-anime'); + CropResizeParameters crop(double strength) => ParameterConverter.toCropResize( + ParameterConverter.fromCropResize(CropResizeParameters( + enabled: true, + useIntegerUpscale: true, + upscaleMethod: UpscaleMethod.realEsrgan, + aiStrength: strength))); + final blended = await generateScriptViaWorker( + buildJob(testName: 'ai_strength', cropResize: crop(0.7))); + expect(blended, contains('core.ncnn.Model(_ai_rgb, network_path=_ai_model')); + expect(blended, contains('clip = core.std.Merge(core.resize.Spline36(_ai_src, ' + 'width=clip.width, height=clip.height), clip, weight=0.700)')); + final full = await generateScriptViaWorker( + buildJob(testName: 'ai_strength_full', cropResize: crop(1.0))); + expect(full, isNot(contains('core.std.Merge(core.resize.Spline36(_ai_src'))); + expect(full, isNot(contains('AI_STRENGTH'))); + }, timeout: const Timeout(Duration(minutes: 2)), tags: ['gpu']); + + // --- Add-on AI upscalers (1.7.0) --- + // Real-ESRGAN General (4x only: resampled to 2x after the network, the + // denoise variant on request) and Live Action SPAN (2x only: twice for + // 4x), each from the upscale-general add-on's directory. Installs the + // add-on (11 MB) with the app's own installer, so a dead link or changed + // hash fails here. + test('crop_resize: add-on upscalers load their models with their factor rules', () async { + if (!await requireGpuPlugin('--probe-upscale')) return; + await WorkerHarness.ensureAddon('upscale-general'); + // The models the script runs, by file name, each checked to come from + // the add-on's own directory. + List models(String script) { + final line = RegExp(r'for _ai_model in \[(.*)\]:').firstMatch(script)!.group(1)!; + final names = []; + for (final m in RegExp(r"'([^']*)'").allMatches(line)) { + final path = m.group(1)!.replaceAll(r'\\', '/'); + expect(path, contains('upscale-general/'), reason: path); + names.add(path.split('/').last); + } + return names; + } + + Future script(UpscaleMethod m, int factor, {bool denoise = false}) => + generateScriptViaWorker(buildJob( + testName: 'addon_upscale_${m.name}_$factor', + cropResize: CropResizeParameters( + enabled: true, + useIntegerUpscale: true, + upscaleMethod: m, + upscaleFactor: factor, + generalDenoise: denoise))); + final general2 = await script(UpscaleMethod.realEsrganGeneral, 2); + expect(models(general2), ['upscale-realesr-general-x4v3.onnx']); + expect(general2, contains('_ai_rgb = core.resize.Spline36(_ai_rgb, width=_ai_rgb.width * 2 // _ai_scale')); + final general4 = await script(UpscaleMethod.realEsrganGeneral, 4, denoise: true); + expect(models(general4), ['upscale-realesr-general-wdn-x4v3.onnx']); + expect(general4, isNot(contains('// _ai_scale'))); + final span4 = await script(UpscaleMethod.liveAction, 4); + expect(models(span4), List.filled(2, '2xLiveActionV1_SPAN_490000.onnx')); + expect(span4, isNot(contains('// _ai_scale'))); + }, timeout: const Timeout(Duration(minutes: 5)), tags: ['gpu']); }); } diff --git a/app/test/integration_frame_mapping_test.dart b/app/test/integration_frame_mapping_test.dart index 05ccbb21..a42c3780 100644 --- a/app/test/integration_frame_mapping_test.dart +++ b/app/test/integration_frame_mapping_test.dart @@ -569,9 +569,12 @@ void main() { }, timeout: const Timeout(Duration(minutes: 10))); }); - // RIFE needs a Vulkan device. CI provisions one wherever RIFE ships (see - // support/gpu_expectations.dart), so there this group must run; elsewhere it - // skips, and integration_gpu_plugins_test.dart covers the refusal. Its count must match FrameMap's + // RIFE needs a Vulkan device, so each test here is tagged `gpu` and the GPU-less + // CI runners leave the group out, encode included (dart_test.yaml; a group + // with no selected tests never runs its setUpAll). Elsewhere it skips without a + // device, or fails under VAPOURBOX_EXPECT_VULKAN=1 (see + // support/gpu_expectations.dart); integration_gpu_plugins_test.dart covers + // the refusal. Its count must match FrameMap's // Retime exactly — floor(n * 2) here — and since every other output frame is // a real one passed through untouched, the preview of source frame S must be // output frame 2S, picture for picture. @@ -607,7 +610,10 @@ void main() { available = probe.stdout.toString().contains('"available":true'); print(' RIFE probe: ${probe.stdout.toString().trim()}'); if (!available) return; - // On the GPU-less runners RIFE runs on lavapipe (CPU Vulkan), alongside + // RIFE's model is an add-on (frame-rate-rife), installed with the app's + // own installer from the URL its manifest pins. + await WorkerHarness.ensureAddon('frame-rate-rife'); + // On a CPU Vulkan driver (lavapipe) RIFE is slow, alongside // the other heavy files. This 75-frame encode passed in under two // minutes on one Linux job and hit the default 5 in another, so it gets // the AI-upscale encodes' budget. @@ -620,10 +626,10 @@ void main() { }); test('25 -> 50 emits exactly twice the frames', () async { - // Skips only where no Vulkan is expected; fails where CI provisions it. + // Skips without a device, unless VAPOURBOX_EXPECT_VULKAN=1. if (!available && !await requireGpuPlugin('--probe-rife')) return; expect(await _countFrames(encoded), _interlacedFrames * 2); - }); + }, tags: ['gpu']); for (final sourceFrame in [12, 36]) { test('source frame $sourceFrame previews as output frame ${sourceFrame * 2}', @@ -650,7 +656,7 @@ void main() { expect(candidates[expectedIndex]!, lessThan(2.0), reason: 'output frame $expectedIndex is a real frame passed ' 'through, so the preview should be the same picture'); - }, timeout: const Timeout(Duration(minutes: 6))); + }, timeout: const Timeout(Duration(minutes: 6)), tags: ['gpu']); } // Issue #129: the frame RIFE makes between two source frames can be @@ -675,6 +681,6 @@ void main() { expect(best.key, 25); // RIFE runs on the GPU in fp16; allow its run-to-run noise, no more. expect(candidates[25]!, lessThan(2.0)); - }, timeout: const Timeout(Duration(minutes: 6))); + }, timeout: const Timeout(Duration(minutes: 6)), tags: ['gpu']); }); } diff --git a/app/test/integration_gpu_plugins_test.dart b/app/test/integration_gpu_plugins_test.dart index 8f52ab67..38e9f7dd 100644 --- a/app/test/integration_gpu_plugins_test.dart +++ b/app/test/integration_gpu_plugins_test.dart @@ -1,7 +1,9 @@ // RIFE and AI upscaling (Real-ESRGAN / Real-CUGAN / AnimeJaNai), through the -// real worker. AnimeJaNai's models are an add-on: the harness installs them -// from the upstream download the app's manifest pins, so this also checks that -// the pinned URL and hashes still hold. +// real worker. Every model is an add-on (the plugins are in the deps bundle, +// their models not): the harness installs each with the app's own installer +// from the URL its manifest pins, so this also checks that the pinned URLs and +// hashes still hold — and the probes must answer before any is installed, +// since they run a stand-in network of their own. // // Both run a network through ncnn on Vulkan, and ncnn segfaults without a // Vulkan device rather than raising. So what must hold on EVERY platform is @@ -70,17 +72,18 @@ VideoJob _upscaleJob(UpscaleMethod method, {String noise = 'conservative'}) => _ 'upscale_${method.name}', ProcessingPipeline( deinterlace: const QTGMCParameters(enabled: false), - // Cropped to 240x192 first: on a CPU Vulkan driver (lavapipe, on the - // GPU-less CI runners) one whole 720x576 Real-CUGAN frame took over - // five minutes on Windows. This test only needs a real frame out of - // the GPU at the right size; full-size encodes are the heavy suite's. + // Cropped to 120x96 first: on a CPU Vulkan driver (Windows lavapipe, + // which CI once used) one whole 720x576 Real-CUGAN frame took over + // five minutes on Windows, and even 240x192 took 2.5 to over 5. This + // test only needs a real frame out of the GPU at the right size; + // full-size encodes are the heavy suite's. cropResize: CropResizeParameters( enabled: true, cropEnabled: true, - cropLeft: 240, - cropRight: 240, - cropTop: 192, - cropBottom: 192, + cropLeft: 300, + cropRight: 300, + cropTop: 240, + cropBottom: 240, useIntegerUpscale: true, upscaleMethod: method, upscaleFactor: 2, @@ -98,34 +101,45 @@ VideoJob _upscaleJob(UpscaleMethod method, {String noise = 'conservative'}) => _ class _Feature { final GpuPlugin plugin; final VideoJob Function() job; + + /// The plugin's directory in the deps bundle. final String dir; + + /// The add-on holding its models, which the script must read them from. + final String addon; final List scriptMust; final (int, int) previewSize; - final bool needsAnimeJanai; - const _Feature(this.plugin, this.job, this.dir, this.scriptMust, this.previewSize, - {this.needsAnimeJanai = false}); + const _Feature(this.plugin, this.job, this.dir, this.addon, this.scriptMust, this.previewSize); + + Future install() => addon == 'animejanai' + ? WorkerHarness.ensureAnimeJanai() + : WorkerHarness.ensureAddon(addon); } final _features = { - 'RIFE': _Feature(GpuPlugin.rife, _rifeJob, 'rife', [ + 'RIFE': _Feature(GpuPlugin.rife, _rifeJob, 'rife', rifeAddonId, [ 'core.rife.RIFE(_rife_rgb, model_path=', + 'rife-v4.26_ensembleFalse', 'fps_num=50, fps_den=1, sc=True)', 'core.misc.SCDetect(clip, threshold=0.1)', ], (720, 576)), - 'Real-ESRGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realEsrgan), 'ncnn', [ - 'RealESRGANv2-animevideo-xsx2.onnx', + 'Real-ESRGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realEsrgan), 'ncnn', + 'upscale-anime', [ + 'upscale-RealESRGANv2-animevideo-xsx2.onnx', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', - ], (480, 384)), - 'Real-CUGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realCugan, noise: 'denoise3x'), 'ncnn', [ - 'up2x-latest-denoise3x.onnx', + ], (240, 192)), + 'Real-CUGAN': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.realCugan, noise: 'denoise3x'), + 'ncnn', 'upscale-anime', [ + 'upscale-cugan-up2x-latest-denoise3x.onnx', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', - ], (480, 384)), - 'AnimeJaNai': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.animeJanai), 'animejanai', [ + ], (240, 192)), + 'AnimeJaNai': _Feature(GpuPlugin.upscale, () => _upscaleJob(UpscaleMethod.animeJanai), 'ncnn', + 'animejanai', [ '2x_AnimeJaNai_SD_V1beta34_Compact.onnx', '2x_AnimeJaNai_HD_V3_Compact.onnx', '_ai_model = _ai_model[0] if _ai_rgb.height <= 576 else _ai_model[1]', 'core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True', - ], (480, 384), needsAnimeJanai: true), + ], (240, 192)), }; void main() { @@ -147,7 +161,7 @@ void main() { reason: '$platformId ships no ${plugin.label}, and must say so'); } else if (expectVulkan) { expect(probe['available'], isTrue, - reason: 'CI provisions Vulkan here: ${probe['message']}'); + reason: 'VAPOURBOX_EXPECT_VULKAN=1: ${probe['message']}'); } if (probe['available'] != true) { final status = GpuPluginStatus.fromJson(plugin, probe); @@ -159,7 +173,7 @@ void main() { 'reason from the user: ${status.message}'); } // The first probe initialises the device and compiles shaders; on a - // CPU Vulkan driver (lavapipe on CI) that took 30.4 s on Windows. + // CPU Vulkan driver (Mesa's lavapipe) that took 30.4 s on Windows. }, timeout: const Timeout(Duration(minutes: 3))); }); } @@ -206,25 +220,31 @@ void main() { test('where it runs, the script loads it explicitly and calls it', () async { if (!await requireGpuPlugin(f.plugin.probeFlag)) return; - if (f.needsAnimeJanai) await WorkerHarness.ensureAnimeJanai(); + await f.install(); final script = await WorkerHarness.generateScript(f.job().toJson()); expect(script, contains('core.std.LoadPlugin(')); - expect(script, contains('${p.separator}${f.dir}${p.separator}'.replaceAll(r'\', r'\\'))); + String inPath(String s) => s.replaceAll(r'\', r'\\'); + expect(script, contains(inPath('${p.separator}${f.dir}${p.separator}'))); for (final must in f.scriptMust) { expect(script, contains(must)); } + // The model comes from its add-on, never from the deps bundle (which + // carried RIFE's and the anime upscalers' until deps 1.16.0). + expect(script, contains(inPath('${p.separator}addons${p.separator}${f.addon}${p.separator}'))); + expect(script, isNot(contains(inPath('${p.separator}models${p.separator}'))), + reason: 'a model path inside the deps bundle'); if (platformId.startsWith('macos')) { expect(script, contains('moltenvk'), reason: 'MoltenVK must be preloaded on macOS'); } - }, timeout: const Timeout(Duration(minutes: 5))); + }, timeout: const Timeout(Duration(minutes: 5)), tags: ['gpu']); test('where it runs, a real frame comes out of the GPU at the right size', () async { if (!await requireGpuPlugin(f.plugin.probeFlag)) return; - if (f.needsAnimeJanai) await WorkerHarness.ensureAnimeJanai(); + await f.install(); final pv = await WorkerHarness.runPreview(f.job().toJson(), frame: 21, label: 'gpu_$name'); expect(pv.success, isTrue, reason: pv.errorTail); expect(_pngSize(pv.png!), f.previewSize); - }, timeout: const Timeout(Duration(minutes: 5))); + }, timeout: const Timeout(Duration(minutes: 5)), tags: ['gpu']); }); } } diff --git a/app/test/integration_high_bit_depth_filters_test.dart b/app/test/integration_high_bit_depth_filters_test.dart index cd3e31e1..bfb7b1cd 100644 --- a/app/test/integration_high_bit_depth_filters_test.dart +++ b/app/test/integration_high_bit_depth_filters_test.dart @@ -46,6 +46,7 @@ import 'package:vapourbox/models/chroma_fix_parameters.dart'; import 'package:vapourbox/models/color_correction_parameters.dart'; import 'package:vapourbox/models/crop_resize_parameters.dart'; import 'package:vapourbox/models/deband_parameters.dart'; +import 'package:vapourbox/models/anti_alias_parameters.dart'; import 'package:vapourbox/models/deblock_parameters.dart'; import 'package:vapourbox/models/dehalo_parameters.dart'; import 'package:vapourbox/models/deflicker_parameters.dart'; @@ -289,6 +290,7 @@ ProcessingPipeline _only({ GhostRemovalParameters ghostRemoval = const GhostRemovalParameters(), FrameRateParameters frameRate = const FrameRateParameters(), DescaleParameters descale = const DescaleParameters(), + AntiAliasParameters antiAlias = const AntiAliasParameters(), }) { return ProcessingPipeline( deinterlace: deinterlace, @@ -308,6 +310,7 @@ ProcessingPipeline _only({ ghostRemoval: ghostRemoval, frameRate: frameRate, descale: descale, + antiAlias: antiAlias, ); } @@ -470,6 +473,51 @@ List<_Pass> _passes() => [ needsKnlmDevice: true, ), + // NLMeans on the CPU (vszip, deps 1.16.0): KNLMeansCL's arguments, every + // depth natively, h depth-independent. Same h as the KNLMeans case above + // so the two read alike; unlike it, this one runs on every runner. + _Pass( + 'noise_nlmeans_cpu', + _only( + noiseReduction: const NoiseReductionParameters( + enabled: true, + method: NoiseReductionMethod.nlMeans, + knlmH: 3.0, + ), + ), + ), + + // SangNom AA: `aa` is in 8-bit units and the plugin scales it itself + // (measured: an 8-bit and a 10-bit run agree to 0.05 levels); the 2x + // Spline36 round trip adds the rest. + _Pass( + 'anti_alias_sangnom', + _only( + antiAlias: const AntiAliasParameters( + enabled: true, + method: AntiAliasMethod.sangnom, + ), + ), + ), + + // The IVTC fade fix works in float, so a correct block is depth-blind. + // This fixture holds no fade, and on a frame that is not fading the fix + // changes ~0.1 levels by design — so it cannot be required to change + // the picture; the parity check is what this case is for (the fix's + // effect is integration_classical_filters_test's). + _Pass( + 'ivtc_fix_fades', + _only( + deinterlace: const QTGMCParameters( + enabled: true, + method: DeinterlaceMethod.ivtc, + tff: true, + ivtcFixFades: true, + ), + ), + changesPicture: false, + ), + // Deflicker measures frame brightness against a windowed average, so its // correction is a ratio and should be depth-independent by construction. _Pass( diff --git a/app/test/integration_noise_halo_test.dart b/app/test/integration_noise_halo_test.dart new file mode 100644 index 00000000..85c0d968 --- /dev/null +++ b/app/test/integration_noise_halo_test.dart @@ -0,0 +1,210 @@ +/// The pass list's noise and halo measurement (`vapourbox-worker +/// --detect-noise-halo`, app 1.7.0), run through the real worker on fixtures +/// whose noise and sharpening are known. +/// +/// Built here from the repo's real footage: the VHS capture +/// `interlaced_test.avi` (interlaced 4:2:2) as it is and with Gaussian noise of +/// sigma 6 added to luma (8-bit, and decoded to 10-bit first), and the DVB-T +/// broadcast as it is and through ffmpeg's `unsharp` at amount 1.5. Every +/// fixture is stored losslessly. The scores must land on the side of the +/// worker's thresholds the report says they do +/// (docs/reports/2026-11-noise-halo-scores): clean sources recommend nothing, +/// heavy noise recommends Noise Reduction only, heavy sharpening Dehalo only. +/// +/// The unit tests in `worker/src/noise_halo.rs` cover the same traps on +/// synthetic frames; this file proves the decode (in the source's own format +/// and depth), the per-field measurement and the JSON as the app runs them. +/// +/// Heavy (decodes and encodes video) — runs in the nightly workflow. +@Tags(['heavy']) +library; + +// ignore_for_file: avoid_print — these tests print diagnostics to the test log. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; +import 'dart:typed_data'; + +import 'package:flutter_test/flutter_test.dart'; +import 'package:uuid/uuid.dart'; +import 'package:vapourbox/models/noise_halo_report.dart'; +import 'package:vapourbox/models/pass_relevance.dart'; +import 'package:vapourbox/models/processing_pipeline.dart'; +import 'package:vapourbox/services/field_order_detector.dart'; + +import 'support/worker_harness.dart'; + +String get _outDir => '${WorkerHarness.outputDir}/noise_halo'; +String get _vhs => '${WorkerHarness.repoRoot}/Tests/TestResources/interlaced_test.avi'; +String get _dvbt => '${WorkerHarness.repoRoot}/Tests/TestResources/pal-dvbt-fieldcoded-25i.ts'; + +const _w = 720; +const _h = 576; +const _frames = 60; + +/// Run ffmpeg, failing the test on an error. +Future _ffmpeg(List args) async { + final r = await Process.run(WorkerHarness.ffmpegPath, ['-y', '-v', 'error', ...args], + environment: WorkerHarness.ffmpegEnv); + if (r.exitCode != 0) throw Exception('ffmpeg ${args.join(' ')}: ${r.stderr}'); +} + +/// The first [_frames] frames of [src], stored losslessly, optionally through +/// an ffmpeg filter. +Future _copy(String src, String name, {String? vf}) async { + final path = '$_outDir/$name.mkv'; + await _ffmpeg([ + '-i', src, '-map', '0:v:0', '-frames:v', '$_frames', + if (vf != null) ...['-vf', vf], + '-field_order', 'tt', '-c:v', 'ffv1', path, + ]); + return path; +} + +/// [src] decoded as planar 4:2:2 at [bits], with Gaussian noise of [sigma] +/// (8-bit levels) added to luma, stored losslessly. +Future _noisy(String src, String name, double sigma, {int bits = 8}) async { + final pix = bits == 8 ? 'yuv422p' : 'yuv422p${bits}le'; + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-v', 'error', '-i', src, '-frames:v', '$_frames', '-f', 'rawvideo', '-pix_fmt', pix, '-'], + environment: WorkerHarness.ffmpegEnv, + stdoutEncoding: null, + ); + if (r.exitCode != 0) throw Exception('decoding $src: ${r.stderr}'); + final bytes = Uint8List.fromList(r.stdout as List); + final bps = bits == 8 ? 1 : 2; + final frame = _w * _h * 2 * bps; // 4:2:2: luma + two half-width planes + final scale = 1 << (bits - 8); + final max = (1 << bits) - 1; + final rnd = Random(11); + double gauss() { + final u = rnd.nextDouble() + 1e-12, v = rnd.nextDouble(); + return sqrt(-2 * log(u)) * cos(2 * pi * v); + } + + final data = ByteData.sublistView(bytes); + for (var off = 0; off + frame <= bytes.length; off += frame) { + for (var i = 0; i < _w * _h; i++) { + if (bps == 1) { + bytes[off + i] = (bytes[off + i] + sigma * gauss()).round().clamp(0, max); + } else { + final p = off + 2 * i; + final v = data.getUint16(p, Endian.little); + data.setUint16(p, (v + sigma * scale * gauss()).round().clamp(0, max), Endian.little); + } + } + } + final path = '$_outDir/$name.mkv'; + final p = await Process.start( + WorkerHarness.ffmpegPath, + [ + '-y', '-v', 'error', '-f', 'rawvideo', '-pix_fmt', pix, '-s', '${_w}x$_h', '-r', '25', + '-i', '-', '-field_order', 'tt', '-c:v', 'ffv1', '-pix_fmt', pix, path, + ], + environment: WorkerHarness.ffmpegEnv, + ); + final err = p.stderr.transform(utf8.decoder).join(); + p.stdout.drain(); + p.stdin.add(bytes); + await p.stdin.close(); + if (await p.exitCode != 0) throw Exception('encoding $name: ${await err}'); + return path; +} + +/// Run `--detect-noise-halo` on [path] as the app does; returns the report. +Future> _detect(String path, String pixFmt, {String fieldOrder = 'tff'}) async { + final job = { + 'id': const Uuid().v4(), + 'inputPath': path, + 'outputPath': '$_outDir/unused.mkv', + 'qtgmcParameters': {}, + 'encodingSettings': {}, + 'detectedFieldOrder': fieldOrder, + 'totalFrames': _frames, + 'inputFrameRate': 25.0, + 'inputWidth': _w, + 'inputHeight': _h, + 'inputPixelFormat': pixFmt, + }; + final config = File('$_outDir/${job['id']}.json'); + await config.writeAsString(jsonEncode(job)); + final sw = Stopwatch()..start(); + final r = await Process.run( + WorkerHarness.workerPath, + ['--detect-noise-halo', '--config', config.path], + environment: WorkerHarness.workerEnv, + workingDirectory: File(WorkerHarness.workerPath).parent.path, + ); + sw.stop(); + await config.delete(); + final line = const LineSplitter() + .convert(r.stdout as String) + .lastWhere((l) => l.trim().startsWith('{'), orElse: () => '{}'); + final report = jsonDecode(line) as Map; + expect(report['error'], isNull, reason: '${r.stderr}'); + print(' ${path.split('/').last}: ${sw.elapsedMilliseconds} ms, noise ${report['noise']} ' + '(${report['noiseHigh']}), halo ${report['halo']} (${report['haloHigh']}, ' + '${report['haloEdges']} edges), pairs ${report['usablePairs']}'); + return report; +} + +/// What the pass list would badge for [report] on a PAL interlaced source. +Set _suggested(Map report) { + const info = VideoInfo( + width: _w, height: _h, frameRate: 25, duration: 2.4, frameCount: _frames, + codec: 'ffv1', pixelFormat: 'yuv422p', scanType: ScanType.interlaced, hasAudio: false, + ); + final measured = NoiseHaloReport.fromJson(report); + return { + for (final p in [PassType.noiseReduction, PassType.dehalo]) + if (relevanceFor(p, info, measured: measured).isRecommended) p, + }; +} + +void main() { + setUpAll(() async { + await WorkerHarness.ensureReady(); + await Directory(_outDir).create(recursive: true); + }); + + const timeout = Timeout(Duration(minutes: 5)); + + test('the VHS capture as it is: nothing recommended', () async { + final r = await _detect(await _copy(_vhs, 'vhs'), 'yuv422p'); + expect(r['interlaced'], isTrue); + expect(r['noise'] as num, lessThan(2.5), reason: 'report: 1.40 on this clip'); + expect(r['noiseHigh'], isFalse); + expect(r['haloHigh'], isFalse); + expect(_suggested(r), isEmpty); + }, timeout: timeout); + + test('the VHS capture with sigma 6 noise: Noise Reduction only', () async { + final r = await _detect(await _noisy(_vhs, 'vhs_n6', 6), 'yuv422p'); + // Motion compensation reads white noise about 6 % low (report). + expect(r['noise'] as num, inInclusiveRange(4.8, 7.0)); + expect(r['noiseHigh'], isTrue); + expect(r['haloHigh'], isFalse, reason: 'noise is not a halo'); + expect(_suggested(r), {PassType.noiseReduction}); + }, timeout: timeout); + + test('10-bit: the same noise reads the same in 8-bit units', () async { + final r8 = await _detect(await _noisy(_vhs, 'vhs_n6_8', 6), 'yuv422p'); + final r10 = await _detect(await _noisy(_vhs, 'vhs_n6_10', 6, bits: 10), 'yuv422p10le'); + expect(r10['noiseHigh'], isTrue); + expect(((r10['noise'] as num) - (r8['noise'] as num)).abs(), lessThan(0.4)); + }, timeout: timeout); + + test('the DVB-T broadcast: clean, and sharpened reads as halos', () async { + final clean = await _detect(await _copy(_dvbt, 'dvbt'), 'yuv420p'); + expect(clean['haloEdges'] as int, greaterThanOrEqualTo(200)); + expect(clean['halo'] as num, lessThan(3.0), reason: 'report: -2.2'); + expect(_suggested(clean), isEmpty); + final sharp = await _detect(await _copy(_dvbt, 'dvbt_usm', vf: 'unsharp=5:5:1.5:5:5:0'), 'yuv420p'); + expect(sharp['halo'] as num, greaterThan(10.0), reason: 'report: 15.7'); + expect(sharp['haloHigh'], isTrue); + expect(sharp['noiseHigh'], isFalse); + expect(_suggested(sharp), {PassType.dehalo}); + }, timeout: timeout); +} diff --git a/app/test/manifest_addon_test.dart b/app/test/manifest_addon_test.dart new file mode 100644 index 00000000..3db854c1 --- /dev/null +++ b/app/test/manifest_addon_test.dart @@ -0,0 +1,421 @@ +// Manifest add-ons (assets/addons/*.json, ManifestAddonInstaller): checked +// without the network, through the mirror (VAPOURBOX_ADDON_MIRROR's code +// path) and a fake downloader. Every file is hash-checked before anything is +// unpacked, an install is all or nothing, a manifest version bump reinstalls, +// and the app and the worker see the same set of manifests. +import 'dart:convert'; +import 'dart:io'; + +import 'package:archive/archive.dart'; +import 'package:crypto/crypto.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:path/path.dart' as p; +import 'package:vapourbox/services/addon_manager.dart'; +import 'package:vapourbox/services/addon_service.dart'; +import 'package:vapourbox/services/manifest_addon.dart'; + +String get _root => + Directory.current.path.endsWith('app') ? Directory.current.parent.path : Directory.current.path; + +/// Serves files from a map of url → bytes, checking the hash as AddonManager does. +class _Serve extends AddonManager { + final Map> bodies; + final List fetched = []; + _Serve(this.bodies); + @override + Future downloadFile(String url, File destination, + {int? expectedSize, String? expectedSha256, Map? extraHeaders}) async { + fetched.add(url); + final body = bodies[url]!; + if (expectedSha256 != null && sha256.convert(body).toString() != expectedSha256) { + throw StateError('SHA256 mismatch'); + } + await destination.writeAsBytes(body); + } +} + +List _zip(Map> files) { + final archive = Archive(); + for (final MapEntry(key: name, value: bytes) in files.entries) { + archive.addFile(ArchiveFile(name, bytes.length, bytes)); + } + return ZipEncoder().encode(archive)!; +} + +String _sha(List b) => sha256.convert(b).toString(); + +Map _entry(String name, List bytes, {List? platforms, Object? extract}) => { + 'name': name, + 'url': 'https://example.invalid/dl/$name', + 'size': bytes.length, + 'sha256': _sha(bytes), + if (platforms != null) 'platforms': platforms, + if (extract != null) 'extract': extract, + }; + +Map _manifest({ + String id = 'test-addon', + String version = '1.0.0', + String kind = 'files', + Object platforms = 'all', + required List> files, + List requires = const [], +}) => + { + 'id': id, + 'version': version, + 'name': 'Test add-on', + 'feature': 'Testing', + 'description': 'A test.', + 'licence': 'MIT', + 'credit': 'Tests', + 'noticeName': 'Test', + 'homepage': 'https://example.invalid', + 'requires': requires, + 'kind': kind, + 'platforms': platforms, + 'files': files, + }; + +void main() { + late Directory tmp; + late Directory addons; + // Every installer here ignores VAPOURBOX_ADDON_MIRROR (an empty mirror), + // so a run with it set for the integration tests still exercises the + // fake server; the mirror test passes its own directory. + ManifestAddonInstaller newInstaller(AddonManifest m, String platformId, + {String? osVersion, String? mirror}) => + ManifestAddonInstaller(m, platformId, osVersion: osVersion, mirror: mirror ?? ''); + + setUp(() { + tmp = Directory.systemTemp.createTempSync('vb_manifest_addon_'); + addons = Directory(p.join(tmp.path, 'addons'))..createSync(); + }); + tearDown(() => tmp.deleteSync(recursive: true)); + + final model = utf8.encode('model weights'); + final other = utf8.encode('other model'); + + group('files add-on', () { + test('installs each file, hash-checked, behind a marker; removes cleanly', () async { + final m = AddonManifest.fromJson(_manifest(files: [_entry('a.onnx', model)])); + final server = _Serve({'https://example.invalid/dl/a.onnx': model}); + final service = AddonService.forTest( + addonsDir: addons, + installers: [newInstaller(m, 'linux-x64')], + downloader: server); + await service.load(); + expect(service.isInstalled('test-addon'), isFalse); + expect(service.info('test-addon')!.downloadBytes, model.length); + expect(await service.install('test-addon'), isTrue); + final dir = Directory(p.join(addons.path, 'test-addon')); + expect(File(p.join(dir.path, 'a.onnx')).readAsBytesSync(), model); + final marker = jsonDecode( + File(p.join(dir.path, ManifestAddonInstaller.markerName)).readAsStringSync()); + expect(marker, {'id': 'test-addon', 'version': '1.0.0', 'platform': 'linux-x64'}); + expect(service.isInstalled('test-addon'), isTrue); + expect(addons.listSync().map((e) => p.basename(e.path)).toSet(), {'test-addon', '.downloads'}); + expect(Directory(p.join(addons.path, '.downloads')).listSync(), isEmpty, + reason: 'downloads are deleted once installed'); + await service.remove('test-addon'); + expect(dir.existsSync(), isFalse); + }); + + test('a file can install into a subdirectory, as RIFE\'s model must', () async { + // RIFE reads the model's version (and with it the padding) from its + // directory name, so frame-rate-rife.json installs into one. + final m = AddonManifest.fromJson(_manifest(files: [ + _entry('rife-v4.26_x/flownet.param', model), + _entry('rife-v4.26_x/flownet.bin', other), + ])); + final server = _Serve({ + 'https://example.invalid/dl/rife-v4.26_x/flownet.param': model, + 'https://example.invalid/dl/rife-v4.26_x/flownet.bin': other, + }); + final service = AddonService.forTest( + addonsDir: addons, installers: [newInstaller(m, 'linux-x64')], downloader: server); + await service.load(); + expect(await service.install('test-addon'), isTrue); + final dir = p.join(addons.path, 'test-addon', 'rife-v4.26_x'); + expect(File(p.join(dir, 'flownet.param')).readAsBytesSync(), model); + expect(File(p.join(dir, 'flownet.bin')).readAsBytesSync(), other); + expect(service.isInstalled('test-addon'), isTrue); + // A missing file in the subdirectory is a damaged install. + File(p.join(dir, 'flownet.bin')).deleteSync(); + await service.refresh(); + expect(service.isInstalled('test-addon'), isFalse); + }); + + test('only this platform\'s files are fetched', () async { + final m = AddonManifest.fromJson(_manifest( + platforms: {'linux-x64': {}, 'windows-x64': {}}, + files: [ + _entry('linux.onnx', model, platforms: ['linux-x64']), + _entry('windows.onnx', other, platforms: ['windows-x64']), + ])); + final server = _Serve({ + 'https://example.invalid/dl/linux.onnx': model, + 'https://example.invalid/dl/windows.onnx': other, + }); + final installer = newInstaller(m, 'windows-x64'); + expect(installer.info.downloadBytes, other.length); + await installer.install(addons, server); + expect(server.fetched, ['https://example.invalid/dl/windows.onnx']); + expect(installer.isInstalled(addons), isTrue); + expect(newInstaller(m, 'linux-x64').isInstalled(addons), isFalse, + reason: 'another platform\'s install is not this one\'s'); + }); + + test('an archive keeps only its listed members, each hash-checked', () async { + final zip = _zip({'pack/a.onnx': model, 'pack/b.pth': utf8.encode('checkpoint')}); + final m = AddonManifest.fromJson(_manifest(files: [ + _entry('pack.zip', zip, extract: [ + {'path': 'pack/a.onnx', 'sha256': _sha(model)}, + ]), + ])); + final installer = newInstaller(m, 'macos-arm64', osVersion: '15.0'); + await installer.install(addons, _Serve({'https://example.invalid/dl/pack.zip': zip})); + final dir = Directory(p.join(addons.path, 'test-addon')); + expect(dir.listSync().map((e) => p.basename(e.path)).toSet(), + {'a.onnx', ManifestAddonInstaller.markerName}); + expect(installer.isInstalled(addons), isTrue); + }); + + test('a wrong member hash installs nothing', () async { + final zip = _zip({'pack/a.onnx': utf8.encode('tampered')}); + final m = AddonManifest.fromJson(_manifest(files: [ + _entry('pack.zip', zip, extract: [ + {'path': 'pack/a.onnx', 'sha256': _sha(model)}, + ]), + ])); + final installer = newInstaller(m, 'linux-x64'); + await expectLater(installer.install(addons, _Serve({'https://example.invalid/dl/pack.zip': zip})), + throwsA(predicate((e) => '$e'.contains('does not match its expected hash')))); + expect(Directory(p.join(addons.path, 'test-addon')).existsSync(), isFalse); + expect(Directory(p.join(addons.path, 'test-addon.new')).existsSync(), isFalse); + expect(installer.isInstalled(addons), isFalse); + }); + + test('a version bump counts as missing, and reinstalling replaces the old install', () async { + final v1 = AddonManifest.fromJson(_manifest(files: [_entry('a.onnx', model)])); + final v2 = AddonManifest.fromJson(_manifest(version: '2.0.0', files: [_entry('b.onnx', other)])); + final server = _Serve({ + 'https://example.invalid/dl/a.onnx': model, + 'https://example.invalid/dl/b.onnx': other, + }); + await newInstaller(v1, 'linux-x64').install(addons, server); + final upgraded = newInstaller(v2, 'linux-x64'); + expect(upgraded.isInstalled(addons), isFalse); + await upgraded.install(addons, server); + expect(upgraded.isInstalled(addons), isTrue); + final dir = Directory(p.join(addons.path, 'test-addon')); + expect(File(p.join(dir.path, 'a.onnx')).existsSync(), isFalse, reason: 'replaced, not merged'); + expect(Directory(p.join(addons.path, 'test-addon.old')).existsSync(), isFalse); + }); + + test('the mirror replaces the download, and is still verified', () async { + final mirror = Directory(p.join(tmp.path, 'mirror'))..createSync(); + File(p.join(mirror.path, 'a.onnx')).writeAsBytesSync(model); + final m = AddonManifest.fromJson(_manifest(files: [_entry('a.onnx', model)])); + final server = _Serve({'https://example.invalid/dl/a.onnx': model}); + final installer = newInstaller(m, 'linux-x64', mirror: mirror.path); + await installer.install(addons, server); + expect(server.fetched, isEmpty); + expect(installer.isInstalled(addons), isTrue); + + await installer.remove(addons); + File(p.join(mirror.path, 'a.onnx')).writeAsBytesSync(other); + await expectLater(installer.install(addons, server), + throwsA(predicate((e) => '$e'.contains('does not match the pinned size and hash')))); + expect(installer.isInstalled(addons), isFalse); + + // A file the mirror doesn't hold (an upstream wheel, say) is downloaded. + File(p.join(mirror.path, 'a.onnx')).deleteSync(); + await installer.install(addons, server); + expect(server.fetched, ['https://example.invalid/dl/a.onnx']); + expect(installer.isInstalled(addons), isTrue); + }); + }); + + group('python add-on', () { + test('unpacks each wheel into site-packages, as pip lays it out', () async { + final wheel = _zip({ + 'pkg/__init__.py': utf8.encode('x = 1'), + 'pkg-1.0.dist-info/METADATA': utf8.encode('Name: pkg'), + 'pkg-1.0.data/platlib/pkg_native.so': utf8.encode('elf'), + 'pkg-1.0.data/scripts/tool': utf8.encode('#!'), + }); + final m = AddonManifest.fromJson( + _manifest(kind: 'python', files: [_entry('pkg-1.0-cp312-cp312-any.whl', wheel)])); + final installer = newInstaller(m, 'linux-arm64'); + await installer.install( + addons, _Serve({'https://example.invalid/dl/pkg-1.0-cp312-cp312-any.whl': wheel})); + final site = Directory(p.join(addons.path, 'test-addon', 'site-packages')); + expect(File(p.join(site.path, 'pkg', '__init__.py')).readAsStringSync(), 'x = 1'); + expect(File(p.join(site.path, 'pkg_native.so')).existsSync(), isTrue); + expect(File(p.join(site.path, 'tool')).existsSync(), isFalse); + expect(installer.isInstalled(addons), isTrue); + }); + + test('a wheel entry escaping its directory is refused', () async { + final wheel = _zip({'../evil.py': utf8.encode('boom')}); + final m = AddonManifest.fromJson(_manifest(kind: 'python', files: [_entry('evil.whl', wheel)])); + await expectLater( + newInstaller(m, 'linux-x64') + .install(addons, _Serve({'https://example.invalid/dl/evil.whl': wheel})), + throwsStateError); + expect(File(p.join(addons.path, 'evil.py')).existsSync(), isFalse); + expect(Directory(p.join(addons.path, 'test-addon')).existsSync(), isFalse); + }); + }); + + group('availability', () { + final m = AddonManifest.fromJson(_manifest( + platforms: {'macos-x64': {'minOs': '13.0'}, 'linux-x64': {}}, + files: [_entry('a.onnx', model)])); + + test('a platform outside the table is unavailable, and refused', () async { + final installer = newInstaller(m, 'windows-x64'); + expect(installer.unavailableReason, contains('isn\'t available for Windows')); + final service = AddonService.forTest(addonsDir: addons, installers: [installer]); + await service.load(); + await expectLater(service.install('test-addon'), throwsStateError); + }); + + test('minOs is checked against the running macOS', () { + expect(newInstaller(m, 'macos-x64', osVersion: '12.7.6').unavailableReason, + contains('needs macOS 13.0 or later')); + expect(newInstaller(m, 'macos-x64', osVersion: '13.0').unavailableReason, isNull); + expect(newInstaller(m, 'macos-x64', osVersion: '15.1').unavailableReason, isNull); + expect(ManifestAddonInstaller.compareOsVersions('13.10', '13.9'), greaterThan(0)); + }); + }); + + group('executable files', () { + test('a program is installed runnable (and signed on macOS)', () async { + // A real program, so there is something to run and sign. On macOS a + // plain C program with its signature stripped: an unsigned program, as + // upstream DeepFilterNet's Intel build is, which Apple Silicon refuses + // to run until it is signed. Not a copy of a system binary: those are + // arm64e, which macOS 15 kills once Apple's signature is gone, signed or + // not, so the test would fail for a reason no add-on can have. + final unsigned = File(p.join(tmp.path, 'unsigned')); + if (Platform.isMacOS) { + final src = File(p.join(tmp.path, 'tool.c'))..writeAsStringSync('int main(void) { return 0; }\n'); + final cc = await Process.run('xcrun', ['clang', '-o', unsigned.path, src.path]); + expect(cc.exitCode, 0, reason: '${cc.stderr}'); + final strip = await Process.run('codesign', ['--remove-signature', unsigned.path]); + expect(strip.exitCode, 0, reason: '${strip.stderr}'); + } else { + final source = File(Platform.isWindows ? r'C:\Windows\System32\whoami.exe' : '/bin/echo'); + unsigned.writeAsBytesSync(source.readAsBytesSync()); + } + final bytes = unsigned.readAsBytesSync(); + final m = AddonManifest.fromJson(_manifest(files: [ + {..._entry('tool', bytes), 'executable': true}, + ])); + expect(m.files.single.executable, isTrue); + final installer = newInstaller(m, 'linux-x64'); + await installer.install(addons, _Serve({'https://example.invalid/dl/tool': bytes})); + final tool = File(p.join(addons.path, 'test-addon', 'tool')); + if (!Platform.isWindows) { + expect(tool.statSync().mode & 0x49, 0x49, reason: 'executable by user, group and other'); + } + if (Platform.isMacOS) { + final verify = await Process.run('codesign', ['--verify', tool.path]); + expect(verify.exitCode, 0, reason: 'an installed program verifies: ${verify.stderr}'); + } + final run = await Process.run(tool.path, Platform.isWindows ? [] : ['ok']); + expect(run.exitCode, 0, reason: 'the installed program runs: ${run.stderr}'); + }); + + test('only a plain file of a files add-on may be executable', () { + expect( + () => AddonManifest.fromJson(_manifest(kind: 'python', files: [ + {..._entry('pkg-1.0-cp312-cp312-any.whl', model), 'executable': true}, + ])), + throwsFormatException); + expect( + () => AddonManifest.fromJson(_manifest(files: [ + { + ..._entry('pack.zip', model, extract: [ + {'path': 'a', 'sha256': _sha(model)}, + ]), + 'executable': true, + }, + ])), + throwsFormatException); + }); + }); + + group('validation', () { + Map good() => _manifest(files: [_entry('a.onnx', model)]); + void bad(void Function(Map) edit, String why) { + final json = good(); + edit(json); + expect(() => AddonManifest.fromJson(json), throwsFormatException, reason: why); + } + + test('malformed manifests are refused', () { + expect(AddonManifest.fromJson(good()).id, 'test-addon'); + bad((j) => j['kind'] = 'binary', 'unknown kind'); + bad((j) => j['id'] = 'Test Addon', 'id format'); + bad((j) => (j['files'] as List).first['sha256'] = 'abc', 'short hash'); + bad((j) => (j['files'] as List).first['url'] = 'http://example.invalid/a', 'plain http'); + bad((j) => (j['files'] as List).first['name'] = '../a.onnx', 'path in a name'); + bad((j) => (j['files'] as List).first['name'] = 'd/../../a.onnx', 'escaping subdirectory'); + bad((j) => (j['files'] as List).first['name'] = '/a.onnx', 'absolute path'); + bad((j) => (j['files'] as List).first['name'] = 'd//a.onnx', 'empty segment'); + bad((j) => (j['files'] as List).first['name'] = r'd\a.onnx', 'backslash'); + expect( + AddonManifest.fromJson(_manifest(files: [_entry('d/a.onnx', model)])).files.single.name, + 'd/a.onnx', + reason: 'a relative path of plain names is allowed'); + bad((j) => j['platforms'] = {'windows-arm64': {}}, 'unknown platform'); + bad((j) { + j['platforms'] = {'linux-x64': {}, 'macos-arm64': {}}; + (j['files'] as List).first['platforms'] = ['linux-x64']; + }, 'a platform with no files'); + bad((j) => j['kind'] = 'python', 'a python add-on lists wheels only'); + }); + }); + + group('shipped manifests', () { + final dir = Directory(p.join(_root, 'app', 'assets', 'addons')); + final manifests = AddonManifest.loadDirectory(dir); + + test('all parse, one per id, named after it, requirements known', () { + expect(manifests, isNotEmpty); + final ids = manifests.map((m) => m.id).toSet(); + expect(ids.length, manifests.length); + for (final m in manifests) { + expect(File(p.join(dir.path, '${m.id}.json')).existsSync(), isTrue, reason: m.id); + for (final r in m.requires) { + expect(ids, contains(r), reason: '${m.id} requires unknown $r'); + } + // Not the ids the hand-written installers use. + expect({'whisper', 'animejanai'}, isNot(contains(m.id))); + expect(m.id.startsWith('whisper-model-'), isFalse); + } + }); + + test('pubspec ships the directory, so AddonService finds every one', () { + final pubspec = File(p.join(_root, 'app', 'pubspec.yaml')).readAsStringSync(); + expect(pubspec, contains('- assets/addons/')); + }); + + test('the worker compiled in the same manifests, at the same versions', () async { + final ext = Platform.isWindows ? '.exe' : ''; + final worker = [ + p.join(_root, 'worker', 'target', 'debug', 'vapourbox-worker$ext'), + p.join(_root, 'worker', 'target', 'release', 'vapourbox-worker$ext'), + ].firstWhere((w) => File(w).existsSync(), + orElse: () => fail('Build the worker first: (cd worker && cargo build)')); + final result = await Process.run(worker, ['--list-addons']); + expect(result.exitCode, 0, reason: '${result.stderr}'); + final listed = jsonDecode((result.stdout as String).trim()) as Map; + expect(listed, {for (final m in manifests) m.id: m.version}, + reason: 'a stale worker binary? rebuild it (cd worker && cargo build)'); + }); + }); +} diff --git a/app/test/new_badges_shipped_test.dart b/app/test/new_badges_shipped_test.dart index e2d0c2bb..de253e9c 100644 --- a/app/test/new_badges_shipped_test.dart +++ b/app/test/new_badges_shipped_test.dart @@ -55,7 +55,15 @@ void main() { test('Noise Reduction: BM3D and KNLMeans in the method list', () { final s = _load('noise_reduction.json'); - expect(s.methods.where(service.isMethodNew).map((m) => m.id).toSet(), {'bm3d', 'knlmeans'}); + // Only the 1.5.0 stamps: NLMeans (CPU) carries 1.7.0 and is not part of + // this upgrade's story. + expect( + s.methods + .where((m) => m.sinceAppVersion == '1.5.0') + .where(service.isMethodNew) + .map((m) => m.id) + .toSet(), + {'bm3d', 'knlmeans'}); expect(service.isMethodChoiceNew(s), isTrue); expect(service.isSchemaNew(s), isTrue); }); @@ -73,4 +81,31 @@ void main() { test('a filter with nothing new in 1.5.0 is not badged', () { expect(service.isSchemaNew(_load('deband.json')), isFalse); }); + + group('1.7.0', () { + setUp(() async { + // Upgraded from 1.6.0 to 1.7.0. + SharedPreferences.setMockInitialValues({ + 'lastRunAppVersion': '1.6.0', + 'lastSeenAppVersion': '1.6.0', + }); + PackageInfo.setMockInitialValues( + appName: 'VapourBox', packageName: 'app.vapourbox.vapourbox', + version: '1.7.0', buildNumber: '1', buildSignature: ''); + service.resetForTesting(); + await service.initialize(); + }); + + test('Face Restoration is a new filter', () { + expect(service.isSchemaNew(_load('face_restore.json')), isTrue); + }); + + test('Colourisation is a new filter', () { + expect(service.isSchemaNew(_load('colorize.json')), isTrue); + }); + + test('Descale, new in 1.5.0, is no longer badged', () { + expect(service.isSchemaNew(_load('descale.json')), isFalse); + }); + }); } diff --git a/app/test/new_badges_widget_test.dart b/app/test/new_badges_widget_test.dart index 5e463363..e6bf417d 100644 --- a/app/test/new_badges_widget_test.dart +++ b/app/test/new_badges_widget_test.dart @@ -78,7 +78,7 @@ void main() { await tester.tap(find.text('NNEDI3').last); await tester.pumpAndSettle(); - for (final o in ['Real-ESRGAN', 'Real-CUGAN', 'AnimeJaNai (download, non-commercial)']) { + for (final o in ['Real-ESRGAN (download)', 'Real-CUGAN (download)', 'AnimeJaNai (download, non-commercial)']) { expect(badged(tester, o), isTrue, reason: o); } expect(badged(tester, 'EEDI3'), isFalse, reason: 'shipped in 1.4.0'); diff --git a/app/test/parameter_copy_with_test.dart b/app/test/parameter_copy_with_test.dart index 4d6539dc..6c785376 100644 --- a/app/test/parameter_copy_with_test.dart +++ b/app/test/parameter_copy_with_test.dart @@ -26,6 +26,7 @@ import 'dart:io'; import 'package:flutter_test/flutter_test.dart'; import 'package:vapourbox/models/anti_alias_parameters.dart'; +import 'package:vapourbox/models/audio_restoration_parameters.dart'; import 'package:vapourbox/models/chroma_denoise_parameters.dart'; import 'package:vapourbox/models/chroma_fix_parameters.dart'; import 'package:vapourbox/models/color_correction_parameters.dart'; @@ -36,6 +37,8 @@ import 'package:vapourbox/models/deflicker_parameters.dart'; import 'package:vapourbox/models/dehalo_parameters.dart'; import 'package:vapourbox/models/descratch_parameters.dart'; import 'package:vapourbox/models/edge_repair_parameters.dart'; +import 'package:vapourbox/models/colorize_parameters.dart'; +import 'package:vapourbox/models/face_restore_parameters.dart'; import 'package:vapourbox/models/frame_rate_parameters.dart'; import 'package:vapourbox/models/geometry_parameters.dart'; import 'package:vapourbox/models/ghost_removal_parameters.dart'; @@ -72,6 +75,9 @@ void main() { final models = <_Model>[ _Model('AntiAliasParameters', () => const AntiAliasParameters().toJson(), (j) => AntiAliasParameters.fromJson(j).copyWith().toJson()), + _Model('AudioRestorationParameters', + () => const AudioRestorationParameters().toJson(), + (j) => AudioRestorationParameters.fromJson(j).copyWith().toJson()), _Model('ChromaDenoiseParameters', () => const ChromaDenoiseParameters().toJson(), (j) => ChromaDenoiseParameters.fromJson(j).copyWith().toJson()), @@ -94,6 +100,10 @@ void main() { (j) => DeScratchParameters.fromJson(j).copyWith().toJson()), _Model('EdgeRepairParameters', () => const EdgeRepairParameters().toJson(), (j) => EdgeRepairParameters.fromJson(j).copyWith().toJson()), + _Model('ColorizeParameters', () => const ColorizeParameters().toJson(), + (j) => ColorizeParameters.fromJson(j).copyWith().toJson()), + _Model('FaceRestoreParameters', () => const FaceRestoreParameters().toJson(), + (j) => FaceRestoreParameters.fromJson(j).copyWith().toJson()), _Model('FrameRateParameters', () => const FrameRateParameters().toJson(), (j) => FrameRateParameters.fromJson(j).copyWith().toJson()), _Model('GeometryParameters', () => const GeometryParameters().toJson(), @@ -140,7 +150,7 @@ void main() { test('every parameter model is covered', () { // The count is the guard: a new model with a forgetful copyWith would // otherwise be caught by nothing at all. - expect(models.length, 21, + expect(models.length, 24, reason: 'a parameter model was added or removed — add it to this ' 'list and update the count'); }); diff --git a/app/test/pass_advice_test.dart b/app/test/pass_advice_test.dart index b1df7cbb..8b626b78 100644 --- a/app/test/pass_advice_test.dart +++ b/app/test/pass_advice_test.dart @@ -10,6 +10,7 @@ import 'package:flutter_test/flutter_test.dart'; import 'package:vapourbox/models/chroma_denoise_parameters.dart'; import 'package:vapourbox/models/color_correction_parameters.dart'; +import 'package:vapourbox/models/colorize_parameters.dart'; import 'package:vapourbox/models/deband_parameters.dart'; import 'package:vapourbox/models/geometry_parameters.dart'; import 'package:vapourbox/models/dehalo_parameters.dart'; @@ -381,6 +382,29 @@ void main() { }); }); + group('chroma denoise before colourisation', () { + test('warns on Chroma Denoise, whose colour Colourisation replaces', () { + final pipeline = const ProcessingPipeline( + chromaDenoise: ChromaDenoiseParameters(enabled: true), + colorize: ColorizeParameters(enabled: true), + ); + final advice = adviceFor(PassType.chromaDenoise, pipeline); + expect(advice, isNotNull); + expect(advice, contains('replaces the colour')); + // The order it states is the pipeline's. + final passes = pipeline.enabledPasses; + expect(passes.indexOf(PassType.chromaDenoise), lessThan(passes.indexOf(PassType.colorize))); + // And Colourisation itself carries no warning. + expect(adviceFor(PassType.colorize, pipeline), isNull); + }); + + test('silent with either alone', () { + expect(adviseOn(const ProcessingPipeline(colorize: ColorizeParameters(enabled: true))), isEmpty); + expect(adviseOn(const ProcessingPipeline(chromaDenoise: ChromaDenoiseParameters(enabled: true))), + isEmpty); + }); + }); + group('advice is always attached to an enabled pass', () { test('no advice names a pass that is switched off', () { // Otherwise a user sees a banner on a pass they are not using, which diff --git a/app/test/pass_list_stages_test.dart b/app/test/pass_list_stages_test.dart index 29d00367..bbd4c1a9 100644 --- a/app/test/pass_list_stages_test.dart +++ b/app/test/pass_list_stages_test.dart @@ -50,13 +50,16 @@ void main() { PassType.deband, PassType.antiAlias, PassType.chromaFixes, + PassType.colorize, PassType.colorCorrection, PassType.stabilize, PassType.geometry, PassType.cropResize, + PassType.faceRestore, PassType.sharpen, PassType.grain, PassType.frameRate, + PassType.audioRestoration, PassType.subtitles, ]); }); @@ -103,6 +106,30 @@ void main() { }); }); + test('faces are restored after the resize and before sharpening', () { + // At the delivered resolution: an upscale gives the restorer more pixels + // per face, and nothing resamples its detail away afterwards. Sharpening + // and grain follow, so they act on the restored face rather than the one + // it replaced. + expect(flattened.indexOf(PassType.faceRestore), + greaterThan(flattened.indexOf(PassType.cropResize))); + expect(flattened.indexOf(PassType.faceRestore), + lessThan(flattened.indexOf(PassType.sharpen))); + expect(flattened.indexOf(PassType.faceRestore), + lessThan(flattened.indexOf(PassType.grain))); + }); + + test('colourisation follows Chroma Fixes and precedes Colour Correction', () { + // The model judges the cleaned-up grey picture, its colour replaces the + // chroma Chroma Fixes repaired, and Colour Correction can grade it. + expect(flattened.indexOf(PassType.colorize), + greaterThan(flattened.indexOf(PassType.chromaFixes))); + expect(flattened.indexOf(PassType.colorize), + greaterThan(flattened.indexOf(PassType.noiseReduction))); + expect(flattened.indexOf(PassType.colorize), + lessThan(flattened.indexOf(PassType.colorCorrection))); + }); + test('grain is the last video pass', () { // Grain added before the resize is resampled away and before the deband is // smoothed away, so it has to follow both. diff --git a/app/test/pass_relevance_test.dart b/app/test/pass_relevance_test.dart index 74ee32fa..37118857 100644 --- a/app/test/pass_relevance_test.dart +++ b/app/test/pass_relevance_test.dart @@ -8,6 +8,7 @@ // Run with: flutter test test/pass_relevance_test.dart import 'package:flutter_test/flutter_test.dart'; +import 'package:vapourbox/models/noise_halo_report.dart'; import 'package:vapourbox/models/pass_relevance.dart'; import 'package:vapourbox/models/processing_pipeline.dart'; import 'package:vapourbox/services/field_order_detector.dart'; @@ -196,8 +197,10 @@ void main() { test('passes detection cannot judge are always neutral', () { // Dirt, scratches, grain, halos, banding and colour casts need eyes on - // the picture. If one of these ever starts making claims, the badge has - // stopped meaning anything. + // the picture. If one of these ever starts making claims from metadata + // alone, the badge has stopped meaning anything. (Noise and halos can be + // badged, but only from a measurement of the picture — see 'measured + // noise and halos' — never from what ffprobe says.) const invisible = [ PassType.descratch, PassType.spotless, @@ -208,6 +211,7 @@ void main() { PassType.sharpen, PassType.colorCorrection, PassType.subtitles, + PassType.audioRestoration, ]; for (final scan in ScanType.values) { @@ -231,4 +235,100 @@ void main() { } }); }); + + group('measured noise and halos', () { + const high = NoiseHaloReport( + noise: 6.2, halo: 14.0, haloEdges: 900, usablePairs: 24, + noiseHigh: true, haloHigh: true); + const low = NoiseHaloReport( + noise: 1.4, halo: 2.1, haloEdges: 900, usablePairs: 24); + // A score above the worker's threshold whose verdict is false is still + // "not high": the app never re-derives the verdict from the numbers. + const numbersOnly = NoiseHaloReport( + noise: 9.0, halo: 40.0, haloEdges: 900, usablePairs: 24); + + test('unmeasured stays neutral', () { + for (final pass in [PassType.noiseReduction, PassType.dehalo]) { + expect(relevanceFor(pass, source()).level, PassRelevance.neutral); + expect(relevanceFor(pass, source(), measured: null).reason, isNull); + } + }); + + test('below the threshold stays neutral, never "not applicable"', () { + for (final measured in [low, numbersOnly]) { + for (final pass in [PassType.noiseReduction, PassType.dehalo]) { + final r = relevanceFor(pass, source(), measured: measured); + expect(r.level, PassRelevance.neutral, reason: '$pass'); + expect(r.reason, isNull); + } + } + }); + + test('a failed measurement says nothing', () { + const failed = NoiseHaloReport.failed('decode error'); + for (final pass in [PassType.noiseReduction, PassType.dehalo]) { + expect(relevanceFor(pass, source(), measured: failed).level, + PassRelevance.neutral); + } + // And a report claiming high with an error is still a failure. + const odd = NoiseHaloReport(noiseHigh: true, haloHigh: true, error: 'x'); + expect(relevanceFor(PassType.noiseReduction, source(), measured: odd).level, + PassRelevance.neutral); + }); + + test('above the threshold recommends, with a reason', () { + final nr = relevanceFor(PassType.noiseReduction, source(), measured: high); + expect(nr.isRecommended, true); + expect(nr.reason, 'noise measured high'); + final dh = relevanceFor(PassType.dehalo, source(), measured: high); + expect(dh.isRecommended, true); + expect(dh.reason, 'halos measured around edges'); + // Each verdict badges only its own pass. + const noisyOnly = NoiseHaloReport(noise: 6.0, noiseHigh: true); + expect(relevanceFor(PassType.dehalo, source(), measured: noisyOnly).level, + PassRelevance.neutral); + expect(relevanceFor(PassType.noiseReduction, source(), measured: noisyOnly) + .isRecommended, true); + }); + + test('nothing loaded says nothing, measurement or not', () { + for (final pass in PassType.values) { + expect(relevanceFor(pass, null, measured: high).level, + PassRelevance.neutral); + } + }); + + test('a measurement changes no other pass', () { + for (final scan in ScanType.values) { + final info = source(scanType: scan, sar: '16:15'); + for (final pass in PassType.values) { + if (pass == PassType.noiseReduction || pass == PassType.dehalo) { + continue; + } + final without = relevanceFor(pass, info); + final with_ = relevanceFor(pass, info, measured: high); + expect(with_.level, without.level, reason: '$pass / $scan'); + expect(with_.reason, without.reason, reason: '$pass / $scan'); + } + } + }); + + test('a noisy, haloed PAL DVD capture suggests at most six passes', () { + final info = source( + height: 576, + codec: 'mpeg2video', + scanType: ScanType.interlaced, + sar: '16:15', + ); + final suggested = PassType.values + .where((p) => relevanceFor(p, info, measured: high).isRecommended) + .toSet(); + expect(suggested.length, lessThanOrEqualTo(6)); + expect(suggested, containsAll([PassType.noiseReduction, PassType.dehalo])); + final notApplicable = PassType.values + .where((p) => relevanceFor(p, info, measured: high).isNotApplicable) + .toSet(); + expect(suggested.intersection(notApplicable), isEmpty); + }); + }); } diff --git a/app/test/preset_visibility_test.dart b/app/test/preset_visibility_test.dart index ec0aca5f..1db9a95b 100644 --- a/app/test/preset_visibility_test.dart +++ b/app/test/preset_visibility_test.dart @@ -66,6 +66,7 @@ const _filterIds = { PassType.cropResize: 'crop_resize', PassType.grain: 'grain', PassType.subtitles: 'subtitles', + PassType.audioRestoration: 'audio_restoration', PassType.edgeRepair: 'edge_repair', PassType.deflicker: 'deflicker', PassType.ghostRemoval: 'ghost_removal', diff --git a/app/test/processing_preset_test.dart b/app/test/processing_preset_test.dart index dc4cdbae..16024b04 100644 --- a/app/test/processing_preset_test.dart +++ b/app/test/processing_preset_test.dart @@ -10,6 +10,7 @@ // Run with: flutter test test/processing_preset_test.dart import 'package:flutter_test/flutter_test.dart'; +import 'package:vapourbox/models/crop_resize_parameters.dart'; import 'package:vapourbox/models/dehalo_parameters.dart'; import 'package:vapourbox/models/encoding_settings.dart'; import 'package:vapourbox/models/processing_pipeline.dart'; @@ -34,6 +35,7 @@ void main() { 'builtin-balanced', 'builtin-high-quality', 'builtin-vhs-cleanup', + 'builtin-sd-tape-to-hd', 'builtin-dv-camcorder', 'builtin-pal-dvd', 'builtin-dvd-ivtc', @@ -102,6 +104,7 @@ void main() { .toSet(); expect(bySource, { 'builtin-vhs-cleanup', + 'builtin-sd-tape-to-hd', 'builtin-dv-camcorder', 'builtin-pal-dvd', 'builtin-dvd-ivtc', @@ -263,6 +266,54 @@ void main() { expect(low.pipeline.deinterlace.enabled, false); }); + test('SD tape to HD delivers square-pixel 1080-line BT.709 a TV plays', () { + final hd = byId('builtin-sd-tape-to-hd'); + expect(hd.category, PresetCategory.source); + final p = hd.pipeline; + // A tape capture is interlaced. + expect(p.deinterlace.enabled, true); + expect(p.deinterlace.method, DeinterlaceMethod.qtgmc); + // Light cleanup: lighter than VHS Cleanup's, both luma and chroma. + expect(p.noiseReduction.enabled, true); + expect(p.noiseReduction.smDegrainThSAD, + lessThan(byId('builtin-vhs-cleanup').pipeline.noiseReduction.smDegrainThSAD)); + expect(p.chromaDenoise.enabled, true); + // HD: an edge-directed doubling, resized into a 1080-line box with + // square pixels, and BT.709 colour. + final cr = p.cropResize; + expect(cr.enabled, true); + expect(cr.useIntegerUpscale, true); + expect(cr.upscaleMethod, UpscaleMethod.nnedi3Rpow2); + expect(cr.resizeEnabled, true); + expect(cr.targetHeight, 1080); + expect(cr.maintainAspect, true); + expect(cr.pixelAspect, PixelAspectMode.square); + expect(cr.convertToBt709, isNot(Bt709Conversion.off)); + // Light CAS, after the upscale. + expect(p.sharpen.enabled, true); + expect(p.sharpen.method, SharpenMethod.cas); + expect(p.sharpen.casSharpness, lessThanOrEqualTo(0.4)); + // What every TV, browser and upload service plays: H.264, 8-bit 4:2:0, + // MP4 with AAC, at a quality that survives YouTube's re-encode. + final e = hd.encodingSettings; + expect(e.codec, VideoCodec.h264); + expect(e.container, ContainerFormat.mp4); + expect(e.chromaSubsampling, ChromaSubsampling.yuv420); + expect(e.audioMode, AudioMode.convert); + expect(e.audioCodec, AudioCodec.aac); + expect(e.quality, inInclusiveRange(17, 18)); + }); + + test('no other preset upscales, so the BT.709 default changes none of them', () { + // "Auto" converts only an SD source taken to HD (app 1.7.0); a preset + // that never resizes is untouched by it. + for (final preset in presets.where((p) => p.id != 'builtin-sd-tape-to-hd')) { + final cr = preset.pipeline.cropResize; + expect(cr.enabled && (cr.resizeEnabled || cr.useIntegerUpscale), false, + reason: preset.id); + } + }); + test('DV camcorder measures its chroma shift rather than guessing', () { final dv = byId('builtin-dv-camcorder'); expect(dv.pipeline.chromaFixes.applyAutoChroma, true); diff --git a/app/test/support/audio_fixtures.dart b/app/test/support/audio_fixtures.dart new file mode 100644 index 00000000..233c06e4 --- /dev/null +++ b/app/test/support/audio_fixtures.dart @@ -0,0 +1,193 @@ +// Audio fixtures and measurements for the audio restoration tests: sources +// made with ffmpeg's lavfi generators (nothing committed), decoded back to +// float samples, and measured here in Dart — level per channel, energy in a +// narrow band, and the lag between two signals by cross-correlation. + +import 'dart:convert'; +import 'dart:io'; +import 'dart:math' as math; +import 'dart:typed_data'; + +import 'package:path/path.dart' as p; +import 'package:uuid/uuid.dart'; + +import 'worker_harness.dart'; + +class AudioFixtures { + static Directory get dir => + Directory(p.join(WorkerHarness.outputDir, 'audio_fixtures'))..createSync(recursive: true); + + /// Run the bundle's ffmpeg; throws with its stderr on failure. + static Future ffmpeg(List args) async { + final r = await Process.run( + WorkerHarness.ffmpegPath, ['-hide_banner', '-nostdin', '-y', '-v', 'error', ...args], + environment: WorkerHarness.ffmpegEnv); + if (r.exitCode != 0) throw StateError('ffmpeg ${args.join(' ')}\n${r.stderr}'); + } + + /// A source with a small picture (25 fps) and the audio [filterComplex] + /// labels `[a]` (or several `[a0]`, `[a1]`… when [audioStreams] > 1). + /// Made once per name. + static Future source( + String name, { + required String filterComplex, + double seconds = 6, + int audioStreams = 1, + String audioCodec = 'flac', + }) async { + final out = p.join(dir.path, '$name.mkv'); + if (File(out).existsSync()) return out; + await ffmpeg([ + '-f', 'lavfi', '-i', 'testsrc2=s=160x120:r=25:d=$seconds', + '-filter_complex', filterComplex, + '-map', '0:v', + if (audioStreams == 1) ...['-map', '[a]'] else + for (var i = 0; i < audioStreams; i++) ...['-map', '[a$i]'], + '-c:v', 'libx264', '-pix_fmt', 'yuv420p', '-preset', 'ultrafast', + '-c:a', audioCodec, + '-t', '$seconds', + out, + ]); + return out; + } + + /// A job for [input] with [audio] as its audio restoration settings and + /// nothing done to the picture. + static Map job( + String input, + Map audio, { + String container = 'mkv', + String label = 'audio', + Map encoding = const {}, + Map extra = const {}, + Map pipeline = const {}, + }) { + final out = p.join(WorkerHarness.outputDir, 'audio_restoration', '$label.$container'); + Directory(p.dirname(out)).createSync(recursive: true); + return { + 'id': const Uuid().v4(), + 'inputPath': input, + 'outputPath': out, + 'qtgmcParameters': {'enabled': false}, + 'processingPipeline': { + 'deinterlace': {'enabled': false}, + ...pipeline, + 'audio': {'enabled': true, ...audio}, + }, + 'encodingSettings': { + 'container': container, + 'codec': 'libx264', + 'encoderPreset': 'ultrafast', + ...encoding, + }, + 'inputFrameRate': 25.0, + 'inputWidth': 160, + 'inputHeight': 120, + 'inputPixelFormat': 'yuv420p', + ...extra, + }; + } + + /// `vapourbox-worker --audio-plan` for [jobJson]: the decoded JSON. + static Future> plan(Map jobJson) async { + final cfg = File(p.join(Directory.systemTemp.path, 'vb_plan_${jobJson['id']}.json')) + ..writeAsStringSync(jsonEncode(jobJson)); + try { + final r = await Process.run(WorkerHarness.workerPath, ['--audio-plan', '--config', cfg.path], + environment: WorkerHarness.workerEnv, stdoutEncoding: utf8, stderrEncoding: utf8); + final line = (r.stdout as String).trim().split('\n').last; + return jsonDecode(line) as Map; + } finally { + cfg.deleteSync(); + } + } + + /// Decode audio stream [stream] of [path] to float samples at [rate], + /// one list per channel (or downmixed to one with [mono]). + static Future> decode(String path, + {int stream = 0, int rate = 48000, bool mono = false}) async { + final probe = await WorkerHarness.ffprobeJson( + ['-v', 'error', '-of', 'json', '-select_streams', 'a:$stream', '-show_entries', 'stream=channels', path]); + final channels = mono ? 1 : ((probe['streams'] as List).first['channels'] as int); + final r = await Process.run( + WorkerHarness.ffmpegPath, + ['-v', 'error', '-nostdin', '-i', path, '-map', '0:a:$stream', '-ac', '$channels', + '-ar', '$rate', '-f', 'f32le', '-'], + stdoutEncoding: null, + environment: WorkerHarness.ffmpegEnv, + ); + if (r.exitCode != 0) throw StateError('decode $path: ${r.stderr}'); + final bytes = Uint8List.fromList(r.stdout as List); + final all = bytes.buffer.asFloat32List(0, bytes.lengthInBytes ~/ 4); + final frames = all.length ~/ channels; + return [ + for (var c = 0; c < channels; c++) + Float32List.fromList([for (var i = 0; i < frames; i++) all[i * channels + c]]), + ]; + } + + static double rmsDb(Float32List x, {int from = 0, int? to}) { + final end = to ?? x.length; + var sum = 0.0; + for (var i = from; i < end; i++) { + sum += x[i] * x[i]; + } + final rms = math.sqrt(sum / math.max(1, end - from)); + return 20 * math.log(rms + 1e-12) / math.ln10; + } + + /// Energy (dB) at [freq], by the Goertzel algorithm with a Hann window, + /// over [x] sampled at [rate]. + static double bandDb(Float32List x, int rate, double freq) { + final n = x.length; + final w = 2 * math.pi * freq / rate; + final coeff = 2 * math.cos(w); + var s1 = 0.0, s2 = 0.0; + for (var i = 0; i < n; i++) { + final win = 0.5 - 0.5 * math.cos(2 * math.pi * i / (n - 1)); + final s = x[i] * win + coeff * s1 - s2; + s2 = s1; + s1 = s; + } + final power = (s1 * s1 + s2 * s2 - coeff * s1 * s2) / (n * n); + return 10 * math.log(power + 1e-30) / math.ln10; + } + + /// Positions (seconds) of the onsets of the clicks in [x]: the first sample + /// above [threshold] after at least [gap] seconds below it. + static List onsets(Float32List x, int rate, {double threshold = 0.3, double gap = 0.1}) { + final out = []; + var last = -1e9; + for (var i = 0; i < x.length; i++) { + if (x[i].abs() > threshold) { + final t = i / rate; + if (t - last > gap) out.add(t); + last = t; + } + } + return out; + } + + /// Lag of [sig] against [ref] in samples (+ = [sig] late), by direct + /// cross-correlation within ±[maxLag]. + static int lag(Float32List ref, Float32List sig, int maxLag) { + final n = math.min(ref.length, sig.length); + var best = 0; + var bestScore = -double.infinity; + for (var l = -maxLag; l <= maxLag; l++) { + var score = 0.0; + for (var i = math.max(0, -l); i < n && i + l < n; i += 4) { + score += ref[i] * sig[i + l]; + } + if (score > bestScore) { + bestScore = score; + best = l; + } + } + return best; + } + + /// The output's duration in seconds, by its audio sample count. + static Future audioSeconds(String path, {int stream = 0}) async => + (await decode(path, stream: stream, mono: true)).first.length / 48000; +} diff --git a/app/test/support/gpu_expectations.dart b/app/test/support/gpu_expectations.dart index 49a0f6ec..7fa5b3d9 100644 --- a/app/test/support/gpu_expectations.dart +++ b/app/test/support/gpu_expectations.dart @@ -1,14 +1,16 @@ // Where the ncnn-on-Vulkan features (RIFE, AI upscaling) must work, and the // one rule every GPU test applies. // -// The hosted CI runners have no GPU. The workflows give the Linux and Windows -// runners Mesa's software Vulkan driver (lavapipe) and set -// VAPOURBOX_EXPECT_VULKAN=1 wherever a Vulkan device is provisioned. Under that -// variable, on a platform that ships the feature, "not available" is a test -// failure — so a broken bundle, plugin or probe cannot hide behind a skip, which -// is how it would read on a runner that simply has no GPU. Without it (a local -// run, a machine with no driver) the GPU tests skip and say why. On a platform -// that does not ship the feature, the probe must say exactly that. +// The hosted CI runners have no GPU, so every test that needs a Vulkan device +// is tagged `gpu` and the workflows exclude that tag (dart_test.yaml). What CI +// still runs without a device: each probe must answer with a real reason, and +// a job that cannot run must be refused with the probe's wording. +// +// On a machine with a GPU, set VAPOURBOX_EXPECT_VULKAN=1: then, on a platform +// that ships the feature, "not available" is a test failure, so a broken +// bundle, plugin or probe cannot hide behind a skip. Without it (a machine with +// no driver) the GPU tests skip and say why. On a platform that does not ship +// the feature, the probe must say exactly that. library; import 'dart:convert'; @@ -36,8 +38,8 @@ String get platformId { return abi.toString(); } -/// Whether this run has a Vulkan device provisioned and so must exercise the -/// GPU features rather than skip them. +/// Whether this run has a Vulkan device (VAPOURBOX_EXPECT_VULKAN=1) and so +/// must exercise the GPU features rather than skip them. bool get expectVulkan => Platform.environment['VAPOURBOX_EXPECT_VULKAN'] == '1'; /// Whether this platform's bundle ships the GPU features at all. @@ -69,8 +71,8 @@ Future requireGpuPlugin(String flag) async { final probe = await probeGpuPlugin(flag); if (probe['available'] == true) return true; if (platformShipsGpuPlugins && expectVulkan) { - fail('$flag says unavailable on $platformId, where CI provisions Vulkan ' - '(VAPOURBOX_EXPECT_VULKAN=1): ${probe['reason']}: ${probe['message']}'); + fail('$flag says unavailable on $platformId, where a Vulkan device is ' + 'expected (VAPOURBOX_EXPECT_VULKAN=1): ${probe['reason']}: ${probe['message']}'); } markTestSkipped('$flag: not available on $platformId (${probe['reason']})'); return false; diff --git a/app/test/support/worker_harness.dart b/app/test/support/worker_harness.dart index 138f9110..84a7f7cd 100644 --- a/app/test/support/worker_harness.dart +++ b/app/test/support/worker_harness.dart @@ -30,7 +30,11 @@ import 'package:path/path.dart' as p; // importing this file does NOT initialize the app's rhttp HTTP client, which is // why the harness does its own dart:io download below (rhttp's native bindings // don't load under the headless `flutter test` VM). +import 'package:vapourbox/services/addon_manager.dart'; import 'package:vapourbox/services/addon_service.dart'; +import 'package:vapourbox/services/manifest_addon.dart'; +import 'package:vapourbox/services/process_tree.dart'; +import 'package:vapourbox/services/whisper_addon_manager.dart'; import 'package:vapourbox/services/dependency_manager.dart' show DepsVersionInfo, DependencyManager; @@ -139,6 +143,82 @@ class WorkerHarness { return target; } + /// The add-ons directory the debug worker reads: `$VAPOURBOX_ADDONS_DIR`, + /// else the nearest `addons/` above the repo root (as the worker searches + /// upward from its binary — a git worktree inside the main checkout shares + /// the checkout's), else `/addons`, created. + static Directory get addonsDirectory { + final found = _addonsDir; + if (found != null) return Directory(found); + var dir = Directory(repoRoot).parent; + for (var i = 0; i < 8; i++) { + final candidate = Directory(p.join(dir.path, 'addons')); + if (candidate.existsSync()) return candidate; + if (dir.parent.path == dir.path) break; + dir = dir.parent; + } + return Directory(p.join(repoRoot, 'addons'))..createSync(recursive: true); + } + + /// Install the manifest add-on [id] (`app/assets/addons/.json`) and what + /// it requires where the worker finds it, with the app's own + /// [ManifestAddonInstaller] — so a test of an add-on also tests that its + /// pinned URLs and hashes still hold. Honours `VAPOURBOX_ADDON_MIRROR` + /// (files taken from a local directory instead of downloaded, still + /// verified), which is how an add-on is tested before its files are + /// uploaded. No-op for what is already installed at this version. + static Future ensureAddon(String id) async { + final manifests = { + for (final m in AddonManifest.loadDirectory( + Directory(p.join(repoRoot, 'app', 'assets', 'addons')))) + m.id: m, + }; + final addons = addonsDirectory; + final order = []; + void visit(String x) { + final m = manifests[x]; + if (m == null) throw ArgumentError('No add-on manifest for "$x"'); + m.requires.forEach(visit); + if (!order.contains(x)) order.add(x); + } + + visit(id); + for (final x in order) { + final installer = ManifestAddonInstaller(manifests[x]!, platform); + if (installer.isInstalled(addons)) continue; + final mirror = installer.mirror; + stderr.writeln('[harness] installing add-on $x into ${addons.path}' + '${mirror != null ? ' from mirror $mirror' : ''}'); + await installer.install(addons, _IoDownloader()); + if (!installer.isInstalled(addons)) { + throw StateError('Add-on $x did not install completely'); + } + } + return Directory(p.join(addons.path, id)); + } + + /// Install the Whisper engine and one language model where the worker finds + /// them, with the app's own installers (from `assets/whisper-addon.json`). + /// For a local run of the subtitle tests (CI provisions the add-on itself); + /// the small model is 466 MB. No-op for what is already installed. + static Future ensureWhisper({String model = 'small'}) async { + final addons = addonsDirectory; + final manager = WhisperAddonManager( + jsonDecode(File(p.join(repoRoot, 'app', 'assets', 'whisper-addon.json')).readAsStringSync()) + as Map, + platform); + for (final installer in manager.installers()) { + final id = installer.info.id; + if (id != WhisperAddonManager.engineAddonId && id != WhisperAddonManager.modelAddonId(model)) { + continue; + } + if (installer.isInstalled(addons)) continue; + stderr.writeln('[harness] installing $id into ${addons.path}'); + await installer.install(addons, _IoDownloader()); + } + return Directory(p.join(addons.path, WhisperAddonManager.dirName)); + } + /// Whether the whisper add-on (whisper-cli) is available. Subtitle tests that /// run the full pipeline must skip when this is false (local dev w/o add-on). static bool get whisperAvailable { @@ -358,8 +438,10 @@ class WorkerHarness { final exitCode = await process.exitCode.timeout( timeout, - onTimeout: () { - process.kill(); + onTimeout: () async { + // The whole tree: on Windows killing the worker alone leaves its + // vspipe running a network on every core, starving later tests. + await ProcessTree.killTree(process, ProcessSignal.sigkill); throw TimeoutException('$label timed out after ${timeout.inSeconds}s'); }, ); @@ -434,7 +516,7 @@ class WorkerHarness { if (logLines.any((l) => l.contains('"type":"error"'))) break; } - process.kill(); + await ProcessTree.killTree(process); await process.exitCode.timeout(const Duration(seconds: 5), onTimeout: () { Process.killPid(process.pid, ProcessSignal.sigkill); return -1; @@ -567,8 +649,8 @@ class WorkerHarness { .fold(BytesBuilder(), (b, chunk) => b..add(chunk)); final stderrFuture = process.stderr.transform(utf8.decoder).join(); - final exitCode = await process.exitCode.timeout(timeout, onTimeout: () { - process.kill(); + final exitCode = await process.exitCode.timeout(timeout, onTimeout: () async { + await ProcessTree.killTree(process, ProcessSignal.sigkill); throw TimeoutException('$label timed out after ${timeout.inSeconds}s'); }); final png = (await stdoutFuture).takeBytes(); @@ -1053,3 +1135,53 @@ class PreviewResult { @override String toString() => success ? 'SUCCESS (${png!.length} bytes)' : 'FAILED: $error\n$errorTail'; } + +/// [AddonManager] over dart:io, for [WorkerHarness.ensureAddon]: the app's +/// downloader uses rhttp, whose native library does not load under the +/// headless `flutter test` VM. Streams to the file (models run to hundreds of +/// MB) and checks size and hash as the app's does. +class _IoDownloader extends AddonManager { + @override + Future downloadFile(String url, File destination, + {int? expectedSize, String? expectedSha256, Map? extraHeaders}) async { + final client = HttpClient()..connectionTimeout = const Duration(seconds: 30); + final partial = File('${destination.path}.partial'); + try { + var current = Uri.parse(url); + for (var redirects = 0;; redirects++) { + if (redirects > 6) throw StateError('too many redirects for $url'); + final request = await client.getUrl(current); + request.followRedirects = false; + extraHeaders?.forEach(request.headers.set); + final response = await request.close(); + if (response.isRedirect) { + final loc = response.headers.value(HttpHeaders.locationHeader); + await response.drain(); + if (loc == null) throw StateError('redirect without Location ($current)'); + current = current.resolve(loc); + continue; + } + if (response.statusCode != 200) { + await response.drain(); + throw HttpException('HTTP ${response.statusCode} for $current'); + } + await response.pipe(partial.openWrite()); + break; + } + if (expectedSize != null && await partial.length() != expectedSize) { + throw StateError('$url: got ${await partial.length()} bytes, expected $expectedSize'); + } + if (expectedSha256 != null) { + final digest = (await sha256.bind(partial.openRead()).first).toString(); + if (digest != expectedSha256) { + throw StateError('$url: SHA256 mismatch, expected $expectedSha256, got $digest'); + } + } + if (await destination.exists()) await destination.delete(); + await partial.rename(destination.path); + } finally { + client.close(force: true); + if (await partial.exists()) await partial.delete(); + } + } +} diff --git a/app/test/vapoursynth_integration_test.dart b/app/test/vapoursynth_integration_test.dart index 2776e64a..130b822b 100644 --- a/app/test/vapoursynth_integration_test.dart +++ b/app/test/vapoursynth_integration_test.dart @@ -129,7 +129,12 @@ required = ['std', 'resize', 'mv', 'znedi3', 'eedi3m', 'fmtc', 'descratch', 'vivtc', 'ttmpsm', 'tmedian', 'fft3dfilter', 'flux', 'bifrost', 'retinex', 'bwdif', 'fb', 'removedirt', 'dedot', 'lghost', 'zsmooth', - 'vszip', 'bm3d', 'descale'] + 'vszip', 'bm3d', 'descale', 'sangnom'] + +# A namespace is not a version: vszip 22.1.0 (deps <= 1.15.0) has CLAHE but no +# NLMeans, which Noise Reduction's NLMeans method and the vine dehalo need. A +# stale bundle would pass the namespace check and fail those jobs. +required_functions = [('vszip', 'NLMeans'), ('vszip', 'CLAHE')] # On ARM, `nnedi3` is load-bearing and `znedi3` is only the fallback: znedi3's # SIMD is x86-only, so the ARM bundles build it scalar and both the templates' @@ -163,6 +168,9 @@ missing = [] for plugin in required: if not hasattr(core, plugin): missing.append(plugin) +for ns, fn in required_functions: + if hasattr(core, ns) and not hasattr(getattr(core, ns), fn): + missing.append(f'{ns}.{fn}') if missing: raise Exception(f"Missing plugins: {missing}") diff --git a/app/windows/runner/Runner.rc b/app/windows/runner/Runner.rc index 063fd76b..6101c6a6 100644 --- a/app/windows/runner/Runner.rc +++ b/app/windows/runner/Runner.rc @@ -73,8 +73,8 @@ IDI_APP_ICON ICON "resources\\app_icon.ico" #endif VS_VERSION_INFO VERSIONINFO - FILEVERSION 1,6,0,0 - PRODUCTVERSION 1,6,0,0 + FILEVERSION 1,7,0,0 + PRODUCTVERSION 1,7,0,0 FILEFLAGSMASK VS_FFI_FILEFLAGSMASK #ifdef _DEBUG FILEFLAGS VS_FF_DEBUG diff --git a/docs/ADDONS.md b/docs/ADDONS.md new file mode 100644 index 00000000..967e411d --- /dev/null +++ b/docs/ADDONS.md @@ -0,0 +1,334 @@ +# Add-ons + +Anything VapourBox can use but does not ship in the app or the deps bundle is +an **add-on**: an optional download the user asks for, from Settings → Add-ons +or from the prompt a feature raises when it needs one (`AddonPrompt.ensure`). +**Every AI model is an add-on, never part of the deps bundle** — a model's +size or licence should never be a cost to users who don't use it, and the deps +bundle must stay freely redistributable. + +There are three kinds of installer behind `AddonService`: + +| Add-on | Installer | Described by | +|---|---|---| +| Whisper engine + models | `WhisperAddonManager` (hand-written) | `app/assets/whisper-addon.json` | +| AnimeJaNai models | `AnimeJanaiInstaller` (hand-written) | `app/assets/animejanai-addon.json` | +| **Everything new** | `ManifestAddonInstaller` (generic) | `app/assets/addons/.json` | + +New add-ons are **manifest add-ons**. This file documents them. + +## The manifest + +One JSON file per add-on, `app/assets/addons/.json`, named after its id. +The app finds every file in that directory through the asset manifest +(`assets/addons/` is registered in `pubspec.yaml`), and the worker compiles the +same files in (`worker/build.rs` → `worker/src/addons.rs`), so there is one +source of truth for what an add-on is and what an install of it holds. + +```json +{ + "id": "face-restore-gfpgan", + "version": "1.0.0", + "name": "GFPGAN face restoration model", + "feature": "Face restoration", + "description": "One or two sentences for Settings → Add-ons.", + "licence": "Apache-2.0", + "licenceUrl": "https://github.com/TencentARC/GFPGAN/blob/master/LICENSE", + "usageNote": "Free for non-commercial use only.", + "credit": "Tencent ARC Lab", + "noticeName": "GFPGAN model", + "homepage": "https://github.com/TencentARC/GFPGAN", + "components": [ + { + "noticeName": "Some bundled part", + "licence": "MIT", + "licenceUrl": "https://…", + "credit": "Its authors", + "homepage": "https://…", + "platforms": ["windows-x64"] + } + ], + "requires": ["ai-runtime"], + "kind": "files", + "platforms": "all", + "files": [ + { + "name": "GFPGANv1.4.onnx", + "url": "https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/GFPGANv1.4.onnx", + "size": 348632874, + "sha256": "…64 lower-case hex…" + }, + { + "name": "upstream-pack.zip", + "url": "https://example.org/releases/pack.zip", + "size": 24674026, + "sha256": "…", + "platforms": ["macos-arm64", "windows-x64", "linux-x64"], + "extract": [ + { "path": "pack/model.onnx", "name": "model.onnx", "sha256": "…" } + ] + } + ] +} +``` + +| Field | | +|---|---| +| `id` | lower-case-with-dashes; the directory it installs to (`addons//`) and the id features ask for. Must not collide with the hand-written ids (`whisper`, `whisper-model-*`, `animejanai`). | +| `version` | the add-on's own version. **Bump it whenever its files change**: an install of another version counts as missing (the app offers it again and the install replaces it; the worker refuses with "out of date"). | +| `name`, `feature`, `description` | what Settings → Add-ons shows; `feature` groups the list. | +| `licence`, `licenceUrl`, `credit`, `noticeName`, `homepage` | the main component's credit (see "Credits" below). | +| `usageNote` | required exactly when the licence is non-commercial (`NC`), and shown bold wherever the add-on is offered. | +| `components` | further components credited separately, each optionally limited to `platforms`. | +| `requires` | add-on ids installed first, and removed with this one's dependants. A model run through ONNX Runtime requires `ai-runtime`; one run by a bundled ncnn plugin (VSNCNN for the Crop & Resize upscalers, `upscale-*.json`; RIFE for `frame-rate-rife.json`) requires nothing, and lists only the platforms that plugin ships on. | +| `kind` | `files` — a model pack: each file is saved as `addons//`, or for an entry with `extract`, only the listed members are kept (each checked against its own hash; nothing else in the archive is written). `python` — every file is a wheel, unpacked into `addons//site-packages/` as pip lays it out (`*.data/purelib`/`platlib` at the top, other `.data` dropped). | +| `python` | for a python add-on, the wheel tag it targets (`cp312`: the deps bundle's Python on every platform). | +| `platforms` | `"all"`, or a table of the deps platform ids (`macos-arm64`, `macos-x64`, `windows-x64`, `linux-x64`, `linux-arm64`) it supports, each with optional `minOs` (macOS only, checked at run time against the running system). Anything else gets an `unavailableReason` and cannot be installed. | +| `files[]` | `name` (a plain file name, or a relative path of plain names joined by `/` — see below), `url` (https), `size` (bytes), `sha256`; optional `platforms` (default: every supported platform), `extract` (members: `path` in the archive, `name` to install as, default its basename, and `sha256`) and `executable` (see below). Every supported platform must have at least one file. | + +`AddonManifest.fromJson` validates all of this, so a malformed manifest fails +`manifest_addon_test.dart` rather than half-working. + +### Programs: `executable` + +A plain file (not an archive entry, not in a `python` add-on) marked +`"executable": true` is a program the worker runs — DeepFilterNet's +`deep-filter` (`app/assets/addons/deepfilternet.json`) is the first. On install +the app sets its executable bit (downloads have none) and, on macOS, signs it +ad hoc when `codesign --verify` fails: Apple Silicon refuses to run an unsigned +arm64 binary, and upstream builds are often unsigned (DeepFilterNet's Intel one +is; its arm64 one is linker-signed and left alone). Quarantine is cleared with +the rest of the install. Each platform's program can keep upstream's own file +name: the worker finds it with `addons::Manifest::executable_for(platform)`. + +### Subdirectories: a path in `name` + +A file's `name` (or an archive member's) may be a relative path such as +`rife-v4.26_ensembleFalse/flownet.param`: every `/`-separated part must be a +plain name, so nothing can land outside the add-on's directory. It exists for +a plugin that reads meaning from its model's directory name — the RIFE plugin +takes the model's version, and with it the padding it runs at, from +substrings of the path it is given, so `frame-rate-rife` installs into +`addons/frame-rate-rife/rife-v4.26_ensembleFalse/`. The mirror still looks a +file up by the last segment of its URL. Prefer plain names otherwise. + +### What an install looks like + +``` +addons/ + / + + .vapourbox-addon.json {"id", "version", "platform"} — written last + .downloads/-/ kept across a cancel so the download resumes; + deleted once installed +``` + +An install downloads and verifies **every** file first, builds the add-on in +`addons/.new`, writes the marker, then renames the old install aside and +the new one into place. A failure or cancel at any point leaves the previous +state; the app and the worker both treat a directory without a matching marker +(id, version, *platform*) as not installed. + +### Testing before the files exist: the mirror + +`VAPOURBOX_ADDON_MIRROR=` makes the installer take each file from +`/` instead of downloading it — still +checked against the manifest's size and hash. A file the mirror doesn't hold +is downloaded as usual, so it can be set for a whole test run: add-ons whose +files come from upstream (the AI runtime's wheels) still install. Use it to +test an add-on whose files are converted locally and not uploaded yet. It +applies to the app and to `WorkerHarness.ensureAddon`. + +## Hosting files we convert ourselves + +Where upstream publishes only a PyTorch checkpoint (`.pth`), we convert it to +ONNX and host the result on this repository's releases: + +``` +https://github.com/StuartCameronCode/VapourBox/releases/download/addon-models-v1/ +``` + +That release is created by the project owner, not by a change. + +Each model sits beside its upstream licence text, `LICENSE-.txt` +(the licences require it to accompany a copy), and the release notes have one +row per file: add-on, upstream, licence, licence file, and the script that +made it. + +**While a release is in development**, converted files may be hosted on the +deps release candidate instead (a prerelease), with the manifests pointing +there, so CI and nightly can install them before the owner uploads them for +good. Before the app release they move to `addon-models-v1` and the manifests +are repointed: `Scripts/release.sh` refuses to cut a release while any +manifest downloads from an `-rc` tag, exactly as it does for +`deps-version.json`. (App 1.7.0's files moved from `deps-v1.16.0-rc1` on +2026-11-09.) + +Before a file is uploaded anywhere, put it in `addon-staging/` at the repo +root (gitignored) and test through the mirror: + +```bash +VAPOURBOX_ADDON_MIRROR=$PWD/addon-staging \ +VAPOURBOX_DEPS_DIR=$PWD/deps/macos-arm64 \ + (cd app && flutter test test/integration__test.dart) +``` + +### The conversion environment + +A shared Python venv for converting checkpoints lives at +`addon-staging/convert-venv/` (gitignored; Python 3.14 with CPU `torch`, +`torchvision`, `onnx`, `onnxruntime`, `onnxscript`, `numpy`). Recreate it with: + +```bash +python3.14 -m venv addon-staging/convert-venv +addon-staging/convert-venv/bin/pip install torch torchvision onnx onnxruntime onnxscript numpy +``` + +Add a model's own packages to it as needed. Every conversion is a script +committed under `Scripts/models/` (see `make_ai_runtime_selftest.py` for the +shape): it must reproduce the hosted file from the upstream checkpoint, pin the +checkpoint by URL and hash, and state the opset and IR version (keep them +loadable by the oldest ONNX Runtime the `ai-runtime` add-on pins — 1.23.2 on +Intel Macs). + +## Models for the bundled ncnn plugins (RIFE, VSNCNN) + +The RIFE and VSNCNN plugins are in the deps bundle; every model they run is +an add-on — RIFE's (`frame-rate-rife`, upstream's ncnn conversion fetched at +a pinned commit) and all of Crop & Resize's upscalers' (`upscale-anime`, +`upscale-general`, `upscale-esrgan-plus`, and the hand-written `animejanai`). +Until deps 1.16.0 RIFE's model and the Real-ESRGAN anime / Real-CUGAN models +were bundled; the rule now has no exceptions, and `attribution_test.dart` +fails if `Scripts/deps-expected-plugins.json` lists a model or a +`download-deps-*` script fetches one. + +Their probes (`--probe-rife`, `--probe-upscale`, `worker/src/gpu_plugin.rs`) +need no add-on: each runs its plugin on a stand-in network of its own and +checks the answer, so "this computer can't run it" is told before the user +downloads anything, and a missing add-on is a separate refusal through +`required_addons()`: + +- **VSNCNN**: `Tests/TestResources/ncnn_selftest.onnx` (264 bytes, one 1x1 + convolution that swaps the first and third channels and adds 0.125; built by + `Scripts/models/make_ncnn_selftest.py`), compiled into the worker and handed + to `ncnn.Model` as bytes (`path_is_serialization=True`). +- **RIFE**: a two-layer ncnn network with RIFE's v4 inputs and output + (`in0`, `in1`, `in2` → `out0`, the average of the two frames, no weights), + written beside the probe script in a directory named `rife-v4.26_selftest`. + +## The worker side + +`worker/src/addons.rs`: + +- `addon_dir(addons_root, id)`, `site_packages(addons_root, id)` — where it is. +- `state(addons_root, id, platform)` → `Installed | Missing | Outdated | Unsupported | Unknown`, + and `is_installed(...)`: the marker for this id, version and platform, plus + every expected path (a model pack's files; a python add-on's `site-packages`). +- `require(addons_root, &[ids], feature, platform) -> anyhow::Result<()>` — + refuses with a message naming the add-on(s) and Settings → Add-ons + (`DependencyLocator::require_addons(&[ids], feature)` fills in the root and + platform). + +A pass declares what it needs in `ProcessingPipeline::required_addons()` +(`AddonNeed { feature, ids }`). `ScriptGenerator::ensure_methods_ready` refuses +the job unless all are installed, exactly as for AnimeJaNai, and emits the +`{{#AI_RUNTIME}}` block when any of them (or what they require) is +`ai-runtime`. + +## The AI runtime add-on (`ai-runtime`) + +ONNX Runtime and NumPy, as wheels for the deps bundle's Python 3.12, for +passes that run a model the VSNCNN plugin can't (per-crop inference, several +inputs, NumPy around the network): + +| Platform | onnxruntime | Execution providers | +|---|---|---| +| macOS arm64 | 1.31.1 | Core ML, CPU | +| macOS x64 | 1.23.2 (the last Intel wheel; needs macOS 13) | Core ML, CPU | +| Windows x64 | onnxruntime-directml 1.24.4 | DirectML (any DX12 GPU), CPU | +| Linux x64 / arm64 | 1.31.1 | CPU | + +plus NumPy 2.5.4. `import onnxruntime` and an `InferenceSession` need nothing +else (checked: no sympy, protobuf, flatbuffers or coloredlogs at run time, +though the wheels declare them). + +### Using it from a pass + +Both templates carry, near the top: + +```python +{{#AI_RUNTIME}} +import ai_runtime +ai_runtime.setup({{AI_RUNTIME_DIR}}) +{{/AI_RUNTIME}} +``` + +`ScriptGenerator` keeps it (with `{{AI_RUNTIME_DIR}}` the add-on's directory as +a Python string literal) when the job needs the runtime, and removes it +otherwise. A pass's own block then uses `templates/ai_runtime.py`: + +| Call | | +|---|---| +| `setup(*addon_dirs)` | puts each add-on's `site-packages` on `sys.path` (first argument wins) and, on Windows, registers onnxruntime's DLL directory. Done by the block; never rely on `PYTHONPATH` — Windows' embeddable Python ignores it. | +| `session(model, force_cpu=False, providers=None, dims=None)` | a cached `InferenceSession` for a path or the model's bytes, on TensorRT → CUDA → DirectML → Core ML → CPU (those available), retried on the CPU if a GPU provider fails (`ai_runtime.last_fallback` says why). `dims` (`{"height": 512, "width": 512}`) fixes a model's named dynamic dimensions for the session (free-dimension overrides), so one exported file serves several input sizes and Core ML still gets static shapes (the colourisation models use it). `Runner(model, force_cpu, lock, dims)` takes the same. | +| `preferred_providers(force_cpu=False)`, `available_providers()`, `session_provider(sess)` | the provider choice. `VAPOURBOX_AI_FORCE_CPU=1` forces the CPU everywhere. | +| `frame_to_nchw(frame)` / `nchw_to_frame(array, frame)` | planar RGBS `VideoFrame` ↔ float32 `(1, 3, h, w)`. | +| `map_frames(clip, fn, width=None, height=None)` | apply `fn(nchw) -> nchw` to every frame of an RGBS clip, at an output size, keeping frame properties. | +| `to_rgbs(clip, matrix, range_)` / `from_rgbs(rgbs, fmt, matrix, range_)` | any clip ↔ planar float RGB with the source's matrix and range (zimg names), dithered on the way back — the conversion the AI upscaling block does, for a pass that works in RGB. | +| `shifted(clip, offset)` | the clip moved by `offset` frames, clamped at the ends. Pass several to `std.ModifyFrame` and a callback gets its neighbours' frames without calling `get_frame()` inside a request. | +| `FrameMemo(capacity)` | `.get(key, compute)`: a bounded cache of a pure per-frame computation, computed once even when several requests want it at the same time (several nodes reading one result, neighbours sharing a detection). | + +A pass that reads neighbouring frames must stay a **pure function of its +input frames**: VapourSynth requests frames in parallel and out of order, so +nothing may depend on which frame was computed first. Take neighbours through +`shifted` clips, share work through a `FrameMemo` (losing an entry only costs +recomputing it), and declare the reach as the pass's `FrameMap` radius so the +preview decodes enough context (`worker/templates/face_restore.py` is the +worked example). + +TensorRT/CUDA are hooks for a future NVIDIA add-on (its directory passed to +`setup()` before `ai-runtime`'s); today's wheels don't offer them. + +### The probe + +`vapourbox-worker --probe-ai-runtime` imports the runtime the way a job does, +runs the committed self-test model (`Tests/TestResources/ai_runtime_selftest.onnx`, +embedded in `ai_runtime.py`, built by `Scripts/models/make_ai_runtime_selftest.py`) +directly and through `map_frames`, and prints JSON: `available`, `reason`, +`detail`, `message` (user-facing, with a remedy), `version`, `providers`, +`provider`, `fallback`. Only a positive result is cached, per boot, add-on +version and force-CPU setting (`/ai-runtime-probe.json`). The app reads +it with `WorkerManager.probeAiRuntime()` → `AiRuntimeStatus`. + +## Adding an add-on: checklist + +1. **Conversion** (if upstream ships no ONNX): a script in `Scripts/models/`, + run in the conversion venv, writing into `addon-staging/`. +2. **Manifest**: `app/assets/addons/.json`, every file pinned by size and + SHA-256, `requires: ["ai-runtime"]` for an ONNX Runtime model. Self-hosted + files use the `addon-models-v1` URL above. +3. **Credits** — `addon_service_test.dart` ("credits agree everywhere") fails + until all exist for the main component and each `components` entry: + a `licenses/NOTICES.txt` entry headed ` — ` whose body + names the credit and says it is an add-on; an About tile in + `app/lib/views/about_dialog.dart` named ` (optional download…)` + with the same licence; the credit in the README's Acknowledgments. Take + holders and licences from upstream's LICENSE, never from a handle (see + CLAUDE.md, "Attribution"). A non-commercial licence needs `usageNote`. +4. **Worker**: the pass adds an `AddonNeed` in `ProcessingPipeline::required_addons()` + and loads its files from `addons::addon_dir(...)`; its template block uses + `ai_runtime` (both templates). +5. **App**: the feature calls `AddonPrompt.ensure(context, [''], reason: …)` + when it is chosen. +6. **Tests**: a heavy integration test that calls + `WorkerHarness.ensureAddon('')` (installs it and its requirements with + the app's own installer, so a dead URL or changed hash fails CI) and runs + the feature. Before upload, run it with `VAPOURBOX_ADDON_MIRROR=addon-staging`. +7. **Upload** (the owner): attach the files in `addon-staging/` to the + `addon-models-v1` release (`gh release upload addon-models-v1 `; + the release is created once with `--latest=false`, like every deps and + whisper release, so it never takes Latest from the app). Attach the + upstream licence as `LICENSE-.txt` if it is not there yet, and add + the file's row to the release notes. Never replace a + file a released manifest already pins — a changed file gets a new name and + the manifest a new `version`. Then run the test again without the mirror. diff --git a/docs/BUILDING.md b/docs/BUILDING.md index b2fb3533..fdea36cf 100644 --- a/docs/BUILDING.md +++ b/docs/BUILDING.md @@ -46,6 +46,7 @@ VapourBox/ │ │ ├── services/ # WorkerManager, PresetService, FilterLoader │ │ └── widgets/ # Reusable UI widgets │ ├── assets/filters/ # Built-in filter schemas (JSON) + manifest.json +│ ├── assets/addons/ # Add-on manifests, one JSON per add-on (docs/ADDONS.md) │ ├── assets/deps-version.json# Which deps release the app expects │ ├── test/ # Unit, shell-out and integration tests │ ├── macos/ # macOS platform config @@ -74,6 +75,7 @@ VapourBox/ ├── packaging/ # macOS + Windows packaging assets (entitlements etc.) ├── licenses/ # License files (GPL, LGPL, NOTICES) ├── Scripts/ # Build, packaging, and release scripts +│ └── models/ # Model conversion scripts for self-hosted add-on files ├── Tests/TestResources/ # Committed test fixtures (small_clip.mp4, etc.) ├── docs/ # Additional documentation │ └── reports/ # Investigation reports: screenshots, measurements, decisions @@ -337,7 +339,7 @@ Do **not** run `app/build/macos/Build/Products/Release/vapourbox.app` directly ## Running Tests -There are three suites. Both `cargo test` and `flutter test --exclude-tags heavy` +There are three suites. Both `cargo test` and `flutter test --exclude-tags heavy,gpu` run in the CI gate (`ci-test.yml`) on macOS arm64, macOS x64, Windows x64 and Linux x64. @@ -345,11 +347,15 @@ Linux x64. # 1. Rust worker tests — script generation, preview, subtitles cd worker && cargo test -# 2. Flutter suite, minus the encode-heavy tests — what the push gate runs -cd app && flutter test --exclude-tags heavy +# 2. Flutter suite, minus the encode-heavy and GPU tests — what the push gate runs +cd app && flutter test --exclude-tags heavy,gpu # 3. Everything, including the full-encode integration tests (slow) cd app && flutter test + +# The GPU tests (RIFE, AI upscaling) on a machine with a GPU; the variable +# turns "no Vulkan device" from a skip into a failure +cd app && VAPOURBOX_EXPECT_VULKAN=1 flutter test --tags gpu ``` - `worker/tests/filter_integration_test.rs` **only generates and inspects the @@ -357,8 +363,12 @@ cd app && flutter test or plugin-behaviour bugs. Those need the Dart integration tests. - `app/test/integration_*_test.dart` spawn the worker end-to-end via `app/test/support/worker_harness.dart`. The encode-heavy ones are - `@Tags(['heavy'])` and run in `nightly.yml` (`flutter test --tags heavy`), not - the push gate. + `@Tags(['heavy'])` and run in `nightly.yml` (`flutter test --tags heavy + --exclude-tags gpu`), not the push gate. +- Tests that need a Vulkan device are tagged `gpu`. The hosted CI runners have + no GPU, so neither workflow runs them: run them yourself on a machine with a + GPU after changing RIFE, AI upscaling or their models, and on Windows and + Linux before a release. - The harness honours `$VAPOURBOX_DEPS_DIR`, else repo-root `deps/`, else **downloads the deps release pinned in `app/assets/deps-version.json`** (opt out with `$VAPOURBOX_SKIP_DEPS_DOWNLOAD=1`). Call diff --git a/docs/ENGINEERING_NOTES.md b/docs/ENGINEERING_NOTES.md index a5eeb270..858dc50b 100644 --- a/docs/ENGINEERING_NOTES.md +++ b/docs/ENGINEERING_NOTES.md @@ -92,6 +92,13 @@ them, and neither can move. > original duration; audio is only ever `-ss` seeked and `-shortest` truncated. > If a pass is ever added that *does* retime audio (an `atempo` for a declared > frame-rate change, say), this breaks and the subtitles drift. +> +> **2026-11-08: one now does.** Audio Restoration's sync offset and drift +> correction move audio in time. It renders the output's soundtrack before the +> encode, and transcription reads that rendered track instead of the source +> window, so cues follow the retiming by construction (see "Audio restoration" +> below). The rule that survives: transcription hears exactly the output's +> audio, and anything that moves audio lives in `audio_restoration`. `SubtitleOutput::{burns_in, muxes, keeps_srt_file}` answer the three independent questions rather than matching the enum in three places. A test asserts every @@ -2474,9 +2481,10 @@ Pixel checks (heavy test), against a plain Spline36 upscale of the same frames: centre on both axes (0.75 px), and that Spline36 itself lands on factor x source (within 0.25 px), so the reference is known-good. -CI runs it for real on Linux and Windows: Mesa lavapipe (ncnn tags a CPU -device as type 3 but does not skip it) and `VAPOURBOX_EXPECT_VULKAN=1`, so an -unavailable feature is a failure there, not a skip. The first lavapipe probe +CI ran it for real on Linux and Windows until 2026-11-09 (see "GPU tests +leave CI" below): Mesa lavapipe (ncnn tags a CPU device as type 3 but does not +skip it) and `VAPOURBOX_EXPECT_VULKAN=1`, so an unavailable feature was a +failure there, not a skip. The first lavapipe probe takes ~30 s (Windows: 30.4 s), past flutter_test's default timeout, and one whole 720x576 Real-CUGAN frame took over five minutes, so the push gate crops to 240x192 first. @@ -2882,6 +2890,346 @@ MPEG-2 at 1000/1500/3000 kbps with `+ildct+ilme`: - The film fixture's reference carries `small_clip.mp4`'s own (upscaled) H.264 blocks; PSNR rewards deblocking's smoothing, and VMAF dips slightly. +### AI models become add-ons; the AI runtime is ONNX Runtime wheels (2026-11-08) + +Policy (owner's decision): every AI model ships as an optional add-on, never +in the deps bundle, through one generic manifest format (docs/ADDONS.md). The +first manifest add-on is `ai-runtime`, ONNX Runtime + NumPy for passes that +need per-crop inference or NumPy around a network, which VSNCNN can't do. +The facts the manifest rests on, checked against PyPI and the wheels on this +date (groundwork, not a quality comparison, so no report): + +- **What `import onnxruntime` needs is NumPy, nothing else.** The wheels + declare flatbuffers, packaging, protobuf (and on 1.23.2 / DirectML + coloredlogs and sympy), but importing the package and running an + `InferenceSession` in the deps Python with only onnxruntime + numpy on + `sys.path` loaded no other third-party module; a scan of the 1.23.2 and + DirectML wheels' runtime modules found no top-level import of them + either (they are used by `tools/`, `transformers/`, `quantization/`). +- **Intel Macs stop at onnxruntime 1.23.2**, whose wheel is + `macosx_13_0_x86_64` — above the x64 app floor (12.0), so the add-on carries + `minOs: 13.0` there and an older Mac is told so. NumPy 2.5.4's Intel wheel + is `macosx_10_13`, and its arm64 one `macosx_14_0` (Accelerate), both + within their app floors. +- **Windows uses onnxruntime-directml 1.24.4**, so any DirectX 12 GPU works. + Its `DirectML.dll` (1.15.4) is under Microsoft's proprietary DirectML + licence (read from the Microsoft.AI.DirectML 1.15.4 NuGet package — the + wheel ships no licence text for it), credited as its own component. +- **Windows' embeddable Python ignores PYTHONPATH** (its `._pth` fixes + `sys.path`), so `ai_runtime.setup()` inserts the add-on's `site-packages` + itself, and registers `onnxruntime/capi` with `os.add_dll_directory`. + Untested on Windows from macOS: the nightly heavy run is the first check. +- On an M-series Mac the self-test session reports `CoreMLExecutionProvider` + first; whether Core ML or the CPU fallback executed the two-node model is + not distinguishable from `get_providers()`. + +### Live-action AI upscalers as add-ons, and AI strength (2026-11-08) + +Full report with crops, every number and the scripts: +[docs/reports/2026-11-ai-upscale-addons/](reports/2026-11-ai-upscale-addons/index.html). +Crop & Resize's AI upscalers were all trained on anime; three more methods +now cover real-world footage, run by the same VSNCNN block, with their +models in two manifest add-ons (`upscale-general`, `upscale-esrgan-plus`). +Measured on an M1 Pro (16 GB): + +- **Converting Real-ESRGAN's `.pth`** (`Scripts/models/convert_realesrgan.py`): + SRVGGNetCompact and RRDBNet transcribed, opset 13 / IR 7 like vs-mlrt's own + conversions. PyTorch vs ONNX Runtime: at most 0.017 of an 8-bit level. The + TorchScript export of x2plus's `pixel_unshuffle` is a 6-D Reshape/Transpose + ncnn can't import; a one-hot stride-2 convolution is exactly equal and + imports. Two runs are byte-identical. +- **VSNCNN fp32 reproduces ONNX Runtime exactly** (≤0.005 levels) for all + five models; fp16 (what the template uses) differs by a mean 0.05–0.12 + levels, the same order as the bundled anime model's 0.03. +- **2xVHS2HD (RealPLKSR) can't run on VSNCNN.** Its official ONNX (dynamic + shapes, in-place slice assignment) has 225 Shape, 56 Range/Where/Equal/ + ConstantOfShape/Cast and 28 ScatterND nodes, none of which ncnn's ONNX + importer supports ("load param failed"). Re-exporting from the `.pth` with + concatenation instead of slice assignment might work; not attempted (the + brief was to stop there). +- **Real-ESRGAN x4plus needs tiles at any size.** Whole-frame at 720x576 lost + the Vulkan device (`vkWaitForFences failed -4`, then a segfault); 512-px + tiles thrashed at 29 s a frame; 384 is the fastest (6.8 s a frame), 256 + gave 10 s. x2plus runs a DVD frame whole at 0.8 fps. +- **A tile larger than the frame segfaults VSNCNN** (found converting the + PSNR run: 384 tiles on a 360x288 frame). The template tiled whenever the + area passed the threshold, so an 800x200 picture (x4plus) — or, for every + AI method, a frame above 720x576 in area but under 512 lines — would have + crashed. Tiles are now `min(tile, width/height)`; the heavy suite runs + x4plus 4x on an 800x200 source. +- **General at 2x = 4x, then resample down**, as upstream's own `--outscale`. + The kernel moves PSNR by under 0.7 dB, and softer kernels score higher + (bilinear 28.40, Spline36 27.75, Lanczos 27.70) because PSNR rewards blur; + Spline36 is the pipeline's kernel and was kept. +- **Flat colour vs Spline36** on the synthetic pattern (worst channel): General + 2.9 levels, Live Action SPAN 0.6 at 2x / 1.3 at 4x, x2plus 4.2, x4plus 12.2. + Bar centre within 0.3 px for all. **AI strength** 0.5 lands within 0.38 + levels of the mid-point of network and Spline36 and halves Real-ESRGAN 4x's + tint (11.2 → 5.8). +- Speed at 720x576 (fp16, two streams, quiet machine): SPAN 2x 8.9 fps, General 2x 4.0, + x2plus 0.9, SPAN 4x 1.5, General 4x 3.9, x4plus 4x 0.15; bundled anime + Real-ESRGAN 2x 8.3 and CUGAN 4.8 for scale. + +### Audio restoration: rendered before the encode; four filter traps measured (2026-11-08) + +Full report with scripts: +[docs/reports/2026-11-audio-restoration/](reports/2026-11-audio-restoration/index.html). + +A new pass, Audio Restoration, renders one audio stream to a WAV before the +video encode (channel fix, sync offset, drift, declip/declick/rumble/hum, +afftdn, DeepFilterNet as an add-on, two-pass loudness, limiter); the encoder +maps that track first and keeps the other streams. Rendering first, not an +`-af` on the encode, is what DeepFilterNet, two-pass loudness, negative offsets +and subtitle alignment all need. Measured with ffmpeg 9.0.2 and deep-filter +0.5.6 on Apple Silicon, on lavfi-generated signals: + +- **afftdn delays by 25 ms at every rate and strength** (1200 samples at + 48 kHz, 1102 at 44.1, 800 at 32; Video-Enhancer's 1200 confirmed) — trimmed + with `atrim=start_sample`. Its default `nf=-50` took 0.7 dB off −39 dB white + noise; with the measured floor (quietest tenth of 50 ms windows) 5.5/9.8/ + 13.5/15.2 dB at nr 6/12/20/30, a 440 Hz tone unchanged; `tn=1` (noise + tracking) undid the measured floor entirely (0.7 dB again), so it is off. +- **atempo moved every click ~20 ms early** (≈ −19 to −21 ms after the first, + at 1.004, 0.996 and 1.02), on top of the stretch. Drift is now an exact + resample (`aresample` to 4–8× → integer `asetrate` → `aresample` back, + ratio within ~1e-11): click positions within 0.01 ms. Resampling is also + the physically right inverse of a sample-clock error (it restores pitch). +- **`alimiter` delays by its attack unless `latency=1`, and normalises to + 0 dBFS unless `level=0`**: measured 239 samples (5 ms) late at 48 kHz, and a + limit of −6 dB came out peaking at 0.00 dBFS; with both set, 0 lag and + −6.02 dBFS. loudnorm, adeclick, adeclip, highpass, the notches and the + drift resample measured zero lag; loudnorm outputs 192 kHz. +- **A late-starting audio stream** (capture audio 0.5 s after the video) lost + its gap in a WAV: `aresample=first_pts=0` restores it with no seek; with an + input seek the window starts at the seek point as before. +- **DeepFilterNet** delays 30 ms (1440 samples at 48 kHz); `-D` compensates + and shortens the output by as much, padded back. It takes 44.1 kHz input and + always writes 48 kHz. On a synthetic voiced signal in pink noise: pauses + 25.3/22.5/13.7 dB quieter at `-a` 100/30/15, voiced level −1.1 dB, ~4 s per + minute (mono, M-series CPU). Its SNR *inside* the synthetic voice dropped + (20.9 → 13.8 dB): a harmonic test tone is not speech, which is the network's + domain — judge it by ear on real dialogue. Not tested on Windows/Linux, or + the macOS x64 build (unsigned; the installer signs it). +- Hum notches (`bandreject` Q 8 at f/2f/3f): 47/61/71 dB down at 50/100/150 Hz, + 1 kHz unchanged. Two-pass linear loudnorm hit −23 and −16 LUFS to 0.0 LU. +- Found on the way, not fixed (another area): `whisper-addon.json` states + `ggml-small.bin` as 466,616,853 bytes; HuggingFace serves 487,601,967. The + app only uses the size for progress, but `WorkerHarness`'s downloader + enforces it, so `WorkerHarness.ensureWhisper(model: 'small')` fails until + the manifest is corrected. + +### SangNom, vszip 24 (NLMeans, WNNM) and three vs-jetpack ports (2026-11-08, deps 1.16.0 unreleased) + +Report: [docs/reports/2026-11-classical-filters](reports/2026-11-classical-filters/index.html). +Probed on macOS arm64 against a private deps copy with vszip 24.0.0 and +SangNom r45; fixtures are three Kodak images panned 2 px/frame (exact ground +truth, real motion). + +- **havsfunc's `santiag(type="sangnom")` is broken against SangNom r45.** Its + strength-0 step calls `SangNom(order=0)`; r45 documents order 0 as + double-rate and refuses a clip without `_FieldBased`. So SangNom became its + own Anti-aliasing method (2x Spline36, `SangNom(order=1)` per axis, back + down). Every depth runs; 8 vs 10-bit 0.44 levels (santiag's own: 0.37). +- **SangNom's licence is split.** Source header + LICENSE: MIT (james1201, + Victor Efimov); the same header: "Binaries are GPL v2"; PyPI metadata: + GPL-2.0. GitHub's SPDX detection only sees the MIT file. Credited as both. +- **vszip: NLMeans arrived in 24.0.0, WNNM in 22.0.0.** 23.0.0 moved to Zig + 0.17 (the macOS x64 sdist build needs `-Doptimize=fast` and the headerpad + patch on the 8-space `use_llvm` inside `if (build_vs)`; cross-built and + loaded under Rosetta with the 1.10.0 x64 VapourSynth) and made CLAHE accept + every depth. The arm64 wheel is now minos 15.0, exactly the floor. +- **vszip's NLMeans is KNLMeansCL on the CPU.** Same arguments, identical + output (0.0000 levels on the probe, 0.0003 through the worker), h + depth-independent (8 vs 10-bit 0.005). Hence one set of `knlm_*` parameters + for both methods. +- **WNNM was measured and not added.** At each method's best sigma, against + the shipped BM3D path: luma +0.7 to −2.3 dB (ahead only at sigma 4 with + `radius=1`), chroma 2.6–7.2 dB worse. Widening its sweep to 4x the noise + found nothing past 2–2.5x. It is fast (10.3 fps at 1080p with `radius=1`; + the shipped BM3D path ran 0.2 fps there on arm64, all three planes at + sigma 16), but NLMeans (CPU) is faster (23.0 fps) with better chroma. + SangNom AA turned out the slowest AA method at 1080p (8.6 fps; daa 15.9, + santiag 19.6) — an early clip.frames() probe under load had it fastest, + and the first draft said so. WNNM needs 32-bit float and divides sigma by 255, + so its float conversion must keep sigma in 8-bit units: with the default + (limited) range, luma spans 219 levels per 1.0 and chroma 224. Passing + `range_in_s="full"` instead broke at 10-bit (a 7-level offset) because the + clip carried `_ColorRange=limited`. +- **vszip's own CPU BM3D** (also new in 24.0.0) scored 1.1–2.2 dB below the + bm3d r12 path on every fixture; not used. +- **The ports were checked against vs-jetpack itself** (vsjetpack 2.2.7 on the + bundle's Python 3.12; its three functions are byte-identical to 3.0.0's): + mpeg2stinx 0.04 levels, FixInterlacedFades exact in both modes. vine could + not run there — vs-jetpack's Catrom is a custom resize2 kernel. +- **mpeg2stinx lost PSNR on every fixture** (progressive pans as interlaced + MPEG-2, and 3:2-telecined MPEG-2 after VIVTC): −0.1 to −3.0 dB, about −4.5 dB + on cleanly coded frames and +0.1 on starved ones. It clamps each field to a + bob of the other, which costs vertical detail; the fixtures lack the + comb-like noise it targets. Shipped advanced-only with that in its + description. VFM flagged every frame of the detailed pan `_Combed`. +- **FixInterlacedFades closes the comb of a fade applied after the field + split**: field gap 5.27 → 0.29 levels (average), 0.00004 (match); through the + worker on a real clip 4.12 → 0.06. Two traps: `average` leaves the last + frame of a fade to full black half-lit (one field's distance is 0, so it is + left alone and the other set to half); and `std.DoubleWeave` sets + `_FieldBased=2`, which made a later 4:2:0 resize treat the progressive + frames as fields ("dimensions must be divisible by subsampling factor" at + an odd field-chroma height) until the block reset it. +- **The first fade fix crashed or hung real jobs.** It made an akarin + `Expr` per frame inside two concurrent FrameEvals. Under vspipe that + aborted (SIGABRT, 10 frames), segfaulted (SIGSEGV, a 720x576 IVTC job + through the worker) or spun one core indefinitely (97 minutes on 30 + frames, found when the benchmark hung); single-threaded, or with + `std.Expr`, it was fine, and the 320x240 heavy test and the Python-side + probes never tripped it. Fixed by stamping the gains as frame properties + (`ModifyFrame`, no node creation) and applying them with one akarin + expression; the std.Expr FrameEval remains only where akarin is absent. + Colour Correction's auto white balance makes an akarin node per frame in a + single FrameEval the same way; it did not reproduce (30 frames, vspipe), but + it is the same pattern. A heavy test now times a full-size IVTC + fade-fix + job (2.5 s), and fails on the old block with the SIGSEGV. +- **vine** on an oversharpened pan: +0.9 to +1.1 dB, against DeHalo_alpha's + +4.1 to +4.4 (which blurs the amplified texture away). Advanced-only. +### Face restoration: RestoreFormer++ on ONNX Runtime, steadied in aligned space (2026-11-08) + +Full report with images and scripts: +[docs/reports/2026-11-face-restoration/](reports/2026-11-face-restoration/index.html). + +A Face Restoration pass (after Crop & Resize) on the `ai-runtime` add-on, its +models as add-ons: `face-restoration` (RetinaFace R50, BiSeNet, +RestoreFormer++; Apache-2.0/MIT), `face-gfpgan` and `face-codeformer` (both +non-commercial: NVIDIA's licence on GFPGAN's StyleGAN2 code, S-Lab License +1.0). Measured on an M1 Pro with a VHS-degraded clip of NASA public-domain +footage (`Tests/TestResources/face_restore_test.mp4`): + +- **Conversions are exact**: ONNX vs PyTorch ≤2.5e-4 on [-1, 1] (0.03 of an + 8-bit level), reproducible byte for byte. Two departures from upstream: + GFPGAN exported with its fixed noise buffers (upstream draws fresh noise + per call — new texture every frame), and CodeFormer's `w` as a graph input + (the traced `w > 0` branch adds exactly zero at `w = 0`). +- **The NumPy pipeline matches facexlib + OpenCV**: landmarks 6e-5 px, + alignment 7e-5, final frame ≤0.16 levels for the same restored face. +- **Detector: ResNet-50 over mobilenet0.25.** Both found the face in every + frame, but mobilenet's landmarks jittered 1.5x (SD) to 2.2x (1080p wide + shot) as much; ResNet-50 costs 87 ms vs 7 ms per frame on Core ML. +- **Temporal stability**: smoothing landmarks over n±1 (the obvious fix) + changed nothing measurable here (flick 2.287 → 2.284; this detector was + already steady). What works is blending each restored face with its + neighbours' in the aligned 512 space, weighted to zero where the input + crops moved: RestoreFormer++'s excess flicker over the clean source halved + (2.287 → 1.799; 1.571 at the default strength 0.8), GFPGAN's fell 86%, + with no rise in distance from the clean source (no ghosting the metric + sees). One mostly still clip; fast motion untested. +- **Bit-exact outside the faces** (MaskedMerge in the source format): 0 + samples changed outside the face box over 16 frames at 8 and 10 bits; + preview = encode exactly. +- **Speed** (one face per frame, RestoreFormer++): 2.2 fps SD and + 1.8 fps 1080p on Core ML; 0.085 / 0.072 fps on the CPU, + ~25x slower (Linux has only the CPU). +- **Core ML traps**: concurrent predictions from VapourSynth's threads ran + the GPU out of memory (`kIOGPUCommandBufferCallbackErrorOutOfMemory`), so a + pass's model runs are serialised (`ai_runtime.Runner`, one lock) with a CPU + retry; `ModelCacheDirectory` made the second session fail + (`output_features has no value`), so every job compiles — CodeFormer takes + 30–50 s, the others 2–8 s. +- Untested: Windows (DirectML), Linux (CPU), macOS x64; crowds, profiles and + scene cuts across a face. + +### Colourisation: DDColor and DeOldify Video, luma kept bit-exact (2026-11-08) + +Full report with images and scripts: +[docs/reports/2026-11-colourisation/](reports/2026-11-colourisation/index.html). + +A Colourisation pass (between Chroma Fixes and Color Correction) on the +`ai-runtime` add-on, one add-on per model: `colorize-ddcolor` (DDColor-T, +Apache-2.0, 220.7 MB) and `colorize-deoldify` (DeOldify's Video generator, +MIT, 872.8 MB; DeepAI's copy cross-checked by SHA-256 against a Hugging Face +mirror). Measured on an M1 Pro with 5 s of "Diary of a Sergeant" (1945, US +War Department, public domain; `Tests/TestResources/colorize_test.mp4`): + +- **Conversions are exact and reproducible byte for byte.** DDColor ≤1.8e-3 + CIELAB units from PyTorch at 256-640; DeOldify ≤1.4e-6. DeOldify had to be + rebuilt without fastai (v1 doesn't install with a current torch); checked + against FaceFusion's independent export it first differed by 0.016 mean — + fastai's `PixelShuffle_ICNR(blur=False)` blurs anyway (it tests the module, + not the flag) — then agreed to 1.8e-6. +- **Core ML needs the graph rewritten, and still compiles DDColor slowly.** + Edge-mode `Pad`, `Expand` (`Tensor.repeat`) and `Einsum` aren't in ML + Program: DDColor split into six partitions. Rewritten exactly, each model is + one partition and DDColor runs at 101 ms per frame at 512 (261 ms before), but compiling it + takes 77 s in every process (DeOldify: 3 s). Previews (a process each) run + DDColor on the CPU (5 frames in 3.8 s); `ModelCacheDirectory` cut a second + session to 14.5 s but cached 898 MB per render size, keyed by model path. + Core ML refused DDColor with free dimensions outright, so one dynamic-shape + file serves every size through ONNX Runtime's free-dimension overrides + (`ai_runtime.session(dims=)`). +- **Luma bit-exact** at 8, 10 and 16 bits, field-based, and on a colour source: + the colour becomes YUV in the source's matrix and only U/V are taken. + DDColor's CIELAB recombination (std.Expr) matches the exact formulas to + 0.0014 levels; OpenCV's float path (what upstream calls) differs by up to + 3.8 levels near black because it tabulates the sRGB curve. Upstream scales + DDColor's a/b nearest-neighbour; the pass uses bilinear. +- **Flicker**: per frame, colour flickers by 1.45 levels where the picture is + still (DDColor; DeOldify 0.33, but with 4.5x less colour). Averaging each + prediction with 2 neighbours either side, weighted to zero where the grey + picture moved, cuts both by 84%; the motion weighting changes still flicker + by <0.01 but cuts the colour lag in moving areas by a third (2.00 → 1.33). +- **Speed** (whole pass, steady colour 2): DDColor 9.1 fps SD / 8.4 fps 1080p + on Core ML (1.3 on the CPU); DeOldify 6.0 / 6.0 (0.65 on the CPU). +- No preset turns it on (8 mm home movies are mostly colour). Untested: + Windows (DirectML), Linux, macOS x64; fast pans. +### Crop & Resize: BT.601 → BT.709 on SD → HD, and Detect borders (2026-11-08) + +Full report with images and scripts: +[docs/reports/2026-11-crop-detect-709/](reports/2026-11-crop-detect-709/index.html). + +**BT.709 conversion.** An SD source resized to HD kept its BT.601 matrix and +tag; a player that ignores the tag on an HD picture (many do) decodes it as +709. Measured: a 75% green comes back 30 levels dark (191 → 161), 100% green +40, magenta and red 25–30. Crop & Resize now converts the matrix in the +script when the output is HD (`convertToBt709`, default `auto`), and +`VideoJob::output_color_metadata` is the one decision every site after the +conversion follows. Measured through the real worker: every patch within +0.67 levels of the source decoded as 601, whether the output is read through +its tags or as 709 regardless; preview = encode exactly. The conversion costs +~7–10 ms per 1350x1080 frame (8 threads, M1 Pro); a 601 → 709 → 601 round +trip changes luma by 0.25/255 on average. + +- **Matrix only, primaries kept and tagged as the source's.** Also converting + SMPTE C → BT.709 primaries moved a 75% green to (55,188,0) and a 75% magenta + to (187,47,191) when decoded without colour management (most players): a + hue shift of its own, as large as the one being fixed. EBU (PAL) primaries + differ only in green, and still moved 75% green to (0,191,30). A + colour-managed player reads the kept primaries tag and shows the source's + colour either way. +- **Default `auto`, deliberately changing existing output**: an SD job that + resizes to HD now writes 709-tagged, converted samples where it wrote + 601-tagged ones. The 601-tagged HD file was correct but shown wrong by the + players people use; the converted one is shown right by all of them. No + built-in preset resized before, so none changes; the new `SD Tape → HD + (YouTube / TV)` preset sets `on`. +- **VapourSynth R78 trap**: a frame's `_Matrix` property overrides + `matrix_in_s`. The first measurement script read "601 decoded as 709" equal + to the source because of it; the template's block removes `_Matrix`/`_Range` + before converting. + +**Detect borders (`--detect-crop`).** VCG Deinterlacer 2.0's `edges.rs` (MIT) +was read for the signals; the code is our own (nothing ported, so no NOTICES +entry). On fixtures built from real VHS picture with known borders every edge +came back exact (noisy blanking 14/10 px, a 2-line top, an 8-line band; 12 +lines per field interlaced; 72-line letterbox bars reported apart; subtitles +40 lines up and fades to black ignored); on a real DVB-T broadcast 7/8 black +columns and 4 black + 1 dim bottom line, as measured by eye; ~0.5–1.5 s per SD +source and ~2.9 s at 1080p (release worker, 24 frames). Traps designed +against, each with a unit test: dark scenes (98th percentile over all frames, +dark frames dropped), noisy blanking (absolute detail floor), letterbox +(bars over 5% are framing), subtitles (band must span 6 of 8 width segments in +half the frames, from the bottom), interlaced (compare within the field). +Untested: a real capture with a visible head-switching band (none in +`Tests/TestResources/`; the real VHS fixture's bottom is too dark to show +one), and Windows/Linux (the analysis is plain Rust; only the decode is +platform-specific). + ### Testing a deps change before publishing A PR that changes `deps/` has a chicken-and-egg problem: `ci-test.yml` and @@ -2966,3 +3314,149 @@ names an `-rc` tag. That is the one way this bites: an rc escaping into a shippe build, where later deleting the prerelease breaks the download for everyone who installed it. +### GPU tests leave CI: the `gpu` tag (2026-11-09) + +From 2026-10-25 the Linux and Windows CI jobs ran the RIFE and AI upscaling +tests on Mesa's lavapipe, a CPU Vulkan driver, with `VAPOURBOX_EXPECT_VULKAN=1`. +On the 1.7.0 branch that stopped being workable, almost all of it on Windows: + +- One 240x192 Real-CUGAN preview frame took 2m21s on `main` and from 3 to over + 5 minutes on the branch, with nothing in our code to account for it (CI runs + 42484522476, 42499908163). Real-ESRGAN x4plus at 4x took ~18 minutes for five + frames on Linux and ran past 20 on Windows (nightly 42484525297). +- When a timed-out test's harness killed only the worker, Windows left vspipe + running the network on every core and the next test timed out too. The + harness now kills the whole tree (`ProcessTree.killTree`). Even so, run + 42516550724 timed out both AnimeJaNai tests at 5 minutes; that test had + taken about 15 s before. + +These timings measure the runner's CPU, not the GPU path a user has, and every +budget raise was followed by another timeout. So tests that need a Vulkan device +are tagged `gpu` and both workflows exclude the tag, and +`.github/scripts/provision-vulkan.sh` is deleted (it is in git history if a GPU +runner ever appears). Without a device CI keeps the probe-answer and refusal +tests. What is lost: CI no longer proves that RIFE and VSNCNN load and run on +the Windows and Linux bundles. That now has to be checked by hand on a machine +with a GPU (`VAPOURBOX_EXPECT_VULKAN=1 flutter test --tags gpu`), as MoltenVK +always was. + +### RIFE's and the anime upscalers' models leave the deps bundle; probes on stand-in models (2026-11-09, deps 1.16.0 rc2) + +The 2026-11-08 policy (every AI model an add-on) had two exceptions left from +deps 1.14.0: RIFE 4.26's model in `vapoursynth/rife/models/` (11 MB) and +vs-mlrt's Real-ESRGAN anime / Real-CUGAN ONNX files in +`vapoursynth/ncnn/models/` (20 MB). They are now the `frame-rate-rife` and +`upscale-anime` manifest add-ons; the plugins (librife, VSNCNN, MoltenVK) +stay in the bundle. Mechanism findings, read from the pinned sources and +checked on an M1 Pro against the deps 1.16.0-rc1 bundle with its model +directories deleted (groundwork, not a comparison, so no report): + +- **RIFE's model directory name is load-bearing.** The plugin + (`styler00dollar/VapourSynth-RIFE-ncnn-Vulkan` r9_mod_v33, `plugin.cpp`) + decides the model family from substrings of `model_path` (`rife-v2`, + `rife-v3.9`, `rife-v3`, `rife-v4`, `rife4`) and the padding from more + (`rife-v4.25` 64, `rife-v4.25-lite` 128, `rife-v4.26` 64, else 32). An + add-on id cannot hold a dot, so the manifest format now allows a file + `name` to be a relative path of plain names, and the add-on installs as + `addons/frame-rate-rife/rife-v4.26_ensembleFalse/flownet.{param,bin}`. +- **VSNCNN takes a model as bytes**: `network_path` with + `path_is_serialization=True` (vs-mlrt v15.16 `vs_ncnn.cpp`). `loadONNX` + needs one 4-D input and output; it fixes N/H/W itself, so dynamic dims + (`n`, `h`, `w`) are fine. A 264-byte ONNX (`Tests/TestResources/ + ncnn_selftest.onnx`, one 1x1 Conv swapping channels 0 and 2, bias 0.125) + ran on Metal through MoltenVK: input (0.2, 0.4, 0.6) gave + (0.72510, 0.52490, 0.32495) with `fp16=True` (exact: 0.725, 0.525, 0.325). +- **RIFE runs a weightless stand-in.** Its v4 path feeds `in0`, `in1` (the + two padded frames, 0-255), `in2` (the timestep plane) and extracts `out0`, + through its own preproc / timestep / postproc shaders. A two-layer ncnn + param file (`BinaryOp` add of `in0`+`in1`, then multiply by 0.5) with an + empty `flownet.bin`, in a directory named `rife-v4.26_selftest`, turned + frames (0.2, 0.4, 0.6) and (0.6, 0.4, 0.2) into exactly 0.4 in every + channel at frame 1 of a 25 → 50 retime. +- So both probes now run a stand-in network and check the answer + (`WrongResult` if the GPU computes garbage, tolerance 0.01), and report + available on a bundle with no model in it and no model add-on installed. + "Can't run here" (no Vulkan, missing VC++/libgomp, old CPU) is therefore + known before the user downloads anything; a missing add-on is a separate + refusal, through `required_addons()`, naming the add-on and Settings → + Add-ons. A stand-in that fails to load (`failed to load model`, `load param + failed`) is now `LoadFailed`, not `NotInstalled`: nothing the user installs + would fix it. +- The anime models are hosted by us, byte for byte as vs-mlrt ships them + (same SHA-256 as the deps scripts pinned), because vs-mlrt publishes them + only inside `models.v15.16.7z` (~810 MB). RIFE's come from upstream's + repository at the commit the deps scripts pinned. +- Upgrade path: deps 1.14/1.15 users are moved to 1.16.0 by the app anyway, + which removes the bundled copies. A saved pipeline with RIFE or an anime + upscaler selected shows the add-on notice with a Download button in its + panel, prompts when the pass is switched on, and a job is refused with the + add-on's name rather than failing in the plugin. No built-in preset selects + either. + +### Noise and halo scores for the pass list (2026-11-09) + +Full report with images and scripts: +[docs/reports/2026-11-noise-halo-scores/](reports/2026-11-noise-halo-scores/index.html). + +Noise Reduction and Dehalo were always neutral in `pass_relevance.dart`. A +survey of VCG Deinterlacer (MIT) suggested measuring both; its measures were +tried first and rejected: ffmpeg `signalstats` TOUT stays at 0.000-0.012 +whatever noise is added (it counts vertical outliers, not noise), YDIF follows +motion and isn't scaled for 10-bit, and its halo ratio read a clean, dense +photo (0.149) above a sharpened one (0.066). + +- **Noise is measured in time, not space.** Immerkaer's spatial estimator + (block 10th percentile) is exact for white noise but read tape-like noise + (blurred along the line) and grain-like noise of sigma 6 at 1.2-2.5, and + dense texture at 3.6 — spatially they are the same thing. A + motion-compensated frame difference (16x16 blocks, +-10 px search, match + chosen on 3x3-smoothed pictures) sees noise at every frequency and cancels + texture that moved: white sigma 4 reads 3.66-3.79, tape sigma 6 5.0-5.4, + texture panned by 1.37 px a frame at most 2.06. +-6 px was too small for + interlaced fields 8 px apart (clean read 5.5-8.3). +- **Halos: per edge, two-sided, over 7 lines.** Overshoot and undershoot + against plateaus 6-10 px out, with equal near/far sample counts so noise + cancels; min of the two, so outlines score 0; plateau flatness also allows + 3% of the step, or strongly sharpened pictures lose their edges. Clean max + 1.48 (3.30 on real 1080p footage), unsharp amount 1 on sharp SD pictures + 11-71. Sharpening of soft edges (Kodak 23, NASA 1080p, the VHS capture) + rings past the plateau zone and is missed. +- **Thresholds 3.5 and 6.0**, decided in the worker; no clean fixture of 154 + crossed either. Both under-report (compressed sources' skipped blocks read + as noiseless), so only a high score says anything. On the repo's real + footage neither fires (VHS noise 1.40, 1968 film grain 1.57). +- The Rust matches the NumPy reference on the same frame pairs to 0.002 + (noise) and 0.49 (halo; median under 1e-6 — edges on the flatness limit + fall either way between float32 and f64); no verdict differs. 0.5-0.9 s on + SD, 2.7-3.4 s at 1080p, 2.9 s at 2160p (release, M1 Pro). Untested: non-rigid motion (water, foliage), real footage + with known halos, sources with many shots. + +### deps 1.16.0 and `addon-models-v1` published (2026-11-09) + +**deps 1.16.0** is the `deps-v1.16.0-rc2` bundles with only the version +changed, as 1.15.0 was from its rc1. The re-stamp is now committed as +`Scripts/promote-deps-rc.py`, which verifies every entry but `version.json` +against the candidate (CRC, size, permissions, timestamp, compression). One +trap it hit: Python's `zipfile` (3.15, in both `writestr` and `open(..., 'w')`) +gives an entry whose attributes are 0 a mode of `0600`, and every entry of the +Windows bundle (written by PowerShell) has 0, so a naive copy changed every +Windows file's attributes. The script puts each original value back before the +central directory is written. rc2 had passed CI on all four platforms; the +nightly's one failure was the known KNLMeans `std::out_of_range` flake on +linux-x64-v2 (QTGMC's KNLMeansCL path on the runner's CPU OpenCL). + +**`addon-models-v1`** now hosts the sixteen model files we convert or +re-host (face restoration, colourisation, the Real-ESRGAN / Real-CUGAN +upscalers), moved from `deps-v1.16.0-rc1` byte for byte (hashes checked against +the manifests before upload, two re-downloaded and checked after), each family +beside its upstream LICENSE as `LICENSE-.txt` — GFPGAN's includes +NVIDIA's Source Code License terms, which with CodeFormer's S-Lab licence +require the text to accompany a redistributed copy. The release notes list +every file's add-on, upstream, licence and the script that made it. The +manifests point there; the rc1 copies were left in place. + +While uploading the anime models to rc1 on the same day, github.com's +anonymous download link for one asset returned 504 for a while (a +GitHub incident; the API served the file) — re-uploading did not help, waiting +did. A manifest's `url` is independent of its installed `name`, so a stuck +asset could have been re-hosted under another name without touching code. diff --git a/docs/reports/2026-11-ai-upscale-addons/img/01-original.png b/docs/reports/2026-11-ai-upscale-addons/img/01-original.png new file mode 100644 index 00000000..b1272b1c Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/01-original.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/img/02-spline36.png b/docs/reports/2026-11-ai-upscale-addons/img/02-spline36.png new file mode 100644 index 00000000..016cf6bd Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/02-spline36.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/img/03-general.png b/docs/reports/2026-11-ai-upscale-addons/img/03-general.png new file mode 100644 index 00000000..d0e818b8 Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/03-general.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/img/04-general-denoise.png b/docs/reports/2026-11-ai-upscale-addons/img/04-general-denoise.png new file mode 100644 index 00000000..8c84b376 Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/04-general-denoise.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/img/05-live-action-span.png b/docs/reports/2026-11-ai-upscale-addons/img/05-live-action-span.png new file mode 100644 index 00000000..cf0af867 Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/05-live-action-span.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/img/06-x2plus.png b/docs/reports/2026-11-ai-upscale-addons/img/06-x2plus.png new file mode 100644 index 00000000..365b8cc6 Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/06-x2plus.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/img/07-anime-esrgan.png b/docs/reports/2026-11-ai-upscale-addons/img/07-anime-esrgan.png new file mode 100644 index 00000000..4fe42338 Binary files /dev/null and b/docs/reports/2026-11-ai-upscale-addons/img/07-anime-esrgan.png differ diff --git a/docs/reports/2026-11-ai-upscale-addons/index.html b/docs/reports/2026-11-ai-upscale-addons/index.html new file mode 100644 index 00000000..d5ef8d78 --- /dev/null +++ b/docs/reports/2026-11-ai-upscale-addons/index.html @@ -0,0 +1,294 @@ + + + + + +Live-action AI upscalers as add-ons, and AI strength + + + +
+

Live-action AI upscalers as add-ons, and AI strength

+

2026-11-08 · app 1.7.0 · measured on an M1 Pro (16 GB, macOS arm64) against the +deps 1.15.0 bundle's VSNCNN (vs-mlrt v15.16, ncnn on MoltenVK) · scripts in +scripts/

+ +
+Outcome. Crop & Resize gains three AI upscalers for real-world footage, run +by the same VSNCNN block as the bundled anime models, with their models in two +manifest add-ons (never the deps bundle): +
    +
  • upscale-general (11 MB): Real-ESRGAN General +(realesr-general-x4v3, plus its denoise variant -wdn-; BSD-3-Clause, +converted here) and Live Action SPAN (jcj83429's +2xLiveActionV1_SPAN, Apache-2.0, upstream's own ONNX).
  • +
  • upscale-esrgan-plus (134 MB): Real-ESRGAN x4plus / x2plus +(BSD-3-Clause, converted here).
  • +
+Every converted model matches PyTorch to 0.017 of an 8-bit level, +and VSNCNN in fp32 matches ONNX Runtime to 0.012. An AI Strength control +(0–1) blends any AI upscale with Spline36; 0.5 halves Real-ESRGAN 4x's flat-colour tint. +2xVHS2HD (RealPLKSR) was rejected: ncnn's ONNX importer cannot load it +(section 4). +
+ +

1. Converting Real-ESRGAN's checkpoints

+

Upstream (xinntao/Real-ESRGAN) publishes +these models only as PyTorch .pth. Scripts/models/convert_realesrgan.py +transcribes the two architectures (SRVGGNetCompact; RRDBNet with 23 blocks), loads each checkpoint +(pinned by URL and SHA-256) and exports ONNX in the shape VSNCNN already runs: one +input, one output, float32 NCHW RGB, batch/height/width dynamic, opset 13, +IR 7 — vs-mlrt's own conversions use the same. Two details:

+
    +
  • x2plus starts with pixel_unshuffle(2), which the TorchScript +exporter writes as a 6-D Reshape/Transpose that ncnn's importer can't handle. It is replaced by a +stride-2 convolution with one-hot weights, checked torch.equal to +F.pixel_unshuffle on every run (each output is one input sample times 1.0).
  • +
  • With torch 2.15.0 / onnx 1.23.2, two runs produce byte-identical files (the hashes the +manifests pin).
  • +
+ + + + + + +
ModelONNX bytesopsPyTorch vs ORT, random image (max, levels)smooth image (max)
realesr-general-x4v34,866,424Conv PRelu DepthToSpace Resize Add0.00120.0069
realesr-general-wdn-x4v34,866,424same0.00560.0089
RealESRGAN_x4plus67,051,646Conv LeakyRelu Concat Mul Add Resize0.00060.0171
RealESRGAN_x2plus67,073,184same (unshuffle as Conv)0.00060.0118
+

In 8-bit levels (255 × the 0..1 difference); 96x128 inputs, seeded. Printed by the +conversion script itself.

+ +

2. VSNCNN against ONNX Runtime

+

The same 320x240 frame of small_clip.mp4 through VSNCNN (as the template runs it) +and through ONNX Runtime on the CPU in fp32. fp32 shows the conversion and ncnn's import are exact; +fp16 is what the template uses (as for the bundled models, ~3x faster).

+ + + + + + + + +
Modelfp32 maxfp16 maxfp16 meanfp16 99.9th pct
Real-ESRGAN General0.0012.3490.0460.726
Real-ESRGAN General denoise0.01210.2810.1122.256
Live Action SPAN0.0001.4470.0520.662
Real-ESRGAN x2plus0.0011.9460.1121.111
Real-ESRGAN x4plus0.0055.1420.1221.433
(bundled) Real-ESRGAN anime 4x0.0003.0060.0300.512
+

run.sh parity. Both clipped to 0..1 first, as the template's +conversion back does. The fp16 maxima are single pixels at hard edges; the mean is what a viewer +sees. The bundled anime model is the reference for what VSNCNN's fp16 already does in shipped +builds.

+ +

3. Fidelity, colour and position

+

3a. Restoring a downscaled broadcast frame (PSNR)

+

A 720x576 DVB-T frame (pal-dvbt-fieldcoded-25i.ts, bwdif) is the ground truth; its +360x288 bicubic downscale is upscaled 2x by each method. PSNR rewards +softness: the source is itself an H.264 broadcast, and a model that removes its compression +noise or redraws texture scores lower. Read this as fidelity to the source, not as +"which looks better" — and it is one frame.

+ + + + + + + + + + + + + +
MethodPSNRmean abs (levels)
Spline3630.991.77
Real-ESRGAN General 4x, then Spline36 to 2x (shipped)27.753.74
General 4x, then Lanczos to 2x27.703.75
General 4x, then Bicubic to 2x27.893.70
General 4x, then Bilinear to 2x28.403.57
General denoise 4x, Spline36 to 2x28.313.00
Live Action SPAN 2x32.042.01
Real-ESRGAN x2plus 2x (shipped for 2x)27.223.73
Real-ESRGAN x4plus 4x, Spline36 to 2x28.603.58
(bundled) Real-ESRGAN anime 2x27.424.83
(bundled) Real-CUGAN conservative 2x30.552.52
+

run.sh psnr, 8-pixel border excluded.

+

General at 2x. The network is 4x only. Running it at 4x and resampling down is +what upstream's own --outscale does (with Lanczos). The downscale kernel moves PSNR by +under 0.7 dB, and in the order of their softness (bilinear best), which is PSNR preferring blur +rather than a reason to pick bilinear. The pipeline's own kernel, Spline36, was kept: it sits next +to Lanczos (0.05 dB apart). The resample runs in float RGB, before the conversion back to the +source format ({{#AI_RESAMPLE}}).

+

Live Action SPAN is the only AI method here that beats Spline36 on PSNR: it was +trained to repair compression, blur and halos without denoising or shifting colour, and it +does what it says. Its author notes it "generally cannot handle VHS degradation" — the method +description in the app says so.

+ +

3b. Crops

+

240x200 from the frame (face, glasses, hair), shown 2x with nearest-neighbour +scaling so pixels are real. Click for full size.

+
+
Original 720x576 crop
1. The original frame (the ground truth).
+
Spline36 2x
2. Spline36 from 360x288: soft, as expected.
+
Real-ESRGAN General
3. Real-ESRGAN General (4x, Spline36 to 2x): crisp edges, skin and hair texture redrawn smoother.
+
Real-ESRGAN General denoise
4. General's denoise variant: smoother again — for noisy sources only.
+
Live Action SPAN
5. Live Action SPAN: closest to the original; keeps texture and colour.
+
Real-ESRGAN x2plus
6. Real-ESRGAN x2plus: the most "restored" look, painterly in flat areas.
+
Bundled anime Real-ESRGAN
7. For contrast, the bundled anime Real-ESRGAN: flattens the face into cel shading — why live action needed its own models.
+
+ +

3c. Colour and position on the synthetic pattern

+

The pattern of integration_ai_upscale_test.dart (six mid-saturation patches and a +white bar on grey, 320x240, encoded through the whole pipeline to FFV1) against a Spline36 upscale +of the same frames. Worst channel of the worst patch, and the bar's centre.

+ + + + + + + + + + +
MethodFlat colour drift (levels)Bar centre offset (px, x / y)Test bound
Real-ESRGAN General 2x2.90.10 / 0.034.0
Real-ESRGAN General 4x2.90.29 / 0.094.0
Live Action SPAN 2x0.60.03 / 0.021.5
Live Action SPAN 4x (twice)1.30.11 / 0.062.0
Real-ESRGAN x2plus4.20.13 / 0.226.0
Real-ESRGAN x4plus12.20.27 / 0.1114.0
(bundled) Real-ESRGAN anime 4x11.2—14.0
AI strength 0.5, anime Real-ESRGAN 4x5.8—half ±1
+

From flutter test test/integration_ai_upscale_test.dart (heavy) with +VAPOURBOX_EXPECT_VULKAN=1, which prints every value. Every bar centre is within 0.3 +px of Spline36's (the test allows 0.75 px). General's 2x and 4x patches are identical: its 2x is the +same 4x network, resampled. x4plus's tint is the model's, as with the anime Real-ESRGAN: the +pipeline is shared with CUGAN and AnimeJaNai, which keep flat colour within 2 and 0.5 levels.

+ +

3d. AI strength

+

aiStrength below 1 adds one std.Merge of a Spline36 upscale of the +source frame with the network's result, at the clip's own format, before the frame properties are +copied back. At strength 0.5 the result is within 0.38 levels (mean) of the +per-pixel mid-point of the two, and the anime Real-ESRGAN 4x tint falls from 11.2 to 5.8 levels. +At 1.0 (the default) the block is removed: the script is byte-for-byte the one a job without the +setting produces (test_188).

+ +

4. 2xVHS2HD (RealPLKSR): rejected

+

starinspace/StarinspaceUpscale's +2xVHS2HD-RealPLKSR.onnx (release tag Models, 29,722,757 bytes, SHA-256 +2b7ec70d…c826; CC BY-SA 4.0; upstream names no author other than the account) is +exported with dynamic shapes and in-place slice assignment (RealPLKSR's partial large-kernel +convolution, x[:, :idx] = conv(x[:, :idx])). ncnn's ONNX importer, inside VSNCNN, +rejects it at load:

+
  225 Shape not supported yet!
+   84 Expand not supported yet!
+   56 Where not supported yet!
+   56 Range not supported yet!
+   56 Equal not supported yet!
+   56 ConstantOfShape not supported yet!
+   56 Cast not supported yet!
+   28 ScatterND not supported yet!
+    1 Tile not supported yet!
+    1 layer Shape not exists or registered
+vapoursynth.Error: operator(): load param failed
+

run.sh vhs2hd. Counted import messages; the job fails with +operator(): load param failed.

+

So it was not built, and no add-on was made. A re-export from the published .pth +with concatenation instead of slice assignment and static shape handling might import; that is a +conversion of a CC BY-SA model (the result would be an adaptation under the same licence) and was +not attempted.

+ +

5. Speed and GPU memory

+ + + + + + + + + + +
720x576 source, 10 framesfps, run.sh fps (GPU shared)fps, first run (machine quiet)
(bundled) Real-ESRGAN anime 2x5.078.3
(bundled) Real-CUGAN 2x3.514.8
Real-ESRGAN General 2x (4x, then resample)2.694.0
Real-ESRGAN General 4x2.833.9
Live Action SPAN 2x3.808.9
Live Action SPAN 4x (twice)0.861.5
Real-ESRGAN x2plus 2x0.450.9
Real-ESRGAN x4plus 4x (384 tiles)0.080.15
+

run.sh fps: fp16, num_streams=2, whole frames except +x4plus — the template's settings for an SD source. Network time only (no decode or encode). One +machine: an M1 Pro's GPU through MoltenVK; Windows and Linux GPUs will differ, and lavapipe on the CI +runners is far slower. The committed-script run shared the GPU with other +jobs (another agent's Core ML work); the first column is that run. The second is the same +measurement made earlier the same day, through a scratch copy of these scripts, on a quiet machine — +kept here because it is the better estimate of real speed. Ratios between methods agree.

+

Real-ESRGAN x4plus needs tiles at any size

+ + + + + + + + + + + + + +
Configuration (4 frames)Result
x4plus, whole framesegfault after vkWaitForFences failed -4 (device lost)
x4plus, 512 tiles, 2 streams0.01 fps (632 s for 4 frames: thrashing)
x4plus, 512 tiles, 1 stream0.05 fps
x4plus, 384 tiles, 2 streams0.12 fps (fastest)
x4plus, 384 tiles, 1 stream0.09 fps
x4plus, 256 tiles, 2 / 1 streams0.10 / 0.06 fps
x4plus, 192 / 128 tiles0.07 / 0.07 fps
x2plus, whole frame, 2 / 1 streams0.79 / 0.38 fps
x2plus, 512 / 256 tiles0.32 / 0.27 fps
General on 360x288, 384 tile as given (larger than the frame)segfault
General on 360x288, tile clamped to the frameruns (10.6 fps)
+

run.sh tiles. "tile=0" is the whole frame.

+

Whole-frame x4plus at 720x576 lost the Vulkan device +(vkWaitForFences failed -4, VK_ERROR_DEVICE_LOST, then a segfault) — its +last layers run 64 channels at 2880x2304. 512-pixel tiles ran but thrashed; 384 is fastest. So +x4plus at 4x always runs in 384-pixel tiles (CropResizeParameters::ai_tiling) whenever the frame is above 384x384. x2plus works at a quarter of the area until its last layers and runs a DVD frame whole, +so it keeps the common rule (512 tiles above SD).

+
Found on the way: a tile larger than the frame segfaults +VSNCNN. The first committed run of run.sh psnr passed x4plus's 384 tile to a +360x288 frame and crashed. The template only tiles above an area threshold, but a wide, short picture +passes the area test while being shorter than a tile (800x200 for x4plus; any frame above 720x576 in +area but under 512 lines for every AI method, the bundled ones included). Tiles are now +min(tile, width/height) in both templates, and the heavy suite runs x4plus 4x on an +800x200 source.
+ +

6. Decisions

+
    +
  • Two add-ons, not five. General and SPAN are small (11 MB together) and both +for real-world footage; x4plus/x2plus are 134 MB and slow, so they are their own download.
  • +
  • VSNCNN, not ONNX Runtime. All four architectures import into ncnn and run +exactly, on the same GPU path as the bundled models, so the add-ons need no ai-runtime +and are offered exactly where VSNCNN ships (macOS arm64, Windows x64, Linux x64; asserted).
  • +
  • Factor rules: General runs 4x and resamples for 2x (as upstream); SPAN runs +twice for 4x; x2plus / x4plus by factor (x2plus is 4x faster than x4plus-then-downscale).
  • +
  • The denoise variant is a switch on General, not a method: same network, +different weights, 4.9 MB.
  • +
  • AI strength is one control for every AI method, 0–1 in 0.05 steps.
  • +
  • Not gated to advanced mode. The four-methods-in-simple-mode rule applies to a +schema's methods; the upscaler is an enum parameter, and the schema has no per-option +advanced flag. The x4plus option's label says "slow".
  • +
+ +

7. Not tested here

+
    +
  • Windows (DirectX/Vulkan drivers) and Linux GPUs; the CI nightly runs the heavy suite on lavapipe. +x4plus's tile size was chosen on a 16 GB unified-memory Mac; a discrete GPU with less memory may +want smaller tiles.
  • +
  • Real VHS, film scans and noisy low-light video — the quality comparison is one broadcast frame +and a synthetic pattern.
  • +
  • The hosted files: until the owner uploads them to addon-models-v1, every test ran +through VAPOURBOX_ADDON_MIRROR, which checks the same sizes and hashes.
  • +
+
+ + diff --git a/docs/reports/2026-11-ai-upscale-addons/scripts/metrics.py b/docs/reports/2026-11-ai-upscale-addons/scripts/metrics.py new file mode 100644 index 00000000..60c9dc1b --- /dev/null +++ b/docs/reports/2026-11-ai-upscale-addons/scripts/metrics.py @@ -0,0 +1,47 @@ +"""Two measurements on vspipe's raw RGBS output (planes G, B, R). + + metrics.py ort W H + VSNCNN against ONNX Runtime (CPU, fp32) on the same input: max / mean / + 99.9th-percentile absolute difference in 8-bit levels, after clipping + both to 0..1 as the pipeline's conversion back does. + + metrics.py psnr W H + PSNR and mean absolute difference against a ground-truth frame, 8 px + border excluded. PSNR rewards softness: a model that removes the + source's compression noise or redraws texture scores lower however good + it looks, so this ranks fidelity, not appearance. + +Run with the conversion venv's Python (numpy, onnxruntime). +""" +import sys + +import numpy as np + + +def read_gbr(path, w, h): + return np.fromfile(path, dtype=np.float32)[: 3 * w * h].reshape(3, h, w) + + +def main(): + mode = sys.argv[1] + if mode == 'ort': + import onnxruntime as ort + frame, w, h, models, vsout = sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5], sys.argv[6] + x = read_gbr(frame, w, h)[[2, 0, 1]][None] + for m in models.split(','): + s = ort.InferenceSession(m, providers=['CPUExecutionProvider']) + x = s.run(None, {s.get_inputs()[0].name: x})[0] + ref = x[0] + got = read_gbr(vsout, ref.shape[2], ref.shape[1])[[2, 0, 1]] + d = np.abs(np.clip(ref, 0, 1) - np.clip(got, 0, 1)) * 255 + print(f'max {d.max():.3f} mean {d.mean():.4f} p99.9 {np.percentile(d, 99.9):.3f}') + elif mode == 'psnr': + truth, w, h, out = sys.argv[2], int(sys.argv[3]), int(sys.argv[4]), sys.argv[5] + d = (np.clip(read_gbr(out, w, h), 0, 1) - np.clip(read_gbr(truth, w, h), 0, 1))[:, 8:-8, 8:-8] * 255 + print(f'PSNR {10 * np.log10(255 ** 2 / (d ** 2).mean()):.2f} dB mean abs {np.abs(d).mean():.2f}') + else: + sys.exit(__doc__) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/2026-11-ai-upscale-addons/scripts/run.sh b/docs/reports/2026-11-ai-upscale-addons/scripts/run.sh new file mode 100644 index 00000000..f769639f --- /dev/null +++ b/docs/reports/2026-11-ai-upscale-addons/scripts/run.sh @@ -0,0 +1,161 @@ +#!/bin/bash +# Every number in the report, from the committed fixtures. +# +# run.sh [parity|psnr|fps|tiles|vhs2hd|crops|all] +# +# Environment: +# VAPOURBOX_DEPS_DIR a deps bundle (deps/macos-arm64) with VSNCNN and ffmpeg +# CONVERT_PY the conversion venv's python (numpy, onnxruntime) +# ADDON_FILES a directory holding the add-on models: the four +# upscale-*.onnx from Scripts/models/convert_realesrgan.py +# and 2xLiveActionV1_SPAN_490000.onnx (an add-on mirror, +# or the installed addons// directories' files) +# SCRATCH where frames and outputs go (default: a new temp dir) +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${VAPOURBOX_DEPS_DIR:?set VAPOURBOX_DEPS_DIR to a deps bundle}" +: "${CONVERT_PY:?set CONVERT_PY to the conversion venv python}" +: "${ADDON_FILES:?set ADDON_FILES to the directory holding the add-on ONNX files}" +SCRATCH="${SCRATCH:-$(mktemp -d)}" +mkdir -p "$SCRATCH" +VS="$VAPOURBOX_DEPS_DIR/vapoursynth/vspipe" +FF="$VAPOURBOX_DEPS_DIR/ffmpeg/ffmpeg" +NC="$VAPOURBOX_DEPS_DIR/vapoursynth/ncnn/models" +RES="$REPO/Tests/TestResources" + +# The models by short name (macOS ships bash 3.2: no associative arrays). +model() { + case $1 in + general) echo "$ADDON_FILES/upscale-realesr-general-x4v3.onnx" ;; + wdn) echo "$ADDON_FILES/upscale-realesr-general-wdn-x4v3.onnx" ;; + span) echo "$ADDON_FILES/2xLiveActionV1_SPAN_490000.onnx" ;; + x2plus) echo "$ADDON_FILES/upscale-RealESRGAN_x2plus.onnx" ;; + x4plus) echo "$ADDON_FILES/upscale-RealESRGAN_x4plus.onnx" ;; + anime2) echo "$NC/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx" ;; + anime4) echo "$NC/RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx" ;; + cugan) echo "$NC/cugan/up2x-latest-conservative.onnx" ;; + esac +} + +frames() { + # small: 320x240 frame of the progressive clip (parity input). + # dvb: a 720x576 broadcast frame (bwdif-deinterlaced) as ground truth, and + # its 360x288 bicubic downscale as the input a 2x upscale must restore. + [ -f "$SCRATCH/small.gbrpf" ] || "$FF" -v error -y -ss 20 -i "$RES/small_clip.mp4" \ + -frames:v 1 -vf format=gbrpf32le -f rawvideo "$SCRATCH/small.gbrpf" + [ -f "$SCRATCH/dvb.gbrpf" ] || "$FF" -v error -y -ss 1 -i "$RES/pal-dvbt-fieldcoded-25i.ts" \ + -frames:v 1 -vf bwdif=0,format=gbrpf32le -f rawvideo "$SCRATCH/dvb.gbrpf" + [ -f "$SCRATCH/dvb-half.gbrpf" ] || "$FF" -v error -y -ss 1 -i "$RES/pal-dvbt-fieldcoded-25i.ts" \ + -frames:v 1 -vf bwdif=0,scale=360:288:flags=bicubic,format=gbrpf32le -f rawvideo "$SCRATCH/dvb-half.gbrpf" +} + +# vs : run upscale.vpy, print vspipe's timing line. +vs() { + local out=$1; shift + local args=(-a deps="$VAPOURBOX_DEPS_DIR") + for a in "$@"; do args+=(-a "$a"); done + "$VS" "${args[@]}" "$HERE/upscale.vpy" "$out" 2>"$SCRATCH/vspipe.log" || { + grep -v "VK_\|mvk-info" "$SCRATCH/vspipe.log" | tail -5; return 1; } + grep -E "^Output" "$SCRATCH/vspipe.log" || true +} + +parity() { + echo "== VSNCNN vs ONNX Runtime (CPU fp32), 320x240 frame, 8-bit levels" + for k in general wdn span x2plus x4plus anime4; do + for fp16 in 0 1; do + vs "$SCRATCH/p.raw" infile="$SCRATCH/small.gbrpf" w=320 h=240 model="$(model $k)" fp16=$fp16 >/dev/null + printf '%-8s fp16=%s ' "$k" "$fp16" + "$CONVERT_PY" -I "$HERE/metrics.py" ort "$SCRATCH/small.gbrpf" 320 240 "$(model $k)" "$SCRATCH/p.raw" + done + done +} + +psnr() { + echo "== 360x288 -> 720x576 against the original frame (PSNR rewards softness)" + one() { local label=$1; shift + vs "$SCRATCH/q.raw" infile="$SCRATCH/dvb-half.gbrpf" w=360 h=288 tw=720 th=576 "$@" >/dev/null + printf '%-34s ' "$label" + "$CONVERT_PY" -I "$HERE/metrics.py" psnr "$SCRATCH/dvb.gbrpf" 720 576 "$SCRATCH/q.raw"; } + one "Spline36" model= + for k in Spline36 Lanczos Bicubic Bilinear; do one "General 4x, then $k to 2x" model="$(model general)" down=$k; done + one "General denoise 4x, Spline36 to 2x" model="$(model wdn)" + one "Live Action SPAN 2x" model="$(model span)" + one "x2plus 2x" model="$(model x2plus)" + one "x4plus 4x, Spline36 to 2x" model="$(model x4plus)" + one "(bundled) Real-ESRGAN anime 2x" model="$(model anime2)" + one "(bundled) Real-CUGAN conservative 2x" model="$(model cugan)" +} + +fps() { + echo "== 720x576 -> 2x / 4x, 10 frames, fp16, num_streams=2 (the template's settings)" + one() { local label=$1; shift; printf '%-28s ' "$label" + vs /dev/null infile="$SCRATCH/dvb.gbrpf" w=720 h=576 n=10 "$@"; } + one "(bundled) anime 2x" model="$(model anime2)" + one "(bundled) CUGAN 2x" model="$(model cugan)" + one "General 2x (4x + resample)" model="$(model general)" tw=1440 th=1152 + one "General 4x" model="$(model general)" + one "SPAN 2x" model="$(model span)" + one "SPAN 4x (twice)" model="$(model span),$(model span)" + one "x2plus 2x" model="$(model x2plus)" + one "x4plus 4x (384 tiles)" model="$(model x4plus)" tile=384 +} + +tiles() { + echo "== x4plus at 720x576: tile size and streams (4 frames each)" + for t in "0 2" "512 2" "512 1" "384 2" "384 1" "256 2" "256 1" "192 2" "128 2"; do + set -- $t + printf 'tile=%-4s streams=%s ' "$1" "$2" + vs /dev/null infile="$SCRATCH/dvb.gbrpf" w=720 h=576 n=4 model="$(model x4plus)" tile=$1 streams=$2 \ + || echo "(failed)" + done + echo "== x2plus at 720x576" + for t in "0 2" "0 1" "512 2" "256 2"; do + set -- $t + printf 'tile=%-4s streams=%s ' "$1" "$2" + vs /dev/null infile="$SCRATCH/dvb.gbrpf" w=720 h=576 n=4 model="$(model x2plus)" tile=$1 streams=$2 \ + || echo "(failed)" + done + echo "== a tile larger than the frame (General on 360x288)" + printf 'tile=384 as given ' + vs /dev/null infile="$SCRATCH/dvb-half.gbrpf" w=360 h=288 model="$(model general)" tile=384 clamp=0 \ + || echo "(failed)" + printf 'tile=384 clamped ' + vs /dev/null infile="$SCRATCH/dvb-half.gbrpf" w=360 h=288 model="$(model general)" tile=384 \ + || echo "(failed)" +} + +vhs2hd() { + echo "== 2xVHS2HD-RealPLKSR through VSNCNN" + local f="$SCRATCH/2xVHS2HD-RealPLKSR.onnx" + [ -f "$f" ] || curl -sSfL -o "$f" \ + https://github.com/starinspace/StarinspaceUpscale/releases/download/Models/2xVHS2HD-RealPLKSR.onnx + shasum -a 256 "$f" + vs /dev/null infile="$SCRATCH/small.gbrpf" w=320 h=240 model="$f" || true + echo "-- ncnn's import messages, counted:" + grep -E "not supported|not exists|load param" "$SCRATCH/vspipe.log" | sort | uniq -c | sort -rn +} + +crops() { + # Zoomed, nearest-neighbour crops of the 2x restorations for the report. + local out="$HERE/../img" + crop() { local name=$1; shift + vs "$SCRATCH/c.raw" infile="$SCRATCH/dvb-half.gbrpf" w=360 h=288 tw=720 th=576 "$@" >/dev/null + "$FF" -v error -y -f rawvideo -pix_fmt gbrpf32le -s 720x576 -i "$SCRATCH/c.raw" \ + -vf "crop=240:200:360:180,scale=480:400:flags=neighbor,format=rgb24" -frames:v 1 "$out/$name.png"; } + "$FF" -v error -y -f rawvideo -pix_fmt gbrpf32le -s 720x576 -i "$SCRATCH/dvb.gbrpf" \ + -vf "crop=240:200:360:180,scale=480:400:flags=neighbor,format=rgb24" -frames:v 1 "$out/01-original.png" + crop 02-spline36 model= + crop 03-general model="$(model general)" + crop 04-general-denoise model="$(model wdn)" + crop 05-live-action-span model="$(model span)" + crop 06-x2plus model="$(model x2plus)" + crop 07-anime-esrgan model="$(model anime2)" +} + +frames +case "${1:-all}" in + parity) parity ;; psnr) psnr ;; fps) fps ;; tiles) tiles ;; vhs2hd) vhs2hd ;; crops) crops ;; + all) parity; psnr; fps; tiles; vhs2hd; crops ;; + *) echo "unknown section $1"; exit 1 ;; +esac diff --git a/docs/reports/2026-11-ai-upscale-addons/scripts/upscale.vpy b/docs/reports/2026-11-ai-upscale-addons/scripts/upscale.vpy new file mode 100644 index 00000000..551ddd4e --- /dev/null +++ b/docs/reports/2026-11-ai-upscale-addons/scripts/upscale.vpy @@ -0,0 +1,62 @@ +# Run ONNX models through VSNCNN (ncnn.Model) the way the worker's +# {{#UPSCALE_AI}} block does, on one planar-float frame read from a file, and +# output RGBS (vspipe writes RGB planes in G, B, R order). +# +# vspipe -a deps= -a infile= -a w=W -a h=H +# -a model= [-a fp16=0|1] [-a tile=N] [-a streams=N] +# [-a n=frames] [-a tw=W -a th=H -a down=Spline36] upscale.vpy out.raw +# +# `infile` is ffmpeg's gbrpf32le rawvideo (planes G, B, R). An empty `model` +# runs no network. tw/th resample the result (the Real-ESRGAN General 2x path). +import ctypes +import os +import vapoursynth as vs + +core = vs.core +g = globals() +D = os.path.join(g['deps'], 'vapoursynth') +molten = os.path.join(D, 'moltenvk', 'libMoltenVK.dylib') +if os.path.exists(molten): # macOS: VSNCNN resolves Vulkan from MoltenVK + ctypes.CDLL(molten, mode=ctypes.RTLD_GLOBAL) +for name in ('libvsncnn.dylib', 'libvsncnn.so', 'vsncnn.dll'): + path = os.path.join(D, 'ncnn', name) + if os.path.exists(path): + core.std.LoadPlugin(path) + break + +w, h = int(g['w']), int(g['h']) +n = int(g.get('n', 1)) +with open(g['infile'], 'rb') as fh: + data = fh.read() +plane = w * h * 4 +blank = core.std.BlankClip(width=w, height=h, format=vs.RGBS, length=n, fpsnum=25, fpsden=1) + + +def fill(n, f): + f = f.copy() + for p, src in ((0, 2), (1, 0), (2, 1)): # R <- file plane 2, G <- 0, B <- 1 + mv = memoryview(f[p]).cast('B') + stride = f.get_stride(p) + off = src * plane + for y in range(h): + mv[y * stride:y * stride + w * 4] = data[off + y * w * 4: off + (y + 1) * w * 4] + return f + + +clip = core.std.ModifyFrame(blank, blank, fill) +fp16 = g.get('fp16', '1') == '1' +tile = int(g.get('tile', '0')) +# clamp=1 (default) keeps a tile within the frame, as the template does; +# clamp=0 passes it as given (a tile larger than the frame segfaults VSNCNN). +clamp = g.get('clamp', '1') == '1' +for m in [m for m in g.get('model', '').split(',') if m]: + kw = {} + if tile: + size = [min(tile, clip.width), min(tile, clip.height)] if clamp else [tile, tile] + kw = dict(tilesize=size, overlap=[16, 16]) + clip = core.ncnn.Model(clip, network_path=m, fp16=fp16, + num_streams=int(g.get('streams', '2')), **kw) +tw, th = int(g.get('tw', clip.width)), int(g.get('th', clip.height)) +if (tw, th) != (clip.width, clip.height): + clip = getattr(core.resize, g.get('down', 'Spline36'))(clip, tw, th) +clip.set_output() diff --git a/docs/reports/2026-11-audio-restoration/index.html b/docs/reports/2026-11-audio-restoration/index.html new file mode 100644 index 00000000..ec41597e --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/index.html @@ -0,0 +1,214 @@ + + + + + +Audio Restoration Probe + + + +
+

Audio restoration: where the sound moves, and what each step does

+

2026-11-08 · ffmpeg 9.0.2 (deps bundle, macOS arm64) · deep-filter 0.5.6 · Apple Silicon · +signals generated with ffmpeg's lavfi sources · scripts in scripts/ (run.sh reproduces every number)

+ +
+

Question. VapourBox copied or re-encoded audio and never processed it. The Audio Restoration pass adds channel fixes, a sync offset, drift correction, declip/declick, rumble and hum removal, broadband denoise (afftdn), DeepFilterNet speech enhancement, loudness normalisation and a limiter. Before building it on ffmpeg's filters: which of them move the sound in time, which defaults are wrong for this use, and does each actually do its job on a known signal?

+

Outcome: built. Four filters move or mis-handle the audio with their defaults; each is handled in the one place every filter is chosen (AudioPlan::new, worker/src/audio_restoration.rs):

+
    +
  • afftdn outputs its input 25 ms late at every rate and strength → trimmed with atrim=start_sample; its default noise floor (nf=-50) barely touched real hiss → the source's measured floor is passed; its noise tracking (tn=1) undid that → off.
  • +
  • atempo moved every click ~20 ms early on top of the stretch → drift is corrected by an exact resample instead.
  • +
  • alimiter delays by its attack and normalises back up to 0 dBFS by default → latency=1:level=0.
  • +
  • Rendering to a WAV loses a late-starting audio stream's gap → aresample=first_pts=0 first.
  • +
+

DeepFilterNet (an add-on) delays by 30 ms; its own -D compensates. The architecture decision — render the soundtrack once, before the video encode, rather than filtering in the encode — is what DeepFilterNet, two-pass loudness, negative offsets and subtitle alignment all need.

+
+ +

1. Which steps move the sound in time

+

A click track (3 ms bursts of 3 kHz at irregular times, plus a 440 Hz tone and pink noise) is filtered and cross-correlated with itself (delay_probe.py). Positive lag = output late.

+
+ + + + + + + + + +
Lag in samples
Filter48 kHz44.1 kHz32 kHz
afftdn=nr=6 / 12 / 24:nf=-4012001102800
alimiter=limit=0.5 (defaults)239219159
alimiter=limit=0.5:level=0:latency=1000
adeclick, adeclip000
highpass=f=40, bandreject ×2000
loudnorm (+ aresample back from 192 kHz)000
drift resample (aresample 4× → asetrate → aresample)000
+

afftdn's delay is exactly 25 ms (rate / 40 samples) whatever its strength — Video-Enhancer's 1200 samples at 48 kHz is the same number. The plan trims round(rate × 0.025) samples at the source's own rate (1103 at 44.1 kHz; the measured 1102 is the correlation peak's integer position, within a sample).

+

alimiter's default also normalises: a click track peaking at −1.34 dBFS, limited to −6 dB, came out peaking at 0.00 dBFS with defaults and −6.02 dBFS with level=0:latency=1.

+ +

DeepFilterNet

+
+ + + + + + +
deep-filter 0.5.6 on a 12 s click track
InputFlagsOutput samples (48 kHz)Lag (samples at 48 kHz)
48 kHz, 576000 samples—5760001440
48 kHz-D5745600
44.1 kHz—5760001439
44.1 kHz-D574560−1
+

It accepts 44.1 kHz and always writes 48 kHz. -D removes its 30 ms STFT + lookahead delay and leaves the track that much short; the plan pads every track to the picture's exact sample count at the end (apad=whole_len + atrim=end_sample), so that is harmless.

+ +

atempo stretches and shifts

+

tempo_denoise_probe.py: each click's position after the stretch against the expected t / factor.

+
+ + + + + + +
Click position error
ChainMeanFirst clickLast click
atempo=1.004−19.13 ms0.00 ms−21.18 ms
atempo=0.996−19.38 ms0.00 ms−21.42 ms
atempo=1.02−19.04 ms0.00 ms−19.21 ms
asetrate + aresample (×1.004, ×0.996, ×1.02)0.01 ms0.00 ms≤0.01 ms
+

The brief said atempo; the measurement says no. Drift between a capture's audio and its picture is a sample-clock error, so resampling is the exact inverse — it restores the original pitch along with the length (at most 0.5% by default, about 9 cents). asetrate only takes an integer rate, which straight from 48 kHz would be off by up to 1e-5 (37 ms over an hour), so the plan upsamples to the rate between 4× and 8× where round(r1 × factor) / r1 is closest to the factor — within ~1e-11 (resample_ratio, unit-tested to under 0.4 ms an hour). The heavy test measures the encoded result: a 20 s picture with 20.08 s of audio puts every click within 2 ms of t / 1.004.

+ +

2. afftdn's noise floor

+

White noise at −38.8 dBFS RMS with clicks; the noise level measured in a click-free stretch after afftdn has adapted (tempo_denoise_probe.py).

+
+ + + + + + +
Noise floor after afftdn, nr=20
ChainFloorDown by
afftdn=nr=20:nf=-50:tn=1 (first build)−39.6 dB0.7 dB
afftdn=nr=20:nf=-50 (default floor)−39.9 dB1.1 dB
afftdn=nr=20:nf=-39 (measured floor)−52.4 dB13.6 dB
afftdn=nr=20:nf=-39:tn=1−39.6 dB0.7 dB
+

Correction, kept visible: the first build passed nf=-50:tn=1, reasoning that noise tracking would find the floor itself. The first heavy-test run showed 0.7 dB of reduction. Tracking does not rescue a wrong floor here — it defeats a right one.

+

The floor afftdn is given is the source's: the RMS of the quietest tenth of 50 ms windows in a minute from the middle of the window, decoded at full bandwidth (white hiss is 8 dB quieter below 4 kHz than across 24, so an 8 kHz analysis would underestimate it), clamped to −80…−30 dB. The upper clamp guards against content with no pauses (continuous music), whose "floor" is the content itself.

+
+ + + + 0 + 5 + 10 + 15 + + + + + + + 5.5 + 9.8 + 13.5 + 15.2 + nr 6 + nr 12 + nr 20 + nr 30 + dB down in a pause + +
afftdn with the measured floor (−38.8 dB) on a gated 440 Hz tone over white noise (quality_probe.py): noise removed in the pauses at each strength. The tone's level in the bursts changed by less than 0.01 dB at every strength.
+
+ +

3. The other steps on known signals

+
+ + + + + + +
Hum notches, bandreject Q 8 at f, 2f, 3f (1 kHz tone + 50/100/150 Hz)
FrequencyDown by
50 Hz47.1 dB
100 Hz60.7 dB
150 Hz70.5 dB
1 kHz0.0 dB
+

Automatic hum detection reads the same minute as the noise floor: Goertzel power at 50/100/150 and 60/120/180 Hz, each in 2 s blocks (0.5 Hz resolution, so mains wandering by a tenth of a hertz stays in the bin), against six neighbours 4–7 Hz away. A line 12 dB above its neighbours counts; whichever mains frequency stands out more wins; nothing standing out means nothing is notched (and the job log says so). Unit tests cover 50.1 Hz hum with harmonics under noise and a tone, 60 Hz hum whose third harmonic dominates, and clean audio.

+ +
+ + + + +
Loudness, two passes (print_format=json, then linear=true with the measurement), 20 s syllable-modulated 200 Hz signal at −34.4 LUFS
TargetResult (ebur128)Error
−23 LUFS−23.0 LUFS0.0 LU
−16 LUFS−16.0 LUFS0.0 LU
+ +
+ + + + + +
DeepFilterNet (-D) on a voiced, syllable-modulated harmonic signal in pink noise; input: pauses −44.8 dB, voiced −24.0 dB, SNR in the voiced bursts 20.9 dB
Attenuation limit (-a)Pauses down byVoiced levelSNR in voicedLagTime per minute (mono)
100 (no limit)25.3 dB−1.1 dB13.8 dB03.8 s
3022.5 dB−1.1 dB14.0 dB04.0 s
1513.7 dB−0.9 dB15.4 dB03.9 s
+

What this metric can't tell you. The signal is a test tone with a speech-like envelope, not speech. DeepFilterNet is trained on speech; it removed the noise in the pauses as expected, but it distorted the synthetic "voice" (SNR inside the bursts fell from 20.9 to ~14 dB), which says little about real dialogue. Judge it by ear on real recordings; the default (no limit) may be stronger than a natural-sounding result wants, which is why the limit is exposed.

+ +
+ + + + + + + + +
Speed: seconds to process one minute of 48 kHz stereo (pink noise), Apple Silicon
StepSeconds
adeclip2.94
loudnorm (one pass)2.08
adeclick1.18
afftdn0.29
drift resample0.29
hum notches0.10
+

Everything is far faster than the video it accompanies; a two-hour tape with every step on adds a few minutes.

+ +

4. A late-starting audio stream

+

start_offset_probe.py: a file whose audio starts 0.5 s after its video. Mapping the source's stream straight into the output (as the encoder always did) keeps that gap through the container's timestamps; a WAV has no timestamps.

+
+ + + + +
First sound and length of the rendered WAV
RenderNo seekSeek to 2 s
anull0.000 s, 9.5 s long0.000 s, 8.0 s
aresample=first_pts=00.500 s, 10.0 s long0.000 s, 8.0 s
+

So aresample=first_pts=0 is the first filter of every render. The heavy test encodes such a file and finds every click 0.5 s later than in the stream itself.

+ +

5. Decisions

+
    +
  • Render the soundtrack before the encode (one WAV, 32-bit float so adeclip's rebuilt peaks and later gain don't clip), and map it into the encode — never an -af on the encode. DeepFilterNet is a separate program; two-pass loudness must measure the whole track first; a negative offset needs audio from before the trim point; and the subtitles transcribe this file, which is the output's audio sample for sample, so their cues follow offset, drift and trim by construction. (Heavy test: a 2 s offset moved Whisper's cues by a median of 2.3 s — its timestamps are coarse.)
  • +
  • One decision site, fixed order, one -af per ffmpeg run — the encoder's -vf rule, for the same reason. Order: start padding → channels → declip → declick → rumble → hum → afftdn (+ its trim) → drift resample → lead silence → [DeepFilterNet] → loudness → limiter → length. Asserted for all 512 combinations of the toggles, with no, positive and negative offsets.
  • +
  • Drift factor = whole source audio length / whole encoded picture length, the picture's from output frames over output rate (FrameMap), applied only above 0.05 s and up to a cap (0.5% default) — a bigger mismatch is a missing stretch or the wrong track, not drift.
  • +
  • Offset and drift compose on the source timeline: source time = (in-point + t − offset) × factor. A positive offset from the start becomes silence; a negative one reads later — never adelay alone.
  • +
  • The track is cut or padded to the picture's exact length.
  • +
  • One stream restored, first in the output; the others copied untouched with their own tags (naming one stream's -map_metadata stopped ffmpeg copying the rest's — found by the heavy test). With audio set to copy, the restored stream is encoded (FLAC/MKV, AAC/MP4·MOV, PCM/AVI).
  • +
  • DeepFilterNet as a manifest add-on, upstream's own v0.5.6 binaries pinned by hash (MIT OR Apache-2.0). macOS builds need macOS 13 (their minos); the arm64 build is linker-signed, the x64 build unsigned — the installer signs a program that fails codesign --verify.
  • +
  • No preview: the preview is picture only, and the panel says so.
  • +
+ +

6. Not tested

+
    +
  • Windows and Linux (the filters are ffmpeg's, the same 9.0 series everywhere; DeepFilterNet's Windows, Linux musl and Linux aarch64 binaries never ran here), and the macOS x64 DeepFilterNet build (needs macOS 13 on Intel).
  • +
  • Real recordings: every number here is from generated signals. Hum detection on real ground-loop hum with a strong bass line, the noise-floor estimate on content without pauses, and DeepFilterNet on real dialogue all deserve a listen.
  • +
  • Very long sources: drift accuracy was measured over 20 s; the resample ratio's accuracy is computed (≤0.4 ms an hour), not measured over hours.
  • +
  • Sources with more than two channels: the channel fix applies to stereo only and says so; the other steps run on every channel.
  • +
+
+ + diff --git a/docs/reports/2026-11-audio-restoration/scripts/common.py b/docs/reports/2026-11-audio-restoration/scripts/common.py new file mode 100644 index 00000000..3ed6598d --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/scripts/common.py @@ -0,0 +1,62 @@ +"""Shared helpers for the audio restoration probes. + +Environment: + FFMPEG path to the deps bundle's ffmpeg (default: ffmpeg on PATH) + DEEP_FILTER path to a deep-filter v0.5.6 binary (optional; DFN probes skip) + WORK scratch directory for generated fixtures (required) +""" +import os +import subprocess +import sys + +import numpy as np + +FFMPEG = os.environ.get("FFMPEG", "ffmpeg") +DEEP_FILTER = os.environ.get("DEEP_FILTER") +WORK = os.environ.get("WORK") +if not WORK: + sys.exit("set WORK to a scratch directory") +os.makedirs(WORK, exist_ok=True) + + +def ff(*args): + """Run ffmpeg quietly; return stderr text.""" + r = subprocess.run([FFMPEG, "-hide_banner", "-nostdin", "-y", *args], + capture_output=True, text=True) + if r.returncode != 0: + raise RuntimeError(r.stderr[-2000:]) + return r.stderr + + +def decode(path, rate, channels=1, af=None): + """Decode to float32 numpy array (frames, channels).""" + cmd = [FFMPEG, "-hide_banner", "-nostdin", "-v", "error", "-i", path] + if af: + cmd += ["-af", af] + cmd += ["-ac", str(channels), "-ar", str(rate), "-f", "f32le", "-"] + raw = subprocess.run(cmd, capture_output=True, check=True).stdout + return np.frombuffer(raw, dtype=np.float32).reshape(-1, channels) + + +def lag_samples(ref, sig, max_lag): + """Lag (samples) of sig relative to ref by cross-correlation; + = sig late.""" + n = min(len(ref), len(sig)) + a = ref[:n] - ref[:n].mean() + b = sig[:n] - sig[:n].mean() + size = 1 << int(np.ceil(np.log2(2 * n))) + c = np.fft.irfft(np.fft.rfft(b, size) * np.conj(np.fft.rfft(a, size)), size) + lags = np.concatenate([c[: max_lag + 1], c[-max_lag:]]) + idx = np.concatenate([np.arange(0, max_lag + 1), np.arange(-max_lag, 0)]) + return int(idx[np.argmax(lags)]) + + +def band_db(x, rate, f, width=2.0): + """Energy (dB) in a narrow band around f, from one long FFT.""" + spec = np.abs(np.fft.rfft(x * np.hanning(len(x)))) ** 2 + freqs = np.fft.rfftfreq(len(x), 1.0 / rate) + sel = (freqs > f - width) & (freqs < f + width) + return 10 * np.log10(spec[sel].sum() + 1e-30) + + +def rms_db(x): + return 20 * np.log10(np.sqrt(np.mean(np.square(x))) + 1e-12) diff --git a/docs/reports/2026-11-audio-restoration/scripts/delay_probe.py b/docs/reports/2026-11-audio-restoration/scripts/delay_probe.py new file mode 100644 index 00000000..d451e32f --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/scripts/delay_probe.py @@ -0,0 +1,78 @@ +"""How far does each processing step move the audio in time? + +Measures, by cross-correlating a click track before and after: + - afftdn at several sample rates and strengths (Video-Enhancer reported 1200 + samples at 48 kHz); + - DeepFilterNet's deep-filter, with and without -D (--compensate-delay); + - atempo, adeclick, adeclip, highpass/bandreject, loudnorm (expected 0). + +Run: WORK=/scratch FFMPEG=.../ffmpeg [DEEP_FILTER=.../deep-filter] python delay_probe.py +""" +import os +import shutil +import subprocess + +from common import WORK, DEEP_FILTER, ff, decode, lag_samples + +CLICKS = ("aevalsrc='if(lt(mod(t\\,0.37)\\,0.002)\\,0.8*sin(2*PI*2000*t)\\,0)" + "+0.05*sin(2*PI*440*t)':s={rate}:d=12") + + +def fixture(rate): + path = os.path.join(WORK, f"clicks_{rate}.wav") + ff("-f", "lavfi", "-i", CLICKS.format(rate=rate) + ",aformat=channel_layouts=mono", + "-f", "lavfi", "-i", f"anoisesrc=c=pink:a=0.02:r={rate}:d=12", + "-filter_complex", "[0][1]amix=inputs=2:normalize=0", "-c:a", "pcm_f32le", path) + return path + + +def measure(src, rate, af): + ref = decode(src, rate)[:, 0] + out = decode(src, rate, af=af)[:, 0] + return lag_samples(ref, out, rate // 2) + + +def peak(src, rate, af): + import numpy as np + return 20 * np.log10(np.abs(decode(src, rate, af=af)[:, 0]).max()) + + +def main(): + src48 = fixture(48000) + print("== alimiter level: peak dBFS of a 0.85-peak click track, limit 0.5 (-6 dB) ==") + for af in ("anull", "alimiter=limit=0.5", "alimiter=limit=0.5:level=0:latency=1"): + print(f" {af:40s} peak {peak(src48, 48000, af):6.2f} dBFS") + print("== ffmpeg filters (lag in samples, + = output late) ==") + for rate in (48000, 44100, 32000): + src = fixture(rate) + for af in ["afftdn=nr=12", "afftdn=nr=24:nf=-40", "afftdn=nr=6", + "adeclick", "adeclip", "highpass=f=40", + "bandreject=f=50:width_type=q:w=4,bandreject=f=100:width_type=q:w=4", + "loudnorm=I=-16:TP=-1.5:LRA=11,aresample=%d" % rate, + "alimiter=limit=0.5", "alimiter=limit=0.5:level=0:latency=1", + "aresample=%d,asetrate=%d,aresample=%d" % (rate * 4, rate * 4, rate)]: + print(f"{rate:6d} Hz {af:60s} lag={measure(src, rate, af):6d}") + + if not DEEP_FILTER: + print("DEEP_FILTER not set; skipping DeepFilterNet") + return + print("== deep-filter ==") + for rate in (48000, 44100): + src = fixture(rate) + for flags in ([], ["-D"]): + out_dir = os.path.join(WORK, "dfn_out") + shutil.rmtree(out_dir, ignore_errors=True) + r = subprocess.run([DEEP_FILTER, *flags, "-o", out_dir, src], + capture_output=True, text=True) + if r.returncode != 0: + print(f"{rate} Hz {flags}: FAILED {r.stderr.strip()[-300:]}") + continue + out = os.path.join(out_dir, os.path.basename(src)) + ref = decode(src, 48000)[:, 0] + got = decode(out, 48000)[:, 0] + print(f"{rate:6d} Hz in, flags={flags}: out frames={len(got)} " + f"(in {len(ref)}), lag={lag_samples(ref, got, 24000)} samples @48k") + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-audio-restoration/scripts/quality_probe.py b/docs/reports/2026-11-audio-restoration/scripts/quality_probe.py new file mode 100644 index 00000000..845fef2f --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/scripts/quality_probe.py @@ -0,0 +1,132 @@ +"""What each step does to a known signal, and how fast it runs. + +Uses the filter strings the worker generates (see worker/src/audio_restoration.rs). + + - hum notches: attenuation at 50/100/150 Hz, change at 1 kHz; + - afftdn: noise floor drop at nr 6/12/20/30 with the measured floor; + - loudnorm (two passes, linear) + aresample: integrated loudness error; + - DeepFilterNet: noise drop in speech pauses, change in the voiced bursts, + delay with -D, at attenuation limits 100/30/15; + - speed: seconds per minute of 48 kHz stereo for each step. + +Run: WORK=/scratch FFMPEG=.../ffmpeg [DEEP_FILTER=.../deep-filter] python quality_probe.py +""" +import json +import os +import re +import shutil +import subprocess +import time + +import numpy as np + +from common import WORK, FFMPEG, DEEP_FILTER, ff, decode, band_db, rms_db, lag_samples + +RATE = 48000 + + +def make(name, graph, secs): + path = os.path.join(WORK, f"q_{name}.wav") + ff("-filter_complex", graph, "-map", "[a]", "-t", str(secs), "-c:a", "pcm_f32le", path) + return path + + +def timed(args): + t = time.time() + ff(*args) + return time.time() - t + + +def main(): + # --- hum + hum = make("hum", "sine=f=1000:r=48000:d=10[t];sine=f=50:r=48000:d=10[a1];" + "sine=f=100:r=48000:d=10[a2];sine=f=150:r=48000:d=10[a3];" + "[t][a1][a2][a3]amix=inputs=4:weights='1 0.8 0.4 0.3':normalize=0[a]", 10) + notch = ",".join(f"bandreject=f={f}:width_type=q:w=8" for f in (50, 100, 150)) + x = decode(hum, RATE)[2 * RATE:8 * RATE, 0] + y = decode(hum, RATE, af=notch)[2 * RATE:8 * RATE, 0] + print("== hum notches (Q 8) ==") + for f in (50, 100, 150, 1000): + print(f" {f:5d} Hz: {band_db(x, RATE, f) - band_db(y, RATE, f):6.1f} dB down") + + # --- afftdn + noisy = make("noise", "sine=f=440:r=48000:d=10,volume=0.3," + "aeval='val(0)*lt(mod(t\\,2)\\,1)'[t];" + "anoisesrc=c=white:a=0.02:r=48000:d=10:seed=7[n];[t][n]amix=inputs=2:normalize=0[a]", 10) + x = decode(noisy, RATE)[:, 0] + frames = x[: len(x) // 2400 * 2400].reshape(-1, 2400) + floor = np.percentile(20 * np.log10(np.sqrt((frames ** 2).mean(axis=1)) + 1e-12), 10) + gap = slice(int(5.2 * RATE), int(5.9 * RATE)) # a pause, after adaptation + tone = slice(int(6.2 * RATE), int(6.9 * RATE)) + print(f"== afftdn (measured floor {floor:.1f} dB) ==") + for nr in (6, 12, 20, 30): + y = decode(noisy, RATE, af=f"afftdn=nr={nr}:nf={floor:.0f}")[:, 0] + y = y[1200:] # its 25 ms delay + print(f" nr={nr:2d}: pause {rms_db(x[gap]) - rms_db(y[gap]):5.1f} dB down, " + f"tone {band_db(y[tone], RATE, 440) - band_db(x[tone], RATE, 440):+5.2f} dB") + + # --- loudnorm + print("== loudnorm, two passes, linear ==") + quiet = make("quiet", "aevalsrc='0.05*sin(2*PI*200*t)*(0.6+0.4*sin(2*PI*4*t))" + "*lt(mod(t\\,1.5)\\,1.0)':s=48000:d=20[a]", 20) + for target in (-23, -16): + r = subprocess.run([FFMPEG, "-hide_banner", "-nostats", "-i", quiet, "-af", + f"loudnorm=I={target}:TP=-1.5:LRA=11:print_format=json", "-f", "null", "-"], + capture_output=True, text=True).stderr + m = json.loads(r[r.rfind("{"):r.rfind("}") + 1]) + out = os.path.join(WORK, "q_ln.wav") + ff("-i", quiet, "-af", + f"loudnorm=I={target}:TP=-1.5:LRA=11:measured_I={m['input_i']}:measured_TP={m['input_tp']}:" + f"measured_LRA={m['input_lra']}:measured_thresh={m['input_thresh']}:offset={m['target_offset']}:" + f"linear=true,aresample=48000", "-c:a", "pcm_f32le", out) + r = subprocess.run([FFMPEG, "-hide_banner", "-nostats", "-i", out, "-af", "ebur128", "-f", "null", "-"], + capture_output=True, text=True).stderr + got = float(re.findall(r"I:\s+(-?[\d.]+) LUFS", r)[-1]) + print(f" target {target}: input {m['input_i']} LUFS -> {got} LUFS (error {got - target:+.1f} LU)") + + # --- speed (60 s stereo) + print("== speed: seconds per minute of 48 kHz stereo ==") + minute = make("minute", "anoisesrc=c=pink:a=0.1:r=48000:d=60,pan=stereo|c0=c0|c1=c0[a]", 60) + for label, af in [("adeclick", "adeclick"), ("adeclip", "adeclip"), + ("afftdn", "afftdn=nr=12:nf=-40"), ("hum notches", notch), + ("drift resample", "aresample=192000,asetrate=192768,aresample=48000"), + ("loudnorm (one pass)", "loudnorm=I=-16:TP=-1.5:LRA=11,aresample=48000")]: + secs = timed(["-i", minute, "-af", af, "-f", "null", "-"]) + print(f" {label:22s} {secs:6.2f} s") + + if not DEEP_FILTER: + print("DEEP_FILTER not set; skipping DeepFilterNet") + return + # --- DeepFilterNet + speech = make("speech", "aevalsrc='(0.12*sin(2*PI*140*t)+0.08*sin(2*PI*280*t)+0.05*sin(2*PI*420*t)" + "+0.03*sin(2*PI*840*t))*(0.5+0.5*sin(2*PI*5*t))*lt(mod(t\\,1.6)\\,0.9)':s=48000:d=16[v];" + "anoisesrc=c=pink:a=0.03:r=48000:d=16:seed=3[n];[v][n]amix=inputs=2:normalize=0[a]", 16) + clean = make("speech_clean", "aevalsrc='(0.12*sin(2*PI*140*t)+0.08*sin(2*PI*280*t)+0.05*sin(2*PI*420*t)" + "+0.03*sin(2*PI*840*t))*(0.5+0.5*sin(2*PI*5*t))*lt(mod(t\\,1.6)\\,0.9)':s=48000:d=16[a]", 16) + x = decode(speech, RATE)[:, 0] + c = decode(clean, RATE)[:, 0] + pauses = np.concatenate([np.arange(int((k * 1.6 + 0.95) * RATE), int((k * 1.6 + 1.55) * RATE)) + for k in range(1, 9)]) + voiced = np.concatenate([np.arange(int((k * 1.6 + 0.1) * RATE), int((k * 1.6 + 0.8) * RATE)) + for k in range(1, 9)]) + print("== DeepFilterNet (deep-filter 0.5.6, -D) ==") + print(f" input: pause {rms_db(x[pauses]):.1f} dB, voiced {rms_db(x[voiced]):.1f} dB, " + f"SNR in voiced {rms_db(c[voiced]) - rms_db((x - c)[voiced]):.1f} dB") + for atten in (100, 30, 15): + out_dir = os.path.join(WORK, "q_dfn") + shutil.rmtree(out_dir, ignore_errors=True) + t = time.time() + subprocess.run([DEEP_FILTER, "-D", "-a", str(atten), "-o", out_dir, speech], + check=True, capture_output=True) + secs = time.time() - t + y = decode(os.path.join(out_dir, os.path.basename(speech)), RATE)[:, 0] + y = np.pad(y, (0, len(x) - len(y))) + err = y - c + print(f" -a {atten:3d}: pause {rms_db(x[pauses]) - rms_db(y[pauses]):5.1f} dB down, " + f"voiced {rms_db(y[voiced]) - rms_db(x[voiced]):+5.1f} dB, " + f"SNR in voiced {rms_db(c[voiced]) - rms_db(err[voiced]):5.1f} dB, " + f"lag {lag_samples(c, y, 4800)} samples, {secs / 16 * 60:5.1f} s per minute (mono)") + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-audio-restoration/scripts/run.sh b/docs/reports/2026-11-audio-restoration/scripts/run.sh new file mode 100755 index 00000000..a9abc0d2 --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/scripts/run.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# Reproduce every number in the report. +# +# WORK=/scratch/dir FFMPEG=/ffmpeg/ffmpeg [DEEP_FILTER=] ./run.sh +# +# Needs Python 3 with NumPy. FFMPEG defaults to the macOS arm64 deps bundle +# beside the repo; DEEP_FILTER to the add-on as the app installs it, if present. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${WORK:?set WORK to a scratch directory}" +: "${FFMPEG:=$REPO/deps/macos-arm64/ffmpeg/ffmpeg}" +export WORK FFMPEG +if [ -z "$DEEP_FILTER" ]; then + for f in "$REPO"/addons/deepfilternet/deep-filter-0.5.6-*; do + [ -x "$f" ] && DEEP_FILTER="$f" + done +fi +[ -n "$DEEP_FILTER" ] && export DEEP_FILTER +PY="${PYTHON:-python3}" +cd "$HERE" +"$PY" delay_probe.py +"$PY" start_offset_probe.py +"$PY" tempo_denoise_probe.py +"$PY" quality_probe.py diff --git a/docs/reports/2026-11-audio-restoration/scripts/start_offset_probe.py b/docs/reports/2026-11-audio-restoration/scripts/start_offset_probe.py new file mode 100644 index 00000000..13cb71ad --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/scripts/start_offset_probe.py @@ -0,0 +1,44 @@ +"""Does rendering audio to a separate WAV keep a container's audio start offset? + +The encode used to map the source's audio straight into the output, where +the muxer keeps an audio stream that starts later than the video (common in +captures) at its real position. Rendering the restored audio to a WAV loses +timestamps, so the render has to pad that gap itself. This checks which +aresample settings do so, with and without an input seek. +""" +import os +import subprocess + +import numpy as np + +from common import WORK, FFMPEG, ff, decode + +FFPROBE = os.environ.get("FFPROBE", os.path.join(os.path.dirname(FFMPEG), "ffprobe")) + + +def first_sound(path): + x = decode(path, 48000)[:, 0] + idx = np.argmax(np.abs(x) > 0.05) + return idx / 48000.0, len(x) / 48000.0 + + +def main(): + src = os.path.join(WORK, "late_audio.mkv") + ff("-f", "lavfi", "-i", "testsrc2=s=320x240:r=25:d=10", + "-itsoffset", "0.5", "-f", "lavfi", "-i", "sine=f=1000:d=9.5:r=48000", + "-map", "0", "-map", "1", "-c:v", "libx264", "-c:a", "flac", src) + print(subprocess.run([FFPROBE, "-v", "error", "-show_entries", + "stream=codec_type,start_time:format=start_time", + "-of", "compact", src], capture_output=True, text=True).stdout) + for seek in (None, "2"): + for af in ("anull", "aresample=first_pts=0", "aresample=async=1:first_pts=0"): + out = os.path.join(WORK, "late_out.wav") + args = (["-ss", seek] if seek else []) + ["-i", src, "-map", "0:a", + "-af", af, "-c:a", "pcm_f32le", out] + ff(*args) + onset, dur = first_sound(out) + print(f"seek={seek!s:4s} af={af:32s} first sound at {onset:.3f}s, length {dur:.3f}s") + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-audio-restoration/scripts/tempo_denoise_probe.py b/docs/reports/2026-11-audio-restoration/scripts/tempo_denoise_probe.py new file mode 100644 index 00000000..dd2b26b0 --- /dev/null +++ b/docs/reports/2026-11-audio-restoration/scripts/tempo_denoise_probe.py @@ -0,0 +1,67 @@ +"""Two follow-ups from the first heavy-test run. + +1. Does atempo shift the audio as well as stretching it? (The drift test + found every click ~20 ms early.) +2. Why did afftdn take only 0.7 dB off white noise at -39 dBFS with nr=20? + Its noise floor `nf` defaults to -50 dB; try the measured floor. + +Run: WORK=/scratch FFMPEG=.../ffmpeg python tempo_denoise_probe.py +""" +import os + +import numpy as np + +from common import WORK, ff, decode, rms_db + +RATE = 48000 +CLICKS = ("aevalsrc='0.8*sin(2*PI*3000*t)*(lt(mod(t\\,0.7)\\,0.003)" + "+lt(mod(t+0.23\\,1.1)\\,0.003))':s=48000:d=8") + + +def onsets(x, thr=0.3, gap=0.1): + out, last = [], -1e9 + for i in np.flatnonzero(np.abs(x) > thr): + t = i / RATE + if t - last > gap: + out.append(t) + last = t + return np.array(out) + + +def main(): + src = os.path.join(WORK, "probe_clicks.wav") + ff("-f", "lavfi", "-i", CLICKS, "-c:a", "pcm_f32le", src) + ref = onsets(decode(src, RATE)[:, 0]) + print("== atempo: first click positions vs expected t / factor ==") + for factor in (1.004, 0.996, 1.02): + for chain in (f"atempo={factor}", f"asetrate={RATE*factor:.3f},aresample={RATE}", + f"aresample={RATE}:async=0,rubberband=tempo={factor}" if False else None): + if chain is None: + continue + got = onsets(decode(src, RATE, af=chain)[:, 0]) + exp = ref / factor + n = min(len(got), len(exp)) + err = got[:n] - exp[:n] + print(f" {chain:40s} error mean {err.mean()*1000:7.2f} ms, " + f"first {err[0]*1000:7.2f} ms, last {err[n-1]*1000:7.2f} ms") + + print("== afftdn on white noise (-39 dBFS RMS) + clicks ==") + noisy = os.path.join(WORK, "probe_noise.wav") + ff("-f", "lavfi", "-i", CLICKS, "-f", "lavfi", "-i", + "anoisesrc=c=white:a=0.02:r=48000:d=8:seed=7", + "-filter_complex", "[0][1]amix=inputs=2:normalize=0", "-c:a", "pcm_f32le", noisy) + x = decode(noisy, RATE)[:, 0] + win = slice(int(4.0 * RATE), int(4.12 * RATE)) + # The floor afftdn would be told: the 10th percentile of 50 ms RMS. + frames = x[: len(x) // 2400 * 2400].reshape(-1, 2400) + floor = np.percentile(20 * np.log10(np.sqrt((frames ** 2).mean(axis=1)) + 1e-12), 10) + print(f" input floor {rms_db(x[win]):.1f} dB; 10th-percentile 50 ms RMS {floor:.1f} dB") + for chain in ("afftdn=nr=20:nf=-50:tn=1", "afftdn=nr=20:nf=-50", + f"afftdn=nr=20:nf={floor:.0f}", f"afftdn=nr=20:nf={floor:.0f}:tn=1", + f"afftdn=nr=12:nf={floor:.0f}:tn=1"): + y = decode(noisy, RATE, af=chain)[:, 0] + print(f" {chain:34s} floor {rms_db(y[win]):6.1f} dB (down {rms_db(x[win]) - rms_db(y[win]):5.1f})") + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-classical-filters/img/01-nr-kodim08-sigma8.png b/docs/reports/2026-11-classical-filters/img/01-nr-kodim08-sigma8.png new file mode 100644 index 00000000..dea1e505 Binary files /dev/null and b/docs/reports/2026-11-classical-filters/img/01-nr-kodim08-sigma8.png differ diff --git a/docs/reports/2026-11-classical-filters/img/02-dehalo-kodim23.png b/docs/reports/2026-11-classical-filters/img/02-dehalo-kodim23.png new file mode 100644 index 00000000..0a00390b Binary files /dev/null and b/docs/reports/2026-11-classical-filters/img/02-dehalo-kodim23.png differ diff --git a/docs/reports/2026-11-classical-filters/img/03-dehalo-kodim08.png b/docs/reports/2026-11-classical-filters/img/03-dehalo-kodim08.png new file mode 100644 index 00000000..ce22dd9b Binary files /dev/null and b/docs/reports/2026-11-classical-filters/img/03-dehalo-kodim08.png differ diff --git a/docs/reports/2026-11-classical-filters/img/04-mpeg2stinx-kodim08-1000k.png b/docs/reports/2026-11-classical-filters/img/04-mpeg2stinx-kodim08-1000k.png new file mode 100644 index 00000000..81df1be8 Binary files /dev/null and b/docs/reports/2026-11-classical-filters/img/04-mpeg2stinx-kodim08-1000k.png differ diff --git a/docs/reports/2026-11-classical-filters/img/05-aa-kodim23.png b/docs/reports/2026-11-classical-filters/img/05-aa-kodim23.png new file mode 100644 index 00000000..219be938 Binary files /dev/null and b/docs/reports/2026-11-classical-filters/img/05-aa-kodim23.png differ diff --git a/docs/reports/2026-11-classical-filters/img/06-fades-kodim23.png b/docs/reports/2026-11-classical-filters/img/06-fades-kodim23.png new file mode 100644 index 00000000..7800a535 Binary files /dev/null and b/docs/reports/2026-11-classical-filters/img/06-fades-kodim23.png differ diff --git a/docs/reports/2026-11-classical-filters/index.html b/docs/reports/2026-11-classical-filters/index.html new file mode 100644 index 00000000..c40b2344 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/index.html @@ -0,0 +1,262 @@ + + + + + +Classical filters for deps 1.16.0 + + + +
+ +

Classical filters for deps 1.16.0

+

Probe and build, 2026-11-08. SangNom, the vszip upgrade (CPU NLMeans, and whether WNNM earns a place), and three ports from vs-jetpack: FixInterlacedFades, vine_dehalo and mpeg2stinx. Apple M1 Pro (macOS arm64), private copy of the deps bundle with vszip 24.0.0 and SangNom r45.

+ +
+ Outcome +
    +
  • SangNom: built, as its own Anti-aliasing method. havsfunc's santiag(type="sangnom") cannot be used: at strength 0 it calls SangNom(order=0), which SangNom r45 treats as double-rate and refuses without _FieldBased (measured: "clip's frames must have the _FieldBased property when order is 0" at every depth). The method supersamples 2x, runs SangNom (order=1) along each axis and comes back down. It runs at 8/10/12/16-bit and float; aa is scaled by the plugin (raw SangNom, 8 vs 10-bit: 0.047 levels).
  • +
  • vszip 22.1.0 → 24.0.0: built. 24.0.0 is the first release with CPU NLMeans; WNNM arrived in 22.0.0. NLMeans (CPU) is an advanced Noise Reduction method sharing KNLMeansCL's settings: on the same input and arguments it gives KNLMeansCL's picture (see the parity line below), with no OpenCL device. The KNLMeans "no device" message now points at it. CLAHE still passes its depth-parity test (1.34 levels against a 3.0 tolerance).
  • +
  • WNNM: not added. Each at its best sigma, its luma runs from +0.7 to −2.3 dB against the shipped BM3D (ahead only at sigma 4, with a temporal radius), and its chroma is 2.6–7.2 dB worse. A wider sweep (to 4× the noise, scripts/wnnm_wide.py) found nothing better than 2–2.5×. It is fast — 10.3 fps at 1080p with a temporal radius, where the shipped BM3D path manages 0.2 — but NLMeans (CPU), added in this change, is faster still (23.0 fps) with chroma 1–4 dB better than WNNM's, so the fast-CPU slot is taken.
  • +
  • FixInterlacedFades: built, as an IVTC option. Matches vs-jetpack exactly (0.0000 levels, both modes). On a fade applied after the field split it closes the field-brightness gap from 5.27 to 0.29 levels (average) / 0.00004 (match); on ordinary frames it changes nothing measurable.
  • +
  • vine_dehalo: built, advanced-only. A faithful port (vs-jetpack's own function could not run against the bundle; see §5). On the oversharpened fixtures it recovers +0.9 to +1.1 dB, where DeHalo_alpha recovers +4.1 to +4.4 dB by blurring, at a fraction of vine's cost.
  • +
  • mpeg2stinx: built, advanced-only, with a warning in its description. Matches vs-jetpack to 0.04 levels, but it lost PSNR on every fixture tried, up to −3.0 dB on detailed content: it clamps each field to a bob of the other, which costs vertical detail. None of these fixtures carries the specific artefact it is written for, so it ships for the user who has that artefact, and says what it costs.
  • +
+
+ +

1. Fixtures and method

+

Three Kodak PhotoCD images (kodim08 buildings, kodim19 fence, kodim23 parrots) panned by a 640×480 window, 2 px per frame, 30 frames, lossless (scripts/make_fixtures.sh). A whole-pixel pan keeps every frame an exact crop, so there is exact ground truth and real motion for the temporal methods. Noise is AddGrain at a known sigma (8-bit levels; measured 4.02 / 8.01 / 12.0). Each method is swept over strengths that scale with the noise and reported at its best luma PSNR. The MPEG-2 fixtures are the pans 3:2-telecined and encoded as interlaced MPEG-2 at 1000 and 2000 kbps (scripts/make_tc_fixtures.sh), then inverse-telecined with VIVTC as the IVTC method does; every kept frame was checked to match its own clean frame far better than its neighbours.

+

The template code is measured as shipped: scripts/ports_compare.py lifts each block out of worker/templates/pipeline_template.vpy and runs it. Speeds come from scripts/bench.py, one fresh vspipe per measurement, median of three; the fps column printed by nr_compare.py (in results/nr_compare.txt) was taken while other probes ran and through cached frames, and is not used. That run also crashed in its last step (a float compared with == 0) before writing its JSON, so the tables are parsed from its printed log (scripts/parse_nr_log.py) and the depth check was re-run alone.

+
What the numbers cannot tell you. PSNR rewards softness, and it is the only metric here; the crops are there to be looked at. Three still images panned are not a population of real footage, and synthetic Gaussian noise is not film grain or tape noise. Speeds are from one Apple Silicon machine with other jobs running on it at times, so treat them as ratios, not throughput. Nothing here ran on x86, Windows or Linux.
+ +

2. Noise reduction: BM3D, NLMeans (CPU and GPU), WNNM

+ +
+ + + + + + + + +
kodim08_pan: luma / chroma (U) PSNR in dB, each method at its best strength
methodσ 4: YUstrengthσ 8: YUstrengthσ 12: YUstrength
noisy input36.0530.0526.54
BM3D (bm3d r12, r=1, Final) - shipped40.4445.77835.8942.671633.3440.8824
NLMeans vszip CPU (d=1 a=2 s=4)39.9344.783.534.8441.68732.1739.8510.5
KNLMeansCL GPU (d=1 a=2 s=4)39.9344.783.534.8441.68732.1739.8510.5
WNNM vszip (r=0)37.8043.23833.4338.251631.0034.8724
WNNM vszip (r=1)41.0143.17835.8537.311632.9233.6524
BM3D vszip (r=1)38.2443.95534.0040.001031.6437.5515
+
+ + + + + + + + +
kodim19_pan: luma / chroma (U) PSNR in dB, each method at its best strength
methodσ 4: YUstrengthσ 8: YUstrengthσ 12: YUstrength
noisy input36.0530.0526.53
BM3D (bm3d r12, r=1, Final) - shipped40.6146.68836.3043.191234.1441.1718
NLMeans vszip CPU (d=1 a=2 s=4)40.6245.584.535.9342.87733.2440.8510.5
KNLMeansCL GPU (d=1 a=2 s=4)40.6245.584.535.9342.87733.2440.8510.5
WNNM vszip (r=0)38.6544.02834.4738.981632.1635.6224
WNNM vszip (r=1)41.3343.671036.1438.001633.2034.4124
BM3D vszip (r=1)39.1544.79535.2140.931032.9640.1918
+
+ + + + + + + + +
kodim23_pan: luma / chroma (U) PSNR in dB, each method at its best strength
methodσ 4: YUstrengthσ 8: YUstrengthσ 12: YUstrength
noisy input36.0530.0526.53
BM3D (bm3d r12, r=1, Final) - shipped43.1547.34840.1244.001638.1841.9424
NLMeans vszip CPU (d=1 a=2 s=4)42.8146.783.539.3043.26737.2441.1413.5
KNLMeansCL GPU (d=1 a=2 s=4)42.8146.783.539.3043.26737.2441.1413.5
WNNM vszip (r=0)41.4144.23837.4239.231634.8235.8924
WNNM vszip (r=1)42.6244.131037.8738.422034.7934.8130
BM3D vszip (r=1)41.9745.60638.5641.791236.4739.3918
+
+ + + +
Depth parity: mean |8-bit run − N-bit run|, 8-bit levels (luma), kodim08 sigma 8
method10-bit16-bit
NLMeans vszip h=20.0050.006
WNNM sigma=80.2680.272
BM3D template sigma=80.2730.277
+

vszip NLMeans vs KNLMeansCL, same args: mean |diff| Y 0.0000 U 0.0000 levels

+ +
+ Five crops of the kodim08 pan at sigma 8 +
kodim08, sigma 8, frame 3, 2× nearest. Left to right: clean, noisy, BM3D (sigma 16, r=1, Final — what ships), NLMeans CPU (h 7), WNNM (sigma 16, r=1). WNNM keeps the wall texture as well as BM3D on luma; look at the colour noise it leaves in the shutters, which is its chroma deficit in the table.
+
+

Speed

+ +
+ + + + + + + + + + + + + + + + +
Speed: vspipe, median of runs, frames per second (M1 Pro)
method640x4801920x1080
BM3D (bm3d r12, r=1, Final, sigma 16) - shipped1.50.2
BM3D (bm3d r12, r=0, Final, sigma 16)1.10.1
NLMeans vszip CPU (d=1 a=2 s=4, h 7)181.323.0
KNLMeansCL GPU (d=1 a=2 s=4, h 7)126.518.5
WNNM vszip (r=0, sigma 16)159.116.7
WNNM vszip (r=1, sigma 16)90.310.3
BM3D vszip (r=1, sigma 10)150.416.9
input only (decode + noise)1999.41146.7
AA: daa60.715.9
AA: santiag (nnedi3, strength 1/1)27.219.6
AA: SangNom (aa 48)79.98.6
Dehalo: DeHalo_alpha (defaults)1399.0211.5
Dehalo: vine (16, 0.5, 1.0)31.615.3
Deblock: Deblock_QED (defaults)1073.5151.7
Deblock: mpeg2stinx36.110.0
IVTC fade fix (average), alone588.275.8
+ +
Reading the speed table. The 640×480 rows ran while other jobs shared the machine: their three runs spread widely (daa 8.3–16.5 fps) and several are slower than the same method at 1080p, which cannot be the method. The 1080p rows ran on a quieter machine and their runs agree within a few percent; they are only 10 frames each, so per-run start-up weighs on the fastest methods (the "input only" row is not a real 1146 fps). Use the 1080p column for comparisons. The shipped BM3D path processes all three planes at sigma 16 here, which is slower than its luma-only default — but 0.2 fps at 1080p (0.9 at 640×480) is far below the ~14 fps at SD a code comment claims for it on the defaults, and r=0 measured slower than r=1. That is worth a dedicated look on arm64; it is not explained here.
+

NLMeans (CPU and GPU, identical) sits 0.3–1.2 dB below BM3D on luma. The vszip BM3D (new in 24.0.0) was measured for the record only: 1.1–2.2 dB below the shipped bm3d r12 path on every fixture and sigma, so it is not used.

+ +

3. SangNom anti-aliasing

+

Speed, corrected. An early probe (Python clip.frames(), 60 frames, other jobs running) put SangNom at 45 fps against santiag's 17 and daa's 26, and the first draft of this change called it the fastest method. The vspipe benchmark disagrees: at 1080p SangNom runs 8.6 fps against daa's 15.9 and santiag's 19.6 — it works on a picture twice the size in each direction. It is offered as the sharpest, not the fastest, and the method's description says it is slower on HD.

+

Every depth runs: raw SangNom changes the picture by 4.97 levels at 8, 10, 12 and 16-bit alike, and the method by 4.37–4.39. The method's 8 vs 10/16-bit difference is 0.44–0.48 levels (santiag's own is 0.37), most of it the 2× Spline36 round trip. A still frame repeated gives bit-identical output, so it does not shimmer.

+ +
+ + + +
Anti-aliasing (kodim08 pan)
methodmean change, levels
daa3.41
santiag (nnedi3, 1/1)4.18
SangNom AA (48)4.11
+ +

The speeds are in §2's speed table (run on the clean pan).

+
+ Aliased crop and three anti-aliased versions +
kodim23 point-resampled to half size and back (jagged edges), 2× nearest. Left to right: aliased, daa, santiag (nnedi3, 1/1), SangNom (aa 48). SangNom rebuilds the diagonal edges of the leaves most firmly; daa is softest.
+
+ +

4. FixInterlacedFades (the IVTC fade fix)

+

A fade applied to telecined video changes brightness between a frame's two fields, so no field match is clean. The fixture fades the kodim23 pan to black per field (the top field at its time, the bottom half a frame later). The heavy test does the same to a real clip through the whole worker: the gap between field brightnesses goes from 4.12 to 0.06 levels.

+ +
+ + + + + + + + +
IVTC fade fix (kodim23 pan, faded after the field split)
measurevalue
field_gap_faded5.2735
field_gap_fixed0.2856
field_gap_fixed_match0.0000
field_gap_clean0.0011
psnr_faded_vs_truth38.9722
psnr_fixed_vs_truth52.9206
unfaded_change_mad0.0000
unfaded_change_mad_mpeg20.0000
+ +

field_gap_*: mean |top-field mean − bottom-field mean| in levels; psnr_*_vs_truth: luma PSNR against each frame faded uniformly to its two fields' mean level; unfaded_change_mad: mean change the fix makes to the clean (not fading) pan, and to the decoded MPEG-2 pan.

+
+ A combing fade frame and two fixed versions +
A frame whose top field sits at 70% of the fade and the bottom at 45%, 2× nearest. Left to right: input (the fade combs), fixed with average (both fields at their mean), fixed with match (the brighter to the darker). Both are comb-free; match is darker.
+
+

A crash found by the benchmark, and fixed. The first version of the block made an akarin Expr per frame inside two concurrent FrameEvals. Through vspipe that aborted (SIGABRT on 10 frames), segfaulted (SIGSEGV on a 720×576 IVTC job through the worker) or spun one core indefinitely — the speed benchmark sat on 30 frames for 97 minutes before it was killed. Single-threaded, or with std.Expr, the same graph took a second, which is why the Python-side probes above and the first 320×240 heavy test never saw it. The block now stamps the gains as frame properties (ModifyFrame, which creates no nodes) and applies them with one akarin expression; output is unchanged (still 0.0000 levels from vs-jetpack). A heavy test now times a full 720×576 IVTC + fade-fix job (2.5 s) and fails on the old block with the SIGSEGV. Every number in this section after the fix was re-measured.

+

A correction found during the work: the first run reported a residual gap of 3.3 levels for average. It was real but specific: on the last frame of a fade that reaches full black one field is already black, upstream's expression leaves a zero-distance field alone, and average then sets the other to half. Match handles it. That is why both modes are offered. A second trap: std.DoubleWeave flags its output interlaced, so without resetting _FieldBased a later resize treated the fixed (progressive) frames as fields; the block resets it.

+ +

5. vine_dehalo

+

vs-jetpack's own vine_dehalo could not be run against the bundle as a reference: its Catrom downscale is a custom kernel on the resize2 plugin, which the bundle does not ship (loading a downloaded resize2 binary just for the comparison was not done). The port was checked line by line instead; its NLMeans calls are vs-jetpack 3.0.0's CPU backend verbatim (vszip.NLMeans), and mpeg2stinx, ported the same way, matched vs-jetpack to 0.04 levels.

+

The fixture is the clean pan with an unsharp mask (Gaussian sigma 1.5, amount 1.5), which brightens edges and fine texture alike, so PSNR against the clean pan rewards undoing sharpening of any kind. DeHalo_alpha wins on that measure by blurring; vine keeps texture, which is its purpose.

+ +
+ + + + + + +
Dehalo, kodim08_pan (oversharpened): luma PSNR vs the clean pan
methodY dB
haloed (no filter)21.93
vine (16, 0.5, 1.0)22.80
vine (24, 0.5, 1.0)23.06
vine (8, 0.5, 1.0)22.45
DeHalo_alpha (defaults)26.28
FineDehalo (defaults)22.00
+
+ + + + + + +
Dehalo, kodim23_pan (oversharpened): luma PSNR vs the clean pan
methodY dB
haloed (no filter)31.99
vine (16, 0.5, 1.0)33.12
vine (24, 0.5, 1.0)33.26
vine (8, 0.5, 1.0)32.83
DeHalo_alpha (defaults)36.11
FineDehalo (defaults)32.25
+ +
+ Dehalo comparison on kodim08 +
kodim08, 2× nearest. Left to right: clean, oversharpened, vine (16, 0.5, 1.0), DeHalo_alpha (defaults), FineDehalo (defaults). vine softens the amplified wall texture and the rails' halo while keeping the shutter slats crisp; DeHalo_alpha is closer to the clean picture here because the "halo" is mostly amplified texture.
+
+
+ Dehalo comparison on kodim23 +
kodim23 (soft background), same panel order. Here the oversharpening is mostly grain-like texture; vine smooths it, DeHalo_alpha and FineDehalo barely touch it away from edges.
+
+ +

6. mpeg2stinx

+ +
+ + + + + + + + +
mpeg2stinx: PSNR vs the clean pan (Y / U dB)
fixturedecoded (no filter)mpeg2stinx (TFF)mpeg2stinx (wrong order: BFF)Deblock_QED (defaults)Deblock (quant 25)
kodim08 telecined @1000k33.30 / 41.6131.54 / 41.6831.53 / 41.6833.24 / 41.7733.27 / 41.88
kodim08 progressive-as-interlaced @1000k42.17 / 46.1338.29 / 46.2138.26 / 46.2141.99 / 45.9542.07 / 46.06
kodim08 telecined @2000k39.32 / 44.6036.31 / 44.7136.29 / 44.7139.24 / 44.6139.29 / 44.78
kodim08 progressive-as-interlaced @2000k42.39 / 46.2038.38 / 46.2838.36 / 46.2842.18 / 46.0142.28 / 46.13
kodim23 telecined @1000k43.35 / 47.0143.08 / 47.0743.06 / 47.0743.22 / 47.1243.25 / 47.27
kodim23 progressive-as-interlaced @1000k44.19 / 47.6043.90 / 47.6543.88 / 47.6544.00 / 47.7544.01 / 47.89
kodim23 telecined @2000k44.33 / 47.6643.99 / 47.7443.96 / 47.7444.09 / 47.7344.12 / 47.86
kodim23 progressive-as-interlaced @2000k44.19 / 47.6043.90 / 47.6543.88 / 47.6544.00 / 47.7544.01 / 47.89
+ +

Per frame (kodim08, telecined, 1000 kbps), mpeg2stinx costs about 4.5 dB on the cleanly coded frames (42.4 → 38.1) and gains 0.1 dB on the starved ones (26.4 → 26.5); the first and last frames are untouched because the temporal limit compares a frame with itself there. The wrong field order changes the result by 0.01–0.03 dB: on progressive film frames both fields come from the same instant.

+
+ mpeg2stinx comparison +
kodim08 telecined, MPEG-2 1000 kbps, IVTC'd, frame 10, 2× nearest. Left to right: clean, decoded, mpeg2stinx, Deblock_QED. mpeg2stinx softens the roof tiles and window bars vertically; the compression smear it is not designed for remains.
+
+ +

7. Depth parity of the ports

+ +
+ + + + +
Ports, depth parity: mean |8-bit − N-bit|, levels
block10-bit16-bit
vine0.4500.455
mpeg2stinx0.0090.010
FixInterlacedFades0.2510.257
SangNom AA0.4360.476
+ +

The heavy high-bit-depth suite (app/test/integration_high_bit_depth_filters_test.dart) agrees, through the whole worker at 8 vs 10-bit (4:2:2): NLMeans (CPU) 0.301, KNLMeansCL 0.301, SangNom 0.491, the fade fix 0.277, vine 0.406, mpeg2stinx 0.287, CLAHE 1.336 levels (tolerance 2.0, CLAHE 3.0).

+ +

8. Deps and platforms

+
    +
  • SangNom r45 from its PyPI wheels on every platform, pinned by hash. The x64 builds (macOS, Linux, Windows) contain SSE2 code only — no ymm/zmm register anywhere (objdump) and no static initializers (Scripts/check-load-time-simd.py), so one file serves both CPU tiers. Minimum macOS 11.0 (arm64) / 10.15 (x64); manylinux_2_28; the Windows DLL imports only the UCRT.
  • +
  • Licence: the source header and LICENSE grant MIT (© 2016 james1201, © 2013 Victor Efimov); the same header says "Binaries are GPL v2", and the wheels' metadata says GPL-2.0. Credited as both.
  • +
  • vszip 24.0.0: the baseline x64 builds contain no AVX (and no longer call fmaf). The arm64 wheel is minos 15.0 — exactly the arm64 floor — with 4 KB of header room. macOS x64 builds the sdist: it now needs Zig 0.17.0 and -Doptimize=fast, and the headerpad patch had to follow lib.use_llvm into if (build_vs). Cross-built here with that exact recipe: minos 12.0, 4104 bytes of header room, signs cleanly; loaded under Rosetta into the deps 1.10.0 x64 VapourSynth R78, NLMeans, WNNM, CLAHE and the x64 SangNom all ran.
  • +
  • Not verified: anything on Windows or Linux, and the x64 builds on a real Intel CPU (Rosetta executes AVX, so it proves nothing about CPU features; the objdump scan is the evidence there). The deps CI run is the test.
  • +
+ +

9. Decisions

+
    +
  • SangNom as a separate method, santiag's interpolator stays pinned to nnedi3 — because havsfunc's sangnom path is incompatible with SangNom r45.
  • +
  • NLMeans (CPU) reuses the KNLMeans parameters rather than its own — same algorithm, same arguments, same picture; switching between them keeps the settings.
  • +
  • WNNM not added — no quality or speed niche BM3D does not already fill.
  • +
  • vine and mpeg2stinx advanced-only — both are specialist tools, and the simple Dehalo slots are full.
  • +
  • mpeg2stinx never takes the Deblock pass's per-field path — it is a cross-field filter by design.
  • +
  • The fade fix after VFM and before VDecimate, off by default — it only helps a fade that combs.
  • +
+ +

10. Not tested

+
    +
  • Real footage with a combing fade or with the comb-like MPEG-2 noise mpeg2stinx targets: no such clip was available.
  • +
  • vine against vs-jetpack's own output (needs resize2).
  • +
  • Any platform but macOS arm64 (and macOS x64 load under Rosetta).
  • +
+ +

Scripts

+

scripts/make_fixtures.sh, scripts/make_tc_fixtures.sh build the fixtures into $SCRATCH/fx (the Kodak images are fetched from r0k.us). scripts/run.sh <probe> runs a probe under a deps bundle's Python: nr_compare.py (§2), ports_compare.py (§3–7), ports_vs_reference.py (the vs-jetpack comparison; needs vsjetpack 2.2.7, whose three functions are identical to 3.0.0's), bench.py (speeds), images.py (the figures), and tables.py prints the tables above from the probes' JSON.

+ +
+ + diff --git a/docs/reports/2026-11-classical-filters/results/bench.txt b/docs/reports/2026-11-classical-filters/results/bench.txt new file mode 100644 index 00000000..76e88bdc --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/bench.txt @@ -0,0 +1,32 @@ +640x480 BM3D (bm3d r12, r=1, Final, sigma 16) - shipped 1.5 fps [1.47, 1.5, 1.51] +640x480 BM3D (bm3d r12, r=0, Final, sigma 16) 1.1 fps [1.14, 1.14, 1.14] +640x480 NLMeans vszip CPU (d=1 a=2 s=4, h 7) 181.3 fps [182.17, 178.15, 181.27] +640x480 KNLMeansCL GPU (d=1 a=2 s=4, h 7) 126.5 fps [126.55, 125.96, 127.03] +640x480 WNNM vszip (r=0, sigma 16) 159.1 fps [156.9, 164.16, 159.12] +640x480 WNNM vszip (r=1, sigma 16) 90.3 fps [90.32, 91.42, 89.53] +640x480 BM3D vszip (r=1, sigma 10) 150.4 fps [150.38, 151.37, 146.78] +640x480 input only (decode + noise) 1999.4 fps [1999.44, 2015.16, 1992.33] +640x480 AA: daa 60.7 fps [61.18, 60.67, 60.53] +640x480 AA: santiag (nnedi3, strength 1/1) 27.2 fps [27.45, 27.11, 27.21] +640x480 AA: SangNom (aa 48) 79.9 fps [81.26, 78.09, 79.93] +640x480 Dehalo: DeHalo_alpha (defaults) 1399.0 fps [1398.99, 1431.08, 1383.68] +640x480 Dehalo: vine (16, 0.5, 1.0) 31.6 fps [31.3, 31.58, 31.73] +640x480 Deblock: Deblock_QED (defaults) 1073.5 fps [1066.51, 1076.15, 1073.55] +640x480 Deblock: mpeg2stinx 36.1 fps [36.1, 35.46, 36.39] +640x480 IVTC fade fix (average), alone 588.2 fps [584.98, 588.18, 600.31] +1920x1080 BM3D (bm3d r12, r=1, Final, sigma 16) - shipped 0.2 fps [0.21, 0.22, 0.21] +1920x1080 BM3D (bm3d r12, r=0, Final, sigma 16) 0.1 fps [0.14, 0.14, 0.13] +1920x1080 NLMeans vszip CPU (d=1 a=2 s=4, h 7) 23.0 fps [22.64, 23.44, 22.99] +1920x1080 KNLMeansCL GPU (d=1 a=2 s=4, h 7) 18.5 fps [20.07, 18.49, 17.86] +1920x1080 WNNM vszip (r=0, sigma 16) 16.7 fps [15.03, 16.7, 16.77] +1920x1080 WNNM vszip (r=1, sigma 16) 10.3 fps [10.28, 10.42, 10.17] +1920x1080 BM3D vszip (r=1, sigma 10) 16.9 fps [17.43, 16.86, 14.99] +1920x1080 input only (decode + noise) 1146.7 fps [1146.66, 1138.22, 1149.18] +1920x1080 AA: daa 15.9 fps [15.87, 15.75, 16.33] +1920x1080 AA: santiag (nnedi3, strength 1/1) 19.6 fps [19.53, 19.66, 19.56] +1920x1080 AA: SangNom (aa 48) 8.6 fps [8.23, 8.59, 8.8] +1920x1080 Dehalo: DeHalo_alpha (defaults) 211.5 fps [190.72, 211.54, 226.6] +1920x1080 Dehalo: vine (16, 0.5, 1.0) 15.3 fps [15.32, 15.4, 14.8] +1920x1080 Deblock: Deblock_QED (defaults) 151.7 fps [151.61, 151.72, 152.85] +1920x1080 Deblock: mpeg2stinx 10.0 fps [9.96, 10.18, 8.96] +1920x1080 IVTC fade fix (average), alone 75.8 fps [74.15, 75.75, 76.01] diff --git a/docs/reports/2026-11-classical-filters/results/bench_results.json b/docs/reports/2026-11-classical-filters/results/bench_results.json new file mode 100644 index 00000000..ae1da04f --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/bench_results.json @@ -0,0 +1,262 @@ +{ + "640x480": { + "BM3D (bm3d r12, r=1, Final, sigma 16) - shipped": { + "median_fps": 1.5, + "runs": [ + 1.47, + 1.5, + 1.51 + ] + }, + "BM3D (bm3d r12, r=0, Final, sigma 16)": { + "median_fps": 1.14, + "runs": [ + 1.14, + 1.14, + 1.14 + ] + }, + "NLMeans vszip CPU (d=1 a=2 s=4, h 7)": { + "median_fps": 181.27, + "runs": [ + 182.17, + 178.15, + 181.27 + ] + }, + "KNLMeansCL GPU (d=1 a=2 s=4, h 7)": { + "median_fps": 126.55, + "runs": [ + 126.55, + 125.96, + 127.03 + ] + }, + "WNNM vszip (r=0, sigma 16)": { + "median_fps": 159.12, + "runs": [ + 156.9, + 164.16, + 159.12 + ] + }, + "WNNM vszip (r=1, sigma 16)": { + "median_fps": 90.32, + "runs": [ + 90.32, + 91.42, + 89.53 + ] + }, + "BM3D vszip (r=1, sigma 10)": { + "median_fps": 150.38, + "runs": [ + 150.38, + 151.37, + 146.78 + ] + }, + "input only (decode + noise)": { + "median_fps": 1999.44, + "runs": [ + 1999.44, + 2015.16, + 1992.33 + ] + }, + "AA: daa": { + "median_fps": 60.67, + "runs": [ + 61.18, + 60.67, + 60.53 + ] + }, + "AA: santiag (nnedi3, strength 1/1)": { + "median_fps": 27.21, + "runs": [ + 27.45, + 27.11, + 27.21 + ] + }, + "AA: SangNom (aa 48)": { + "median_fps": 79.93, + "runs": [ + 81.26, + 78.09, + 79.93 + ] + }, + "Dehalo: DeHalo_alpha (defaults)": { + "median_fps": 1398.99, + "runs": [ + 1398.99, + 1431.08, + 1383.68 + ] + }, + "Dehalo: vine (16, 0.5, 1.0)": { + "median_fps": 31.58, + "runs": [ + 31.3, + 31.58, + 31.73 + ] + }, + "Deblock: Deblock_QED (defaults)": { + "median_fps": 1073.55, + "runs": [ + 1066.51, + 1076.15, + 1073.55 + ] + }, + "Deblock: mpeg2stinx": { + "median_fps": 36.1, + "runs": [ + 36.1, + 35.46, + 36.39 + ] + }, + "IVTC fade fix (average), alone": { + "median_fps": 588.18, + "runs": [ + 584.98, + 588.18, + 600.31 + ] + } + }, + "1920x1080": { + "BM3D (bm3d r12, r=1, Final, sigma 16) - shipped": { + "median_fps": 0.21, + "runs": [ + 0.21, + 0.22, + 0.21 + ] + }, + "BM3D (bm3d r12, r=0, Final, sigma 16)": { + "median_fps": 0.14, + "runs": [ + 0.14, + 0.14, + 0.13 + ] + }, + "NLMeans vszip CPU (d=1 a=2 s=4, h 7)": { + "median_fps": 22.99, + "runs": [ + 22.64, + 23.44, + 22.99 + ] + }, + "KNLMeansCL GPU (d=1 a=2 s=4, h 7)": { + "median_fps": 18.49, + "runs": [ + 20.07, + 18.49, + 17.86 + ] + }, + "WNNM vszip (r=0, sigma 16)": { + "median_fps": 16.7, + "runs": [ + 15.03, + 16.7, + 16.77 + ] + }, + "WNNM vszip (r=1, sigma 16)": { + "median_fps": 10.28, + "runs": [ + 10.28, + 10.42, + 10.17 + ] + }, + "BM3D vszip (r=1, sigma 10)": { + "median_fps": 16.86, + "runs": [ + 17.43, + 16.86, + 14.99 + ] + }, + "input only (decode + noise)": { + "median_fps": 1146.66, + "runs": [ + 1146.66, + 1138.22, + 1149.18 + ] + }, + "AA: daa": { + "median_fps": 15.87, + "runs": [ + 15.87, + 15.75, + 16.33 + ] + }, + "AA: santiag (nnedi3, strength 1/1)": { + "median_fps": 19.56, + "runs": [ + 19.53, + 19.66, + 19.56 + ] + }, + "AA: SangNom (aa 48)": { + "median_fps": 8.59, + "runs": [ + 8.23, + 8.59, + 8.8 + ] + }, + "Dehalo: DeHalo_alpha (defaults)": { + "median_fps": 211.54, + "runs": [ + 190.72, + 211.54, + 226.6 + ] + }, + "Dehalo: vine (16, 0.5, 1.0)": { + "median_fps": 15.32, + "runs": [ + 15.32, + 15.4, + 14.8 + ] + }, + "Deblock: Deblock_QED (defaults)": { + "median_fps": 151.72, + "runs": [ + 151.61, + 151.72, + 152.85 + ] + }, + "Deblock: mpeg2stinx": { + "median_fps": 9.96, + "runs": [ + 9.96, + 10.18, + 8.96 + ] + }, + "IVTC fade fix (average), alone": { + "median_fps": 75.75, + "runs": [ + 74.15, + 75.75, + 76.01 + ] + } + } +} \ No newline at end of file diff --git a/docs/reports/2026-11-classical-filters/results/nr_compare.txt b/docs/reports/2026-11-classical-filters/results/nr_compare.txt new file mode 100644 index 00000000..96ff3289 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/nr_compare.txt @@ -0,0 +1,79 @@ + +== kodim08_pan sigma 4.0 (measured 4.02); noisy Y 36.05 dB + BM3D (bm3d r12, r=1, Final) - shipped best 8.0 Y 40.44 U 45.77 1.2 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 3.5 Y 39.93 U 44.78 162.5 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 3.5 Y 39.93 U 44.78 36.1 fps + WNNM vszip (r=0) best 8.0 Y 37.80 U 43.23 97.9 fps + WNNM vszip (r=1) best 8.0 Y 41.01 U 43.17 67.2 fps + BM3D vszip (r=1) best 5.0 Y 38.24 U 43.95 99.2 fps + +== kodim08_pan sigma 8.0 (measured 8.01); noisy Y 30.05 dB + BM3D (bm3d r12, r=1, Final) - shipped best 16.0 Y 35.89 U 42.67 1.4 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 7.0 Y 34.84 U 41.68 144.8 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 7.0 Y 34.84 U 41.68 100.5 fps + WNNM vszip (r=0) best 16.0 Y 33.43 U 38.25 101.8 fps + WNNM vszip (r=1) best 16.0 Y 35.85 U 37.31 71.7 fps + BM3D vszip (r=1) best 10.0 Y 34.00 U 40.00 99.1 fps + +== kodim08_pan sigma 12.0 (measured 12.00); noisy Y 26.54 dB + BM3D (bm3d r12, r=1, Final) - shipped best 24.0 Y 33.34 U 40.88 0.9 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 10.5 Y 32.17 U 39.85 79.1 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 10.5 Y 32.17 U 39.85 58.1 fps + WNNM vszip (r=0) best 24.0 Y 31.00 U 34.87 94.9 fps + WNNM vszip (r=1) best 24.0 Y 32.92 U 33.65 37.6 fps + BM3D vszip (r=1) best 15.0 Y 31.64 U 37.55 98.9 fps + +== kodim19_pan sigma 4.0 (measured 4.02); noisy Y 36.05 dB + BM3D (bm3d r12, r=1, Final) - shipped best 8.0 Y 40.61 U 46.68 1.0 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 4.5 (edge of sweep) Y 40.62 U 45.58 129.8 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 4.5 (edge of sweep) Y 40.62 U 45.58 53.8 fps + WNNM vszip (r=0) best 8.0 Y 38.65 U 44.02 85.7 fps + WNNM vszip (r=1) best 10.0 (edge of sweep) Y 41.33 U 43.67 61.9 fps + BM3D vszip (r=1) best 5.0 Y 39.15 U 44.79 75.3 fps + +== kodim19_pan sigma 8.0 (measured 8.02); noisy Y 30.05 dB + BM3D (bm3d r12, r=1, Final) - shipped best 12.0 Y 36.30 U 43.19 0.7 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 7.0 Y 35.93 U 42.87 33.3 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 7.0 Y 35.93 U 42.87 44.0 fps + WNNM vszip (r=0) best 16.0 Y 34.47 U 38.98 69.5 fps + WNNM vszip (r=1) best 16.0 Y 36.14 U 38.00 22.9 fps + BM3D vszip (r=1) best 10.0 Y 35.21 U 40.93 22.9 fps + +== kodim19_pan sigma 12.0 (measured 12.02); noisy Y 26.53 dB + BM3D (bm3d r12, r=1, Final) - shipped best 18.0 Y 34.14 U 41.17 0.8 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 10.5 Y 33.24 U 40.85 123.0 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 10.5 Y 33.24 U 40.85 28.9 fps + WNNM vszip (r=0) best 24.0 Y 32.16 U 35.62 81.6 fps + WNNM vszip (r=1) best 24.0 Y 33.20 U 34.41 57.4 fps + BM3D vszip (r=1) best 18.0 Y 32.96 U 40.19 61.6 fps + +== kodim23_pan sigma 4.0 (measured 4.02); noisy Y 36.05 dB + BM3D (bm3d r12, r=1, Final) - shipped best 8.0 Y 43.15 U 47.34 0.8 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 3.5 Y 42.81 U 46.78 118.5 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 3.5 Y 42.81 U 46.78 46.6 fps + WNNM vszip (r=0) best 8.0 Y 41.41 U 44.23 94.6 fps + WNNM vszip (r=1) best 10.0 (edge of sweep) Y 42.62 U 44.13 65.7 fps + BM3D vszip (r=1) best 6.0 Y 41.97 U 45.60 104.5 fps + +== kodim23_pan sigma 8.0 (measured 8.02); noisy Y 30.05 dB + BM3D (bm3d r12, r=1, Final) - shipped best 16.0 Y 40.12 U 44.00 0.9 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 7.0 Y 39.30 U 43.26 85.2 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 7.0 Y 39.30 U 43.26 64.6 fps + WNNM vszip (r=0) best 16.0 Y 37.42 U 39.23 79.0 fps + WNNM vszip (r=1) best 20.0 (edge of sweep) Y 37.87 U 38.42 26.2 fps + BM3D vszip (r=1) best 12.0 Y 38.56 U 41.79 35.9 fps + +== kodim23_pan sigma 12.0 (measured 12.02); noisy Y 26.53 dB + BM3D (bm3d r12, r=1, Final) - shipped best 24.0 Y 38.18 U 41.94 0.8 fps + NLMeans vszip CPU (d=1 a=2 s=4) best 13.5 (edge of sweep) Y 37.24 U 41.14 106.4 fps + KNLMeansCL GPU (d=1 a=2 s=4) best 13.5 (edge of sweep) Y 37.24 U 41.14 44.6 fps + WNNM vszip (r=0) best 24.0 Y 34.82 U 35.89 96.1 fps + WNNM vszip (r=1) best 30.0 (edge of sweep) Y 34.79 U 34.81 45.1 fps + BM3D vszip (r=1) best 18.0 Y 36.47 U 39.39 80.1 fps + +== depth parity: mean |8-bit run - N-bit run|, 8-bit levels (luma) +Traceback (most recent call last): + File "/docs/reports/2026-11-classical-filters/scripts/nr_compare.py", line 157, in + assert mad8(rt, up, 0) == 0, 'float round trip is not exact' + ^^^^^^^^^^^^^^^^^^^^ +AssertionError: float round trip is not exact diff --git a/docs/reports/2026-11-classical-filters/results/nr_parity.txt b/docs/reports/2026-11-classical-filters/results/nr_parity.txt new file mode 100644 index 00000000..6c5cbc8e --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/nr_parity.txt @@ -0,0 +1,10 @@ + +== depth parity: mean |8-bit run - N-bit run|, 8-bit levels (luma) + float round trip at 8-bit: 0.0000 levels + float round trip at 10-bit: 0.0000 levels + float round trip at 16-bit: 0.0000 levels + NLMeans vszip h=2 10-bit 0.005 16-bit 0.006 + WNNM sigma=8 10-bit 0.268 16-bit 0.272 + BM3D template sigma=8 10-bit 0.273 16-bit 0.277 + +vszip NLMeans vs KNLMeansCL, same args: mean |diff| Y 0.0000 U 0.0000 levels diff --git a/docs/reports/2026-11-classical-filters/results/nr_results.json b/docs/reports/2026-11-classical-filters/results/nr_results.json new file mode 100644 index 00000000..14c75505 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/nr_results.json @@ -0,0 +1,390 @@ +{ + "nfr": 10, + "fixtures": { + "kodim08_pan": { + "4.0": { + "measured_sigma": 4.02, + "noisy_psnr_y": 36.05, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 8.0, + "psnr_y": 40.44, + "psnr_u": 45.77, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 3.5, + "psnr_y": 39.93, + "psnr_u": 44.78, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 3.5, + "psnr_y": 39.93, + "psnr_u": 44.78, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 8.0, + "psnr_y": 37.8, + "psnr_u": 43.23, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 8.0, + "psnr_y": 41.01, + "psnr_u": 43.17, + "edge_of_sweep": false + }, + "BM3D vszip (r=1)": { + "best_strength": 5.0, + "psnr_y": 38.24, + "psnr_u": 43.95, + "edge_of_sweep": false + } + } + }, + "8.0": { + "measured_sigma": 8.01, + "noisy_psnr_y": 30.05, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 16.0, + "psnr_y": 35.89, + "psnr_u": 42.67, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 7.0, + "psnr_y": 34.84, + "psnr_u": 41.68, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 7.0, + "psnr_y": 34.84, + "psnr_u": 41.68, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 16.0, + "psnr_y": 33.43, + "psnr_u": 38.25, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 16.0, + "psnr_y": 35.85, + "psnr_u": 37.31, + "edge_of_sweep": false + }, + "BM3D vszip (r=1)": { + "best_strength": 10.0, + "psnr_y": 34.0, + "psnr_u": 40.0, + "edge_of_sweep": false + } + } + }, + "12.0": { + "measured_sigma": 12.0, + "noisy_psnr_y": 26.54, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 24.0, + "psnr_y": 33.34, + "psnr_u": 40.88, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 10.5, + "psnr_y": 32.17, + "psnr_u": 39.85, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 10.5, + "psnr_y": 32.17, + "psnr_u": 39.85, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 24.0, + "psnr_y": 31.0, + "psnr_u": 34.87, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 24.0, + "psnr_y": 32.92, + "psnr_u": 33.65, + "edge_of_sweep": false + }, + "BM3D vszip (r=1)": { + "best_strength": 15.0, + "psnr_y": 31.64, + "psnr_u": 37.55, + "edge_of_sweep": false + } + } + } + }, + "kodim19_pan": { + "4.0": { + "measured_sigma": 4.02, + "noisy_psnr_y": 36.05, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 8.0, + "psnr_y": 40.61, + "psnr_u": 46.68, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 4.5, + "psnr_y": 40.62, + "psnr_u": 45.58, + "edge_of_sweep": true + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 4.5, + "psnr_y": 40.62, + "psnr_u": 45.58, + "edge_of_sweep": true + }, + "WNNM vszip (r=0)": { + "best_strength": 8.0, + "psnr_y": 38.65, + "psnr_u": 44.02, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 10.0, + "psnr_y": 41.33, + "psnr_u": 43.67, + "edge_of_sweep": true + }, + "BM3D vszip (r=1)": { + "best_strength": 5.0, + "psnr_y": 39.15, + "psnr_u": 44.79, + "edge_of_sweep": false + } + } + }, + "8.0": { + "measured_sigma": 8.02, + "noisy_psnr_y": 30.05, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 12.0, + "psnr_y": 36.3, + "psnr_u": 43.19, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 7.0, + "psnr_y": 35.93, + "psnr_u": 42.87, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 7.0, + "psnr_y": 35.93, + "psnr_u": 42.87, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 16.0, + "psnr_y": 34.47, + "psnr_u": 38.98, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 16.0, + "psnr_y": 36.14, + "psnr_u": 38.0, + "edge_of_sweep": false + }, + "BM3D vszip (r=1)": { + "best_strength": 10.0, + "psnr_y": 35.21, + "psnr_u": 40.93, + "edge_of_sweep": false + } + } + }, + "12.0": { + "measured_sigma": 12.02, + "noisy_psnr_y": 26.53, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 18.0, + "psnr_y": 34.14, + "psnr_u": 41.17, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 10.5, + "psnr_y": 33.24, + "psnr_u": 40.85, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 10.5, + "psnr_y": 33.24, + "psnr_u": 40.85, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 24.0, + "psnr_y": 32.16, + "psnr_u": 35.62, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 24.0, + "psnr_y": 33.2, + "psnr_u": 34.41, + "edge_of_sweep": false + }, + "BM3D vszip (r=1)": { + "best_strength": 18.0, + "psnr_y": 32.96, + "psnr_u": 40.19, + "edge_of_sweep": false + } + } + } + }, + "kodim23_pan": { + "4.0": { + "measured_sigma": 4.02, + "noisy_psnr_y": 36.05, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 8.0, + "psnr_y": 43.15, + "psnr_u": 47.34, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 3.5, + "psnr_y": 42.81, + "psnr_u": 46.78, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 3.5, + "psnr_y": 42.81, + "psnr_u": 46.78, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 8.0, + "psnr_y": 41.41, + "psnr_u": 44.23, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 10.0, + "psnr_y": 42.62, + "psnr_u": 44.13, + "edge_of_sweep": true + }, + "BM3D vszip (r=1)": { + "best_strength": 6.0, + "psnr_y": 41.97, + "psnr_u": 45.6, + "edge_of_sweep": false + } + } + }, + "8.0": { + "measured_sigma": 8.02, + "noisy_psnr_y": 30.05, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 16.0, + "psnr_y": 40.12, + "psnr_u": 44.0, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 7.0, + "psnr_y": 39.3, + "psnr_u": 43.26, + "edge_of_sweep": false + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 7.0, + "psnr_y": 39.3, + "psnr_u": 43.26, + "edge_of_sweep": false + }, + "WNNM vszip (r=0)": { + "best_strength": 16.0, + "psnr_y": 37.42, + "psnr_u": 39.23, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 20.0, + "psnr_y": 37.87, + "psnr_u": 38.42, + "edge_of_sweep": true + }, + "BM3D vszip (r=1)": { + "best_strength": 12.0, + "psnr_y": 38.56, + "psnr_u": 41.79, + "edge_of_sweep": false + } + } + }, + "12.0": { + "measured_sigma": 12.02, + "noisy_psnr_y": 26.53, + "methods": { + "BM3D (bm3d r12, r=1, Final) - shipped": { + "best_strength": 24.0, + "psnr_y": 38.18, + "psnr_u": 41.94, + "edge_of_sweep": false + }, + "NLMeans vszip CPU (d=1 a=2 s=4)": { + "best_strength": 13.5, + "psnr_y": 37.24, + "psnr_u": 41.14, + "edge_of_sweep": true + }, + "KNLMeansCL GPU (d=1 a=2 s=4)": { + "best_strength": 13.5, + "psnr_y": 37.24, + "psnr_u": 41.14, + "edge_of_sweep": true + }, + "WNNM vszip (r=0)": { + "best_strength": 24.0, + "psnr_y": 34.82, + "psnr_u": 35.89, + "edge_of_sweep": false + }, + "WNNM vszip (r=1)": { + "best_strength": 30.0, + "psnr_y": 34.79, + "psnr_u": 34.81, + "edge_of_sweep": true + }, + "BM3D vszip (r=1)": { + "best_strength": 18.0, + "psnr_y": 36.47, + "psnr_u": 39.39, + "edge_of_sweep": false + } + } + } + } + }, + "source": "parsed from nr_compare.txt" +} \ No newline at end of file diff --git a/docs/reports/2026-11-classical-filters/results/ports_compare.txt b/docs/reports/2026-11-classical-filters/results/ports_compare.txt new file mode 100644 index 00000000..b680f3ab --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/ports_compare.txt @@ -0,0 +1,70 @@ +vine kodim08_pan: haloed (no filter) Y 21.93 dB 0.0 fps +vine kodim08_pan: vine (16, 0.5, 1.0) Y 22.80 dB 14.2 fps +vine kodim08_pan: vine (24, 0.5, 1.0) Y 23.06 dB 0.0 fps +vine kodim08_pan: vine (8, 0.5, 1.0) Y 22.45 dB 0.0 fps +vine kodim08_pan: DeHalo_alpha (defaults) Y 26.28 dB 1201.2 fps +vine kodim08_pan: FineDehalo (defaults) Y 22.00 dB 884.5 fps +vine kodim23_pan: haloed (no filter) Y 31.99 dB 0.0 fps +vine kodim23_pan: vine (16, 0.5, 1.0) Y 33.12 dB 55.4 fps +vine kodim23_pan: vine (24, 0.5, 1.0) Y 33.26 dB 0.0 fps +vine kodim23_pan: vine (8, 0.5, 1.0) Y 32.83 dB 0.0 fps +vine kodim23_pan: DeHalo_alpha (defaults) Y 36.11 dB 1221.4 fps +vine kodim23_pan: FineDehalo (defaults) Y 32.25 dB 853.2 fps +mpeg2stinx kodim08 telecined @1000k: decoded (no filter) Y 33.30 U 41.61 +mpeg2stinx kodim08 telecined @1000k: mpeg2stinx (TFF) Y 31.54 U 41.68 +mpeg2stinx kodim08 telecined @1000k: mpeg2stinx (wrong order: BFF) Y 31.53 U 41.68 +mpeg2stinx kodim08 telecined @1000k: Deblock_QED (defaults) Y 33.24 U 41.77 +mpeg2stinx kodim08 telecined @1000k: Deblock (quant 25) Y 33.27 U 41.88 +mpeg2stinx kodim08 progressive-as-interlaced @1000k: decoded (no filter) Y 42.17 U 46.13 +mpeg2stinx kodim08 progressive-as-interlaced @1000k: mpeg2stinx (TFF) Y 38.29 U 46.21 +mpeg2stinx kodim08 progressive-as-interlaced @1000k: mpeg2stinx (wrong order: BFF) Y 38.26 U 46.21 +mpeg2stinx kodim08 progressive-as-interlaced @1000k: Deblock_QED (defaults) Y 41.99 U 45.95 +mpeg2stinx kodim08 progressive-as-interlaced @1000k: Deblock (quant 25) Y 42.07 U 46.06 +mpeg2stinx kodim08 telecined @2000k: decoded (no filter) Y 39.32 U 44.60 +mpeg2stinx kodim08 telecined @2000k: mpeg2stinx (TFF) Y 36.31 U 44.71 +mpeg2stinx kodim08 telecined @2000k: mpeg2stinx (wrong order: BFF) Y 36.29 U 44.71 +mpeg2stinx kodim08 telecined @2000k: Deblock_QED (defaults) Y 39.24 U 44.61 +mpeg2stinx kodim08 telecined @2000k: Deblock (quant 25) Y 39.29 U 44.78 +mpeg2stinx kodim08 progressive-as-interlaced @2000k: decoded (no filter) Y 42.39 U 46.20 +mpeg2stinx kodim08 progressive-as-interlaced @2000k: mpeg2stinx (TFF) Y 38.38 U 46.28 +mpeg2stinx kodim08 progressive-as-interlaced @2000k: mpeg2stinx (wrong order: BFF) Y 38.36 U 46.28 +mpeg2stinx kodim08 progressive-as-interlaced @2000k: Deblock_QED (defaults) Y 42.18 U 46.01 +mpeg2stinx kodim08 progressive-as-interlaced @2000k: Deblock (quant 25) Y 42.28 U 46.13 +mpeg2stinx kodim23 telecined @1000k: decoded (no filter) Y 43.35 U 47.01 +mpeg2stinx kodim23 telecined @1000k: mpeg2stinx (TFF) Y 43.08 U 47.07 +mpeg2stinx kodim23 telecined @1000k: mpeg2stinx (wrong order: BFF) Y 43.06 U 47.07 +mpeg2stinx kodim23 telecined @1000k: Deblock_QED (defaults) Y 43.22 U 47.12 +mpeg2stinx kodim23 telecined @1000k: Deblock (quant 25) Y 43.25 U 47.27 +mpeg2stinx kodim23 progressive-as-interlaced @1000k: decoded (no filter) Y 44.19 U 47.60 +mpeg2stinx kodim23 progressive-as-interlaced @1000k: mpeg2stinx (TFF) Y 43.90 U 47.65 +mpeg2stinx kodim23 progressive-as-interlaced @1000k: mpeg2stinx (wrong order: BFF) Y 43.88 U 47.65 +mpeg2stinx kodim23 progressive-as-interlaced @1000k: Deblock_QED (defaults) Y 44.00 U 47.75 +mpeg2stinx kodim23 progressive-as-interlaced @1000k: Deblock (quant 25) Y 44.01 U 47.89 +mpeg2stinx kodim23 telecined @2000k: decoded (no filter) Y 44.33 U 47.66 +mpeg2stinx kodim23 telecined @2000k: mpeg2stinx (TFF) Y 43.99 U 47.74 +mpeg2stinx kodim23 telecined @2000k: mpeg2stinx (wrong order: BFF) Y 43.96 U 47.74 +mpeg2stinx kodim23 telecined @2000k: Deblock_QED (defaults) Y 44.09 U 47.73 +mpeg2stinx kodim23 telecined @2000k: Deblock (quant 25) Y 44.12 U 47.86 +mpeg2stinx kodim23 progressive-as-interlaced @2000k: decoded (no filter) Y 44.19 U 47.60 +mpeg2stinx kodim23 progressive-as-interlaced @2000k: mpeg2stinx (TFF) Y 43.90 U 47.65 +mpeg2stinx kodim23 progressive-as-interlaced @2000k: mpeg2stinx (wrong order: BFF) Y 43.88 U 47.65 +mpeg2stinx kodim23 progressive-as-interlaced @2000k: Deblock_QED (defaults) Y 44.00 U 47.75 +mpeg2stinx kodim23 progressive-as-interlaced @2000k: Deblock (quant 25) Y 44.01 U 47.89 +fades: { + "field_gap_faded": 5.27347265625, + "field_gap_fixed": 0.2856132812500012, + "field_gap_fixed_match": 3.841145833617665e-05, + "field_gap_clean": 0.0010820312500012808, + "psnr_faded_vs_truth": 38.972162613525306, + "psnr_fixed_vs_truth": 52.9205588884955, + "unfaded_change_mad": 0.0, + "unfaded_change_mad_mpeg2": 0.0, + "fps": 478.61060078735665 +} +AA daa 53.9 fps, mean change 3.41 +AA santiag (nnedi3, 1/1) 20.9 fps, mean change 4.18 +AA SangNom AA (48) 60.1 fps, mean change 4.11 +parity vine 10-bit 0.450 16-bit 0.455 +parity mpeg2stinx 10-bit 0.009 16-bit 0.010 +parity FixInterlacedFades 10-bit 0.251 16-bit 0.257 +parity SangNom AA 10-bit 0.436 16-bit 0.476 diff --git a/docs/reports/2026-11-classical-filters/results/ports_results.json b/docs/reports/2026-11-classical-filters/results/ports_results.json new file mode 100644 index 00000000..79b21d53 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/ports_results.json @@ -0,0 +1,279 @@ +{ + "vine": { + "kodim08_pan": { + "haloed (no filter)": { + "psnr_y": 21.933134712608727 + }, + "vine (16, 0.5, 1.0)": { + "psnr_y": 22.797082828237052, + "fps": 14.21292564101782 + }, + "vine (24, 0.5, 1.0)": { + "psnr_y": 23.0619950719995 + }, + "vine (8, 0.5, 1.0)": { + "psnr_y": 22.453524109191623 + }, + "DeHalo_alpha (defaults)": { + "psnr_y": 26.277493614694684, + "fps": 1201.2211816593635 + }, + "FineDehalo (defaults)": { + "psnr_y": 22.0037554682307, + "fps": 884.4824023112124 + } + }, + "kodim23_pan": { + "haloed (no filter)": { + "psnr_y": 31.98589798603438 + }, + "vine (16, 0.5, 1.0)": { + "psnr_y": 33.11719114650669, + "fps": 55.43943871967178 + }, + "vine (24, 0.5, 1.0)": { + "psnr_y": 33.25691627829084 + }, + "vine (8, 0.5, 1.0)": { + "psnr_y": 32.82610218076776 + }, + "DeHalo_alpha (defaults)": { + "psnr_y": 36.10776928033571, + "fps": 1221.4403447974605 + }, + "FineDehalo (defaults)": { + "psnr_y": 32.24800621416794, + "fps": 853.2465366071973 + } + } + }, + "mpeg2stinx": { + "kodim08 telecined @1000k": { + "decoded (no filter)": { + "psnr_y": 33.29538368376405, + "psnr_u": 41.61371834325565 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 31.5384839201852, + "psnr_u": 41.67942121155347, + "fps": 32.77849936307724 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 31.529313223774377, + "psnr_u": 41.67841814310286 + }, + "Deblock_QED (defaults)": { + "psnr_y": 33.23652487333629, + "psnr_u": 41.7661195749458 + }, + "Deblock (quant 25)": { + "psnr_y": 33.273421092982424, + "psnr_u": 41.87678402726829 + } + }, + "kodim08 progressive-as-interlaced @1000k": { + "decoded (no filter)": { + "psnr_y": 42.17042351197701, + "psnr_u": 46.12772009077852 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 38.28563393500911, + "psnr_u": 46.20638031515618, + "fps": 33.83420023490236 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 38.25890561044511, + "psnr_u": 46.20539825836086 + }, + "Deblock_QED (defaults)": { + "psnr_y": 41.985965749803604, + "psnr_u": 45.952495619483884 + }, + "Deblock (quant 25)": { + "psnr_y": 42.07005602997155, + "psnr_u": 46.06458855197737 + } + }, + "kodim08 telecined @2000k": { + "decoded (no filter)": { + "psnr_y": 39.316846574844575, + "psnr_u": 44.5996358673874 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 36.310117129411786, + "psnr_u": 44.71130472527608, + "fps": 32.142649683539695 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 36.294308619850526, + "psnr_u": 44.70883283607603 + }, + "Deblock_QED (defaults)": { + "psnr_y": 39.24210307098453, + "psnr_u": 44.60998398058343 + }, + "Deblock (quant 25)": { + "psnr_y": 39.289751955392695, + "psnr_u": 44.77698864169139 + } + }, + "kodim08 progressive-as-interlaced @2000k": { + "decoded (no filter)": { + "psnr_y": 42.38708793130139, + "psnr_u": 46.19888749957376 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 38.38442160526226, + "psnr_u": 46.278817013176415, + "fps": 35.60378591740588 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 38.35863219416915, + "psnr_u": 46.27792290870529 + }, + "Deblock_QED (defaults)": { + "psnr_y": 42.184874187058085, + "psnr_u": 46.01040531606005 + }, + "Deblock (quant 25)": { + "psnr_y": 42.27585808977774, + "psnr_u": 46.12996339886284 + } + }, + "kodim23 telecined @1000k": { + "decoded (no filter)": { + "psnr_y": 43.351684182023, + "psnr_u": 47.00520084461393 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 43.075304671267595, + "psnr_u": 47.07324243122878, + "fps": 139.27346624340873 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 43.05620409633267, + "psnr_u": 47.073004520420376 + }, + "Deblock_QED (defaults)": { + "psnr_y": 43.21752044052841, + "psnr_u": 47.124156664486456 + }, + "Deblock (quant 25)": { + "psnr_y": 43.24926678121209, + "psnr_u": 47.26724623836998 + } + }, + "kodim23 progressive-as-interlaced @1000k": { + "decoded (no filter)": { + "psnr_y": 44.19061404946899, + "psnr_u": 47.601377887411346 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 43.89827824541628, + "psnr_u": 47.65378552415274, + "fps": 137.27961247667986 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 43.882195202328795, + "psnr_u": 47.652395446575476 + }, + "Deblock_QED (defaults)": { + "psnr_y": 43.997027638305916, + "psnr_u": 47.74729014411055 + }, + "Deblock (quant 25)": { + "psnr_y": 44.0123657215253, + "psnr_u": 47.890436524807704 + } + }, + "kodim23 telecined @2000k": { + "decoded (no filter)": { + "psnr_y": 44.325646142417064, + "psnr_u": 47.6599805275344 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 43.98534339248991, + "psnr_u": 47.73734149836887, + "fps": 137.4248952350372 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 43.95996432355227, + "psnr_u": 47.73705551346618 + }, + "Deblock_QED (defaults)": { + "psnr_y": 44.08857576993642, + "psnr_u": 47.732755056861045 + }, + "Deblock (quant 25)": { + "psnr_y": 44.11728596679578, + "psnr_u": 47.85766363275266 + } + }, + "kodim23 progressive-as-interlaced @2000k": { + "decoded (no filter)": { + "psnr_y": 44.19061404946899, + "psnr_u": 47.601377887411346 + }, + "mpeg2stinx (TFF)": { + "psnr_y": 43.89827824541628, + "psnr_u": 47.65378552415274, + "fps": 142.50296434966074 + }, + "mpeg2stinx (wrong order: BFF)": { + "psnr_y": 43.882195202328795, + "psnr_u": 47.652395446575476 + }, + "Deblock_QED (defaults)": { + "psnr_y": 43.997027638305916, + "psnr_u": 47.74729014411055 + }, + "Deblock (quant 25)": { + "psnr_y": 44.0123657215253, + "psnr_u": 47.890436524807704 + } + } + }, + "fades": { + "field_gap_faded": 5.27347265625, + "field_gap_fixed": 0.2856132812500012, + "field_gap_fixed_match": 3.841145833617665e-05, + "field_gap_clean": 0.0010820312500012808, + "psnr_faded_vs_truth": 38.972162613525306, + "psnr_fixed_vs_truth": 52.9205588884955, + "unfaded_change_mad": 0.0, + "unfaded_change_mad_mpeg2": 0.0, + "fps": 478.61060078735665 + }, + "sangnom": { + "daa": { + "fps": 53.89259011321249, + "change_vs_src": 3.4142727864583335 + }, + "santiag (nnedi3, 1/1)": { + "fps": 20.92812553171, + "change_vs_src": 4.182309895833333 + }, + "SangNom AA (48)": { + "fps": 60.099355920303196, + "change_vs_src": 4.105525065104167 + } + }, + "parity": { + "vine": { + "10": 0.4503635660807292, + "16": 0.4546001211802165 + }, + "mpeg2stinx": { + "10": 0.009453328450520833, + "16": 0.010067272186279296 + }, + "FixInterlacedFades": { + "10": 0.2506402587890625, + "16": 0.2568229039510091 + }, + "SangNom AA": { + "10": 0.43591512044270836, + "16": 0.4755617618560791 + } + } +} \ No newline at end of file diff --git a/docs/reports/2026-11-classical-filters/results/wnnm_wide.txt b/docs/reports/2026-11-classical-filters/results/wnnm_wide.txt new file mode 100644 index 00000000..3615892d --- /dev/null +++ b/docs/reports/2026-11-classical-filters/results/wnnm_wide.txt @@ -0,0 +1,18 @@ +kodim08_pan sigma 4.0 WNNM r=0: best 8 -> Y 37.80 U 43.23 sweep [(8.0, 37.8), (10.0, 37.68), (12.0, 37.18), (14.0, 36.53), (16.0, 35.85)] +kodim08_pan sigma 4.0 WNNM r=1: best 8 -> Y 41.01 U 43.17 sweep [(8.0, 41.01), (10.0, 40.9), (12.0, 40.5), (14.0, 40.03), (16.0, 39.52)] +kodim08_pan sigma 8.0 WNNM r=0: best 16 -> Y 33.43 U 38.25 sweep [(16.0, 33.43), (20.0, 33.2), (24.0, 32.59), (28.0, 31.92), (32.0, 31.27)] +kodim08_pan sigma 8.0 WNNM r=1: best 16 -> Y 35.85 U 37.31 sweep [(16.0, 35.85), (20.0, 35.8), (24.0, 35.42), (28.0, 34.96), (32.0, 34.48)] +kodim08_pan sigma 12.0 WNNM r=0: best 24 -> Y 31.00 U 34.87 sweep [(24.0, 31.0), (30.0, 30.7), (36.0, 30.08), (42.0, 29.48), (48.0, 28.97)] +kodim08_pan sigma 12.0 WNNM r=1: best 24 -> Y 32.92 U 33.65 sweep [(24.0, 32.92), (30.0, 32.86), (36.0, 32.51), (42.0, 32.12), (48.0, 31.76)] +kodim19_pan sigma 4.0 WNNM r=0: best 8 -> Y 38.65 U 44.02 sweep [(8.0, 38.65), (10.0, 38.58), (12.0, 37.83), (14.0, 36.81), (16.0, 35.87)] +kodim19_pan sigma 4.0 WNNM r=1: best 10 -> Y 41.33 U 43.67 sweep [(8.0, 41.33), (10.0, 41.33), (12.0, 40.92), (14.0, 40.19), (16.0, 39.38)] +kodim19_pan sigma 8.0 WNNM r=0: best 16 -> Y 34.47 U 38.98 sweep [(16.0, 34.47), (20.0, 33.9), (24.0, 33.37), (28.0, 33.03), (32.0, 32.77)] +kodim19_pan sigma 8.0 WNNM r=1: best 16 -> Y 36.14 U 38.00 sweep [(16.0, 36.14), (20.0, 35.84), (24.0, 35.42), (28.0, 35.17), (32.0, 34.99)] +kodim19_pan sigma 12.0 WNNM r=0: best 24 -> Y 32.16 U 35.62 sweep [(24.0, 32.16), (30.0, 31.89), (36.0, 31.63), (42.0, 31.42), (48.0, 31.25)] +kodim19_pan sigma 12.0 WNNM r=1: best 24 -> Y 33.20 U 34.41 sweep [(24.0, 33.2), (30.0, 33.14), (36.0, 33.01), (42.0, 32.9), (48.0, 32.82)] +kodim23_pan sigma 4.0 WNNM r=0: best 8 -> Y 41.41 U 44.23 sweep [(8.0, 41.41), (10.0, 41.29), (12.0, 40.98), (14.0, 40.69), (16.0, 40.4)] +kodim23_pan sigma 4.0 WNNM r=1: best 10 -> Y 42.62 U 44.13 sweep [(8.0, 42.59), (10.0, 42.62), (12.0, 42.48), (14.0, 42.34), (16.0, 42.2)] +kodim23_pan sigma 8.0 WNNM r=0: best 16 -> Y 37.42 U 39.23 sweep [(16.0, 37.42), (20.0, 37.37), (24.0, 37.19), (28.0, 36.99), (32.0, 36.78)] +kodim23_pan sigma 8.0 WNNM r=1: best 20 -> Y 37.87 U 38.42 sweep [(16.0, 37.81), (20.0, 37.87), (24.0, 37.81), (28.0, 37.74), (32.0, 37.64)] +kodim23_pan sigma 12.0 WNNM r=0: best 24 -> Y 34.82 U 35.89 sweep [(24.0, 34.82), (30.0, 34.78), (36.0, 34.64), (42.0, 34.47), (48.0, 34.31)] +kodim23_pan sigma 12.0 WNNM r=1: best 30 -> Y 34.79 U 34.81 sweep [(24.0, 34.73), (30.0, 34.79), (36.0, 34.74), (42.0, 34.68), (48.0, 34.62)] diff --git a/docs/reports/2026-11-classical-filters/scripts/bench.py b/docs/reports/2026-11-classical-filters/scripts/bench.py new file mode 100644 index 00000000..60dda5ba --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/bench.py @@ -0,0 +1,91 @@ +"""Speed of each method, measured the way a job runs: vspipe, with the +graph's own parallelism, writing nowhere. One fresh vspipe per measurement +(so nothing is cached between methods), each run RUNS times, median kept. + + SCRATCH=... ./run.sh bench.py + +Methods at the strengths nr_compare.py found best for sigma 8 on kodim08. +The source is the 640x480 Kodak pan (30 frames) with noise added, and the +same pan scaled to 1920x1080 (its first 10 frames) for an HD figure. +""" +import json +import os +import re +import statistics +import subprocess +import sys +import tempfile + +HERE = os.path.dirname(os.path.abspath(__file__)) +S = os.environ['SCRATCH'] +REPO = os.environ.get('REPO', os.path.abspath(os.path.join(HERE, '..', '..', '..', '..'))) +RUNS = int(os.environ.get('RUNS', '3')) + +PRELUDE = r''' +import os, sys +sys.path.insert(0, {here!r}) +os.environ['SCRATCH'] = {scratch!r} +os.environ['REPO'] = {repo!r} +import vapoursynth as vs +core = vs.core +from probelib import load +src = load(os.path.join({scratch!r}, 'fx', 'kodim08_pan.mkv'), 'yuv420p', frames=30) +if {hd}: + src = core.resize.Spline36(src[:10], 1920, 1080) # 10 frames: BM3D runs ~0.1 fps at HD here +noisy = core.grain.Add(src, var=64, uvar=64, seed=7) +# the shipped template blocks, as ports_compare.py runs them +_pc = open(os.path.join({here!r}, 'ports_compare.py')).read() +_pc = _pc[:_pc.index('# ---- vine_dehalo')].replace('os.path.dirname(os.path.abspath(__file__))', repr({here!r})) +exec(_pc) +''' + +NR = open(os.path.join(HERE, 'nr_compare.py')).read() +NR_FUNCS = NR[NR.index('def bm3d_template'):NR.index('MULT = ')] + +METHODS = { + 'BM3D (bm3d r12, r=1, Final, sigma 16) - shipped': 'bm3d_template(noisy, 16.0)', + 'BM3D (bm3d r12, r=0, Final, sigma 16)': 'bm3d_template(noisy, 16.0, radius=0)', + 'NLMeans vszip CPU (d=1 a=2 s=4, h 7)': 'nlm_vszip(noisy, 7.0)', + 'KNLMeansCL GPU (d=1 a=2 s=4, h 7)': 'nlm_knlm(noisy, 7.0)', + 'WNNM vszip (r=0, sigma 16)': 'wnnm(noisy, 16.0, 0)', + 'WNNM vszip (r=1, sigma 16)': 'wnnm(noisy, 16.0, 1)', + 'BM3D vszip (r=1, sigma 10)': 'bm3d_vszip(noisy, 10.0)', + 'input only (decode + noise)': 'noisy', + # The other new blocks, on the clean pan. + 'AA: daa': 'haf.daa(core.std.SetFieldBased(src, 0))', + 'AA: santiag (nnedi3, strength 1/1)': 'haf.santiag(core.std.SetFieldBased(src, 0))', + 'AA: SangNom (aa 48)': "run_block('AA_SANGNOM', src, {'AA_SANGNOM_AA': 48, 'AA_FIELD_BASED_VALUE': 0})", + 'Dehalo: DeHalo_alpha (defaults)': 'haf.DeHalo_alpha(src)', + 'Dehalo: vine (16, 0.5, 1.0)': "run_block('DEHALO_VINE', src, {'DEHALO_VINE_STRENGTH': 16.0, " + "'DEHALO_VINE_SHARP': 0.5, 'DEHALO_VINE_SIGMA': 1.0})", + 'Deblock: Deblock_QED (defaults)': 'haf.Deblock_QED(src)', + 'Deblock: mpeg2stinx': "run_block('DEBLOCK_MPEG2STINX', src, {'DEBLOCK_STINX_TFF': True})", + 'IVTC fade fix (average), alone': "run_block('DEINT_IVTC_FIX_FADES', src, {'IVTC_FADE_MODE': 'average'})", +} + + +def vspipe_fps(body, hd): + code = PRELUDE.format(here=HERE, scratch=S, repo=REPO, hd=hd) + NR_FUNCS + '\nclip = ' + body + '\nclip.set_output()\n' + with tempfile.NamedTemporaryFile('w', suffix='.vpy', dir=S, delete=False) as f: + f.write(code) + path = f.name + try: + # Bounded: the first fade-fix block hung vspipe for 97 minutes here. + r = subprocess.run(['vspipe', '-p', path, '--'], capture_output=True, text=True, timeout=900) + m = re.search(r'Output (\d+) frames in ([\d.]+) seconds \(([\d.]+) fps\)', r.stderr) + if not m: + raise RuntimeError(r.stderr[-800:]) + return float(m.group(3)) + finally: + os.unlink(path) + + +out = {} +for hd in (False, True): + label = '1920x1080' if hd else '640x480' + out[label] = {} + for name, body in METHODS.items(): + runs = [vspipe_fps(body, hd) for _ in range(RUNS)] + out[label][name] = {'median_fps': statistics.median(runs), 'runs': runs} + print(f'{label} {name:50s} {statistics.median(runs):7.1f} fps {runs}', flush=True) +json.dump(out, open(os.path.join(S, 'bench_results.json'), 'w'), indent=1) diff --git a/docs/reports/2026-11-classical-filters/scripts/fill_report.py b/docs/reports/2026-11-classical-filters/scripts/fill_report.py new file mode 100644 index 00000000..6c5d09c8 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/fill_report.py @@ -0,0 +1,35 @@ +"""Fills index.html's table markers from tables.py's output, so every number +in a table comes straight from the probes' results. + + python3 fill_report.py (rewrites ../index.html in place; idempotent) +""" +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +html_path = os.path.join(HERE, '..', 'index.html') +out = subprocess.run([sys.executable, os.path.join(HERE, 'tables.py')], capture_output=True, text=True, + check=True).stdout + +# Split into blocks: each ...
, and loose

lines. +blocks = re.findall(r'(.*?
|

.*?

)', out, re.S) +SECTIONS = { + 'NR_TABLES': lambda b: 'luma / chroma (U) PSNR' in b or 'Depth parity: mean |8-bit run' in b + or 'vszip NLMeans vs KNLMeansCL' in b, + 'BENCH': lambda b: 'Speed' in b, + 'AA_TABLES': lambda b: 'Anti-aliasing' in b, + 'FADE_TABLES': lambda b: 'IVTC fade fix' in b, + 'VINE_TABLES': lambda b: 'Dehalo' in b, + 'STINX_TABLES': lambda b: 'mpeg2stinx' in b, + 'PARITY_TABLES': lambda b: 'Ports, depth parity' in b, +} +html = open(html_path).read() +for marker, pick in SECTIONS.items(): + body = '\n'.join(f'
{b}
' if b.startswith('.*?|', re.S) + html = pattern.sub(lambda _m: f'\n{body}\n', html) +open(html_path, 'w').write(html) +print('filled', ', '.join(SECTIONS)) diff --git a/docs/reports/2026-11-classical-filters/scripts/images.py b/docs/reports/2026-11-classical-filters/scripts/images.py new file mode 100644 index 00000000..58301ab7 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/images.py @@ -0,0 +1,82 @@ +"""The report's comparison images: crops side by side, 2x nearest-neighbour, +panels in the order each caption gives. Writes into the report's img/. + + SCRATCH=... ./run.sh images.py +""" +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.argv = [sys.argv[0]] +os.environ.setdefault('NFR', '24') +S = os.environ['SCRATCH'] +IMG = os.path.join(HERE, '..', 'img') +src = open(os.path.join(HERE, 'ports_compare.py')).read() +exec(src[:src.index('# ---- vine_dehalo')]) +exec(src[src.index('def gauss('):src.index("out['vine'] = {}")]) +exec(src[src.index('def stinx('):src.index("out['mpeg2stinx'] = {}")]) +exec(src[src.index('def ivtc('):src.index('def stinx_rows(')]) +exec(src[src.index('def fades('):src.index('def field_gap(')]) +nr = open(os.path.join(HERE, 'nr_compare.py')).read() +exec(nr[nr.index('def bm3d_template'):nr.index('MULT = ')]) + + +def panels(clips, frame, x, y, w, h, scale=2): + out = [] + for c in clips: + c = core.std.CropAbs(c[frame], w, h, x, y) + c = core.resize.Point(c, w * scale, h * scale, format=vs.RGB24, matrix_in_s='170m') + out.append(c) + out.append(core.std.BlankClip(c, width=4, color=[255, 255, 255])) + return core.std.StackHorizontal(out[:-1]) + + +def save(clip, name): + f = clip.get_frame(0) + data = b''.join(bytes(f[p]) for p in (1, 2, 0)) # gbrp plane order + subprocess.run(['ffmpeg', '-v', 'error', '-y', '-f', 'rawvideo', '-pix_fmt', 'gbrp', + '-s', f'{clip.width}x{clip.height}', '-i', '-', os.path.join(IMG, name)], + input=data, check=True) + print('wrote', name, clip.width, 'x', clip.height) + + +os.makedirs(IMG, exist_ok=True) + +# 1. noise reduction, kodim08 sigma 8: clean | noisy | BM3D | NLMeans | WNNM r=1 +clean = load(os.path.join(S, 'fx', 'kodim08_pan.mkv'), 'yuv420p', frames=6) +noisy = core.grain.Add(clean, var=64, uvar=64, seed=7) +save(panels([clean, noisy, bm3d_template(noisy, 16.0), nlm_vszip(noisy, 7.0), wnnm(noisy, 16.0, 1)], + 3, 300, 150, 150, 130), '01-nr-kodim08-sigma8.png') + +# 2. dehalo, kodim23: clean | haloed | vine | DeHalo_alpha | FineDehalo +clean = load(os.path.join(S, 'fx', 'kodim23_pan.mkv'), 'yuv420p', frames=2) +halo = expr([clean, gauss(clean, 1.5)], 'x x y - 1.5 * +') +save(panels([clean, halo, vine(halo), haf.DeHalo_alpha(halo), haf.FineDehalo(halo)], + 0, 250, 120, 150, 130), '02-dehalo-kodim23.png') +clean = load(os.path.join(S, 'fx', 'kodim08_pan.mkv'), 'yuv420p', frames=2) +halo = expr([clean, gauss(clean, 1.5)], 'x x y - 1.5 * +') +save(panels([clean, halo, vine(halo), haf.DeHalo_alpha(halo), haf.FineDehalo(halo)], + 0, 260, 200, 150, 130), '03-dehalo-kodim08.png') + +# 3. mpeg2stinx, kodim08 telecined @1000k after IVTC: clean | decoded | mpeg2stinx | Deblock_QED +clean = load(os.path.join(S, 'fx', 'kodim08_pan.mkv'), 'yuv420p', frames=24) +enc = ivtc(load(os.path.join(S, 'fx', 'kodim08_tc_1000.mkv'), 'yuv420p', frames=30)) +save(panels([clean, enc, stinx(enc), haf.Deblock_QED(enc)], 10, 300, 150, 150, 130), + '04-mpeg2stinx-kodim08-1000k.png') + +# 4. anti-aliasing: a point-resampled (aliased) kodim23 | daa | santiag | SangNom AA +clean = load(os.path.join(S, 'fx', 'kodim23_pan.mkv'), 'yuv420p', frames=2) +aliased = core.resize.Point(core.resize.Point(clean, 320, 240), 640, 480) +aliased = core.std.SetFieldBased(aliased, 0) +save(panels([aliased, haf.daa(aliased), haf.santiag(aliased), + run_block('AA_SANGNOM', aliased, {'AA_SANGNOM_AA': 48, 'AA_FIELD_BASED_VALUE': 0})], + 0, 250, 120, 150, 130), '05-aa-kodim23.png') + +# 5. fades: a frame fading after the field split | fixed (average) | fixed (match) +c = load(os.path.join(S, 'fx', 'kodim23_pan.mkv'), 'yuv420p', frames=2) +fl = core.std.SeparateFields(core.std.SetFieldBased(core.resize.Point(c, format=vs.YUV420PS), 0), tff=True) +top = expr([fl[::2]], ['x 0.7 *'] * 3) +bot = expr([fl[1::2]], ['x 0.45 *'] * 3) +mixed = core.resize.Point(core.std.SetFieldBased(core.std.DoubleWeave(core.std.Interleave([top, bot]), tff=True)[::2], 0), format=c.format.id) +save(panels([mixed, fades(mixed), fades(mixed, 'match')], 0, 250, 120, 150, 130), '06-fades-kodim23.png') diff --git a/docs/reports/2026-11-classical-filters/scripts/make_fixtures.sh b/docs/reports/2026-11-classical-filters/scripts/make_fixtures.sh new file mode 100755 index 00000000..30c405a4 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/make_fixtures.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# Builds the clean fixtures for the noise-reduction comparison into $SCRATCH/fx. +# +# Three Kodak PhotoCD images (kodim08 buildings, kodim19 fence/lighthouse, +# kodim23 parrots: real detail, flat sky, fine texture), each panned by a +# 640x480 window two pixels per frame for 30 frames. A whole-pixel pan keeps +# every frame a clean crop of the original — the temporal methods get true +# motion to track and the ground truth stays exact. Stored as lossless FFV1, +# 4:2:0 8-bit. +# +# The Kodak set is fetched from r0k.us (the usual mirror; Kodak released the +# images for unrestricted use). Only the fixtures need the network; nothing in +# the report does. +set -e +S="${SCRATCH:?set SCRATCH to your scratch directory}" +FF="${FFMPEG:-ffmpeg} -v error -y" +mkdir -p "$S/fx" "$S/kodak" +for k in 08 19 23; do + f="$S/kodak/kodim$k.png" + [ -f "$f" ] || curl -fsSL -o "$f" "https://r0k.us/graphics/kodak/kodak/kodim$k.png" +done +# kodim19 is portrait (512x768): pan vertically instead. +$FF -loop 1 -i "$S/kodak/kodim08.png" -vf "crop=640:480:'2*n':16,format=yuv420p" \ + -frames:v 30 -r 25 -c:v ffv1 "$S/fx/kodim08_pan.mkv" +$FF -loop 1 -i "$S/kodak/kodim23.png" -vf "crop=640:480:'2*n':16,format=yuv420p" \ + -frames:v 30 -r 25 -c:v ffv1 "$S/fx/kodim23_pan.mkv" +$FF -loop 1 -i "$S/kodak/kodim19.png" -vf "transpose=1,crop=640:480:'2*n':16,format=yuv420p" \ + -frames:v 30 -r 25 -c:v ffv1 "$S/fx/kodim19_pan.mkv" +ls -la "$S/fx" + +# mpeg2stinx fixture: the progressive pans encoded the way a hard-telecined +# film DVD is (interlaced DCT + interlaced motion estimation, TFF, 4:2:0) at +# starved bitrates, then decoded. The progressive pan is the ground truth. +for k in 08 23; do + for r in 1000 2000; do + $FF -i "$S/fx/kodim${k}_pan.mkv" -vf setfield=tff -c:v mpeg2video -flags +ildct+ilme \ + -g 12 -bf 2 -b:v ${r}k -maxrate ${r}k -bufsize $((r * 2))k "$S/fx/kodim${k}_mpeg2_${r}.m2v" + $FF -i "$S/fx/kodim${k}_mpeg2_${r}.m2v" -c:v ffv1 "$S/fx/kodim${k}_mpeg2_${r}.mkv" + done +done +ls -la "$S/fx" diff --git a/docs/reports/2026-11-classical-filters/scripts/make_tc_fixtures.sh b/docs/reports/2026-11-classical-filters/scripts/make_tc_fixtures.sh new file mode 100755 index 00000000..a77d5c38 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/make_tc_fixtures.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Hard-telecined MPEG-2 fixtures for mpeg2stinx, from make_fixtures.sh's pans: +# 24 pan frames as 24p, 3:2 pulldown to 30i (TFF), then interlaced MPEG-2 +# (interlaced DCT + motion estimation, as a film DVD is) at starved bitrates. +# This is the material mpeg2stinx is written for; ports_compare.py +# inverse-telecines it with VIVTC (as the IVTC method does) and compares with +# the 24 clean frames. +set -e +S="${SCRATCH:?set SCRATCH to your scratch directory}" +FF="${FFMPEG:-ffmpeg} -v error -y" +for k in 08 23; do + $FF -i "$S/fx/kodim${k}_pan.mkv" -vf "select=lt(n\,24),setpts=N/24/TB,telecine=first_field=t:pattern=23" \ + -r 30 -c:v ffv1 "$S/fx/kodim${k}_tc.mkv" + for r in 1000 2000; do + $FF -i "$S/fx/kodim${k}_tc.mkv" -vf setfield=tff -c:v mpeg2video -flags +ildct+ilme \ + -g 12 -bf 2 -b:v ${r}k -maxrate ${r}k -bufsize $((r * 2))k "$S/fx/kodim${k}_tc_${r}.m2v" + $FF -i "$S/fx/kodim${k}_tc_${r}.m2v" -c:v ffv1 "$S/fx/kodim${k}_tc_${r}.mkv" + done +done +ls -la "$S/fx" | grep _tc diff --git a/docs/reports/2026-11-classical-filters/scripts/nr_compare.py b/docs/reports/2026-11-classical-filters/scripts/nr_compare.py new file mode 100644 index 00000000..952ecb52 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/nr_compare.py @@ -0,0 +1,181 @@ +"""Noise-reduction comparison for the vszip upgrade (deps 1.16.0). + +Methods, each at its best strength from a sweep (sweeps scale with the noise): + - BM3D: the bm3d plugin r12 through the exact code of the NR_BM3D template + block (16-bit, per GRAY plane, radius 1, Final step) - what ships today + - NLMeans (vszip, CPU) and KNLMeansCL (GPU, the reference it ports) + - WNNM (vszip), spatial and radius 1 + - BM3D (vszip, CPU, new in 24.0.0), radius 1 - for the record only + +Each clean Kodak pan (make_fixtures.sh) gets AddGrain Gaussian noise of a known +sigma (8-bit levels, luma and chroma); PSNR (luma, then chroma U) against the +clean pan; fps from clip.frames() (VapourSynth's own parallel prefetch). + +Run under the bundled interpreter with the deps environment, e.g. + SCRATCH=... DEPS=deps/macos-arm64 scripts/run.sh nr_compare.py +Writes $SCRATCH/nr_results.json. +""" +import json +import math +import os +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import vapoursynth as vs # noqa: E402 +from probelib import load, psnr, mad8, expr # noqa: E402 + +core = vs.core +S = os.environ['SCRATCH'] +FIXTURES = ['kodim08_pan', 'kodim19_pan', 'kodim23_pan'] +NFR = int(os.environ.get('NFR', '10')) +SIGMAS = [float(x) for x in os.environ.get('SIGMAS', '4,8,12').split(',')] + + +def add_noise(c, sigma): + return core.grain.Add(c, var=sigma * sigma, uvar=sigma * sigma, seed=7, constant=False) + + +def measured_sigma(noisy, clean): + d = expr([core.std.ShufflePlanes(noisy, 0, vs.GRAY), core.std.ShufflePlanes(clean, 0, vs.GRAY)], + 'x y - dup *', vs.GRAYS) + st = core.std.PlaneStats(d) + return math.sqrt(sum(st.get_frame(i).props['PlaneStatsAverage'] for i in range(3)) / 3) + + +# --- methods (each returns a clip in the input's format) ------------------- + +def bm3d_template(c, sigma, radius=1, final=True, profile='np'): + """Verbatim logic of the NR_BM3D block in worker/templates/*.vpy.""" + fmt = c.format + + def plane(p, s): + if radius > 0: + ref = core.bm3d.VAggregate(core.bm3d.VBasic(p, sigma=s, radius=radius, profile=profile), radius=radius) + if final: + ref = core.bm3d.VAggregate(core.bm3d.VFinal(p, ref, sigma=s, radius=radius, profile=profile), + radius=radius) + return ref + ref = core.bm3d.Basic(p, sigma=s, profile=profile) + return core.bm3d.Final(p, ref, sigma=s, profile=profile) if final else ref + work = core.resize.Point(c, format=fmt.replace(bits_per_sample=16, sample_type=vs.INTEGER).id) + planes = [core.std.ShufflePlanes(work, i, vs.GRAY) for i in range(fmt.num_planes)] + planes = [plane(p, sigma) for p in planes] + work = core.std.ShufflePlanes(planes, [0, 0, 0], fmt.color_family) + return core.resize.Point(work, format=fmt.id, dither_type='error_diffusion') + + +def nlm_vszip(c, h, d=1, a=2, s=4): + c = core.vszip.NLMeans(c, d=d, a=a, s=s, h=h, channels='Y') + return core.vszip.NLMeans(c, d=d, a=a, s=s, h=h, channels='UV') + + +def nlm_knlm(c, h, d=1, a=2, s=4): + c = core.knlm.KNLMeansCL(c, d=d, a=a, s=s, h=h, channels='Y', device_type='gpu') + return core.knlm.KNLMeansCL(c, d=d, a=a, s=s, h=h, channels='UV', device_type='gpu') + + +def _to_float(c): + # Default (limited) range both ways, so the round trip is exact (asserted + # below). The float picture then spans 219 (luma) / 224 (chroma) 8-bit + # levels per 1.0, so sigma (which WNNM divides by 255) is rescaled to stay + # in 8-bit units. Passing range_in_s="full" instead is NOT safe: a clip + # carrying _ColorRange=limited round-tripped 7 levels off at 10-bit. + return core.resize.Point(c, format=c.format.replace(bits_per_sample=32, sample_type=vs.FLOAT).id) + + +def _from_float(f, fmt): + return core.resize.Point(f, format=fmt.id, dither_type='error_diffusion') + + +def wnnm(c, sigma, radius=0): + f = core.vszip.WNNM(_to_float(c), sigma=[sigma * 255 / 219, sigma * 255 / 224, sigma * 255 / 224], + radius=radius) + return _from_float(f, c.format) + + +def bm3d_vszip(c, sigma, radius=1): + f = core.vszip.BM3D(_to_float(c), sigma=[sigma * 255 / 219, sigma * 255 / 224, sigma * 255 / 224], + radius=radius) + return _from_float(f, c.format) + + +MULT = [0.5, 0.75, 1.0, 1.25, 1.5, 2.0, 2.5] # x noise sigma +NLM_H = [0.5, 0.75, 1.0, 1.5, 2.0, 2.75, 3.5, 4.5] # x noise sigma / 4 + +METHODS = { + 'BM3D (bm3d r12, r=1, Final) - shipped': (bm3d_template, MULT), + 'NLMeans vszip CPU (d=1 a=2 s=4)': (nlm_vszip, None), + 'KNLMeansCL GPU (d=1 a=2 s=4)': (nlm_knlm, None), + 'WNNM vszip (r=0)': (lambda c, s: wnnm(c, s, 0), MULT), + 'WNNM vszip (r=1)': (lambda c, s: wnnm(c, s, 1), MULT), + 'BM3D vszip (r=1)': (bm3d_vszip, MULT), +} + + +def fps(clip): + clip = core.std.Trim(clip, 0, NFR - 1) + t = time.time() + for _ in clip.frames(close=True): + pass + return NFR / (time.time() - t) + + +results = {'nfr': NFR, 'fixtures': {}} +# PARITY_ONLY=1 skips the (two-hour) sweep and runs only the depth checks. +for fx in ([] if os.environ.get('PARITY_ONLY') else FIXTURES): + clean = load(os.path.join(S, 'fx', fx + '.mkv'), 'yuv420p', frames=NFR) + results['fixtures'][fx] = {} + for sigma in SIGMAS: + noisy = add_noise(clean, sigma) + row = {'measured_sigma': measured_sigma(noisy, clean), 'noisy_psnr_y': psnr(noisy, clean, 0), + 'methods': {}} + print(f'\n== {fx} sigma {sigma} (measured {row["measured_sigma"]:.2f}); noisy Y {row["noisy_psnr_y"]:.2f} dB', + flush=True) + for name, (fn, mult) in METHODS.items(): + strengths = [round(m * sigma, 2) for m in mult] if mult else [round(k * sigma / 4, 2) for k in NLM_H] + sweep = [] + for st in strengths: + sweep.append((st, psnr(fn(noisy, st), clean, 0))) + best = max(sweep, key=lambda r: r[1]) + out = fn(noisy, best[0]) + m = {'sweep': sweep, 'best_strength': best[0], 'psnr_y': best[1], 'psnr_u': psnr(out, clean, 1), + 'fps': fps(fn(noisy, best[0]))} # a fresh node: 'out' is cached + edge = ' (edge of sweep)' if best[0] in (strengths[0], strengths[-1]) else '' + print(f' {name:40s} best {best[0]:>5}{edge:16s} Y {m["psnr_y"]:.2f} U {m["psnr_u"]:.2f} ' + f'{m["fps"]:6.1f} fps', flush=True) + row['methods'][name] = m + results['fixtures'][fx][str(sigma)] = row + # Written as it goes: a failure later must not lose the sweep. + json.dump(results, open(os.path.join(S, 'nr_results.json'), 'w'), indent=1) + +# --- depth parity: the same noisy picture at 8, 10 and 16 bits -------------- +print('\n== depth parity: mean |8-bit run - N-bit run|, 8-bit levels (luma)', flush=True) +clean = load(os.path.join(S, 'fx', FIXTURES[0] + '.mkv'), 'yuv420p', frames=4) +noisy8 = add_noise(clean, 8) +results['float_round_trip'] = {} +for bits in (8, 10, 16): + # How much of the WNNM row is the float conversion itself (it dithers). + up = core.resize.Point(noisy8, format=noisy8.format.replace(bits_per_sample=bits).id) + results['float_round_trip'][bits] = mad8(_from_float(_to_float(up), up.format), up, 0) + print(f' float round trip at {bits}-bit: {results["float_round_trip"][bits]:.4f} levels', flush=True) +results['parity'] = {} +for name, fn in [('NLMeans vszip h=2', lambda c: nlm_vszip(c, 2.0)), + ('WNNM sigma=8', lambda c: wnnm(c, 8.0)), + ('BM3D template sigma=8', lambda c: bm3d_template(c, 8.0))]: + o8 = fn(noisy8) + row = {} + for bits in (10, 16): + n = core.resize.Point(noisy8, format=noisy8.format.replace(bits_per_sample=bits).id) + row[bits] = mad8(fn(n), o8, 0) + results['parity'][name] = row + print(f' {name:28s} 10-bit {row[10]:.3f} 16-bit {row[16]:.3f}', flush=True) + +o1 = nlm_vszip(noisy8, 2.0) +o2 = nlm_knlm(noisy8, 2.0) +results['nlm_vs_knlm_mad'] = {'y': mad8(o1, o2, 0), 'u': mad8(o1, o2, 1)} +print(f'\nvszip NLMeans vs KNLMeansCL, same args: mean |diff| Y {results["nlm_vs_knlm_mad"]["y"]:.4f} ' + f'U {results["nlm_vs_knlm_mad"]["u"]:.4f} levels') +json.dump(results, open(os.path.join(S, 'nr_parity.json' if os.environ.get('PARITY_ONLY') else + 'nr_results.json'), 'w'), indent=1) diff --git a/docs/reports/2026-11-classical-filters/scripts/parse_nr_log.py b/docs/reports/2026-11-classical-filters/scripts/parse_nr_log.py new file mode 100644 index 00000000..fcbc5fb7 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/parse_nr_log.py @@ -0,0 +1,33 @@ +"""Rebuilds nr_results.json from nr_compare.py's printed log. + +The 2026-11-08 run crashed in its final parity step (a round-trip assertion +that compared a float mean with == 0) after the two-hour sweep, before the +script wrote its JSON; the sweep's per-strength rows existed only in memory. +The log (results/nr_compare.txt) has every best-strength row, which is what +the report's tables use; the parity step was re-run on its own +(PARITY_ONLY=1, results/nr_parity.txt). nr_compare.py now writes its JSON +after every fixture. + + python3 parse_nr_log.py results/nr_compare.txt > $SCRATCH/nr_results.json +""" +import json +import re +import sys + +out = {'nfr': 10, 'fixtures': {}, 'source': 'parsed from nr_compare.txt'} +head = re.compile(r'== (\S+) sigma ([\d.]+) \(measured ([\d.]+)\); noisy Y ([\d.]+) dB') +row = re.compile(r' (.+?)\s+best\s+([\d.]+)(?: \(edge of sweep\))?\s+Y ([\d.]+)\s+U ([\d.]+)\s+([\d.]+) fps') +cur = None +for line in open(sys.argv[1]): + m = head.search(line) + if m: + fx, sigma = m.group(1), m.group(2) + cur = {'measured_sigma': float(m.group(3)), 'noisy_psnr_y': float(m.group(4)), 'methods': {}} + out['fixtures'].setdefault(fx, {})[sigma] = cur + continue + m = row.match(line) + if m and cur is not None: + cur['methods'][m.group(1).strip()] = { + 'best_strength': float(m.group(2)), 'psnr_y': float(m.group(3)), 'psnr_u': float(m.group(4)), + 'edge_of_sweep': '(edge of sweep)' in line} +json.dump(out, sys.stdout, indent=1) diff --git a/docs/reports/2026-11-classical-filters/scripts/ports_compare.py b/docs/reports/2026-11-classical-filters/scripts/ports_compare.py new file mode 100644 index 00000000..d9700c45 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/ports_compare.py @@ -0,0 +1,236 @@ +"""Checks the three vs-jetpack ports and SangNom AA against their fixtures. + + vine_dehalo - a clean Kodak pan oversharpened into halos (unsharp mask, + Gaussian sigma 1.5, amount 1.5); PSNR vs the clean pan for + vine and the existing Dehalo methods. + mpeg2stinx - the pans encoded as interlaced MPEG-2 (make_fixtures.sh); + PSNR vs the clean pan for mpeg2stinx and the existing + Deblock methods. + FixInterlacedFades - a pan faded to black AFTER a field split: the top field + at the frame's fade level, the bottom field half a frame + later, as a fade applied to telecined video is. Residual + "combing" (mean |top - bottom| field brightness gap) and the + change it makes to ordinary, unfaded frames. + depth parity - each port at 10 and 16 bits against its 8-bit run. + +The port functions are imported from the worker's own template (the block +between the markers named below is exec'd), so this measures the shipped code. + + SCRATCH=... ./run.sh ports_compare.py +""" +import json +import math +import os +import re +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +import vapoursynth as vs # noqa: E402 +import havsfunc as haf # noqa: E402 +from probelib import load, psnr, mad8, expr # noqa: E402 + +core = vs.core +S = os.environ['SCRATCH'] +REPO = os.environ.get('REPO', os.path.abspath(os.path.join(HERE, '..', '..', '..', '..'))) +TPL = open(os.path.join(REPO, 'worker', 'templates', 'pipeline_template.vpy')).read() + + +def template_block(name): + """The Python between {{#NAME}} and {{/NAME}} in the shipped template.""" + m = re.search(r'\{\{#' + name + r'\}\}\n(.*?)\{\{/' + name + r'\}\}', TPL, re.S) + return m.group(1) + + +# The helpers every template block relies on. +ns = {'vs': vs, 'core': core, 'haf': haf, 'math': math, 'functools': __import__('functools')} +exec(TPL[TPL.index('import platform as _platform'):TPL.index('# PASS 2: DEINTERLACING')].rsplit('# ====', 1)[0], ns) + + +def run_block(name, clip, subs): + code = template_block(name) + for k, v in subs.items(): + code = code.replace('{{' + k + '}}', str(v)) + code = re.sub(r'\{\{[#/][A-Z0-9_]+\}\}\n', '', code) + env = dict(ns) + env['clip'] = clip + exec(code, env) + return env['clip'] + + +def fps(clip, n): + clip = core.std.Trim(clip, 0, n - 1) + t = time.time() + for _ in clip.frames(close=True): + pass + return n / (time.time() - t) + + +NFR = int(os.environ.get('NFR', '10')) +out = {} + +# ---- vine_dehalo ------------------------------------------------------------ +def gauss(c, sigma): + r = int(math.ceil(sigma * 3)) + k = [math.exp(-(x * x) / (2 * sigma * sigma)) for x in range(-r, r + 1)] + return core.std.Convolution(core.std.Convolution(c, matrix=k, mode='h'), matrix=k, mode='v') + + +def vine(c, strength=16.0, sharp=0.5, sigma=1.0): + return run_block('DEHALO_VINE', c, {'DEHALO_VINE_STRENGTH': strength, 'DEHALO_VINE_SHARP': sharp, + 'DEHALO_VINE_SIGMA': sigma}) + + +out['vine'] = {} +for fx in ['kodim08_pan', 'kodim23_pan']: + clean = load(os.path.join(S, 'fx', fx + '.mkv'), 'yuv420p', frames=NFR) + halo = expr([clean, gauss(clean, 1.5)], 'x x y - 1.5 * +') + rows = {'haloed (no filter)': halo, + 'vine (16, 0.5, 1.0)': vine(halo), + 'vine (24, 0.5, 1.0)': vine(halo, 24.0), + 'vine (8, 0.5, 1.0)': vine(halo, 8.0), + 'DeHalo_alpha (defaults)': haf.DeHalo_alpha(halo), + 'FineDehalo (defaults)': haf.FineDehalo(halo)} + res = {name: {'psnr_y': psnr(c, clean, 0)} for name, c in rows.items()} + # Speed on fresh nodes (the ones above are cached by the PSNR pass). + res['vine (16, 0.5, 1.0)']['fps'] = fps(vine(halo), NFR) + res['DeHalo_alpha (defaults)']['fps'] = fps(haf.DeHalo_alpha(halo), NFR) + res['FineDehalo (defaults)']['fps'] = fps(haf.FineDehalo(halo), NFR) + for name, r in res.items(): + print(f'vine {fx}: {name:28s} Y {r["psnr_y"]:.2f} dB {r.get("fps", 0):.1f} fps', flush=True) + out['vine'][fx] = res + +# ---- mpeg2stinx --------------------------------------------------------------- +def stinx(c, tff=True): + return run_block('DEBLOCK_MPEG2STINX', c, {'DEBLOCK_STINX_TFF': tff}) + + +out['mpeg2stinx'] = {} + + +def ivtc(c): + # As the IVTC method does it (VFM + VDecimate, defaults, TFF). + return core.vivtc.VDecimate(core.vivtc.VFM(c, order=1)) + + +def stinx_rows(enc, clean, label): + rows = {'decoded (no filter)': enc, + 'mpeg2stinx (TFF)': stinx(enc), + 'mpeg2stinx (wrong order: BFF)': stinx(enc, False), + 'Deblock_QED (defaults)': haf.Deblock_QED(enc), + 'Deblock (quant 25)': core.deblock.Deblock(enc, quant=25)} + res = {n: {'psnr_y': psnr(c, clean, 0), 'psnr_u': psnr(c, clean, 1)} for n, c in rows.items()} + res['mpeg2stinx (TFF)']['fps'] = fps(stinx(enc), min(NFR, enc.num_frames)) + for n, r in res.items(): + print(f'mpeg2stinx {label}: {n:30s} Y {r["psnr_y"]:.2f} U {r["psnr_u"]:.2f}', flush=True) + return res + + +for fx in ['kodim08', 'kodim23']: + clean24 = load(os.path.join(S, 'fx', fx + '_pan.mkv'), 'yuv420p', frames=24) + for rate in (1000, 2000): + # Hard-telecined, then inverse-telecined: what mpeg2stinx is for. + enc = ivtc(load(os.path.join(S, 'fx', f'{fx}_tc_{rate}.mkv'), 'yuv420p', frames=30)) + assert enc.num_frames == 24, enc.num_frames + # VDecimate must have kept the right frames: each matches its clean + # frame far better than either neighbour. + for i in (1, 12, 22): + here = psnr(enc[i], clean24[i]) + assert here > psnr(enc[i], clean24[i - 1]) + 3 and here > psnr(enc[i], clean24[i + 1]) + 3, i + out['mpeg2stinx'][f'{fx} telecined @{rate}k'] = stinx_rows(enc, clean24, f'{fx} telecined @{rate}k') + # Progressive frames encoded as interlaced, no pulldown. + clean = clean24[:NFR] if NFR <= 24 else clean24 + enc = load(os.path.join(S, 'fx', f'{fx}_mpeg2_{rate}.mkv'), 'yuv420p', frames=clean.num_frames) + out['mpeg2stinx'][f'{fx} progressive-as-interlaced @{rate}k'] = stinx_rows( + enc, clean, f'{fx} progressive-as-interlaced @{rate}k') + +# ---- FixInterlacedFades ----------------------------------------------------- +def fades(c, mode='average'): + return run_block('DEINT_IVTC_FIX_FADES', c, {'IVTC_FADE_MODE': mode}) + + +def field_gap(c): + """Mean |top-field mean - bottom-field mean| of luma, 8-bit levels.""" + f = core.std.SeparateFields(core.std.SetFieldBased(c, 0), tff=True) + st = core.std.PlaneStats(f) + vals = [st.get_frame(i).props['PlaneStatsAverage'] for i in range(st.num_frames)] + peak = (1 << c.format.bits_per_sample) - 1 + gaps = [abs(vals[2 * i] - vals[2 * i + 1]) * peak / (1 << (c.format.bits_per_sample - 8)) + for i in range(len(vals) // 2)] + return sum(gaps) / len(gaps) + + +clean = load(os.path.join(S, 'fx', 'kodim23_pan.mkv'), 'yuv420p', frames=NFR) +n = clean.num_frames + + +def faded_after_split(c): + """Fade to black over the clip, applied per field at field time.""" + flds = core.std.SeparateFields(core.std.SetFieldBased(c, 0), tff=True) + total = 2 * n + + def lvl(k): + return max(0.0, 1.0 - k / (total - 1)) + parts = [] + for k in range(total): + a = lvl(k) + # fade in the float domain about black (limited range: 16 -> 0.0) + fl = core.resize.Point(flds[k], format=vs.YUV420PS) + fl = expr([fl], [f'x {a} *', f'x {a} *', f'x {a} *']) + parts.append(core.resize.Point(fl, format=c.format.id)) + return core.std.DoubleWeave(core.std.Splice(parts), tff=True)[::2] + + +def truth(c): + """What a perfect fix gives: each frame at its two fields' mean level.""" + total = 2 * n + parts = [] + for i in range(n): + a = (max(0.0, 1.0 - (2 * i) / (total - 1)) + max(0.0, 1.0 - (2 * i + 1) / (total - 1))) / 2 + fl = core.resize.Point(c[i], format=vs.YUV420PS) + parts.append(core.resize.Point(expr([fl], [f'x {a} *'] * 3), format=c.format.id)) + return core.std.Splice(parts) + + +faded = faded_after_split(clean) +gt = truth(clean) +fixed = fades(faded) +fixed_min = fades(faded, 'match') +out['fades'] = { + 'field_gap_faded': field_gap(faded), 'field_gap_fixed': field_gap(fixed), + 'field_gap_fixed_match': field_gap(fixed_min), 'field_gap_clean': field_gap(clean), + 'psnr_faded_vs_truth': psnr(faded, gt, 0), 'psnr_fixed_vs_truth': psnr(fixed, gt, 0), + 'unfaded_change_mad': mad8(fades(clean), clean, 0), + 'unfaded_change_mad_mpeg2': mad8(fades(load(os.path.join(S, 'fx', 'kodim08_mpeg2_1000.mkv'), 'yuv420p', frames=NFR)), + load(os.path.join(S, 'fx', 'kodim08_mpeg2_1000.mkv'), 'yuv420p', frames=NFR), 0), + 'fps': fps(fades(faded), NFR), +} +print('fades:', json.dumps(out['fades'], indent=1), flush=True) + +# ---- SangNom AA speed/effect ----------------------------------------------------- +out['sangnom'] = {} +c = load(os.path.join(S, 'fx', 'kodim08_pan.mkv'), 'yuv420p', frames=NFR) +for name, f in [('daa', lambda x: haf.daa(core.std.SetFieldBased(x, 0))), + ('santiag (nnedi3, 1/1)', lambda x: haf.santiag(core.std.SetFieldBased(x, 0))), + ('SangNom AA (48)', lambda x: run_block('AA_SANGNOM', x, {'AA_SANGNOM_AA': 48, 'AA_FIELD_BASED_VALUE': 0}))]: + out['sangnom'][name] = {'fps': fps(f(c), NFR), 'change_vs_src': mad8(f(c), c, 0)} + print(f'AA {name:24s} {out["sangnom"][name]["fps"]:.1f} fps, mean change {out["sangnom"][name]["change_vs_src"]:.2f}', + flush=True) + +# ---- depth parity --------------------------------------------------------------- +out['parity'] = {} +base = load(os.path.join(S, 'fx', 'kodim08_mpeg2_1000.mkv'), 'yuv420p', frames=4) +halo8 = expr([base, gauss(base, 1.5)], 'x x y - 1.5 * +') +for name, f, src in [('vine', vine, halo8), ('mpeg2stinx', stinx, base), ('FixInterlacedFades', fades, faded[:4]), + ('SangNom AA', lambda x: run_block('AA_SANGNOM', x, {'AA_SANGNOM_AA': 48, 'AA_FIELD_BASED_VALUE': 0}), + base)]: + o8 = f(src) + row = {} + for bits in (10, 16): + up = core.resize.Point(src, format=src.format.replace(bits_per_sample=bits).id) + row[bits] = mad8(f(up), o8, 0) + out['parity'][name] = row + print(f'parity {name:20s} 10-bit {row[10]:.3f} 16-bit {row[16]:.3f}', flush=True) + +json.dump(out, open(os.path.join(S, 'ports_results.json'), 'w'), indent=1) diff --git a/docs/reports/2026-11-classical-filters/scripts/ports_vs_reference.py b/docs/reports/2026-11-classical-filters/scripts/ports_vs_reference.py new file mode 100644 index 00000000..1dc10db9 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/ports_vs_reference.py @@ -0,0 +1,86 @@ +"""The ported template blocks against vs-jetpack itself. + +vs-jetpack 3.0.0 needs Python 3.13; the bundle's is 3.12, so the reference is +vsjetpack 2.2.7, whose three functions are byte-identical to 3.0.0's (checked +with a diff). Install it beside this script's scratch: + + $DEPS/python/bin/python3 -m pip install --target "$SCRATCH/jp" --no-deps \\ + vsjetpack==2.2.7 'jetpytools>=3.1,<4' 'numpy>=2' psutil rich \\ + typing-extensions markdown-it-py mdurl pygments + SCRATCH=... ./run.sh ports_vs_reference.py + +Reports mean |port - reference| in 8-bit levels (luma, chroma) on the fixtures. +""" +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +S = os.environ['SCRATCH'] +sys.path.append(os.path.join(S, 'jp')) +sys.argv = [sys.argv[0]] +os.environ.setdefault('NFR', '6') +# Reuse ports_compare's loaders and the template-block runner, without running it. +src = open(os.path.join(HERE, 'ports_compare.py')).read() +exec(src[:src.index('# ---- vine_dehalo')]) +exec(src[src.index('def gauss('):src.index("out['vine'] = {}")]) +exec(src[src.index('def stinx('):src.index("out['mpeg2stinx'] = {}")]) +exec(src[src.index('def fades('):src.index('def field_gap(')]) + +# vs-jetpack's scalers need the resize2 plugin, which the bundle does not ship +# (the ports do not use it), so vine_dehalo's reference cannot run against the +# bundle and is reported as not run. mpeg2stinx and FixInterlacedFades need +# only bundled plugins. resize2 is a superset of the core resizers, so its +# kernels are pointed at core.resize for the reference run (vine_dehalo's +# NNEDI3 supersample and Catrom downscale pass no resize2-only arguments). +import vskernels # noqa: E402 + +for _name in dir(vskernels): + _cls = getattr(vskernels, _name) + if not isinstance(_cls, type) or 'zimg' not in getattr(_cls, '__module__', ''): + continue + for _attr in ('scale_function', 'resample_function'): + _ref = getattr(getattr(_cls, _attr, None), 'func_ref', None) + if _ref is not None and hasattr(core.resize, _ref[1]): + setattr(_cls, _attr, getattr(core.resize, _ref[1])) +# 2.2.7's nl_means predates vszip's CPU NLMeans and finds no backend here; +# 3.0.0's CPU backend is exactly `clip.vszip.NLMeans(*args, **kwargs)`, so +# route it there (wmode/wref/channels/rclip pass straight through). +from vsdenoise import nlm as _nlm # noqa: E402 + +_nlm.NLMeans.Backend.NLMeans = lambda self, clip, *a, **k: clip.vszip.NLMeans(*a, **k) +from vsdehalo import vine_dehalo # noqa: E402 +from vsdenoise import mpeg2stinx # noqa: E402 +from vsdeinterlace import FixInterlacedFades # noqa: E402 + +res = {} +clean = load(os.path.join(S, 'fx', 'kodim08_pan.mkv'), 'yuv420p', frames=NFR) +halo = core.std.SetFieldBased(expr([clean, gauss(clean, 1.5)], 'x x y - 1.5 * +'), 0) +for bits in (8, 10): + h = halo if bits == 8 else core.resize.Point(halo, format=halo.format.replace(bits_per_sample=bits).id) + try: + port, ref = vine(h), vine_dehalo(h) + res[f'vine {bits}-bit'] = (mad8(port, ref, 0), mad8(port, ref, 1)) + except (vs.Error, AttributeError) as e: + print(f'vine {bits}-bit: reference not run ({str(e).splitlines()[0][:90]})') + +enc = core.std.SetFieldBased(load(os.path.join(S, 'fx', 'kodim08_mpeg2_1000.mkv'), 'yuv420p', frames=NFR), 0) +for bits in (8, 10): + e = enc if bits == 8 else core.resize.Point(enc, format=enc.format.replace(bits_per_sample=bits).id) + port, ref = stinx(e), mpeg2stinx(e) + res[f'mpeg2stinx {bits}-bit'] = (mad8(port, ref, 0), mad8(port, ref, 1)) + +c = core.resize.Point(clean, format=vs.YUV420PS) +faded = core.std.SetFieldBased(expr([core.std.SeparateFields(core.std.SetFieldBased(c, 0), tff=True)], + ['x 0.6 *', 'x 0.6 *', 'x 0.6 *']), 0) +# a frame whose fields sit at different fade levels: top 0.8, bottom 0.5 +fl = core.std.SeparateFields(core.std.SetFieldBased(c, 0), tff=True) +top = expr([fl[::2]], ['x 0.8 *'] * 3) +bot = expr([fl[1::2]], ['x 0.5 *'] * 3) +mixed = core.std.SetFieldBased(core.std.DoubleWeave(core.std.Interleave([top, bot]), tff=True)[::2], 0) +for mode, ref_fn in (('average', FixInterlacedFades.AVERAGE), ('match', FixInterlacedFades.MATCH)): + port = fades(mixed, mode) + ref = ref_fn(mixed) + res[f'FixInterlacedFades {mode} (float)'] = (mad8(port, ref, 0), mad8(port, ref, 1)) + +for k, (y, u) in res.items(): + print(f'{k:36s} |port - vs-jetpack| Y {y:.4f} U {u:.4f} levels') diff --git a/docs/reports/2026-11-classical-filters/scripts/probelib.py b/docs/reports/2026-11-classical-filters/scripts/probelib.py new file mode 100644 index 00000000..fd62688d --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/probelib.py @@ -0,0 +1,100 @@ +"""Probe helpers: load a source through ffmpeg into a VapourSynth clip, metrics.""" +import ctypes +import math +import os +import subprocess +import time + +import vapoursynth as vs + +core = vs.core + +FMTS = { + 'yuv420p': vs.YUV420P8, 'yuv422p': vs.YUV422P8, 'yuv444p': vs.YUV444P8, + 'yuv420p10le': vs.YUV420P10, 'yuv422p10le': vs.YUV422P10, + 'yuv420p12le': vs.YUV420P12, 'yuv420p16le': vs.YUV420P16, + 'yuv422p16le': vs.YUV422P16, +} + + +def load(path, pix_fmt='yuv420p', frames=10, start=0, vf=None): + fmt = core.get_video_format(FMTS[pix_fmt]) + probe = subprocess.run(['ffprobe', '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=width,height', '-of', 'csv=p=0', path], + capture_output=True, text=True, check=True).stdout.strip().split(',') + w, h = int(probe[0]), int(probe[1]) + args = ['ffmpeg', '-v', 'error', '-apply_cropping', 'codec', '-i', path] + filt = f'select=gte(n\\,{start})' + if vf: + filt += ',' + vf + args += ['-vf', filt, '-frames:v', str(frames), '-pix_fmt', pix_fmt, '-f', 'rawvideo', '-'] + data = subprocess.run(args, capture_output=True, check=True).stdout + if vf and 'scale=' in vf: + sc = vf.split('scale=')[1].split(',')[0].split(':') + w, h = int(sc[0]), int(sc[1]) + bps = fmt.bytes_per_sample + sizes = [w * h * bps] + [(w >> fmt.subsampling_w) * (h >> fmt.subsampling_h) * bps] * 2 + fsize = sum(sizes) + n = len(data) // fsize + blank = core.std.BlankClip(width=w, height=h, format=fmt.id, length=n, fpsnum=25, fpsden=1) + + def fill(n, f): + fout = f.copy() + off = n * fsize + for p in range(3): + pw = w >> (fmt.subsampling_w if p else 0) + ph = h >> (fmt.subsampling_h if p else 0) + row = pw * bps + stride = fout.get_stride(p) + ptr = fout.get_write_ptr(p) + dst = ptr if isinstance(ptr, int) else ptr.value + buf = (ctypes.c_char * sizes[p]).from_buffer_copy(data, off) + for r in range(ph): + ctypes.memmove(dst + r * stride, ctypes.addressof(buf) + r * row, row) + off += sizes[p] + return fout + + return core.std.ModifyFrame(blank, blank, fill) + + +def expr(clips, e, fmt=None): + kw = {} if fmt is None else {'format': fmt} + if hasattr(core, 'akarin'): + return core.akarin.Expr(clips, e, **kw) + return core.std.Expr(clips, e, **kw) + + +def _diff8(a, b, plane, op): + a = core.std.ShufflePlanes(a, plane, vs.GRAY) + b = core.std.ShufflePlanes(b, plane, vs.GRAY) + sa = 1.0 / (1 << (a.format.bits_per_sample - 8)) + sb = 1.0 / (1 << (b.format.bits_per_sample - 8)) + return expr([a, b], f'x {sa} * y {sb} * - ' + op, vs.GRAYS) + + +def psnr(a, b, plane=0, frames=None): + """Mean PSNR over frames, in 8-bit units (peak 255).""" + st = core.std.PlaneStats(_diff8(a, b, plane, 'dup *')) + n = st.num_frames if frames is None else min(frames, st.num_frames) + vals = [] + for i in range(n): + mse = st.get_frame(i).props['PlaneStatsAverage'] + vals.append(99.0 if mse <= 0 else 10 * math.log10(255 * 255 / mse)) + return sum(vals) / len(vals) + + +def mad8(a, b, plane=0, frames=None): + """Mean absolute difference in 8-bit levels.""" + st = core.std.PlaneStats(_diff8(a, b, plane, 'abs')) + n = st.num_frames if frames is None else min(frames, st.num_frames) + return sum(st.get_frame(i).props['PlaneStatsAverage'] for i in range(n)) / n + + +def timed(clip, frames=None): + n = clip.num_frames if frames is None else min(frames, clip.num_frames) + t = time.time() + for f in clip.frames(close=True): + n -= 1 + if n <= 0: + break + return time.time() - t diff --git a/docs/reports/2026-11-classical-filters/scripts/run.sh b/docs/reports/2026-11-classical-filters/scripts/run.sh new file mode 100755 index 00000000..3e64498d --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Runs one of this report's Python probes under a deps bundle's own Python and +# VapourSynth (macOS/Linux layout), with the environment the worker gives vspipe. +# +# SCRATCH=/path/to/scratch [DEPS=/path/to/deps/] ./run.sh nr_compare.py +# +# DEPS defaults to the repo's deps/macos-arm64. The probes in this report need a +# bundle with vszip >= 24.0.0 and SangNom (deps 1.16.0). +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +D="${DEPS:-$REPO/deps/macos-arm64}" +: "${SCRATCH:?set SCRATCH to your scratch directory}" +export PYTHONHOME="$D/python" +export PYTHONPATH="$HERE:$D:$D/python-packages" +export PYTHONNOUSERSITE=1 +export XDG_CONFIG_HOME="$D/config" +export PATH="$D/python/bin:$D/vapoursynth:$D/ffmpeg:$PATH" +export DYLD_LIBRARY_PATH="$D/vapoursynth:$D/python/lib" +export LD_LIBRARY_PATH="$D/vapoursynth:$D/python/lib:$D/lib" +export REPO +exec python3 "$HERE/$1" "${@:2}" diff --git a/docs/reports/2026-11-classical-filters/scripts/tables.py b/docs/reports/2026-11-classical-filters/scripts/tables.py new file mode 100644 index 00000000..db167146 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/tables.py @@ -0,0 +1,89 @@ +"""Prints the report's result tables as HTML from the probes' output in +../results (nr_results.json from parse_nr_log.py, nr_parity.txt, +ports_results.json, bench_results.json). + + python3 tables.py > tables.html +""" +import json +import os +import re + +HERE = os.path.dirname(os.path.abspath(__file__)) +S = os.environ.get('RESULTS', os.path.join(HERE, '..', 'results')) + + +def load(name): + p = os.path.join(S, name) + return json.load(open(p)) if os.path.exists(p) else None + + +nr = load('nr_results.json') +if nr: + for fx, by_sigma in nr['fixtures'].items(): + print(f'') + sigmas = list(by_sigma) + print('' + ''.join(f'' for s in sigmas) + + '') + print('' + ''.join( + f'' for s in sigmas) + '') + methods = list(by_sigma[sigmas[0]]['methods']) + for m in methods: + cells = '' + for s in sigmas: + r = by_sigma[s]['methods'][m] + best_y = max(v['psnr_y'] for v in by_sigma[s]['methods'].values()) + mark = ' class="b"' if r['psnr_y'] == best_y else '' + cells += f'{r["psnr_y"]:.2f}' + print(f'{cells}') + print('
{fx}: luma / chroma (U) PSNR in dB, each method at its best strength
methodσ {float(s):g}: YUstrength
noisy input{by_sigma[s]["noisy_psnr_y"]:.2f}
{r["psnr_u"]:.2f}{r["best_strength"]:g}
{m}
') +par = os.path.join(S, 'nr_parity.txt') +if os.path.exists(par): + print('') + for line in open(par): + m = re.match(r'\s+(.+?)\s+10-bit ([\d.]+)\s+16-bit ([\d.]+)', line) + if m: + print(f'') + elif 'vszip NLMeans vs KNLMeansCL' in line: + print('
Depth parity: mean |8-bit run − N-bit run|, 8-bit levels (luma), kodim08 sigma 8' + '
method10-bit16-bit
{m.group(1)}{m.group(2)}{m.group(3)}

' + line.strip() + '

') + +bench = load('bench_results.json') +if bench: + print('' + '' + ''.join(f'' for res in bench) + '') + first = list(bench)[0] + for m in bench[first]: + print(f'' + ''.join(f'' for res in bench) + '') + print('
Speed: vspipe, median of runs, frames per second (M1 Pro)
method{res}
{m}{bench[res][m]["median_fps"]:.1f}
') + +ports = load('ports_results.json') +if ports: + for fx, rows in ports['vine'].items(): + print(f'' + '') + for n, r in rows.items(): + print(f'') + print('
Dehalo, {fx} (oversharpened): luma PSNR vs the clean pan
methodY dB
{n}{r["psnr_y"]:.2f}
') + print('' + + ''.join(f'' for n in next(iter(ports['mpeg2stinx'].values()))) + '') + for fx, rows in ports['mpeg2stinx'].items(): + print(f'' + ''.join(f'' for r in rows.values()) + + '') + print('
mpeg2stinx: PSNR vs the clean pan (Y / U dB)
fixture{n}
{fx}{r["psnr_y"]:.2f} / {r["psnr_u"]:.2f}
') + f = ports['fades'] + print('' + '') + for k, v in f.items(): + if k == 'fps': # taken under load; bench.py has the speeds + continue + print(f'') + print('
IVTC fade fix (kodim23 pan, faded after the field split)
measurevalue
{k}{v:.4f}
') + print('') + for n, r in ports['sangnom'].items(): + print(f'') + print('
Anti-aliasing (kodim08 pan)
methodmean change, levels
{n}{r["change_vs_src"]:.2f}
') + print('' + '') + for n, r in ports['parity'].items(): + print(f'') + print('
Ports, depth parity: mean |8-bit − N-bit|, levels
block10-bit16-bit
{n}{r["10"]:.3f}{r["16"]:.3f}
') diff --git a/docs/reports/2026-11-classical-filters/scripts/wnnm_wide.py b/docs/reports/2026-11-classical-filters/scripts/wnnm_wide.py new file mode 100644 index 00000000..d047d217 --- /dev/null +++ b/docs/reports/2026-11-classical-filters/scripts/wnnm_wide.py @@ -0,0 +1,31 @@ +"""WNNM's best sigma sometimes sat at the top of nr_compare.py's sweep (2.5x +the noise). This widens it to 4x for WNNM only, on every fixture and sigma, +so the comparison with BM3D is not cut short. + + SCRATCH=... ./run.sh wnnm_wide.py -> $SCRATCH/wnnm_wide.json +""" +import json +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, HERE) +src = open(os.path.join(HERE, 'nr_compare.py')).read() +exec(src[:src.index('results = {')]) + +out = {} +for fx in FIXTURES: + clean = load(os.path.join(S, 'fx', fx + '.mkv'), 'yuv420p', frames=NFR) + out[fx] = {} + for sigma in SIGMAS: + noisy = add_noise(clean, sigma) + out[fx][str(sigma)] = {} + for r in (0, 1): + sweep = [(m * sigma, psnr(wnnm(noisy, m * sigma, r), clean, 0)) for m in (2.0, 2.5, 3.0, 3.5, 4.0)] + best = max(sweep, key=lambda t: t[1]) + out[fx][str(sigma)][f'r={r}'] = {'sweep': sweep, 'best': best, + 'psnr_u': psnr(wnnm(noisy, best[0], r), clean, 1)} + print(f'{fx} sigma {sigma} WNNM r={r}: best {best[0]:g} -> Y {best[1]:.2f} ' + f'U {out[fx][str(sigma)][f"r={r}"]["psnr_u"]:.2f} sweep {[(s, round(p, 2)) for s, p in sweep]}', + flush=True) +json.dump(out, open(os.path.join(S, 'wnnm_wide.json'), 'w'), indent=1) diff --git a/docs/reports/2026-11-colourisation/data/colorize-ddcolor.conversion.json b/docs/reports/2026-11-colourisation/data/colorize-ddcolor.conversion.json new file mode 100644 index 00000000..f3be4d46 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/colorize-ddcolor.conversion.json @@ -0,0 +1,57 @@ +{ + "opset": 17, + "irVersion": 8, + "torch": "2.15.0", + "models": [ + { + "file": "colorize-ddcolor-tiny.onnx", + "size": 220687346, + "sha256": "9300e48e7612fec4a0fffc38eb005fdd65bcc7a859390d6de0db3abf44e46d1c", + "checkpoint": "https://huggingface.co/piddnad/ddcolor_paper_tiny/resolve/cf9fd99c1d7472689ec7413441c1b799a51866a3/pytorch_model.bin", + "checkpointSha256": "8a1277bc90a1bfbb6d2d83933a9a6bc821931879ca93e26e4fcec12165d41fce", + "source": "https://github.com/piddnad/DDColor/tree/2adb63f2656ac41cbdf7b894cddd94121a3faf13", + "rewrites": { + "replicationPadsRewritten": 4, + "einsumRewritten": true + }, + "parity": { + "512": { + "max": 0.001644134521484375, + "mean": 4.178590315567776e-05, + "unit": "CIELAB a/b", + "refRange": [ + -84.84383392333984, + 129.04098510742188 + ] + }, + "256": { + "max": 0.00128173828125, + "mean": 7.835150522073064e-05, + "unit": "CIELAB a/b", + "refRange": [ + -146.1870880126953, + 154.69163513183594 + ] + }, + "384": { + "max": 0.0017547607421875, + "mean": 4.324595244635058e-05, + "unit": "CIELAB a/b", + "refRange": [ + -101.91814422607422, + 134.89495849609375 + ] + }, + "640": { + "max": 0.0009765625, + "mean": 4.7702506353983765e-05, + "unit": "CIELAB a/b", + "refRange": [ + -88.1981430053711, + 148.0986785888672 + ] + } + } + } + ] +} \ No newline at end of file diff --git a/docs/reports/2026-11-colourisation/data/colorize-deoldify.conversion.json b/docs/reports/2026-11-colourisation/data/colorize-deoldify.conversion.json new file mode 100644 index 00000000..ec9f31fa --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/colorize-deoldify.conversion.json @@ -0,0 +1,60 @@ +{ + "checkpointTensors": 754, + "strictLoad": true, + "verification": { + "reference": "https://github.com/facefusion/facefusion-assets/releases/download/models-3.0.0/deoldify.onnx", + "referenceSha256": "a00de09afacbed28b30ea63d174e1ade855b497c0fc61372217953c429696a9a", + "inputShape": [ + 1, + 3, + 256, + 256 + ], + "max": 1.8477439880371094e-06, + "mean": 9.928839972417336e-08, + "unit": "[0, 1] RGB" + }, + "replicationPadsRewritten": 5, + "opset": 17, + "irVersion": 8, + "torch": "2.15.0", + "models": [ + { + "file": "colorize-deoldify-video.onnx", + "size": 872841884, + "sha256": "aadb46c7d44af94ba0d01b1d3069f90ec83d08c14c60f0d203b3962094835e5a", + "checkpoint": "https://data.deepai.org/deoldify/ColorizeVideo_gen.pth", + "checkpointSha256": "e3d98bb6a222354c79f95485c2f078a89dc724e9183662506d9e0f5aafac83ad", + "source": "https://github.com/jantic/DeOldify/tree/5f86c28923799036ad4b1bf7af8c629ac65efd75", + "parity": { + "336": { + "max": 1.2367963790893555e-06, + "mean": 1.1977064370467008e-07, + "unit": "[0, 1] RGB", + "refRange": [ + 0.11664381623268127, + 0.7518526315689087 + ] + }, + "256": { + "max": 1.430511474609375e-06, + "mean": 2.0082567668093057e-07, + "unit": "[0, 1] RGB", + "refRange": [ + 0.12117752432823181, + 0.7370831966400146 + ] + }, + "512": { + "max": 9.5367431640625e-07, + "mean": 1.0259155184636863e-07, + "unit": "[0, 1] RGB", + "refRange": [ + 0.1116834282875061, + 0.7427202463150024 + ] + } + } + } + ] +} \ No newline at end of file diff --git a/docs/reports/2026-11-colourisation/data/coreml_notes.md b/docs/reports/2026-11-colourisation/data/coreml_notes.md new file mode 100644 index 00000000..c5c3d259 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/coreml_notes.md @@ -0,0 +1,26 @@ +# Core ML observations not in the JSON files + +Exploratory runs of `scripts/coreml_variants.py` on colorize-ddcolor-tiny.onnx +(the shipped, one-partition export), 512x512, conversion venv, ONNX Runtime +1.31.1, M1 Pro, idle machine: + +| Options | Session | ms/run | +|---|---|---| +| MLProgram, ALL | 80.2 s | 101.0 | +| MLProgram, CPUAndGPU | 50.8 s | 102.1 | +| MLProgram, CPUAndNeuralEngine | 56.3 s | 389.3 | +| NeuralNetwork, ALL | 9.9 s | 3017.6 | +| MLProgram, ALL, ModelCacheDirectory (first session) | 80.0 s | 101.2 | +| MLProgram, ALL, ModelCacheDirectory (second session) | 14.5 s | 101.8 | + +The cache directory held 898 MB after the first session (one subdirectory, +named from a hash; its key is the model's path, so another render size of the +same file would look the same to it). + +`provider_bench_unrewritten_ddcolor.jsonl`: the same options as the first +row, on the export without the Core ML rewrites +(`scripts/export_unrewritten_ddcolor.py`): 75.6 s session, 261.0 ms per run. + +Earlier, with free height and width (no overrides), Core ML failed to build +the six-partition DDColor at all ("Failed to build the model execution plan +using a model architecture file"). diff --git a/docs/reports/2026-11-colourisation/data/flicker.json b/docs/reports/2026-11-colourisation/data/flicker.json new file mode 100644 index 00000000..d4510832 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/flicker.json @@ -0,0 +1,87 @@ +{ + "block": 8, + "still": 1.0, + "moving": 4.0, + "rows": { + "ddcolor_r0": { + "staticFlicker": 1.446, + "staticChange": 1.828, + "stillShare": 0.501, + "chroma": 31.837, + "motionLag": 0.0, + "movingShare": 0.056 + }, + "ddcolor_r1": { + "staticFlicker": 0.499, + "staticChange": 0.9, + "stillShare": 0.501, + "chroma": 31.839, + "motionLag": 0.965, + "movingShare": 0.056 + }, + "ddcolor_r2": { + "staticFlicker": 0.233, + "staticChange": 0.578, + "stillShare": 0.501, + "chroma": 31.844, + "motionLag": 1.326, + "movingShare": 0.056 + }, + "ddcolor_r3": { + "staticFlicker": 0.153, + "staticChange": 0.439, + "stillShare": 0.501, + "chroma": 31.851, + "motionLag": 1.466, + "movingShare": 0.056 + }, + "ddcolor_r2nomotion": { + "staticFlicker": 0.227, + "staticChange": 0.575, + "stillShare": 0.501, + "chroma": 31.838, + "motionLag": 2.0, + "movingShare": 0.056 + }, + "deoldify_r0": { + "staticFlicker": 0.325, + "staticChange": 0.399, + "stillShare": 0.501, + "chroma": 7.145, + "motionLag": 0.0, + "movingShare": 0.056 + }, + "deoldify_r1": { + "staticFlicker": 0.101, + "staticChange": 0.186, + "stillShare": 0.501, + "chroma": 7.138, + "motionLag": 0.195, + "movingShare": 0.056 + }, + "deoldify_r2": { + "staticFlicker": 0.053, + "staticChange": 0.123, + "stillShare": 0.501, + "chroma": 7.135, + "motionLag": 0.276, + "movingShare": 0.056 + }, + "deoldify_r3": { + "staticFlicker": 0.033, + "staticChange": 0.092, + "stillShare": 0.501, + "chroma": 7.133, + "motionLag": 0.318, + "movingShare": 0.056 + }, + "deoldify_r2nomotion": { + "staticFlicker": 0.049, + "staticChange": 0.121, + "stillShare": 0.501, + "chroma": 7.134, + "motionLag": 0.481, + "movingShare": 0.056 + } + } +} diff --git a/docs/reports/2026-11-colourisation/data/lab_parity.json b/docs/reports/2026-11-colourisation/data/lab_parity.json new file mode 100644 index 00000000..f7cc4635 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/lab_parity.json @@ -0,0 +1 @@ +{"opencv": "5.0.0", "pixels": 65536, "exprVsOpenCV": {"maxLevels": [3.8456, 3.7817, 2.0103], "meanLevels": [0.11864, 0.18737, 0.13204]}, "exprVsExactFormula": {"maxLevels": [0.0014, 0.0006, 0.0002], "meanLevels": [1e-05, 1e-05, 1e-05]}, "exactFormulaVsOpenCV": {"maxLevels": [3.8471, 3.7816, 2.0101], "meanLevels": [0.11864, 0.18737, 0.13203]}} diff --git a/docs/reports/2026-11-colourisation/data/provider_bench.jsonl b/docs/reports/2026-11-colourisation/data/provider_bench.jsonl new file mode 100644 index 00000000..b6c95429 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/provider_bench.jsonl @@ -0,0 +1,12 @@ +{"model": "colorize-ddcolor-tiny.onnx", "size": 256, "provider": "CPUExecutionProvider", "fixedShape": true, "sessionSeconds": 0.23, "msPerRun": 205.0, "onnxruntime": "1.31.1"} +{"model": "colorize-ddcolor-tiny.onnx", "size": 256, "provider": "CoreMLExecutionProvider", "fixedShape": true, "sessionSeconds": 76.19, "msPerRun": 30.8, "onnxruntime": "1.31.1", "maxDiffVsCpu": 0.00066375732421875, "meanDiffVsCpu": 3.057322101085447e-05} +{"model": "colorize-ddcolor-tiny.onnx", "size": 384, "provider": "CPUExecutionProvider", "fixedShape": true, "sessionSeconds": 0.24, "msPerRun": 442.8, "onnxruntime": "1.31.1"} +{"model": "colorize-ddcolor-tiny.onnx", "size": 384, "provider": "CoreMLExecutionProvider", "fixedShape": true, "sessionSeconds": 76.47, "msPerRun": 61.9, "onnxruntime": "1.31.1", "maxDiffVsCpu": 0.0003509521484375, "meanDiffVsCpu": 1.0953920536849182e-05} +{"model": "colorize-ddcolor-tiny.onnx", "size": 512, "provider": "CPUExecutionProvider", "fixedShape": true, "sessionSeconds": 0.27, "msPerRun": 746.9, "onnxruntime": "1.31.1"} +{"model": "colorize-ddcolor-tiny.onnx", "size": 512, "provider": "CoreMLExecutionProvider", "fixedShape": true, "sessionSeconds": 75.93, "msPerRun": 100.6, "onnxruntime": "1.31.1", "maxDiffVsCpu": 0.0002899169921875, "meanDiffVsCpu": 1.257876920135459e-05} +{"model": "colorize-deoldify-video.onnx", "size": 256, "provider": "CPUExecutionProvider", "fixedShape": true, "sessionSeconds": 0.27, "msPerRun": 895.8, "onnxruntime": "1.31.1"} +{"model": "colorize-deoldify-video.onnx", "size": 256, "provider": "CoreMLExecutionProvider", "fixedShape": true, "sessionSeconds": 4.33, "msPerRun": 88.0, "onnxruntime": "1.31.1", "maxDiffVsCpu": 9.834766387939453e-07, "meanDiffVsCpu": 1.5411698939260532e-07} +{"model": "colorize-deoldify-video.onnx", "size": 336, "provider": "CPUExecutionProvider", "fixedShape": true, "sessionSeconds": 0.44, "msPerRun": 1538.2, "onnxruntime": "1.31.1"} +{"model": "colorize-deoldify-video.onnx", "size": 336, "provider": "CoreMLExecutionProvider", "fixedShape": true, "sessionSeconds": 3.62, "msPerRun": 154.3, "onnxruntime": "1.31.1", "maxDiffVsCpu": 0.005722701549530029, "meanDiffVsCpu": 0.0012882634764537215} +{"model": "colorize-deoldify-video.onnx", "size": 448, "provider": "CPUExecutionProvider", "fixedShape": true, "sessionSeconds": 0.32, "msPerRun": 2797.9, "onnxruntime": "1.31.1"} +{"model": "colorize-deoldify-video.onnx", "size": 448, "provider": "CoreMLExecutionProvider", "fixedShape": true, "sessionSeconds": 3.66, "msPerRun": 255.5, "onnxruntime": "1.31.1", "maxDiffVsCpu": 9.238719940185547e-07, "meanDiffVsCpu": 8.040913002105299e-08} diff --git a/docs/reports/2026-11-colourisation/data/provider_bench_unrewritten_ddcolor.jsonl b/docs/reports/2026-11-colourisation/data/provider_bench_unrewritten_ddcolor.jsonl new file mode 100644 index 00000000..ca977833 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/provider_bench_unrewritten_ddcolor.jsonl @@ -0,0 +1 @@ +{"opt": {"ModelFormat": "MLProgram", "MLComputeUnits": "ALL"}, "session": 75.58, "ms": 261.0} diff --git a/docs/reports/2026-11-colourisation/data/results.jsonl b/docs/reports/2026-11-colourisation/data/results.jsonl new file mode 100644 index 00000000..2a2bc5b6 --- /dev/null +++ b/docs/reports/2026-11-colourisation/data/results.jsonl @@ -0,0 +1,24 @@ +{"tag": "ddcolor_r0", "model": "ddcolor", "renderSize": 512, "radius": 0, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 76.98, "seconds": 12.63, "fps": 9.503, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "ddcolor_r1", "model": "ddcolor", "renderSize": 512, "radius": 1, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 77.0, "seconds": 13.15, "fps": 9.125, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "ddcolor_r2", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 77.72, "seconds": 13.27, "fps": 9.042, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "ddcolor_r3", "model": "ddcolor", "renderSize": 512, "radius": 3, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 77.14, "seconds": 13.17, "fps": 9.112, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "ddcolor_r2nomotion", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": 1000000000.0, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 76.67, "seconds": 13.08, "fps": 9.176, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "deoldify_r0", "model": "deoldify", "renderSize": 336, "radius": 0, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 4.23, "seconds": 19.46, "fps": 6.166, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "deoldify_r1", "model": "deoldify", "renderSize": 336, "radius": 1, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 3.61, "seconds": 19.29, "fps": 6.22, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "deoldify_r2", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 3.17, "seconds": 19.35, "fps": 6.203, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "deoldify_r3", "model": "deoldify", "renderSize": 336, "radius": 3, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 3.16, "seconds": 19.42, "fps": 6.18, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "deoldify_r2nomotion", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": 1000000000.0, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 3.18, "seconds": 19.36, "fps": 6.199, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_sd_ddcolor_coreml", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 77.6, "seconds": 13.22, "fps": 9.076, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_sd_deoldify_coreml", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 120, "setupSeconds": 3.48, "seconds": 20.13, "fps": 5.962, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_hd_ddcolor_coreml", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "1920x1080", "frames": 24, "setupSeconds": 78.82, "seconds": 2.86, "fps": 8.38, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_hd_deoldify_coreml", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "1920x1080", "frames": 24, "setupSeconds": 3.87, "seconds": 4.01, "fps": 5.979, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_sd_ddcolor_cpu", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 24, "setupSeconds": 0.41, "seconds": 18.01, "fps": 1.333, "provider": "CPUExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_sd_deoldify_cpu", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 24, "setupSeconds": 0.3, "seconds": 37.07, "fps": 0.647, "provider": "CPUExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_hd_ddcolor_cpu", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "1920x1080", "frames": 12, "setupSeconds": 0.39, "seconds": 9.07, "fps": 1.323, "provider": "CPUExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_hd_deoldify_cpu", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "1920x1080", "frames": 12, "setupSeconds": 0.26, "seconds": 18.67, "fps": 0.643, "provider": "CPUExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_preview_ddcolor", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": true, "size": "640x480", "frames": 5, "setupSeconds": 0.4, "seconds": 3.84, "fps": 1.301, "provider": "CPUExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "speed_preview_deoldify", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": true, "size": "640x480", "frames": 5, "setupSeconds": 3.21, "seconds": 1.05, "fps": 4.782, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "colour_ddcolor", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 1, "setupSeconds": 76.7, "seconds": 0.35, "fps": 2.869, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "colour_deoldify", "model": "deoldify", "renderSize": 336, "radius": 2, "strength": 1.0, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 1, "setupSeconds": 3.38, "seconds": 0.28, "fps": 3.591, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "ddcolor_s05", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 0.5, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 5, "setupSeconds": 75.72, "seconds": 0.75, "fps": 6.665, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} +{"tag": "ddcolor_s15", "model": "ddcolor", "renderSize": 512, "radius": 2, "strength": 1.5, "depth": 8, "motionLimit": null, "preview": false, "size": "640x480", "frames": 5, "setupSeconds": 76.01, "seconds": 0.74, "fps": 6.787, "provider": "CoreMLExecutionProvider", "fallback": null, "lumaBitExact": true} diff --git a/docs/reports/2026-11-colourisation/img/01-models.png b/docs/reports/2026-11-colourisation/img/01-models.png new file mode 100644 index 00000000..891660d3 Binary files /dev/null and b/docs/reports/2026-11-colourisation/img/01-models.png differ diff --git a/docs/reports/2026-11-colourisation/img/02-smoothing.png b/docs/reports/2026-11-colourisation/img/02-smoothing.png new file mode 100644 index 00000000..266d1462 Binary files /dev/null and b/docs/reports/2026-11-colourisation/img/02-smoothing.png differ diff --git a/docs/reports/2026-11-colourisation/img/03-strength.png b/docs/reports/2026-11-colourisation/img/03-strength.png new file mode 100644 index 00000000..00d9fcc9 Binary files /dev/null and b/docs/reports/2026-11-colourisation/img/03-strength.png differ diff --git a/docs/reports/2026-11-colourisation/img/04-colour-source.png b/docs/reports/2026-11-colourisation/img/04-colour-source.png new file mode 100644 index 00000000..91c62cec Binary files /dev/null and b/docs/reports/2026-11-colourisation/img/04-colour-source.png differ diff --git a/docs/reports/2026-11-colourisation/index.html b/docs/reports/2026-11-colourisation/index.html new file mode 100644 index 00000000..bc1c6721 --- /dev/null +++ b/docs/reports/2026-11-colourisation/index.html @@ -0,0 +1,141 @@ + + + + + +Colourisation + + + +
+

Colourisation

+

DDColor-T and DeOldify Video on ONNX Runtime, as a VapourSynth pass that keeps the source's luma bit-exact. 2026-11-08, macOS 27 on an M1 Pro (16 GB). Scripts: scripts/; raw results: data/. To reproduce: Scripts/models/make_colorize_ddcolor.py and make_colorize_deoldify.py (conversion venv), scripts/make_refs.sh, scripts/bench.sh, scripts/flicker.py, scripts/lab_reference.py + lab_parity.py, Scripts/models/colorize_provider_bench.py, scripts/export_unrewritten_ddcolor.py + coreml_variants.py (notes) and scripts/make_figures.sh.

+ +
+Outcome: built. A Colourisation pass between Chroma Fixes and Color Correction, on the ai-runtime add-on, with one add-on per model: colorize-ddcolor (DDColor-T, Apache-2.0, 220.7 MB, the default) and colorize-deoldify (DeOldify's Video generator, MIT, 872.8 MB). The source's luma is kept bit-exact (8, 10 and 16 bits); only U and V are replaced. Both conversions match PyTorch (DDColor ≤1.8e-3 CIELAB units, DeOldify ≤1.4e-6) and reproduce byte for byte; DeOldify, rebuilt without fastai, matches an independent export to 1.8e-6. Averaging each frame's colour with two neighbours either side, weighted to zero where the picture moved, cuts colour flicker by 84% for both models. On Core ML: DDColor 9.1 fps SD / 8.4 fps 1080p after a 77 s compile per process (so previews run it on the CPU), DeOldify 6.0 fps after 3 s; 7–9× the CPU. +
+ +

1. Licences, read upstream

+ + + + + + +
ComponentLicence (as upstream states it)Holder namedDecision
DDColor code (piddnad/DDColor @2adb63f)Apache-2.0 (LICENSE)none: the LICENSE's copyright line is the unfilled template; authors per the README's citation (Kang, Yang, Ouyang, Ren, Li, Xie)converted
DDColor-T weights (huggingface.co/piddnad/ddcolor_paper_tiny @cf9fd99c)Apache-2.0 (model card's license field)as abovecolorize-ddcolor add-on, 220.7 MB
DeOldify code (jantic/DeOldify @5f86c28)MIT (LICENSE)Copyright (c) 2018 Jason Anticnetwork rebuilt from it
DeOldify Video generator (ColorizeVideo_gen.pth)MIT: the README's "Pretrained Weights" are "trained by ourselves and are released under the MIT license"Jason Anticcolorize-deoldify add-on, 872.8 MB
+

Neither is non-commercial, so neither add-on carries a usageNote. DDColor's network code adapts ConvNeXt (MIT), DETR and Mask2Former; DeOldify's builds on fastai (Apache-2.0) and torchvision's ResNet-101 (BSD-3-Clause); NOTICES says so in each entry. The DeOldify weights' host (data.deepai.org) has been unreliable in the past; the file found there (874,066,230 bytes) has the same SHA-256 (e3d98bb6…) as an independent mirror at huggingface.co/spensercai/DeOldify, so the pin is cross-checked, and since MIT permits it the converted ONNX is hosted with the other add-on files rather than fetched from upstream.

+ +

2. Conversion to ONNX

+

Both exports take one grey channel in [0, 1] (luma, (1, 1, H, W)) with H and W dynamic, named height/width; DDColor returns CIELAB a/b (ab), DeOldify an RGB picture in [0, 1] (rgb). TorchScript exporter, opset 17 / IR 8 (loadable by ONNX Runtime 1.23.2, the Intel-Mac floor). Re-running each script reproduces its file byte for byte (checked twice for each final file).

+ + + + +
FileSizeParity, ONNX Runtime (CPU) vs PyTorch, at render sizesNotes
colorize-ddcolor-tiny.onnx220.7 MBmax |Δ| 1.6e-3 / 1.3e-3 / 1.8e-3 / 9.8e-4 at 512 / 256 / 384 / 640 (CIELAB units; outputs span ±150); mean ≤7.8e-5loaded strict (upstream loads strict=False; nothing was missing)
colorize-deoldify-video.onnx872.8 MBmax |Δ| 1.2e-6 / 1.4e-6 / 1.0e-6 at 336 / 256 / 512 ([0, 1] RGB)verified against FaceFusion's independent export: max 1.8e-6
+

DeOldify without fastai

+

DeOldify is written against fastai 1.0.60, which does not install with a current PyTorch. The generator is rebuilt module for module in plain PyTorch (DynamicUnetWide on a ResNet-101, nf_factor 2, spectral norm, self-attention in the second decoder block, SigmoidRange(-3, 3)); the checkpoint's 754 tensors load with strict=True. To catch what a key-for-key load cannot, the result is compared with FaceFusion's deoldify.onnx (models-3.0.0), a conversion of the same weights made with fastai itself, used only for this check. The first comparison failed: mean 0.016, max 0.12 on [0, 1] RGB. The cause was fastai's PixelShuffle_ICNR: it takes blur=False but always builds the blur layer and tests the module (always true) rather than the flag, so the final upsample is always blurred. With the blur, the two agree to 1.8e-6.

+

Rewrites for Core ML

+

Core ML's ML Program format has no edge-mode Pad (ReplicationPad2d, in every blur), no Expand (Tensor.repeat) and no Einsum (DDColor's colour decoder). Each splits the graph: DDColor's first export ran as six Core ML partitions, at 261 ms per frame at 512. The exports write each as an exact equivalent (pads as slices and concatenations, the repeat as a concatenation, the einsum bqc,bchw→bqhw as a matrix product over the flattened image), parity unchanged; both graphs are now one partition (DDColor 936 of 936 nodes, DeOldify 341 of 341), and DDColor runs at 101 ms per frame, 2.6× faster (data; the compile time did not change). Spectral and weight norms are baked into the weights (remove_spectral_norm, which computes the same quotient), which also removed 50 MB of duplicated weights from DDColor's file.

+

Fixed shapes from one file

+

Core ML refuses DDColor with free height and width (Failed to build the model execution plan), and Face Restoration found Core ML needs static shapes. Rather than ship a file per render size, each session fixes the named dimensions with ONNX Runtime's free-dimension overrides (ai_runtime.session(..., dims={"height": n, "width": n}), new), so one 870 MB file serves every DeOldify render factor.

+ +

3. The pass

+
    +
  • Input to the model: the luma plane as a grey picture in [0, 1] (range-expanded), stretched with a bilinear resize to a square, as both upstreams do (DeOldify: "a simple stretch to fit a square really makes a big difference"). DDColor at 512 (upstream's input_size; 256–768 in steps of 32), DeOldify at render factor 21 × 16 = 336 (upstream's default for video; 10–40).
  • +
  • DDColor's colour: a and b are scaled to the frame (bilinear: upstream's F.interpolate default is nearest-neighbour, which leaves 2-pixel colour blocks at SD and 4-pixel ones at 1080p) and recombined with each pixel's own lightness, as upstream's pipeline does with the original L. The CIELAB → sRGB conversion is a chain of six std.Expr stages (akarin's JIT where present) at the luma's resolution, with OpenCV's constants.
  • +
  • DeOldify's colour: the RGB picture is scaled to the frame (bilinear, as upstream's _unsquare). Upstream then keeps its chroma via OpenCV's BT.601 YUV with the source's Y. The pass does the same in the source's own matrix.
  • +
  • Luma kept bit-exact: the colour picture becomes float YUV with the source's matrix; U and V are scaled by Colour Strength; the result is converted to the source's own format (depth, subsampling, chroma siting, dithered) and only U and V are taken (ShufflePlanes([clip, colour, colour], [0, 1, 2])). Verified bit-exact on every frame at 8, 10 and 16 bits and on a field-based clip (lumaBitExact in results.jsonl, and the heavy test).
  • +
+
Source, DDColor and DeOldify Video at frames 20 and 110
Figure 1. Frames 20 (the close-up) and 110 (mid-dissolve) of the fixture: the source, DDColor, DeOldify Video, each at its defaults. DDColor is far more colourful (31.8 levels of chroma from neutral on average, against DeOldify's 7.1) and warms everything, white bedding included; DeOldify is muted, with the same skin and hair choices. Neither colour is a record of the 1945 set.
+
DDColor at colour strength 0.5, 1 and 1.5
Figure 3. Colour Strength 0.5, 1 (the default) and 1.5 on DDColor, frame 110. Strength scales U and V about neutral, so 0 returns the input exactly and the luma never changes.
+ +

Checking the CIELAB chain

+

On 65,536 synthetic (grey value, a, b) triples (lab_reference.py → lab_parity.py, data): the Expr chain matches the same conversion in float64 with the exact sRGB curve to 0.0014 of an 8-bit level. Against OpenCV's float COLOR_LAB2RGB (what DDColor's pipeline calls) the mean difference is 0.12–0.19 levels per channel, max 3.8 — all of it OpenCV's: its float path tabulates the sRGB curve, which is coarse near black (the worst pixel: linear 0.0026, exact sRGB 0.0338, OpenCV 0.0187). The exact formula is kept.

+ +

4. Temporal stability

+

Both models colour each frame afresh, and nothing ties one frame's guess to the next. The pass averages each frame's prediction (at the render size, before it is scaled up) with the predictions of the r frames either side, weighted by distance (r+1−|k|) and by a motion weight: the grey pictures are compared over 8×8-pixel blocks (of the render-size picture) and a neighbour counts for nothing where its block differs from this frame's by 10 levels or more (3×3-block average, falling linearly from 1 at no difference). A neighbour past a cut is excluded the same way. Everything is a pure function of frames n−r..n+r (ai_runtime.shifted clips into std.ModifyFrame, each frame's prediction shared through a FrameMemo), so the preview, which decodes r frames either side (the pass's FrameMap radius), shows the encode's frame.

+

Measured on the committed fixture (120 frames: a close-up with small head movements, a dissolve, a two-shot) with flicker.py, the face report's metric adapted to colour, in 8-bit chroma levels over 8×8 blocks of the chroma planes. Still flicker is the second difference |C[t] − (C[t−1] + C[t+1])/2| where the grey picture did not move (mean luma change under 1 level either side; 50% of blocks): there any change of colour is the model's. Motion lag is how far the smoothed colour sits from the per-frame colour where the picture moved by 4 levels or more (5.6% of blocks): ghosting, or colour trailing a moving edge, raises it. Colour is the mean distance of U and V from neutral, to show smoothing does not just fade the picture. data/flicker.json.

+ + + + + + + + + + + + +
VariantStill flickerStill frame-to-frame changeMotion lagColour
DDColor, per frame (r = 0)1.4461.828031.8
DDColor, r = 10.4990.9000.9731.8
DDColor, r = 2 (default)0.2330.5781.3331.8
DDColor, r = 30.1530.4391.4731.9
DDColor, r = 2, no motion weighting0.2270.5752.0031.8
DeOldify Video, per frame (r = 0)0.3250.39907.1
DeOldify Video, r = 10.1010.1860.207.1
DeOldify Video, r = 2 (default)0.0530.1230.287.1
DeOldify Video, r = 30.0330.0920.327.1
DeOldify Video, r = 2, no motion weighting0.0490.1210.487.1
+
    +
  • Smoothing works: at the default radius 2, DDColor's still-picture flicker falls 84% (1.446 → 0.233) and DeOldify's 84% (0.325 → 0.053), with no loss of colour.
  • +
  • DeOldify is steadier in absolute terms, not relative ones. Per frame it flickers 4.5× less than DDColor, but it also has 4.5× less colour (7.1 vs 31.8 levels from neutral): relative to how much colour each puts down, they flicker alike. DeOldify's "Video" training buys muted colour more than steadiness.
  • +
  • The motion weighting earns its place in motion, not in still areas: switching it off changes still flicker by under 0.01 but raises motion lag by half (DDColor 1.33 → 2.00, DeOldify 0.28 → 0.48) — that is colour from where an object was, painted where it is.
  • +
  • Radius 2 is the default: radius 3 removes another third of what flicker is left at the cost of more lag and a slower preview (each preview frame colours 2r+1 frames).
  • +
+
DDColor frame 110 per frame and smoothed
Figure 2. DDColor at frame 110 of the fixture (the two-shot): per frame (left) and with steady colour 2 (right). Single frames look alike; the difference is in motion, which a still cannot show.
+ +

5. Speed and providers

+

The whole pass (decode excluded), steady colour 2, default render sizes, on the fixture (SD, 640×480) and on the same seconds of the source at its own 1920×1080 (make_refs.sh). Setup is session creation, which on Core ML includes compiling the graph for this machine; it happens once per vspipe process. data/results.jsonl.

+ + + + + + +
ModelProviderSetupSD fps1080p fps
DDColor (512)Core ML77–79 s9.18.4
DDColor (512)CPU0.4 s1.331.32
DeOldify Video (336)Core ML3–4 s6.06.0
DeOldify Video (336)CPU0.3 s0.650.64
+
    +
  • The cost is the model, at a fixed size, so 1080p runs nearly as fast as SD: the CIELAB chain and the conversions at full resolution are a few percent.
  • +
  • Core ML is 7× (DDColor) and 9× (DeOldify) faster than the CPU here. DirectML (Windows) and CUDA are untested; Linux has only the CPU.
  • +
  • Core ML's compile of DDColor takes 77 s in every process. For an encode that is a one-off pause before the first frame (the script says so in the worker's debug log). For a preview, which is a process of its own that colours 2r+1 = 5 frames, it would be 77 s per click, so the preview runs DDColor on the CPU instead: 5 frames in 3.8 s. DeOldify compiles in 3 s and previews on Core ML (5 frames in 1.1 s after setup). For DDColor, Core ML and the CPU agree to under 0.001 CIELAB units (table below), far inside the preview = encode test's 0.5-level tolerance. Core ML's model cache (ModelCacheDirectory) brought a second DDColor session down to 14.5 s, but wrote 898 MB per render size, and its cache key is the model's path, which free-dimension overrides do not change; it is not used (notes). Face Restoration found the same option broke a second session outright.
  • +
+

The models alone, per provider and size

+

colorize_provider_bench.py, five runs each after a warm-up, sessions made as ai_runtime makes them (Core ML as an ML Program on all compute units, the size fixed by free-dimension overrides). data/provider_bench.jsonl.

+ + + + + + + + +
ModelSizeCPU msCore ML msCore ML sessionCore ML vs CPU, max / mean |Δ|
DDColor2562053176 s6.6e-4 / 3.1e-5 (CIELAB)
DDColor3844436276 s3.5e-4 / 1.1e-5
DDColor51274710176 s2.9e-4 / 1.3e-5
DeOldify Video256896884.3 s9.8e-7 / 1.5e-7 (RGB, [0, 1])
DeOldify Video33615381543.6 s5.7e-3 / 1.3e-3
DeOldify Video44827982563.7 s9.2e-7 / 8.0e-8
+

Core ML is 7× (DDColor) to 10–11× (DeOldify) faster than the CPU. Its output matches the CPU's to float precision except DeOldify at 336, where it differs by up to 1.5 8-bit levels (mean 0.33) — probably because Core ML places that size on different hardware (the Neural Engine computes in 16-bit floats); still far below what a viewer or the preview = encode check (0.5-level mean) would notice. The pipeline runs below these model rates (DDColor at 512 is 101 ms, the pass 9 fps) because model runs are serialised (ai_runtime.Runner) and the rest of the frame's work overlaps them. Untested here: DirectML (Windows) and CUDA; Linux has only the CPU.

+ +

6. Decisions

+
    +
  • Pipeline position: after Chroma Fixes, before Color Correction (Detail & Color stage). After the cleanup, because noise, blocks and dirt in the grey picture are what the model judges colour from, and colour added before a denoise or deblock would be smeared by it. After Chroma Fixes and Chroma Denoise because both work on the chroma the pass replaces (the app now says so when Chroma Denoise and Colourisation are both on). Before Color Correction, so the result can be graded — the obvious fix for a model that is too warm or too pale. Before Crop & Resize: the model sees a fixed-size square either way, and the colour is rebuilt at the source's size and then resized with the rest of the picture; it also means the pass reads the source's own colour tags, not the BT.709 conversion's.
  • +
  • A source that already has colour is coloured from its luma all the same: its chroma is replaced (figure 4; the heavy test checks the luma stays bit-exact). The alternatives were worse: blocking or skipping would be a non-advisory refusal; blending the model's colour into the existing colour would produce neither. Re-colouring a faded or badly tinted colour film from its luma is also a legitimate use. The panel's description says so; nothing is blocked.
  • +
  • Presets: none turns it on, including 8mm / Super 8 Film Scan. Most 8 mm home movies of the era are colour; switching on Colourisation would replace their real colour, and require a 220–870 MB download on first use. It stays a deliberate choice.
  • +
  • Defaults: DDColor (smaller download, more natural colour, runs faster on Core ML once compiled), strength 1, steady colour 2 frames either side.
  • +
+ +
A colour source, recoloured by DDColor and DeOldify Video
Figure 4. A source that is already in colour (the face-restoration fixture, NASA public-domain footage): the source, DDColor, DeOldify Video. The flags lose their real colours and the grey shirt turns blue — the models know only the brightness.
+ +

7. Not tested here

+
    +
  • Windows (DirectML), Linux (CPU only, where DDColor at 512 is about 0.8 s per frame), macOS x64 (Core ML on Intel, ONNX Runtime 1.23.2).
  • +
  • Whether the colour is right: there is no ground truth for a 1945 black-and-white film, and the flicker metric only measures steadiness.
  • +
  • One clip: a close-up, a dissolve and a two-shot, with little camera motion. Fast pans, where the motion weighting excludes most neighbours, are untested.
  • +
+
+ + diff --git a/docs/reports/2026-11-colourisation/scripts/bench.sh b/docs/reports/2026-11-colourisation/scripts/bench.sh new file mode 100644 index 00000000..f4952245 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/bench.sh @@ -0,0 +1,50 @@ +#!/bin/bash +# Every run behind the report's flicker and speed tables, in order. Run on an +# otherwise idle machine: the speed numbers are wall-clock. +# +# DEPS, ADDONS, VAPOURBOX_ADDON_STAGING as for run.sh +# WORK= outputs (.npy, .png, results.jsonl) +# HD= built by make_refs.sh +# +# The SD fixture is the committed Tests/TestResources/colorize_test.mp4. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${WORK:?set WORK to a scratch directory}" "${HD:?}" +FIX="$REPO/Tests/TestResources/colorize_test.mp4" +OUT="$WORK/runs" +mkdir -p "$OUT" +R() { timeout 1800 "$HERE/run.sh" colorize_run.py "$@" 2>/dev/null | grep '^{' | tee -a "$WORK/results.jsonl"; } + +# Flicker: each model per frame (radius 0) and smoothed over 1, 2 and 3 frames +# either side; and radius 2 with the motion weighting off (every neighbour +# blended in full), to show what the weighting is for. +for m in ddcolor deoldify; do + R --clip "$FIX" --model $m --radius 0 --save-uv --out "$OUT" --tag ${m}_r0 --png 20 --png 110 + R --clip "$FIX" --model $m --radius 1 --save-uv --out "$OUT" --tag ${m}_r1 + R --clip "$FIX" --model $m --radius 2 --save-uv --out "$OUT" --tag ${m}_r2 --png 20 --png 110 + R --clip "$FIX" --model $m --radius 3 --save-uv --out "$OUT" --tag ${m}_r3 + R --clip "$FIX" --model $m --radius 2 --motion-limit 1e9 --save-uv --out "$OUT" --tag ${m}_r2nomotion +done +# Strength, for the figure (frame 110, its smoothing neighbours included). +R --clip "$FIX" --model ddcolor --strength 0.5 --frames 108:113 --out "$OUT" --tag ddcolor_s05 --png 2 +R --clip "$FIX" --model ddcolor --strength 1.5 --frames 108:113 --out "$OUT" --tag ddcolor_s15 --png 2 + +# Speed: SD (the fixture) and 1080p, the preferred provider (Core ML) vs the +# CPU, at each model's default size and radius 2. setupSeconds is session +# creation, which includes Core ML's compilation. +R --clip "$FIX" --model ddcolor --tag speed_sd_ddcolor_coreml +R --clip "$FIX" --model deoldify --tag speed_sd_deoldify_coreml +R --clip "$HD" --model ddcolor --tag speed_hd_ddcolor_coreml +R --clip "$HD" --model deoldify --tag speed_hd_deoldify_coreml +VAPOURBOX_AI_FORCE_CPU=1 R --clip "$FIX" --model ddcolor --frames 0:24 --tag speed_sd_ddcolor_cpu +VAPOURBOX_AI_FORCE_CPU=1 R --clip "$FIX" --model deoldify --frames 0:24 --tag speed_sd_deoldify_cpu +VAPOURBOX_AI_FORCE_CPU=1 R --clip "$HD" --model ddcolor --frames 0:12 --tag speed_hd_ddcolor_cpu +VAPOURBOX_AI_FORCE_CPU=1 R --clip "$HD" --model deoldify --frames 0:12 --tag speed_hd_deoldify_cpu +# A preview's process: DDColor on the CPU (preview=True), five frames. +R --clip "$FIX" --model ddcolor --preview --frames 50:55 --tag speed_preview_ddcolor +R --clip "$FIX" --model deoldify --preview --frames 50:55 --tag speed_preview_deoldify + +# Already-colour input: the face fixture (colour), recoloured by each model. +R --clip "$REPO/Tests/TestResources/face_restore_test.mp4" --model ddcolor --frames 60:61 --out "$OUT" --tag colour_ddcolor --png 0 +R --clip "$REPO/Tests/TestResources/face_restore_test.mp4" --model deoldify --frames 60:61 --out "$OUT" --tag colour_deoldify --png 0 diff --git a/docs/reports/2026-11-colourisation/scripts/colorize_run.py b/docs/reports/2026-11-colourisation/scripts/colorize_run.py new file mode 100755 index 00000000..4612ac63 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/colorize_run.py @@ -0,0 +1,128 @@ +#!/usr/bin/env python3 +"""Run worker/templates/colorize.py on a clip inside the deps bundle's Python, +time it, and save what the flicker and image comparisons need. + +Run with the deps bundle's Python and VapourSynth importable, the repo's +worker/templates and the ai-runtime add-on's site-packages on PYTHONPATH, +ffmpeg/ffprobe on PATH and VAPOURBOX_ADDON_STAGING pointing at the converted +.onnx files (run.sh does all of this). VAPOURBOX_AI_FORCE_CPU=1 runs the +model on the CPU. + + colorize_run.py --clip FILE [--model ddcolor|deoldify] [--size N] + [--radius R] [--strength S] [--frames A:B] [--depth 8|10|16] + [--png N ...] [--save-uv] [--out DIR] [--tag NAME] + [--preview] [--field-based 1|2] + +Prints one JSON line: frames, seconds, fps, provider, any CPU fallback, and +whether the output's Y plane equals the input's (it must). +""" +import argparse +import json +import os +import subprocess +import time + +import numpy as np +import vapoursynth as vs + +core = vs.core +STAGING = os.environ.get("VAPOURBOX_ADDON_STAGING", "") +FILES = {"ddcolor": "colorize-ddcolor-tiny.onnx", "deoldify": "colorize-deoldify-video.onnx"} + + +def load(path, frames=None): + """A clip of `path` decoded by ffmpeg to 8-bit 4:2:0.""" + info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=width,height", "-of", "json", path]))["streams"][0] + w, h = info["width"], info["height"] + raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + per = w * h * 3 // 2 + data = np.frombuffer(raw, np.uint8) + n = len(data) // per + a, b = frames if frames else (0, n) + blank = core.std.BlankClip(width=w, height=h, format=vs.YUV420P8, length=b - a, fpsnum=24, fpsden=1) + sizes = [(h, w), (h // 2, w // 2), (h // 2, w // 2)] + + def fill(n, f): + out = f.copy() + off = (a + n) * per + for p, (ph, pw) in enumerate(sizes): + np.copyto(np.asarray(out[p]), data[off:off + ph * pw].reshape(ph, pw)) + off += ph * pw + return out + + return core.std.ModifyFrame(blank, blank, fill) + + +def save_png(clip, n, path): + rgb = core.resize.Bicubic(clip[n], format=vs.RGB24, matrix_in_s="470bg", dither_type="error_diffusion") + f = rgb.get_frame(0) + arr = np.ascontiguousarray(np.stack([np.asarray(f[p]) for p in range(3)], -1)) + subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", + f"{clip.width}x{clip.height}", "-i", "-", "-frames:v", "1", path], + input=arr.tobytes(), check=True) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--clip", required=True) + ap.add_argument("--model", default="ddcolor", choices=sorted(FILES)) + ap.add_argument("--size", type=int, default=None) + ap.add_argument("--radius", type=int, default=2) + ap.add_argument("--strength", type=float, default=1.0) + ap.add_argument("--frames", default=None) + ap.add_argument("--depth", type=int, default=8) + ap.add_argument("--png", type=int, action="append", default=[]) + ap.add_argument("--save-uv", action="store_true") + ap.add_argument("--out", default=".") + ap.add_argument("--tag", default="run") + ap.add_argument("--preview", action="store_true") + ap.add_argument("--motion-limit", type=float, default=None, help="8-bit levels") + ap.add_argument("--field-based", type=int, default=None) + args = ap.parse_args() + import colorize + + size = args.size or (512 if args.model == "ddcolor" else 336) + frames = tuple(int(x) for x in args.frames.split(":")) if args.frames else None + src = load(args.clip, frames) + if args.depth != 8: + src = core.resize.Point(src, format=core.query_video_format(vs.YUV, vs.INTEGER, args.depth, 1, 1).id) + if args.field_based is not None: + # As the templates' SET_FIELD_BASED leaves a still-interlaced clip. + src = core.std.SetFieldBased(src, args.field_based) + t0 = time.perf_counter() + # Mirrors the generated script's call (worker/templates/*_template.vpy). + out = colorize.colorize(src, model_path=os.path.join(STAGING, FILES[args.model]), model=args.model, + render_size=size, strength=args.strength, temporal_radius=args.radius, + matrix="470bg", color_range="limited", preview=args.preview, + **({"motion_limit": args.motion_limit / 255.0} if args.motion_limit else {})) + setup = time.perf_counter() - t0 + os.makedirs(args.out, exist_ok=True) + ys, us, vs_, y_exact = [], [], [], True + t0 = time.perf_counter() + for i, (f, g) in enumerate(zip(out.frames(close=True), src.frames(close=True))): + if not np.array_equal(np.asarray(f[0]), np.asarray(g[0])): + y_exact = False + if args.save_uv: + ys.append(np.asarray(f[0]).copy()) + us.append(np.asarray(f[1]).copy()) + vs_.append(np.asarray(f[2]).copy()) + dt = time.perf_counter() - t0 + for n in args.png: + save_png(out, n, os.path.join(args.out, f"{args.tag}_{n}.png")) + if args.save_uv: + np.save(os.path.join(args.out, f"{args.tag}_y.npy"), np.stack(ys)) + np.save(os.path.join(args.out, f"{args.tag}_u.npy"), np.stack(us)) + np.save(os.path.join(args.out, f"{args.tag}_v.npy"), np.stack(vs_)) + runner = colorize.last_runner + print(json.dumps({ + "tag": args.tag, "model": args.model, "renderSize": size, "radius": args.radius, + "strength": args.strength, "depth": args.depth, "motionLimit": args.motion_limit, "preview": args.preview, + "size": f"{src.width}x{src.height}", "frames": out.num_frames, + "setupSeconds": round(setup, 2), "seconds": round(dt, 2), "fps": round(out.num_frames / dt, 3), + "provider": runner.provider, "fallback": runner.fallback, "lumaBitExact": y_exact, + }), flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-colourisation/scripts/coreml_variants.py b/docs/reports/2026-11-colourisation/scripts/coreml_variants.py new file mode 100644 index 00000000..2f37bad9 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/coreml_variants.py @@ -0,0 +1,48 @@ +#!/usr/bin/env python3 +"""Session time and per-run time of one model on Core ML with given provider +options, at a fixed square size (free-dimension overrides, as ai_runtime does). +Used for the unrewritten-DDColor comparison and the model-cache trial. + +Run in the conversion venv: + + python coreml_variants.py '' ['' ...] + +e.g. '{"ModelFormat": "MLProgram", "MLComputeUnits": "ALL"}', or with +"ModelCacheDirectory": "" twice in a row to time a cached second session. +Prints one JSON line per option set. +""" +import json +import sys +import time + +import numpy as np +import onnxruntime as ort + + +def main(): + path, size = sys.argv[1], int(sys.argv[2]) + x = np.random.default_rng(0).uniform(0, 1, (1, 1, size, size)).astype(np.float32) + for opt in (json.loads(v) for v in sys.argv[3:]): + opts = ort.SessionOptions() + opts.log_severity_level = 3 + opts.add_free_dimension_override_by_name("height", size) + opts.add_free_dimension_override_by_name("width", size) + t0 = time.perf_counter() + try: + s = ort.InferenceSession(path, sess_options=opts, + providers=[("CoreMLExecutionProvider", opt), "CPUExecutionProvider"]) + except Exception as e: + print(json.dumps({"opt": opt, "error": str(e)[:200]}), flush=True) + continue + created = time.perf_counter() - t0 + feeds = {s.get_inputs()[0].name: x} + s.run(None, feeds) + t0 = time.perf_counter() + for _ in range(3): + s.run(None, feeds) + print(json.dumps({"opt": opt, "session": round(created, 2), + "ms": round((time.perf_counter() - t0) / 3 * 1000, 1)}), flush=True) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-colourisation/scripts/export_unrewritten_ddcolor.py b/docs/reports/2026-11-colourisation/scripts/export_unrewritten_ddcolor.py new file mode 100644 index 00000000..408235a3 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/export_unrewritten_ddcolor.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +"""DDColor exported as make_colorize_ddcolor.py does, but WITHOUT the Core ML +rewrites (Tensor.repeat, ReplicationPad2d and the einsum left as they are): +the six-partition graph the report times against the shipped one with +coreml_variants.py. Not shipped. + +Run in the conversion venv from the repository root: + + VAPOURBOX_ADDON_STAGING= python docs/reports/2026-11-colourisation/scripts/export_unrewritten_ddcolor.py +""" +import os +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "..", "Scripts", "models")) + +import torch # noqa: E402 + +import colorize_models_common as cm # noqa: E402 +import face_models_common as fm # noqa: E402 +import make_colorize_ddcolor as m # noqa: E402 + + +class Repeat(torch.nn.Module): + def __init__(self, wrapped): + super().__init__() + self.inner = wrapped.inner + + def forward(self, luma): + return self.inner(luma.repeat(1, 3, 1, 1)) + + +def main(): + net = m.ddcolor() + cm.bake_norms(net) + x = cm.test_input(m.EXPORT_SIZE) + print(fm.export(Repeat(net), [torch.from_numpy(x)], ["luma"], ["ab"], sys.argv[1], + dynamic_axes={"luma": {2: "height", 3: "width"}, "ab": {2: "height", 3: "width"}})) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-colourisation/scripts/flicker.py b/docs/reports/2026-11-colourisation/scripts/flicker.py new file mode 100644 index 00000000..5091893a --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/flicker.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Colour flicker and colour lag, from the planes colorize_run.py --save-uv +writes (_y.npy, _u.npy, _v.npy: (frames, h, w) each, 8-bit +4:2:0 here). Adapted from the face report's metric +(docs/reports/2026-11-face-restoration/scripts/flicker.py) to chroma: the +luma is the source's, bit-exact, so all that can flicker is the colour. + +Per variant, in 8-bit chroma levels, over 8x8-pixel blocks of the chroma +planes (16x16 of the picture): + +- staticFlicker: mean |C[t] - (C[t-1] + C[t+1]) / 2| (U and V averaged) over + blocks whose luma is still — mean luma moves by under STILL levels from + t-1 to t and from t to t+1. There the picture has not changed, so any + change of colour is the model's, i.e. flicker. The source's own value is 0. +- staticChange: mean |C[t] - C[t-1]| over the same still blocks. +- motionLag: over blocks whose luma MOVED by at least MOVING levels, the + mean |C - C_ref| against the per-frame variant of the same model (radius + 0) — how far smoothing pulled moving colour away from what the model saw + in that frame (ghosting or lag would raise it; 0 by definition for the + reference itself). +- chroma: mean |C - 128|, how much colour there is at all (a variant that + flickered less only by fading toward grey would show it here). + +What the metric can't tell you: whether the colour is right (there is no +ground truth for a black-and-white film); it is one clip; and blocks that +are "still" by mean luma can still hold small motion. + + python flicker.py -- +""" + +import json +import os +import sys + +import numpy as np + +BLOCK = 8 +STILL = 1.0 +MOVING = 4.0 + + +def blocks(a): + """(frames, h, w) -> (frames, h/B, w/B) block means (edges cropped).""" + f, h, w = a.shape + h, w = h // BLOCK * BLOCK, w // BLOCK * BLOCK + return a[:, :h, :w].reshape(f, h // BLOCK, BLOCK, w // BLOCK, BLOCK).mean((2, 4)) + + +def luma_blocks(y): + """Luma at chroma resolution (2x2 means), in blocks.""" + f, h, w = y.shape + y = y[:, : h // 2 * 2, : w // 2 * 2].astype(np.float64) + y = y.reshape(f, h // 2, 2, w // 2, 2).mean((2, 4)) + return blocks(y) + + +def load(d, tag): + return [np.load(os.path.join(d, f"{tag}_{p}.npy")).astype(np.float64) for p in "yuv"] + + +def metrics(d, tag, ref=None): + y, u, v = load(d, tag) + ly = luma_blocks(y) + cu, cv = blocks(u), blocks(v) + dy_prev = np.abs(ly[1:-1] - ly[:-2]) + dy_next = np.abs(ly[2:] - ly[1:-1]) + still = (dy_prev < STILL) & (dy_next < STILL) + flick = sum(np.abs(c[1:-1] - (c[:-2] + c[2:]) / 2) for c in (cu, cv)) / 2 + change = sum(np.abs(c[1:-1] - c[:-2]) for c in (cu, cv)) / 2 + out = { + "staticFlicker": round(float(flick[still].mean()), 3), + "staticChange": round(float(change[still].mean()), 3), + "stillShare": round(float(still.mean()), 3), + "chroma": round(float((np.abs(u - 128).mean() + np.abs(v - 128).mean()) / 2), 3), + } + if ref is not None: + _, ru, rv = load(d, ref) + moving = np.abs(ly[1:] - ly[:-1]) >= MOVING + lag = (np.abs(cu - blocks(ru)) + np.abs(cv - blocks(rv)))[1:] / 2 + out["motionLag"] = round(float(lag[moving].mean()), 3) + out["movingShare"] = round(float(moving.mean()), 3) + return out + + +def main(): + d = sys.argv[1] + sep = sys.argv.index("--") + refs = dict(a.split("=", 1) for a in sys.argv[2:sep]) + rows = {} + for tag in sys.argv[sep + 1:]: + model = tag.split("_")[0] + rows[tag] = metrics(d, tag, refs.get(model)) + print(json.dumps({"block": BLOCK, "still": STILL, "moving": MOVING, "rows": rows}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-colourisation/scripts/lab_parity.py b/docs/reports/2026-11-colourisation/scripts/lab_parity.py new file mode 100644 index 00000000..4f59ce09 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/lab_parity.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +"""Check colorize.lab_stages (the std.Expr chain that recombines DDColor's +a and b with the frame's lightness) on the planes lab_reference.py wrote: + +- against OpenCV (cv2.COLOR_LAB2RGB on float32, what DDColor's own pipeline + calls), and +- against the same conversion written out in float64 NumPy with the exact + sRGB curve (OpenCV's float path tabulates its gamma, which is where the two + part company, near black). + +Run inside the deps bundle's Python (run.sh): + + ./run.sh lab_parity.py + +Prints max and mean |difference| per channel in 8-bit levels. +""" +import json +import sys + +import numpy as np +import vapoursynth as vs + +import colorize + +core = vs.core + + +def plane(arr): + h, w = arr.shape + blank = core.std.BlankClip(width=w, height=h, format=vs.GRAYS, length=1) + + def fill(n, f): + out = f.copy() + np.copyto(np.asarray(out[0]), arr) + return out + + return core.std.ModifyFrame(blank, blank, fill) + + +def exact(y, a, b): + """CIELAB (L of the grey value y, a, b) -> sRGB, float64, exact curves, + OpenCV's constants (colorize.py's XN, ZN, XYZ_TO_RGB).""" + ylin = np.where(y <= 0.04045, y / 12.92, ((y + 0.055) / 1.055) ** 2.4) + fy = np.where(ylin > colorize.LAB_T, np.cbrt(ylin), colorize.LAB_K * ylin + 16 / 116) + + def finv(f): + return np.where(f > colorize.LAB_F, f ** 3, (f - 16 / 116) / colorize.LAB_K) + + x = colorize.XN * finv(fy + a / 500) + z = colorize.ZN * finv(fy - b / 200) + lin = np.stack([r[0] * x + r[1] * ylin + r[2] * z for r in colorize.XYZ_TO_RGB], -1).clip(0, 1) + return np.where(lin <= 0.0031308, 12.92 * lin, 1.055 * lin ** (1 / 2.4) - 0.055) + + +def stats(d): + return {"maxLevels": [round(float(d[..., c].max()), 4) for c in range(3)], + "meanLevels": [round(float(d[..., c].mean()), 5) for c in range(3)]} + + +def main(): + ref = np.load(sys.argv[1]) + rgb = colorize.lab_stages(core, plane(ref["y"]), plane(ref["a"]), plane(ref["b"])) + f = rgb.get_frame(0) + got = np.stack([np.asarray(f[p]) for p in range(3)], -1).astype(np.float64) + y, a, b = (ref[k].astype(np.float64) for k in "yab") + out = {"opencv": str(ref["cv2"]), "pixels": int(y.size), + "exprVsOpenCV": stats(np.abs(got - ref["rgb"].astype(np.float64)) * 255), + "exprVsExactFormula": stats(np.abs(got - exact(y, a, b)) * 255), + "exactFormulaVsOpenCV": stats(np.abs(exact(y, a, b) - ref["rgb"].astype(np.float64)) * 255)} + print(json.dumps(out)) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-colourisation/scripts/lab_reference.py b/docs/reports/2026-11-colourisation/scripts/lab_reference.py new file mode 100644 index 00000000..3ce5c54a --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/lab_reference.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""The reference half of the CIELAB check: OpenCV's own recombination, as +DDColor's ColorizationPipeline does it, on synthetic planes. + +A grey picture Y (a value in [0, 1] repeated in R, G and B) gives its L +(cv2.COLOR_RGB2Lab on float32); L with predicted a and b goes back to RGB +(cv2.COLOR_LAB2RGB). lab_parity.py feeds the same Y, a and b to +colorize.lab_stages (std.Expr) and compares. + +Run in the conversion venv (it has opencv-python-headless): + + python lab_reference.py +""" +import sys + +import cv2 +import numpy as np + + +def main(): + rng = np.random.default_rng(0) + h, w = 256, 256 + y = np.concatenate([np.linspace(0, 1, h * w // 2), rng.uniform(0, 1, h * w // 2)]).reshape(h, w) + y = y.astype(np.float32) + a = rng.uniform(-110, 110, (h, w)).astype(np.float32) + b = rng.uniform(-110, 110, (h, w)).astype(np.float32) + grey = np.repeat(y[..., None], 3, axis=2) + lightness = cv2.cvtColor(grey, cv2.COLOR_RGB2Lab)[:, :, :1] + lab = np.concatenate([lightness, a[..., None], b[..., None]], axis=2) + rgb = cv2.cvtColor(lab, cv2.COLOR_LAB2RGB) + np.savez(sys.argv[1], y=y, a=a, b=b, rgb=rgb.astype(np.float32), cv2=cv2.__version__) + print("cv2", cv2.__version__, "rgb range", float(rgb.min()), float(rgb.max())) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-colourisation/scripts/make_figures.sh b/docs/reports/2026-11-colourisation/scripts/make_figures.sh new file mode 100644 index 00000000..38c380d6 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/make_figures.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# The report's figures, from the PNGs bench.sh wrote. +# +# WORK= ./make_figures.sh (ffmpeg on PATH) +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${WORK:?}" +IMG="${1:?img dir}" +R="$WORK/runs" +FIX="$REPO/Tests/TestResources/colorize_test.mp4" +mkdir -p "$IMG" +src() { ffmpeg -v error -y -i "$FIX" -vf "select=eq(n\,$1)" -frames:v 1 "$WORK/src_$1.png"; } +src 20 +src 110 + +# 1: source | DDColor | DeOldify Video (defaults: radius 2), frames 20 and 110. +for n in 20 110; do + ffmpeg -v error -y -i "$WORK/src_$n.png" -i "$R/ddcolor_r2_$n.png" -i "$R/deoldify_r2_$n.png" \ + -filter_complex "[0][1][2]hstack=3,scale=1440:-2" "$WORK/row_$n.png" +done +ffmpeg -v error -y -i "$WORK/row_20.png" -i "$WORK/row_110.png" -filter_complex vstack \ + -compression_level 9 "$IMG/01-models.png" + +# 2: DDColor per frame (radius 0) vs smoothed (radius 2), frame 110. +ffmpeg -v error -y -i "$R/ddcolor_r0_110.png" -i "$R/ddcolor_r2_110.png" \ + -filter_complex "[0][1]hstack=2,scale=1280:-2" -compression_level 9 "$IMG/02-smoothing.png" + +# 3: strength 0.5 | 1 | 1.5 (DDColor), frame 110. +ffmpeg -v error -y -i "$R/ddcolor_s05_2.png" -i "$R/ddcolor_r2_110.png" -i "$R/ddcolor_s15_2.png" \ + -filter_complex "[0][1][2]hstack=3,scale=1440:-2" -compression_level 9 "$IMG/03-strength.png" + +# 4: a colour source: the source | DDColor | DeOldify Video (face fixture, frame 60). +ffmpeg -v error -y -i "$REPO/Tests/TestResources/face_restore_test.mp4" -vf "select=eq(n\,60)" \ + -frames:v 1 "$WORK/colour_src.png" +ffmpeg -v error -y -i "$WORK/colour_src.png" -i "$R/colour_ddcolor_0.png" -i "$R/colour_deoldify_0.png" \ + -filter_complex "[0][1][2]hstack=3,scale=1440:-2" -compression_level 9 "$IMG/04-colour-source.png" +ls -l "$IMG" diff --git a/docs/reports/2026-11-colourisation/scripts/make_refs.sh b/docs/reports/2026-11-colourisation/scripts/make_refs.sh new file mode 100644 index 00000000..e0c16c8c --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/make_refs.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# The 1080p clip for the speed table: the same seconds of "Diary of a +# Sergeant" (1945, public domain; see Tests/TestResources/generate_colorize_test.sh) +# as the committed SD fixture, at the source's own 1920x1080 (pillarboxed 4:3), +# chroma made neutral. Also a colour clip (the face fixture) for the +# already-colour check. +# +# ./make_refs.sh (ffmpeg on PATH) +set -euo pipefail +SRC="${1:?source webm}" OUT="${2:?output directory}" +mkdir -p "$OUT" +ffmpeg -y -v error -ss 156 -t 1 -i "$SRC" -an -vf "format=gray,format=yuv420p" \ + -c:v libx264 -preset slow -crf 18 -color_primaries bt709 -color_trc bt709 -colorspace bt709 \ + "$OUT/colorize_hd.mp4" +ls -l "$OUT" diff --git a/docs/reports/2026-11-colourisation/scripts/run.sh b/docs/reports/2026-11-colourisation/scripts/run.sh new file mode 100755 index 00000000..60544172 --- /dev/null +++ b/docs/reports/2026-11-colourisation/scripts/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Run a script (colorize_run.py) inside the deps bundle's Python with VapourSynth +# importable, as the vspipe wrapper sets it up, plus the repo's templates and +# the ai-runtime add-on on PYTHONPATH and the bundle's ffmpeg on PATH. +# +# DEPS= ADDONS= +# VAPOURBOX_ADDON_STAGING= +# ./run.sh colorize_run.py --clip ... [args] +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${DEPS:?set DEPS to the deps bundle (deps/macos-arm64)}" +: "${ADDONS:?set ADDONS to the add-ons directory holding ai-runtime}" +: "${VAPOURBOX_ADDON_STAGING:?set VAPOURBOX_ADDON_STAGING to the converted models}" +export PATH="$DEPS/ffmpeg:$DEPS/python/bin:$PATH" +export PYTHONHOME="$DEPS/python" +export PYTHONPATH="$DEPS:$DEPS/python-packages:$REPO/worker/templates:$ADDONS/ai-runtime/site-packages" +export DYLD_LIBRARY_PATH="$DEPS/vapoursynth:$DEPS/python/lib" +export LD_LIBRARY_PATH="$DEPS/vapoursynth:$DEPS/python/lib" +export XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-$DEPS/config}" +script="$1"; shift +exec "$DEPS/python/bin/python3" "$HERE/$script" "$@" diff --git a/docs/reports/2026-11-crop-detect-709/img/01-bt709-patches.png b/docs/reports/2026-11-crop-detect-709/img/01-bt709-patches.png new file mode 100644 index 00000000..04f9211c Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/01-bt709-patches.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/02-vhs-fixture-detected.png b/docs/reports/2026-11-crop-detect-709/img/02-vhs-fixture-detected.png new file mode 100644 index 00000000..db9fc9c9 Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/02-vhs-fixture-detected.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/03-vhs-fixture-bottom-left-x4.png b/docs/reports/2026-11-crop-detect-709/img/03-vhs-fixture-bottom-left-x4.png new file mode 100644 index 00000000..a9aa6cdf Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/03-vhs-fixture-bottom-left-x4.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/04-letterbox-detected.png b/docs/reports/2026-11-crop-detect-709/img/04-letterbox-detected.png new file mode 100644 index 00000000..abe66789 Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/04-letterbox-detected.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/05-subtitles-detected.png b/docs/reports/2026-11-crop-detect-709/img/05-subtitles-detected.png new file mode 100644 index 00000000..1efeca15 Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/05-subtitles-detected.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/06-real-vhs-detected.png b/docs/reports/2026-11-crop-detect-709/img/06-real-vhs-detected.png new file mode 100644 index 00000000..7e7482d1 Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/06-real-vhs-detected.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/07-real-vhs-left-x4.png b/docs/reports/2026-11-crop-detect-709/img/07-real-vhs-left-x4.png new file mode 100644 index 00000000..a798d1d0 Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/07-real-vhs-left-x4.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/08-dvbt-detected.png b/docs/reports/2026-11-crop-detect-709/img/08-dvbt-detected.png new file mode 100644 index 00000000..925be1f4 Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/08-dvbt-detected.png differ diff --git a/docs/reports/2026-11-crop-detect-709/img/09-dvbt-bottom-left-x4.png b/docs/reports/2026-11-crop-detect-709/img/09-dvbt-bottom-left-x4.png new file mode 100644 index 00000000..c7dc76ed Binary files /dev/null and b/docs/reports/2026-11-crop-detect-709/img/09-dvbt-bottom-left-x4.png differ diff --git a/docs/reports/2026-11-crop-detect-709/index.html b/docs/reports/2026-11-crop-detect-709/index.html new file mode 100644 index 00000000..d4d19040 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/index.html @@ -0,0 +1,283 @@ + + + + + +BT.709 conversion and border detection + + + +
+ +

Crop & Resize: BT.601 → BT.709 on SD → HD, and Detect borders

+

Build report, 2026-11-08, app 1.7.0. Two Crop & Resize improvements suggested by a survey of VapourSynth restoration tools (sd-video-restore, VCG Deinterlacer, vhs-vapourizer, Video-Enhancer all convert SD colour to BT.709 on upscale; VCG Deinterlacer 2.0 has auto-crop). Measured on an M1 Pro, macOS arm64, deps bundle 1.16.0 (unreleased).

+ +
+ Outcome: +
    +
  • BT.709 conversion, built. An SD (BT.601) source upscaled to HD is converted to the BT.709 matrix in the script when the output is at least 720 lines or 1280 wide (Auto, the default; also Always and Never), and both sides of the encoder say 709. Through the real worker, every colour patch comes back within 0.67 levels of the source decoded as 601, whether the output is read through its tags or as 709 regardless. Without it, a player that ignores the tag on HD shows a 75% green 30 levels dark and 100% green 40.
  • +
  • Only the matrix is converted. Converting the primaries too (SMPTE C → BT.709) moved a 75% green by 55 levels on a player without colour management — a colour shift as large as the one being fixed. The primaries tag keeps the source's, so colour-managed players stay exact. This departs from the brief, which asked for the primaries too; the reason is §1.3.
  • +
  • Detect borders, built (--detect-crop). On fixtures of real VHS picture with known borders, every edge came back exact: noisy blanking, a 2-line top, an 8-line head-switching band (12 per field when interlaced), 72-line letterbox bars reported apart, subtitles and fades ignored. On a DVB-T broadcast it matches a by-eye measurement. About 1 s per SD source, 2.9 s at 1080p.
  • +
  • New preset: SD Tape → HD (YouTube / TV).
  • +
+
+ +

1. BT.601 → BT.709

+ +

1.1 The problem, measured

+

An SD picture's YCbCr samples are coded with the BT.601 matrix. VapourBox carried the source's -colorspace tag through to the output, so an SD video resized to 1080 lines was a correctly tagged BT.601 HD file. Players and TVs that pick a matrix by frame size and ignore the tag decode it as BT.709. Eight patches, made in RGB, encoded as 601 limited range, and decoded as 709 (scripts/gamut.py, 16-bit intermediates so rounding doesn't muddy it):

+ +
+ + + + + + + + + + + + + +
8-bit RGB of each patch: the source decoded as 601, and the same samples decoded as 709 ("misread")
PatchSource (601)Misread as 709Worst channel error
75% red191, 0, 0208, 18, 018
75% green0, 191, 00, 161, 030
75% blue0, 0, 1910, 11, 19911
75% yellow191, 191, 0194, 180, 011
75% cyan0, 191, 1910, 173, 19418
75% magenta191, 0, 191205, 30, 19630
100% red255, 0, 0255, 25, 025
100% green0, 255, 00, 215, 040
Skin tone224, 172, 140229, 175, 1385
Grey128, 128, 128128, 128, 1280
+
+ +

1.2 What was built

+
    +
  • Option: Crop & Resize → Resize → Convert Colour to BT.709 (HD standard): Auto (default), Always, Never. Auto converts a BT.601 source (tagged bt470bg or smpte170m, or untagged below 720 lines, which is how the app and players read an untagged SD file) when the output is ≥ 720 lines or ≥ 1280 wide. The output size is computed in Rust by CropResizeParameters::output_size, which mirrors the script's crop, upscale, fitting and bars; the decision has to be made before the script runs because the encoder's tags depend on it.
  • +
  • One decision point: VideoJob::converts_to_bt709, and VideoJob::output_color_metadata() for everything that describes what the script delivers: the encoder's input and output flags (still identical, per the 2026-09-26 rule), the preview's swscale step, and the passes after Crop & Resize (Face Restoration, RIFE, the border fill). The AI upscale runs inside the resize, before the conversion, and keeps the source's.
  • +
  • Script: a {{#COLOR_709}} block in both templates right after the resize: resize.Spline36 with matrix_in_s = the source's, matrix_s="709", the source's range, error-diffusion dither, after removing _Matrix/_Range frame properties (§1.4).
  • +
  • Tags: matrix bt709, transfer bt709 (BT.601's transfer is the same curve), primaries the source's (tagged, or EBU for 576/288 lines, SMPTE C otherwise), range the source's (tv when untagged).
  • +
+ +

1.3 Why only the matrix (a departure from the brief)

+

The brief asked for primaries and transfer to be converted too where they differ. Both were tried. The table shows each patch decoded as 709 without colour management, which is how most players show video:

+
+ + + + + + + + + + +
75%/100% patches after conversion, decoded as 709. "Matrix only" is what shipped.
PatchSource (601)Matrix onlyMatrix + primaries, SMPTE C → 709Matrix + primaries, EBU → 709
75% red191, 0, 0191, 0, 0186, 36, 0194, 0, 0
75% green0, 191, 00, 191, 055, 188, 00, 191, 30
75% blue0, 0, 1910, 0, 19128, 34, 1910, 0, 190
75% cyan0, 191, 1910, 191, 19159, 190, 1910, 191, 191
75% magenta191, 0, 191191, 0, 191187, 47, 191194, 0, 190
100% green0, 255, 00, 255, 073, 251, 00, 255, 40
Skin tone224, 172, 140224, 172, 140221, 173, 140226, 172, 140
+
+

A gamut conversion is colorimetrically correct only for a viewer whose player also colour-manages the source; for everyone else it changes what the picture looks like by as much as the misread matrix does. Keeping the primaries and tagging them as the source's gives every kind of player the source's colour: one that ignores tags shows the same RGB the SD file showed, and one that colour-manages reads the primaries tag and converts on display, exactly as it did for the SD file. VCG Deinterlacer converts the matrix only too (it tags the primaries 709, which is a small mislabel this avoids).

+

Limit: the comparison is of decoded RGB code values, not of perceived colour on a calibrated display. It says how far a non-colour-managed player's output moves; it says nothing about which result a colourist would prefer on a reference monitor.

+ +

1.4 A trap: R78 lets frame properties override *_in_s

+

The first run of gamut.py printed "misread as 709" equal to the source. VapourSynth R78's resizers take a clip's _Matrix property in preference to matrix_in_s (the prefer_props argument is deprecated and ignored), and the test clip carried _Matrix=6 from its own creation. The template block therefore removes _Matrix, _Range and _ColorRange before converting, so the worker's reading of the source's tags is what decides. The other colour conversions in the templates (AI upscale, RIFE, border fill) are unaffected today because nothing upstream sets those properties, but any new conversion that relies on *_in_s needs the same care.

+ +

1.5 The pixel test, through the real worker

+

app/test/integration_bt709_conversion_test.dart (heavy) makes a 720×576 BT.601 FFV1 clip of eight patches, encodes it to a 1920×1080 box (1350×1080, square pixels) with the conversion on and off, and decodes each output to RGB with swscale's accurate path (accurate_rnd+full_chroma_int), averaging a 9×9 block at each patch centre:

+
+ + + + + + + + + + + +
Converted output against the source decoded as 601 (8-bit RGB, worst channel)
PatchSource (601)Output, through its tagsΔ
75% red191, 0, 1191, 0, 10.26
75% green0, 191, 00, 191, 00.01
75% blue0, 1, 1920, 0, 1910.64
75% yellow191, 190, 0191, 191, 00.67
75% cyan0, 191, 1900, 191, 1900.37
75% magenta191, 0, 192191, 0, 1920.06
Skin tone224, 172, 140224, 172, 1400.36
Grey128, 128, 128128, 128, 1280.00
+
+
    +
  • Tolerance asserted: 3 levels. The same holds decoding the output as 709 regardless of its tags (what the tag-ignoring player does).
  • +
  • Tags written: color_space bt709, color_transfer bt709, color_primaries smpte170m, color_range tv.
  • +
  • Off: tags stay smpte170m; both files decoded as 601 agree within 1 level (asserted), and the Y/U/V samples at every patch centre are the source's exactly (scripts/patch_yuv.py: e.g. 75% green 112/72/58 in both; converted, 133/62/52). Read as 709, the worst patch (75% magenta) is off by 30 — the control that the test can see the problem.
  • +
  • Auto at SD size (a 700-line box) does not convert; an untagged 480-line source is read as 601 and tagged with NTSC primaries.
  • +
  • Preview = encode: the worker's preview of frame 2 against the encode decoded by ffmpeg's default path: mean absolute difference 0.000/255. (A first version of the test compared the preview with the accurate decode path and measured 1.55/255: that was swscale's default yuv→rgb path, not the conversion, which is why the comparison is now like for like.)
  • +
+
+ Three panels of eight colour patches +
source, decoded as 601unconverted HD, read as 709converted HD, read as 709
+
Figure 1. The fixture's eight patches, three ways (frame 2, each decoded explicitly, scaled nearest-neighbour). Middle: today's HD output in a player that ignores its 601 tag — green, red and magenta visibly darker and shifted. Right: the converted output in the same player, indistinguishable from the left.
+
+ +

1.6 Cost

+

scripts/convert_cost.py, 20 frames of the VHS fixture resized to 1350×1080 4:2:0:

+
    +
  • The conversion runs at 97–140 fps (7–10 ms per frame, 8 threads) across runs — small beside QTGMC and an NNEDI3 upscale at that size.
  • +
  • A 601 → 709 → 601 round trip (two conversions, each through 4:4:4 and back with error-diffusion dither) changes luma by 0.25/255 on average and chroma by 0.06/255, so one conversion costs at most that.
  • +
+

Limit: Apple Silicon only; one clip; the round-trip number bounds the loss from above rather than measuring a single conversion.

+ +

1.7 Default and presets

+
    +
  • Default Auto, knowingly changing existing output. Any job that resizes an SD source to HD now writes 709-tagged, converted samples where it wrote 601-tagged ones, including presets saved before 1.7.0 (the field is absent, so it reads as Auto). The old output was correct and shown wrong by many players; the new one is shown right by all of them, at a cost of ≤ 0.25/255. A user who needs the old behaviour picks Never.
  • +
  • No built-in preset resized before (asserted in processing_preset_test.dart), so none changes.
  • +
  • New preset, SD Tape → HD (YouTube / TV): QTGMC Slow, light SMDegrain, CCD, square pixels, NNEDI3 2x then Spline36 into a 1920×1080 box (4:3 lands at 1440×1080), BT.709 Always, CAS 0.3, H.264 CRF 17 slow, 8-bit 4:2:0, MP4, AAC. 4:2:0 8-bit because a 4:2:2 capture-card source would otherwise be encoded as H.264 High 4:2:2, which TVs and browsers refuse.
  • +
+ +

2. Detect borders (--detect-crop)

+ +

2.1 Method

+

24 frames spread over the middle 90% of the source (or its trim) are decoded as 8-bit 4:4:4 by ffmpeg, four at a time, and measured line by line in plain Rust (worker/src/crop_detect.rs). VCG Deinterlacer 2.0's analysis/edges.rs (MIT) was read as a reference for which signals separate blanking, ramp and head switching; nothing was ported, and the thresholds were set on our own fixtures.

+
    +
  • Black and blanking: a line whose 98th-percentile brightness over every usable frame is within 30% of the way from black to the picture's typical line, and whose detail is under half the picture's or 6 levels. The percentile is over all frames, so a line that shows picture in any of them is picture. One line counts only if it is true black (≤ 4 above black) in every frame — a capture card's first line.
  • +
  • Ramp: up to 6 more lines, each dimmer than 85% of the picture just inside and rising.
  • +
  • Bars: a dark run wider than 5% of the dimension is framing (letterbox / pillarbox), reported in cropWithBars and never in the recommended crop.
  • +
  • Head switching: a bottom line is torn in a frame when it differs from the line above in the same field by 2.2× the picture's typical line-to-line difference (and ≥ 4 levels luma or ≥ 2.5 chroma) across 6 of 8 width segments, or matches the line above best shifted sideways by ≥ 3 px. A line is band when torn in ≥ 50% of usable frames; the band must start within 2 lines of the bottom (above any black lines) and may contain one untorn line.
  • +
  • Dark frames (central 90th percentile below 48) are left out of every statistic.
  • +
  • Output: the recommended crop rounded outward to steps the Crop pass accepts (even; fours vertically for 4:2:0 still interlaced at the crop) and mapped through Rotate / Flip; the crop with bars; and, for a band of up to 16 lines, Edge Repair's widths to repair it instead. A per-edge confidence is the share of usable frames whose picture reaches that edge.
  • +
+ +

2.2 Accuracy

+

Fixtures (app/test/integration_crop_detect_test.dart, heavy): 30 frames of the VHS capture interlaced_test.avi, cut to its inner 664×536 (no borders of its own), resized into a 720×576 canvas, and given known edges; FFV1 4:2:2. The band shifts each bottom line sideways by 5–13 px from a line 10 above, with ±25 luma noise and ±40 chroma noise, different every frame. Measured = unrounded; release worker:

+
+ + + + + + + + + +
Detection on fixtures with known edges (drawn → measured, in lines/px; asserted within ±2)
FixtureLeftRightTopBottomTime
VHS: noisy blanking, band 814 → 1410 → 102 → 2band 8 → 81.5 s*
Interlaced (TFF), band 1214 → 1410 → 100 → 0band 12 → 121.0 s
Letterbox 72 + blanking 88 → 88 → 8bar 72 → 72bar 72 → 720.9 s
Subtitles 40 lines up, 2 frames in 38 → 88 → 80 → 00 → 0 (not band)1.1 s
Fades: every 3rd frame near black12 → 1212 → 124 → 40 → 00.9 s
1080p, 240-px pillarboxbar 240 → 240bar 240 → 2400 → 00 → 02.9 s
+
+

* The first run includes cold disk caches. Debug builds (what the heavy tests run) take 0.8–1.9 s per SD fixture. Fades: 16 of 24 sampled frames were usable. Letterbox: the bottom bar's confidence is 0.17, because the VHS picture's dashboard just above it is dark in most frames — the edge is right, the evidence thin, and the panel says so (it warns below 0.25).

+ +

2.3 Real files

+
+ + + + + +
Real sources: by-eye ground truth from edges_dump-style line statistics (98th-percentile column / row brightness)
SourceBy eyeMeasuredRecommended cropTime
DVB-T broadcast, 720×576 TFF 4:2:0left 7 black, right 8 black, top 0, bottom 4 black + 1 dim7, 8, 0, 4 + 1 ramp8, 8, 0, 8 (fours: still interlaced 4:2:0)0.5 s
VHS capture interlaced_test.avi, 720×576 TFF 4:2:2left: 10 dark columns (19–30 levels, grainy) then a 9-column rise; right: a 7-column rise from 44; top: one line at 0 in every frameleft 10 + 6 ramp, right 3 + 3, top 1, bottom 4 (confidence 0.12)16, 6, 2, 41.0 s
+
+
    +
  • The VHS capture's left ramp runs about 9 columns; the detector follows 6 (its cap) and stops at 16, leaving a few columns of slightly dim picture. Whether those columns are ramp or picture is a judgement; the cap keeps the detector from eating into genuinely dark content.
  • +
  • Its bottom 4 lines are reported as blanking with confidence 0.12: the dashboard above them is dark in nearly every frame, so few frames show where the picture ends. No head-switching band is visible in this capture (Figure 6), and none was reported.
  • +
+ +
+ VHS fixture with the detected crop box +
Figure 2. The VHS fixture, frame 15, with the recommended crop drawn as a red 1-px box: 14 px of noisy blanking at the left, 10 at the right, 2 lines at the top, and the 8-line band at the bottom outside the box.
+
+
+ Bottom-left corner, 4x +
Figure 3. Its bottom-left corner (128×40, 4× nearest-neighbour): the noisy blanking at the left edge and the torn, colour-wrecked band below the red crop line.
+
+
+ Letterbox fixture +
Figure 4. Letterbox fixture: red = the recommended crop (blanking only, bars kept), yellow = with the bars (offered separately as "Crop the bars too").
+
+
+ Subtitle fixture +
Figure 5. Subtitle fixture, a frame with the text block: bright, sharp-edged lines 40–62 lines above the bottom, in two frames of three. Not reported as head switching: the band test needs a tear across 6 of 8 width segments, in half the frames, starting at the bottom.
+
+
+ Real VHS capture +
Figure 6. The real VHS capture with its recommended crop: left 16 (blanking and ramp), right 6, top 2 (one black line, rounded to two), bottom 4.
+
+
+ Left edge, 4x +
Figure 7. Its left edge (48×96, 4×): grainy dark blanking, the soft ramp, and the crop line at column 16.
+
+
+ DVB-T broadcast +
Figure 8. The DVB-T broadcast with its recommended crop (8, 8, 0, 8).
+
+
+ DVB-T bottom-left, 4x +
Figure 9. Its bottom-left corner (64×24, 4×): the 7 black columns and the black bottom lines inside the crop line.
+
+ +

2.4 Traps, and what holds each off

+
+ + + + + + + + + + + +
Each trap has a unit test in crop_detect.rs (synthetic frames) and, where marked, a fixture in the heavy test
TrapDefenceHeavy fixture
Dark scenes, fades98th percentile over all frames; dark frames dropped; confidence says how many frames show the edgefades
Letterbox inside 4:3Runs over 5% are bars: reported, not cropped by defaultletterbox
Noisy blankingAbsolute detail floor (6 levels) beside the relative oneVHS
Subtitles near the bottomBand = torn across 6 of 8 segments, in half the frames, from the bottomsubtitles
Interlaced sourcesLines compared within the field; crop in fours for 4:2:0 still interlacedinterlaced, DVB-T
A band under a letterboxBand cropped, bar above it kept apart(unit test)
One dark line that is pictureA single line counts only if true black in every frame(unit test)
Rotate / Flip before the cropThe crop is mapped into the turned orientation; Edge Repair's stays in the source's(unit test)
+
+ +

2.5 App

+

Crop & Resize gains a Detect borders button (NEW badge, 1.7.0). The panel lists each edge ("Left: 10 px of black / blanking and a 2-px soft edge", "Bottom: 8 lines of head-switching noise", "Top: 72-line letterbox bar"), the crop, and up to three buttons: Apply crop, Crop the bars too, Repair the band instead (Edge Repair at the bottom, the rest cropped — keeps the frame height for DVD authoring). Nothing changes until one is pressed (crop_detect_widget_test.dart). The crop sliders' maximum rose from 200 to 480 so a 1080p pillarbox fits.

+ +

3. Decisions

+
    +
  • Matrix-only conversion, primaries tag kept: §1.3.
  • +
  • HD threshold 720 lines or 1280 columns: the same height the app already guesses 709 from for an untagged file, and the width catches a scope picture cropped short (1280×536).
  • +
  • Default Auto: §1.7.
  • +
  • Bars not in the recommended crop: a letterbox is the picture's framing; cropping it changes the aspect, which is the user's call.
  • +
  • Analysis in Rust, not VapourSynth: per-line statistics need no plugin, run in about a second, and are unit-testable on synthetic frames.
  • +
  • No NOTICES entry for VCG: its code was read, not ported.
  • +
+ +

4. Not tested

+
    +
  • A real capture with a clearly visible head-switching band: none is in Tests/TestResources/; the band in the fixtures is simulated (a sideways tear with noise), modelled on how it looks, not on a measured capture.
  • +
  • Windows and Linux, and macOS x64. The analysis is plain Rust; the decode uses the same ffmpeg arguments as the preview. The BT.709 block is plain zimg.
  • +
  • HDR sources are never converted (asserted), and were not run.
  • +
  • A colour-managed player's rendering of the converted file was not checked on screen.
  • +
+ +

5. Reproducing

+
cd docs/reports/2026-11-crop-detect-709
+scripts/run_vs.sh scripts/gamut.py          # §1.1, §1.3
+scripts/run_vs.sh scripts/convert_cost.py   # §1.6
+python3 scripts/patch_yuv.py ../../../Tests/TestOutput/bt709_conversion/*.mkv   # §1.5, after the heavy test
+SCRATCH=/some/dir scripts/crop_detect.sh    # §2.2, §2.3 (runs the heavy test, then times the release worker)
+(cd ../../../app && flutter test test/integration_bt709_conversion_test.dart)   # §1.5
+scripts/images.sh                           # the figures, from the two heavy tests' outputs
+

All scripts take the deps bundle from DEPS (default deps/macos-arm64) and find the repo from their own location.

+ +
+ + diff --git a/docs/reports/2026-11-crop-detect-709/scripts/convert_cost.py b/docs/reports/2026-11-crop-detect-709/scripts/convert_cost.py new file mode 100644 index 00000000..34dcdde9 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/convert_cost.py @@ -0,0 +1,77 @@ +"""The price of the BT.709 conversion: time per frame, and what the extra +resample does to the picture. + +Run with scripts/run_vs.sh. Takes 20 frames of the VHS fixture +(Tests/TestResources/interlaced_test.avi, 720x576 4:2:2, decoded to 4:2:0 as +most HD deliveries are), resizes them to 1350x1080 with Spline36 (what Crop & +Resize's 1920x1080 box does to a 720x576 square-pixel picture), and then + + * times the template's conversion (the COLOR_709 block, copied here) over + 200 frames, against the resize alone; + * converts 601 -> 709 and straight back 709 -> 601, and compares each plane + with the unconverted resize. That round trip is TWO conversions, so it + bounds what one costs from above. +""" +import ctypes +import os +import subprocess +import time + +import vapoursynth as vs + +core = vs.core +REPO = os.environ['REPO'] +FF = os.environ['FFMPEG'] +SRC = os.path.join(REPO, 'Tests', 'TestResources', 'interlaced_test.avi') +W, H, N = 720, 576, 20 + +raw = subprocess.check_output([FF, '-v', 'error', '-i', SRC, '-frames:v', str(N), + '-f', 'rawvideo', '-pix_fmt', 'yuv420p', '-']) +FRAME = W * H * 3 // 2 + + +def load(): + blank = core.std.BlankClip(width=W, height=H, format=vs.YUV420P8, length=N) + + def fill(n, f): + fo = f.copy() + data = raw[n * FRAME:(n + 1) * FRAME] + offset = 0 + for p in range(3): + pw, ph = (W, H) if p == 0 else (W // 2, H // 2) + ptr, stride = fo.get_write_ptr(p).value, fo.get_stride(p) + for y in range(ph): + ctypes.memmove(ptr + y * stride, data[offset + y * pw: offset + (y + 1) * pw], pw) + offset += pw * ph + return fo + return core.std.ModifyFrame(blank, blank, fill) + + +def convert(clip, m_in, m_out): + # The template's block, with the matrices as arguments. + clip = core.std.RemoveFrameProps(clip, props=["_Matrix", "_Range", "_ColorRange"]) + return core.resize.Spline36(clip, format=clip.format.id, matrix_in_s=m_in, matrix_s=m_out, + range_in_s="limited", range_s="limited", dither_type="error_diffusion") + + +src = load() +hd = core.resize.Spline36(src, 1350, 1080) +on = convert(hd, '170m', '709') +back = convert(on, '709', '170m') + +for p, name in enumerate('YUV'): + a = core.std.ShufflePlanes(hd, p, vs.GRAY) + b = core.std.ShufflePlanes(back, p, vs.GRAY) + diff = core.std.PlaneStats(a, b) + mae = sum(f.props['PlaneStatsDiff'] for f in diff.frames()) / N * 255 + print(f'round trip 601->709->601, plane {name}: mean abs diff {mae:.3f}/255') + +# The 20 resized frames are cached after the first pass, so "resize only" +# measures almost nothing and the second line is the conversion's own cost. +looped = core.std.Loop(hd, 10) +for label, clip in (('resize only', looped), ('resize + conversion', convert(looped, '170m', '709'))): + start = time.perf_counter() + for _ in clip.frames(): + pass + dt = time.perf_counter() - start + print(f'{label}: {clip.num_frames / dt:.1f} fps ({1000 * dt / clip.num_frames:.2f} ms/frame), {core.num_threads} threads') diff --git a/docs/reports/2026-11-crop-detect-709/scripts/crop_detect.sh b/docs/reports/2026-11-crop-detect-709/scripts/crop_detect.sh new file mode 100755 index 00000000..454bead5 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/crop_detect.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# Detection accuracy and speed for --detect-crop. +# +# 1. Builds the worker (release, for the timings) and runs the heavy test +# app/test/integration_crop_detect_test.dart, which builds the fixtures +# (real VHS picture on a canvas with known borders) into +# Tests/TestOutput/crop_detect/ and asserts every edge within 2 lines. +# 2. Times the release worker on each fixture, on the two real files, and on +# a 1080p pillarbox fixture made here in $SCRATCH. +# +# Needs: deps/macos-arm64 (or DEPS=...), Flutter, Rust. Every number in the +# report's detection table is printed by this script. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../../.." && pwd)" +S="${SCRATCH:?set SCRATCH to your scratch directory}" +export DEPS="${DEPS:-$ROOT/deps/macos-arm64}" +FF="$DEPS/ffmpeg/ffmpeg" +mkdir -p "$S" + +(cd "$ROOT/worker" && cargo build --release -q && cargo build -q) +(cd "$ROOT/app" && VAPOURBOX_DEPS_DIR="$DEPS" flutter test test/integration_crop_detect_test.dart) + +export WORKER="$ROOT/worker/target/release/vapourbox-worker" +OUT="$ROOT/Tests/TestOutput/crop_detect" +echo "== release worker ==" +for f in vhs letterbox subtitles fades; do + python3 "$HERE/detect.py" "$OUT/$f.mkv" progressive +done +python3 "$HERE/detect.py" "$OUT/vhs_interlaced.mkv" tff +python3 "$HERE/detect.py" "$ROOT/Tests/TestResources/pal-dvbt-fieldcoded-25i.ts" tff +python3 "$HERE/detect.py" "$ROOT/Tests/TestResources/interlaced_test.avi" tff + +# 1080p: the VHS picture as 4:3 inside HD, 240-px pillarbox bars. +"$FF" -v error -y -i "$ROOT/Tests/TestResources/interlaced_test.avi" \ + -vf "crop=664:536:28:12,scale=1440:1080,pad=1920:1080:240:0:black" \ + -frames:v 60 -c:v ffv1 -pix_fmt yuv420p "$S/pillarbox_1080.mkv" +python3 "$HERE/detect.py" "$S/pillarbox_1080.mkv" progressive diff --git a/docs/reports/2026-11-crop-detect-709/scripts/detect.py b/docs/reports/2026-11-crop-detect-709/scripts/detect.py new file mode 100644 index 00000000..41478452 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/detect.py @@ -0,0 +1,75 @@ +"""Run `vapourbox-worker --detect-crop` on one video and print the report. + +usage: python3 detect.py VIDEO [tff|bff|progressive] [--json] + +The worker is $WORKER, else the repo's release build, else its debug build. +Prints the time taken, the recommended crop and each edge as measured; with +--json, the report itself on one line. +""" +import json +import os +import subprocess +import sys +import tempfile +import time +import uuid + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..')) +DEPS = os.environ.get('DEPS', os.path.join(ROOT, 'deps', 'macos-arm64')) +FFPROBE = os.path.join(DEPS, 'ffmpeg', 'ffprobe') + + +def worker(): + if os.environ.get('WORKER'): + return os.environ['WORKER'] + for build in ('release', 'debug'): + p = os.path.join(ROOT, 'worker', 'target', build, 'vapourbox-worker') + if os.path.exists(p): + return p + sys.exit('build the worker first (cargo build --release)') + + +def detect(path, field='progressive'): + info = json.loads(subprocess.check_output([ + FFPROBE, '-v', 'error', '-select_streams', 'v:0', '-count_packets', '-show_entries', + 'stream=width,height,pix_fmt,r_frame_rate,nb_read_packets,color_range', '-of', 'json', path]))['streams'][0] + num, den = map(int, info['r_frame_rate'].split('/')) + fps = num / den + if field != 'progressive' and fps > 30: + fps /= 2 # a field-coded stream reports its field rate + job = { + 'id': str(uuid.uuid4()), 'inputPath': path, 'outputPath': os.path.join(tempfile.gettempdir(), 'unused.mkv'), + 'qtgmcParameters': {'enabled': False}, 'encodingSettings': {}, + 'processingPipeline': {'deinterlace': {'enabled': False}}, + 'detectedFieldOrder': field, + 'totalFrames': int(info.get('nb_read_packets') or 100), 'inputFrameRate': fps, + 'inputWidth': info['width'], 'inputHeight': info['height'], + 'inputPixelFormat': info['pix_fmt'], 'inputColorRange': info.get('color_range'), + } + with tempfile.TemporaryDirectory() as d: + cfg = os.path.join(d, 'job.json') + with open(cfg, 'w') as f: + json.dump(job, f) + start = time.perf_counter() + out = subprocess.run([worker(), '--detect-crop', '--config', cfg], capture_output=True, text=True, timeout=900) + dt = time.perf_counter() - start + line = [l for l in out.stdout.splitlines() if l.startswith('{')][-1] + return json.loads(line), dt + + +if __name__ == '__main__': + path = sys.argv[1] + field = sys.argv[2] if len(sys.argv) > 2 and not sys.argv[2].startswith('--') else 'progressive' + report, dt = detect(path, field) + if '--json' in sys.argv: + print(json.dumps(report)) + sys.exit(0) + print(f'{os.path.basename(path)}: {dt * 1000:.0f} ms, usable {report.get("usableFrames")}/{len(report.get("frames", []))}') + if 'error' in report: + print(' error:', report['error']) + sys.exit(1) + print(f' crop {report["crop"]} bars {report.get("cropWithBars")} repair {report.get("edgeRepair")}') + for side, e in report['edges'].items(): + print(f' {side:6s} {e["kind"]:13s} measured {e["crop"]:3d} (with bars {e["cropWithBars"]:3d}) ' + f'blank {e["blank"]} ramp {e["ramp"]} band {e["band"]} conf {e["confidence"]:.2f}') diff --git a/docs/reports/2026-11-crop-detect-709/scripts/gamut.py b/docs/reports/2026-11-crop-detect-709/scripts/gamut.py new file mode 100644 index 00000000..8d59c818 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/gamut.py @@ -0,0 +1,62 @@ +"""What converting a BT.601 picture to BT.709 does to saturated colours. + +Run with scripts/run_vs.sh. For each patch (8-bit RGB), encodes it as BT.601 +limited-range YUV (16-bit, so rounding does not muddy the comparison), then: + + matrixOnly - converted 601 -> 709 matrix only (what VapourBox does), decoded as 709 + full - matrix AND primaries converted (SMPTE C or EBU -> BT.709), decoded as 709 + misread - the unconverted 601 samples decoded as 709: a player that + ignores the tag on an HD picture + +and prints each as 8-bit RGB next to the source decoded as 601. + +Trap found doing this: VapourSynth R78's resizers let a frame's own _Matrix / +_Primaries properties override matrix_in_s / primaries_in_s, so every clip +here has those properties removed before the arguments are trusted. The first +run of this script did not, and printed "misread" equal to the source. +""" +import vapoursynth as vs + +core = vs.core + +PATCHES = { + 'red75': (191, 0, 0), 'green75': (0, 191, 0), 'blue75': (0, 0, 191), + 'yellow75': (191, 191, 0), 'cyan75': (0, 191, 191), 'magenta75': (191, 0, 191), + 'red100': (255, 0, 0), 'green100': (0, 255, 0), 'skin': (224, 172, 140), + 'grey': (128, 128, 128), +} + + +def clean(c): + return core.std.RemoveFrameProps(c, ['_Matrix', '_Primaries', '_Transfer']) + + +def rgb_of(c): + f = c.get_frame(0) + return tuple(int(f[p][8, 8]) for p in range(3)) + + +def decode(c, matrix): + return rgb_of(core.resize.Point(clean(c), format=vs.RGB24, matrix_in_s=matrix, range_in_s='limited')) + + +def worst(a, b): + return max(abs(x - y) for x, y in zip(a, b)) + + +for prim in ('170m', '470bg'): + print(f'source primaries {prim}') + print(f' {"patch":10s} {"source601":15s} {"matrixOnly":15s} {"full":15s} {"misread":15s} worst: mOnly full misread') + for name, rgb in PATCHES.items(): + patch = core.std.BlankClip(width=16, height=16, format=vs.RGB24, color=list(rgb), length=1) + src = clean(core.resize.Point(patch, format=vs.YUV444P16, matrix_s='170m', range_s='limited')) + m_only = core.resize.Spline36(src, format=vs.YUV444P16, matrix_in_s='170m', matrix_s='709', + range_in_s='limited', range_s='limited') + full = core.resize.Spline36(src, format=vs.YUV444P16, matrix_in_s='170m', matrix_s='709', + primaries_in_s=prim, primaries_s='709', + transfer_in_s='709', transfer_s='709', + range_in_s='limited', range_s='limited') + s601 = decode(src, '170m') + mo, fu, mis = decode(m_only, '709'), decode(full, '709'), decode(src, '709') + print(f' {name:10s} {str(s601):15s} {str(mo):15s} {str(fu):15s} {str(mis):15s}' + f' {worst(mo, s601):5d} {worst(fu, s601):4d} {worst(mis, s601):7d}') diff --git a/docs/reports/2026-11-crop-detect-709/scripts/images.sh b/docs/reports/2026-11-crop-detect-709/scripts/images.sh new file mode 100755 index 00000000..29d21655 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/images.sh @@ -0,0 +1,62 @@ +#!/bin/bash +# Renders the report's images into ../img from the outputs of the two heavy +# tests (run them first: crop_detect.sh, and +# `flutter test test/integration_bt709_conversion_test.dart`). +# +# Detected crops are drawn as a 1-px box: red = the recommended crop, yellow = +# with the letterbox / pillarbox bars. Zooms are 4x nearest-neighbour, so +# every pixel shown is a real one. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../../.." && pwd)" +export DEPS="${DEPS:-$ROOT/deps/macos-arm64}" +FF="$DEPS/ffmpeg/ffmpeg -v error -y" +IMG="$HERE/../img" +OUT="$ROOT/Tests/TestOutput" +mkdir -p "$IMG" + +# box FILE FIELD -> "drawbox filters" for the detected crop +box() { + python3 - "$1" "$2" <<'EOF' +import json, subprocess, sys, os +here = os.environ['HERE'] +r = json.loads(subprocess.check_output([sys.executable, os.path.join(here, 'detect.py'), sys.argv[1], sys.argv[2], '--json'])) +w, h = r['width'], r['height'] +def rect(c, colour): + return (f"drawbox=x={c['left']}:y={c['top']}:w={w - c['left'] - c['right']}:" + f"h={h - c['top'] - c['bottom']}:color={colour}:t=1") +parts = [rect(r['crop'], 'red')] +if r.get('cropWithBars'): + parts.insert(0, rect(r['cropWithBars'], 'yellow')) +print(','.join(parts)) +EOF +} +export HERE + +# frame FILE FIELD N OUT [ZOOM_CROP] (ZOOM_CROP = w:h:x:y, shown 4x) +frame() { + local B; B=$(box "$1" "$2") + local Z="" + [ -n "$5" ] && Z=",crop=$5,scale=iw*4:ih*4:flags=neighbor" + $FF -i "$1" -vf "select=eq(n\,$3),format=yuv444p,$B$Z" -frames:v 1 "$4" +} + +C="$OUT/crop_detect" +frame "$C/vhs.mkv" progressive 15 "$IMG/02-vhs-fixture-detected.png" +frame "$C/vhs.mkv" progressive 15 "$IMG/03-vhs-fixture-bottom-left-x4.png" "128:40:0:536" +frame "$C/letterbox.mkv" progressive 15 "$IMG/04-letterbox-detected.png" +frame "$C/subtitles.mkv" progressive 13 "$IMG/05-subtitles-detected.png" +frame "$ROOT/Tests/TestResources/interlaced_test.avi" tff 40 "$IMG/06-real-vhs-detected.png" +frame "$ROOT/Tests/TestResources/interlaced_test.avi" tff 40 "$IMG/07-real-vhs-left-x4.png" "48:96:0:240" +frame "$ROOT/Tests/TestResources/pal-dvbt-fieldcoded-25i.ts" tff 60 "$IMG/08-dvbt-detected.png" +frame "$ROOT/Tests/TestResources/pal-dvbt-fieldcoded-25i.ts" tff 60 "$IMG/09-dvbt-bottom-left-x4.png" "64:24:0:552" + +# BT.601 -> 709: the same eight patches three ways, each decoded explicitly. +B="$OUT/bt709_conversion" +D="flags=accurate_rnd+full_chroma_int,format=rgb24,scale=450:360:flags=neighbor" +$FF -i "$B/sd601_patches.mkv" -i "$B/off_1080.mkv" -i "$B/on_1080.mkv" -filter_complex \ + "[0:v]select=eq(n\,2),scale=in_color_matrix=bt601:in_range=tv:out_range=pc:$D[a];\ + [1:v]select=eq(n\,2),scale=in_color_matrix=bt709:in_range=tv:out_range=pc:$D[b];\ + [2:v]select=eq(n\,2),scale=in_color_matrix=bt709:in_range=tv:out_range=pc:$D[c];\ + [a][b][c]hstack=3" -frames:v 1 "$IMG/01-bt709-patches.png" +ls -la "$IMG" diff --git a/docs/reports/2026-11-crop-detect-709/scripts/patch_yuv.py b/docs/reports/2026-11-crop-detect-709/scripts/patch_yuv.py new file mode 100644 index 00000000..081a2e14 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/patch_yuv.py @@ -0,0 +1,31 @@ +"""Print the Y/U/V samples at each patch centre of the BT.709 test's files. + +usage: python3 patch_yuv.py FILE... (e.g. Tests/TestOutput/bt709_conversion/*.mkv) + +Decodes the first frame to yuv444p (chroma upsampled, which is exact on a +flat patch) and prints the centre sample of each of the 4x2 patches. Used to +check that the unconverted output stores the source's own samples, and to see +how the converted one differs. +""" +import os +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +ROOT = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..')) +DEPS = os.environ.get('DEPS', os.path.join(ROOT, 'deps', 'macos-arm64')) +FF = os.path.join(DEPS, 'ffmpeg', 'ffmpeg') +FP = os.path.join(DEPS, 'ffmpeg', 'ffprobe') + +for path in sys.argv[1:]: + w, h = map(int, subprocess.check_output([FP, '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=width,height', '-of', 'csv=p=0', path]).decode().split(',')) + raw = subprocess.check_output([FF, '-v', 'error', '-i', path, '-frames:v', '1', '-f', 'rawvideo', + '-pix_fmt', 'yuv444p', '-']) + n = w * h + samples = [] + for i in range(8): + cx, cy = int(((i % 4) + 0.5) * w / 4), int(((i // 4) + 0.5) * h / 2) + o = cy * w + cx + samples.append((raw[o], raw[n + o], raw[2 * n + o])) + print(f'{os.path.basename(path)} {w}x{h}: {samples}') diff --git a/docs/reports/2026-11-crop-detect-709/scripts/run_vs.sh b/docs/reports/2026-11-crop-detect-709/scripts/run_vs.sh new file mode 100755 index 00000000..d23771d0 --- /dev/null +++ b/docs/reports/2026-11-crop-detect-709/scripts/run_vs.sh @@ -0,0 +1,17 @@ +#!/bin/bash +# Run one of this report's Python scripts with the deps bundle's own Python and +# VapourSynth (macOS / Linux layout). DEPS overrides the bundle directory. +# +# scripts/run_vs.sh scripts/gamut.py +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +ROOT="$(cd "$HERE/../../../.." && pwd)" +D="${DEPS:-$ROOT/deps/macos-arm64}" +export PYTHONHOME="$D/python" +export PYTHONPATH="$D:$D/python-packages" +export DYLD_LIBRARY_PATH="$D/vapoursynth:$D/python/lib" +export LD_LIBRARY_PATH="$D/vapoursynth:$D/python/lib" +export XDG_CONFIG_HOME="$D/config" +export FFMPEG="$D/ffmpeg/ffmpeg" +export REPO="$ROOT" +exec "$D/python/bin/python" "$@" diff --git a/docs/reports/2026-11-face-restoration/data/face-codeformer.conversion.json b/docs/reports/2026-11-face-restoration/data/face-codeformer.conversion.json new file mode 100644 index 00000000..73cbf631 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/face-codeformer.conversion.json @@ -0,0 +1,67 @@ +{ + "opset": 17, + "irVersion": 8, + "torch": "2.15.0", + "models": [ + { + "file": "face-codeformer-v0.1.0.onnx", + "size": 376827102, + "sha256": "536db9b0d73400d09ca19ba1ea4978ba3b084e3bc0c3ab951597e5536c031e18", + "checkpoint": "https://github.com/sczhou/CodeFormer/releases/download/v0.1.0/codeformer.pth", + "parity": { + "w=0.0": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 3.2580789999919944, + "outputs": [ + { + "output": "output", + "max": 4.0531158447265625e-05, + "mean": 2.581076167406058e-06 + } + ] + }, + "w=0.5": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 3.893039582995698, + "outputs": [ + { + "output": "output", + "max": 4.4345855712890625e-05, + "mean": 3.16973939147902e-06 + } + ] + }, + "w=0.7": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 10.734747875016183, + "outputs": [ + { + "output": "output", + "max": 6.389617919921875e-05, + "mean": 2.88713418880058e-06 + } + ] + }, + "w=1.0": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 6.94589916698169, + "outputs": [ + { + "output": "output", + "max": 0.00010955333709716797, + "mean": 2.8745563086505894e-06 + } + ] + } + } + } + ] +} \ No newline at end of file diff --git a/docs/reports/2026-11-face-restoration/data/face-gfpgan.conversion.json b/docs/reports/2026-11-face-restoration/data/face-gfpgan.conversion.json new file mode 100644 index 00000000..ad916ff6 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/face-gfpgan.conversion.json @@ -0,0 +1,31 @@ +{ + "opset": 17, + "irVersion": 8, + "torch": "2.15.0", + "models": [ + { + "file": "face-gfpgan-v1.4.onnx", + "size": 340357340, + "sha256": "c84d5c2bf1ddc5e61cfc71524b1a155b297c1c3e884255e6a8c0ce5230a91d94", + "checkpoint": "https://github.com/TencentARC/GFPGAN/releases/download/v1.3.0/GFPGANv1.4.pth", + "parity": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 1.4851923339883797, + "outputs": [ + { + "output": "output", + "max": 1.239776611328125e-05, + "mean": 2.645203949214192e-07 + } + ] + }, + "fixedVsRandomNoise": { + "max": 0.18705010414123535, + "mean": 0.00460228743031621, + "unit": "[-1, 1] output" + } + } + ] +} \ No newline at end of file diff --git a/docs/reports/2026-11-face-restoration/data/face-restoration.conversion.json b/docs/reports/2026-11-face-restoration/data/face-restoration.conversion.json new file mode 100644 index 00000000..53dc9c50 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/face-restoration.conversion.json @@ -0,0 +1,104 @@ +{ + "opset": 17, + "irVersion": 8, + "torch": "2.15.0", + "models": [ + { + "file": "face-restoration-retinaface-r50.onnx", + "size": 109106945, + "sha256": "da4d5fa7473e3043b5b15df3aa3586d25c5cd94d5cbb53cb8d6309f525381098", + "checkpoint": "https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_Resnet50_Final.pth", + "parity": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 0.32051425002282485, + "outputs": [ + { + "output": "loc", + "max": 1.1354684829711914e-05, + "mean": 9.606442592006e-07 + }, + { + "output": "conf", + "max": 1.1920928955078125e-07, + "mean": 7.331378323475032e-10 + }, + { + "output": "landmarks", + "max": 1.8596649169921875e-05, + "mean": 1.4495417096787561e-06 + } + ] + } + }, + { + "file": "face-restoration-retinaface-mnet.onnx", + "size": 1729296, + "sha256": "92d3bdd20d0a31bfff78e9312974637abe4471b86ad6470ad663c731b7489210", + "checkpoint": "https://github.com/xinntao/facexlib/releases/download/v0.1.0/detection_mobilenet0.25_Final.pth", + "parity": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 0.012891915976069868, + "outputs": [ + { + "output": "loc", + "max": 1.2129545211791992e-05, + "mean": 1.2732803146978509e-06 + }, + { + "output": "conf", + "max": 1.1920928955078125e-07, + "mean": 2.5632617247798147e-09 + }, + { + "output": "landmarks", + "max": 2.3603439331054688e-05, + "mean": 1.7438624740428903e-06 + } + ] + } + }, + { + "file": "face-restoration-bisenet.onnx", + "size": 52598302, + "sha256": "2e8a24164b64477200bf7d0c7022858a736f36409e184e4bf3c8cb39844bf048", + "checkpoint": "https://github.com/xinntao/facexlib/releases/download/v0.2.0/parsing_bisenet.pth", + "parity": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 0.08913283300353214, + "outputs": [ + { + "output": "parsing", + "max": 1.2159347534179688e-05, + "mean": 1.3916364676476664e-06 + } + ], + "argmaxAgreement": 1.0 + } + }, + { + "file": "face-restoration-restoreformerpp.onnx", + "size": 295272527, + "sha256": "640da24f9eaae52fe4e840b4394c65583b69e0091947a0d659ccd2176c8cd86b", + "checkpoint": "https://github.com/wzhouxiff/RestoreFormerPlusPlus/releases/download/v1.0.0/RestoreFormer++.ckpt", + "parity": { + "providers": [ + "CPUExecutionProvider" + ], + "seconds": 2.610279541986529, + "outputs": [ + { + "output": "output", + "max": 0.0002478957176208496, + "mean": 2.636080659354356e-06 + } + ] + } + } + ] +} \ No newline at end of file diff --git a/docs/reports/2026-11-face-restoration/data/flicker.json b/docs/reports/2026-11-face-restoration/data/flicker.json new file mode 100644 index 00000000..52dbcc02 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/flicker.json @@ -0,0 +1,69 @@ +{ + "rect": [ + 277, + 176, + 342, + 258 + ], + "rows": { + "clean reference": { + "tdiff": 2.448, + "flick": 1.273 + }, + "codeformer_r0": { + "tdiff": 3.537, + "flick": 2.421, + "errVsClean": 6.973 + }, + "codeformer_r1": { + "tdiff": 3.158, + "flick": 1.789, + "errVsClean": 6.867 + }, + "gfpgan_r0": { + "tdiff": 2.938, + "flick": 1.914, + "errVsClean": 5.957 + }, + "gfpgan_r1": { + "tdiff": 2.642, + "flick": 1.359, + "errVsClean": 5.885 + }, + "input": { + "tdiff": 1.998, + "flick": 1.14, + "errVsClean": 5.473 + }, + "rf_r0": { + "tdiff": 3.453, + "flick": 2.287, + "errVsClean": 7.893 + }, + "rf_r1_landmarks": { + "tdiff": 3.514, + "flick": 2.284, + "errVsClean": 7.923 + }, + "rf_r1_mnet": { + "tdiff": 3.371, + "flick": 1.951, + "errVsClean": 7.978 + }, + "rf_r1_s08": { + "tdiff": 2.878, + "flick": 1.571, + "errVsClean": 7.13 + }, + "rf_r1": { + "tdiff": 3.228, + "flick": 1.799, + "errVsClean": 7.864 + }, + "rf_r2": { + "tdiff": 3.125, + "flick": 1.657, + "errVsClean": 7.833 + } + } +} diff --git a/docs/reports/2026-11-face-restoration/data/pipeline_parity.json b/docs/reports/2026-11-face-restoration/data/pipeline_parity.json new file mode 100644 index 00000000..d3ff3645 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/pipeline_parity.json @@ -0,0 +1,74 @@ +[ + { + "frame": 0, + "faces": [ + 1, + 1 + ], + "boxMaxDiff": 3.0517578125e-05, + "landmarkMaxDiff": 3.0517578125e-05, + "scoreDiff": 0.0, + "affineMaxDiff": 4.851950421880247e-05, + "alignedCropMaxDiff8bit": 0.002272278070449829, + "frameMaxDiff8bit": 0.16514296752603214, + "frameMeanDiff8bitInFace": 0.00460679355055863, + "maskReachPixels": [ + 21599, + 21112 + ] + }, + { + "frame": 40, + "faces": [ + 1, + 1 + ], + "boxMaxDiff": 3.0517578125e-05, + "landmarkMaxDiff": 6.103515625e-05, + "scoreDiff": 0.0, + "affineMaxDiff": 5.0527419375612226e-05, + "alignedCropMaxDiff8bit": 0.002416670322418213, + "frameMaxDiff8bit": 0.10665217677869987, + "frameMeanDiff8bitInFace": 0.004561936156879923, + "maskReachPixels": [ + 21023, + 20579 + ] + }, + { + "frame": 80, + "faces": [ + 1, + 1 + ], + "boxMaxDiff": 3.0517578125e-05, + "landmarkMaxDiff": 3.0517578125e-05, + "scoreDiff": 0.0, + "affineMaxDiff": 7.1380790927833e-05, + "alignedCropMaxDiff8bit": 0.001991093158721924, + "frameMaxDiff8bit": 0.11140976347372572, + "frameMeanDiff8bitInFace": 0.005738668381720679, + "maskReachPixels": [ + 22069, + 21590 + ] + }, + { + "frame": 120, + "faces": [ + 1, + 1 + ], + "boxMaxDiff": 3.0517578125e-05, + "landmarkMaxDiff": 3.0517578125e-05, + "scoreDiff": 0.0, + "affineMaxDiff": 3.851226750839487e-05, + "alignedCropMaxDiff8bit": 0.001869499683380127, + "frameMaxDiff8bit": 0.1550988369548001, + "frameMeanDiff8bitInFace": 0.00432491451527014, + "maskReachPixels": [ + 21715, + 21242 + ] + } +] diff --git a/docs/reports/2026-11-face-restoration/data/provider_bench_1.jsonl b/docs/reports/2026-11-face-restoration/data/provider_bench_1.jsonl new file mode 100644 index 00000000..95475611 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/provider_bench_1.jsonl @@ -0,0 +1,14 @@ +{"model": "face-restoration-retinaface-r50.onnx", "provider": "CPUExecutionProvider", "sessionSeconds": 0.16, "msPerRun": 405.2} +{"model": "face-restoration-retinaface-r50.onnx", "provider": "CoreMLExecutionProvider", "sessionSeconds": 2.05, "msPerRun": 87.2, "maxDiffVsCpu": 1.430511474609375e-05} +{"model": "face-restoration-retinaface-mnet.onnx", "provider": "CPUExecutionProvider", "sessionSeconds": 0.12, "msPerRun": 16.0} +{"model": "face-restoration-retinaface-mnet.onnx", "provider": "CoreMLExecutionProvider", "sessionSeconds": 0.37, "msPerRun": 6.7, "maxDiffVsCpu": 1.1086463928222656e-05} +{"model": "face-restoration-bisenet.onnx", "provider": "CPUExecutionProvider", "sessionSeconds": 0.08, "msPerRun": 98.3} +{"model": "face-restoration-bisenet.onnx", "provider": "CoreMLExecutionProvider", "sessionSeconds": 0.48, "msPerRun": 33.3, "maxDiffVsCpu": 1.0441988706588745e-05} +{"model": "face-restoration-restoreformerpp.onnx", "provider": "CPUExecutionProvider", "sessionSeconds": 0.6, "msPerRun": 6775.4} +{"model": "face-restoration-restoreformerpp.onnx", "provider": "CoreMLExecutionProvider", "sessionSeconds": 5.57, "msPerRun": 292.6, "maxDiffVsCpu": 0.00019106268882751465} +{"model": "face-gfpgan-v1.4.onnx", "provider": "CPUExecutionProvider", "sessionSeconds": 2.58, "msPerRun": 2012.2} +{"model": "face-gfpgan-v1.4.onnx", "provider": "CoreMLExecutionProvider", "sessionSeconds": 7.92, "msPerRun": 638.5, "maxDiffVsCpu": 8.225440979003906e-06} +{"model": "face-codeformer-v0.1.0.onnx", "provider": "CPUExecutionProvider", "sessionSeconds": 0.83, "msPerRun": 6094.5} +{"model": "face-codeformer-v0.1.0.onnx", "provider": "CoreMLExecutionProvider", "sessionSeconds": 46.02, "msPerRun": 676.6, "maxDiffVsCpu": 7.164478302001953e-05} + +[exited with code 0] diff --git a/docs/reports/2026-11-face-restoration/data/results.jsonl b/docs/reports/2026-11-face-restoration/data/results.jsonl new file mode 100644 index 00000000..2f75fef0 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/data/results.jsonl @@ -0,0 +1,17 @@ +{"tag": "rf_r0", "model": "restoreformer", "radius": 0, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 2.67, "seconds": 62.28, "fps": 2.408, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "rf_r1_landmarks", "model": "restoreformer", "radius": 1, "blend": false, "size": "640x480", "frames": 150, "setupSeconds": 2.47, "seconds": 59.49, "fps": 2.522, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "rf_r1", "model": "restoreformer", "radius": 1, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 2.36, "seconds": 61.51, "fps": 2.439, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "rf_r2", "model": "restoreformer", "radius": 2, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 2.43, "seconds": 63.8, "fps": 2.351, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "rf_r1_s08", "model": "restoreformer", "radius": 1, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 4.71, "seconds": 68.88, "fps": 2.178, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "input", "model": "restoreformer", "radius": 1, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 3.77, "seconds": 66.19, "fps": 2.266, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "rf_r1_mnet", "model": "restoreformer", "radius": 1, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 2.8, "seconds": 56.12, "fps": 2.673, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "gfpgan_r0", "model": "gfpgan", "radius": 0, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 2.31, "seconds": 106.27, "fps": 1.412, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "gfpgan_r1", "model": "gfpgan", "radius": 1, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 2.79, "seconds": 124.11, "fps": 1.209, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "codeformer_r0", "model": "codeformer", "radius": 0, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 46.09, "seconds": 70.02, "fps": 2.142, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "codeformer_r1", "model": "codeformer", "radius": 1, "blend": true, "size": "640x480", "frames": 150, "setupSeconds": 57.51, "seconds": 70.28, "fps": 2.134, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "speed_sd_coreml", "model": "restoreformer", "radius": 1, "blend": true, "size": "640x480", "frames": 60, "setupSeconds": 3.43, "seconds": 27.85, "fps": 2.155, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "speed_hd_coreml", "model": "restoreformer", "radius": 1, "blend": true, "size": "1920x1080", "frames": 60, "setupSeconds": 3.41, "seconds": 32.68, "fps": 1.836, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "speed_sd_cpu", "model": "restoreformer", "radius": 1, "blend": true, "size": "640x480", "frames": 12, "setupSeconds": 0.35, "seconds": 140.6, "fps": 0.085, "providers": {"detector": "CPUExecutionProvider", "parser": "CPUExecutionProvider", "restorer": "CPUExecutionProvider"}, "fallback": {}} +{"tag": "speed_hd_cpu", "model": "restoreformer", "radius": 1, "blend": true, "size": "1920x1080", "frames": 12, "setupSeconds": 0.31, "seconds": 165.93, "fps": 0.072, "providers": {"detector": "CPUExecutionProvider", "parser": "CPUExecutionProvider", "restorer": "CPUExecutionProvider"}, "fallback": {}} +{"tag": "speed_sd_coreml_gfpgan", "model": "gfpgan", "radius": 1, "blend": true, "size": "640x480", "frames": 60, "setupSeconds": 4.6, "seconds": 75.09, "fps": 0.799, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} +{"tag": "speed_sd_coreml_codeformer", "model": "codeformer", "radius": 1, "blend": true, "size": "640x480", "frames": 60, "setupSeconds": 49.25, "seconds": 164.38, "fps": 0.365, "providers": {"detector": "CoreMLExecutionProvider", "parser": "CoreMLExecutionProvider", "restorer": "CoreMLExecutionProvider"}, "fallback": {}} diff --git a/docs/reports/2026-11-face-restoration/img/01-models.png b/docs/reports/2026-11-face-restoration/img/01-models.png new file mode 100644 index 00000000..3b620ff6 Binary files /dev/null and b/docs/reports/2026-11-face-restoration/img/01-models.png differ diff --git a/docs/reports/2026-11-face-restoration/img/02-strength.png b/docs/reports/2026-11-face-restoration/img/02-strength.png new file mode 100644 index 00000000..a47a4c38 Binary files /dev/null and b/docs/reports/2026-11-face-restoration/img/02-strength.png differ diff --git a/docs/reports/2026-11-face-restoration/img/03-frame.png b/docs/reports/2026-11-face-restoration/img/03-frame.png new file mode 100644 index 00000000..cae50148 Binary files /dev/null and b/docs/reports/2026-11-face-restoration/img/03-frame.png differ diff --git a/docs/reports/2026-11-face-restoration/index.html b/docs/reports/2026-11-face-restoration/index.html new file mode 100644 index 00000000..38fe2880 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/index.html @@ -0,0 +1,246 @@ + + + + + +Face Restoration + + + +
+

Face Restoration

+

RestoreFormer++, GFPGAN and CodeFormer on ONNX Runtime, as a VapourSynth pass. 2026-11-08, macOS 27 on an M1 Pro (16 GB). Scripts: scripts/; raw results: data/ (the bench runs, flicker metrics, conversion and pipeline parity records). To reproduce: scripts/make_refs.sh, then scripts/bench.sh, scripts/flicker.py, scripts/tables.py and scripts/make_figures.sh.

+ +
+Outcome: built. A Face Restoration pass after Crop & Resize, on the ai-runtime add-on, with three model add-ons: +face-restoration (RetinaFace ResNet-50 + BiSeNet + RestoreFormer++, the default, Apache-2.0/MIT), +face-gfpgan and face-codeformer (both non-commercial). The NumPy reimplementation of facexlib's +pipeline matches it to 0.16/255; every pixel outside the faces is bit-exact; a temporal blend in aligned face space cuts +the restored face's frame-to-frame flicker by half (RestoreFormer++, excess over the clean source) to 86% (GFPGAN); RestoreFormer++ runs at 2.2 fps on SD and +1.8 fps at 1080p on Core ML (about 25× slower on the CPU). +
+ +

1. Licences, read upstream

+ + + + + + + + + + +
ComponentLicence (as its LICENSE file states it)Holder namedDecision
RestoreFormer++ (wzhouxiff/RestoreFormerPlusPlus v1.0.0)Apache-2.0none (template line left unfilled); authors per its citationdefault model, base add-on
RetinaFace (biubug6/Pytorch_Retinaface; weights as published by facexlib)MIT"Copyright (c) 2019", no namedetector, base add-on
BiSeNet (zllrunning/face-parsing.PyTorch; facexlib's parsing_bisenet.pth)MITzll, 2019face parser, base add-on
facexlib (xinntao/facexlib)MITXintao Wang, 2020hosts the weights; alignment template
GFPGAN v1.4 (TencentARC/GFPGAN)Apache-2.0 except third-party code: the StyleGAN2-derived network is under NVIDIA's licence, §3.3 "may be used … non-commercially … research or evaluation purposes only"THL A29 Limited, a Tencent company, 2021own add-on, non-commercial usage note
CodeFormer (sczhou/CodeFormer v0.1.0)S-Lab License 1.0: redistribution and use for non-commercial purposes, with the noticeS-Lab, 2022own add-on, non-commercial usage note
ParseNet (facexlib's default parser)derived from PSFRGAN, CC BY-NC-SAnot used
InsightFace / FaceFusion retinaface_10gnon-commercialnot used
+ +

2. Conversion to ONNX

+

Each model is exported by a committed script (Scripts/models/make_face_restoration.py, +make_face_gfpgan.py, make_face_codeformer.py) from the upstream checkpoint pinned by SHA-256 and +the network source pinned by commit, with the TorchScript exporter at opset 17 / IR 8 (loadable by ONNX Runtime 1.23.2, the +Intel-Mac floor). Shapes are static (detector 640×640, faces 512×512) so Core ML can take whole graphs. Re-running the +script reproduces the files byte for byte (checked for the detector and parser). Parity is ONNX Runtime (CPU) against +PyTorch on the same input, in the networks' own units:

+ + + + + + + +
FileSizeOutputmax |Δ|mean |Δ|
face-restoration-retinaface-r50.onnx109.1 MBloc / conf / landmarks1.1e-5 / 1.2e-7 / 1.9e-59.6e-7 / 7.3e-10 / 1.5e-6
face-restoration-bisenet.onnx52.6 MBlogits (argmax agrees on 100% of pixels)1.2e-51.4e-6
face-restoration-restoreformerpp.onnx295.3 MBimage, [-1, 1]2.5e-42.6e-6
face-gfpgan-v1.4.onnx340.4 MBimage, [-1, 1]1.2e-52.6e-7
face-codeformer-v0.1.0.onnx376.8 MBimage at w = 0 / 0.5 / 0.7 / 14.1e-5 / 4.4e-5 / 6.4e-5 / 1.1e-4≤3.2e-6
+

The worst case, 2.5e-4 on a [-1, 1] scale, is 0.03 of an 8-bit level.

+

Two deliberate departures from upstream inference

+
    +
  • GFPGAN with fixed noise. GFPGANer runs the StyleGAN2 decoder with randomize_noise=True: +fresh random noise on every call, i.e. new texture on every frame of a video. The export uses the generator's fixed noise +buffers (part of the checkpoint), which makes the model a pure function of its input. For scale, upstream's random-noise +output differs from the fixed-noise one by up to 0.19 (mean 0.0046) on the [-1, 1] scale.
  • +
  • CodeFormer's fidelity w as an input. CodeFormer uses w only in each fusion +block's out = dec + w·(dec·scale + shift), behind if w > 0. The export traces the w > 0 +branch; at w = 0 it adds exactly zero, so one graph is the network at every w (parity above at four +values, including 0 against upstream's skipped branch), where fixed exports would cost 377 MB per offered value.
  • +
+ +

3. The pipeline without OpenCV

+

Neither the deps bundle nor the ai-runtime add-on has OpenCV, and nothing needs it: RetinaFace's priors, box/landmark +decoding and NMS, the five-point similarity transform (Umeyama, in place of cv2.estimateAffinePartial2D), +bilinear warps, the Gaussian feathering and the paste-back are NumPy in worker/templates/face_restore.py. +scripts/pipeline_parity.py runs facexlib's PyTorch detect_faces and OpenCV beside it on four frames +of the fixture:

+ + + + + + + +
Stepmax difference over 4 frames
Box / landmarks (pixels)3.1e-5 / 6.1e-5
Detection score0
Alignment matrix7.1e-5
Aligned 512×512 crop (8-bit levels)0.0024
Final frame given the same restored face (8-bit levels)0.16 (mean in the face 0.006)
+

The pass's mask reaches ~2% fewer pixels than facexlib's (21,112 vs 21,599 on frame 0): values below 1/1024 are cut to +exactly zero, so that the blend has a finite reach and everything past it can be left bit-exact (section 6).

+ +

4. Detector: ResNet-50, not mobilenet0.25

+ + + + + + + + +
ResNet-50mobilenet0.25
File109 MB1.7 MB
640×640 on Core ML / CPU (ms)87 / 4056.7 / 16
WIDER FACE val, hard, original scale (upstream README)84.4%73.8%
Faces found ≥ 0.8, SD fixture / 1080p wide shot150/150, 90/90150/150, 90/90
Landmark jitter (px, second difference), SD fixture0.1660.243
Landmark jitter, 1080p wide shot (face ~125 px)0.2010.445
+

Both find this clip's one face in every frame; the difference is in how steadily. Mobilenet's landmarks jitter 1.5–2.2× +as much, and jitter is what a restorer turns into a face that swims; on hard faces (small, blurred, turned) the published +gap is 10 points. Against that, ResNet-50 costs ~80 ms more per frame on Core ML, which is small beside the restorer's +~300 ms per face — but it is the whole cost of a frame with no face, and on the CPU (Linux) 405 ms. Chosen for quality; +and its steadier landmarks show in the restored face too: flick 1.799 against 1.951 with mobilenet (section 5).

+ +

5. Temporal stability

+

A per-frame restorer re-invents fine texture on every frame from noisy input, and a face that is +detected a pixel differently each frame is aligned, and so restored, differently. On a mostly still talking head +(scripts/flicker.py, a fixed rectangle in the middle of the face, 150 frames, luma in 8-bit levels):

+
    +
  • flick = mean |Y[t] − (Y[t−1] + Y[t+1]) / 2|: change from frame to frame that smooth motion does not +explain — shimmer, texture that comes and goes, a face that jumps.
  • +
  • tdiff = mean |Y[t] − Y[t−1]|: all temporal change, motion included.
  • +
  • vs clean = mean |Y − clean source|: a check that blending neighbours didn't ghost or smear; not +a quality score (restorers invent plausible detail, which a pixel metric counts as error — every restorer scores +worse than the soft input here).
  • +
+ + + + + + + + + + + + + + +
Variantflicktdiffvs clean
Clean source, same crop (the floor: real motion)1.2732.448—
Input (degraded fixture)1.1401.9985.47
RestoreFormer++, no smoothing2.2873.4537.89
RestoreFormer++, landmarks smoothed only (r = 1)2.2843.5147.92
RestoreFormer++, landmarks + aligned-space blend (r = 1)1.7993.2287.86
RestoreFormer++, landmarks + blend (r = 2)1.6573.1257.83
RestoreFormer++, r = 1, strength 0.8 (the default)1.5712.8787.13
RestoreFormer++, r = 1, mobilenet0.25 detector1.9513.3717.98
GFPGAN, no smoothing1.9142.9385.96
GFPGAN, r = 11.3592.6425.88
CodeFormer (w 0.7), no smoothing2.4213.5376.97
CodeFormer (w 0.7), r = 11.7893.1586.87
+

Correction, found during the work. The brief's minimum — smoothing landmarks +over n−1, n, n+1 — was built first and changed nothing measurable here: flick 2.287 → 2.284. On this clip the +detector is already steady (landmark jitter 0.17 px), so the restorer's own per-frame texture is the whole problem. +That is what led to the second step, which is what the pass ships: each restored face is averaged with the restored +faces it matches in the neighbouring frames, in the aligned 512×512 space where they line up, weighted down to +nothing wherever the input crops differ by more than 15 levels (a moving mouth or eye keeps its own frame). +Landmark smoothing stays: it costs nothing and matters where detection is less steady than here (the mobilenet row: +flick 1.951 against 1.799 with ResNet-50's steadier landmarks).

+

The blend cuts RestoreFormer++'s excess flicker (above the clean source's 1.273) by half at r = 1 +(2.287 → 1.799) and GFPGAN's by 86% (1.914 → 1.359), with "vs clean" unchanged or lower, i.e. no ghosting the metric +can see. r = 2 gains another 0.14 but reads four frames either side for the preview and widens the window over which +a turning head is averaged; r = 1 ships. Strength 0.8 (the default) keeps a fifth of the real face's texture and +takes flicker to 1.571. Limits: one clip of one mostly still face; no measurement of fast motion, +turning heads or scene cuts, where the motion weight is what stops ghosting and is untested beyond this clip.

+
Five face crops side by side +
Frame 75, 200×200 crop at 2× (nearest neighbour). Left to right: the degraded input; the clean source +(same crop and scale, no damage — not available to the pass); RestoreFormer++; GFPGAN v1.4; CodeFormer w 0.7 — all at +strength 1 with smoothing. Look at the eyes and teeth: every model recovers them; GFPGAN and CodeFormer invent a +lighter iris colour than the source has.
+
Input, strength 1, strength 0.8 +
Input | RestoreFormer++ at strength 1 | at 0.8, the default. 0.8 keeps some of the real face's skin texture +and flickers less (table above).
+
Full frame before and after +
The whole frame, input above, restored (default settings) below: only the face changes; the background, +hair and clothes are the input's pixels.
+ +

6. Bit-exact outside the faces

+

The pass converts the frame to float RGB (the source's matrix and range), restores and pastes the faces there, converts +back, and composites with std.MaskedMerge in the source's own format using the face mask (luma-sized, +first_plane=True, so 4:2:0 chroma takes it resampled). Wherever the mask is zero — everything outside the +feathered faces, and every frame without one — the output is the input. integration_face_restore_test.dart +compares a lossless encode with the pass on against one with it off, sample by sample in every plane:

+ + + + + + + +
Case (16 frames, FFV1)Face mean |Δ| (8-bit levels)Samples changed outside the face box
RestoreFormer++, defaults, 8-bit 4:2:04.700
Strength 000
GFPGAN, smoothing on2.920
CodeFormer, smoothing on / off3.59 / 3.650 / 0
RestoreFormer++, 10-bit 4:2:0 source (stays 10-bit)4.680
+

The "face box" is generous (x 190–430, y 70–370 of 640×480, about half the face's size added on every side), +because the feathered mask reaches past the face and the head moves; the test proves nothing changed outside it, +not where exactly inside it the change stops. Preview against encode at two frames: mean |Δ| 0.0 (identical).

+ +

7. Speed and providers

+

One face in every frame, scripts/face_run.py in the deps bundle's Python (the code the +generated script runs), M1 Pro, smoothing on. Other agents' jobs shared the GPU during parts of this session; these +runs were made with no other VapourSynth process running, but the numbers are one sample each.

+ + + + + + + + +
RunSizeFramesfpsSetup (s)Note
RestoreFormer++, Core ML640x480602.153.4
RestoreFormer++, Core ML1920x1080601.843.4detection on a 640 copy; conversions and paste-back at 1080p
RestoreFormer++, CPU (VAPOURBOX_AI_FORCE_CPU=1)640x480120.090.3
RestoreFormer++, CPU1920x1080120.070.3
GFPGAN, Core ML640x480600.804.6
CodeFormer, Core ML640x480600.3649.2setup includes Core ML compiling the model
+

Core ML runs RestoreFormer++ about 25× faster than the CPU. The cost is per face, so a +frame with two faces takes about twice as long, and 1080p costs little more than SD (the face is restored at 512×512 +either way). Per-model latency on this Mac (Scripts/models/face_provider_bench.py, one 512² face or one 640² +detection, after warm-up):

+ + + + + + + +
ModelCore ML (ms)CPU (ms)Core ML vs CPU max |Δ|Core ML session (s)
RetinaFace R50874051.4e-52.1
BiSeNet33981.0e-50.5
RestoreFormer++2936,7751.9e-45.6
GFPGAN v1.46392,0128.2e-67.9
CodeFormer6776,0957.2e-546 (30 with CPUAndGPU units)
+

Linux has only the CPU provider, so there a minute of SD video with a face throughout takes on the order of +5.9 hours; Windows uses DirectML (untested here).

+ +

8. What wasn't tested

+
    +
  • Windows (DirectML), Linux (CPU), macOS x64: only macOS arm64 was run. The models are plain opset-17 +graphs and ran on ONNX Runtime's CPU provider here, but DirectML's operator coverage for the transformer models +(RestoreFormer++, CodeFormer) is unverified; a failure there falls back to the CPU at session creation, or at the first run +(ai_runtime.Runner).
  • +
  • TensorRT/CUDA: hooks only, as for every ai-runtime pass.
  • +
  • One clip, one face, frontal, mostly still. Crowds, profiles, occlusion, scene cuts across a face, and faces smaller +than ~60 px at 640 were not measured.
  • +
  • Faithfulness: a restorer invents plausible detail. No metric here says the restored face is the person; Strength is +the user's control for that.
  • +
+ +

9. Decisions

+
    +
  • Pass position: after Crop & Resize, before Sharpen. Faces are restored at the delivered +resolution (an upscale gives the restorer more pixels per face; nothing resamples the result afterwards); sharpening +before would sharpen a face that is then replaced; grain after puts texture back over the smooth result.
  • +
  • RestoreFormer++ as the default — the only one of the three with a clean licence.
  • +
  • BiSeNet, not ParseNet (licence), with facexlib's classes for the face (skin, brows, eyes, glasses, +ears, nose, mouth, lips) and its feathering.
  • +
  • CodeFormer w as a graph input (section 2).
  • +
  • Static shapes, the detector fed a padded 640 square: Core ML takes static graphs whole.
  • +
  • Runs serialised per pass (ai_runtime.Runner, one lock for the three models) with a CPU retry: +two jobs' worth of concurrent Core ML predictions ran the GPU out of memory (kIOGPUCommandBufferCallbackErrorOutOfMemory).
  • +
  • Default strength 0.8, smoothing on (section 5).
  • +
  • No Core ML model cache. ONNX Runtime's ModelCacheDirectory cut RestoreFormer++'s session +creation but the second session from the cache failed (output_features has no value), so each job compiles: +RestoreFormer++ 2–6 s, GFPGAN 2–8 s, CodeFormer 30–50 s.
  • +
+
+ + diff --git a/docs/reports/2026-11-face-restoration/scripts/bench.sh b/docs/reports/2026-11-face-restoration/scripts/bench.sh new file mode 100755 index 00000000..a608d885 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/bench.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# Every run behind the report's speed and flicker tables, in order. +# +# DEPS, ADDONS, VAPOURBOX_ADDON_STAGING as for run.sh +# WORK= outputs (.npy, .png, results.jsonl) +# CLEAN= HD= built by make_refs.sh +# +# The fixture is the committed Tests/TestResources/face_restore_test.mp4. +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${WORK:?set WORK to a scratch directory}" "${CLEAN:?}" "${HD:?}" +FIX="$REPO/Tests/TestResources/face_restore_test.mp4" +OUT="$WORK/runs" +mkdir -p "$OUT" +R() { "$HERE/run.sh" face_run.py "$@" | tee -a "$WORK/results.jsonl"; } + +# Flicker: smoothing off / landmarks only / landmarks + blend, radius 1 and 2. +R --clip "$FIX" --radius 0 --save-y --out "$OUT" --tag rf_r0 --png 30 --png 75 +R --clip "$FIX" --radius 1 --no-blend --save-y --out "$OUT" --tag rf_r1_landmarks --png 30 +R --clip "$FIX" --radius 1 --save-y --out "$OUT" --tag rf_r1 --png 30 --png 75 +R --clip "$FIX" --radius 2 --save-y --out "$OUT" --tag rf_r2 --png 30 +R --clip "$FIX" --radius 1 --strength 0.8 --save-y --out "$OUT" --tag rf_r1_s08 --png 75 +R --clip "$FIX" --radius 1 --strength 0 --save-y --out "$OUT" --tag input +R --clip "$FIX" --radius 1 --det mnet --save-y --out "$OUT" --tag rf_r1_mnet +for m in gfpgan codeformer; do + R --clip "$FIX" --model $m --radius 0 --save-y --out "$OUT" --tag ${m}_r0 --png 75 + R --clip "$FIX" --model $m --radius 1 --save-y --out "$OUT" --tag ${m}_r1 --png 75 +done +"$HERE/run.sh" luma_npy.py "$CLEAN" "$WORK/clean_y.npy" 2>/dev/null || true + +# Speed: SD (the fixture) and 1080p, Core ML (default) vs the CPU. +R --clip "$FIX" --frames 0:60 --tag speed_sd_coreml +R --clip "$HD" --frames 0:60 --tag speed_hd_coreml +VAPOURBOX_AI_FORCE_CPU=1 R --clip "$FIX" --frames 0:12 --tag speed_sd_cpu +VAPOURBOX_AI_FORCE_CPU=1 R --clip "$HD" --frames 0:12 --tag speed_hd_cpu +for m in gfpgan codeformer; do + R --clip "$FIX" --model $m --frames 0:60 --tag speed_sd_coreml_$m +done diff --git a/docs/reports/2026-11-face-restoration/scripts/face_rect.py b/docs/reports/2026-11-face-restoration/scripts/face_rect.py new file mode 100644 index 00000000..a10aa3ce --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/face_rect.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""The fixed face rectangle flicker.py measures in: RetinaFace (the add-on's +ONNX model, through face_restore.detect) on every frame of the clean +reference, the median box, shrunk to its middle 60% so it holds only face. +Also prints per-frame detection counts and the landmark jitter (frame-to-frame +movement of the five points, pixels) on the clip given, for the detector +comparison. + + VAPOURBOX_ADDON_STAGING= \\ + python face_rect.py [r50|mnet] (ffmpeg on PATH; conversion venv) +""" +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..", "..", "..", "..", "worker", "templates")) +import face_restore as fr # noqa: E402 +import onnxruntime as ort # noqa: E402 + + +def main(): + clip = sys.argv[1] + kind = sys.argv[2] if len(sys.argv) > 2 else "r50" + staging = os.environ["VAPOURBOX_ADDON_STAGING"] + sess = ort.InferenceSession(os.path.join(staging, f"face-restoration-retinaface-{kind}.onnx"), + providers=["CPUExecutionProvider"]) + info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=width,height", "-of", "json", clip]))["streams"][0] + w, h = info["width"], info["height"] + # As face_restore does: scaled (bicubic) to fit the detector's 640 square. + scale = min(1.0, fr.DET_SIZE / max(w, h)) + w, h = round(w * scale), round(h * scale) + raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", clip, "-vf", f"scale={w}:{h}:flags=bicubic", + "-f", "rawvideo", "-pix_fmt", "rgb24", "-"]) + frames = np.frombuffer(raw, np.uint8).reshape(-1, h, w, 3) + boxes, marks, scores, counts = [], [], [], [] + for f in frames: + s, b, m = fr.detect(sess, f.transpose(2, 0, 1).astype(np.float32) / 255.0, (scale, scale), 0.5) + counts.append(int((s >= 0.8).sum())) + if len(s): + boxes.append(b[0]) + marks.append(m[0]) + scores.append(float(s[0])) + med = np.median(np.array(boxes), 0) + cx, cy = (med[0] + med[2]) / 2, (med[1] + med[3]) / 2 + hw, hh = (med[2] - med[0]) * 0.3, (med[3] - med[1]) * 0.3 + rect = [int(cx - hw), int(cy - hh), int(cx + hw), int(cy + hh)] + marks = np.array(marks) + jitter = np.abs(marks[1:-1] - (marks[:-2] + marks[2:]) / 2).mean() if len(marks) > 2 else None + print(json.dumps({"detector": kind, "frames": len(frames), "framesWithFace>=0.8": int(sum(c > 0 for c in counts)), + "minScore": min(scores), "meanScore": float(np.mean(scores)), + "landmarkJitterPx": None if jitter is None else round(float(jitter), 3), + "medianBox": [round(float(v), 1) for v in med], "rect": rect})) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-face-restoration/scripts/face_run.py b/docs/reports/2026-11-face-restoration/scripts/face_run.py new file mode 100644 index 00000000..a6373c72 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/face_run.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +"""Run worker/templates/face_restore.py on a clip inside the deps bundle's +Python, time it, and save what the flicker and image comparisons need. + +Run with the deps bundle's Python and VapourSynth importable (as the vspipe +wrapper sets them up), the repo's worker/templates and the ai-runtime add-on's +site-packages on PYTHONPATH, ffmpeg/ffprobe on PATH, and +VAPOURBOX_ADDON_STAGING pointing at the converted .onnx files (run.sh does +all of this). VAPOURBOX_AI_FORCE_CPU=1 runs every model on the CPU. + + face_run.py --clip FILE [--model restoreformer|gfpgan|codeformer] + [--radius R] [--no-blend] [--strength S] [--fidelity W] + [--frames A:B] [--png N ...] [--save-y] [--out DIR] [--tag NAME] + [--det r50|mnet] + +Prints one JSON line: frames, seconds, fps, the provider each model ran on, +and any CPU fallback. +""" +import argparse +import json +import os +import subprocess +import time + +import numpy as np +import vapoursynth as vs + +core = vs.core +STAGING = os.environ.get("VAPOURBOX_ADDON_STAGING", "") +FILES = { + "restoreformer": "face-restoration-restoreformerpp.onnx", + "gfpgan": "face-gfpgan-v1.4.onnx", + "codeformer": "face-codeformer-v0.1.0.onnx", +} + + +def load(path, frames=None): + """A clip of `path` decoded by ffmpeg to 8-bit 4:2:0 (the fixtures' format).""" + info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=width,height", "-of", "json", path]))["streams"][0] + w, h = info["width"], info["height"] + raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "yuv420p", "-"]) + per = w * h * 3 // 2 + data = np.frombuffer(raw, np.uint8) + n = len(data) // per + a, b = frames if frames else (0, n) + blank = core.std.BlankClip(width=w, height=h, format=vs.YUV420P8, length=b - a, fpsnum=30000, fpsden=1001) + sizes = [(h, w), (h // 2, w // 2), (h // 2, w // 2)] + + def fill(n, f): + out = f.copy() + off = (a + n) * per + for p, (ph, pw) in enumerate(sizes): + np.copyto(np.asarray(out[p]), data[off:off + ph * pw].reshape(ph, pw)) + off += ph * pw + return out + + return core.std.ModifyFrame(blank, blank, fill) + + +def save_png(clip, n, path): + rgb = core.resize.Bicubic(clip[n], format=vs.RGB24, matrix_in_s="470bg") + f = rgb.get_frame(0) + arr = np.ascontiguousarray(np.stack([np.asarray(f[p]) for p in range(3)], -1)) + subprocess.run(["ffmpeg", "-v", "error", "-y", "-f", "rawvideo", "-pix_fmt", "rgb24", "-s", + f"{clip.width}x{clip.height}", "-i", "-", "-frames:v", "1", path], + input=arr.tobytes(), check=True) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--clip", required=True) + ap.add_argument("--model", default="restoreformer", choices=sorted(FILES)) + ap.add_argument("--radius", type=int, default=1) + ap.add_argument("--no-blend", action="store_true") + ap.add_argument("--strength", type=float, default=1.0) + ap.add_argument("--fidelity", type=float, default=0.7) + ap.add_argument("--threshold", type=float, default=0.8) + ap.add_argument("--frames", default=None) + ap.add_argument("--png", type=int, action="append", default=[]) + ap.add_argument("--save-y", action="store_true") + ap.add_argument("--out", default=".") + ap.add_argument("--tag", default="run") + ap.add_argument("--det", default="r50", choices=["r50", "mnet"]) + args = ap.parse_args() + import face_restore + + frames = tuple(int(x) for x in args.frames.split(":")) if args.frames else None + src = load(args.clip, frames) + t0 = time.perf_counter() + # Mirrors the generated script's call (worker/templates/*_template.vpy). + out = face_restore.restore_faces( + src, + detector=os.path.join(STAGING, f"face-restoration-retinaface-{args.det}.onnx"), + parser=os.path.join(STAGING, "face-restoration-bisenet.onnx"), + restorer=os.path.join(STAGING, FILES[args.model]), + model=args.model, strength=args.strength, fidelity=args.fidelity, threshold=args.threshold, + temporal_radius=args.radius, temporal_blend=not args.no_blend, + matrix="470bg", color_range="limited") + setup = time.perf_counter() - t0 + os.makedirs(args.out, exist_ok=True) + ys = [] + t0 = time.perf_counter() + for f in out.frames(close=True): + if args.save_y: + ys.append(np.asarray(f[0]).copy()) + dt = time.perf_counter() - t0 + for n in args.png: + save_png(out, n, os.path.join(args.out, f"{args.tag}_{n}.png")) + if args.save_y: + np.save(os.path.join(args.out, f"{args.tag}_y.npy"), np.stack(ys)) + runners = face_restore.last_restorer + print(json.dumps({ + "tag": args.tag, "model": args.model, "radius": args.radius, "blend": not args.no_blend, + "size": f"{src.width}x{src.height}", "frames": out.num_frames, + "setupSeconds": round(setup, 2), "seconds": round(dt, 2), "fps": round(out.num_frames / dt, 3), + "providers": {k: r.provider for k, r in runners.items()}, + "fallback": {k: r.fallback for k, r in runners.items() if r.fallback}, + })) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-face-restoration/scripts/flicker.py b/docs/reports/2026-11-face-restoration/scripts/flicker.py new file mode 100644 index 00000000..84be1d70 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/flicker.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +"""Flicker in the face region, from the luma .npy files face_run.py --save-y +writes (one (frames, h, w) array per variant). + +Three numbers per variant, all over a fixed face rectangle (the median of the +detected boxes on the clean reference, shrunk to the face's middle 60%, so +hair and background never enter it), in 8-bit luma levels: + +- tdiff: mean |Y[t] - Y[t-1]| (all temporal change, incl. motion) +- flick: mean |Y[t] - (Y[t-1] + Y[t+1])/2| (frame-to-frame change that is not + explained by smooth motion: noise, texture that comes and goes, + a face that jumps) +- errVsClean: mean |Y - clean| (distance from the undegraded source) + +The clean reference (the undegraded source, same crop and scale) gives the +floor: what the true motion of the face costs. What the metric can't tell +you: it is one clip of one talking head; it doesn't separate "restorer +hallucinated new texture" from "restorer followed real motion" beyond the +second difference; lower is not better past the clean reference's value +(a frozen face would score 0). + + python flicker.py [x0 y0 x1 y1] +""" + +import json +import os +import sys + +import numpy as np + + +def metrics(y, rect, clean=None): + x0, y0, x1, y1 = rect + r = y[:, y0:y1, x0:x1].astype(np.float64) + tdiff = np.abs(np.diff(r, axis=0)).mean() + flick = np.abs(r[1:-1] - (r[:-2] + r[2:]) / 2).mean() + out = {"tdiff": round(float(tdiff), 3), "flick": round(float(flick), 3)} + if clean is not None: + # Mean distance from the undegraded source: not a quality score (a + # restorer invents plausible detail, which a pixel metric counts as + # error), but a blend that ghosted or smeared motion would raise it. + c = clean[: len(y), y0:y1, x0:x1].astype(np.float64) + out["errVsClean"] = round(float(np.abs(r[: len(c)] - c).mean()), 3) + return out + + +def main(): + d, clean_path = sys.argv[1], sys.argv[2] + rect = [int(v) for v in sys.argv[3:7]] if len(sys.argv) >= 7 else None + clean = np.load(clean_path) + if rect is None: + raise SystemExit("pass the face rectangle x0 y0 x1 y1") + rows = {"clean reference": metrics(clean, rect)} + for name in sorted(os.listdir(d)): + if name.endswith("_y.npy"): + y = np.load(os.path.join(d, name)) + n = min(len(y), len(clean)) + rows[name[:-6]] = metrics(y[:n], rect, clean) + print(json.dumps({"rect": rect, "rows": rows}, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-face-restoration/scripts/luma_npy.py b/docs/reports/2026-11-face-restoration/scripts/luma_npy.py new file mode 100644 index 00000000..21324090 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/luma_npy.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Decode a clip's luma to a (frames, h, w) uint8 .npy (ffmpeg on PATH), for +flicker.py's reference rows. + + python luma_npy.py +""" +import json +import subprocess +import sys + +import numpy as np + + +def main(): + clip, out = sys.argv[1], sys.argv[2] + info = json.loads(subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=width,height", "-of", "json", clip]))["streams"][0] + w, h = info["width"], info["height"] + raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", clip, "-f", "rawvideo", "-pix_fmt", "gray", "-"]) + np.save(out, np.frombuffer(raw, np.uint8).reshape(-1, h, w)) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-face-restoration/scripts/make_figures.sh b/docs/reports/2026-11-face-restoration/scripts/make_figures.sh new file mode 100755 index 00000000..4fee9285 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/make_figures.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# The report's images, from bench.sh's PNGs: face crops (200x200 from the +# 640x480 frame, scaled 2x nearest-neighbour) side by side, and one full frame. +# +# WORK= CLEAN= ./make_figures.sh [ffmpeg] +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +FF="${1:-ffmpeg}" +: "${WORK:?}" "${CLEAN:?}" +IMG="$HERE/../img" +R="$WORK/runs" +FIX="$REPO/Tests/TestResources/face_restore_test.mp4" +mkdir -p "$IMG" "$WORK/fig" +CROP="crop=200:200:210:100,scale=400:400:flags=neighbor" +frame() { "$FF" -v error -y -i "$1" -vf "select=eq(n\,$2),$CROP" -frames:v 1 "$3"; } +crop() { "$FF" -v error -y -i "$1" -vf "$CROP" -frames:v 1 "$2"; } + +frame "$FIX" 75 "$WORK/fig/in.png" +frame "$CLEAN" 75 "$WORK/fig/clean.png" +for t in rf_r1_75 rf_r1_s08_75 gfpgan_r1_75 codeformer_r1_75; do crop "$R/$t.png" "$WORK/fig/$t.png"; done +# 1: input | clean source | RestoreFormer++ | GFPGAN | CodeFormer (strength 1, smoothing on) +"$FF" -v error -y -i "$WORK/fig/in.png" -i "$WORK/fig/clean.png" -i "$WORK/fig/rf_r1_75.png" \ + -i "$WORK/fig/gfpgan_r1_75.png" -i "$WORK/fig/codeformer_r1_75.png" -filter_complex hstack=5 "$IMG/01-models.png" +# 2: input | RestoreFormer++ strength 1 | strength 0.8 (the default) +"$FF" -v error -y -i "$WORK/fig/in.png" -i "$WORK/fig/rf_r1_75.png" -i "$WORK/fig/rf_r1_s08_75.png" \ + -filter_complex hstack=3 "$IMG/02-strength.png" +# 3: the whole frame, input above restored (default strength) +"$FF" -v error -y -i "$FIX" -i "$R/rf_r1_s08_75.png" -filter_complex \ + "[0:v]select=eq(n\,75),scale=640:480[a];[a][1:v]vstack=2,scale=480:-1" -frames:v 1 "$IMG/03-frame.png" +ls -l "$IMG" diff --git a/docs/reports/2026-11-face-restoration/scripts/make_refs.sh b/docs/reports/2026-11-face-restoration/scripts/make_refs.sh new file mode 100755 index 00000000..28175136 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/make_refs.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# The two clips the report needs besides the committed fixture, from the same +# NASA public-domain source (see Tests/TestResources/generate_face_test.sh): +# +# face_clean.mkv the fixture's crop and scale with none of its damage +# (the flicker reference), lossless +# face_1080.mp4 3 s of the uncropped 1080p original (the 1080p speed run) +# +# ./make_refs.sh [ffmpeg] +set -euo pipefail +SRC="${1:?source .webm}"; OUT="${2:?output dir}"; FF="${3:-ffmpeg}" +mkdir -p "$OUT" +"$FF" -y -v error -ss 50 -t 5 -i "$SRC" -an \ + -vf "fps=30000/1001,crop=720:540:728:140,scale=640:480:flags=lanczos,format=yuv420p" \ + -c:v ffv1 "$OUT/face_clean.mkv" +"$FF" -y -v error -ss 50 -t 3 -i "$SRC" -an -vf "fps=30000/1001,format=yuv420p" \ + -c:v libx264 -crf 16 -preset fast "$OUT/face_1080.mp4" +ls -l "$OUT" diff --git a/docs/reports/2026-11-face-restoration/scripts/pipeline_parity.py b/docs/reports/2026-11-face-restoration/scripts/pipeline_parity.py new file mode 100644 index 00000000..8dd86805 --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/pipeline_parity.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Check worker/templates/face_restore.py's NumPy pipeline against the +reference it reimplements without OpenCV: facexlib's PyTorch RetinaFace +(detect_faces), cv2.estimateAffinePartial2D + cv2.warpAffine (alignment), and +facexlib's paste-back (BiSeNet mask, two Gaussian blurs, cv2.warpAffine back). + +Run in the conversion venv (torch, opencv-python-headless, onnxruntime) after +Scripts/models/make_face_restoration.py, with: + + FACE_MODELS_WORK= (facexlib source + checkpoints) + VAPOURBOX_ADDON_STAGING= + python pipeline_parity.py [frame ...] + +Prints JSON: per frame, detection differences (pixels), alignment matrix +difference, aligned-crop difference, and the final frame difference (8-bit +levels) between the two paste-backs given the same restored face. +""" + +import json +import os +import subprocess +import sys + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, "..", "..", "..", "..")) +sys.path.insert(0, os.path.join(REPO, "worker", "templates")) +sys.path.insert(0, os.path.join(REPO, "Scripts", "models")) + +import cv2 # noqa: E402 +import onnxruntime as ort # noqa: E402 +import torch # noqa: E402 + +import face_models_common as fm # noqa: E402 +import face_restore as fr # noqa: E402 +import make_face_restoration as mfr # noqa: E402 + + +def frames_rgb(path, wanted): + probe = subprocess.check_output(["ffprobe", "-v", "error", "-select_streams", "v:0", "-show_entries", + "stream=width,height", "-of", "csv=p=0", path]).decode().split(",") + w, h = int(probe[0]), int(probe[1]) + raw = subprocess.check_output(["ffmpeg", "-v", "error", "-i", path, "-f", "rawvideo", "-pix_fmt", "rgb24", "-"]) + allf = np.frombuffer(raw, np.uint8).reshape(-1, h, w, 3) + return {n: allf[n] for n in wanted} + + +def main(): + clip = sys.argv[1] + wanted = [int(x) for x in sys.argv[2:]] or [0, 40, 80, 120] + staging = fm.STAGING + det_pt = mfr.detector("r50") + det_onnx = ort.InferenceSession(os.path.join(staging, "face-restoration-retinaface-r50.onnx"), + providers=["CPUExecutionProvider"]) + parse_onnx = ort.InferenceSession(os.path.join(staging, "face-restoration-bisenet.onnx"), + providers=["CPUExecutionProvider"]) + restore_onnx = ort.InferenceSession(os.path.join(staging, "face-restoration-restoreformerpp.onnx"), + providers=["CPUExecutionProvider"]) + out = [] + for n, rgb8 in frames_rgb(clip, wanted).items(): + h, w, _ = rgb8.shape + assert max(h, w) <= fr.DET_SIZE, "use a clip that needs no detection resize (<= 640)" + bgr = rgb8[..., ::-1].copy() + with torch.no_grad(): + ref = det_pt.detect_faces(bgr, conf_threshold=0.8) # x1 y1 x2 y2 score, 5 landmarks + rgb = rgb8.transpose(2, 0, 1).astype(np.float32) / 255.0 + scores, boxes, marks = fr.detect(det_onnx, rgb, (1.0, 1.0), 0.8) + row = {"frame": n, "faces": [len(ref), len(scores)]} + if len(ref) and len(scores): + # face_restore maps detection pixels with the half-pixel convention; + # at scale 1 that is the identity, so these compare directly. + row["boxMaxDiff"] = float(np.abs(ref[0, :4] - boxes[0]).max()) + row["landmarkMaxDiff"] = float(np.abs(ref[0, 5:].reshape(5, 2) - marks[0]).max()) + row["scoreDiff"] = float(abs(ref[0, 4] - scores[0])) + lm = marks[0] + m_cv = cv2.estimateAffinePartial2D(lm, fr.FFHQ_TEMPLATE, method=cv2.LMEDS)[0] + m_np = fr.similarity(lm, fr.FFHQ_TEMPLATE) + row["affineMaxDiff"] = float(np.abs(m_cv - m_np).max()) + crop_cv = cv2.warpAffine(rgb.transpose(1, 2, 0), m_np, (512, 512), borderMode=cv2.BORDER_CONSTANT, + borderValue=tuple(float(v) for v in fr.BORDER_RGB)).transpose(2, 0, 1) + crop_np = fr.align(rgb, m_np) + row["alignedCropMaxDiff8bit"] = float(np.abs(crop_cv - crop_np).max() * 255) + restored = np.clip((restore_onnx.run(None, {"input": (crop_np * 2 - 1)[None]})[0][0] + 1) / 2, 0, 1) + # facexlib's paste-back (use_parse branch), with BiSeNet and the + # face classes face_restore uses. + x = ((restored - fr.PARSE_MEAN) / fr.PARSE_STD)[None].astype(np.float32) + labels = parse_onnx.run(None, {"input": x})[0][0].argmax(0) + mask = np.isin(labels, fr.FACE_CLASSES).astype(np.float64) * 255 + mask = cv2.GaussianBlur(mask, (101, 101), 11) + mask = cv2.GaussianBlur(mask, (101, 101), 11) + mask[:10], mask[-10:], mask[:, :10], mask[:, -10:] = 0, 0, 0, 0 + mask = mask / 255.0 + inv = cv2.invertAffineTransform(m_np) + inv_face = cv2.warpAffine(restored.transpose(1, 2, 0), inv, (w, h)) + inv_mask = cv2.warpAffine(mask, inv, (w, h), flags=3)[..., None] + ref_frame = inv_mask * inv_face + (1 - inv_mask) * rgb.transpose(1, 2, 0) + # face_restore's: paste the face where its mask reaches, blend once. + o = rgb.copy() + a = np.zeros((h, w), np.float32) + fr.paste(rgb, o, a, restored, fr.face_mask(parse_onnx.run(None, {"input": x})[0]), m_np) + ours = (a[None] * o + (1 - a[None]) * rgb).transpose(1, 2, 0) + d = np.abs(ref_frame - ours) * 255 + row["frameMaxDiff8bit"] = float(d.max()) + row["frameMeanDiff8bitInFace"] = float(d[a > 0].mean()) + ref_reach = inv_mask[..., 0] > 1 / 1024 + row["maskReachPixels"] = [int(ref_reach.sum()), int((a > 0).sum())] + out.append(row) + print(json.dumps(out, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-face-restoration/scripts/run.sh b/docs/reports/2026-11-face-restoration/scripts/run.sh new file mode 100755 index 00000000..e67fe1df --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Run a script (face_run.py) inside the deps bundle's Python with VapourSynth +# importable, as the vspipe wrapper sets it up, plus the repo's templates and +# the ai-runtime add-on on PYTHONPATH and the bundle's ffmpeg on PATH. +# +# DEPS= ADDONS= +# VAPOURBOX_ADDON_STAGING= +# ./run.sh face_run.py --clip ... [args] +set -euo pipefail +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/../../../.." && pwd)" +: "${DEPS:?set DEPS to the deps bundle (deps/macos-arm64)}" +: "${ADDONS:?set ADDONS to the add-ons directory holding ai-runtime}" +: "${VAPOURBOX_ADDON_STAGING:?set VAPOURBOX_ADDON_STAGING to the converted models}" +export PATH="$DEPS/ffmpeg:$DEPS/python/bin:$PATH" +export PYTHONHOME="$DEPS/python" +export PYTHONPATH="$DEPS:$DEPS/python-packages:$REPO/worker/templates:$ADDONS/ai-runtime/site-packages" +export DYLD_LIBRARY_PATH="$DEPS/vapoursynth:$DEPS/python/lib" +export LD_LIBRARY_PATH="$DEPS/vapoursynth:$DEPS/python/lib" +export XDG_CONFIG_HOME="${XDG_CONFIG_HOME:-$DEPS/config}" +script="$1"; shift +exec "$DEPS/python/bin/python3" "$HERE/$script" "$@" diff --git a/docs/reports/2026-11-face-restoration/scripts/tables.py b/docs/reports/2026-11-face-restoration/scripts/tables.py new file mode 100644 index 00000000..d690c9db --- /dev/null +++ b/docs/reports/2026-11-face-restoration/scripts/tables.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Print the report's flicker and speed tables (HTML) from bench.sh's +results.jsonl and flicker.py's JSON. + + python tables.py +""" +import json +import sys + +LABELS = { + "input": "Input (pass at strength 0)", + "rf_r0": "RestoreFormer++, smoothing off", + "rf_r1_landmarks": "RestoreFormer++, landmarks smoothed only (r=1)", + "rf_r1": "RestoreFormer++, landmarks + blend (r=1, strength 1)", + "rf_r2": "RestoreFormer++, landmarks + blend (r=2)", + "rf_r1_s08": "RestoreFormer++, r=1, strength 0.8 (the default)", + "rf_r1_mnet": "RestoreFormer++, r=1, mobilenet detector", + "gfpgan_r0": "GFPGAN, smoothing off", + "gfpgan_r1": "GFPGAN, r=1", + "codeformer_r0": "CodeFormer (w 0.7), smoothing off", + "codeformer_r1": "CodeFormer (w 0.7), r=1", +} + + +def main(): + runs = {} + for line in open(sys.argv[1]): + d = json.loads(line) + runs[d["tag"]] = d + fl = json.load(open(sys.argv[2]))["rows"] + clean = fl["clean reference"] + print('\n' + '') + print(f'' + f'') + for tag, label in LABELS.items(): + if tag in fl: + r = fl[tag] + print(f'' + f'') + print('
Variantflicktdiffvs clean
Clean source (the floor){clean["flick"]:.3f}{clean["tdiff"]:.3f}—
{label}{r["flick"]:.3f}{r["tdiff"]:.3f}{r["errVsClean"]:.3f}
') + print('\n' + '') + for tag, d in runs.items(): + if tag.startswith("speed"): + print(f'' + f'' + f'') + print('
RunSizeFramesfpsSession setup (s)Restorer on
{tag}{d["size"]}{d["frames"]}{d["fps"]:.2f}{d["setupSeconds"]:.1f}{d["providers"]["restorer"]}
') + + +if __name__ == "__main__": + main() diff --git a/docs/reports/2026-11-noise-halo-scores/img/01-noise-kinds.png b/docs/reports/2026-11-noise-halo-scores/img/01-noise-kinds.png new file mode 100644 index 00000000..2eb32233 Binary files /dev/null and b/docs/reports/2026-11-noise-halo-scores/img/01-noise-kinds.png differ diff --git a/docs/reports/2026-11-noise-halo-scores/img/02-halo-kinds.png b/docs/reports/2026-11-noise-halo-scores/img/02-halo-kinds.png new file mode 100644 index 00000000..5afce91f Binary files /dev/null and b/docs/reports/2026-11-noise-halo-scores/img/02-halo-kinds.png differ diff --git a/docs/reports/2026-11-noise-halo-scores/img/03-cel-outline.png b/docs/reports/2026-11-noise-halo-scores/img/03-cel-outline.png new file mode 100644 index 00000000..4a0e994c Binary files /dev/null and b/docs/reports/2026-11-noise-halo-scores/img/03-cel-outline.png differ diff --git a/docs/reports/2026-11-noise-halo-scores/img/04-real.png b/docs/reports/2026-11-noise-halo-scores/img/04-real.png new file mode 100644 index 00000000..25cff30c Binary files /dev/null and b/docs/reports/2026-11-noise-halo-scores/img/04-real.png differ diff --git a/docs/reports/2026-11-noise-halo-scores/index.html b/docs/reports/2026-11-noise-halo-scores/index.html new file mode 100644 index 00000000..f9c266e5 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/index.html @@ -0,0 +1,678 @@ + + + + + +Noise and halo scores + + + +
+ +

Noise and halo scores for the pass list

+

Probe-before-building report, 2026-11-09, app 1.7.0. Can the app measure how noisy a loaded source is, and how strongly its edges ring, well enough to badge Noise Reduction or Dehalo as Suggested? Both passes were hard-coded neutral because "the app has no opinion". The idea came from a survey of VCG Deinterlacer (MIT, © 2026 Video-Capture-Guide), whose v1 scores noise from ffmpeg signalstats (TOUT / YDIF) and halos from luma overshoot beside strong edges; its code was read, none of it is used.

+ +
+ Outcome: built (vapourbox-worker --detect-noise-halo, worker/src/noise_halo.rs), run in the background when a source loads. +
    +
  • Noise = motion-compensated frame difference: each 16×16 block matched in the next frame (±10 px), σ = std(difference)/√2, the 10th percentile of blocks, the 25th percentile of 24 sampled pairs; per field unless the source is known to be progressive. Suggested at σ ≥ 3.5 (8-bit levels). On 154 fixtures: the noisiest clean one measured 2.06; 0 clean fixtures crossed the threshold; 19 of 54 "clearly noisy" fixtures were missed — every one of them σ 3 (white or grain-like, reading 2.0–2.9) except tape-like σ 4 on the flat cartoon (3.34). From σ 4 white and σ 6 tape-like or grain-like up, every fixture was caught.
  • +
  • Halo = per-edge two-sided overshoot beside isolated step edges followed over 7 lines, % of the step, averaged over every edge. Suggested at ≥ 6. The worst clean fixture measured 1.48 (the highest real footage, a 1080p camera file labelled grey, 3.30); 0 clean fixtures crossed; 8 of 39 sharpened ones were missed — all sharpening of already-soft edges.
  • +
  • VCG's measures do not separate these fixtures: TOUT does not respond to added noise at all, YDIF follows motion, and its halo ratio reads a dense photo higher than a sharpened one (see Candidates).
  • +
  • Both scores under-report rather than over-report, so a low score means nothing: the badge appears only when a score is clearly high, and nothing is ever marked "not needed" from a measurement. On the repo's real footage (VHS capture, DVB-T, a 1968 film scan, NASA 1080p) neither fires.
  • +
+
+ +

What was measured, and against what

+

154 fixtures, 8 frames each, stored losslessly (FFV1) and measured as 4 pairs of consecutive frames. Degradations were added to luma in float with known parameters, so each fixture carries its ground truth and a class: high (clearly needs the pass), clean, or grey (either answer is acceptable).

+
    +
  • Clean pictures: seven Kodak PhotoCD images panned 2 px a frame; synthetic flat animation (flat fills, 3 px dark outlines, antialiased; also through x264); Lanczos-3 2× upscales; NASA 1080p downlink footage.
  • +
  • Noise: white Gaussian σ 1–8; tape-like (blurred along the line, σ 1.2 px, rescaled) σ 2/4/6; grain-like (blurred both ways 0.7 px, strongest in mid-tones) σ 3/6; σ 4 through x264 CRF 23. High = σ ≥ 3, grey = σ 2.
  • +
  • Halos: unsharp mask radius 1 and 2 at amount 0.5/1/2; horizontal-only "peaking" radius 2 (the tape / camcorder kind). High = amount ≥ 1, grey = 0.5.
  • +
  • Traps: dark scenes (luma at 20%), letterbox bars, interlaced frames whose fields are 4 px apart (combing on every edge), 10-bit, sharpening on top of noise and noise on top of sharpening, and dense texture panned by a fraction of a pixel (1.37 px a frame) so motion compensation can't cancel it.
  • +
  • Real footage (truth by eye, mostly grey): the VHS capture and DVB-T broadcast from Tests/TestResources, the other test clips there, NASA 1080p, and Night of the Living Dead (1968, 35 mm, 1440×1080 10-bit AV1) for real film grain.
  • +
+ +
Four crops of the parrot picture: clean, white noise, tape-like noise, grain-like noise +
Kinds of noise, Kodak 23 at 3× (nearest), left to right: clean; white σ 4; tape-like σ 6; grain-like σ 6. A spatial high-pass sees the second clearly and barely sees the last two — which is why the chosen measure is temporal.
+
Four crops of the house picture: clean and three sharpened versions +
Halos, Kodak 08 at 3×: clean; unsharp radius 1 amount 1; radius 2 amount 1; horizontal peaking radius 2 amount 1. Look at the window frames and roof edge: bright and dark bands on both sides.
+
Synthetic cartoon outline, clean and sharpened +
The animation trap, 3×: a dark outline (left) dips below the fill on one side only and must not read as a halo; sharpened (right), the fills ring on both sides of it.
+
Crops of the VHS capture, the 1968 film and the NASA footage +
Real footage at 3×: the VHS capture (soft, streaky noise, no visible halos); Night of the Living Dead (grain, softened by AV1); NASA 1080p (clean, compressed).
+ +

Result: separation

+

Each dot is a fixture, at its score, in the row of its ground-truth class (hover for its name); the dashed line is the threshold.

+

Noise (σ, 8-bit levels)

+ +clean + +grey + +high + +kodak01_clean: 0.00 +kodak03_clean: 0.00 +kodak05_clean: 0.00 +kodak08_clean: 0.00 +kodak13_clean: 0.00 +kodak19_clean: 0.00 +kodak23_clean: 0.00 +cel_clean: 0.00 +cel_x264: 0.00 +kodak23_lanczos2x: 0.00 +kodak08_lanczos2x: 0.00 +kodak08_n1: 0.98 +kodak08_n2: 1.90 +kodak08_n3: 2.83 +kodak08_n4: 3.77 +kodak08_n6: 5.63 +kodak08_n8: 7.51 +kodak08_tape2: 1.80 +kodak08_tape4: 3.57 +kodak08_tape6: 5.40 +kodak08_grain3: 2.06 +kodak08_grain6: 4.21 +kodak23_n1: 0.98 +kodak23_n2: 1.91 +kodak23_n3: 2.85 +kodak23_n4: 3.79 +kodak23_n6: 5.63 +kodak23_n8: 7.49 +kodak23_tape2: 1.84 +kodak23_tape4: 3.58 +kodak23_tape6: 5.29 +kodak23_grain3: 2.34 +kodak23_grain6: 4.58 +kodak05_n1: 0.98 +kodak05_n2: 1.90 +kodak05_n3: 2.83 +kodak05_n4: 3.77 +kodak05_n6: 5.62 +kodak05_n8: 7.51 +kodak05_tape2: 1.80 +kodak05_tape4: 3.56 +kodak05_tape6: 5.38 +kodak05_grain3: 2.04 +kodak05_grain6: 4.10 +cel_n1: 0.95 +cel_n2: 1.84 +cel_n3: 2.74 +cel_n4: 3.66 +cel_n6: 5.49 +cel_n8: 7.28 +cel_tape2: 1.68 +cel_tape4: 3.34 +cel_tape6: 5.00 +cel_grain3: 2.07 +cel_grain6: 4.09 +nasa_n1: 0.99 +nasa_n2: 1.91 +nasa_n3: 2.84 +nasa_n4: 3.76 +nasa_n6: 5.61 +nasa_n8: 7.46 +nasa_tape2: 1.82 +nasa_tape4: 3.54 +nasa_tape6: 5.26 +nasa_grain3: 1.98 +nasa_grain6: 3.86 +kodak08_n4_x264: 0.00 +kodak23_n4_x264: 0.45 +kodak08_usm1_0.5: 0.00 +kodak08_usm1_1: 0.00 +kodak08_usm1_2: 0.00 +kodak08_usm2_0.5: 0.00 +kodak08_usm2_1: 0.00 +kodak08_usm2_2: 0.00 +kodak08_peak2_0.5: 0.00 +kodak08_peak2_1: 0.00 +kodak08_peak2_2: 0.00 +kodak23_usm1_0.5: 0.00 +kodak23_usm1_1: 0.00 +kodak23_usm1_2: 0.00 +kodak23_usm2_0.5: 0.00 +kodak23_usm2_1: 0.00 +kodak23_usm2_2: 0.00 +kodak23_peak2_0.5: 0.00 +kodak23_peak2_1: 0.00 +kodak23_peak2_2: 0.00 +kodak05_usm1_0.5: 0.00 +kodak05_usm1_1: 0.00 +kodak05_usm1_2: 0.00 +kodak05_usm2_0.5: 0.00 +kodak05_usm2_1: 0.00 +kodak05_usm2_2: 0.00 +kodak05_peak2_0.5: 0.00 +kodak05_peak2_1: 0.00 +kodak05_peak2_2: 0.00 +cel_usm1_0.5: 0.00 +cel_usm1_1: 0.00 +cel_usm1_2: 0.00 +cel_usm2_0.5: 0.00 +cel_usm2_1: 0.00 +cel_usm2_2: 0.00 +cel_peak2_0.5: 0.00 +cel_peak2_1: 0.00 +cel_peak2_2: 0.00 +nasa_usm1_0.5: 0.00 +nasa_usm1_1: 0.00 +nasa_usm1_2: 0.00 +nasa_usm2_0.5: 0.00 +nasa_usm2_1: 0.00 +nasa_usm2_2: 0.00 +nasa_peak2_0.5: 0.00 +nasa_peak2_1: 0.00 +nasa_peak2_2: 0.00 +kodak08_usm1.5_1_n3: 2.67 +kodak08_n3_usm1.5_1: 5.18 +kodak23_usm1.5_1_n3: 2.83 +kodak23_n3_usm1.5_1: 5.47 +kodak13_subpix: 2.06 +kodak13_subpix_tape4: 4.38 +kodak05_subpix: 1.56 +kodak05_subpix_tape4: 4.21 +kodak08_subpix: 1.17 +kodak08_subpix_tape4: 4.17 +kodak08_dark: 0.00 +kodak08_dark_n3: 2.84 +kodak23_dark: 0.00 +kodak23_dark_n3: 2.79 +kodak08_letterbox: 0.00 +kodak08_letterbox_n3: 2.78 +kodak08_letterbox_usm: 0.00 +kodak23_letterbox: 0.00 +kodak23_letterbox_n3: 2.80 +kodak23_letterbox_usm: 0.00 +kodak08_woven: 0.00 +kodak08_woven_n3: 2.83 +kodak08_woven_usm: 0.00 +kodak05_woven: 0.00 +kodak05_woven_n3: 2.83 +kodak05_woven_usm: 0.00 +kodak08_10bit_n0: 0.00 +kodak08_10bit_n2: 1.88 +kodak08_10bit_n4: 3.75 +kodak08_10bit_usm1_1: 0.00 +real_vhs: 1.40 +real_dvbt: 0.00 +real_bars10: 0.00 +real_testsrc_mpeg2: 0.00 +real_small_clip: 0.00 +real_face_test: 0.00 +real_colorize_test: 0.00 +real_nasa_mc: 0.00 +real_nasa_iss: 0.00 +real_notld_road: 1.58 +real_notld_car: 1.56 + +threshold 3.5 +0 +1.33333 +2.66667 +4 +5.33333 +6.66667 +8 +σ, 8-bit levels +

Missed "high" fixtures: kodak08_n3, kodak08_grain3, kodak23_n3, kodak23_grain3, kodak05_n3, kodak05_grain3, cel_n3, cel_tape4, cel_grain3, nasa_n3, nasa_grain3, kodak08_usm1.5_1_n3, kodak23_usm1.5_1_n3, kodak08_dark_n3, kodak23_dark_n3, kodak08_letterbox_n3, kodak23_letterbox_n3, kodak08_woven_n3, kodak05_woven_n3.

+

Halo (% of the step)

+ +clean + +grey + +high + +kodak01_clean: -5.77 +kodak03_clean: -1.07 +kodak05_clean: -2.65 +kodak08_clean: -1.42 +kodak13_clean: -8.53 +kodak19_clean: -7.48 +kodak23_clean: -4.70 +cel_clean: 0.00 +cel_x264: 1.48 +kodak23_lanczos2x: -2.50 +kodak08_lanczos2x: 0.79 +kodak08_n1: -1.61 +kodak08_n2: -2.23 +kodak08_n3: -2.47 +kodak08_n4: -2.54 +kodak08_n6: -4.73 +kodak08_n8: -6.34 +kodak08_tape2: -1.74 +kodak08_tape4: -2.20 +kodak08_tape6: -4.39 +kodak08_grain3: -2.66 +kodak08_grain6: -3.84 +kodak23_n1: -4.59 +kodak23_n2: -5.06 +kodak23_n3: -5.52 +kodak23_n4: -6.56 +kodak23_n6: -6.58 +kodak23_n8: -6.09 +kodak23_tape2: -4.76 +kodak23_tape4: -5.92 +kodak23_tape6: -5.65 +kodak23_grain3: -4.96 +kodak23_grain6: -7.07 +kodak05_n1: -2.65 +kodak05_n2: -2.78 +kodak05_n3: -2.88 +kodak05_n4: -3.87 +kodak05_n6: -3.22 +kodak05_n8: -3.24 +kodak05_tape2: -3.02 +kodak05_tape4: -3.35 +kodak05_tape6: -2.97 +kodak05_grain3: -2.62 +kodak05_grain6: -3.92 +cel_n1: -0.02 +cel_n2: -0.13 +cel_n3: -0.10 +cel_n4: -0.27 +cel_n6: 0.01 +cel_n8: 0.19 +cel_tape2: -0.06 +cel_tape4: 0.02 +cel_tape6: 0.15 +cel_grain3: 0.16 +cel_grain6: -0.11 +nasa_n1: -3.82 +nasa_n2: -4.10 +nasa_n3: -4.57 +nasa_n4: -4.94 +nasa_n6: -5.11 +nasa_n8: -5.44 +nasa_tape2: -3.83 +nasa_tape4: -4.36 +nasa_tape6: -4.41 +nasa_grain3: -4.25 +nasa_grain6: -5.33 +kodak08_n4_x264: -3.49 +kodak23_n4_x264: -4.43 +kodak08_usm1_0.5: 4.79 +kodak08_usm1_1: 15.13 +kodak08_usm1_2: 30.11 +kodak08_usm2_0.5: 12.63 +kodak08_usm2_1: 27.82 +kodak08_usm2_2: 46.60 +kodak08_peak2_0.5: 11.63 +kodak08_peak2_1: 28.32 +kodak08_peak2_2: 39.34 +kodak23_usm1_0.5: -1.74 +kodak23_usm1_1: 1.84 +kodak23_usm1_2: 5.30 +kodak23_usm2_0.5: 1.73 +kodak23_usm2_1: 6.45 +kodak23_usm2_2: 8.53 +kodak23_peak2_0.5: 1.25 +kodak23_peak2_1: 7.21 +kodak23_peak2_2: 9.90 +kodak05_usm1_0.5: 3.80 +kodak05_usm1_1: 11.17 +kodak05_usm1_2: 24.06 +kodak05_usm2_0.5: 7.19 +kodak05_usm2_1: 19.04 +kodak05_usm2_2: 34.87 +kodak05_peak2_0.5: 7.59 +kodak05_peak2_1: 19.72 +kodak05_peak2_2: 41.43 +cel_usm1_0.5: 33.70 +cel_usm1_1: 70.93 +cel_usm1_2: 107.65 +cel_usm2_0.5: 41.50 +cel_usm2_1: 39.38 +cel_usm2_2: 79.34 +cel_peak2_0.5: 43.76 +cel_peak2_1: 39.69 +cel_peak2_2: 79.65 +nasa_usm1_0.5: -2.62 +nasa_usm1_1: -0.82 +nasa_usm1_2: 3.89 +nasa_usm2_0.5: 0.81 +nasa_usm2_1: 6.55 +nasa_usm2_2: 19.29 +nasa_peak2_0.5: 0.35 +nasa_peak2_1: 5.82 +nasa_peak2_2: 17.96 +kodak08_usm1.5_1_n3: 24.69 +kodak08_n3_usm1.5_1: 19.36 +kodak23_usm1.5_1_n3: 4.14 +kodak23_n3_usm1.5_1: 4.98 +kodak13_subpix: -8.45 +kodak13_subpix_tape4: -4.82 +kodak05_subpix: -2.94 +kodak05_subpix_tape4: -3.00 +kodak08_subpix: -1.98 +kodak08_subpix_tape4: -3.19 +kodak08_dark: -6.95 +kodak08_dark_n3: -9.18 +kodak23_dark: -6.40 +kodak23_dark_n3: -7.08 +kodak08_letterbox: -2.72 +kodak08_letterbox_n3: -3.67 +kodak08_letterbox_usm: 18.64 +kodak23_letterbox: -4.70 +kodak23_letterbox_n3: -5.24 +kodak23_letterbox_usm: 5.87 +kodak08_woven: -1.94 +kodak08_woven_n3: -2.15 +kodak08_woven_usm: 26.66 +kodak05_woven: -6.01 +kodak05_woven_n3: -5.32 +kodak05_woven_usm: 9.85 +kodak08_10bit_n0: -1.42 +kodak08_10bit_n2: -2.05 +kodak08_10bit_n4: -2.43 +kodak08_10bit_usm1_1: 15.30 +real_vhs: -4.46 +real_dvbt: -1.83 +real_bars10: -0.00 +real_testsrc_mpeg2: -0.08 +real_small_clip: -4.81 +real_face_test: -3.51 +real_colorize_test: -6.80 +real_nasa_mc: 3.30 +real_nasa_iss: -3.70 +real_notld_road: -4.38 +real_notld_car: -8.23 + +threshold 6 +-10 +0 +10 +20 +30 +40 +50 +% of the step (clipped at 50) +

Missed "high" fixtures: kodak23_usm1_1, kodak23_usm1_2, nasa_usm1_1, nasa_usm1_2, nasa_peak2_1, kodak23_usm1.5_1_n3, kodak23_n3_usm1.5_1, kodak23_letterbox_usm.

+ +
What these numbers can't tell you. Every fixture is 8 frames of one shot; the app samples 24 pairs across the whole source and takes the 25th percentile, which should make a single hard shot matter less — that was not tested. No fixture has non-rigid motion (water, leaves in wind), the likeliest way texture could survive motion compensation and read as noise. The Kodak pans are stills moved across the frame, so their own film grain moves with the picture and reads as zero temporal noise; real grain changes every frame. There is no real footage with known halos: the halo ground truth is synthetic sharpening. All runs are Apple Silicon.
+ +

How the measures were chosen

+

Noise

+
+ + + + + + + + +
Candidates (8-bit levels; TOUT is a fraction): the noisiest clean fixture against white σ 4 and tape-like σ 6 on the five bases
measureclean maxwhite σ 4tape σ 6
Immerkær, whole picture7.910 (kodak05_woven)4.071–6.4231.470–4.723
Immerkær, 10th pct of 16×16 blocks3.588 (kodak13_clean)3.590–4.4201.174–2.410
frame difference, no motion compensation13.799 (kodak05_woven)3.782–8.1855.385–9.243
signalstats TOUT (VCG v1)0.012 (kodak08_10bit_n0)0.000–0.0070.005–0.011
signalstats YDIF (VCG v1)86.985 (kodak08_10bit_n0)4.538–22.8286.789–23.869
motion-compensated, 10th pct of blocks, 25th pct of pairs (chosen)2.061 (kodak13_subpix)3.664–3.7895.002–5.401
+
    +
  • signalstats TOUT counts pixels unlike the lines above and below; it stays at 0.000–0.012 whatever noise is added (VCG's cut-offs are 0.012 / 0.03 / 0.08). YDIF is the mean frame difference: it follows motion (clean pans 5–35) and is not scaled for 10-bit (the 10-bit copy of a fixture reads 4× higher).
  • +
  • Immerkær's estimator (a 3×3 Laplacian-difference high-pass) per 16×16 block, 10th percentile, is accurate for white noise and robust to texture — but blind to noise that is band-limited along the line or clumped: tape-like σ 6 read 1.2–2.4, grain σ 6 1.2–2.5. Dense texture (Kodak 13, foliage and water) read 3.6, above white σ 3. explore.py noise-spatial: a plane-fit standard deviation sees every frequency but read Kodak 05's texture as 10.2; Immerkær at 2× coarser scales caught tape noise and read the same texture as 7.4. Spatially, correlated noise and texture are the same thing; only time separates them.
  • +
  • Motion-compensated temporal (explore.py noise-mc): texture that moved cancels, noise at every frequency stays. The match is chosen on 3×3-smoothed pictures so the choice barely depends on the noise measured; white noise reads about 6% low (σ 4 → 3.75). At ±6 px the interlaced fixtures (fields 8 px apart between frames) read 5.5 and 8.3 clean; ±10 px (coarse search at half resolution, ±1 px refine) reads them 0.00.
  • +
+
Corrected during the work. The first choice was the spatial Immerkær score at σ ≥ 4.5; it was dropped when the tape and grain fixtures showed it misses exactly the noise analogue sources have. The threshold for the temporal score was first 4.0 and lowered to 3.5 once the sub-pixel texture fixtures (worst clean 2.06) showed the margin allowed it. The VHS capture was first labelled "clearly noisy" by assumption; looking at it (figure 4) it is lightly noisy, and it is labelled grey.
+ +

Halo

+
+ + + + + +
Candidates: the highest clean fixture against unsharp radius 1 amount 1 on each base
measureclean maxkodak08_usm1_1kodak23_usm1_1kodak05_usm1_1cel_usm1_1nasa_usm1_1
share of strong edges with a two-sided overshoot (VCG v1 idea)0.1980.0660.0230.1210.0170.105
mean edge profile, no line coherence4.0064.640-3.27515.67855.6171.833
per-edge two-sided overshoot, 7 lines (chosen)1.48015.1271.83711.17170.929-0.820
+
    +
  • VCG's ratio (share of strong edges with a two-sided overshoot ≥ 14 levels 4–7 px out, against plateaus 11–14 px out) read the clean, dense Kodak 13 at 0.149 and its sharpened Kodak 08 at 0.066: its zones sit past where radius-1 sharpening rings, and its fixed level threshold counts texture.
  • +
  • A mean edge profile over every isolated edge (no line coherence) mixes edges with thin features beside them; clean Kodak 08 already has a 10% bump on the dark side.
  • +
  • Per edge, followed over 7 lines: an edge must continue within a pixel on the 3 lines above and below in the same field (texture rarely does), and the averaged profile cuts noise by √7. Overshoot = max(near zone) − max(far plateau), with equal sample counts so noise lifts both alike; each edge scores min(overshoot, undershoot), so a cartoon outline (one-sided) scores 0. Over 1 line each way instead of 3, σ 8 noise biased clean Kodak 08 to −9.8 and hid noise-then-sharpen (4.5); over 3, −6.7 and 11.3 (explore.py halo-reach). Averaging near-zone means instead of maxima (halo-zone) failed outright: soft edges' ramps dominate.
  • +
  • Plateau flatness is ≤ max(2 levels, 2σ/√7, 3% of the step): without the relative term a strongly sharpened picture's plateaus (sharpened texture) rejected most edges — Kodak 08 USM 2/2 kept 72 — and soft pictures' sharpening read low; with it, 256 edges and Kodak 23 USM 2/2 rose from 4.0 to 8.5, while the worst clean rose 3.19 → 3.30 (halo-flat).
  • +
+
Corrected during the work. Halos were first measured on only the first frame of each pair, which left the strongest-sharpened fixtures under the 200-edge minimum (no score at all); both frames are measured now. Unsharp radius 1 amount 0.5 was first labelled clean; it is mild rather than absent (4.8 on Kodak 08, under the threshold; radius 2 at the same amount reads 12.6), so both amount-0.5 rows are grey.
+ +

The worker against the reference

+

worker_parity.py runs --detect-noise-halo on every fixture and measures the frame pairs the worker reports with the NumPy reference: largest difference 0.0016 levels in noise and 0.49 in halo (the median is under 1e-6: an edge sitting exactly on the flatness limit can fall either way between NumPy's float32 and the worker's f64, so a few fixtures count a handful of edges differently — Kodak 08 peaking 2198 against 2184); verdicts that differ: 0. On whole real clips:

+
+ + + + + + +
clipnoisehaloedgespairssuggests NR / Dehalo
vhs_full1.42-5.01639724no / no
dvbt_full0.00-3.183126223no / no
small_clip_full0.00-7.05617024no / no
nasa_partial0.00-1.7328294924no / no
notld_partial1.56-4.244571023no / no
+
+ + + + + + + +
Time per source, release worker, M1 Pro (median of 3)
sourcetimepairs sampled
SD VHS (Lagarith 4:2:2, 79 frames)0.9 s24
SD DVB-T (H.264 4:2:0)0.5 s24
1080p NASA (AV1 8-bit, first 60 s)2.7 s24
1440x1080 NOTLD (AV1 10-bit, first 60 s)3.4 s24
2160p (H.264, 20 s)2.9 s12
+ +

Every fixture

+

Green: the verdict agrees with the ground truth (or the truth is grey); red: it doesn't. ● = above the threshold.

+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
fixturegroupσ addedsharpeningnoise truthnoisehalo truthhaloedges
kodak01_cleanclean photo0clean0.00clean-5.77537
kodak03_cleanclean photo0clean0.00clean-1.077961
kodak05_cleanclean photo0clean0.00clean-2.652623
kodak08_cleanclean photo0clean0.00clean-1.424623
kodak13_cleanclean photo0clean0.00clean-8.53401
kodak19_cleanclean photo0clean0.00clean-7.481113
kodak23_cleanclean photo0clean0.00clean-4.702921
cel_cleanflat animation0clean0.00clean0.004784
cel_x264flat animation0clean0.00clean1.484774
kodak23_lanczos2xupscaled0clean0.00clean-2.506344
kodak08_lanczos2xupscaled0clean0.00clean0.798027
kodak08_n1white noise1clean0.98clean-1.614566
kodak08_n2white noise2grey1.90clean-2.234508
kodak08_n3white noise3high2.83clean-2.474392
kodak08_n4white noise4high3.77 ●clean-2.544875
kodak08_n6white noise6high5.63 ●clean-4.734730
kodak08_n8white noise8high7.51 ●clean-6.345345
kodak08_tape2tape-like noise2grey1.80clean-1.744535
kodak08_tape4tape-like noise4high3.57 ●clean-2.205132
kodak08_tape6tape-like noise6high5.40 ●clean-4.395273
kodak08_grain3synthetic grain3high2.06clean-2.664135
kodak08_grain6synthetic grain6high4.21 ●clean-3.844172
kodak23_n1white noise1clean0.98clean-4.592934
kodak23_n2white noise2grey1.91clean-5.062975
kodak23_n3white noise3high2.85clean-5.522896
kodak23_n4white noise4high3.79 ●clean-6.563179
kodak23_n6white noise6high5.63 ●clean-6.583015
kodak23_n8white noise8high7.49 ●clean-6.092962
kodak23_tape2tape-like noise2grey1.84clean-4.762923
kodak23_tape4tape-like noise4high3.58 ●clean-5.923116
kodak23_tape6tape-like noise6high5.29 ●clean-5.653159
kodak23_grain3synthetic grain3high2.34clean-4.962814
kodak23_grain6synthetic grain6high4.58 ●clean-7.072873
kodak05_n1white noise1clean0.98clean-2.652542
kodak05_n2white noise2grey1.90clean-2.782431
kodak05_n3white noise3high2.83clean-2.882435
kodak05_n4white noise4high3.77 ●clean-3.872950
kodak05_n6white noise6high5.62 ●clean-3.223573
kodak05_n8white noise8high7.51 ●clean-3.242901
kodak05_tape2tape-like noise2grey1.80clean-3.022504
kodak05_tape4tape-like noise4high3.56 ●clean-3.353148
kodak05_tape6tape-like noise6high5.38 ●clean-2.973873
kodak05_grain3synthetic grain3high2.04clean-2.622327
kodak05_grain6synthetic grain6high4.10 ●clean-3.922803
cel_n1white noise1clean0.95clean-0.024795
cel_n2white noise2grey1.84clean-0.134791
cel_n3white noise3high2.74clean-0.104771
cel_n4white noise4high3.66 ●clean-0.274379
cel_n6white noise6high5.49 ●clean0.013089
cel_n8white noise8high7.28 ●clean0.192552
cel_tape2tape-like noise2grey1.68clean-0.064798
cel_tape4tape-like noise4high3.34clean0.024737
cel_tape6tape-like noise6high5.00 ●clean0.153059
cel_grain3synthetic grain3high2.07clean0.164661
cel_grain6synthetic grain6high4.09 ●clean-0.112610
nasa_n1white noise1clean0.99clean-3.8259295
nasa_n2white noise2grey1.91clean-4.1058707
nasa_n3white noise3high2.84clean-4.5759274
nasa_n4white noise4high3.76 ●clean-4.9461282
nasa_n6white noise6high5.61 ●clean-5.1157521
nasa_n8white noise8high7.46 ●clean-5.4451632
nasa_tape2tape-like noise2grey1.82clean-3.8358546
nasa_tape4tape-like noise4high3.54 ●clean-4.3660373
nasa_tape6tape-like noise6high5.26 ●clean-4.4157368
nasa_grain3synthetic grain3high1.98clean-4.2554182
nasa_grain6synthetic grain6high3.86 ●clean-5.3348209
kodak08_n4_x264compressed noise4grey0.00clean-3.494460
kodak23_n4_x264compressed noise4grey0.45clean-4.432751
kodak08_usm1_0.5unsharp mask0USM radius 1 amount 0.5clean0.00grey4.793141
kodak08_usm1_1unsharp mask0USM radius 1 amount 1clean0.00high15.13 ●2153
kodak08_usm1_2unsharp mask0USM radius 1 amount 2clean0.00high30.11 ●888
kodak08_usm2_0.5unsharp mask0USM radius 2 amount 0.5clean0.00grey12.63 ●2322
kodak08_usm2_1unsharp mask0USM radius 2 amount 1clean0.00high27.82 ●1138
kodak08_usm2_2unsharp mask0USM radius 2 amount 2clean0.00high46.60 ●256
kodak08_peak2_0.5line peaking0horizontal-only USM radius 2 amount 0.5 (tape/camcorder peaking)clean0.00grey11.63 ●2509
kodak08_peak2_1line peaking0horizontal-only USM radius 2 amount 1 (tape/camcorder peaking)clean0.00high28.32 ●1248
kodak08_peak2_2line peaking0horizontal-only USM radius 2 amount 2 (tape/camcorder peaking)clean0.00high39.34 ●256
kodak23_usm1_0.5unsharp mask0USM radius 1 amount 0.5clean0.00grey-1.742504
kodak23_usm1_1unsharp mask0USM radius 1 amount 1clean0.00high1.842117
kodak23_usm1_2unsharp mask0USM radius 1 amount 2clean0.00high5.301420
kodak23_usm2_0.5unsharp mask0USM radius 2 amount 0.5clean0.00grey1.732141
kodak23_usm2_1unsharp mask0USM radius 2 amount 1clean0.00high6.45 ●1772
kodak23_usm2_2unsharp mask0USM radius 2 amount 2clean0.00high8.53 ●983
kodak23_peak2_0.5line peaking0horizontal-only USM radius 2 amount 0.5 (tape/camcorder peaking)clean0.00grey1.252215
kodak23_peak2_1line peaking0horizontal-only USM radius 2 amount 1 (tape/camcorder peaking)clean0.00high7.21 ●1706
kodak23_peak2_2line peaking0horizontal-only USM radius 2 amount 2 (tape/camcorder peaking)clean0.00high9.90 ●1078
kodak05_usm1_0.5unsharp mask0USM radius 1 amount 0.5clean0.00grey3.801943
kodak05_usm1_1unsharp mask0USM radius 1 amount 1clean0.00high11.17 ●1470
kodak05_usm1_2unsharp mask0USM radius 1 amount 2clean0.00high24.06 ●779
kodak05_usm2_0.5unsharp mask0USM radius 2 amount 0.5clean0.00grey7.19 ●1576
kodak05_usm2_1unsharp mask0USM radius 2 amount 1clean0.00high19.04 ●996
kodak05_usm2_2unsharp mask0USM radius 2 amount 2clean0.00high34.87 ●382
kodak05_peak2_0.5line peaking0horizontal-only USM radius 2 amount 0.5 (tape/camcorder peaking)clean0.00grey7.59 ●1712
kodak05_peak2_1line peaking0horizontal-only USM radius 2 amount 1 (tape/camcorder peaking)clean0.00high19.72 ●1085
kodak05_peak2_2line peaking0horizontal-only USM radius 2 amount 2 (tape/camcorder peaking)clean0.00high41.43 ●477
cel_usm1_0.5unsharp mask0USM radius 1 amount 0.5clean0.00grey33.70 ●3848
cel_usm1_1unsharp mask0USM radius 1 amount 1clean0.00high70.93 ●2952
cel_usm1_2unsharp mask0USM radius 1 amount 2clean0.00high107.65 ●2552
cel_usm2_0.5unsharp mask0USM radius 2 amount 0.5clean0.00grey41.50 ●1408
cel_usm2_1unsharp mask0USM radius 2 amount 1clean0.00high39.38 ●880
cel_usm2_2unsharp mask0USM radius 2 amount 2clean0.00high79.34 ●872
cel_peak2_0.5line peaking0horizontal-only USM radius 2 amount 0.5 (tape/camcorder peaking)clean0.00grey43.76 ●1488
cel_peak2_1line peaking0horizontal-only USM radius 2 amount 1 (tape/camcorder peaking)clean0.00high39.69 ●880
cel_peak2_2line peaking0horizontal-only USM radius 2 amount 2 (tape/camcorder peaking)clean0.00high79.65 ●872
nasa_usm1_0.5unsharp mask0USM radius 1 amount 0.5clean0.00grey-2.6257676
nasa_usm1_1unsharp mask0USM radius 1 amount 1clean0.00high-0.8253911
nasa_usm1_2unsharp mask0USM radius 1 amount 2clean0.00high3.8947336
nasa_usm2_0.5unsharp mask0USM radius 2 amount 0.5clean0.00grey0.8151849
nasa_usm2_1unsharp mask0USM radius 2 amount 1clean0.00high6.55 ●43993
nasa_usm2_2unsharp mask0USM radius 2 amount 2clean0.00high19.29 ●31941
nasa_peak2_0.5line peaking0horizontal-only USM radius 2 amount 0.5 (tape/camcorder peaking)clean0.00grey0.3553080
nasa_peak2_1line peaking0horizontal-only USM radius 2 amount 1 (tape/camcorder peaking)clean0.00high5.8246210
nasa_peak2_2line peaking0horizontal-only USM radius 2 amount 2 (tape/camcorder peaking)clean0.00high17.96 ●35707
kodak08_usm1.5_1_n3sharpened + noise3USM 1.5 / 1.0, then sigma 3high2.67high24.69 ●1307
kodak08_n3_usm1.5_1sharpened + noise3sigma 3, then USM 1.5 / 1.0 (sharpening the noise too)high5.18 ●high19.36 ●1879
kodak23_usm1.5_1_n3sharpened + noise3USM 1.5 / 1.0, then sigma 3high2.83high4.141731
kodak23_n3_usm1.5_1sharpened + noise3sigma 3, then USM 1.5 / 1.0 (sharpening the noise too)high5.47 ●high4.982399
kodak13_subpixsub-pixel pan0clean2.06clean-8.45516
kodak13_subpix_tape4sub-pixel pan4high4.38 ●clean-4.82698
kodak05_subpixsub-pixel pan0clean1.56clean-2.942675
kodak05_subpix_tape4sub-pixel pan4high4.21 ●clean-3.003743
kodak08_subpixsub-pixel pan0clean1.17clean-1.984718
kodak08_subpix_tape4sub-pixel pan4high4.17 ●clean-3.195518
kodak08_darkdark scene0clean0.00clean-6.952894
kodak08_dark_n3dark scene3high2.84clean-9.182681
kodak23_darkdark scene0clean0.00clean-6.40464
kodak23_dark_n3dark scene3high2.79clean-7.08454
kodak08_letterboxletterbox0clean0.00clean-2.723711
kodak08_letterbox_n3letterbox3high2.78clean-3.673508
kodak08_letterbox_usmletterbox0USM 1.5 / 1.0 inside the barsclean0.00high18.64 ●1136
kodak23_letterboxletterbox0clean0.00clean-4.702897
kodak23_letterbox_n3letterbox3high2.80clean-5.242878
kodak23_letterbox_usmletterbox0USM 1.5 / 1.0 inside the barsclean0.00high5.871760
kodak08_woveninterlaced0clean0.00clean-1.944206
kodak08_woven_n3interlaced3high2.83clean-2.153959
kodak08_woven_usminterlaced0USM 1.5 / 1.0 per fieldclean0.00high26.66 ●1400
kodak05_woveninterlaced0clean0.00clean-6.01838
kodak05_woven_n3interlaced3high2.83clean-5.32818
kodak05_woven_usminterlaced0USM 1.5 / 1.0 per fieldclean0.00high9.85 ●276
kodak08_10bit_n010-bit0clean0.00clean-1.424505
kodak08_10bit_n210-bit2grey1.88clean-2.054430
kodak08_10bit_n410-bit4high3.75 ●clean-2.434853
kodak08_10bit_usm1_110-bit0USM 1 / 1.0clean0.00high15.30 ●2113
real_vhsrealgrey1.40grey-4.46881
real_dvbtrealgrey0.00grey-1.835213
real_bars10realclean0.00clean-0.0014248
real_testsrc_mpeg2realclean0.00clean-0.0815299
real_small_cliprealgrey0.00grey-4.81668
real_face_testrealgrey0.00clean-3.516261
real_colorize_testrealgrey0.00grey-6.805321
real_nasa_mcrealclean0.00grey3.3032733
real_nasa_issrealclean0.00grey-3.7059674
real_notld_roadrealgrey1.58clean-4.387124
real_notld_carrealgrey1.56clean-8.233581
+ +

Decisions

+
    +
  • Suggest only, and only when clearly high. A low score never makes a pass "not applicable": both measures under-report (compressed sources' skipped blocks read as noiseless; sharpening of soft edges sits past the plateau), so a low score is not evidence.
  • +
  • The worker decides the thresholds and reports noiseHigh / haloHigh; the app never re-derives them.
  • +
  • Measured on the whole source, not the trim, once, in the background after the source loads, one source at a time; a failure suggests nothing.
  • +
  • Per field unless known progressive (progressive or soft telecine), so combing can't read as noise.
  • +
  • Luma only, at the source's own depth, decoded in its own format and the Y plane taken: a yuv → gray conversion in swscale expands the range.
  • +
  • Grain counts as noise. Noise Reduction is the pass that removes it; whether to keep it is the user's choice, and the badge only says it was measured.
  • +
+ +

Not tested

+
    +
  • Long real sources with many shots; non-rigid motion; scene cuts inside a sampled pair (each pair is two frames; the 25th percentile should absorb a few).
  • +
  • Real footage with known halos (no such clip in the repo); edge enhancement that rings wider than 6 px.
  • +
  • Chroma noise (not measured). HDR (scores are in code values, not light).
  • +
  • Windows, Linux, x86.
  • +
+ +

Reproducing

+

scripts/run.sh runs everything in order (SCRATCH for downloads and fixtures, Python 3 with NumPy, the repo's deps bundle and worker builds). probelib.py holds the decoding, fixture building, every candidate and the chosen measure (final_*, mirroring noise_halo.rs); explore.py prints the exploration tables quoted above.

+ +
+ + diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/build_fixtures.py b/docs/reports/2026-11-noise-halo-scores/scripts/build_fixtures.py new file mode 100644 index 00000000..c36d245c --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/build_fixtures.py @@ -0,0 +1,292 @@ +"""Build every fixture the probe measures into $SCRATCH/fx, plus fx/manifest.json. + +Sources ($SCRATCH/src, fetched by fetch_sources.sh): + * Kodak PhotoCD images kodim01/03/05/08/13/19/23, panned 2 px a frame. + * NASA JSC "Anne McClain Talks with Fox13 News" (public domain, 1080p). + * "Night of the Living Dead" (1968, public domain; 1440x1080 10-bit AV1). + * The repo's own fixtures in Tests/TestResources. + +Each fixture is 8 frames (real clips: 8 consecutive frames from a chosen +point), stored as lossless FFV1. Degradations are applied to luma in float and +then quantised; chroma is neutral for the built ones. Every fixture carries +its ground truth: the noise sigma added (8-bit levels) and the sharpening +applied, and the class it ought to land in: + + noise: "high" (clearly needs Noise Reduction), "clean", or "grey" (between: + the score may land either side and neither is wrong); + halo: the same for Dehalo. +""" +import json +import math +import os +import subprocess + +import numpy as np + +import probelib as L + +N = 8 +SRC = L.scratch('src') +FX = L.scratch('fx') +os.makedirs(FX, exist_ok=True) +manifest = [] + + +def add(name, frames, *, group, noise_class, halo_class, sigma=0.0, sharpen='', bits=8, + interlaced=None, note=''): + path = os.path.join(FX, name + '.mkv') + if not os.path.exists(path): + L.write_ffv1(L.quantize(frames, bits), path, bits=bits, interlaced=interlaced) + manifest.append(dict(name=name, path=path, group=group, noise_class=noise_class, + halo_class=halo_class, sigma=sigma, sharpen=sharpen, bits=bits, + interlaced=bool(interlaced), note=note)) + + +def add_file(name, src, start, *, group, noise_class, halo_class, interlaced=False, note=''): + """N consecutive frames of a real clip, stored losslessly in its own format.""" + path = os.path.join(FX, name + '.mkv') + if not os.path.exists(path): + w, h, fmt = L.probe(src) + args = [L.FFMPEG, '-v', 'error', '-y'] + if start > 0: + args += ['-ss', str(start)] + args += ['-i', src, '-map', '0:v:0', '-frames:v', str(N), '-c:v', 'ffv1', '-level', '3', + '-pix_fmt', fmt] + if interlaced: + args += ['-field_order', 'tt'] + subprocess.run(args + [path], check=True) + manifest.append(dict(name=name, path=path, group=group, noise_class=noise_class, + halo_class=halo_class, sigma=None, sharpen='', bits=None, + interlaced=interlaced, note=note)) + + +def roundtrip_x264(frames, crf, name): + """Encode with x264 and decode again: noise as a viewer actually gets it.""" + tmp = os.path.join(FX, f'_tmp_{name}.mkv') + L.write_ffv1(L.quantize(frames), tmp) + enc = os.path.join(FX, f'_tmp_{name}.mp4') + subprocess.run([L.FFMPEG, '-v', 'error', '-y', '-i', tmp, '-c:v', 'libx264', '-preset', 'medium', + '-crf', str(crf), '-pix_fmt', 'yuv420p', enc], check=True) + out = L.luma(enc, 0, len(frames)) + os.remove(tmp) + os.remove(enc) + return out + + +def lanczos2x(a): + """2x Lanczos-3 upscale, separable, centre-aligned (as zimg/ffmpeg do).""" + def kern(x): + x = np.abs(x) + return np.where(x < 3, np.sinc(x) * np.sinc(x / 3), 0.0) + + def up(m): + h, w = m.shape + out = np.zeros((h, 2 * w), np.float32) + p = np.pad(m, ((0, 0), (3, 3)), mode='edge') + for o in range(2 * w): + src = (o + 0.5) / 2 - 0.5 + i0 = int(math.floor(src)) + taps = np.arange(i0 - 2, i0 + 4) + wts = kern(src - taps) + wts = wts / wts.sum() + out[:, o] = (p[:, taps + 3] * wts[None, :]).sum(axis=1) + return out + return up(up(a).T).T + + +def cel(n, w=720, h=480, ss=4): + """Flat animation: flat fills with 3 px dark outlines, antialiased by + supersampling, moving 3 px a frame. No noise, no sharpening.""" + frames = [] + H, W = h * ss, w * ss + yy, xx = np.mgrid[0:H, 0:W].astype(np.float32) / ss + for t in range(n): + img = np.full((H, W), 175.0, np.float32) + img[yy > 330] = 140.0 # ground + shapes = [ + ('ellipse', 220 + 3 * t, 230, 120, 90, 110.0), + ('ellipse', 480 - 2 * t, 200, 70, 70, 205.0), + ('rect', 80 + 3 * t, 300, 180, 120, 60.0), + ('ellipse', 560, 380 - t, 60, 40, 225.0), + ] + for kind, cx, cy, rx, ry, level in shapes: + if kind == 'ellipse': + d = np.sqrt(((xx - cx) / rx) ** 2 + ((yy - cy) / ry) ** 2) + inside = d <= 1.0 + outline = (d > 1.0) & (d <= 1.0 + 3.0 / min(rx, ry)) + else: + inside = (np.abs(xx - cx) <= rx / 2) & (np.abs(yy - cy) <= ry / 2) + outer = (np.abs(xx - cx) <= rx / 2 + 3) & (np.abs(yy - cy) <= ry / 2 + 3) + outline = outer & ~inside + img[inside] = level + img[outline] = 30.0 + frames.append(img.reshape(h, ss, w, ss).mean(axis=(1, 3))) + return frames + + +def main(): + kodak = {} + for k in ['01', '03', '05', '08', '13', '19', '23']: + img = L.kodak_luma(os.path.join(SRC, f'kodim{k}.png')) + if img.shape[0] > img.shape[1]: + img = img.T.copy() + kodak[k] = img + + def kpan(k, w=640, h=480, step=2): + return L.pan(kodak[k], w, h, N, step=step, y0=(kodak[k].shape[0] - h) // 2) + + # --- Clean references ------------------------------------------------- + for k in kodak: + add(f'kodak{k}_clean', kpan(k), group='clean photo', noise_class='clean', halo_class='clean', + note='Kodak PhotoCD scan (fine film grain of its own)') + + anim = cel(N) + add('cel_clean', anim, group='flat animation', noise_class='clean', halo_class='clean', + note='synthetic cels: flat fills, 3 px dark outlines') + add('cel_x264', roundtrip_x264(anim, 18, 'cel'), group='flat animation', noise_class='clean', + halo_class='clean', note='the same, through x264 CRF 18 (ringing / mosquito noise)') + add('kodak23_lanczos2x', [lanczos2x(f) for f in kpan('23', 480, 360)], group='upscaled', + noise_class='clean', halo_class='clean', note='640x360 pan, Lanczos-3 2x to 960x720 (ringing)') + add('kodak08_lanczos2x', [lanczos2x(f) for f in kpan('08', 480, 360)], group='upscaled', + noise_class='clean', halo_class='clean', note='Lanczos-3 2x') + + # --- Noise ladders ---------------------------------------------------- + bases = {'kodak08': kpan('08'), 'kodak23': kpan('23'), 'kodak05': kpan('05'), 'cel': anim} + nasa = L.luma(os.path.join(SRC, 'nasa.webm'), 50.0, N) + bases['nasa'] = nasa + for b, fr in bases.items(): + for s in [1, 2, 3, 4, 6, 8]: + cls = 'clean' if s <= 1 else ('grey' if s <= 2 else 'high') + add(f'{b}_n{s}', L.add_noise(fr, s, seed=s), group='white noise', noise_class=cls, + halo_class='clean', sigma=s) + for s in [2, 4, 6]: + cls = 'grey' if s <= 2 else 'high' + add(f'{b}_tape{s}', L.add_noise(fr, s, seed=10 + s, corr_x=1.2), group='tape-like noise', + noise_class=cls, halo_class='clean', sigma=s, + note='noise blurred along the line (sigma 1.2 px), rescaled') + for s in [3, 6]: + add(f'{b}_grain{s}', L.add_noise(fr, s, seed=20 + s, corr_xy=0.7, luma_dependent=True), + group='synthetic grain', noise_class='high', halo_class='clean', sigma=s, + note='noise blurred 0.7 px both ways, strongest in the mid-tones') + for b in ['kodak08', 'kodak23']: + noisy = L.add_noise(bases[b], 4, seed=99) + add(f'{b}_n4_x264', roundtrip_x264(noisy, 23, b), group='compressed noise', noise_class='grey', + halo_class='clean', sigma=4, note='sigma 4 then x264 CRF 23: the encoder removes much of it') + + # --- Sharpening ladders ----------------------------------------------- + hbases = {'kodak08': bases['kodak08'], 'kodak23': bases['kodak23'], 'kodak05': bases['kodak05'], + 'cel': anim, 'nasa': nasa} + for b, fr in hbases.items(): + for r in [1.0, 2.0]: + for a in [0.5, 1.0, 2.0]: + cls = 'grey' if a <= 0.5 else 'high' + add(f'{b}_usm{r:g}_{a:g}', [L.unsharp(f, r, a) for f in fr], group='unsharp mask', + noise_class='clean', halo_class=cls, sharpen=f'USM radius {r:g} amount {a:g}') + for a in [0.5, 1.0, 2.0]: + cls = 'grey' if a <= 0.5 else 'high' + add(f'{b}_peak2_{a:g}', [L.unsharp(f, 2.0, a, horizontal_only=True) for f in fr], + group='line peaking', noise_class='clean', halo_class=cls, + sharpen=f'horizontal-only USM radius 2 amount {a:g} (tape/camcorder peaking)') + for b in ['kodak08', 'kodak23']: + sh = [L.unsharp(f, 1.5, 1.0) for f in bases[b]] + add(f'{b}_usm1.5_1_n3', L.add_noise(sh, 3, seed=7), group='sharpened + noise', noise_class='high', + halo_class='high', sigma=3, sharpen='USM 1.5 / 1.0, then sigma 3') + add(f'{b}_n3_usm1.5_1', [L.unsharp(f, 1.5, 1.0) for f in L.add_noise(bases[b], 3, seed=8)], + group='sharpened + noise', noise_class='high', halo_class='high', sigma=3, + sharpen='sigma 3, then USM 1.5 / 1.0 (sharpening the noise too)') + + # --- Traps ------------------------------------------------------------ + for k in ['13', '05', '08']: + sp = L.subpixel_pan(kodak[k], 640, 480, N, 1.37, y0=(kodak[k].shape[0] - 480) // 2) + add(f'kodak{k}_subpix', sp, group='sub-pixel pan', noise_class='clean', halo_class='clean', + note='panned 1.37 px a frame (cubic), so texture never lines up on whole pixels') + add(f'kodak{k}_subpix_tape4', L.add_noise(sp, 4, seed=70, corr_x=1.2), group='sub-pixel pan', + noise_class='high', halo_class='clean', sigma=4, note='the same with tape-like noise') + for b in ['kodak08', 'kodak23']: + dark =[16 + (f - 16) * 0.2 for f in bases[b]] + add(f'{b}_dark', dark, group='dark scene', noise_class='clean', halo_class='clean', + note='luma scaled to 20% (peaks ~60)') + add(f'{b}_dark_n3', L.add_noise(dark, 3, seed=31), group='dark scene', noise_class='high', + halo_class='clean', sigma=3) + for b in ['kodak08', 'kodak23']: + pic = kpan(b[-2:], 640, 360) + def box(frames): + out = [] + for f in frames: + c = np.full((480, 640), 16.0, np.float32) + c[60:420] = f + out.append(c) + return out + add(f'{b}_letterbox', box(pic), group='letterbox', noise_class='clean', halo_class='clean', + note='640x360 picture in 640x480, digital black bars') + add(f'{b}_letterbox_n3', box(L.add_noise(pic, 3, seed=41)), group='letterbox', + noise_class='high', halo_class='clean', sigma=3, note='noise in the picture, clean bars') + add(f'{b}_letterbox_usm', box([L.unsharp(f, 1.5, 1.0) for f in pic]), group='letterbox', + noise_class='clean', halo_class='high', sharpen='USM 1.5 / 1.0 inside the bars') + for b in ['kodak08', 'kodak05']: + # Interlaced: each frame's two fields are 4 px apart on the pan. + src = kodak[b[-2:]] + y0 = (src.shape[0] - 480) // 2 + def weave(noise=0.0, sharpen=False, seed=0): + rng = np.random.default_rng(seed) + out = [] + for t in range(N): + a = src[y0:y0 + 480, 8 * t:8 * t + 640].copy() + b2 = src[y0:y0 + 480, 8 * t + 4:8 * t + 644].copy() + f = a.copy() + f[1::2] = b2[1::2] + if sharpen: + top, bot = L.unsharp(f[0::2], 1.5, 1.0), L.unsharp(f[1::2], 1.5, 1.0) + f[0::2], f[1::2] = top, bot + if noise: + f = f + rng.standard_normal(f.shape).astype(np.float32) * noise + out.append(f) + return out + add(f'{b}_woven', weave(), group='interlaced', noise_class='clean', halo_class='clean', + interlaced='tff', note='fields 4 px apart: combing on every edge') + add(f'{b}_woven_n3', weave(3.0, seed=51), group='interlaced', noise_class='high', + halo_class='clean', sigma=3, interlaced='tff') + add(f'{b}_woven_usm', weave(sharpen=True), group='interlaced', noise_class='clean', + halo_class='high', interlaced='tff', sharpen='USM 1.5 / 1.0 per field') + for s in [0, 2, 4]: + cls = 'clean' if s == 0 else ('grey' if s <= 2 else 'high') + add(f'kodak08_10bit_n{s}', L.add_noise(bases['kodak08'], s, seed=60 + s) if s else bases['kodak08'], + group='10-bit', noise_class=cls, halo_class='clean', sigma=s, bits=10) + add('kodak08_10bit_usm1_1', [L.unsharp(f, 1.0, 1.0) for f in bases['kodak08']], group='10-bit', + noise_class='clean', halo_class='high', bits=10, sharpen='USM 1 / 1.0') + + # --- Real footage (truth by eye; see the report) ------------------------ + real = [ + ('real_vhs', os.path.join(L.RES, 'interlaced_test.avi'), 0.5, 'grey', 'grey', True, + 'VHS capture (Lagarith 4:2:2, interlaced)'), + ('real_dvbt', os.path.join(L.RES, 'pal-dvbt-fieldcoded-25i.ts'), 1.0, 'grey', 'grey', True, + 'DVB-T broadcast, H.264 interlaced'), + ('real_bars10', os.path.join(L.RES, 'pal-sd-25.mov'), 1.0, 'clean', 'clean', False, + 'SMPTE bars, ProRes 10-bit (synthetic)'), + ('real_testsrc_mpeg2', os.path.join(L.RES, 'hard_telecine_test.avi'), 0.5, 'clean', 'clean', + True, 'ffmpeg testsrc as telecined MPEG-2'), + ('real_small_clip', os.path.join(L.RES, 'small_clip.mp4'), 30.0, 'grey', 'grey', False, + '320x240 H.264'), + ('real_face_test', os.path.join(L.RES, 'face_restore_test.mp4'), 1.0, 'grey', 'clean', False, + 'NASA clip degraded VHS-like: noise=8 temporal, then soft H.264'), + ('real_colorize_test', os.path.join(L.RES, 'colorize_test.mp4'), 1.0, 'grey', 'grey', False, + 'black-and-white H.264 test clip'), + ('real_nasa_mc', os.path.join(SRC, 'nasa.webm'), 10.0, 'clean', 'grey', False, + 'NASA 1080p, mission control (AV1)'), + ('real_nasa_iss', os.path.join(SRC, 'nasa.webm'), 50.0, 'clean', 'grey', False, + 'NASA 1080p, ISS downlink (AV1)'), + ('real_notld_road', os.path.join(SRC, 'notld.webm'), 30.0, 'grey', 'clean', False, + 'Night of the Living Dead, 1968 35 mm, 10-bit AV1: heavy grain'), + ('real_notld_car', os.path.join(SRC, 'notld.webm'), 100.0, 'grey', 'clean', False, + 'Night of the Living Dead, cemetery'), + ] + for name, src, start, nc, hc, il, note in real: + add_file(name, src, start, group='real', noise_class=nc, halo_class=hc, interlaced=il, note=note) + + with open(os.path.join(FX, 'manifest.json'), 'w') as f: + json.dump(manifest, f, indent=1) + print(len(manifest), 'fixtures in', FX) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/explore.py b/docs/reports/2026-11-noise-halo-scores/scripts/explore.py new file mode 100644 index 00000000..b07c074b --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/explore.py @@ -0,0 +1,101 @@ +"""The exploration behind the chosen measures, one table per mode. Prints the +tables the report quotes in "Candidates"; filter fixtures by name fragments. + + python explore.py noise-spatial [names...] Immerkaer p10, plane-fit std, + Immerkaer at coarser scales + python explore.py noise-mc [names...] spatial p25 against the + motion-compensated estimate + (R=radius, default 6; 10 chosen) + python explore.py halo-zone [names...] mean-zone halo (rejected) + python explore.py halo-reach [names...] max-based halo, 1 vs 3 lines + each way (REACH=1,3) + python explore.py halo-flat [names...] relative plateau tolerance + 0 / 0.03 / 0.05 (needs final.json) +""" +import json +import os +import sys + +import numpy as np + +import probelib as L + +FX = L.scratch('fx') + + +def fixtures(pats, skip_sharpened=False): + for e in json.load(open(os.path.join(FX, 'manifest.json'))): + if pats and not any(p in e['name'] for p in pats): + continue + if skip_sharpened and not pats and ('usm' in e['name'] or 'peak' in e['name']): + continue + yield e, L.luma(e['path'], 0, 8) + + +def sig_label(e): + return '' if e['sigma'] is None else f"{e['sigma']:g}" + + +def noise_spatial(pats): + for e, fr in fixtures(pats, True): + il = e['interlaced'] + imm = np.nanmedian([L.noise_blocks(f, il) for f in fr]) + pl = {p: np.nanmedian([L.noise_plane(f, il, pct=p) for f in fr]) for p in (5, 10, 25)} + cx = np.nanmedian([L.noise_blocks_coarse(f, il, fx=2, fy=1) for f in fr]) + c22 = np.nanmedian([L.noise_blocks_coarse(f, il, fx=2, fy=2) for f in fr]) + print(f"{e['name']:28s} {e['noise_class']:5s} {sig_label(e):>3s} imm10 {imm:5.2f} " + f"plane5 {pl[5]:5.2f} plane10 {pl[10]:5.2f} plane25 {pl[25]:5.2f} x2 {cx:5.2f} 2x2 {c22:5.2f}", + flush=True) + + +def noise_mc(pats): + radius = int(os.environ.get('R', 6)) + for e, fr in fixtures(pats, True): + il = e['interlaced'] + sp = np.percentile([L.noise_blocks(f, il) for f in fr], 25) + mc = [L.noise_mc(fr[i], fr[i + 1], il, radius=radius) for i in range(0, 8, 2)] + print(f"{e['name']:28s} {e['noise_class']:5s} {sig_label(e):>3s} spatial {sp:5.2f} " + f"mc {np.median(mc):5.2f} (min {min(mc):5.2f} max {max(mc):5.2f})", flush=True) + + +def halo_zone(pats): + for e, fr in fixtures(pats): + il = e['interlaced'] + sig = float(np.nanmedian([L.noise_blocks(f, il) for f in fr])) + p = np.concatenate([L.coherent_edges(f, sig, il, profiles=True) for f in fr]) + s = [] + for near in [(1, 3), (2, 4), (1, 5)]: + r = L.halo_mean_zone(p, near) + s.append(f"{near}: {r['score']:6.2f} ({r['over']:6.1f}/{r['under']:6.1f})") + print(f"{e['name']:28s} {e['halo_class']:5s} n {p.shape[0]:6d} " + ' '.join(s), flush=True) + + +def halo_reach(pats): + reaches = [int(r) for r in os.environ.get('REACH', '1,3').split(',')] + for e, fr in fixtures(pats): + il = e['interlaced'] + sig = float(np.nanmedian([L.noise_blocks(f, il) for f in fr])) + s = [] + for reach in reaches: + r = L.halo_score(np.concatenate([L.coherent_edges(f, sig, il, reach=reach) for f in fr])) + s.append(f"r{reach}: n {r['edges']:5d} mean {r['score']:6.2f} med {r['median']:6.2f}") + print(f"{e['name']:28s} {e['halo_class']:5s} sig {sig:5.2f} " + ' | '.join(s), flush=True) + + +def halo_flat(pats): + fin = json.load(open(os.path.join(FX, 'final.json'))) + for e, fr in fixtures(pats): + il = e['interlaced'] + sig = fin[e['name']]['noise'] or 0.0 + s = [] + for rel in (0.0, 0.03, 0.05): + ou = np.concatenate([L.coherent_edges(f, sig, il, reach=3, flat_rel=rel) for f in fr]) + two = np.minimum(ou[:, 0], ou[:, 1]) if len(ou) else np.zeros(0) + s.append(f"rel {rel}: n {len(two):5d} {two.mean() * 100 if len(two) else float('nan'):6.2f}") + print(f"{e['name']:28s} {e['halo_class']:5s} sig {sig:4.2f} " + ' | '.join(s), flush=True) + + +if __name__ == '__main__': + mode, pats = sys.argv[1], sys.argv[2:] + {'noise-spatial': noise_spatial, 'noise-mc': noise_mc, 'halo-zone': halo_zone, + 'halo-reach': halo_reach, 'halo-flat': halo_flat}[mode](pats) diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/fetch_sources.sh b/docs/reports/2026-11-noise-halo-scores/scripts/fetch_sources.sh new file mode 100644 index 00000000..e456df9d --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/fetch_sources.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Fetch the outside footage the fixtures are made from into $SCRATCH/src. +# Only building the fixtures needs the network; nothing in the report does. +# +# * Kodak PhotoCD images (Kodak released them for unrestricted use), from the +# usual mirror. +# * "NASA Astronaut Anne McClain Talks with Fox13 News, Seattle - March 31, +# 2025" (NASA JSC, public domain as a work of NASA), 1080p, Wikimedia +# Commons: the first 80 MB. +# * "Night of the Living Dead" (1968; public domain), 1440x1080 10-bit AV1, +# Wikimedia Commons: the first 90 MB. +set -e +S="${SCRATCH:?set SCRATCH to your scratch directory}/src" +mkdir -p "$S" +cd "$S" +UA="VapourBox-test-fixtures/1.0 (https://github.com/StuartCameronCode/VapourBox)" +[ -f nasa.webm ] || curl -fsS --retry 5 --retry-all-errors -A "$UA" -r 0-80000000 -o nasa.webm \ + "https://upload.wikimedia.org/wikipedia/commons/f/f0/NASA_Astronaut_Anne_McClain_Talks_with_Fox13_News%2C_Seattle_%E2%80%93_March_31%2C_2025_%28iss072m260901634%29.webm" +[ -f notld.webm ] || curl -fsS --retry 5 --retry-all-errors -A "$UA" -r 0-90000000 -o notld.webm \ + "https://upload.wikimedia.org/wikipedia/commons/2/24/Night_of_the_Living_Dead_%281968%29.webm" +for k in 01 03 05 08 13 19 23; do + [ -f kodim$k.png ] || curl -fsSL -o kodim$k.png "https://r0k.us/graphics/kodak/kodak/kodim$k.png" +done +ls -la diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/fill_report.py b/docs/reports/2026-11-noise-halo-scores/scripts/fill_report.py new file mode 100644 index 00000000..7c61ef14 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/fill_report.py @@ -0,0 +1,176 @@ +"""Write ../index.html from $SCRATCH/fx/{final,candidates,parity,timing}.json. + +The prose is in this file; every number in a table or chart comes from those +JSON files (measure_final.py, measure_candidates.py, worker_parity.py, +timing.py). The exploration tables quoted in "How the measures were chosen" +are printed by explore.py. +""" +import html +import json +import os + +import probelib as L + +HERE = os.path.dirname(os.path.abspath(__file__)) +FX = L.scratch('fx') +final = json.load(open(os.path.join(FX, 'final.json'))) +cand = json.load(open(os.path.join(FX, 'candidates.json'))) +parity = json.load(open(os.path.join(FX, 'parity.json'))) +timing = json.load(open(os.path.join(FX, 'timing.json'))) + + +def fmt(v, d=2): + return '–' if v is None else f'{v:.{d}f}' + + +def sep(key, cls_key, values): + """Clean max, grey range, high min for one score.""" + out = {} + for c in ('clean', 'grey', 'high'): + vs = [(v, n) for n, v in values.items() if final[n][cls_key] == c and v is not None] + out[c] = vs + return out + + +def strip(key, cls_key, thr, lo, hi, unit): + """A strip chart: one row per class, a dot per fixture, the threshold.""" + W, H, pad = 900, 150, 110 + rows = {'clean': 30, 'grey': 70, 'high': 110} + col = {'clean': 'var(--c-clean)', 'grey': 'var(--c-grey)', 'high': 'var(--c-high)'} + + def x(v): + v = min(max(v, lo), hi) + return pad + (W - pad - 20) * (v - lo) / (hi - lo) + parts = [f''] + for c, y in rows.items(): + parts.append(f'{c}') + parts.append(f'') + for n, v in final.items(): + s = v[key] + if s is None: + continue + c = v[cls_key] + parts.append(f'' + f'{html.escape(n)}: {s:.2f}') + tx = x(thr) + parts.append(f'') + parts.append(f'threshold {thr:g}') + step = (hi - lo) / 6 + for i in range(7): + v = lo + i * step + parts.append(f'{v:g}') + parts.append(f'{unit}') + return '\n'.join(parts) + + +def fixture_table(): + rows = [] + for n, v in final.items(): + nv, hv = v['noise'], v['halo'] + nc = 'good' if (v['noise_high'] == (v['noise_class'] == 'high')) or v['noise_class'] == 'grey' else 'bad' + hc = 'good' if (v['halo_high'] == (v['halo_class'] == 'high')) or v['halo_class'] == 'grey' else 'bad' + sigma = '' if v['sigma'] is None else f"{v['sigma']:g}" + rows.append( + f"{n}{html.escape(v['group'])}" + f"{sigma}" + f"{html.escape(v['sharpen'] or '')}" + f"{v['noise_class']}{fmt(nv)}{' ●' if v['noise_high'] else ''}" + f"{v['halo_class']}{fmt(hv)}{' ●' if v['halo_high'] else ''}" + f"{v['halo_edges']}") + return '\n'.join(rows) + + +def candidate_noise_table(): + """Clean max vs the white sigma 4 and tape sigma 6 fixtures, per candidate.""" + keys = [('whole', 'Immerkær, whole picture'), ('blk10', 'Immerkær, 10th pct of 16×16 blocks'), + ('temporal', 'frame difference, no motion compensation'), ('tout', 'signalstats TOUT (VCG v1)'), + ('ydif', 'signalstats YDIF (VCG v1)')] + # Classes from the final manifest (candidates.json was measured before two relabels). + clean = [n for n in cand if final[n]['noise_class'] == 'clean' and 'usm' not in n and 'peak' not in n] + n4 = [n for n in cand if n.endswith('_n4') and not n.startswith('kodak08_10bit')] + t6 = [n for n in cand if n.endswith('_tape6')] + rows = [] + for k, label in keys: + cm = max(clean, key=lambda n: cand[n][k]) + rows.append(f"{label}{cand[cm][k]:.3f} ({cm})" + f"{min(cand[n][k] for n in n4):.3f}–{max(cand[n][k] for n in n4):.3f}" + f"{min(cand[n][k] for n in t6):.3f}–{max(cand[n][k] for n in t6):.3f}") + fclean = [n for n, v in final.items() if v['noise_class'] == 'clean' and v['noise'] is not None] + cm = max(fclean, key=lambda n: final[n]['noise']) + rows.append(f"motion-compensated, 10th pct of blocks, 25th pct of pairs " + f"(chosen){final[cm]['noise']:.3f} ({cm})" + f"{min(final[n]['noise'] for n in n4):.3f}–{max(final[n]['noise'] for n in n4):.3f}" + f"{min(final[n]['noise'] for n in t6):.3f}–{max(final[n]['noise'] for n in t6):.3f}") + return '\n'.join(rows) + + +def candidate_halo_table(): + clean = [n for n in cand if final[n]['halo_class'] == 'clean'] + usm1 = [n for n in cand if n.endswith('_usm1_1') and n != 'kodak08_10bit_usm1_1'] + rows = [] + for k, label in [('vcg', 'share of strong edges with a two-sided overshoot (VCG v1 idea)'), + ('esf', 'mean edge profile, no line coherence')]: + vals = [cand[n][k] for n in clean if cand[n][k] == cand[n][k]] + rows.append(f"{label}{max(vals):.3f}" + + ''.join(f"{cand[n][k]:.3f}" for n in usm1) + '') + vals = [final[n]['halo'] for n in final if final[n]['halo_class'] == 'clean' and final[n]['halo'] is not None] + rows.append("per-edge two-sided overshoot, 7 lines (chosen)" + f"{max(vals):.3f}" + ''.join(f"{fmt(final[n]['halo'], 3)}" for n in usm1) + '') + head = ''.join(f'{n}' for n in usm1) + return head, '\n'.join(rows) + + +def parity_table(): + rows = [] + for n, v in parity.items(): + if n.startswith('_') or 'reference' in v: + continue + r = v['worker'] + rows.append(f"{n}{fmt(r.get('noise'))}{fmt(r.get('halo'))}" + f"{r.get('haloEdges')}{r.get('usablePairs')}" + f"{'yes' if r.get('noiseHigh') else 'no'} / {'yes' if r.get('haloHigh') else 'no'}") + return '\n'.join(rows) + + +def timing_table(): + return '\n'.join(f"{n}{v['seconds']:.1f} s{len(v['report'].get('frames', []))}" + for n, v in timing.items()) + + +ns = {n: v['noise'] for n, v in final.items()} +hs = {n: v['halo'] for n, v in final.items()} +nclean = max(v for n, v in ns.items() if v is not None and final[n]['noise_class'] == 'clean') +hclean = max(v for n, v in hs.items() if v is not None and final[n]['halo_class'] == 'clean') +fp_n = [n for n in final if final[n]['noise_class'] == 'clean' and final[n]['noise_high']] +fp_h = [n for n in final if final[n]['halo_class'] == 'clean' and final[n]['halo_high']] +miss_n = [n for n in final if final[n]['noise_class'] == 'high' and not final[n]['noise_high']] +miss_h = [n for n in final if final[n]['halo_class'] == 'high' and not final[n]['halo_high']] +n_high = sum(1 for n in final if final[n]['noise_class'] == 'high') +h_high = sum(1 for n in final if final[n]['halo_class'] == 'high') +worst = parity['_worst'] +differ = [n for n, v in parity.items() if isinstance(v, dict) and v.get('same_verdict') is False] +hhead, hrows = candidate_halo_table() + +template = open(os.path.join(HERE, 'report_template.html')).read() +out = template +for k, v in { + 'NOISE_THR': f'{L.NOISE_HIGH:g}', 'HALO_THR': f'{L.HALO_HIGH:g}', + 'NOISE_CLEAN_MAX': f'{nclean:.2f}', 'HALO_CLEAN_MAX': f'{hclean:.2f}', + 'NOISE_FP': str(len(fp_n)), 'HALO_FP': str(len(fp_h)), + 'NOISE_MISS': f'{len(miss_n)} of {n_high}', 'HALO_MISS': f'{len(miss_h)} of {h_high}', + 'NOISE_MISSED': ', '.join(f'{n}' for n in miss_n), + 'HALO_MISSED': ', '.join(f'{n}' for n in miss_h), + 'N_FIXTURES': str(len(final)), + 'NOISE_STRIP': strip('noise', 'noise_class', L.NOISE_HIGH, 0, 8, 'σ, 8-bit levels'), + 'HALO_STRIP': strip('halo', 'halo_class', L.HALO_HIGH, -10, 50, '% of the step (clipped at 50)'), + 'FIXTURE_ROWS': fixture_table(), + 'CAND_NOISE_ROWS': candidate_noise_table(), + 'CAND_HALO_HEAD': hhead, 'CAND_HALO_ROWS': hrows, + 'PARITY_NOISE': f"{worst['noise']:.2g}", 'PARITY_HALO': f"{worst['halo']:.2g}", + 'PARITY_DIFFER': str(len(differ)), + 'PARITY_ROWS': parity_table(), 'TIMING_ROWS': timing_table(), +}.items(): + out = out.replace('{{' + k + '}}', v) +assert '{{' not in out, out[out.index('{{'):out.index('{{') + 40] +open(os.path.join(HERE, '..', 'index.html'), 'w').write(out) +print('wrote index.html') diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/images.sh b/docs/reports/2026-11-noise-halo-scores/scripts/images.sh new file mode 100644 index 00000000..801a047e --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/images.sh @@ -0,0 +1,48 @@ +#!/bin/bash +# The report's screenshots, from the fixtures in $SCRATCH/fx (build_fixtures.py). +# Each is a row of same-size crops at 3x nearest-neighbour, left to right as +# the report's caption lists them. Labels are in the captions, not the pixels. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +S="${SCRATCH:?set SCRATCH to your scratch directory}" +FF="${FFMPEG:-$HERE/../../../../deps/macos-arm64/ffmpeg/ffmpeg}" +IMG="$HERE/../img" +mkdir -p "$IMG" "$S/crops" + +# crop : frame 2, luma only, 3x. +crop() { + "$FF" -v error -y -i "$S/fx/$1.mkv" \ + -vf "select=eq(n\,2),crop=$4:$5:$2:$3,extractplanes=y,scale=iw*3:ih*3:flags=neighbor" \ + -frames:v 1 "$6" +} +row() { # row + local out=$1; shift + local args=() n=0 + for f in "$@"; do args+=(-i "$f"); n=$((n + 1)); done + "$FF" -v error -y "${args[@]}" -filter_complex "hstack=inputs=$n" "$out" +} + +# 1. Noise: clean, white sigma 4, tape-like sigma 6, synthetic grain sigma 6. +for f in kodak23_clean kodak23_n4 kodak23_tape6 kodak23_grain6; do + crop $f 300 60 110 110 "$S/crops/$f.png" +done +row "$IMG/01-noise-kinds.png" "$S"/crops/kodak23_{clean,n4,tape6,grain6}.png + +# 2. Halos: clean, USM radius 1 amount 1, radius 2 amount 1, line peaking 2/1. +for f in kodak08_clean kodak08_usm1_1 kodak08_usm2_1 kodak08_peak2_1; do + crop $f 250 150 110 110 "$S/crops/$f.png" +done +row "$IMG/02-halo-kinds.png" "$S"/crops/kodak08_{clean,usm1_1,usm2_1,peak2_1}.png + +# 3. Animation: outlines (one-sided) against sharpened outlines (two-sided). +for f in cel_clean cel_usm1_1; do + crop $f 120 140 140 110 "$S/crops/$f.png" +done +row "$IMG/03-cel-outline.png" "$S"/crops/cel_{clean,usm1_1}.png + +# 4. Real footage: VHS capture, Night of the Living Dead grain, NASA 1080p. +crop real_vhs 330 120 150 110 "$S/crops/real_vhs.png" +crop real_notld_car 560 300 150 110 "$S/crops/real_notld_car.png" +crop real_nasa_mc 900 400 150 110 "$S/crops/real_nasa_mc.png" +row "$IMG/04-real.png" "$S"/crops/real_{vhs,notld_car,nasa_mc}.png +ls -la "$IMG" diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/measure_candidates.py b/docs/reports/2026-11-noise-halo-scores/scripts/measure_candidates.py new file mode 100644 index 00000000..9b9af726 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/measure_candidates.py @@ -0,0 +1,98 @@ +"""Measure every candidate metric on every fixture: $SCRATCH/fx/candidates.json. + +Candidates (all on luma, 8-bit units, per field on an interlaced fixture): + +noise + whole Immerkaer's estimator over the whole picture + blk5/10/20/50 the p-th percentile of per-16x16-block Immerkaer estimates, + blocks within 4 levels of black or white left out + temporal the 10th percentile of per-block std(frame difference)/sqrt(2) + tout/ydif ffmpeg signalstats TOUT and YDIF averages (what VCG v1 uses) + +halo + esf the mean normalised edge profile across isolated step edges: + min(overshoot, undershoot) in % of the step; zones 6..10 px out + esf_n/w the same with plateaus 4..8 and 8..12 px out + vcg fraction of strong edges with a two-sided overshoot (the idea of + VCG's analyze_halo_level) + +Each per-frame value is aggregated as the median over the fixture's frames. +""" +import json +import os +import re +import subprocess +import sys + +import numpy as np + +import probelib as L + +FX = L.scratch('fx') + + +def signalstats(path): + r = subprocess.run([L.FFMPEG, '-v', 'error', '-i', path, '-vf', + 'signalstats=stat=tout,metadata=mode=print:file=-', '-f', 'null', '-'], + capture_output=True, text=True) + tout = [float(m) for m in re.findall(r'lavfi\.signalstats\.TOUT=([0-9.eE+-]+)', r.stdout)] + ydif = [float(m) for m in re.findall(r'lavfi\.signalstats\.YDIF=([0-9.eE+-]+)', r.stdout)] + # YDIF is 0 on the first frame (no previous one). + return (float(np.mean(tout)) if tout else float('nan'), + float(np.mean(ydif[1:])) if len(ydif) > 1 else float('nan')) + + +def measure(entry, vcg=True): + frames = L.luma(entry['path'], 0, 8) + il = entry['interlaced'] + out = {} + per = {k: [] for k in ['whole', 'blk5', 'blk10', 'blk20', 'blk50']} + for f in frames: + per['whole'].append(L.noise_whole(f, il)) + for p in [5, 10, 20, 50]: + per[f'blk{p}'].append(L.noise_blocks(f, il, pct=p)) + for k, v in per.items(): + out[k] = float(np.nanmedian(v)) + out['temporal'] = float(np.nanmedian([L.noise_temporal(frames[i], frames[i + 1], il) + for i in range(0, len(frames) - 1, 2)])) + sigma = out['blk10'] + profiles = {'esf': (6, 10), 'esf_n': (4, 8), 'esf_w': (8, 12)} + for key, (d1, d2) in profiles.items(): + ps = [L.edge_profiles(f, noise=sigma, d1=d1, d2=d2) for f in frames] + p = np.concatenate(ps) + r = L.halo_from_profiles(p, d1, d2) + out[key] = r['score'] + out[key + '_over'] = r['over'] + out[key + '_under'] = r['under'] + out[key + '_edges'] = r['edges'] + if key == 'esf': + out['esf_profile'] = p.mean(axis=0).tolist() if p.shape[0] else [] + # Spread over frames: the same statistic per frame. + out['esf_frames'] = [L.halo_from_profiles(q, d1, d2)['score'] for q in ps] + if vcg: + rs = [L.halo_vcg_like(f) for f in frames[::3]] + edges = sum(r['edges'] for r in rs) + out['vcg_edges'] = edges + out['vcg'] = (sum(r['ratio'] * r['edges'] for r in rs if r['edges']) / edges) if edges else float('nan') + out['tout'], out['ydif'] = signalstats(entry['path']) + return out + + +def main(): + manifest = json.load(open(os.path.join(FX, 'manifest.json'))) + only = sys.argv[1:] + path = os.path.join(FX, 'candidates.json') + results = json.load(open(path)) if os.path.exists(path) else {} + for e in manifest: + if only and not any(o in e['name'] for o in only): + continue + r = measure(e) + results[e['name']] = {**e, **r} + print(f"{e['name']:28s} blk10 {r['blk10']:6.2f} temp {r['temporal']:6.2f} whole {r['whole']:6.2f} " + f"tout {r['tout']:.4f} | esf {r['esf']:6.2f} ({r['esf_over']:5.1f}/{r['esf_under']:5.1f}, " + f"{r['esf_edges']}) vcg {r['vcg']:.3f}", flush=True) + json.dump(results, open(path, 'w'), indent=1) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/measure_final.py b/docs/reports/2026-11-noise-halo-scores/scripts/measure_final.py new file mode 100644 index 00000000..17088181 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/measure_final.py @@ -0,0 +1,45 @@ +"""The chosen noise and halo scores on every fixture: $SCRATCH/fx/final.json. + +Same algorithm and thresholds as worker/src/noise_halo.rs (probelib.final_*): +each fixture's 8 frames are measured as 4 pairs of consecutive frames. +Prints each fixture with its ground-truth class and what the score says, then +the separation: the highest score among "clean" fixtures against the lowest +among "high" ones, per family. +""" +import json +import os + +import numpy as np + +import probelib as L + +FX = L.scratch('fx') + + +def main(): + manifest = json.load(open(os.path.join(FX, 'manifest.json'))) + out = {} + for e in manifest: + frames = L.luma(e['path'], 0, 8) + pairs = [(frames[i], frames[i + 1]) for i in range(0, len(frames) - 1, 2)] + r = L.final_source(pairs, e['interlaced']) + out[e['name']] = {**e, **r} + n = ' - ' if r['noise'] is None else f"{r['noise']:5.2f}" + h = ' - ' if r['halo'] is None else f"{r['halo']:6.2f}" + print(f"{e['name']:28s} noise {n} ({e['noise_class']:5s}) {'HIGH' if r['noise_high'] else ' '} " + f"halo {h} ({e['halo_class']:5s}, {r['halo_edges']:6d} edges) {'HIGH' if r['halo_high'] else ''}", + flush=True) + json.dump(out, open(os.path.join(FX, 'final.json'), 'w'), indent=1) + + for key, cls_key, thr in [('noise', 'noise_class', L.NOISE_HIGH), ('halo', 'halo_class', L.HALO_HIGH)]: + clean = [(v[key], n) for n, v in out.items() if v[cls_key] == 'clean' and v[key] is not None] + high = [(v[key], n) for n, v in out.items() if v[cls_key] == 'high' and v[key] is not None] + wrong_pos = [n for s, n in clean if s >= thr] + missed = [n for s, n in high if s < thr] + print(f"\n{key}: threshold {thr}; clean max {max(clean)}; high min {min(high)}") + print(f" false positives (clean at/above threshold): {wrong_pos}") + print(f" missed (high below threshold): {len(missed)} of {len(high)}: {missed}") + + +if __name__ == '__main__': + main() diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/probelib.py b/docs/reports/2026-11-noise-halo-scores/scripts/probelib.py new file mode 100644 index 00000000..1ffc7a2a --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/probelib.py @@ -0,0 +1,642 @@ +"""Shared helpers for the noise / halo score probe. + +Decoding, fixture building and the candidate metrics, all on luma only, in +8-bit units whatever the source's depth. NumPy is the only dependency besides +the bundle's ffmpeg. + +Paths: the repo is found from this file's location; ffmpeg defaults to the +bundle in deps//ffmpeg/ (override with $FFMPEG / $FFPROBE); scratch +files go to $SCRATCH. +""" +import json +import math +import os +import platform +import subprocess + +import numpy as np + +HERE = os.path.dirname(os.path.abspath(__file__)) +REPO = os.path.abspath(os.path.join(HERE, '..', '..', '..', '..')) +RES = os.path.join(REPO, 'Tests', 'TestResources') + + +def _platform_id(): + m = platform.machine().lower() + arch = 'arm64' if m in ('arm64', 'aarch64') else 'x64' + sysname = {'Darwin': 'macos', 'Linux': 'linux', 'Windows': 'windows'}[platform.system()] + return f'{sysname}-{arch}' + + +_FFDIR = os.path.join(REPO, 'deps', _platform_id(), 'ffmpeg') +FFMPEG = os.environ.get('FFMPEG', os.path.join(_FFDIR, 'ffmpeg')) +FFPROBE = os.environ.get('FFPROBE', os.path.join(_FFDIR, 'ffprobe')) + + +def scratch(*parts): + s = os.environ.get('SCRATCH') + if not s: + raise SystemExit('set SCRATCH to a scratch directory') + p = os.path.join(s, *parts) + os.makedirs(os.path.dirname(p) if parts else s, exist_ok=True) + return p + + +# -------------------------------------------------------------------------- +# Decoding + +def probe(path): + out = subprocess.run([FFPROBE, '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=width,height,pix_fmt', '-of', 'json', path], + capture_output=True, text=True, check=True).stdout + s = json.loads(out)['streams'][0] + return s['width'], s['height'], s['pix_fmt'] + + +def luma(path, start=0.0, count=1, pix_fmt=None): + """`count` consecutive frames from `start` seconds, luma plane only, as + float32 in 8-bit units. Decoded in the source's own format and the Y plane + taken, because a yuv -> gray conversion in swscale expands the range.""" + w, h, src_fmt = probe(path) + fmt = pix_fmt or src_fmt + deep = any(fmt.endswith(s) for s in ('9le', '10le', '12le', '14le', '16le')) + depth = int(''.join(c for c in fmt.split('p')[-1] if c.isdigit()) or 8) if deep else 8 + args = [FFMPEG, '-v', 'error'] + if start > 0: + args += ['-ss', f'{start:.6f}'] + args += ['-apply_cropping', 'codec', '-i', path, '-map', '0:v:0', '-frames:v', str(count), + '-f', 'rawvideo', '-pix_fmt', fmt, '-'] + data = subprocess.run(args, capture_output=True, check=True).stdout + n = count + size = len(data) // n + bps = 2 if deep else 1 + frames = [] + for i in range(n): + raw = data[i * size: i * size + w * h * bps] + a = np.frombuffer(raw, np.uint16 if deep else np.uint8).reshape(h, w).astype(np.float32) + if deep: + a /= float(1 << (depth - 8)) + frames.append(a) + return frames + + +# -------------------------------------------------------------------------- +# Fixture building (all in float luma, 8-bit units) + +def kodak_luma(path): + """A Kodak PNG as BT.601 limited-range luma.""" + w, h, _ = probe(path) + data = subprocess.run([FFMPEG, '-v', 'error', '-i', path, '-f', 'rawvideo', '-pix_fmt', 'rgb24', '-'], + capture_output=True, check=True).stdout + rgb = np.frombuffer(data, np.uint8).reshape(h, w, 3).astype(np.float32) + y = 0.299 * rgb[..., 0] + 0.587 * rgb[..., 1] + 0.114 * rgb[..., 2] + return 16.0 + y * 219.0 / 255.0 + + +def pan(img, w, h, n, step=2, x0=0, y0=0): + """`n` frames of a `w`x`h` window panned `step` px a frame across `img`.""" + return [img[y0:y0 + h, x0 + step * i: x0 + step * i + w].copy() for i in range(n)] + + +def gauss_kernel(sigma): + r = max(1, int(math.ceil(3 * sigma))) + x = np.arange(-r, r + 1, dtype=np.float64) + k = np.exp(-x * x / (2 * sigma * sigma)) + return (k / k.sum()).astype(np.float32) + + +def blur_x(a, sigma): + k = gauss_kernel(sigma) + r = len(k) // 2 + p = np.pad(a, ((0, 0), (r, r)), mode='edge') + out = np.zeros_like(a) + for i, kv in enumerate(k): + out += kv * p[:, i:i + a.shape[1]] + return out + + +def blur_y(a, sigma): + return blur_x(a.T, sigma).T + + +def blur(a, sigma): + return blur_y(blur_x(a, sigma), sigma) + + +def unsharp(a, radius, amount, horizontal_only=False): + """Unsharp mask: a + amount * (a - gaussian(a, radius)).""" + b = blur_x(a, radius) if horizontal_only else blur(a, radius) + return a + amount * (a - b) + + +def add_noise(frames, sigma, seed=1, corr_x=0.0, corr_xy=0.0, luma_dependent=False): + """Independent noise per frame with standard deviation `sigma` (8-bit + levels). `corr_x` blurs it horizontally first (tape noise is band-limited + along the line); `corr_xy` blurs it in both directions (film grain clumps); + the result is rescaled to `sigma` either way. `luma_dependent` scales it + by a mid-tone weighting, as film grain is strongest in the mid-tones.""" + rng = np.random.default_rng(seed) + out = [] + for f in frames: + n = rng.standard_normal(f.shape).astype(np.float32) + if corr_x > 0: + n = blur_x(n, corr_x) + if corr_xy > 0: + n = blur(n, corr_xy) + n *= sigma / max(1e-6, float(n.std())) + if luma_dependent: + t = np.clip((f - 16.0) / 219.0, 0, 1) + n *= 0.4 + 2.4 * t * (1 - t) + out.append(f + n) + return out + + +def quantize(frames, bits=8, lo=0.0, hi=255.0): + q = float(1 << (bits - 8)) + return [np.clip(np.round(f * q) / q, lo, hi + (q - 1) / q) for f in frames] + + +def write_ffv1(frames, path, bits=8, fps=25, interlaced=None): + """Lossless FFV1 (4:2:0, neutral chroma). `interlaced` = 'tff' tags it.""" + h, w = frames[0].shape + q = 1 << (bits - 8) + dt = np.uint16 if bits > 8 else np.uint8 + neutral = 128 * q + pix = 'yuv420p' if bits == 8 else f'yuv420p{bits}le' + args = [FFMPEG, '-v', 'error', '-y', '-f', 'rawvideo', '-pix_fmt', pix, '-s', f'{w}x{h}', + '-r', str(fps), '-i', '-'] + if interlaced: + args += ['-vf', f'setfield={interlaced}', '-field_order', 'tt' if interlaced == 'tff' else 'bb'] + args += ['-c:v', 'ffv1', '-level', '3', '-pix_fmt', pix, path] + p = subprocess.Popen(args, stdin=subprocess.PIPE) + c = np.full((h // 2) * (w // 2), neutral, dt).tobytes() + for f in frames: + y = np.clip(np.round(f * q), 0, 255 * q + q - 1).astype(dt) + p.stdin.write(y.tobytes() + c + c) + p.stdin.close() + if p.wait() != 0: + raise RuntimeError(f'encoding {path}') + + +# -------------------------------------------------------------------------- +# Candidate noise metrics + +_SQ = math.sqrt(math.pi / 2) / 6.0 + + +def immerkaer_response(a): + """|a * M| for Immerkaer's mask M = [1 -2 1; -2 4 -2; 1 -2 1] (interior).""" + r = (a[:-2, :-2] - 2 * a[:-2, 1:-1] + a[:-2, 2:] + - 2 * a[1:-1, :-2] + 4 * a[1:-1, 1:-1] - 2 * a[1:-1, 2:] + + a[2:, :-2] - 2 * a[2:, 1:-1] + a[2:, 2:]) + return np.abs(r) + + +def fields(a, interlaced): + return [a[0::2], a[1::2]] if interlaced else [a] + + +def noise_whole(a, interlaced=False): + """Immerkaer (1996) over the whole picture.""" + return float(np.mean([_SQ * immerkaer_response(f).mean() for f in fields(a, interlaced)])) + + +def block_sigmas(a, block=16, black=16.0, white=235.0, margin=4.0): + """Immerkaer per block, with each block's mean. Blocks whose mean is within + `margin` of black or white are left out (clipping hides noise there).""" + r = immerkaer_response(a) + m = a[1:-1, 1:-1] + H = (r.shape[0] // block) * block + W = (r.shape[1] // block) * block + rb = r[:H, :W].reshape(H // block, block, W // block, block).mean(axis=(1, 3)) + mb = m[:H, :W].reshape(H // block, block, W // block, block).mean(axis=(1, 3)) + keep = (mb > black + margin) & (mb < white - margin) + return (_SQ * rb[keep]).ravel(), keep.size + + +def noise_blocks(a, interlaced=False, pct=10, block=16, black=16.0, margin=4.0): + """A low percentile of per-block Immerkaer estimates: the flattest parts of + the picture, where the high-pass is noise rather than detail.""" + vals = [] + for f in fields(a, interlaced): + s, total = block_sigmas(f, block=block, black=black, margin=margin) + if s.size >= max(8, total // 20): + vals.append(float(np.percentile(s, pct))) + return float(np.mean(vals)) if vals else float('nan') + + +def downsample(a, fx=2, fy=1): + """Box-average `fx` x `fy` pixels into one. White noise drops by + sqrt(fx*fy); noise correlated over that span much less.""" + h = (a.shape[0] // fy) * fy + w = (a.shape[1] // fx) * fx + return a[:h, :w].reshape(h // fy, fy, w // fx, fx).mean(axis=(1, 3)) + + +def noise_blocks_coarse(a, interlaced=False, pct=10, fx=2, fy=1, block=16): + """noise_blocks at a coarser scale, rescaled so white noise reads the same + as at full scale.""" + vals = [] + for f in fields(a, interlaced): + d = downsample(f, fx, fy) + s, total = block_sigmas(d, block=block) + if s.size >= max(8, total // 20): + vals.append(float(np.percentile(s, pct)) * math.sqrt(fx * fy)) + return float(np.mean(vals)) if vals else float('nan') + + +def plane_sigmas(a, block=16, black=16.0, white=235.0, margin=4.0): + """Per block: the standard deviation left after a least-squares plane + (mean + horizontal + vertical slope) is taken out, with each block's mean. + Unlike a high-pass, this sees noise at every spatial frequency, so noise + that is band-limited along the line (tape) or clumped (grain) counts in + full.""" + H = (a.shape[0] // block) * block + W = (a.shape[1] // block) * block + b = a[:H, :W].reshape(H // block, block, W // block, block).transpose(0, 2, 1, 3) + c = (np.arange(block) - (block - 1) / 2).astype(np.float32) + yy, xx = np.meshgrid(c, c, indexing='ij') + m = b.mean(axis=(2, 3)) + sx = (b * xx).sum(axis=(2, 3)) / (xx * xx).sum() + sy = (b * yy).sum(axis=(2, 3)) / (yy * yy).sum() + r = b - m[..., None, None] - sx[..., None, None] * xx - sy[..., None, None] * yy + sd = np.sqrt((r * r).sum(axis=(2, 3)) / (block * block - 3)) + keep = (m > black + margin) & (m < white - margin) + return sd[keep].ravel(), keep.size + + +def noise_plane(a, interlaced=False, pct=10, block=16, black=16.0, margin=4.0): + vals = [] + for f in fields(a, interlaced): + s, total = plane_sigmas(f, block=block, black=black, margin=margin) + if s.size >= max(8, total // 20): + vals.append(float(np.percentile(s, pct))) + return float(np.mean(vals)) if vals else float('nan') + + +def box3(a): + p = np.pad(a, 1, mode='edge') + return sum(p[i:i + a.shape[0], j:j + a.shape[1]] for i in range(3) for j in range(3)) / 9.0 + + +def noise_mc(a, b, interlaced=False, pct=10, block=16, radius=6, black=16.0, white=235.0, margin=4.0): + """Motion-compensated temporal estimate. Each block of `a` is matched in + `b` within +-radius px, the match chosen on 3x3-smoothed pictures (so the + choice barely depends on the noise being measured); then + std(a - matched b) / sqrt 2 on the unsmoothed ones. Sees noise at every + spatial frequency, and texture that moved cancels. A low percentile over + blocks, as for the spatial estimate.""" + vals = [] + for fa, fb in zip(fields(a, interlaced), fields(b, interlaced)): + h, w = fa.shape + by = (h - 2 * radius) // block + bx = (w - 2 * radius) // block + if by < 2 or bx < 2: + continue + y0 = x0 = radius + H, W = by * block, bx * block + sa, sb = box3(fa), box3(fb) + ca = sa[y0:y0 + H, x0:x0 + W] + best = np.full((by, bx), np.inf, np.float32) + best_d = np.zeros((by, bx), np.float32) + ra = fa[y0:y0 + H, x0:x0 + W] + for dy in range(-radius, radius + 1): + for dx in range(-radius, radius + 1): + cb = sb[y0 + dy:y0 + dy + H, x0 + dx:x0 + dx + W] + sad = np.abs(ca - cb).reshape(by, block, bx, block).mean(axis=(1, 3)) + d = (ra - fb[y0 + dy:y0 + dy + H, x0 + dx:x0 + dx + W]).reshape(by, block, bx, block) + sd = d.std(axis=(1, 3)) / math.sqrt(2) + better = sad < best + best = np.where(better, sad, best) + best_d = np.where(better, sd, best_d) + mb = ra.reshape(by, block, bx, block).mean(axis=(1, 3)) + keep = (mb > black + margin) & (mb < white - margin) + if keep.sum() >= max(8, keep.size // 20): + vals.append(float(np.percentile(best_d[keep], pct))) + return float(np.mean(vals)) if vals else float('nan') + + +def subpixel_pan(img, w, h, n, step, y0=0): + """A pan by a non-integer `step` px a frame (cubic interpolation along x), + so motion compensation at whole pixels cannot cancel texture exactly.""" + out = [] + for i in range(n): + x = step * i + xi = int(math.floor(x)) + t = x - xi + # Catmull-Rom weights + wts = [(-t + 2 * t * t - t ** 3) / 2, (2 - 5 * t * t + 3 * t ** 3) / 2, + (t + 4 * t * t - 3 * t ** 3) / 2, (-t * t + t ** 3) / 2] + row = img[y0:y0 + h] + f = sum(wv * row[:, xi + k:xi + k + w] for k, wv in zip(range(4), wts)) + out.append(f.astype(np.float32)) + return out + + +def noise_temporal(a, b, interlaced=False, pct=10, block=16, black=16.0, margin=4.0): + """Frame difference over the blocks that changed least: std(b - a) / sqrt(2).""" + vals = [] + for fa, fb in zip(fields(a, interlaced), fields(b, interlaced)): + d = fb - fa + H = (d.shape[0] // block) * block + W = (d.shape[1] // block) * block + db = d[:H, :W].reshape(H // block, block, W // block, block) + mb = fa[:H, :W].reshape(H // block, block, W // block, block).mean(axis=(1, 3)) + sd = db.std(axis=(1, 3)) / math.sqrt(2) + keep = (mb > black + margin) & (mb < 235 - margin) + if keep.sum() >= 8: + vals.append(float(np.percentile(sd[keep], pct))) + return float(np.mean(vals)) if vals else float('nan') + + +# -------------------------------------------------------------------------- +# Candidate halo metrics + +def edge_profiles(a, noise=0.0, t_edge=None, d1=6, d2=10, min_step=None, row_step=1): + """Every isolated horizontal step edge in `a`: rows scanned along the line + (field-safe), an edge where |a[x+1] - a[x-1]| is a local maximum and large, + with flat plateaus d1..d2 px out on both sides that differ by enough. + Returns the profiles oriented bright-side-positive and normalised so the + dark plateau is 0 and the bright one 1, shape (n, 2*d2+1).""" + t_edge = t_edge if t_edge is not None else max(20.0, 6.0 * noise) + min_step = min_step if min_step is not None else max(30.0, 8.0 * noise) + flat = max(2.5, 1.6 * noise) + rows = a[::row_step] + h, w = rows.shape + g = np.zeros_like(rows) + g[:, 1:-1] = rows[:, 2:] - rows[:, :-2] + ag = np.abs(g) + peak = np.zeros_like(rows, dtype=bool) + peak[:, 1:-1] = (ag[:, 1:-1] >= ag[:, :-2]) & (ag[:, 1:-1] > ag[:, 2:]) & (ag[:, 1:-1] >= t_edge) + peak[:, :d2 + 1] = False + peak[:, w - d2 - 1:] = False + ys, xs = np.nonzero(peak) + if ys.size == 0: + return np.zeros((0, 2 * d2 + 1), np.float32) + offs = np.arange(-d2, d2 + 1) + prof = rows[ys[:, None], xs[:, None] + offs[None, :]] + sign = np.sign(g[ys, xs]) + prof = np.where(sign[:, None] > 0, prof, prof[:, ::-1]) + far_d = prof[:, :d2 - d1 + 1] + far_b = prof[:, d2 + d1:] + fd, fb = far_d.mean(axis=1), far_b.mean(axis=1) + ok = (far_d.std(axis=1) <= flat) & (far_b.std(axis=1) <= flat) & (fb - fd >= min_step) + prof, fd, fb = prof[ok], fd[ok], fb[ok] + return ((prof - fd[:, None]) / (fb - fd)[:, None]).astype(np.float32) + + +def halo_from_profiles(p, d1=6, d2=10): + """Overshoot past the bright plateau and undershoot below the dark one, in + the mean profile, as % of the step. A halo is two-sided; an outline (a dark + line beside a fill) is one-sided.""" + if p.shape[0] == 0: + return dict(edges=0, over=float('nan'), under=float('nan'), score=float('nan')) + m = p.mean(axis=0) + c = d2 + over = float(m[c + 1:c + d1].max() - 1.0) * 100 + under = float(-m[c - d1 + 1:c].min()) * 100 + return dict(edges=int(p.shape[0]), over=over, under=under, score=min(over, under)) + + +def halo_vcg_like(a, edge=40, over_t=14, flat_range=20): + """The idea of VCG Deinterlacer's analyze_halo_level (fraction of strong + edges with a two-sided overshoot in 4..7 px against a plateau 11..14 px + out), re-expressed in NumPy for comparison. Not used by VapourBox. + VCG Deinterlacer: MIT License, Copyright (c) 2026 Video-Capture-Guide + (vcg_deinterlacer_v130.py); its thresholds (40, 14, 20) are kept.""" + h, w = a.shape + total = halo = 0 + + def probe_at(x, y): + grad = a[y, x + 2] - a[y, x - 2] + if abs(grad) < edge: + return None + b, d = (1, -1) if grad > 0 else (-1, 1) + bn = max(a[y, x + b * k] for k in range(4, 8)) + bf = [a[y, x + b * k] for k in range(11, 15)] + dn = min(a[y, x + d * k] for k in range(4, 8)) + df = [a[y, x + d * k] for k in range(11, 15)] + if max(bf) - min(bf) > flat_range or max(df) - min(df) > flat_range: + return None + return (bn - sum(bf) / 4 >= over_t) and (sum(df) / 4 - dn >= over_t * 0.7) + + for y in range(h // 8, h * 7 // 8 - 2, 4): + x = 16 + while x < w - 16: + here = probe_at(x, y) + if here is None: + x += 2 + continue + below = None + for dx in (0, -1, 1, -2, 2): + if 16 <= x + dx < w - 16: + below = probe_at(x + dx, y + 2) + if below is not None: + break + if below is not None: + total += 1 + if here and below: + halo += 1 + x += 10 + return dict(edges=total, ratio=(halo / total) if total else float('nan')) + + +def coherent_edges(a, noise=0.0, interlaced=False, d1=6, d2=10, t_edge=None, min_step=None, + profiles=False, flat=None, reach=1, flat_rel=0.0): + """Per-edge halo measurements on isolated, vertically coherent step edges. + + An edge is a local maximum of |a[x+1] - a[x-1]| along a line, at least + `t_edge`, that continues on the lines above and below in the same field + (within a pixel, same sign): texture rarely does, a picture edge does. The + three lines' profiles, each centred on its own edge, are averaged (noise + down by sqrt 3), oriented bright side positive, and kept when both + plateaus `d1..d2` px out are flat and differ by `min_step`. + + For each kept edge, in units of its step: + over = max(near bright zone, 1..d1-1) - max(far bright plateau) + under = min(far dark plateau) - min(near dark zone) + Near and far zones have the same number of samples, so noise lifts the max + (and lowers the min) of both alike and cancels in the difference. + Returns an array of shape (n, 2): over, under.""" + t_edge = t_edge if t_edge is not None else max(16.0, 5.0 * noise) + min_step = min_step if min_step is not None else max(24.0, 8.0 * noise) + flat = flat if flat is not None else max(2.0, 2.0 * noise / math.sqrt(2 * reach + 1)) + out = [] + offs = np.arange(-d2, d2 + 1) + for f in fields(a, interlaced): + h, w = f.shape + g = np.zeros_like(f) + g[:, 1:-1] = f[:, 2:] - f[:, :-2] + ag = np.abs(g) + peak = np.zeros_like(f, dtype=bool) + peak[1:-1, 1:-1] = ((ag[1:-1, 1:-1] >= ag[1:-1, :-2]) & (ag[1:-1, 1:-1] > ag[1:-1, 2:]) + & (ag[1:-1, 1:-1] >= t_edge)) + peak[:, :d2 + reach + 2] = False + peak[:, w - d2 - reach - 2:] = False + peak[:reach, :] = False + peak[h - reach:, :] = False + ys, xs = np.nonzero(peak) + if ys.size == 0: + continue + s = np.sign(g[ys, xs]) + ok = np.ones(ys.size, bool) + rows = [(ys, xs)] + for direction in (-1, 1): + px = xs + for k in range(1, reach + 1): + yy = ys + direction * k + cand = np.stack([ag[yy, px + dx] * (np.sign(g[yy, px + dx]) == s) for dx in (-1, 0, 1)]) + best = cand.argmax(axis=0) + ok &= cand.max(axis=0) >= t_edge * 0.5 + px = px + best - 1 + rows.append((yy, px)) + prof = np.mean([f[yy[:, None], xx[:, None] + offs[None, :]] for yy, xx in rows], axis=0) + prof = np.where(s[:, None] > 0, prof, prof[:, ::-1])[ok] + c = d2 + far_d = prof[:, :d2 - d1 + 1] + far_b = prof[:, c + d1:] + fd, fb = far_d.mean(axis=1), far_b.mean(axis=1) + tol = np.maximum(flat, flat_rel * (fb - fd)) + keep = (far_d.std(axis=1) <= tol) & (far_b.std(axis=1) <= tol) & (fb - fd >= min_step) + prof, fd, fb, far_d, far_b = prof[keep], fd[keep], fb[keep], far_d[keep], far_b[keep] + hgt = fb - fd + if profiles: + out.append((prof - fd[:, None]) / hgt[:, None]) + continue + near_b = prof[:, c + 1:c + d1] + near_d = prof[:, c - d1 + 1:c] + over = (near_b.max(axis=1) - far_b.max(axis=1)) / hgt + under = (far_d.min(axis=1) - near_d.min(axis=1)) / hgt + out.append(np.stack([over, under], axis=1)) + if not out: + return np.zeros((0, 2 * d2 + 1 if profiles else 2), np.float32) + return np.concatenate(out) + + +def halo_mean_zone(p, near=(2, 4), d1=6, d2=10): + """From normalised coherent profiles: the mean over edges of (near-zone + mean - plateau) on the bright side, and of (plateau - near-zone mean) on + the dark side, in % of the step; the score is the smaller. Means, so noise + adds no bias; a one-sided feature (an outline) scores 0.""" + if p.shape[0] == 0: + return dict(edges=0, score=float('nan'), over=float('nan'), under=float('nan')) + c = d2 + a, b = near + o = p[:, c + a:c + b + 1].mean(axis=1) - p[:, c + d1:].mean(axis=1) + u = p[:, :d2 - d1 + 1].mean(axis=1) - p[:, c - b:c - a + 1].mean(axis=1) + O, U = float(o.mean() * 100), float(u.mean() * 100) + return dict(edges=int(p.shape[0]), score=min(O, U), over=O, under=U) + + +def halo_score(ou, frac_at=0.08): + """From coherent_edges: the mean two-sided halo (min of over and under, % + of the step) and the share of edges whose two-sided halo exceeds + `frac_at`.""" + if ou.shape[0] == 0: + return dict(edges=0, score=float('nan'), frac=float('nan'), over=float('nan'), + under=float('nan'), median=float('nan')) + two = np.minimum(ou[:, 0], ou[:, 1]) + return dict(edges=int(ou.shape[0]), score=float(two.mean() * 100), frac=float((two > frac_at).mean()), + over=float(ou[:, 0].mean() * 100), under=float(ou[:, 1].mean() * 100), + median=float(np.median(two) * 100)) + + +# -------------------------------------------------------------------------- +# The chosen measurement, exactly as worker/src/noise_halo.rs does it. + +NOISE_HIGH = 3.5 # 8-bit levels +HALO_HIGH = 6.0 # % of the step +HALO_MIN_EDGES = 200 +HALO_FLAT_REL = 0.03 # plateau flatness may also be 3% of the step +NOISE_PAIR_PCT = 25 # percentile over sampled frame pairs +MC_BLOCK = 16 +MC_COARSE = 5 # search radius at half resolution: +-10 px +MC_MARGIN = 2 * MC_COARSE + 1 +MC_MAX_BLOCKS = 4096 + + +def _box3_edge(a): + p = np.pad(a, 1, mode='edge') + return sum(p[i:i + a.shape[0], j:j + a.shape[1]] for i in range(3) for j in range(3)) / 9.0 + + +def _half(a): + h, w = (a.shape[0] // 2) * 2, (a.shape[1] // 2) * 2 + return a[:h, :w].reshape(h // 2, 2, w // 2, 2).mean(axis=(1, 3)) + + +def final_mc_field(f, g, black, white): + """Motion-compensated noise sigma of one field (or progressive frame) `f` + against the same field of the next frame `g`, or None. Each 16x16 block is + matched at half resolution within +-5 (+-10 px), refined +-1 px at full + resolution, the match chosen on 3x3 box-smoothed pictures; its sigma is + std(f - matched g) / sqrt 2 on the raw ones. The 10th percentile over + blocks not within 4 levels of black or white.""" + h, w = f.shape + B, M = MC_BLOCK, MC_MARGIN + by, bx = (h - 2 * M) // B, (w - 2 * M) // B + if by < 2 or bx < 2: + return None + stride = 1 + while (by // stride) * (bx // stride) > MC_MAX_BLOCKS: + stride += 1 + sf, sg = _box3_edge(f), _box3_edge(g) + cf, cg = _box3_edge(_half(f)), _box3_edge(_half(g)) + sig = [] + for i in range(0, by, stride): + for j in range(0, bx, stride): + y, x = M + i * B, M + j * B + # Coarse: block of 8 at (y/2, x/2). + cy, cx = y // 2, x // 2 + ref = cf[cy:cy + B // 2, cx:cx + B // 2] + best, bv = None, (0, 0) + for dy in range(-MC_COARSE, MC_COARSE + 1): + for dx in range(-MC_COARSE, MC_COARSE + 1): + s = np.abs(ref - cg[cy + dy:cy + dy + B // 2, cx + dx:cx + dx + B // 2]).sum() + if best is None or s < best: + best, bv = s, (dy, dx) + ref = sf[y:y + B, x:x + B] + best, fv = None, (0, 0) + for ry in (-1, 0, 1): + for rx in (-1, 0, 1): + dy, dx = 2 * bv[0] + ry, 2 * bv[1] + rx + s = np.abs(ref - sg[y + dy:y + dy + B, x + dx:x + dx + B]).sum() + if best is None or s < best: + best, fv = s, (dy, dx) + raw = f[y:y + B, x:x + B] + m = raw.mean() + if not (black + 4.0 < m < white - 4.0): + continue + d = raw - g[y + fv[0]:y + fv[0] + B, x + fv[1]:x + fv[1] + B] + sig.append(float(d.std()) / math.sqrt(2)) + total = len(range(0, by, stride)) * len(range(0, bx, stride)) + if len(sig) < max(8, total // 20): + return None + return float(np.percentile(sig, 10, method='lower')) + + +def final_pair(a, b, interlaced=False, full_range=False): + """One sampled pair of consecutive frames: (noise sigma or None, list of + per-edge two-sided halos in both frames).""" + black, white = (0.0, 255.0) if full_range else (16.0, 235.0) + sig = [final_mc_field(fa, fb, black, white) for fa, fb in zip(fields(a, interlaced), fields(b, interlaced))] + sigma = None if any(s is None for s in sig) else float(np.mean(sig)) + halos = [] + for f in (a, b): + ou = coherent_edges(f, sigma or 0.0, interlaced, d1=6, d2=10, reach=3, flat_rel=HALO_FLAT_REL) + halos += list(np.minimum(ou[:, 0], ou[:, 1])) + return sigma, halos + + +def final_source(pairs, interlaced=False, full_range=False): + sigmas, halos = [], [] + for a, b in pairs: + s, h = final_pair(a, b, interlaced, full_range) + if s is not None: + sigmas.append(s) + halos += h + noise = float(np.percentile(sigmas, NOISE_PAIR_PCT, method='lower')) if sigmas else None + halo = float(np.mean(halos) * 100) if len(halos) >= HALO_MIN_EDGES else None + return dict(noise=noise, halo=halo, halo_edges=len(halos), usable=len(sigmas), + noise_high=noise is not None and noise >= NOISE_HIGH, + halo_high=halo is not None and halo >= HALO_HIGH) diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/report_template.html b/docs/reports/2026-11-noise-halo-scores/scripts/report_template.html new file mode 100644 index 00000000..3ceaf041 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/report_template.html @@ -0,0 +1,170 @@ + + + + + +Noise and halo scores + + + +
+ +

Noise and halo scores for the pass list

+

Probe-before-building report, 2026-11-09, app 1.7.0. Can the app measure how noisy a loaded source is, and how strongly its edges ring, well enough to badge Noise Reduction or Dehalo as Suggested? Both passes were hard-coded neutral because "the app has no opinion". The idea came from a survey of VCG Deinterlacer (MIT, © 2026 Video-Capture-Guide), whose v1 scores noise from ffmpeg signalstats (TOUT / YDIF) and halos from luma overshoot beside strong edges; its code was read, none of it is used.

+ +
+ Outcome: built (vapourbox-worker --detect-noise-halo, worker/src/noise_halo.rs), run in the background when a source loads. +
    +
  • Noise = motion-compensated frame difference: each 16×16 block matched in the next frame (±10 px), σ = std(difference)/√2, the 10th percentile of blocks, the 25th percentile of 24 sampled pairs; per field unless the source is known to be progressive. Suggested at σ ≥ {{NOISE_THR}} (8-bit levels). On {{N_FIXTURES}} fixtures: the noisiest clean one measured {{NOISE_CLEAN_MAX}}; {{NOISE_FP}} clean fixtures crossed the threshold; {{NOISE_MISS}} "clearly noisy" fixtures were missed — every one of them σ 3 (white or grain-like, reading 2.0–2.9) except tape-like σ 4 on the flat cartoon (3.34). From σ 4 white and σ 6 tape-like or grain-like up, every fixture was caught.
  • +
  • Halo = per-edge two-sided overshoot beside isolated step edges followed over 7 lines, % of the step, averaged over every edge. Suggested at ≥ {{HALO_THR}}. The worst clean fixture measured {{HALO_CLEAN_MAX}} (the highest real footage, a 1080p camera file labelled grey, 3.30); {{HALO_FP}} clean fixtures crossed; {{HALO_MISS}} sharpened ones were missed — all sharpening of already-soft edges.
  • +
  • VCG's measures do not separate these fixtures: TOUT does not respond to added noise at all, YDIF follows motion, and its halo ratio reads a dense photo higher than a sharpened one (see Candidates).
  • +
  • Both scores under-report rather than over-report, so a low score means nothing: the badge appears only when a score is clearly high, and nothing is ever marked "not needed" from a measurement. On the repo's real footage (VHS capture, DVB-T, a 1968 film scan, NASA 1080p) neither fires.
  • +
+
+ +

What was measured, and against what

+

{{N_FIXTURES}} fixtures, 8 frames each, stored losslessly (FFV1) and measured as 4 pairs of consecutive frames. Degradations were added to luma in float with known parameters, so each fixture carries its ground truth and a class: high (clearly needs the pass), clean, or grey (either answer is acceptable).

+
    +
  • Clean pictures: seven Kodak PhotoCD images panned 2 px a frame; synthetic flat animation (flat fills, 3 px dark outlines, antialiased; also through x264); Lanczos-3 2× upscales; NASA 1080p downlink footage.
  • +
  • Noise: white Gaussian σ 1–8; tape-like (blurred along the line, σ 1.2 px, rescaled) σ 2/4/6; grain-like (blurred both ways 0.7 px, strongest in mid-tones) σ 3/6; σ 4 through x264 CRF 23. High = σ ≥ 3, grey = σ 2.
  • +
  • Halos: unsharp mask radius 1 and 2 at amount 0.5/1/2; horizontal-only "peaking" radius 2 (the tape / camcorder kind). High = amount ≥ 1, grey = 0.5.
  • +
  • Traps: dark scenes (luma at 20%), letterbox bars, interlaced frames whose fields are 4 px apart (combing on every edge), 10-bit, sharpening on top of noise and noise on top of sharpening, and dense texture panned by a fraction of a pixel (1.37 px a frame) so motion compensation can't cancel it.
  • +
  • Real footage (truth by eye, mostly grey): the VHS capture and DVB-T broadcast from Tests/TestResources, the other test clips there, NASA 1080p, and Night of the Living Dead (1968, 35 mm, 1440×1080 10-bit AV1) for real film grain.
  • +
+ +
Four crops of the parrot picture: clean, white noise, tape-like noise, grain-like noise +
Kinds of noise, Kodak 23 at 3× (nearest), left to right: clean; white σ 4; tape-like σ 6; grain-like σ 6. A spatial high-pass sees the second clearly and barely sees the last two — which is why the chosen measure is temporal.
+
Four crops of the house picture: clean and three sharpened versions +
Halos, Kodak 08 at 3×: clean; unsharp radius 1 amount 1; radius 2 amount 1; horizontal peaking radius 2 amount 1. Look at the window frames and roof edge: bright and dark bands on both sides.
+
Synthetic cartoon outline, clean and sharpened +
The animation trap, 3×: a dark outline (left) dips below the fill on one side only and must not read as a halo; sharpened (right), the fills ring on both sides of it.
+
Crops of the VHS capture, the 1968 film and the NASA footage +
Real footage at 3×: the VHS capture (soft, streaky noise, no visible halos); Night of the Living Dead (grain, softened by AV1); NASA 1080p (clean, compressed).
+ +

Result: separation

+

Each dot is a fixture, at its score, in the row of its ground-truth class (hover for its name); the dashed line is the threshold.

+

Noise (σ, 8-bit levels)

+{{NOISE_STRIP}} +

Missed "high" fixtures: {{NOISE_MISSED}}.

+

Halo (% of the step)

+{{HALO_STRIP}} +

Missed "high" fixtures: {{HALO_MISSED}}.

+ +
What these numbers can't tell you. Every fixture is 8 frames of one shot; the app samples 24 pairs across the whole source and takes the 25th percentile, which should make a single hard shot matter less — that was not tested. No fixture has non-rigid motion (water, leaves in wind), the likeliest way texture could survive motion compensation and read as noise. The Kodak pans are stills moved across the frame, so their own film grain moves with the picture and reads as zero temporal noise; real grain changes every frame. There is no real footage with known halos: the halo ground truth is synthetic sharpening. All runs are Apple Silicon.
+ +

How the measures were chosen

+

Noise

+
+ + +{{CAND_NOISE_ROWS}} +
Candidates (8-bit levels; TOUT is a fraction): the noisiest clean fixture against white σ 4 and tape-like σ 6 on the five bases
measureclean maxwhite σ 4tape σ 6
+
    +
  • signalstats TOUT counts pixels unlike the lines above and below; it stays at 0.000–0.012 whatever noise is added (VCG's cut-offs are 0.012 / 0.03 / 0.08). YDIF is the mean frame difference: it follows motion (clean pans 5–35) and is not scaled for 10-bit (the 10-bit copy of a fixture reads 4× higher).
  • +
  • Immerkær's estimator (a 3×3 Laplacian-difference high-pass) per 16×16 block, 10th percentile, is accurate for white noise and robust to texture — but blind to noise that is band-limited along the line or clumped: tape-like σ 6 read 1.2–2.4, grain σ 6 1.2–2.5. Dense texture (Kodak 13, foliage and water) read 3.6, above white σ 3. explore.py noise-spatial: a plane-fit standard deviation sees every frequency but read Kodak 05's texture as 10.2; Immerkær at 2× coarser scales caught tape noise and read the same texture as 7.4. Spatially, correlated noise and texture are the same thing; only time separates them.
  • +
  • Motion-compensated temporal (explore.py noise-mc): texture that moved cancels, noise at every frequency stays. The match is chosen on 3×3-smoothed pictures so the choice barely depends on the noise measured; white noise reads about 6% low (σ 4 → 3.75). At ±6 px the interlaced fixtures (fields 8 px apart between frames) read 5.5 and 8.3 clean; ±10 px (coarse search at half resolution, ±1 px refine) reads them 0.00.
  • +
+
Corrected during the work. The first choice was the spatial Immerkær score at σ ≥ 4.5; it was dropped when the tape and grain fixtures showed it misses exactly the noise analogue sources have. The threshold for the temporal score was first 4.0 and lowered to 3.5 once the sub-pixel texture fixtures (worst clean 2.06) showed the margin allowed it. The VHS capture was first labelled "clearly noisy" by assumption; looking at it (figure 4) it is lightly noisy, and it is labelled grey.
+ +

Halo

+
+ +{{CAND_HALO_HEAD}} +{{CAND_HALO_ROWS}} +
Candidates: the highest clean fixture against unsharp radius 1 amount 1 on each base
measureclean max
+
    +
  • VCG's ratio (share of strong edges with a two-sided overshoot ≥ 14 levels 4–7 px out, against plateaus 11–14 px out) read the clean, dense Kodak 13 at 0.149 and its sharpened Kodak 08 at 0.066: its zones sit past where radius-1 sharpening rings, and its fixed level threshold counts texture.
  • +
  • A mean edge profile over every isolated edge (no line coherence) mixes edges with thin features beside them; clean Kodak 08 already has a 10% bump on the dark side.
  • +
  • Per edge, followed over 7 lines: an edge must continue within a pixel on the 3 lines above and below in the same field (texture rarely does), and the averaged profile cuts noise by √7. Overshoot = max(near zone) − max(far plateau), with equal sample counts so noise lifts both alike; each edge scores min(overshoot, undershoot), so a cartoon outline (one-sided) scores 0. Over 1 line each way instead of 3, σ 8 noise biased clean Kodak 08 to −9.8 and hid noise-then-sharpen (4.5); over 3, −6.7 and 11.3 (explore.py halo-reach). Averaging near-zone means instead of maxima (halo-zone) failed outright: soft edges' ramps dominate.
  • +
  • Plateau flatness is ≤ max(2 levels, 2σ/√7, 3% of the step): without the relative term a strongly sharpened picture's plateaus (sharpened texture) rejected most edges — Kodak 08 USM 2/2 kept 72 — and soft pictures' sharpening read low; with it, 256 edges and Kodak 23 USM 2/2 rose from 4.0 to 8.5, while the worst clean rose 3.19 → 3.30 (halo-flat).
  • +
+
Corrected during the work. Halos were first measured on only the first frame of each pair, which left the strongest-sharpened fixtures under the 200-edge minimum (no score at all); both frames are measured now. Unsharp radius 1 amount 0.5 was first labelled clean; it is mild rather than absent (4.8 on Kodak 08, under the threshold; radius 2 at the same amount reads 12.6), so both amount-0.5 rows are grey.
+ +

The worker against the reference

+

worker_parity.py runs --detect-noise-halo on every fixture and measures the frame pairs the worker reports with the NumPy reference: largest difference {{PARITY_NOISE}} levels in noise and {{PARITY_HALO}} in halo (the median is under 1e-6: an edge sitting exactly on the flatness limit can fall either way between NumPy's float32 and the worker's f64, so a few fixtures count a handful of edges differently — Kodak 08 peaking 2198 against 2184); verdicts that differ: {{PARITY_DIFFER}}. On whole real clips:

+
+ +{{PARITY_ROWS}} +
clipnoisehaloedgespairssuggests NR / Dehalo
+
+ + +{{TIMING_ROWS}} +
Time per source, release worker, M1 Pro (median of 3)
sourcetimepairs sampled
+ +

Every fixture

+

Green: the verdict agrees with the ground truth (or the truth is grey); red: it doesn't. ● = above the threshold.

+
+ +{{FIXTURE_ROWS}} +
fixturegroupσ addedsharpeningnoise truthnoisehalo truthhaloedges
+ +

Decisions

+
    +
  • Suggest only, and only when clearly high. A low score never makes a pass "not applicable": both measures under-report (compressed sources' skipped blocks read as noiseless; sharpening of soft edges sits past the plateau), so a low score is not evidence.
  • +
  • The worker decides the thresholds and reports noiseHigh / haloHigh; the app never re-derives them.
  • +
  • Measured on the whole source, not the trim, once, in the background after the source loads, one source at a time; a failure suggests nothing.
  • +
  • Per field unless known progressive (progressive or soft telecine), so combing can't read as noise.
  • +
  • Luma only, at the source's own depth, decoded in its own format and the Y plane taken: a yuv → gray conversion in swscale expands the range.
  • +
  • Grain counts as noise. Noise Reduction is the pass that removes it; whether to keep it is the user's choice, and the badge only says it was measured.
  • +
+ +

Not tested

+
    +
  • Long real sources with many shots; non-rigid motion; scene cuts inside a sampled pair (each pair is two frames; the 25th percentile should absorb a few).
  • +
  • Real footage with known halos (no such clip in the repo); edge enhancement that rings wider than 6 px.
  • +
  • Chroma noise (not measured). HDR (scores are in code values, not light).
  • +
  • Windows, Linux, x86.
  • +
+ +

Reproducing

+

scripts/run.sh runs everything in order (SCRATCH for downloads and fixtures, Python 3 with NumPy, the repo's deps bundle and worker builds). probelib.py holds the decoding, fixture building, every candidate and the chosen measure (final_*, mirroring noise_halo.rs); explore.py prints the exploration tables quoted above.

+ +
+ + diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/run.sh b/docs/reports/2026-11-noise-halo-scores/scripts/run.sh new file mode 100644 index 00000000..d02484d0 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/run.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Reproduce every number in the report, in order. Needs Python 3 with NumPy, +# the repo's deps bundle (deps/), and worker builds: +# cd worker && cargo build && cargo build --release +# SCRATCH holds the downloads (~180 MB) and fixtures (~1.5 GB). +# Takes about an hour on an M1 Pro, most of it the NumPy reference. +set -e +HERE="$(cd "$(dirname "$0")" && pwd)" +: "${SCRATCH:?set SCRATCH to your scratch directory}" +PY="${PYTHON:-python3}" +cd "$HERE" +bash fetch_sources.sh +"$PY" build_fixtures.py +"$PY" measure_candidates.py # candidates.json: the rejected measures +"$PY" measure_final.py # final.json: the chosen measures +"$PY" worker_parity.py # parity.json: the Rust worker, same pairs +WORKER="${WORKER:-$HERE/../../../../worker/target/release/vapourbox-worker}" "$PY" timing.py +bash images.sh +"$PY" fill_report.py +# The exploration tables quoted in the report: +# $PY explore.py noise-spatial; R=10 $PY explore.py noise-mc +# $PY explore.py halo-zone; $PY explore.py halo-reach; $PY explore.py halo-flat diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/timing.py b/docs/reports/2026-11-noise-halo-scores/scripts/timing.py new file mode 100644 index 00000000..5ffc44ce --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/timing.py @@ -0,0 +1,46 @@ +"""How long `--detect-noise-halo` takes, with a release worker, on SD, 1080p, +1440x1080 10-bit AV1 and a 4K H.264 clip made from the NASA footage: +$SCRATCH/fx/timing.json. Each run three times; the median is reported. + +WORKER= (a release build finds deps in the app's +install location, not the repo's). Timings are one Apple M1 Pro; the measure +itself is single-threaded per pair, the pairs run in parallel. +""" +import json +import os +import subprocess + +import numpy as np + +import probelib as L +import worker_parity as P + +FX = L.scratch('fx') + + +def main(): + uhd = L.scratch('fx', '_uhd.mp4') + if not os.path.exists(uhd): + subprocess.run([L.FFMPEG, '-v', 'error', '-y', '-ss', '50', '-t', '20', '-i', L.scratch('src', 'nasa.webm'), + '-vf', 'scale=3840:2160:flags=lanczos', '-c:v', 'libx264', '-preset', 'veryfast', + '-crf', '18', '-an', uhd], check=True) + clips = [ + ('SD VHS (Lagarith 4:2:2, 79 frames)', os.path.join(L.RES, 'interlaced_test.avi'), True, None), + ('SD DVB-T (H.264 4:2:0)', os.path.join(L.RES, 'pal-dvbt-fieldcoded-25i.ts'), True, None), + ('1080p NASA (AV1 8-bit, first 60 s)', L.scratch('src', 'nasa.webm'), False, 60 * 60), + ('1440x1080 NOTLD (AV1 10-bit, first 60 s)', L.scratch('src', 'notld.webm'), False, 60 * 24), + ('2160p (H.264, 20 s)', uhd, False, None), + ] + out = {} + for name, path, il, frames in clips: + runs = [P.run_worker(path, il, frames=frames) for _ in range(3)] + secs = float(np.median([s for _, s in runs])) + rep = runs[0][0] + out[name] = dict(seconds=secs, report=rep) + print(f"{name:42s} {secs:5.1f} s noise {rep.get('noise')} halo {rep.get('halo')} " + f"pairs {rep.get('usablePairs')}/{len(rep.get('frames', []))}", flush=True) + json.dump(out, open(os.path.join(FX, 'timing.json'), 'w'), indent=1) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/2026-11-noise-halo-scores/scripts/worker_parity.py b/docs/reports/2026-11-noise-halo-scores/scripts/worker_parity.py new file mode 100644 index 00000000..c6d015e3 --- /dev/null +++ b/docs/reports/2026-11-noise-halo-scores/scripts/worker_parity.py @@ -0,0 +1,87 @@ +"""Run the real worker (`vapourbox-worker --detect-noise-halo`) on every +fixture and on the full-length real clips, and check it against the NumPy +reference measured on the same frame pairs: $SCRATCH/fx/parity.json. + +Needs a worker build (`cd worker && cargo build`); $WORKER overrides the path. +""" +import json +import os +import subprocess +import tempfile +import time + +import numpy as np + +import probelib as L + +FX = L.scratch('fx') +WORKER = os.environ.get('WORKER', os.path.join(L.REPO, 'worker', 'target', 'debug', 'vapourbox-worker')) + + +def run_worker(path, interlaced, frames=None): + w, h, pix = L.probe(path) + rate = subprocess.run([L.FFPROBE, '-v', 'error', '-select_streams', 'v:0', '-show_entries', + 'stream=r_frame_rate:format=duration', '-of', 'json', path], + capture_output=True, text=True, check=True).stdout + info = json.loads(rate) + num, den = info['streams'][0]['r_frame_rate'].split('/') + fps = float(num) / float(den) + total = frames or int(float(info['format']['duration']) * fps) + job = { + 'id': '00000000-0000-0000-0000-000000000000', 'inputPath': path, 'outputPath': '/dev/null', + 'qtgmcParameters': {}, 'encodingSettings': {}, 'inputWidth': w, 'inputHeight': h, + 'inputPixelFormat': pix, 'inputFrameRate': fps, 'totalFrames': total, + 'detectedFieldOrder': 'tff' if interlaced else 'progressive', + } + with tempfile.NamedTemporaryFile('w', suffix='.json', delete=False) as f: + json.dump(job, f) + cfg = f.name + t = time.monotonic() + r = subprocess.run([WORKER, '--detect-noise-halo', '--config', cfg], capture_output=True, text=True) + elapsed = time.monotonic() - t + os.remove(cfg) + line = [ln for ln in r.stdout.splitlines() if ln.startswith('{')][-1] + return json.loads(line), elapsed + + +def main(): + manifest = json.load(open(os.path.join(FX, 'manifest.json'))) + out = {} + worst = {'noise': 0.0, 'halo': 0.0} + for e in manifest: + rep, secs = run_worker(e['path'], e['interlaced'], frames=8) + frames = L.luma(e['path'], 0, 8) + pairs = [(frames[f], frames[f + 1]) for f in sorted(set(rep['frames'])) if f + 1 < len(frames)] + ref = L.final_source(pairs, e['interlaced']) + dn = None if rep['noise'] is None or ref['noise'] is None else abs(rep['noise'] - ref['noise']) + dh = None if rep['halo'] is None or ref['halo'] is None else abs(rep['halo'] - ref['halo']) + same = (rep['noiseHigh'] == ref['noise_high']) and (rep['haloHigh'] == ref['halo_high']) + for k, d in (('noise', dn), ('halo', dh)): + if d is not None: + worst[k] = max(worst[k], d) + out[e['name']] = dict(worker=rep, reference=ref, seconds=secs, same_verdict=same) + print(f"{e['name']:28s} worker noise {rep['noise']} halo {rep['halo']} | ref noise {ref['noise']} " + f"halo {ref['halo']} | {'same' if same else 'DIFFERENT'} {secs:.1f}s", flush=True) + real = [ + ('vhs_full', os.path.join(L.RES, 'interlaced_test.avi'), True), + ('dvbt_full', os.path.join(L.RES, 'pal-dvbt-fieldcoded-25i.ts'), True), + ('small_clip_full', os.path.join(L.RES, 'small_clip.mp4'), False), + ('nasa_partial', L.scratch('src', 'nasa.webm'), False), + ('notld_partial', L.scratch('src', 'notld.webm'), False), + ] + for name, path, il in real: + # The partial downloads hold only their first minutes. + frames = None + if 'partial' in name: + frames = 60 * 24 if 'notld' in name else 60 * 60 + rep, secs = run_worker(path, il, frames=frames) + out[name] = dict(worker=rep, seconds=secs) + print(f'{name:28s} {json.dumps(rep)} {secs:.1f}s', flush=True) + out['_worst'] = worst + json.dump(out, open(os.path.join(FX, 'parity.json'), 'w'), indent=1) + print('worst difference worker vs reference:', worst) + print('verdicts that differ:', [n for n, v in out.items() if isinstance(v, dict) and v.get('same_verdict') is False]) + + +if __name__ == '__main__': + main() diff --git a/docs/reports/README.md b/docs/reports/README.md index 5368b2a5..71d790e6 100644 --- a/docs/reports/README.md +++ b/docs/reports/README.md @@ -66,6 +66,13 @@ to it. | Date | Report | Outcome | |---|---|---| +| 2026-11-09 | [Noise and halo scores for the pass list](2026-11-noise-halo-scores/index.html) | Built: `--detect-noise-halo` badges Noise Reduction at a motion-compensated noise sigma ≥ 3.5 and Dehalo at a two-sided edge overshoot ≥ 6% of the step; no false positives on 154 fixtures (clean max 2.06 / 1.48), catches white noise from sigma 4 and tape/grain noise from sigma 6, misses sharpening of already-soft edges. VCG's signalstats TOUT and halo ratio did not separate the fixtures; a spatial (Immerkær) noise score was rejected as blind to tape and grain noise | +| 2026-11-08 | [Crop & Resize: BT.601 → BT.709 on SD → HD, and Detect borders](2026-11-crop-detect-709/index.html) | Built: matrix-only 601 → 709 conversion when the output is HD (default Auto; patches within 0.67 levels of the source; a gamut conversion rejected: it moved 75% green by 55 levels on players without colour management); `--detect-crop` finds blanking, bars and the head-switching band exactly on real-picture fixtures, and matches a by-eye measurement of a DVB-T broadcast | +| 2026-11-08 | [Classical filters for deps 1.16.0: SangNom, vszip NLMeans/WNNM, vs-jetpack ports](2026-11-classical-filters/index.html) | Built: SangNom AA (sharpest; 8/10/16-bit parity; slowest of the three at 1080p), NLMeans (CPU) = KNLMeansCL's output without OpenCL, the IVTC fade fix (field gap 4.12 → 0.06), vine dehalo and mpeg2stinx (both advanced; mpeg2stinx lost PSNR on every fixture). Not built: WNNM (luma ≈ BM3D, chroma 2.6–7.2 dB worse; fast, but NLMeans (CPU) covers that with better chroma); havsfunc's santiag `sangnom` type stays off (calls a double-rate SangNom) | +| 2026-11-08 | [Colourisation: DDColor and DeOldify Video on ONNX Runtime](2026-11-colourisation/index.html) | Built: DDColor-T (default) and DeOldify Video as add-ons, luma kept bit-exact (8/10/16-bit); conversions exact (DeOldify rebuilt without fastai, verified against an independent export); Core ML needs Pad/Expand/Einsum rewritten and still compiles DDColor for 77 s, so previews run it on the CPU; temporal smoothing cuts colour flicker 84% for both | +| 2026-11-08 | [Face restoration: RestoreFormer++, GFPGAN and CodeFormer on ONNX Runtime](2026-11-face-restoration/index.html) | Built: RetinaFace R50 (steadier landmarks than mobilenet0.25) + BiSeNet + RestoreFormer++ as the base add-on, GFPGAN/CodeFormer as non-commercial add-ons; ONNX parity ≤2.5e-4; NumPy pipeline within 0.16/255 of facexlib; bit-exact outside the faces; flicker cut by blending restored faces in aligned space | +| 2026-11-08 | [Live-action AI upscalers as add-ons, and AI strength](2026-11-ai-upscale-addons/index.html) | Built: Real-ESRGAN General (+ denoise), Live Action SPAN and Real-ESRGAN x4plus/x2plus on VSNCNN as two add-ons; conversions match PyTorch to 0.02 levels; 2xVHS2HD rejected (ncnn has no ScatterND/Shape/Range); AI strength blend halves a model's tint | +| 2026-11-08 | [Audio restoration: filter delays, drift and DeepFilterNet](2026-11-audio-restoration/index.html) | Built: render the soundtrack before the encode; trim afftdn's 25 ms delay and give it the measured noise floor; correct drift by resampling (atempo moved audio ~20 ms); alimiter `latency=1:level=0`; DeepFilterNet as an add-on with `-D` | | 2026-11-03 | [Deblock before or after deinterlacing?](2026-11-deblock-order/index.html) | Keep Deblock after Deinterlace (≤0.05 dB gain from moving it); when deinterlacing is off, deblock per field rather than on the woven frame (+0.03 to +0.30 dB; DCTFilter on the woven frame lost 1.5–3.7 dB); built | | 2026-11-03 | [Issue #144: KNLMeansCL fails on 10-bit sources (Windows), unreliable on macOS](issue-144-knlmeanscl/index.html) | Swap every platform to the maintained fork v1.3 (tag on macOS/Linux, PyPI wheel on Windows); verified on macOS arm64; on Windows x64 (AMD GPU) the bug reproduces with v1.1.1 and v1.3 fixes it | | 2026-11-03 | [Issue #117: QTempGaussMC (vs-jetpack) as an alternate deinterlacer](issue-117-qtempgaussmc/index.html) | Worth adding as an advanced-only method (not macOS x64); +0.5–0.8 dB over havsfunc QTGMC Slower at equal speed once tuned; not built yet | diff --git a/licenses/NOTICES.txt b/licenses/NOTICES.txt index fb412757..17ed7aca 100644 --- a/licenses/NOTICES.txt +++ b/licenses/NOTICES.txt @@ -50,6 +50,47 @@ Flutter — BSD 3-Clause https://github.com/flutter/flutter The UI toolkit the application is written in. +ONNX Runtime — MIT + Copyright (c) Microsoft Corporation + https://github.com/microsoft/onnxruntime + Runs AI models for the features that need it. NOT distributed with + VapourBox: part of the optional AI runtime add-on, which the app downloads + as the project's own unmodified Python wheels from PyPI (onnxruntime; on + Windows onnxruntime-directml) only when the user asks (Settings → Add-ons). + ONNX Runtime's ThirdPartyNotices.txt, inside each wheel, lists the + open-source components it includes. + +NumPy — BSD-3-Clause + Copyright (c) 2005-2025, NumPy Developers. + https://github.com/numpy/numpy + The array library ONNX Runtime's Python interface uses. NOT distributed + with VapourBox: part of the optional AI runtime add-on, downloaded as the + project's own unmodified wheels from PyPI. Some platforms' wheels bundle + further libraries (OpenBLAS, LAPACK, the GCC runtime libraries), each under + the licence given in the LICENSE.txt inside the wheel. + +DirectML — Microsoft Software License Terms (DirectML redistributable) + © Microsoft Corporation. All rights reserved. + https://github.com/microsoft/DirectML + Licence: https://www.nuget.org/packages/Microsoft.AI.DirectML/1.15.4/License + The DirectX Machine Learning library (DirectML.dll) that lets ONNX Runtime + use any DirectX 12 graphics card. Windows only, and NOT distributed with + VapourBox: it comes inside the onnxruntime-directml wheel that the optional + AI runtime add-on downloads from PyPI when the user asks. Proprietary; + Microsoft permits its redistribution in applications for Windows. + +DeepFilterNet — MIT OR Apache-2.0 (at your option) + Copyright (c) 2021 Hendrik Schröter + https://github.com/Rikorose/DeepFilterNet + Speech enhancement for Audio Restoration: the deep-filter 0.5.6 program, + with the DeepFilterNet 3 model built in. The repository's code and model + are dual-licensed under the MIT licence (LICENSE-MIT) or the Apache License + 2.0 (LICENSE-APACHE). NOT distributed with VapourBox: an optional add-on, + downloaded unmodified from the project's own v0.5.6 GitHub release only when + the user asks (Settings → Add-ons). The program statically includes the + Rust libraries it is built from (among them Sonos' tract), each under the + licence stated in its own crate. + ------------------------------------------------------------------------------- 2. VapourSynth Python modules @@ -74,6 +115,16 @@ mvsfunc — no licence stated upstream would prefer it not be redistributed, please open an issue and it will be removed. +vs-jetpack (ported code) — MIT + Copyright (c) 2020 - 2025 Jaded-Encoding-Thaumaturgy + https://github.com/Jaded-Encoding-Thaumaturgy/vs-jetpack + Not bundled as a package: three of its functions are ported, as VapourBox's + own template code, from release v3.0.0 — FixInterlacedFades (vsdeinterlace; + upstream describes it as an improved version of the Fix-Telecined-Fades + plugin), vine_dehalo (vsdehalo) and mpeg2stinx (vsdenoise). Used by the + Deinterlace pass's IVTC fade fix, the Dehalo pass's vine method and the + Deblock pass's mpeg2stinx method. + adjust — WTFPL Copyright (c) dubhater https://github.com/dubhater/vapoursynth-adjust @@ -175,7 +226,9 @@ vszip — MIT Copyright (c) 2024 Julek https://github.com/dnjulek/vapoursynth-zip Provides core.vszip.CLAHE (contrast-limited adaptive histogram - equalization), used by Color Correction's local contrast option. + equalization), used by Color Correction's local contrast option, and + core.vszip.NLMeans (non-local means on the CPU), used by Noise Reduction's + NLMeans method and the Dehalo pass's vine method. The binary statically includes two Zig libraries: zigimg — MIT, Copyright (c) 2019-2021 zigimg developers https://github.com/zigimg/zigimg @@ -184,12 +237,22 @@ vszip — MIT https://github.com/dnjulek/vapoursynth-zig Licence text: LGPL-2.1.txt +SangNom — MIT (source); binaries GPL-2.0 + Copyright (c) 2016 james1201 + Copyright (c) 2013 Victor Efimov + https://github.com/dubhatervapoursynth/vapoursynth-sangnom + The source header and LICENSE grant MIT; the same header states "Binaries + are GPL v2", and the PyPI wheels VapourBox ships declare GPL-2.0, so the + binaries are redistributed under GPL-2.0. Licence text: GPL-2.0.txt + Provides core.sangnom.SangNom, used by the Anti-aliasing pass's SangNom + method. Taken unmodified from the project's PyPI wheels (r45). + RIFE (VapourSynth-RIFE-ncnn-Vulkan) — MIT Copyright (c) 2021-2022 HolyWu https://github.com/styler00dollar/VapourSynth-RIFE-ncnn-Vulkan Provides core.rife.RIFE, the Frame Rate pass's "AI interpolation" method. Shipped on macOS arm64, Windows x64 and Linux x64 only, in - vapoursynth/rife/. Based on rife-ncnn-vulkan: + vapoursynth/rife/ (the plugin only; its model is an add-on, below). Based on rife-ncnn-vulkan: rife-ncnn-vulkan — MIT, Copyright (c) 2020 nihui https://github.com/nihui/rife-ncnn-vulkan The binary statically includes: @@ -212,8 +275,11 @@ RIFE 4.26 model — MIT ECCV 2022 paper's code: ECCV2022-RIFE — MIT, Copyright (c) Megvii Inc. https://github.com/hzwer/ECCV2022-RIFE - Shipped as the ncnn conversion (flownet.param / flownet.bin) published in - the VapourSynth-RIFE-ncnn-Vulkan repository, under its licence above. + NOT distributed with VapourBox: the optional "RIFE 4.26 interpolation model" + add-on, which the app downloads only when the user asks (Settings → Add-ons, + or on choosing AI interpolation). It is the ncnn conversion (flownet.param / + flownet.bin) published in the VapourSynth-RIFE-ncnn-Vulkan repository, under + its licence above, fetched from there at a pinned commit, unmodified. MoltenVK — Apache-2.0 Copyright (c) 2015-2026 The Brenwill Workshop Ltd. @@ -226,8 +292,8 @@ VSNCNN (vs-mlrt) — GPL-3.0 No copyright notice is given upstream; published by AmusementClub https://github.com/AmusementClub/vs-mlrt Licence text: GPL-3.0.txt - Provides core.ncnn.Model, which runs Crop & Resize's AI upscaling - (Real-ESRGAN / Real-CUGAN). Shipped on macOS arm64, Windows x64 and Linux + Provides core.ncnn.Model, which runs Crop & Resize's AI upscaling (every + model it runs is an add-on). Shipped on macOS arm64, Windows x64 and Linux x64 only, in vapoursynth/ncnn/: built from the v15.16 source on macOS and Linux, vs-mlrt's own build on Windows. The binary statically includes: ncnn — BSD 3-Clause, Copyright (C) 2017 Tencent. All rights reserved. @@ -248,11 +314,17 @@ VSNCNN (vs-mlrt) — GPL-3.0 https://github.com/llvm/llvm-project/tree/main/openmp Licence text: Apache-2.0-LLVM-Exception.txt -Real-ESRGAN anime video model — BSD 3-Clause +Real-ESRGAN anime video model — BSD-3-Clause Copyright (c) 2021, Xintao Wang https://github.com/xinntao/Real-ESRGAN - The trained network weights of realesr-animevideov3, at 2x and 4x. Shipped as - vs-mlrt's ONNX conversions (RealESRGANv2-animevideo-xsx2/xsx4.onnx). + The trained network weights of realesr-animevideov3, at 2x and 4x, used by + the Real-ESRGAN upscaler. NOT distributed with VapourBox: part of the + optional "Real-ESRGAN anime & Real-CUGAN models" add-on, which the app + downloads only when the user asks (Settings → Add-ons, or on choosing the + upscaler). The files are vs-mlrt's ONNX conversions + (RealESRGANv2-animevideo-xsx2/xsx4.onnx), unmodified, hosted individually by + this project because vs-mlrt publishes them only inside one ~810 MB archive; + redistributed under the BSD 3-Clause licence whose text is given below. AnimeJaNai models — CC BY-NC-SA 4.0 (non-commercial) By the-database (GitHub); upstream names no other copyright holder. @@ -265,11 +337,145 @@ AnimeJaNai models — CC BY-NC-SA 4.0 (non-commercial) upscaler). Free for non-commercial use only; anyone using them must keep to that. +RestoreFormer++ model — Apache-2.0 + By Zhouxia Wang, Jiawei Zhang, Tianshui Chen, Wenping Wang and Ping Luo (the + authors its README cites); its LICENSE names no copyright holder. + https://github.com/wzhouxiff/RestoreFormerPlusPlus + Licence text: Apache-2.0.txt + The trained RestoreFormer++ network (release v1.0.0), the Face Restoration + pass's default model. NOT distributed with VapourBox: part of the optional + face restoration add-on, which the app downloads only when the user asks + (Settings → Add-ons, or on switching the pass on), as an ONNX conversion of + the published weights (Scripts/models/make_face_restoration.py). + +RetinaFace face detector — MIT + By biubug6 (Pytorch_Retinaface). Its LICENSE reads "Copyright (c) 2019" + and names no holder. + https://github.com/biubug6/Pytorch_Retinaface + The RetinaFace ResNet-50 face detector (detection_Resnet50_Final.pth as + facexlib publishes it), which finds the faces and their landmarks for the + Face Restoration pass. NOT distributed with VapourBox: part of the optional + face restoration add-on, downloaded as an ONNX conversion only when the user + asks. + +BiSeNet face parser — MIT + Copyright (c) 2019 zll (face-parsing.PyTorch) + https://github.com/zllrunning/face-parsing.PyTorch + The BiSeNet face-parsing network (parsing_bisenet.pth as facexlib publishes + it), which limits the Face Restoration pass's blend to the face itself. NOT + distributed with VapourBox: part of the optional face restoration add-on, + downloaded as an ONNX conversion only when the user asks. + +facexlib — MIT + Copyright (c) 2020 Xintao Wang + https://github.com/xinntao/facexlib + The face-restoration reference code whose detector and parser weights the + face restoration add-on converts, and whose FFHQ alignment template and + paste-back the pass reproduces (worker/templates/face_restore.py). Its + weights reach users only through that optional add-on, downloaded when the + user asks. + +GFPGAN v1.4 model — Apache-2.0 + NVIDIA Source Code License (NC) + Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + https://github.com/TencentARC/GFPGAN + Licence: https://github.com/TencentARC/GFPGAN/blob/master/LICENSE + The trained GFPGAN v1.4 network (GFPGANv1.4.pth), an alternative model for + the Face Restoration pass. GFPGAN is Apache-2.0 except its third-party + components; its StyleGAN2-derived network code is under NVIDIA's Source Code + License, whose section 3.3 says the work "only may be used or intended for + use non-commercially", where "non-commercially" means "for research or + evaluation purposes only". NOT distributed with VapourBox: an optional + add-on (face-gfpgan), downloaded as an ONNX conversion only when the user + asks, after stating that restriction. Non-commercial use only; anyone using + it must keep to that. + +CodeFormer model — S-Lab License 1.0 (NC) + Copyright 2022 S-Lab + https://github.com/sczhou/CodeFormer + Licence: https://github.com/sczhou/CodeFormer/blob/master/LICENSE + The trained CodeFormer network (codeformer.pth, release v0.1.0), an + alternative model for the Face Restoration pass. The S-Lab License 1.0 + permits redistribution and use for non-commercial purposes, with the + copyright notice and its disclaimer; commercial use requires contacting the + contributors. NOT distributed with VapourBox: an optional add-on + (face-codeformer), downloaded as an ONNX conversion only when the user asks, + after stating that restriction. Non-commercial use only. + +DDColor model — Apache-2.0 + By Xiaoyang Kang, Tao Yang, Wenqi Ouyang, Peiran Ren, Lingzhi Li and Xuansong Xie + (the authors its README cites); its LICENSE names no copyright holder. + https://github.com/piddnad/DDColor + Licence text: Apache-2.0.txt + The trained DDColor-T network (the ddcolor_paper_tiny checkpoint, whose model + card states Apache-2.0), the Colourisation pass's default model. Its network + code adapts ConvNeXt (MIT, Meta Platforms), DETR and Mask2Former (Facebook). + NOT distributed with VapourBox: an optional add-on (colorize-ddcolor), which + the app downloads only when the user asks (Settings → Add-ons, or on + switching the pass on), as an ONNX conversion of the published weights + (Scripts/models/make_colorize_ddcolor.py). + +DeOldify Video model — MIT + Copyright (c) 2018 Jason Antic + https://github.com/jantic/DeOldify + The trained DeOldify Video generator (ColorizeVideo_gen.pth), an alternative + model for the Colourisation pass; DeOldify's README states that its + published weights are released under the MIT licence. Built on fastai + (Apache-2.0) and a ResNet-101 backbone (torchvision, BSD-3-Clause). NOT + distributed with VapourBox: an optional add-on (colorize-deoldify), which the + app downloads only when the user asks, as an ONNX conversion of the + published weights (Scripts/models/make_colorize_deoldify.py). + +Real-ESRGAN general model — BSD-3-Clause + Copyright (c) 2021, Xintao Wang + https://github.com/xinntao/Real-ESRGAN + The trained network weights of realesr-general-x4v3 and + realesr-general-wdn-x4v3, used by the Real-ESRGAN General upscaler. NOT + distributed with VapourBox: part of the optional "General & live-action + upscaling" add-on, which the app downloads only when the user asks (Settings + → Add-ons, or on choosing the upscaler). Upstream publishes PyTorch + checkpoints only; the add-on carries this project's ONNX conversions + (Scripts/models/convert_realesrgan.py), redistributed under the BSD 3-Clause + licence whose text is given below. + +2xLiveActionV1_SPAN model — Apache-2.0 + Copyright 2025 jcj83429 + https://github.com/jcj83429/upscaling/tree/main/2xLiveActionV1_SPAN + Licence text: Apache-2.0.txt + The 2x SPAN model for live-action video and film, used by the Live Action + SPAN upscaler, as upstream's own unmodified ONNX + (2xLiveActionV1_SPAN_490000.onnx). NOT distributed with VapourBox: part of + the optional "General & live-action upscaling" add-on, which the app + downloads from upstream only when the user asks. Relicensed by its author to + the Apache License 2.0 in upstream commit 5d8cdd2e; its NOTICE file reads: + Copyright 2025 jcj83429 + Licensed under the Apache License, Version 2.0 (the "License"); you may + not use this file except in compliance with the License. You may obtain a + copy of the License at http://www.apache.org/licenses/LICENSE-2.0. Unless + required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the + License. + +Real-ESRGAN x4plus / x2plus models — BSD-3-Clause + Copyright (c) 2021, Xintao Wang + https://github.com/xinntao/Real-ESRGAN + The trained network weights of RealESRGAN_x4plus and RealESRGAN_x2plus, used + by the Real-ESRGAN x4plus upscaler. NOT distributed with VapourBox: the + optional "Real-ESRGAN x4plus / x2plus" add-on, which the app downloads only + when the user asks. Upstream publishes PyTorch checkpoints only; the add-on + carries this project's ONNX conversions, redistributed under the BSD + 3-Clause licence whose text is given below. + Real-CUGAN models — MIT Copyright (c) 2022 bilibili https://github.com/bilibili/ailab/tree/main/Real-CUGAN The trained network weights of the 2x models (no-denoise, conservative, - denoise3x). Shipped as vs-mlrt's ONNX conversions (up2x-latest-*.onnx). + denoise3x), used by the Real-CUGAN upscaler. NOT distributed with VapourBox: + part of the optional "Real-ESRGAN anime & Real-CUGAN models" add-on, which + the app downloads only when the user asks. The files are vs-mlrt's ONNX + conversions (up2x-latest-*.onnx), unmodified, hosted individually by this + project. DFTTest — GPL-3.0 Copyright (C) 2007-2010 Kevin Stone (tritical); VapourSynth port by HolyWu @@ -532,10 +738,13 @@ Appendix — full text of the short permissive licences MIT License ----------- -(Applies to zsmooth, vszip and the zigimg library it includes, CAS, +(Applies to zsmooth, vszip and the zigimg library it includes, SangNom's +source, the code ported from vs-jetpack, CAS, DCTFilter, BM3D, descale, the RIFE plugin, rife-ncnn-vulkan, the RIFE model and -ECCV2022-RIFE, the Real-CUGAN models, whisper.cpp, the Whisper models, pip and -the Clense filter +ECCV2022-RIFE, the Real-CUGAN models, whisper.cpp, the Whisper models, ONNX +Runtime, the RetinaFace face detector, the BiSeNet face parser, facexlib, the +DeOldify Video model, pip +and the Clense filter in vs-removegrain, each under the copyright notice given in its entry above.) @@ -596,8 +805,8 @@ DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE, Version 2 BSD 3-Clause License -------------------- -(Applies to Flutter, Zstandard, ncnn, the core of glslang, Protocol Buffers -and the Real-ESRGAN model, under the copyright notices given above.) +(Applies to Flutter, Zstandard, ncnn, the core of glslang, Protocol Buffers, +the Real-ESRGAN models and NumPy, under the copyright notices given above.) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: diff --git a/worker/Cargo.toml b/worker/Cargo.toml index f79801be..b0e85e96 100644 --- a/worker/Cargo.toml +++ b/worker/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "vapourbox-worker" -version = "1.6.0" +version = "1.7.0" edition = "2021" description = "Video processing worker using VapourSynth" authors = ["Stuart Cameron"] diff --git a/worker/build.rs b/worker/build.rs new file mode 100644 index 00000000..520e1130 --- /dev/null +++ b/worker/build.rs @@ -0,0 +1,40 @@ +//! Compiles the add-on manifests (`app/assets/addons/*.json`) into the worker, +//! so the app and the worker read the same files: one source of truth for +//! what each add-on is and what an install of it holds (see `src/addons.rs`). + +use std::env; +use std::fs; +use std::path::PathBuf; + +fn main() { + let crate_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let dir = crate_dir.join("..").join("app").join("assets").join("addons"); + // A directory: Cargo rescans it, so adding a manifest rebuilds too. + println!("cargo:rerun-if-changed={}", dir.display()); + + // Git keeps no empty directory, so none at all means no manifests. + let mut manifests: Vec = fs::read_dir(&dir) + .into_iter() + .flatten() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| p.extension().is_some_and(|x| x == "json")) + .collect(); + manifests.sort(); + + let mut out = String::from( + "/// Every add-on manifest, as (file name, contents). Generated by build.rs.\n\ + pub(crate) const MANIFESTS: &[(&str, &str)] = &[\n", + ); + for path in &manifests { + println!("cargo:rerun-if-changed={}", path.display()); + let name = path.file_name().unwrap().to_string_lossy().to_string(); + let abs = fs::canonicalize(path).unwrap(); + out.push_str(&format!( + " ({name:?}, include_str!({:?})),\n", + abs.to_string_lossy() + )); + } + out.push_str("];\n"); + let dest = PathBuf::from(env::var("OUT_DIR").unwrap()).join("addon_manifests.rs"); + fs::write(dest, out).unwrap(); +} diff --git a/worker/src/addons.rs b/worker/src/addons.rs new file mode 100644 index 00000000..720079f0 --- /dev/null +++ b/worker/src/addons.rs @@ -0,0 +1,431 @@ +//! Manifest-described add-ons: optional third-party downloads (AI models, the +//! AI runtime) that VapourBox can use but never ships in the deps bundle. +//! +//! Each is described by one JSON file in `app/assets/addons/`, which the app +//! installs from (`ManifestAddonInstaller`, `app/lib/services/manifest_addon.dart`) +//! and which `build.rs` compiles in here, so both sides read the same +//! description (docs/ADDONS.md has the format). The app downloads, verifies +//! and unpacks an add-on into `//`, writing [`MARKER`] last; this +//! module only answers whether that happened for this platform and version, +//! and refuses a job that needs an add-on that isn't there, with the same kind +//! of message as `animejanai::MISSING_MESSAGE`. + +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; +use std::sync::OnceLock; + +use serde::Deserialize; + +include!(concat!(env!("OUT_DIR"), "/addon_manifests.rs")); + +/// The commit marker an install ends with: `{id, version, platform}`. +pub const MARKER: &str = ".vapourbox-addon.json"; + +/// The AI runtime: ONNX Runtime and NumPy for the job's Python. +pub const AI_RUNTIME: &str = "ai-runtime"; + +/// DeepFilterNet's `deep-filter` program, for audio restoration's speech +/// enhancement (`audio_restoration.rs`). +pub const DEEPFILTERNET: &str = "deepfilternet"; + +/// Where in the add-ons UI to install them — named in every refusal. +const WHERE: &str = "Settings → Add-ons"; + +/// One file an add-on installs (the fields the worker needs). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ManifestFile { + pub name: String, + #[serde(default)] + pub platforms: Option>, + #[serde(default)] + pub extract: Option>, + /// A program the worker runs: the app marks it executable on install. + #[serde(default)] + pub executable: bool, +} + +impl ManifestFile { + fn is_for(&self, platform: &str) -> bool { + self.platforms.as_ref().map_or(true, |p| p.iter().any(|x| x == platform)) + } +} + +/// A member kept from an archive entry. +#[derive(Debug, Clone, Deserialize)] +pub struct ArchiveMember { + pub path: String, + #[serde(default)] + pub name: Option, +} + +impl ArchiveMember { + fn installed_name(&self) -> &str { + self.name.as_deref().unwrap_or_else(|| self.path.rsplit('/').next().unwrap_or(&self.path)) + } +} + +/// `"all"` or a table of platform id → requirements. +#[derive(Debug, Clone, Deserialize)] +#[serde(untagged)] +enum Platforms { + All(String), + Table(BTreeMap), +} + +/// An add-on manifest (the fields the worker needs). +#[derive(Debug, Clone, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Manifest { + pub id: String, + pub version: String, + pub name: String, + pub feature: String, + pub kind: String, + #[serde(default)] + pub requires: Vec, + platforms: Platforms, + pub files: Vec, +} + +/// Every platform id add-ons are described for. +pub const PLATFORMS: &[&str] = &["macos-arm64", "macos-x64", "windows-x64", "linux-x64", "linux-arm64"]; + +impl Manifest { + pub fn supports(&self, platform: &str) -> bool { + match &self.platforms { + Platforms::All(s) => s == "all" && PLATFORMS.contains(&platform), + Platforms::Table(t) => t.contains_key(platform), + } + } + + /// The paths, relative to the add-on's directory, that a complete install + /// on `platform` holds: a python add-on's `site-packages`, or a model + /// pack's files (an archive's kept members by their installed names). + pub fn expected_paths(&self, platform: &str) -> Vec { + if self.kind == "python" { + return vec!["site-packages".to_string()]; + } + let mut out = Vec::new(); + for f in self.files.iter().filter(|f| f.is_for(platform)) { + match &f.extract { + None => out.push(f.name.clone()), + Some(members) => out.extend(members.iter().map(|m| m.installed_name().to_string())), + } + } + out + } +} + +impl Manifest { + /// The program this add-on installs for `platform` (the file marked + /// `executable`), by its installed name. + pub fn executable_for(&self, platform: &str) -> Option<&str> { + self.files.iter().find(|f| f.executable && f.is_for(platform)).map(|f| f.name.as_str()) + } +} + +/// Every compiled-in manifest, sorted by id. +pub fn manifests() -> &'static [Manifest] { + static ALL: OnceLock> = OnceLock::new(); + ALL.get_or_init(|| { + let mut v: Vec = MANIFESTS + .iter() + .map(|(file, json)| { + serde_json::from_str(json) + .unwrap_or_else(|e| panic!("add-on manifest {file} does not parse: {e}")) + }) + .collect(); + v.sort_by(|a, b| a.id.cmp(&b.id)); + v + }) +} + +pub fn manifest(id: &str) -> Option<&'static Manifest> { + manifests().iter().find(|m| m.id == id) +} + +/// The add-on's directory under the add-ons directory. +pub fn addon_dir(addons_root: &Path, id: &str) -> PathBuf { + addons_root.join(id) +} + +/// A python add-on's packages directory, which the script puts on `sys.path`. +pub fn site_packages(addons_root: &Path, id: &str) -> PathBuf { + addon_dir(addons_root, id).join("site-packages") +} + +/// What is installed of one add-on. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AddonState { + Installed, + /// Nothing, or an incomplete install. + Missing, + /// An install of another version (the manifest was updated since). + Outdated { installed: String }, + /// No build of it for this platform. + Unsupported, + /// No manifest by that id: a bug, not a user problem. + Unknown, +} + +/// Whether `id` is installed for `platform` in `addons_root`. +pub fn state(addons_root: &Path, id: &str, platform: &str) -> AddonState { + let Some(m) = manifest(id) else { return AddonState::Unknown }; + if !m.supports(platform) { + return AddonState::Unsupported; + } + let dir = addon_dir(addons_root, id); + let marker: Option = std::fs::read_to_string(dir.join(MARKER)) + .ok() + .and_then(|s| serde_json::from_str(&s).ok()); + let Some(marker) = marker else { return AddonState::Missing }; + if marker.get("id").and_then(|v| v.as_str()) != Some(id) + || marker.get("platform").and_then(|v| v.as_str()) != Some(platform) + { + return AddonState::Missing; + } + let version = marker.get("version").and_then(|v| v.as_str()).unwrap_or(""); + if version != m.version { + return AddonState::Outdated { installed: version.to_string() }; + } + if m.expected_paths(platform).iter().all(|p| dir.join(p).exists()) { + AddonState::Installed + } else { + AddonState::Missing + } +} + +pub fn is_installed(addons_root: &Path, id: &str, platform: &str) -> bool { + state(addons_root, id, platform) == AddonState::Installed +} + +/// The add-ons one feature of a job needs: what `ProcessingPipeline::required_addons` +/// returns, and what `ScriptGenerator` checks before writing a script. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AddonNeed { + /// The feature as the user knows it, starting the refusal message. + pub feature: &'static str, + /// The add-on ids it uses (their own requirements are added automatically). + pub ids: Vec<&'static str>, +} + +/// `ids` and everything they require, requirements first, each once. +pub fn with_requirements(ids: &[&str]) -> Vec { + fn visit(id: &str, out: &mut Vec) { + if out.iter().any(|x| x == id) { + return; + } + if let Some(m) = manifest(id) { + for r in &m.requires { + visit(r, out); + } + } + out.push(id.to_string()); + } + let mut out = Vec::new(); + for id in ids { + visit(id, &mut out); + } + out +} + +/// Whether a job needing `ids` needs the AI runtime (directly or through a +/// model add-on that requires it) — what decides the script's `AI_RUNTIME` +/// block. +pub fn needs_ai_runtime(ids: &[&str]) -> bool { + with_requirements(ids).iter().any(|i| i == AI_RUNTIME) +} + +/// Refuse, with a message the user can act on, unless every add-on in `ids` +/// (and everything they require) is installed. `feature` is what the user +/// chose ("Face restoration"), at the start of the message. +pub fn require(addons_root: &Path, ids: &[&str], feature: &str, platform: &str) -> anyhow::Result<()> { + match missing_message(addons_root, ids, feature, platform) { + None => Ok(()), + Some(message) => anyhow::bail!("{message}"), + } +} + +/// The refusal [`require`] gives, or `None` when everything is installed. +pub fn missing_message(addons_root: &Path, ids: &[&str], feature: &str, platform: &str) -> Option { + let mut missing = Vec::new(); + let mut outdated = Vec::new(); + for id in with_requirements(ids) { + let name = manifest(&id).map(|m| m.name.clone()).unwrap_or_else(|| id.clone()); + match state(addons_root, &id, platform) { + AddonState::Installed => {} + AddonState::Unknown => { + return Some(format!("{feature} needs an add-on this version of VapourBox doesn't know (\"{id}\").")) + } + AddonState::Unsupported => { + return Some(format!( + "{feature} needs the {name} add-on, which isn't available on this platform." + )) + } + AddonState::Missing => missing.push(name), + AddonState::Outdated { installed } => outdated.push((name, installed)), + } + } + if missing.is_empty() && outdated.is_empty() { + return None; + } + let mut parts = Vec::new(); + match missing.len() { + 0 => {} + 1 => parts.push(format!( + "{feature} needs the {} add-on, which is not installed. It is a separate download: \ + install it in {WHERE}, or choose {feature} again to be offered it.", + missing[0] + )), + _ => parts.push(format!( + "{feature} needs these add-ons, which are not installed: {}. They are separate \ + downloads: install them in {WHERE}, or choose {feature} again to be offered them.", + missing.join(", ") + )), + } + for (name, installed) in outdated { + parts.push(format!( + "The installed {name} add-on (version {installed}) is out of date for this version \ + of VapourBox. Install it again in {WHERE}." + )); + } + Some(parts.join(" ")) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn tmp(tag: &str) -> PathBuf { + let d = std::env::temp_dir().join(format!("vb-addons-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&d); + std::fs::create_dir_all(&d).unwrap(); + d + } + + fn install(root: &Path, id: &str, version: &str, platform: &str) { + let m = manifest(id).unwrap(); + let dir = addon_dir(root, id); + std::fs::create_dir_all(&dir).unwrap(); + for p in m.expected_paths(platform) { + if m.kind == "python" { + std::fs::create_dir_all(dir.join(p)).unwrap(); + } else { + std::fs::write(dir.join(p), b"x").unwrap(); + } + } + std::fs::write( + dir.join(MARKER), + serde_json::json!({"id": id, "version": version, "platform": platform}).to_string(), + ) + .unwrap(); + } + + /// The compiled-in set is exactly the directory's: build.rs found them all. + #[test] + fn compiled_manifests_match_the_app_directory() { + let dir = Path::new(env!("CARGO_MANIFEST_DIR")).join("../app/assets/addons"); + let mut on_disk: Vec = std::fs::read_dir(&dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().to_string()) + .filter(|n| n.ends_with(".json")) + .collect(); + on_disk.sort(); + let compiled: Vec = MANIFESTS.iter().map(|(n, _)| n.to_string()).collect(); + assert_eq!(compiled, on_disk); + for m in manifests() { + // One file per add-on, named after it. + assert!(on_disk.contains(&format!("{}.json", m.id)), "{}", m.id); + assert!(matches!(m.kind.as_str(), "files" | "python"), "{}", m.id); + for r in &m.requires { + assert!(manifest(r).is_some(), "{} requires unknown {r}", m.id); + } + for p in PLATFORMS { + if m.supports(p) { + assert!(!m.expected_paths(p).is_empty(), "{} has nothing for {p}", m.id); + } + } + } + assert!(manifest(AI_RUNTIME).is_some()); + } + + #[test] + fn ai_runtime_is_python_on_every_platform() { + let m = manifest(AI_RUNTIME).unwrap(); + assert_eq!(m.kind, "python"); + for p in PLATFORMS { + assert!(m.supports(p), "{p}"); + assert_eq!(m.expected_paths(p), vec!["site-packages".to_string()]); + assert_eq!(m.files.iter().filter(|f| f.is_for(p)).count(), 2, "onnxruntime + numpy on {p}"); + } + } + + #[test] + fn installed_needs_the_marker_for_this_version_and_platform() { + let root = tmp("state"); + let v = manifest(AI_RUNTIME).unwrap().version.clone(); + assert_eq!(state(&root, AI_RUNTIME, "linux-x64"), AddonState::Missing); + // Files without the marker: an interrupted install. + std::fs::create_dir_all(site_packages(&root, AI_RUNTIME)).unwrap(); + assert_eq!(state(&root, AI_RUNTIME, "linux-x64"), AddonState::Missing); + install(&root, AI_RUNTIME, &v, "linux-x64"); + assert!(is_installed(&root, AI_RUNTIME, "linux-x64")); + // Another platform's install (a copied add-ons directory) is not this one's. + assert_eq!(state(&root, AI_RUNTIME, "linux-arm64"), AddonState::Missing); + install(&root, AI_RUNTIME, "0.0.1", "linux-x64"); + assert_eq!( + state(&root, AI_RUNTIME, "linux-x64"), + AddonState::Outdated { installed: "0.0.1".into() } + ); + assert_eq!(state(&root, "no-such-addon", "linux-x64"), AddonState::Unknown); + std::fs::remove_dir_all(&root).unwrap(); + } + + #[test] + fn require_names_the_addon_and_where_to_get_it() { + let root = tmp("require"); + let err = require(&root, &[AI_RUNTIME], "Face restoration", "macos-arm64").unwrap_err().to_string(); + assert!(err.starts_with("Face restoration needs the AI runtime (ONNX Runtime) add-on"), "{err}"); + assert!(err.contains("Settings → Add-ons"), "{err}"); + let v = manifest(AI_RUNTIME).unwrap().version.clone(); + install(&root, AI_RUNTIME, &v, "macos-arm64"); + require(&root, &[AI_RUNTIME], "Face restoration", "macos-arm64").unwrap(); + require(&root, &[], "Nothing", "macos-arm64").unwrap(); + install(&root, AI_RUNTIME, "0.0.1", "macos-arm64"); + let err = require(&root, &[AI_RUNTIME], "Face restoration", "macos-arm64").unwrap_err().to_string(); + assert!(err.contains("out of date") && err.contains("0.0.1"), "{err}"); + std::fs::remove_dir_all(&root).unwrap(); + } + + /// DeepFilterNet is one program per platform, each the platform's own + /// upstream build, marked executable so the app sets the bit on install. + #[test] + fn deepfilternet_has_one_program_per_platform() { + let m = manifest(DEEPFILTERNET).unwrap(); + assert_eq!(m.kind, "files"); + for p in PLATFORMS { + assert!(m.supports(p), "{p}"); + let exe = m.executable_for(p).unwrap_or_else(|| panic!("no program for {p}")); + assert_eq!(m.expected_paths(p), vec![exe.to_string()]); + assert_eq!(exe.ends_with(".exe"), *p == "windows-x64", "{exe}"); + } + // Nothing else is marked executable. + assert!(manifest(AI_RUNTIME).unwrap().files.iter().all(|f| !f.executable)); + } + + #[test] + fn requirements_come_first() { + assert_eq!(with_requirements(&[AI_RUNTIME, AI_RUNTIME]), vec![AI_RUNTIME.to_string()]); + assert!(needs_ai_runtime(&[AI_RUNTIME])); + assert!(!needs_ai_runtime(&[])); + for m in manifests() { + let order = with_requirements(&[m.id.as_str()]); + assert_eq!(order.last(), Some(&m.id)); + for r in &m.requires { + assert!(order.iter().position(|x| x == r) < order.iter().position(|x| x == &m.id)); + } + } + } +} diff --git a/worker/src/ai_runtime.rs b/worker/src/ai_runtime.rs new file mode 100644 index 00000000..c8b20995 --- /dev/null +++ b/worker/src/ai_runtime.rs @@ -0,0 +1,383 @@ +//! The AI runtime add-on (ONNX Runtime + NumPy, `app/assets/addons/ai-runtime.json`): +//! whether it runs here, and on which execution provider. +//! +//! Like the GPU plugins (`gpu_plugin.rs`), this is found out by running it +//! once in a throwaway vspipe rather than by a job: the probe script imports +//! ONNX Runtime from the add-on through `templates/ai_runtime.py` — exactly +//! the path a job's `{{#AI_RUNTIME}}` block takes — creates a session for the +//! committed self-test model on the preferred provider, runs it directly and +//! through a VapourSynth frame, and reports the result as JSON on stderr. The +//! outcome becomes an [`AiRuntimeStatus`] with a message the app shows +//! verbatim (`--probe-ai-runtime`). Only a positive result is cached, for the +//! boot, as for the GPU plugins. + +use std::collections::HashMap; +use std::path::Path; +use std::time::Duration; + +use serde::{Deserialize, Serialize}; + +use crate::addons::{self, AddonState, AI_RUNTIME}; +use crate::gpu_plugin::{py_str, run_vspipe_probe, ProbeExit}; + +/// Session creation can compile the model for the GPU (Core ML, DirectML); +/// a driver that hangs must not hang the app. +const PROBE_TIMEOUT: Duration = Duration::from_secs(90); + +/// Stage marker the probe prints to stderr, so a silent crash says where. +const MARKER: &str = "AI_RUNTIME_PROBE:"; +/// Prefix of the line carrying the self-test's JSON result. +const RESULT: &str = "AI_RUNTIME_RESULT:"; + +/// Why the AI runtime cannot run here. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum AiRuntimeReason { + /// The add-on has no build for this platform (or this macOS is too old — + /// the app decides that one before offering it). + UnsupportedPlatform, + /// The add-on is not installed. + NotInstalled, + /// An install of an older manifest version. + Outdated, + /// A system library onnxruntime needs is missing (Windows: the VC++ runtime). + MissingLibrary, + /// `import onnxruntime` failed for another reason. + ImportFailed, + /// The self-test model could not be run, or computed the wrong answer. + SessionFailed, + /// An instruction this CPU lacks. + UnsupportedCpu, + /// The probe process crashed. + Crashed, + /// The probe could not be run or did not finish. + ProbeFailed, +} + +/// Whether the AI runtime runs here, and how. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AiRuntimeStatus { + pub available: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub detail: Option, + /// onnxruntime's version. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub version: Option, + /// Every execution provider the installed onnxruntime offers. + #[serde(default)] + pub providers: Vec, + /// The provider the self-test session ran on (its first registered one). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub provider: Option, + /// Why the preferred GPU provider was not used, when it fell back to the CPU. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback: Option, +} + +impl AiRuntimeStatus { + pub fn unavailable(reason: AiRuntimeReason, detail: impl Into>) -> Self { + Self { + available: false, + reason: Some(reason), + detail: detail.into(), + version: None, + providers: Vec::new(), + provider: None, + fallback: None, + } + } + + /// What to tell the user, remedy included; `None` when available. The app + /// shows this verbatim. + pub fn message(&self, platform: &str) -> Option { + use AiRuntimeReason as R; + let reason = self.reason?; + let name = addons::manifest(AI_RUNTIME).map(|m| m.name.as_str()).unwrap_or("The AI runtime"); + let text = match reason { + R::UnsupportedPlatform => format!("The {name} add-on isn't available on this platform."), + R::NotInstalled => format!( + "The {name} add-on is not installed. It is a separate download: install it in \ + Settings → Add-ons." + ), + R::Outdated => format!( + "The installed {name} add-on is out of date for this version of VapourBox. \ + Install it again in Settings → Add-ons." + ), + R::MissingLibrary => { + if platform.starts_with("windows") { + "ONNX Runtime needs the Microsoft Visual C++ Redistributable (x64), which \ + isn't installed. Install the latest version from Microsoft (search for \ + \"Visual C++ Redistributable latest supported downloads\"), then restart \ + VapourBox." + .to_string() + } else { + "ONNX Runtime needs a system library that isn't installed; the details below \ + name it. Install it with your package manager, then restart VapourBox." + .to_string() + } + } + R::ImportFailed => format!( + "The {name} add-on is installed but would not load. Remove it and install it \ + again in Settings → Add-ons; if this persists, please report it with the \ + details below." + ), + R::SessionFailed => format!( + "The {name} add-on loaded but could not run its test model. Updating your \ + graphics driver may help; if this persists, please report it with the details \ + below." + ), + R::UnsupportedCpu => "ONNX Runtime needs a newer processor than this computer has \ + (it uses instructions this CPU lacks)." + .to_string(), + R::Crashed => format!( + "The {name} add-on crashed while testing itself. Updating your graphics driver \ + usually fixes this; restart VapourBox afterwards." + ), + R::ProbeFailed => format!( + "VapourBox couldn't check whether the {name} add-on runs on this computer. \ + Restart VapourBox to try again." + ), + }; + Some(match &self.detail { + Some(d) if !d.is_empty() => format!("{text}\n\nDetails: {d}"), + _ => text, + }) + } + + /// The JSON `--probe-ai-runtime` prints, message included. + pub fn to_json(&self, platform: &str) -> serde_json::Value { + let mut v = serde_json::to_value(self).unwrap_or_default(); + if let (Some(msg), Some(obj)) = (self.message(platform), v.as_object_mut()) { + obj.insert("message".into(), serde_json::Value::String(msg)); + } + v + } +} + +/// What the add-on files alone decide. `None`: installed — run the probe. +pub fn precheck(addons_root: &Path, platform: &str) -> Option { + use AiRuntimeReason as R; + match addons::state(addons_root, AI_RUNTIME, platform) { + AddonState::Installed => None, + AddonState::Unsupported | AddonState::Unknown => { + Some(AiRuntimeStatus::unavailable(R::UnsupportedPlatform, None)) + } + AddonState::Missing => Some(AiRuntimeStatus::unavailable(R::NotInstalled, None)), + AddonState::Outdated { installed } => Some(AiRuntimeStatus::unavailable( + R::Outdated, + format!("installed version {installed}"), + )), + } +} + +/// The probe script: the same import path a job takes, then the self-tests. +pub fn probe_script(templates_dir: &Path, addon_dir: &Path, force_cpu: bool) -> String { + format!( + "import sys, json\n\ + def stage(s):\n print('{MARKER}' + s, file=sys.stderr, flush=True)\n\ + stage('start')\n\ + sys.path.insert(0, {templates})\n\ + import ai_runtime\n\ + stage('setup')\n\ + ai_runtime.setup({addon})\n\ + stage('import')\n\ + ai_runtime.ort()\n\ + stage('session')\n\ + result = ai_runtime.self_test(force_cpu={force})\n\ + stage('frames')\n\ + result['frame'] = ai_runtime.frame_test(force_cpu={force})\n\ + print('{RESULT}' + json.dumps(result), file=sys.stderr, flush=True)\n\ + import vapoursynth as vs\n\ + vs.core.std.BlankClip(width=16, height=16, length=1).set_output()\n", + templates = py_str(templates_dir), + addon = py_str(addon_dir), + force = if force_cpu { "True" } else { "False" }, + ) +} + +/// Turn a finished probe into a status. Pure, so every branch is testable. +pub fn classify(exit: &ProbeExit, stderr: &str) -> AiRuntimeStatus { + use AiRuntimeReason as R; + let result = stderr + .lines() + .filter_map(|l| l.trim().strip_prefix(RESULT)) + .last() + .and_then(|j| serde_json::from_str::(j).ok()); + if exit.success { + if let Some(r) = result { + let s = |k: &str| r.get(k).and_then(|v| v.as_str()).map(str::to_string); + return AiRuntimeStatus { + available: true, + reason: None, + detail: None, + version: s("version"), + providers: r + .get("providers") + .and_then(|v| v.as_array()) + .map(|a| a.iter().filter_map(|x| x.as_str().map(str::to_string)).collect()) + .unwrap_or_default(), + provider: s("provider"), + fallback: s("fallback"), + }; + } + return AiRuntimeStatus::unavailable(R::ProbeFailed, "the self-test reported nothing".to_string()); + } + if exit.timed_out { + return AiRuntimeStatus::unavailable( + R::ProbeFailed, + format!("the test did not finish within {}s", PROBE_TIMEOUT.as_secs()), + ); + } + if exit.status.starts_with("signal 4 ") || exit.status.contains("illegal instruction") { + return AiRuntimeStatus::unavailable(R::UnsupportedCpu, exit.status.clone()); + } + let last = stderr.lines().filter_map(|l| l.trim().strip_prefix(MARKER)).last(); + let error = last_error_line(stderr); + let or_status = |e: Option| e.or_else(|| Some(format!("vspipe exited with {}", exit.status))); + match last { + Some("setup") if stderr.contains("is not installed") => { + AiRuntimeStatus::unavailable(R::NotInstalled, error) + } + Some("import") | Some("setup") + if stderr.contains("DLL load failed") || stderr.contains("cannot open shared object") => + { + AiRuntimeStatus::unavailable(R::MissingLibrary, or_status(error)) + } + Some("import") | Some("setup") if error.is_some() => AiRuntimeStatus::unavailable(R::ImportFailed, error), + Some("session") | Some("frames") if error.is_some() => { + AiRuntimeStatus::unavailable(R::SessionFailed, error) + } + Some("import") | Some("session") | Some("frames") => { + AiRuntimeStatus::unavailable(R::Crashed, or_status(error)) + } + _ => AiRuntimeStatus::unavailable(R::ProbeFailed, or_status(error)), + } +} + +/// The last line of a Python error, which names the actual failure. +fn last_error_line(stderr: &str) -> Option { + stderr + .lines() + .map(str::trim) + .filter(|l| !l.is_empty() && !l.starts_with(MARKER) && !l.starts_with(RESULT)) + .filter(|l| l.contains("Error") || l.contains("error") || l.contains("Exception")) + .last() + .map(|l| l.chars().take(400).collect()) +} + +/// Run the probe through vspipe and classify it. +pub fn run_probe( + vspipe: &Path, + env: &HashMap, + templates_dir: &Path, + addon_dir: &Path, + force_cpu: bool, +) -> AiRuntimeStatus { + let tmp = std::env::temp_dir().join(format!("vapourbox_ai_runtime_probe_{}.vpy", std::process::id())); + if let Err(e) = std::fs::write(&tmp, probe_script(templates_dir, addon_dir, force_cpu)) { + return AiRuntimeStatus::unavailable( + AiRuntimeReason::ProbeFailed, + format!("could not write the probe script: {e}"), + ); + } + let outcome = run_vspipe_probe(vspipe, &tmp, env, &["-s", "0", "-e", "0"], PROBE_TIMEOUT); + let _ = std::fs::remove_file(&tmp); + match outcome { + Ok((exit, stderr)) => classify(&exit, &stderr), + Err(e) => AiRuntimeStatus::unavailable(AiRuntimeReason::ProbeFailed, format!("could not run vspipe: {e}")), + } +} + +/// `VAPOURBOX_AI_FORCE_CPU=1`: run on the CPU only (also read by ai_runtime.py). +pub fn force_cpu_requested() -> bool { + std::env::var("VAPOURBOX_AI_FORCE_CPU") + .map(|v| matches!(v.trim().to_ascii_lowercase().as_str(), "1" | "true" | "yes")) + .unwrap_or(false) +} + +#[cfg(test)] +mod tests { + use super::*; + use AiRuntimeReason as R; + + fn exit(success: bool, status: &str) -> ProbeExit { + ProbeExit { success, timed_out: false, status: status.into() } + } + + #[test] + fn a_successful_probe_reports_its_provider() { + let stderr = "AI_RUNTIME_PROBE:start\nAI_RUNTIME_PROBE:session\nAI_RUNTIME_PROBE:frames\n\ + AI_RUNTIME_RESULT:{\"version\": \"1.31.1\", \"providers\": [\"CoreMLExecutionProvider\", \ + \"CPUExecutionProvider\"], \"provider\": \"CoreMLExecutionProvider\", \"fallback\": null}\n"; + let s = classify(&exit(true, "0"), stderr); + assert!(s.available); + assert_eq!(s.provider.as_deref(), Some("CoreMLExecutionProvider")); + assert_eq!(s.version.as_deref(), Some("1.31.1")); + assert_eq!(s.providers.len(), 2); + assert_eq!(s.fallback, None); + assert_eq!(s.message("macos-arm64"), None); + assert!(!classify(&exit(true, "0"), "AI_RUNTIME_PROBE:start\n").available); + } + + #[test] + fn failures_are_classified_by_stage() { + let cases = [ + ("AI_RUNTIME_PROBE:setup\nRuntimeError: The AI runtime add-on is not installed (no x)", R::NotInstalled), + ("AI_RUNTIME_PROBE:import\nImportError: DLL load failed while importing onnxruntime_pybind11_state", R::MissingLibrary), + ("AI_RUNTIME_PROBE:import\nModuleNotFoundError: No module named 'numpy'", R::ImportFailed), + ("AI_RUNTIME_PROBE:session\nonnxruntime.capi.onnxruntime_pybind11_state.Fail: [ONNXRuntimeError]", R::SessionFailed), + ("AI_RUNTIME_PROBE:frames\nRuntimeError: wrong pixels", R::SessionFailed), + ("AI_RUNTIME_PROBE:session\n", R::Crashed), + ("", R::ProbeFailed), + ]; + for (stderr, want) in cases { + assert_eq!(classify(&exit(false, "exit code 1"), stderr).reason, Some(want), "{stderr}"); + } + assert_eq!( + classify(&exit(false, "signal 4 (SIGILL)"), "AI_RUNTIME_PROBE:import\n").reason, + Some(R::UnsupportedCpu) + ); + let t = ProbeExit { success: false, timed_out: true, status: "timed out".into() }; + assert_eq!(classify(&t, "").reason, Some(R::ProbeFailed)); + } + + #[test] + fn every_reason_has_a_message() { + for r in [ + R::UnsupportedPlatform, R::NotInstalled, R::Outdated, R::MissingLibrary, R::ImportFailed, + R::SessionFailed, R::UnsupportedCpu, R::Crashed, R::ProbeFailed, + ] { + for p in addons::PLATFORMS { + let m = AiRuntimeStatus::unavailable(r, Some("d".into())).message(p).unwrap(); + assert!(m.ends_with("Details: d"), "{m}"); + } + } + let m = AiRuntimeStatus::unavailable(R::NotInstalled, None).message("linux-x64").unwrap(); + assert!(m.contains("Settings → Add-ons"), "{m}"); + let j = AiRuntimeStatus::unavailable(R::NotInstalled, None).to_json("linux-x64"); + assert_eq!(j["available"], false); + assert_eq!(j["reason"], "notInstalled"); + assert!(j["message"].as_str().unwrap().contains("not installed")); + } + + #[test] + fn precheck_needs_an_install() { + let root = std::env::temp_dir().join(format!("vb-ai-precheck-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + assert_eq!(precheck(&root, "linux-x64").unwrap().reason, Some(R::NotInstalled)); + assert_eq!(precheck(&root, "windows-arm64").unwrap().reason, Some(R::UnsupportedPlatform)); + } + + #[test] + fn the_probe_script_uses_the_template_helper() { + let s = probe_script(Path::new("/t"), Path::new("/a/ai-runtime"), true); + assert!(s.contains("sys.path.insert(0, '/t')")); + assert!(s.contains("ai_runtime.setup('/a/ai-runtime')")); + assert!(s.contains("self_test(force_cpu=True)")); + assert!(s.contains("frame_test(force_cpu=True)")); + assert!(s.contains(".set_output()")); + } +} diff --git a/worker/src/audio_restoration.rs b/worker/src/audio_restoration.rs new file mode 100644 index 00000000..c674bb3a --- /dev/null +++ b/worker/src/audio_restoration.rs @@ -0,0 +1,1620 @@ +//! Audio restoration: renders the output's audio track before the encode. +//! +//! VapourBox's picture goes through VapourSynth; its sound never did — the +//! encoder copied or re-encoded the source's audio. This pass changes that for +//! one audio stream, and does it **before** the video encode, into a WAV the +//! encoder then maps in place of the source's stream: +//! +//! ```text +//! source ──ffmpeg (restore chain)──► a.wav ──[deep-filter]──► b.wav ──ffmpeg (loudness, limiter)──► restored.wav +//! │ +//! transcribe restored.wav (Whisper) ◄──────────────────────────────────────────────┤ +//! encode: video from vspipe + restored.wav (+ the source's other audio streams) ◄─┘ +//! ``` +//! +//! Rendering first, rather than adding an `-af` to the encode, is what makes +//! the rest possible: DeepFilterNet is a separate program; two-pass loudness +//! normalisation needs the whole track measured before any of it is written; +//! a negative sync offset needs audio from *before* the trim point; and the +//! subtitles transcribe the rendered file, which is the output's soundtrack +//! sample for sample, so their cues line up whatever offset, drift or trim was +//! applied. It also leaves the audio independent of how the video encode is +//! run (a segmented encode can map the same file). +//! +//! **One decision site.** Every filter is chosen by [`AudioPlan::new`], in one +//! fixed order, and each ffmpeg run gets exactly one `-af`, joined from its +//! share of that list — the same rule as the encoder's `-vf` chain, for the +//! same reason (ffmpeg honours only the last `-af`). +//! +//! The order, and why: +//! 1. `aresample=first_pts=0` — pads the start when the audio stream begins +//! after the video does (common in captures). Mapping the source's stream +//! straight into the output kept that gap through its timestamps; a WAV has +//! none, so it must be made of samples. +//! 2. channel fix (`pan`) — before anything measures or filters the channels. +//! 3. `adeclip` — first of the repairs: it reconstructs peaks from the shape +//! of the clipped waveform, which any filter ahead of it would change. +//! 4. `adeclick` — impulses, before the filters below smear them. +//! 5. rumble `highpass`, 6. hum notches (`bandreject` at f, 2f, 3f). +//! 7. `afftdn`, then `atrim` of its delay: it puts out its input 25 ms late +//! at every sample rate and strength (measured: 1200 samples at 48 kHz, +//! 1102 at 44.1, 800 at 32). Its noise floor (`nf`) is the source's, +//! measured: at its −50 dB default it took 0.7 dB off −39 dB hiss, with +//! the measured floor 13.6 dB; and its noise tracking (`tn`) undid that. +//! 8. timing — drift is corrected by resampling (`aresample` up, `asetrate` +//! to relabel by the exact ratio, `aresample` back), not `atempo`: atempo +//! moved every click ~20 ms early, and drift *is* a sample-clock error, +//! so resampling restores the original pitch as well as the length. Then +//! a silent lead (`adelay`) where a positive offset reaches before the +//! start; a negative one just reads later. +//! 9. DeepFilterNet (a separate program, `-D` compensates its 30 ms delay). +//! 10. `loudnorm` (two passes, linear), `aresample` back from its 192 kHz. +//! 11. `alimiter` with `latency=1` (otherwise it delays by its attack time) +//! and `level=0` (otherwise it normalises back up to 0 dBFS). +//! 12. `apad` + `atrim` to exactly the picture's length. + +use std::io::{BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; +use std::sync::atomic::{AtomicI64, Ordering}; +use std::sync::{Arc, Mutex}; +use std::thread; +use std::time::Duration; + +use anyhow::{bail, Context, Result}; + +use crate::dependency_locator::DependencyLocator; +use crate::models::{ + AudioCodec, AudioMode, AudioRestorationParameters, ChannelFix, ContainerFormat, + EncodingSettings, HumFrequency, LogLevel, ProgressInfo, VideoJob, +}; +use crate::progress_reporter::ProgressReporter; + +/// The feature name the DeepFilterNet add-on refusal starts with. +pub const DEEP_FILTER_FEATURE: &str = "Speech enhancement (DeepFilterNet)"; + +/// DeepFilterNet works at 48 kHz and writes 48 kHz whatever it is given. +pub const DEEP_FILTER_RATE: u32 = 48_000; + +/// `afftdn`'s output delay, as a fraction of a second (measured, see above). +pub const AFFTDN_DELAY_SECONDS: f64 = 0.025; + +/// Drift smaller than this is left alone: under two frames, and within the +/// accuracy of a container's stated durations. +pub const DRIFT_MIN_SECONDS: f64 = 0.05; + +/// A drift factor this close to 1 is not applied: the resample would still +/// run, for nothing. +pub const TEMPO_EPSILON: f64 = 1e-6; + +/// Loudness normalisation's true-peak ceiling and loudness range: the +/// widely used streaming values. Only the integrated target is a setting. +pub const LOUDNORM_TRUE_PEAK: f64 = -1.5; +pub const LOUDNORM_RANGE: f64 = 11.0; + +/// How much audio the automatic checks (channels, hum, noise floor) look at, +/// and the highest rate they decode it at (the noise floor needs the whole +/// band: white hiss is 8 dB quieter below 4 kHz than across 24). +pub const ANALYSIS_SECONDS: f64 = 60.0; +pub const ANALYSIS_MAX_RATE: u32 = 48_000; + +/// `afftdn`'s noise floor is the quietest tenth of 50 ms windows, clamped to +/// what it accepts (-80) and to -30: a "floor" louder than that is the +/// content itself (music with no pauses), which afftdn would eat. +pub const NOISE_FLOOR_PERCENTILE: f64 = 0.10; +pub const NOISE_FLOOR_RANGE: (f64, f64) = (-80.0, -30.0); + +/// Extra source audio read past the end, so the filters that trim their own +/// delay off the front still fill the window. +const READ_MARGIN_SECONDS: f64 = 1.0; + +/// Notch width (Q) for the hum filters: about 6 Hz wide at 50 Hz. +const HUM_Q: f64 = 8.0; + +// --------------------------------------------------------------------------- +// What is known about the source and the output +// --------------------------------------------------------------------------- + +/// The source's audio, as probed. +#[derive(Debug, Clone, PartialEq)] +pub struct SourceAudio { + /// How many audio streams the source has. + pub streams: u32, + /// The restored stream's channel count and sample rate. + pub channels: u32, + pub sample_rate: u32, + /// The restored stream's duration in seconds, when stated or measured. + pub duration: Option, +} + +/// The span of the output on the source's timeline: where the picture starts +/// (the trim's in-point) and how long the encoded picture is. +#[derive(Debug, Clone, Copy, PartialEq)] +pub struct OutputWindow { + pub start: f64, + /// `None` when the length is unknown: the audio then runs to its end. + pub duration: Option, +} + +impl OutputWindow { + /// The window a job's picture covers: from its in-point, as long as the + /// encoded video (its output frame count over its output rate, from the + /// pipeline's `FrameMap`s — not the source's count, which double-rate + /// deinterlacing, IVTC and frame-rate conversion all change). + pub fn for_job(job: &VideoJob) -> Self { + let fps = job.input_frame_rate.unwrap_or(29.97); + let start = job.start_frame.filter(|s| *s > 0).map_or(0.0, |s| s as f64 / fps); + let duration = job + .total_frames + .filter(|n| *n > 0) + .and_then(|n| job.effective_pipeline().output_duration_secs(n as u64, fps)); + Self { start, duration } + } +} + +/// Which channel of a stereo pair is dead. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DeadChannel { + Left, + Right, +} + +/// What the automatic checks found, on a sample of the window. +#[derive(Debug, Clone, Copy, PartialEq, Default)] +pub struct Analysis { + pub dead_channel: Option, + /// 50 or 60 when a mains hum stands out of the spectrum. + pub hum: Option, + /// The noise floor in dBFS (for `afftdn`), when measured. + pub noise_floor_db: Option, +} + + + +/// A channel is dead when it is this far below the other… +pub const DEAD_CHANNEL_GAP_DB: f64 = 30.0; +/// …and the other carries something. +pub const LIVE_CHANNEL_MIN_DB: f64 = -60.0; + +/// Which channel to replace, from each channel's RMS level in dBFS. +pub fn dead_channel(left_db: f64, right_db: f64) -> Option { + let (loud, quiet) = if left_db >= right_db { (left_db, right_db) } else { (right_db, left_db) }; + if loud < LIVE_CHANNEL_MIN_DB || loud - quiet < DEAD_CHANNEL_GAP_DB { + return None; + } + Some(if left_db < right_db { DeadChannel::Left } else { DeadChannel::Right }) +} + +/// RMS level in dBFS of each channel of interleaved samples. +pub fn channel_levels_db(interleaved: &[f32], channels: usize) -> Vec { + let channels = channels.max(1); + let frames = interleaved.len() / channels; + (0..channels) + .map(|c| { + let sum: f64 = (0..frames).map(|i| (interleaved[i * channels + c] as f64).powi(2)).sum(); + let rms = (sum / frames.max(1) as f64).sqrt(); + 20.0 * (rms + 1e-12).log10() + }) + .collect() +} + +/// The noise floor of `mono` in dBFS: the [`NOISE_FLOOR_PERCENTILE`] quietest +/// 50 ms window's RMS level, clamped to [`NOISE_FLOOR_RANGE`] (digital +/// silence reads as the bottom of the range). +pub fn noise_floor_db(mono: &[f32], rate: u32) -> f64 { + let win = (rate as usize / 20).max(1); + let mut levels: Vec = mono + .chunks_exact(win) + .map(|w| { + let rms = (w.iter().map(|v| (*v as f64).powi(2)).sum::() / win as f64).sqrt(); + 20.0 * (rms + 1e-12).log10() + }) + .collect(); + if levels.is_empty() { + return NOISE_FLOOR_RANGE.0; + } + levels.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let idx = ((levels.len() - 1) as f64 * NOISE_FLOOR_PERCENTILE).round() as usize; + levels[idx].clamp(NOISE_FLOOR_RANGE.0, NOISE_FLOOR_RANGE.1) +} + +/// The integer rates that resample by `factor` most exactly: upsample to +/// `r1`, relabel as `s` (`asetrate` takes only an integer), resample back. +/// `r1` is searched between 4× and 8× the rate, so the content is never +/// low-passed and the ratio `s / r1` is within ~1e-11 of `factor` — an +/// integer `asetrate` straight from 48 kHz would be off by up to 1e-5, 37 ms +/// over an hour. +pub fn resample_ratio(rate: u32, factor: f64) -> (u32, u32) { + let mut best = (rate, (rate as f64 * factor).round() as u32); + let mut best_err = f64::MAX; + for r1 in rate * 4..=rate * 8 { + let s = (r1 as f64 * factor).round(); + let err = (s / r1 as f64 - factor).abs(); + if err < best_err { + best_err = err; + best = (r1, s as u32); + if err < 1e-13 { + break; + } + } + } + best +} + +/// Power at `freq` by the Goertzel algorithm, averaged over 2-second blocks +/// (0.5 Hz resolution: mains frequency wanders by a tenth of a hertz, which a +/// single minute-long transform would resolve right past). +pub fn band_power(x: &[f32], rate: u32, freq: f64) -> f64 { + let block = (rate as usize) * 2; + if x.len() < block { + return goertzel(x, rate, freq); + } + let blocks: Vec = x.chunks_exact(block).map(|b| goertzel(b, rate, freq)).collect(); + blocks.iter().sum::() / blocks.len() as f64 +} + +fn goertzel(x: &[f32], rate: u32, freq: f64) -> f64 { + let n = x.len().max(1) as f64; + let w = 2.0 * std::f64::consts::PI * freq / rate as f64; + let coeff = 2.0 * w.cos(); + let (mut s1, mut s2) = (0.0f64, 0.0f64); + for (i, &v) in x.iter().enumerate() { + // Hann window, so a strong neighbour does not leak into the bin. + let win = 0.5 - 0.5 * (2.0 * std::f64::consts::PI * i as f64 / (n - 1.0).max(1.0)).cos(); + let s = v as f64 * win + coeff * s1 - s2; + s2 = s1; + s1 = s; + } + (s1 * s1 + s2 * s2 - coeff * s1 * s2) / (n * n) +} + +/// How far a hum line must stand above its neighbours, in dB. +pub const HUM_PROMINENCE_DB: f64 = 12.0; + +/// How far (dB) the strongest of `f`, 2f and 3f stands above the spectrum +/// either side of it. +pub fn hum_prominence_db(mono: &[f32], rate: u32, mains: f64) -> f64 { + (1..=3) + .map(|k| { + let f = mains * k as f64; + let peak = band_power(mono, rate, f); + // Neighbours 4–7 Hz away: outside the notch, inside the band, and + // clear of the other mains frequency's harmonics. + let mut refs: Vec = + [-7.0, -5.5, -4.0, 4.0, 5.5, 7.0].iter().map(|d| band_power(mono, rate, f + d)).collect(); + refs.sort_by(|a, b| a.partial_cmp(b).unwrap()); + let floor = (refs[2] + refs[3]) / 2.0; + 10.0 * ((peak + 1e-20) / (floor + 1e-20)).log10() + }) + .fold(f64::MIN, f64::max) +} + +/// 50, 60 or nothing: whichever mains frequency's lines stand out more, if +/// either stands out by [`HUM_PROMINENCE_DB`]. +pub fn detect_hum(mono: &[f32], rate: u32) -> Option { + let p50 = hum_prominence_db(mono, rate, 50.0); + let p60 = hum_prominence_db(mono, rate, 60.0); + let (best, p) = if p50 >= p60 { (50, p50) } else { (60, p60) }; + (p >= HUM_PROMINENCE_DB).then_some(best) +} + +/// What drift correction will do. +#[derive(Debug, Clone, PartialEq)] +pub enum DriftDecision { + /// Resample by this factor, audio length / picture length. + Apply(f64), + /// Leave it, with the reason for the job log. + Skip(String), +} + +/// Whether to correct drift between an audio track of `audio_secs` and a +/// picture of `video_secs`, refusing anything above `max_percent`. +pub fn drift_decision(audio_secs: f64, video_secs: f64, max_percent: f64) -> DriftDecision { + if !(audio_secs > 0.0 && video_secs > 0.0) { + return DriftDecision::Skip("the audio or picture length could not be measured".into()); + } + let drift = audio_secs - video_secs; + let factor = audio_secs / video_secs; + let percent = (factor - 1.0).abs() * 100.0; + if drift.abs() <= DRIFT_MIN_SECONDS { + return DriftDecision::Skip(format!( + "the audio is {:.3} s against the picture's {:.3} s — within {} s, nothing to correct", + audio_secs, video_secs, DRIFT_MIN_SECONDS + )); + } + if percent > max_percent { + return DriftDecision::Skip(format!( + "the audio ({:.3} s) and picture ({:.3} s) differ by {:.2}%, more than the {:.2}% \ + limit — that is not clock drift (a missing stretch of audio, or the wrong track?), \ + so it is left uncorrected", + audio_secs, video_secs, percent, max_percent + )); + } + if (factor - 1.0).abs() < TEMPO_EPSILON { + return DriftDecision::Skip("the change would be below 1e-6".into()); + } + DriftDecision::Apply(factor) +} + +// --------------------------------------------------------------------------- +// The plan: every decision, as pure data +// --------------------------------------------------------------------------- + +/// Loudness as measured by `loudnorm`'s first pass. +#[derive(Debug, Clone, PartialEq)] +pub struct LoudnessMeasurement { + pub input_i: f64, + pub input_tp: f64, + pub input_lra: f64, + pub input_thresh: f64, + pub target_offset: f64, +} + +/// Every decision about one job's audio, made up front. +#[derive(Debug, Clone, PartialEq)] +pub struct AudioPlan { + /// The source audio stream (among audio streams) restored. + pub stream: u32, + pub source_rate: u32, + /// Where to start reading the source (seconds, `-ss` before `-i`). + pub read_start: f64, + /// How much to read (`-t`), when the window's length is known. + pub read_duration: Option, + /// The first ffmpeg run's filters, in order. + pub restore_filters: Vec, + /// DeepFilterNet's attenuation limit, when it runs. + pub deep_filter: Option, + /// The loudness target, when normalising. + pub loudness_target: Option, + /// The filters after DeepFilterNet / loudness (limiter, length). + pub finish_filters: Vec, + /// The rendered track's sample rate and exact length in samples. + pub out_rate: u32, + pub out_samples: Option, + /// What was decided and why, for the job log. + pub notes: Vec, +} + +impl AudioPlan { + /// Decide everything. `drift` is the factor to apply, already + /// vetted by [`drift_decision`]. + pub fn new( + p: &AudioRestorationParameters, + source: &SourceAudio, + window: OutputWindow, + analysis: &Analysis, + drift: Option, + ) -> Self { + let rate = source.sample_rate.max(1); + let mut notes = Vec::new(); + let mut f: Vec = Vec::new(); + + // 1. A stream that starts late is padded from the window's start. + f.push("aresample=first_pts=0".to_string()); + + // 2. Channels. + let fix = match p.channel_fix { + ChannelFix::Auto => match (source.channels, analysis.dead_channel) { + (2, Some(DeadChannel::Right)) => { + notes.push("Channel fix: the right channel is silent; using the left for both.".into()); + ChannelFix::LeftOnly + } + (2, Some(DeadChannel::Left)) => { + notes.push("Channel fix: the left channel is silent; using the right for both.".into()); + ChannelFix::RightOnly + } + (2, None) => { + notes.push("Channel fix: both channels carry sound; left as they are.".into()); + ChannelFix::Off + } + _ => ChannelFix::Off, + }, + other => other, + }; + if fix != ChannelFix::Off && source.channels != 2 { + notes.push(format!( + "Channel fix applies to stereo audio; this stream has {} channel(s), so it is skipped.", + source.channels + )); + } else if let Some(pan) = pan_filter(fix) { + f.push(pan.to_string()); + } + + // 3–7. Repairs. + if p.declip { + f.push("adeclip".to_string()); + } + if p.declick { + f.push(format!("adeclick=t={}", num(p.declick_threshold.clamp(1.0, 100.0)))); + } + if p.rumble_filter { + f.push(format!("highpass=f={}:poles=2", p.rumble_cutoff_hz.clamp(10, 300))); + } + if p.hum_removal { + let mains = match p.hum_frequency { + HumFrequency::Hz50 => Some(50), + HumFrequency::Hz60 => Some(60), + HumFrequency::Auto => { + match analysis.hum { + Some(hz) => notes.push(format!("Hum removal: found {hz} Hz mains hum.")), + None => notes.push( + "Hum removal: no 50 or 60 Hz hum stands out, so nothing is notched.".into(), + ), + } + analysis.hum + } + }; + if let Some(hz) = mains { + for k in 1..=3 { + f.push(format!("bandreject=f={}:width_type=q:w={}", hz * k, num(HUM_Q))); + } + } + } + if p.denoise { + let floor = analysis.noise_floor_db.unwrap_or(-50.0).round(); + f.push(format!("afftdn=nr={}:nf={}", num(p.denoise_strength.clamp(0.01, 97.0)), num(floor))); + notes.push(format!("Noise reduction: noise floor {} dB.", num(floor))); + let delay = (rate as f64 * AFFTDN_DELAY_SECONDS).round() as u64; + f.push(format!("atrim=start_sample={delay}")); + f.push("asetpts=PTS-STARTPTS".to_string()); + } + + // 8. Timing. Source time for output time t is (start + t - offset) × factor. + let tempo = drift.filter(|t| (t - 1.0).abs() >= TEMPO_EPSILON); + let factor = tempo.unwrap_or(1.0); + let offset = p.offset_ms as f64 / 1000.0; + let source_start = (window.start - offset) * factor; + let (read_start, lead) = if source_start >= 0.0 { + (source_start, 0.0) + } else { + // A delay reaching before the source's start: silence first. + (0.0, -source_start / factor) + }; + if let Some(t) = tempo { + let (r1, s) = resample_ratio(rate, t); + f.push(format!("aresample={r1}")); + f.push(format!("asetrate={s}")); + f.push(format!("aresample={rate}")); + notes.push(format!( + "Drift correction: the sound is {:.3}% {} than the picture; resampled to fit.", + (t - 1.0).abs() * 100.0, + if t > 1.0 { "longer" } else { "shorter" } + )); + } + let lead_samples = (lead * rate as f64).round() as u64; + if lead_samples > 0 { + f.push(format!("adelay=delays={lead_samples}S:all=1")); + } + if p.offset_ms != 0 { + notes.push(format!( + "Sync offset: sound {} by {} ms.", + if p.offset_ms > 0 { "delayed" } else { "advanced" }, + p.offset_ms.abs() + )); + } + let read_duration = window.duration.map(|d| { + (d - lead).max(0.0) * factor + READ_MARGIN_SECONDS + if p.denoise { AFFTDN_DELAY_SECONDS } else { 0.0 } + }); + + // 9–12. + let deep_filter = p.deep_filter.then(|| p.deep_filter_attenuation.clamp(1, 100)); + let loudness_target = p.loudness_normalize.then(|| p.loudness_target.clamp(-70.0, -5.0)); + let out_rate = if deep_filter.is_some() { DEEP_FILTER_RATE } else { rate }; + let out_samples = window.duration.map(|d| (d * out_rate as f64).round() as u64); + let mut finish = Vec::new(); + if p.limiter { + let limit = 10f64.powf(p.limiter_ceiling_db.clamp(-24.0, 0.0) / 20.0).clamp(0.0625, 1.0); + finish.push(format!("alimiter=limit={}:level=0:latency=1", num(limit))); + } + if let Some(n) = out_samples { + finish.extend(length_filters(n)); + } + // With nothing after the first run, it writes the final track itself. + if deep_filter.is_none() && loudness_target.is_none() { + f.append(&mut finish); + } else if let Some(d) = window.duration { + // The input to DeepFilterNet / loudness already has its length. + f.extend(length_filters((d * rate as f64).round() as u64)); + } + + Self { + stream: p.stream_index, + source_rate: rate, + read_start, + read_duration, + restore_filters: f, + deep_filter, + loudness_target, + finish_filters: finish, + out_rate, + out_samples, + notes, + } + } + + /// Whether the first ffmpeg run writes the finished track. + pub fn single_pass(&self) -> bool { + self.deep_filter.is_none() && self.loudness_target.is_none() + } + + /// The first run: the source's chosen stream through the restore chain. + pub fn restore_args(&self, source: &str, output: &Path) -> Vec { + let mut args = quiet_args(); + if self.read_start > 0.0 { + args.extend(["-ss".into(), format!("{:.6}", self.read_start)]); + } + if let Some(t) = self.read_duration { + args.extend(["-t".into(), format!("{:.6}", t)]); + } + args.extend(["-i".into(), source.to_string()]); + args.extend(["-map".into(), format!("0:a:{}", self.stream)]); + args.extend(["-af".into(), self.restore_filters.join(",")]); + args.extend(wav_output_args(output)); + args + } + + /// DeepFilterNet on the first run's output, into `out_dir` (it keeps the + /// input's file name). + pub fn deep_filter_args(&self, input: &Path, out_dir: &Path) -> Option> { + let atten = self.deep_filter?; + Some(vec![ + // Compensate its STFT + lookahead delay (30 ms, measured). The + // track comes out that much short at the end; `finish` pads it. + "-D".into(), + "-a".into(), + atten.to_string(), + "-o".into(), + out_dir.to_string_lossy().to_string(), + input.to_string_lossy().to_string(), + ]) + } + + /// `loudnorm`'s measuring pass over `input`; its JSON is on stderr. + pub fn measure_args(&self, input: &Path) -> Option> { + let target = self.loudness_target?; + let mut args = quiet_args(); + args.extend(["-i".into(), input.to_string_lossy().to_string()]); + args.extend([ + "-af".into(), + format!( + "loudnorm=I={}:TP={}:LRA={}:print_format=json", + num(target), + num(LOUDNORM_TRUE_PEAK), + num(LOUDNORM_RANGE) + ), + ]); + args.extend(["-f".into(), "null".into(), "-".into()]); + Some(args) + } + + /// The last run, after DeepFilterNet or for loudness: loudness (with the + /// first pass's measurement), back to the track's rate, limiter, length. + pub fn finish_filter_chain(&self, measured: Option<&LoudnessMeasurement>) -> String { + let mut f = Vec::new(); + if let (Some(target), Some(m)) = (self.loudness_target, measured) { + f.push(format!( + "loudnorm=I={}:TP={}:LRA={}:measured_I={}:measured_TP={}:measured_LRA={}:\ + measured_thresh={}:offset={}:linear=true:print_format=summary", + num(target), + num(LOUDNORM_TRUE_PEAK), + num(LOUDNORM_RANGE), + num(m.input_i), + num(m.input_tp), + num(m.input_lra), + num(m.input_thresh), + num(m.target_offset) + )); + } + // loudnorm resamples to 192 kHz; DeepFilterNet's output is already 48. + f.push(format!("aresample={}", self.out_rate)); + f.extend(self.finish_filters.iter().cloned()); + f.join(",") + } + + pub fn finish_args(&self, input: &Path, output: &Path, measured: Option<&LoudnessMeasurement>) -> Vec { + let mut args = quiet_args(); + args.extend(["-i".into(), input.to_string_lossy().to_string()]); + args.extend(["-af".into(), self.finish_filter_chain(measured)]); + args.extend(wav_output_args(output)); + args + } +} + +fn quiet_args() -> Vec { + ["-hide_banner", "-nostdin", "-nostats", "-y"].iter().map(|s| s.to_string()).collect() +} + +/// Intermediate and final tracks are 32-bit float: `adeclip` rebuilds peaks +/// above full scale, and loudness gain comes later, so nothing may clip in +/// between. RF64 past 4 GB (about 3.5 hours of float stereo at 48 kHz). +fn wav_output_args(output: &Path) -> Vec { + [ + "-vn", "-sn", "-dn", "-map_metadata", "-1", "-c:a", "pcm_f32le", "-rf64", "auto", "-f", "wav", + ] + .iter() + .map(|s| s.to_string()) + .chain(std::iter::once(output.to_string_lossy().to_string())) + .collect() +} + +/// Exactly `n` samples: silence added if short, the rest cut if long. +fn length_filters(n: u64) -> [String; 2] { + [format!("apad=whole_len={n}"), format!("atrim=end_sample={n}")] +} + +fn pan_filter(fix: ChannelFix) -> Option<&'static str> { + match fix { + ChannelFix::LeftOnly => Some("pan=stereo|c0=c0|c1=c0"), + ChannelFix::RightOnly => Some("pan=stereo|c0=c1|c1=c1"), + ChannelFix::Mix => Some("pan=stereo|c0=0.5*c0+0.5*c1|c1=0.5*c0+0.5*c1"), + ChannelFix::Off | ChannelFix::Auto => None, + } +} + +/// A number for a filter argument: no trailing zeros, no exponent. +fn num(v: f64) -> String { + let s = format!("{:.4}", v); + let s = s.trim_end_matches('0').trim_end_matches('.'); + if s == "-0" { "0".into() } else { s.to_string() } +} + +/// A tempo factor: drift of 0.01% needs more than four places. +fn num_precise(v: f64) -> String { + let s = format!("{:.9}", v); + s.trim_end_matches('0').trim_end_matches('.').to_string() +} + +/// Parse `loudnorm`'s `print_format=json` block from ffmpeg's stderr. +pub fn parse_loudnorm_json(stderr: &str) -> Option { + let start = stderr.rfind("\"input_i\"").and_then(|i| stderr[..i].rfind('{'))?; + let end = start + stderr[start..].find('}')?; + let v: serde_json::Value = serde_json::from_str(&stderr[start..=end]).ok()?; + let get = |k: &str| -> Option { + let s = v.get(k)?.as_str()?.trim(); + s.parse::().ok().filter(|x| x.is_finite()) + }; + Some(LoudnessMeasurement { + input_i: get("input_i")?, + input_tp: get("input_tp")?, + input_lra: get("input_lra")?, + input_thresh: get("input_thresh")?, + target_offset: get("target_offset")?, + }) +} + +// --------------------------------------------------------------------------- +// The encoder's side: mapping the rendered track +// --------------------------------------------------------------------------- + +/// The rendered track as the encoder needs it. +#[derive(Debug, Clone, PartialEq)] +pub struct RestoredTrack { + pub path: PathBuf, + /// Which source audio stream it replaces, and how many there are. + pub stream: u32, + pub source_streams: u32, +} + +/// The encoder's input for the rendered track (comes after its other inputs). +pub fn encode_input_args(track: Option<&RestoredTrack>) -> Vec { + match track { + Some(t) => vec!["-i".into(), t.path.to_string_lossy().to_string()], + None => Vec::new(), + } +} + +/// The encoder's audio `-map`s. Without a rendered track, every source audio +/// stream (`1:a?`) as always. With one, the rendered track first — it is +/// the one a player picks by default — then the source's other audio streams +/// untouched, in their order, with the restored stream's tags (language, +/// title) carried over to the rendered one. +pub fn encode_map_args(settings: &EncodingSettings, track: Option<&RestoredTrack>, track_input: usize) -> Vec { + if settings.audio_mode == AudioMode::None { + return Vec::new(); + } + let Some(t) = track else { + return vec!["-map".into(), "1:a?".into()]; + }; + let mut args = vec!["-map".into(), format!("{track_input}:a:0")]; + let others: Vec = (0..t.source_streams).filter(|k| *k != t.stream).collect(); + for k in &others { + args.extend(["-map".into(), format!("1:a:{k}")]); + } + // Tags per output stream. Naming one stream's switches off ffmpeg's own + // copying for the rest, so every audio stream is named. + args.extend(["-map_metadata:s:a:0".into(), format!("1:s:a:{}", t.stream)]); + for (j, k) in others.iter().enumerate() { + args.extend([format!("-map_metadata:s:a:{}", j + 1), format!("1:s:a:{k}")]); + } + args +} + +/// The encoder's audio codec arguments. Passthrough copies — except the +/// rendered track, which is new audio and must be encoded: FLAC in MKV, AAC +/// at the chosen quality in MP4/MOV, PCM in AVI. +pub fn encode_codec_args(settings: &EncodingSettings, track: Option<&RestoredTrack>) -> Vec { + let mut args = Vec::new(); + match settings.audio_mode { + AudioMode::Passthrough => { + args.extend(["-c:a".into(), "copy".into()]); + if track.is_some() { + let codec = restored_passthrough_codec(settings.container); + args.extend(["-c:a:0".into(), codec.ffmpeg_name().into()]); + if !codec.is_lossless() { + args.extend(["-b:a:0".into(), format!("{}k", settings.audio_quality.bitrate())]); + } + } + } + AudioMode::Convert => { + args.extend(["-c:a".into(), settings.audio_codec.ffmpeg_name().into()]); + if !settings.audio_codec.is_lossless() { + args.extend(["-b:a".into(), format!("{}k", settings.audio_quality.bitrate())]); + } + } + AudioMode::None => args.push("-an".into()), + } + args +} + +/// What a restored track is encoded as when the audio setting is Passthrough. +pub fn restored_passthrough_codec(container: ContainerFormat) -> AudioCodec { + match container { + ContainerFormat::Mkv => AudioCodec::Flac, + ContainerFormat::Mp4 | ContainerFormat::Mov => AudioCodec::Aac, + ContainerFormat::Avi => AudioCodec::Pcm, + } +} + +// --------------------------------------------------------------------------- +// Running it +// --------------------------------------------------------------------------- + +/// A finished render: the track, and the scratch directory it lives in, +/// removed when this is dropped (after the encode and the subtitle mux). +pub struct RenderedAudio { + pub track: RestoredTrack, + work: PathBuf, +} + +impl Drop for RenderedAudio { + fn drop(&mut self) { + let _ = std::fs::remove_dir_all(&self.work); + } +} + +/// The DeepFilterNet add-on's program for `platform` (its upstream file +/// name, which differs per platform; see its manifest). +pub fn deep_filter_binary(addons_root: &Path, platform: &str) -> Result { + let name = crate::addons::manifest(crate::addons::DEEPFILTERNET) + .and_then(|m| m.executable_for(platform)) + .with_context(|| format!("DeepFilterNet has no program for {platform}"))?; + Ok(crate::addons::addon_dir(addons_root, crate::addons::DEEPFILTERNET).join(name)) +} + +/// Probes the source, plans and renders. Owns nothing but its helpers. +pub struct AudioRestorer { + reporter: ProgressReporter, + deps: DependencyLocator, +} + +impl AudioRestorer { + pub fn new(reporter: ProgressReporter, deps: DependencyLocator) -> Self { + Self { reporter, deps } + } + + /// Plan the job's audio without rendering it (`--audio-plan`), or `None` + /// when the pass does nothing for this job (with the reason logged). + pub fn plan(&self, job: &VideoJob) -> Result> { + let p = job.effective_pipeline().audio; + if !p.has_effect() { + return Ok(None); + } + if p.uses_deep_filter() { + // Refused before anything runs, with the add-on refusal every + // feature gives (the app offers the download when it is chosen). + self.deps.require_addons(&[crate::addons::DEEPFILTERNET], DEEP_FILTER_FEATURE)?; + } + if job.encoding_settings.audio_mode == AudioMode::None { + self.log(LogLevel::Warning, "Audio restoration is on, but the output has no audio (Output → Audio is off); skipping it."); + return Ok(None); + } + let Some(mut source) = self.probe_source(&job.input_path, p.stream_index)? else { + self.log(LogLevel::Warning, "Audio restoration: the source has no audio stream; skipping it."); + return Ok(None); + }; + let mut p = p; + if p.stream_index >= source.streams { + self.log( + LogLevel::Warning, + &format!( + "Audio restoration: the source has {} audio stream(s), so stream {} does not exist; restoring the first.", + source.streams, p.stream_index + 1 + ), + ); + p.stream_index = 0; + source = self.probe_source(&job.input_path, 0)?.context("audio stream vanished")?; + } + let window = OutputWindow::for_job(job); + + let needs_analysis = p.channel_fix == ChannelFix::Auto + || (p.hum_removal && p.hum_frequency == HumFrequency::Auto) + || p.denoise; + let analysis = if needs_analysis { + self.analyse(&job.input_path, &p, &source, window).unwrap_or_else(|e| { + self.log(LogLevel::Warning, &format!("Audio analysis failed ({e}); automatic choices are off.")); + Analysis::default() + }) + } else { + Analysis::default() + }; + + let drift = if p.drift_correction { + match self.drift_factor(job, &source, p.drift_max_percent) { + DriftDecision::Apply(f) => Some(f), + DriftDecision::Skip(why) => { + self.log(LogLevel::Info, &format!("Drift correction: {why}.")); + None + } + } + } else { + None + }; + let plan = AudioPlan::new(&p, &source, window, &analysis, drift); + Ok(Some((plan, source))) + } + + /// Render the job's restored audio track, or `None` when the pass does + /// nothing for this job. + pub fn render bool>(&self, job: &VideoJob, on_cancel: F) -> Result> { + let Some((plan, source)) = self.plan(job)? else { return Ok(None) }; + for note in &plan.notes { + self.log(LogLevel::Info, note); + } + let ffmpeg = self.deps.ffmpeg_path()?; + let work = std::env::temp_dir().join(format!("vb_audio_{}", job.id)); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).context("Failed to create the audio work directory")?; + // From here the directory is removed on any early return. + let rendered = RenderedAudio { + track: RestoredTrack { path: work.join("restored.wav"), stream: plan.stream, source_streams: source.streams }, + work: work.clone(), + }; + let total = plan.out_samples.map(|n| n as f64 / plan.out_rate as f64).or(source.duration); + + self.log(LogLevel::Info, "Restoring audio..."); + let first = if plan.single_pass() { rendered.track.path.clone() } else { work.join("restore.wav") }; + self.run_ffmpeg(&ffmpeg, &plan.restore_args(&job.input_path, &first), total, &on_cancel) + .context("Audio restoration failed")?; + + let mut current = first; + if let Some(args) = plan.deep_filter_args(¤t, &work.join("dfn")) { + self.log(LogLevel::Info, "Running DeepFilterNet speech enhancement..."); + let bin = deep_filter_binary(&self.deps.addons_dir(), self.deps.platform_suffix())?; + self.run_quiet(&bin, &args, &on_cancel).context("DeepFilterNet failed")?; + current = work.join("dfn").join(current.file_name().unwrap()); + if !current.exists() { + bail!("DeepFilterNet wrote no output at {}", current.display()); + } + } + if !plan.single_pass() { + let measured = match plan.measure_args(¤t) { + Some(args) => { + self.log(LogLevel::Info, "Measuring loudness..."); + let stderr = self.run_ffmpeg(&ffmpeg, &args, total, &on_cancel)?; + let m = parse_loudnorm_json(&stderr); + if m.is_none() { + self.log(LogLevel::Warning, "Loudness could not be measured (silent audio?); not normalising it."); + } + m + } + None => None, + }; + if let Some(m) = &measured { + self.log(LogLevel::Info, &format!( + "Loudness: measured {:.1} LUFS, true peak {:.1} dBTP; normalising to {} LUFS.", + m.input_i, m.input_tp, num(plan.loudness_target.unwrap_or_default()) + )); + } + let args = plan.finish_args(¤t, &rendered.track.path, measured.as_ref()); + self.run_ffmpeg(&ffmpeg, &args, total, &on_cancel).context("Audio finishing failed")?; + } + if on_cancel() { + bail!("Job cancelled"); + } + Ok(Some(rendered)) + } + + fn log(&self, level: LogLevel, msg: &str) { + self.reporter.send_log(level, msg); + } + + /// The number of audio streams, and the chosen one's details. + fn probe_source(&self, input: &str, stream: u32) -> Result> { + let ffprobe = self.deps.ffprobe_path()?; + let out = Command::new(&ffprobe) + .args(["-v", "error", "-select_streams", "a", "-show_entries", + "stream=channels,sample_rate,duration:stream_tags=DURATION", "-of", "json", input]) + .envs(self.deps.build_environment()) + .output() + .context("Failed to run ffprobe on the audio")?; + Ok(parse_source_probe(&String::from_utf8_lossy(&out.stdout), stream)) + } + + /// Decode a minute from the middle of the window at 8 kHz and measure it. + fn analyse(&self, input: &str, p: &AudioRestorationParameters, source: &SourceAudio, window: OutputWindow) -> Result { + let ffmpeg = self.deps.ffmpeg_path()?; + let start = window.start + window.duration.map_or(0.0, |d| (d / 2.0 - ANALYSIS_SECONDS / 2.0).max(0.0)); + let rate = source.sample_rate.clamp(8000, ANALYSIS_MAX_RATE); + let out = Command::new(&ffmpeg) + .args(["-v", "error", "-nostdin", "-ss", &format!("{start:.3}"), "-t", &format!("{ANALYSIS_SECONDS}"), + "-i", input, "-map", &format!("0:a:{}", p.stream_index), + "-ar", &rate.to_string(), "-f", "f32le", "-"]) + .envs(self.deps.build_environment()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .context("Failed to decode audio for analysis")?; + let samples: Vec = out.stdout.chunks_exact(4).map(|b| f32::from_le_bytes([b[0], b[1], b[2], b[3]])).collect(); + let ch = source.channels.max(1) as usize; + let mut a = Analysis::default(); + if ch == 2 { + let levels = channel_levels_db(&samples, 2); + a.dead_channel = dead_channel(levels[0], levels[1]); + self.log(LogLevel::Debug, &format!("Audio analysis: channel levels L {:.1} dB, R {:.1} dB", levels[0], levels[1])); + } + let mono: Vec = samples.chunks_exact(ch).map(|f| f.iter().sum::() / ch as f32).collect(); + if p.hum_removal && p.hum_frequency == HumFrequency::Auto { + a.hum = detect_hum(&mono, rate); + self.log(LogLevel::Debug, &format!( + "Audio analysis: hum prominence 50 Hz {:.1} dB, 60 Hz {:.1} dB", + hum_prominence_db(&mono, rate, 50.0), hum_prominence_db(&mono, rate, 60.0) + )); + } + if p.denoise { + a.noise_floor_db = Some(noise_floor_db(&mono, rate)); + } + Ok(a) + } + + /// The whole source's audio length over the whole encoded picture's. + fn drift_factor(&self, job: &VideoJob, source: &SourceAudio, max_percent: f64) -> DriftDecision { + let fps = job.input_frame_rate.unwrap_or(29.97); + let trimmed = job.start_frame.is_some() || job.end_frame.is_some(); + // Drift is a property of the recording, so it is measured over all of + // it: a trimmed window's audio is cut to the window by construction. + let frames = if trimmed { self.deps.probe_frame_count(&job.input_path) } else { job.total_frames }; + let video = frames + .filter(|n| *n > 0) + .and_then(|n| job.effective_pipeline().output_duration_secs(n as u64, fps)); + let audio = source.duration.or_else(|| self.measure_audio_duration(&job.input_path, job.effective_pipeline().audio.stream_index)); + match (audio, video) { + (Some(a), Some(v)) => drift_decision(a, v, max_percent), + _ => DriftDecision::Skip("the audio or picture length could not be measured".into()), + } + } + + /// Decode the stream to nothing and read how long it was. + fn measure_audio_duration(&self, input: &str, stream: u32) -> Option { + let ffmpeg = self.deps.ffmpeg_path().ok()?; + let out = Command::new(&ffmpeg) + .args(["-nostdin", "-v", "error", "-i", input, "-map", &format!("0:a:{stream}"), + "-progress", "pipe:1", "-f", "null", "-"]) + .envs(self.deps.build_environment()) + .output() + .ok()?; + String::from_utf8_lossy(&out.stdout) + .lines() + .filter_map(|l| l.strip_prefix("out_time_us=")) + .filter_map(|v| v.trim().parse::().ok()) + .last() + .map(|us| us as f64 / 1e6) + .filter(|s| *s > 0.0) + } + + /// Run ffmpeg, reporting progress against `total` seconds and honouring + /// cancel. Returns its stderr tail (the loudness measurement is there). + fn run_ffmpeg bool>(&self, ffmpeg: &Path, args: &[String], total: Option, on_cancel: &F) -> Result { + self.log(LogLevel::Debug, &format!("audio ffmpeg: {:?}", args)); + let mut full = vec!["-progress".to_string(), "pipe:1".to_string()]; + full.extend(args.iter().cloned()); + let mut child = Command::new(ffmpeg) + .args(&full) + .envs(self.deps.build_environment()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .context("Failed to start ffmpeg for audio")?; + let out_us = Arc::new(AtomicI64::new(0)); + let stdout = child.stdout.take().context("ffmpeg stdout")?; + let out_us2 = out_us.clone(); + let progress = thread::spawn(move || { + for line in BufReader::new(stdout).lines().map_while(Result::ok) { + if let Some(v) = line.strip_prefix("out_time_us=") { + if let Ok(us) = v.trim().parse::() { + out_us2.store(us, Ordering::Relaxed); + } + } + } + }); + let tail = Arc::new(Mutex::new(String::new())); + let mut stderr = child.stderr.take().context("ffmpeg stderr")?; + let tail2 = tail.clone(); + let errs = thread::spawn(move || { + let mut s = String::new(); + let _ = stderr.read_to_string(&mut s); + let mut keep = s.len().saturating_sub(16_000); + while !s.is_char_boundary(keep) { + keep += 1; + } + *tail2.lock().unwrap() = s[keep..].to_string(); + }); + let status = loop { + if on_cancel() { + let _ = child.kill(); + let _ = child.wait(); + bail!("Job cancelled"); + } + if let Some(status) = child.try_wait()? { + break status; + } + let pct = total + .filter(|t| *t > 0.0) + .map_or(0, |t| ((out_us.load(Ordering::Relaxed) as f64 / 1e6 / t) * 100.0).clamp(0.0, 100.0) as i32); + self.reporter.send_progress_phase(&ProgressInfo::new(pct, if total.is_some() { 100 } else { 0 }, 0.0, 0.0), "audio"); + thread::sleep(Duration::from_millis(250)); + }; + let _ = progress.join(); + let _ = errs.join(); + let stderr = tail.lock().unwrap().clone(); + if !status.success() { + let last: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect(); + let last = last[last.len().saturating_sub(8)..].join("\n"); + bail!("ffmpeg exited with {status}:\n{last}"); + } + Ok(stderr) + } + + /// Run a helper program (DeepFilterNet) with indeterminate progress. + fn run_quiet bool>(&self, bin: &Path, args: &[String], on_cancel: &F) -> Result<()> { + self.log(LogLevel::Debug, &format!("{}: {:?}", bin.display(), args)); + let mut child = Command::new(bin) + .args(args) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("Failed to start {}", bin.display()))?; + let mut stderr = child.stderr.take().context("stderr")?; + let errs = thread::spawn(move || { + let mut s = String::new(); + let _ = stderr.read_to_string(&mut s); + s + }); + let status = loop { + if on_cancel() { + let _ = child.kill(); + let _ = child.wait(); + bail!("Job cancelled"); + } + if let Some(status) = child.try_wait()? { + break status; + } + self.reporter.send_progress_phase(&ProgressInfo::new(0, 0, 0.0, 0.0), "audio"); + thread::sleep(Duration::from_millis(250)); + }; + let stderr = errs.join().unwrap_or_default(); + if !status.success() { + let last: Vec<&str> = stderr.lines().filter(|l| !l.trim().is_empty()).collect(); + bail!("exited with {status}:\n{}", last[last.len().saturating_sub(8)..].join("\n")); + } + Ok(()) + } +} + +/// Parse ffprobe's `-select_streams a` JSON: the stream count and the chosen +/// stream's details (`None` without any audio stream, or without `stream`). +pub fn parse_source_probe(json: &str, stream: u32) -> Option { + let v: serde_json::Value = serde_json::from_str(json).ok()?; + let streams = v.get("streams")?.as_array()?; + let s = streams.get(stream as usize)?; + let int = |k: &str| -> Option { + let x = s.get(k)?; + x.as_u64().map(|n| n as u32).or_else(|| x.as_str()?.parse().ok()) + }; + let duration = s + .get("duration") + .and_then(|d| d.as_str()) + .and_then(|d| d.parse::().ok()) + .or_else(|| s.get("tags")?.get("DURATION")?.as_str().and_then(parse_hms)) + .filter(|d| *d > 0.0); + Some(SourceAudio { + streams: streams.len() as u32, + channels: int("channels").unwrap_or(2), + sample_rate: int("sample_rate").unwrap_or(48_000), + duration, + }) +} + +/// "01:02:03.500000000" → 3723.5 (Matroska's DURATION tag). +fn parse_hms(s: &str) -> Option { + let parts: Vec<&str> = s.trim().split(':').collect(); + if parts.len() != 3 { + return None; + } + let h: f64 = parts[0].parse().ok()?; + let m: f64 = parts[1].parse().ok()?; + let sec: f64 = parts[2].parse().ok()?; + Some(h * 3600.0 + m * 60.0 + sec) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::{AudioQuality, ProcessingPipeline, QTGMCParameters, DeinterlaceMethod, FrameRateParameters, FrameRateTarget}; + + fn stereo48() -> SourceAudio { + SourceAudio { streams: 1, channels: 2, sample_rate: 48_000, duration: Some(60.0) } + } + + fn window(start: f64, d: f64) -> OutputWindow { + OutputWindow { start, duration: Some(d) } + } + + fn on() -> AudioRestorationParameters { + AudioRestorationParameters { enabled: true, ..Default::default() } + } + + fn everything() -> AudioRestorationParameters { + AudioRestorationParameters { + enabled: true, + channel_fix: ChannelFix::Mix, + offset_ms: 120, + drift_correction: true, + declip: true, + declick: true, + hum_removal: true, + hum_frequency: HumFrequency::Hz50, + rumble_filter: true, + denoise: true, + deep_filter: true, + loudness_normalize: true, + limiter: true, + ..Default::default() + } + } + + fn names(filters: &[String]) -> Vec { + filters.iter().map(|f| f.split('=').next().unwrap().to_string()).collect() + } + + fn af_count(args: &[String]) -> usize { + args.iter().filter(|a| *a == "-af").count() + } + + /// The order is the module's documented order whatever is switched on, and + /// every run carries exactly one -af (ffmpeg honours only the last). + #[test] + fn every_combination_keeps_the_order_and_one_af_per_run() { + let order = [ + "aresample", "pan", "adeclip", "adeclick", "highpass", "bandreject", "afftdn", "atrim", + "asetpts", "asetrate", "adelay", "alimiter", "apad", + ]; + let rank = |n: &str| order.iter().position(|o| *o == n).unwrap_or_else(|| panic!("unexpected filter {n}")); + // 9 toggles → 512 combinations, each with and without a negative offset. + for mask in 0u32..512 { + for offset in [0, 250, -250] { + let p = AudioRestorationParameters { + enabled: true, + channel_fix: if mask & 1 != 0 { ChannelFix::LeftOnly } else { ChannelFix::Off }, + declip: mask & 2 != 0, + declick: mask & 4 != 0, + rumble_filter: mask & 8 != 0, + hum_removal: mask & 16 != 0, + hum_frequency: HumFrequency::Hz60, + denoise: mask & 32 != 0, + deep_filter: mask & 64 != 0, + loudness_normalize: mask & 128 != 0, + limiter: mask & 256 != 0, + offset_ms: offset, + ..Default::default() + }; + let drift = if mask % 3 == 0 { Some(1.002) } else { None }; + let plan = AudioPlan::new(&p, &stereo48(), window(1.0, 30.0), &Analysis::default(), drift); + let ranks: Vec = names(&plan.restore_filters) + .iter() + .enumerate() + // The trailing atrim of the length fix is the last filter; + // the first aresample is step 1, the drift resample's two + // aresamples belong with its asetrate. + .filter(|(i, n)| !(n.as_str() == "atrim" && *i == plan.restore_filters.len() - 1)) + .filter(|(i, n)| !(n.as_str() == "aresample" && *i > 0)) + .map(|(_, n)| rank(n)) + .collect(); + assert!(ranks.windows(2).all(|w| w[0] <= w[1]), "mask {mask}: {:?}", plan.restore_filters); + let restore = plan.restore_args("in.mkv", Path::new("a.wav")); + assert_eq!(af_count(&restore), 1, "{restore:?}"); + if !plan.single_pass() { + let finish = plan.finish_args(Path::new("a.wav"), Path::new("b.wav"), None); + assert_eq!(af_count(&finish), 1, "{finish:?}"); + // The limiter only ever runs once, last before the length fix. + assert!(!plan.restore_filters.iter().any(|f| f.starts_with("alimiter"))); + } + // The length is always fixed at the end of the run that writes the track. + let last_chain = if plan.single_pass() { + plan.restore_filters.join(",") + } else { + plan.finish_filter_chain(None) + }; + assert!(last_chain.ends_with(&format!("apad=whole_len={n},atrim=end_sample={n}", n = 30 * plan.out_rate)), "{last_chain}"); + } + } + } + + #[test] + fn full_chain_reads_in_the_documented_order() { + let plan = AudioPlan::new(&everything(), &stereo48(), window(0.0, 10.0), &Analysis::default(), Some(1.001)); + assert_eq!( + plan.restore_filters, + vec![ + "aresample=first_pts=0", + "pan=stereo|c0=0.5*c0+0.5*c1|c1=0.5*c0+0.5*c1", + "adeclip", + "adeclick=t=2", + "highpass=f=40:poles=2", + "bandreject=f=50:width_type=q:w=8", + "bandreject=f=100:width_type=q:w=8", + "bandreject=f=150:width_type=q:w=8", + "afftdn=nr=12:nf=-50", + "atrim=start_sample=1200", + "asetpts=PTS-STARTPTS", + "aresample=192000", + "asetrate=192192", + "aresample=48000", + // 120 ms delay from a start of 0 reaches before the source: lead silence. + "adelay=delays=5760S:all=1", + "apad=whole_len=480000", + "atrim=end_sample=480000", + ] + ); + assert_eq!(plan.deep_filter, Some(100)); + assert_eq!(plan.out_rate, 48_000); + let m = LoudnessMeasurement { input_i: -27.3, input_tp: -6.0, input_lra: 5.5, input_thresh: -37.5, target_offset: 0.2 }; + assert_eq!( + plan.finish_filter_chain(Some(&m)), + "loudnorm=I=-16:TP=-1.5:LRA=11:measured_I=-27.3:measured_TP=-6:measured_LRA=5.5:\ + measured_thresh=-37.5:offset=0.2:linear=true:print_format=summary,aresample=48000,\ + alimiter=limit=0.8913:level=0:latency=1,apad=whole_len=480000,atrim=end_sample=480000" + ); + } + + /// afftdn's delay is trimmed in samples at the source's own rate. + #[test] + fn afftdn_delay_is_trimmed_at_the_source_rate() { + for (rate, samples) in [(48_000, 1200), (44_100, 1103), (32_000, 800)] { + let src = SourceAudio { sample_rate: rate, ..stereo48() }; + let p = AudioRestorationParameters { denoise: true, ..on() }; + let plan = AudioPlan::new(&p, &src, window(0.0, 5.0), &Analysis::default(), None); + assert!(plan.restore_filters.contains(&format!("atrim=start_sample={samples}")), "{:?}", plan.restore_filters); + } + } + + /// A positive offset delays the sound: from a later in-point it reads + /// earlier source audio; from the start it needs silence in front. + /// A negative one reads later audio, never silence (adelay alone could + /// only ever delay). + #[test] + fn offsets_move_the_read_point_or_add_lead_silence() { + let at = |offset_ms: i32, start: f64| { + let p = AudioRestorationParameters { offset_ms, ..on() }; + AudioPlan::new(&p, &stereo48(), window(start, 10.0), &Analysis::default(), None) + }; + let late = at(300, 5.0); + assert!((late.read_start - 4.7).abs() < 1e-9); + assert!(!late.restore_filters.iter().any(|f| f.starts_with("adelay"))); + + let from_zero = at(300, 0.0); + assert_eq!(from_zero.read_start, 0.0); + assert!(from_zero.restore_filters.contains(&"adelay=delays=14400S:all=1".to_string())); + // Read only what fills the window after the lead (plus the margin). + assert!((from_zero.read_duration.unwrap() - (9.7 + READ_MARGIN_SECONDS)).abs() < 1e-9); + + let early = at(-300, 0.0); + assert!((early.read_start - 0.3).abs() < 1e-9); + assert!(!early.restore_filters.iter().any(|f| f.starts_with("adelay"))); + let args = early.restore_args("in.mkv", Path::new("o.wav")); + let ss = args.iter().position(|a| a == "-ss").unwrap(); + let i = args.iter().position(|a| a == "-i").unwrap(); + assert!(ss < i, "the seek is an input option"); + assert_eq!(args[ss + 1], "0.300000"); + } + + /// Drift and offset compose: source time = (start + t − offset) × tempo. + #[test] + fn drift_scales_the_read_point_and_length() { + let p = AudioRestorationParameters { offset_ms: -100, ..on() }; + let plan = AudioPlan::new(&p, &stereo48(), window(10.0, 20.0), &Analysis::default(), Some(1.004)); + assert!((plan.read_start - 10.1 * 1.004).abs() < 1e-9); + assert!((plan.read_duration.unwrap() - (20.0 * 1.004 + READ_MARGIN_SECONDS)).abs() < 1e-9); + let (r1, s) = resample_ratio(48_000, 1.004); + assert!(plan.restore_filters.contains(&format!("asetrate={s}"))); + assert!(plan.restore_filters.contains(&format!("aresample={r1}"))); + } + + #[test] + fn drift_resample_is_skipped_within_1e6_of_one() { + for t in [1.0, 1.0 + 5e-7, 1.0 - 9e-7] { + let plan = AudioPlan::new(&on(), &stereo48(), window(0.0, 1.0), &Analysis::default(), Some(t)); + assert!(!plan.restore_filters.iter().any(|f| f.starts_with("asetrate")), "{t}"); + assert_eq!(plan.restore_filters.iter().filter(|f| f.starts_with("aresample")).count(), 1); + } + let plan = AudioPlan::new(&on(), &stereo48(), window(0.0, 1.0), &Analysis::default(), Some(1.0 + 2e-6)); + assert!(plan.restore_filters.iter().any(|f| f.starts_with("asetrate")), "{:?}", plan.restore_filters); + } + + /// The relabelled rate is an integer, so the ratio is found where it can + /// be exact: within 1e-10 of the factor for any drift, never below 4×. + #[test] + fn resample_ratio_is_exact_enough_for_an_hour() { + for rate in [44_100u32, 48_000] { + for f in [1.004, 0.996, 1.000123456, 0.99987654321, 1.005] { + let (r1, s) = resample_ratio(rate, f); + assert!(r1 >= rate * 4 && r1 <= rate * 8, "{r1}"); + let err = (s as f64 / r1 as f64 - f).abs(); + // Under 0.4 ms over an hour. + assert!(err * 3600.0 < 0.0004, "{rate} {f}: {r1}->{s} err {err:e}"); + } + } + assert_eq!(resample_ratio(48_000, 1.001), (192_000, 192_192)); + } + + #[test] + fn noise_floor_is_the_quiet_tenth() { + let mut seed = 99u64; + // 9 s of -40 dB noise with 1 s bursts of a loud tone every 2 s. + let x = synth(48_000, 9.0, |t| { + let n = 0.01 * 3f64.sqrt() * noise(&mut seed); + if (t as u64) % 2 == 0 && t.fract() < 0.5 { n + 0.5 * (1000.0 * t).sin() } else { n } + }); + let floor = noise_floor_db(&x, 48_000); + assert!((floor - (-40.0)).abs() < 1.0, "{floor}"); + assert_eq!(noise_floor_db(&vec![0.0; 48_000], 48_000), NOISE_FLOOR_RANGE.0); + let p = AudioRestorationParameters { denoise: true, ..on() }; + let a = Analysis { noise_floor_db: Some(-41.4), ..Default::default() }; + let plan = AudioPlan::new(&p, &stereo48(), window(0.0, 1.0), &a, None); + assert!(plan.restore_filters.contains(&"afftdn=nr=12:nf=-41".to_string()), "{:?}", plan.restore_filters); + } + + #[test] + fn drift_needs_more_than_50ms_and_at_most_the_cap() { + // 1 h picture. + let v = 3600.0; + assert!(matches!(drift_decision(v + 0.04, v, 0.5), DriftDecision::Skip(_))); + assert_eq!(drift_decision(v + 3.6, v, 0.5), DriftDecision::Apply((v + 3.6) / v)); + assert_eq!(drift_decision(v - 3.6, v, 0.5), DriftDecision::Apply((v - 3.6) / v)); + // 0.6% is over a 0.5% cap but under a 1% one. + assert!(matches!(drift_decision(v * 1.006, v, 0.5), DriftDecision::Skip(why) if why.contains("0.60%"))); + assert!(matches!(drift_decision(v * 1.006, v, 1.0), DriftDecision::Apply(_))); + assert!(matches!(drift_decision(0.0, v, 0.5), DriftDecision::Skip(_))); + } + + #[test] + fn channel_fix_auto_uses_the_analysis() { + let p = AudioRestorationParameters { channel_fix: ChannelFix::Auto, ..on() }; + let with = |dead| AudioPlan::new(&p, &stereo48(), window(0.0, 1.0), &Analysis { dead_channel: dead, ..Default::default() }, None); + assert!(with(Some(DeadChannel::Right)).restore_filters.contains(&"pan=stereo|c0=c0|c1=c0".to_string())); + assert!(with(Some(DeadChannel::Left)).restore_filters.contains(&"pan=stereo|c0=c1|c1=c1".to_string())); + assert!(!with(None).restore_filters.iter().any(|f| f.starts_with("pan"))); + // Never on anything but stereo. + let mono = SourceAudio { channels: 1, ..stereo48() }; + let p = AudioRestorationParameters { channel_fix: ChannelFix::LeftOnly, ..on() }; + let plan = AudioPlan::new(&p, &mono, window(0.0, 1.0), &Analysis::default(), None); + assert!(!plan.restore_filters.iter().any(|f| f.starts_with("pan"))); + assert!(plan.notes.iter().any(|n| n.contains("1 channel"))); + } + + #[test] + fn dead_channel_needs_a_30db_gap_and_a_live_side() { + assert_eq!(dead_channel(-20.0, -90.0), Some(DeadChannel::Right)); + assert_eq!(dead_channel(-95.0, -25.0), Some(DeadChannel::Left)); + assert_eq!(dead_channel(-20.0, -45.0), None); // quiet, not dead + assert_eq!(dead_channel(-80.0, -120.0), None); // all silence + } + + #[test] + fn hum_auto_notches_only_what_was_found() { + let p = AudioRestorationParameters { hum_removal: true, ..on() }; + let plan = |hum| AudioPlan::new(&p, &stereo48(), window(0.0, 1.0), &Analysis { hum, ..Default::default() }, None); + let f60 = plan(Some(60)).restore_filters; + assert!(f60.iter().any(|f| f == "bandreject=f=180:width_type=q:w=8")); + assert!(!f60.iter().any(|f| f.contains("f=50:"))); + let none = plan(None); + assert!(!none.restore_filters.iter().any(|f| f.starts_with("bandreject"))); + assert!(none.notes.iter().any(|n| n.contains("nothing is notched"))); + } + + fn synth(rate: u32, secs: f64, mut f: impl FnMut(f64) -> f64) -> Vec { + (0..(rate as f64 * secs) as usize).map(|i| f(i as f64 / rate as f64) as f32).collect() + } + + /// A deterministic noise source for the synthetic tests. + fn noise(seed: &mut u64) -> f64 { + *seed ^= *seed << 13; + *seed ^= *seed >> 7; + *seed ^= *seed << 17; + (*seed as f64 / u64::MAX as f64) * 2.0 - 1.0 + } + + #[test] + fn hum_detection_tells_50_from_60_and_ignores_plain_noise() { + use std::f64::consts::PI; + let mut s = 0x1234_5678_9abc_def0u64; + let mut s2 = s; + let mut s3 = s; + // Hum at 50.1 Hz (mains wanders), harmonics, under noise and a 440 Hz tone. + let hum50 = synth(8000, 20.0, |t| { + 0.02 * (2.0 * PI * 50.1 * t).sin() + 0.01 * (2.0 * PI * 100.2 * t).sin() + + 0.2 * (2.0 * PI * 440.0 * t).sin() + 0.05 * noise(&mut s) + }); + assert_eq!(detect_hum(&hum50, 8000), Some(50)); + // 60 Hz mains where the third harmonic dominates (rectifier buzz). + let hum60 = synth(8000, 20.0, |t| { + 0.003 * (2.0 * PI * 60.0 * t).sin() + 0.02 * (2.0 * PI * 180.0 * t).sin() + 0.05 * noise(&mut s2) + }); + assert_eq!(detect_hum(&hum60, 8000), Some(60)); + let clean = synth(8000, 20.0, |t| 0.2 * (2.0 * PI * 440.0 * t).sin() + 0.05 * noise(&mut s3)); + assert_eq!(detect_hum(&clean, 8000), None); + } + + #[test] + fn channel_levels_measure_each_channel() { + let interleaved: Vec = (0..16000).flat_map(|i| [if i % 2 == 0 { 0.5 } else { -0.5 }, 0.0]).collect(); + let l = channel_levels_db(&interleaved, 2); + assert!((l[0] - 20.0 * 0.5f64.log10()).abs() < 0.01); + assert!(l[1] < -200.0); + } + + #[test] + fn loudnorm_json_is_read_from_stderr() { + let stderr = "[Parsed_loudnorm_0 @ 0x1]\n{\n\t\"input_i\" : \"-27.61\",\n\t\"input_tp\" : \"-4.47\",\n\ + \t\"input_lra\" : \"18.06\",\n\t\"input_thresh\" : \"-39.20\",\n\t\"output_i\" : \"-16.58\",\n\ + \t\"output_tp\" : \"-1.50\",\n\t\"output_lra\" : \"14.78\",\n\t\"output_thresh\" : \"-27.71\",\n\ + \t\"normalization_type\" : \"dynamic\",\n\t\"target_offset\" : \"0.58\"\n}\n"; + let m = parse_loudnorm_json(stderr).unwrap(); + assert_eq!(m.input_i, -27.61); + assert_eq!(m.target_offset, 0.58); + // Silence measures -inf: nothing to normalise. + assert!(parse_loudnorm_json(&stderr.replace("-27.61", "-inf")).is_none()); + assert!(parse_loudnorm_json("no json").is_none()); + } + + #[test] + fn source_probe_reads_streams_and_durations() { + let json = r#"{"streams":[{"channels":2,"sample_rate":"48000","duration":"12.5"}, + {"channels":6,"sample_rate":"44100","tags":{"DURATION":"01:00:01.250000000"}}]}"#; + let a = parse_source_probe(json, 0).unwrap(); + assert_eq!((a.streams, a.channels, a.sample_rate, a.duration), (2, 2, 48_000, Some(12.5))); + let b = parse_source_probe(json, 1).unwrap(); + assert_eq!((b.channels, b.sample_rate, b.duration), (6, 44_100, Some(3601.25))); + assert!(parse_source_probe(json, 2).is_none()); + assert!(parse_source_probe(r#"{"streams":[]}"#, 0).is_none()); + } + + /// The picture's length is its output frames over its output rate, so + /// double-rate deinterlacing and frame-rate conversion leave it as long + /// as the source window, and a trim shortens it. + #[test] + fn the_window_is_the_encoded_pictures_length() { + let mut job: VideoJob = serde_json::from_value(serde_json::json!({ + "id": "00000000-0000-0000-0000-000000000001", + "inputPath": "in.mkv", "outputPath": "out.mkv", + "qtgmcParameters": {}, "encodingSettings": {}, + "totalFrames": 250, "inputFrameRate": 25.0 + })) + .unwrap(); + let mut pipeline = ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: true, fps_divisor: Some(1), ..Default::default() }, + ..Default::default() + }; + job.processing_pipeline = Some(pipeline.clone()); + let w = OutputWindow::for_job(&job); + assert_eq!(w.start, 0.0); + assert!((w.duration.unwrap() - 10.0).abs() < 1e-9, "{w:?}"); + + // IVTC from 29.97: 300 frames → 240 at 23.976. + pipeline.deinterlace = QTGMCParameters { enabled: true, method: DeinterlaceMethod::Ivtc, ..Default::default() }; + job.processing_pipeline = Some(pipeline.clone()); + job.input_frame_rate = Some(30000.0 / 1001.0); + job.total_frames = Some(300); + assert!((OutputWindow::for_job(&job).duration.unwrap() - 300.0 * 1001.0 / 30000.0).abs() < 1e-9); + + // 25 → 50 fps, trimmed to 100 frames from frame 50: 200 frames at 50 = 4 s. + pipeline.deinterlace.enabled = false; + pipeline.frame_rate = FrameRateParameters { enabled: true, target: FrameRateTarget::Pal50, ..Default::default() }; + job.processing_pipeline = Some(pipeline.clone()); + job.input_frame_rate = Some(25.0); + job.start_frame = Some(50); + job.end_frame = Some(149); + job.total_frames = Some(100); + let w = OutputWindow::for_job(&job); + assert!((w.start - 2.0).abs() < 1e-9); + assert!((w.duration.unwrap() - 4.0).abs() < 1e-9, "{w:?}"); + + // 25 → 59.94 is not a whole ratio: floor(100 × 2400/1001) = 239 frames + // at 60000/1001 — the encoded length, a frame short of the source's. + pipeline.frame_rate.target = FrameRateTarget::Ntsc5994; + job.processing_pipeline = Some(pipeline); + let d = OutputWindow::for_job(&job).duration.unwrap(); + assert!((d - 239.0 * 1001.0 / 60000.0).abs() < 1e-9, "{d}"); + } + + fn settings(mode: AudioMode, container: ContainerFormat) -> EncodingSettings { + EncodingSettings { audio_mode: mode, container, audio_quality: AudioQuality::High, ..Default::default() } + } + + #[test] + fn encoder_maps_the_restored_track_first_and_keeps_the_others() { + let s = settings(AudioMode::Passthrough, ContainerFormat::Mkv); + assert_eq!(encode_map_args(&s, None, 3), vec!["-map", "1:a?"]); + let t = RestoredTrack { path: "r.wav".into(), stream: 1, source_streams: 3 }; + assert_eq!( + encode_map_args(&s, Some(&t), 3), + vec![ + "-map", "3:a:0", "-map", "1:a:0", "-map", "1:a:2", + "-map_metadata:s:a:0", "1:s:a:1", "-map_metadata:s:a:1", "1:s:a:0", + "-map_metadata:s:a:2", "1:s:a:2", + ] + ); + assert!(encode_map_args(&settings(AudioMode::None, ContainerFormat::Mkv), Some(&t), 3).is_empty()); + assert_eq!(encode_input_args(Some(&t)), vec!["-i", "r.wav"]); + assert!(encode_input_args(None).is_empty()); + } + + #[test] + fn passthrough_copies_the_others_and_encodes_the_restored_track() { + let t = RestoredTrack { path: "r.wav".into(), stream: 0, source_streams: 2 }; + assert_eq!( + encode_codec_args(&settings(AudioMode::Passthrough, ContainerFormat::Mkv), Some(&t)), + vec!["-c:a", "copy", "-c:a:0", "flac"] + ); + assert_eq!( + encode_codec_args(&settings(AudioMode::Passthrough, ContainerFormat::Mp4), Some(&t)), + vec!["-c:a", "copy", "-c:a:0", "aac", "-b:a:0", "192k"] + ); + assert_eq!( + encode_codec_args(&settings(AudioMode::Passthrough, ContainerFormat::Avi), Some(&t)), + vec!["-c:a", "copy", "-c:a:0", "pcm_s16le"] + ); + // Without a track, exactly what the encoder always did. + assert_eq!(encode_codec_args(&settings(AudioMode::Passthrough, ContainerFormat::Mp4), None), vec!["-c:a", "copy"]); + let mut convert = settings(AudioMode::Convert, ContainerFormat::Mp4); + convert.audio_codec = AudioCodec::Opus; + assert_eq!(encode_codec_args(&convert, Some(&t)), vec!["-c:a", "libopus", "-b:a", "192k"]); + assert_eq!(encode_codec_args(&settings(AudioMode::None, ContainerFormat::Mp4), None), vec!["-an"]); + } + + #[test] + fn deep_filter_and_loudness_args() { + let p = AudioRestorationParameters { deep_filter: true, deep_filter_attenuation: 24, loudness_normalize: true, loudness_target: -23.0, ..on() }; + let plan = AudioPlan::new(&p, &SourceAudio { sample_rate: 44_100, ..stereo48() }, window(0.0, 2.0), &Analysis::default(), None); + assert_eq!( + plan.deep_filter_args(Path::new("w/a.wav"), Path::new("w/dfn")).unwrap(), + vec!["-D", "-a", "24", "-o", "w/dfn", "w/a.wav"] + ); + // The DeepFilterNet input keeps the source rate; the output is 48 kHz. + assert!(plan.restore_filters.ends_with(&["apad=whole_len=88200".to_string(), "atrim=end_sample=88200".to_string()])); + assert_eq!(plan.out_rate, 48_000); + let m = plan.measure_args(Path::new("b.wav")).unwrap(); + assert!(m.contains(&"loudnorm=I=-23:TP=-1.5:LRA=11:print_format=json".to_string())); + assert_eq!(&m[m.len() - 3..], ["-f", "null", "-"]); + // Without a measurement (silence), no loudnorm at all — never one-pass. + assert!(!plan.finish_filter_chain(None).contains("loudnorm")); + } + + #[test] + fn numbers_are_plain() { + assert_eq!(num(12.0), "12"); + assert_eq!(num(-1.5), "-1.5"); + assert_eq!(num(0.891251), "0.8913"); + assert_eq!(num(-0.0), "0"); + assert_eq!(num_precise(1.000999), "1.000999"); + } +} diff --git a/worker/src/crop_detect.rs b/worker/src/crop_detect.rs new file mode 100644 index 00000000..d4b54b98 --- /dev/null +++ b/worker/src/crop_detect.rs @@ -0,0 +1,1350 @@ +//! Find what is not picture at the edges of a video: black borders and analog +//! blanking, the dim ramp beside them, wide letterbox / pillarbox bars, and the +//! head-switching band at the bottom of a tape capture (app 1.7.0, +//! `--detect-crop`). The result fills Crop & Resize's crop controls when the +//! user clicks Apply, the way Detect fills the Descale panel; nothing here is +//! ever applied on its own. +//! +//! Frames are sampled across the source (or its trim) and measured line by +//! line in plain Rust; no VapourSynth is involved. VCG Deinterlacer 2.0's +//! `analysis/edges.rs` (MIT) was read as a reference for which signals +//! separate these things; the code and its thresholds are our own, measured on +//! our fixtures (docs/reports/2026-11-crop-detect-709/). +//! +//! What each test is built against: +//! +//! * **Black and blanking** are dark AND flat. Brightness alone would eat into +//! a night scene, so a line's brightness is a high percentile over every +//! sampled frame — a line that carries picture in any of them is picture — +//! and its detail must be low as well. Noisy blanking (grain out of the +//! blanking interval) is dark but not perfectly flat, so detail has an +//! absolute floor rather than only a fraction of the picture's. +//! * **The ramp**: past analog blanking the picture fades in over a few lines. +//! It is followed only while each line is dimmer than the picture just inside +//! it and the next is brighter, for at most a few lines, so dark content is +//! never mistaken for one. +//! * **Bars wider than blanking ever is** (`BLANKING_MAX_FRACTION`) are the +//! picture's framing — a letterboxed film inside 4:3, a 4:3 picture inside +//! HD. They are reported separately and only cropped if the user asks. +//! * **Head switching** is a band of lines at the very bottom torn out of +//! sequence with the picture: unlike the line above it in luma or chroma, or +//! shifted sideways against it. Three things keep a picture from being read +//! as one: the band must be in at least half the usable frames (a burnt-in +//! subtitle or a hard edge comes and goes), torn across most of the width (a +//! subtitle is a short run in the middle), and touch the bottom (a subtitle +//! sits above the title-safe margin). On an interlaced source each field has +//! its own band, so lines are compared with the line above *in the same +//! field*. +//! * **Dark scenes and fades**: frames too dark to judge are left out of every +//! statistic, and the report says how many frames were usable. + +use std::process::{Command, Stdio}; + +use anyhow::{bail, Context, Result}; +use serde::{Deserialize, Serialize}; + +use crate::dependency_locator::DependencyLocator; +use crate::models::{FieldOrder, ProcessingPipeline, Rotation, VideoJob}; +use crate::source_decode; + +/// Frames sampled across the source. +pub const SAMPLE_FRAMES: usize = 24; + +/// Fraction of each dimension left out at both ends when measuring along the +/// other, so a bar on one axis cannot contaminate the other's statistics (the +/// head-switching band spans the whole width, for one). +const INSET: f64 = 0.12; + +/// A line's brightness: this percentile of its samples over every usable +/// frame. A mean would be dragged down by dark content. +const LEVEL_PERCENTILE: f64 = 0.98; + +/// A frame whose picture (the central region's 90th percentile) is darker than +/// this, on the 8-bit scale, tells nothing about where the picture stops. +const DARK_FRAME: f64 = 48.0; + +/// A black/blanking line is no brighter than this fraction of the way from +/// black to the picture's typical line brightness... +const BLANK_LEVEL: f64 = 0.30; + +/// ...and carries no more than this fraction of the picture's detail, or +/// `BLANK_DETAIL_FLOOR` levels, whichever is more. +const BLANK_DETAIL: f64 = 0.5; +const BLANK_DETAIL_FLOOR: f64 = 6.0; + +/// Fewer dark lines than this is not a border (one dim line is a hard edge)... +const MIN_RUN: usize = 2; + +/// ...unless it is black in every sampled frame: no sample brighter than +/// this many levels above black. Capture cards leave exactly one such line at +/// the top of a field (measured on our VHS fixture: 0 in every frame, where +/// the line below is picture). +const HARD_BLACK: f64 = 4.0; + +/// Analog blanking and overscan borders are narrower than this fraction of the +/// dimension; anything wider is a bar (letterbox / pillarbox), reported but +/// not cropped unless asked. Blanking on a tape capture measures 1-3 %. +pub const BLANKING_MAX_FRACTION: f64 = 0.05; + +/// No side is ever more than this fraction of its dimension: a "bar" wider +/// than that is a black picture, not framing. +const BAR_MAX_FRACTION: f64 = 0.35; + +/// The ramp: at most this many lines, each below this fraction of the +/// picture's brightness just inside it. +const RAMP_MAX: usize = 6; +const RAMP_KNEE: f64 = 0.85; + +/// The head-switching search covers this fraction of the height, from the +/// bottom (35 lines of 576: an interlaced band is two fields' worth). +const BAND_SEARCH_FRACTION: f64 = 0.06; + +/// A band line is torn across at least this many of these segments of the +/// width — a subtitle is a short run in the middle. +const BAND_SEGMENTS: usize = 8; +const BAND_SEGMENTS_NEEDED: usize = 6; + +/// A segment is torn when it differs from the line above by this many times +/// the picture's typical line-to-line difference, and by at least the floor +/// (8-bit levels; chroma's typical difference is far smaller than luma's). +const BAND_RATIO: f64 = 2.2; +const BAND_LUMA_FLOOR: f64 = 4.0; +const BAND_CHROMA_FLOOR: f64 = 2.5; + +/// The sideways tear: the best horizontal match against the line above is at +/// least this far off, and much better than no shift. +const BAND_SHIFT_MAX: i32 = 16; +const BAND_SHIFT_MIN: i32 = 3; +const BAND_SHIFT_GAIN: f64 = 0.6; + +/// A line is band when it is torn in at least this fraction of usable frames. +const BAND_FRAME_FRACTION: f64 = 0.5; + +/// Lines at the very bottom that may be neither blank nor band before the band +/// starts, and lines inside the band that may miss the test. +const BAND_SKIP: usize = 2; +const BAND_GAP: usize = 1; + +/// The widest band offered to Edge Repair instead of the crop: the Edge +/// Repair panel's own maximum. A band wider than that is cropped. +pub const EDGE_REPAIR_MAX: i32 = 16; + +/// A frame agrees with an edge when its own measurement is within this. +const AGREEMENT: usize = 2; + +/// The four sides of something. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Sides { + pub left: i32, + pub right: i32, + pub top: i32, + pub bottom: i32, +} + +impl Sides { + pub fn is_zero(&self) -> bool { + self.left == 0 && self.right == 0 && self.top == 0 && self.bottom == 0 + } +} + +/// What one edge was found to be. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub enum EdgeKind { + #[default] + None, + /// Black border or analog blanking, narrow: cropped. + Blanking, + /// A letterbox / pillarbox bar: the picture's framing, cropped only if the + /// user asks. + Bar, + /// The torn band at the bottom of a tape capture. + HeadSwitching, +} + +/// One edge, as measured (unrounded source lines). +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EdgeFinding { + pub kind: EdgeKind, + /// Dark, flat lines from the frame edge (blanking or bar). + pub blank: i32, + /// The dim ramp beside them. + pub ramp: i32, + /// Head-switching lines (bottom only), above the `blank` black lines at + /// the very bottom, counting the up to two untorn lines between them. + pub band: i32, + /// A bar beyond the band (bottom only: a letterbox above a torn band). + pub bar: i32, + /// What the recommended crop removes here, before rounding. + pub crop: i32, + /// The same with any bar included. + pub crop_with_bars: i32, + /// Fraction of usable frames whose own measurement lands within 2 lines + /// of this edge (0-1): the frames whose picture reaches it. The rest do + /// not contradict it — their picture is dark there too — so a low value + /// means thin evidence (a dark film), not a wrong edge. + pub confidence: f64, +} + +/// The four edges of the source, measured in its own orientation. +#[derive(Debug, Clone, Copy, Default, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct EdgeFindings { + pub left: EdgeFinding, + pub right: EdgeFinding, + pub top: EdgeFinding, + pub bottom: EdgeFinding, +} + +/// One decoded frame, 8-bit 4:4:4 (chroma at luma resolution, so a chroma line +/// can be compared with the one above it). +pub struct Frame { + pub width: usize, + pub height: usize, + pub y: Vec, + pub u: Vec, + pub v: Vec, +} + +impl Frame { + /// From planar `yuv444p` bytes, or `None` for a short decode. + pub fn from_yuv444p(data: &[u8], width: usize, height: usize) -> Option { + let n = width * height; + if data.len() < 3 * n { + return None; + } + Some(Frame { + width, + height, + y: data[..n].to_vec(), + u: data[n..2 * n].to_vec(), + v: data[2 * n..3 * n].to_vec(), + }) + } +} + +/// The inclusive-exclusive middle of one axis. +fn inset(n: usize) -> (usize, usize) { + let lo = (n as f64 * INSET) as usize; + let hi = n - lo; + if hi > lo + 2 { + (lo, hi) + } else { + (0, n) + } +} + +fn median(values: &[f64]) -> f64 { + if values.is_empty() { + return 0.0; + } + let mut v = values.to_vec(); + v.sort_by(|a, b| a.total_cmp(b)); + v[v.len() / 2] +} + +/// The value at `pct` of a 256-bin histogram. +fn percentile(hist: &[u32], pct: f64) -> f64 { + let total: u64 = hist.iter().map(|&c| c as u64).sum(); + if total == 0 { + return 0.0; + } + let want = ((total as f64) * pct).ceil().max(1.0) as u64; + let mut seen = 0u64; + for (v, &c) in hist.iter().enumerate() { + seen += c as u64; + if seen >= want { + return v as f64; + } + } + 255.0 +} + +/// Per-line statistics of one axis: brightness percentile and detail. +#[derive(Clone, Default)] +struct Lines { + level: Vec, + detail: Vec, +} + +/// One frame's measurements. +struct FrameStats { + cols: Lines, + rows: Lines, + /// Per bottom line (offset 0 = last line): torn in this frame. + torn: Vec, +} + +/// Columns (over the middle rows) and rows (over the middle columns), with +/// their histograms added into `col_hist` / `row_hist`. +fn line_stats(f: &Frame, col_hist: &mut [u32], row_hist: &mut [u32]) -> (Lines, Lines) { + let (w, h) = (f.width, f.height); + let (y0, y1) = inset(h); + let (x0, x1) = inset(w); + let mut ch = vec![0u32; w * 256]; + let mut col_detail = vec![0f64; w]; + for y in y0..y1 { + let row = &f.y[y * w..(y + 1) * w]; + for (x, &v) in row.iter().enumerate() { + ch[x * 256 + v as usize] += 1; + } + if y > y0 { + let above = &f.y[(y - 1) * w..y * w]; + for x in 0..w { + col_detail[x] += row[x].abs_diff(above[x]) as f64; + } + } + } + let n = (y1 - y0 - 1).max(1) as f64; + let mut cols = Lines { level: vec![0.0; w], detail: vec![0.0; w] }; + for x in 0..w { + let hist = &ch[x * 256..(x + 1) * 256]; + cols.level[x] = percentile(hist, LEVEL_PERCENTILE); + cols.detail[x] = col_detail[x] / n; + for (i, &c) in hist.iter().enumerate() { + col_hist[x * 256 + i] += c; + } + } + + let mut rows = Lines { level: vec![0.0; h], detail: vec![0.0; h] }; + let mut rh = [0u32; 256]; + for y in 0..h { + rh.fill(0); + let row = &f.y[y * w..(y + 1) * w]; + let mut d = 0f64; + for x in x0..x1 { + rh[row[x] as usize] += 1; + if x > x0 { + d += row[x].abs_diff(row[x - 1]) as f64; + } + } + rows.level[y] = percentile(&rh, LEVEL_PERCENTILE); + rows.detail[y] = d / (x1 - x0 - 1).max(1) as f64; + for (i, &c) in rh.iter().enumerate() { + row_hist[y * 256 + i] += c; + } + } + (cols, rows) +} + +/// How unlike line `y` is from line `r` over the middle columns: per segment +/// in luma and in chroma, and the whole line in luma. +fn line_break(f: &Frame, y: usize, r: usize) -> ([f64; BAND_SEGMENTS], [f64; BAND_SEGMENTS], f64) { + let w = f.width; + let (x0, x1) = inset(w); + let seg = ((x1 - x0) / BAND_SEGMENTS).max(1); + let mut luma = [0f64; BAND_SEGMENTS]; + let mut chroma = [0f64; BAND_SEGMENTS]; + let mut total = 0f64; + for s in 0..BAND_SEGMENTS { + let a = x0 + s * seg; + let b = (a + seg).min(x1); + let (mut l, mut c) = (0f64, 0f64); + for x in a..b { + let (i, j) = (y * w + x, r * w + x); + l += f.y[i].abs_diff(f.y[j]) as f64; + c += (f.u[i].abs_diff(f.u[j]) as f64 + f.v[i].abs_diff(f.v[j]) as f64) / 2.0; + } + let n = (b - a).max(1) as f64; + luma[s] = l / n; + chroma[s] = c / n; + total += l; + } + (luma, chroma, total / (seg * BAND_SEGMENTS) as f64) +} + +/// Whether line `y` is shifted sideways against line `r`: its best match is +/// at least `BAND_SHIFT_MIN` columns off and much better than no shift. +fn shifted(f: &Frame, y: usize, r: usize, floor: f64) -> bool { + let w = f.width; + let (x0, x1) = inset(w); + let lo = x0 as i32 + BAND_SHIFT_MAX; + let hi = x1 as i32 - BAND_SHIFT_MAX; + if hi <= lo + 8 { + return false; + } + let sad = |s: i32| -> f64 { + let mut t = 0f64; + let mut n = 0f64; + let mut x = lo; + while x < hi { + t += f.y[y * w + (x + s) as usize].abs_diff(f.y[r * w + x as usize]) as f64; + n += 1.0; + x += 2; + } + t / n + }; + let zero = sad(0); + if zero < floor { + return false; + } + let (mut best, mut best_s) = (zero, 0); + for s in -BAND_SHIFT_MAX..=BAND_SHIFT_MAX { + let v = sad(s); + if v < best { + best = v; + best_s = s; + } + } + best_s.abs() >= BAND_SHIFT_MIN && best <= zero * BAND_SHIFT_GAIN +} + +/// Which of the bottom `search` lines are torn in this frame. `d` is the +/// distance to the line above in the same field (2 when interlaced). +fn torn_lines(f: &Frame, search: usize, d: usize) -> Vec { + let h = f.height; + let (y0, y1) = inset(h); + // The picture's own line-to-line difference, over the middle rows. + let (mut refs_l, mut refs_c) = (Vec::new(), Vec::new()); + let mut y = y0.max(d); + while y < y1 { + let (l, c, _) = line_break(f, y, y - d); + refs_l.push(l.iter().sum::() / BAND_SEGMENTS as f64); + refs_c.push(c.iter().sum::() / BAND_SEGMENTS as f64); + y += 2; + } + let ref_l = median(&refs_l); + let ref_c = median(&refs_c); + let luma_bar = (ref_l * BAND_RATIO).max(BAND_LUMA_FLOOR); + let chroma_bar = (ref_c * BAND_RATIO).max(BAND_CHROMA_FLOOR); + (0..search) + .map(|o| { + let y = h - 1 - o; + if y < d { + return false; + } + let (l, c, _) = line_break(f, y, y - d); + let torn_l = l.iter().filter(|&&v| v > luma_bar).count(); + let torn_c = c.iter().filter(|&&v| v > chroma_bar).count(); + torn_l >= BAND_SEGMENTS_NEEDED + || torn_c >= BAND_SEGMENTS_NEEDED + || shifted(f, y, y - d, (ref_l * 1.5).max(3.0)) + }) + .collect() +} + +/// The central region's 90th percentile luma: how bright the picture is. +fn picture_level(f: &Frame) -> f64 { + let (x0, x1) = inset(f.width); + let (y0, y1) = inset(f.height); + let mut hist = [0u32; 256]; + for y in (y0..y1).step_by(2) { + for x in (x0..x1).step_by(2) { + hist[f.y[y * f.width + x] as usize] += 1; + } + } + percentile(&hist, 0.9) +} + +/// Dark, flat lines from the start of `level`, up to `limit`. +fn blank_run(level: &[f64], detail: &[f64], level_bar: f64, detail_bar: f64, limit: usize) -> usize { + let mut k = 0; + while k < limit.min(level.len()) && level[k] <= level_bar && detail[k] <= detail_bar { + k += 1; + } + k +} + +/// `blank_run`, less a run too short to be a border. +fn border_run(level: &[f64], detail: &[f64], level_bar: f64, detail_bar: f64, limit: usize, black: f64) -> usize { + let k = blank_run(level, detail, level_bar, detail_bar, limit); + if k >= MIN_RUN || (k == 1 && level[0] <= black + HARD_BLACK) { + k + } else { + 0 + } +} + +/// The dim ramp after `k` dark lines. +fn ramp_after(level: &[f64], k: usize, limit: usize) -> usize { + if k == 0 { + return 0; + } + let inner: Vec = level.iter().skip(k + 6).take(8).copied().collect(); + let reference = median(&inner); + let mut r = 0; + while r < RAMP_MAX + && k + r + 1 < limit.min(level.len()) + && level[k + r] < reference * RAMP_KNEE + && level[k + r + 1] > level[k + r] + { + r += 1; + } + r +} + +/// The measurement, before any rounding or orientation. +#[derive(Debug, Clone, PartialEq)] +pub struct Analysis { + pub edges: EdgeFindings, + pub usable_frames: usize, + /// Why nothing could be measured, when that is the case. + pub inconclusive: Option, +} + +/// Measure `frames` (all the same size). `interlaced` compares each line with +/// the one above it in the same field; `full_range` puts black at 0, not 16. +pub fn analyse(frames: &[Frame], interlaced: bool, full_range: bool) -> Analysis { + let mut result = Analysis { edges: EdgeFindings::default(), usable_frames: 0, inconclusive: None }; + let Some(first) = frames.first() else { + result.inconclusive = Some("No frames could be decoded.".into()); + return result; + }; + let (w, h) = (first.width, first.height); + if w < 32 || h < 32 { + result.inconclusive = Some(format!("The video is too small to analyse ({w}x{h}).")); + return result; + } + let d = if interlaced { 2 } else { 1 }; + let search = ((h as f64 * BAND_SEARCH_FRACTION) as usize).max(4); + + let mut col_hist = vec![0u32; w * 256]; + let mut row_hist = vec![0u32; h * 256]; + let mut per_frame = Vec::new(); + for f in frames.iter().filter(|f| f.width == w && f.height == h) { + if picture_level(f) < DARK_FRAME { + continue; + } + let (cols, rows) = line_stats(f, &mut col_hist, &mut row_hist); + per_frame.push(FrameStats { cols, rows, torn: torn_lines(f, search, d) }); + } + result.usable_frames = per_frame.len(); + if per_frame.is_empty() { + result.inconclusive = Some( + "Every sampled frame is too dark to show where the picture stops.".into(), + ); + return result; + } + + // The whole sample, line by line. + let n = per_frame.len() as f64; + let mean = |pick: &dyn Fn(&FrameStats) -> &Vec, len: usize| -> Vec { + let mut acc = vec![0f64; len]; + for fs in &per_frame { + for (a, v) in acc.iter_mut().zip(pick(fs)) { + *a += v / n; + } + } + acc + }; + let cols = Lines { + level: (0..w).map(|x| percentile(&col_hist[x * 256..(x + 1) * 256], LEVEL_PERCENTILE)).collect(), + detail: mean(&|fs| &fs.cols.detail, w), + }; + let rows = Lines { + level: (0..h).map(|y| percentile(&row_hist[y * 256..(y + 1) * 256], LEVEL_PERCENTILE)).collect(), + detail: mean(&|fs| &fs.rows.detail, h), + }; + + let black = if full_range { 0.0 } else { 16.0 }; + let (x0, x1) = inset(w); + let (y0, y1) = inset(h); + let col_plateau = median(&cols.level[x0..x1]); + let row_plateau = median(&rows.level[y0..y1]); + let col_bar = black + BLANK_LEVEL * (col_plateau - black); + let row_bar = black + BLANK_LEVEL * (row_plateau - black); + let col_flat = (median(&cols.detail[x0..x1]) * BLANK_DETAIL).max(BLANK_DETAIL_FLOOR); + let row_flat = (median(&rows.detail[y0..y1]) * BLANK_DETAIL).max(BLANK_DETAIL_FLOOR); + let rev = |v: &[f64]| -> Vec { v.iter().rev().copied().collect() }; + + // One side: the dark run and the ramp, and how many frames agree. + let side = |level: &[f64], detail: &[f64], bar: f64, flat: f64, len: usize, per: &dyn Fn(&FrameStats) -> (Vec, Vec)| -> EdgeFinding { + let limit = (len as f64 * BAR_MAX_FRACTION) as usize; + let blank = border_run(level, detail, bar, flat, limit, black); + let ramp = ramp_after(level, blank, limit); + let agree = per_frame + .iter() + .filter(|fs| { + let (l, dd) = per(fs); + border_run(&l, &dd, bar, flat, limit, black).abs_diff(blank) <= AGREEMENT + }) + .count(); + let is_bar = blank as f64 > len as f64 * BLANKING_MAX_FRACTION; + let total = (blank + ramp) as i32; + EdgeFinding { + kind: if blank == 0 { + EdgeKind::None + } else if is_bar { + EdgeKind::Bar + } else { + EdgeKind::Blanking + }, + blank: blank as i32, + ramp: ramp as i32, + band: 0, + bar: if is_bar { total } else { 0 }, + crop: if is_bar { 0 } else { total }, + crop_with_bars: total, + confidence: agree as f64 / n, + } + }; + + result.edges.left = side(&cols.level, &cols.detail, col_bar, col_flat, w, + &|fs| (fs.cols.level.clone(), fs.cols.detail.clone())); + result.edges.right = side(&rev(&cols.level), &rev(&cols.detail), col_bar, col_flat, w, + &|fs| (rev(&fs.cols.level), rev(&fs.cols.detail))); + result.edges.top = side(&rows.level, &rows.detail, row_bar, row_flat, h, + &|fs| (fs.rows.level.clone(), fs.rows.detail.clone())); + let mut bottom = side(&rev(&rows.level), &rev(&rows.detail), row_bar, row_flat, h, + &|fs| (rev(&fs.rows.level), rev(&fs.rows.detail))); + + // Head switching, upward from any dark lines at the very bottom. + let torn_fraction: Vec = (0..search) + .map(|o| per_frame.iter().filter(|fs| fs.torn[o]).count() as f64 / n) + .collect(); + let is_band = |o: usize| o < search && torn_fraction[o] >= BAND_FRAME_FRACTION; + let start = if bottom.kind == EdgeKind::Bar { 0 } else { bottom.blank as usize }; + let mut o = start; + while o < start + BAND_SKIP && o < search && !is_band(o) { + o += 1; + } + if is_band(o) { + let first = o; + let mut last = o; + let mut gap = 0; + while o < search { + if is_band(o) { + last = o; + gap = 0; + } else { + gap += 1; + if gap > BAND_GAP { + break; + } + } + o += 1; + } + let rows_torn = (first..=last).filter(|&o| is_band(o)).count(); + if rows_torn >= MIN_RUN { + let band_end = last + 1; + let rrows = rev(&rows.level); + let rdetail = rev(&rows.detail); + // Anything dark above the band: blanking, or a letterbox bar. + let limit = (h as f64 * BAR_MAX_FRACTION) as usize; + let above = border_run(&rrows[band_end..], &rdetail[band_end..], row_bar, row_flat, limit, black); + let ramp = ramp_after(&rrows[band_end..], above, limit); + let above_is_bar = above as f64 > h as f64 * BLANKING_MAX_FRACTION; + let band_conf = (first..=last).map(|o| torn_fraction[o]).sum::() + / (last - first + 1) as f64; + let dark_below = if bottom.kind == EdgeKind::Bar { 0 } else { bottom.blank }; + bottom = EdgeFinding { + kind: EdgeKind::HeadSwitching, + // Black lines below the band, the band, then whatever dark + // lines sit above it: blanking (cropped with it, counted as + // ramp here) or a bar (kept apart). + blank: dark_below, + ramp: if above_is_bar { 0 } else { (above + ramp) as i32 }, + band: (band_end - start) as i32, + bar: if above_is_bar { (above + ramp) as i32 } else { 0 }, + crop: if above_is_bar { + band_end as i32 + } else { + (band_end + above + ramp) as i32 + }, + crop_with_bars: (band_end + above + ramp) as i32, + confidence: band_conf, + }; + } + } + result.edges.bottom = bottom; + result +} + +/// What `--detect-crop` prints. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct CropReport { + /// Whether anything worth cropping was found. + pub found: bool, + /// The source's size. + pub width: i32, + pub height: i32, + /// Whether lines were compared within each field. + pub interlaced: bool, + /// Frames sampled, and how many were bright enough to use. + pub frames: Vec, + pub usable_frames: i32, + /// The recommended crop for the Crop controls: blanking, ramps and the + /// head-switching band, not bars. In the orientation the Crop pass sees + /// (after Rotate / Flip), rounded outward to the steps below. + pub crop: Sides, + /// The same with letterbox / pillarbox bars included; `None` with no bars. + pub crop_with_bars: Option, + /// Edge Repair's widths for repairing the head-switching band instead of + /// cropping it (in the source's own orientation, which is the one Edge + /// Repair works in), with `crop_with_repair` for the rest; `None` with no + /// band. + pub edge_repair: Option, + pub crop_with_repair: Option, + /// Each edge as measured, in the Crop pass's orientation. + pub edges: EdgeFindings, + /// The lowest confidence among the edges that found something. + pub confidence: f64, + /// Horizontal and vertical rounding steps the crop obeys. + pub step_x: i32, + pub step_y: i32, + /// Why nothing was found, in words, when that is the case. + pub message: Option, +} + +/// Round `v` up to a multiple of `step`. +fn round_up(v: i32, step: i32) -> i32 { + (v.max(0) + step - 1) / step * step +} + +/// Map the source's four sides to the orientation after Rotate / Flip, which +/// is the one the Crop pass works in. +pub fn orient(pipeline: &ProcessingPipeline, left: T, right: T, top: T, bottom: T) -> (T, T, T, T) { + let g = &pipeline.geometry; + if !g.has_effect() { + return (left, right, top, bottom); + } + // A clockwise quarter turn puts the source's left edge at the top. + let (l, r, t, b) = match g.rotation { + Rotation::None => (left, right, top, bottom), + Rotation::Cw90 => (bottom, top, left, right), + Rotation::Rotate180 => (right, left, bottom, top), + Rotation::Ccw90 => (top, bottom, right, left), + }; + let (l, r) = if g.flip_horizontal { (r, l) } else { (l, r) }; + let (t, b) = if g.flip_vertical { (b, t) } else { (t, b) }; + (l, r, t, b) +} + +/// The rounding steps for a job's crop: even everywhere (the Crop controls +/// move in twos, which keeps 4:2:0 and 4:2:2 chroma aligned), and a multiple +/// of four vertically when a 4:2:0 picture is still interlaced at the crop — +/// each field's chroma lines alternate too, so cropping two lines would swap +/// them. +pub fn steps(job: &VideoJob) -> (i32, i32) { + let pipeline = job.effective_pipeline(); + let pipe = crate::pixel_format::decode_pixel_format(job.input_pixel_format.as_deref()); + let (class, _) = crate::pixel_format::chroma_and_depth(&pipe.name); + let still_interlaced = + crate::script_generator::ScriptGenerator::still_interlaced_tff(job, &pipeline).is_some(); + let y = if still_interlaced && class == crate::pixel_format::ChromaClass::C420 { 4 } else { 2 }; + (2, y) +} + +/// Turn the measurement into the report: orientation, rounding, the +/// alternatives. +pub fn report_from(analysis: &Analysis, job: &VideoJob, frames: Vec, interlaced: bool) -> CropReport { + let pipeline = job.effective_pipeline(); + let (sx, sy) = steps(job); + let e = &analysis.edges; + let (w, h) = (job.input_width.unwrap_or(0), job.input_height.unwrap_or(0)); + + // Rounded in the Crop pass's orientation: a side that ends up horizontal + // rounds to the horizontal step. + let place = |pick: &dyn Fn(&EdgeFinding) -> i32| -> Sides { + let (l, r, t, b) = orient(&pipeline, pick(&e.left), pick(&e.right), pick(&e.top), pick(&e.bottom)); + Sides { + left: round_up(l, sx), + right: round_up(r, sx), + top: round_up(t, sy), + bottom: round_up(b, sy), + } + }; + let crop = place(&|f| f.crop); + let with_bars = place(&|f| f.crop_with_bars); + let has_bars = [e.left, e.right, e.top, e.bottom].iter().any(|f| f.bar > 0); + + // The band can be repaired instead: Edge Repair rebuilds the bottom lines + // in the source's orientation (it runs before Rotate / Flip), and the crop + // keeps everything else it found. + // Edge Repair works from the frame's edge, so it covers any black lines + // below the band as well as the band. + let (edge_repair, crop_with_repair) = if e.bottom.kind == EdgeKind::HeadSwitching && e.bottom.band > 0 { + let reach = e.bottom.blank + e.bottom.band; + let repair = crate::models::EdgeRepairParameters::even(round_up(reach, 2)); + if repair >= reach && repair <= EDGE_REPAIR_MAX { + let mut rest = *e; + rest.bottom.crop -= reach; + let (l, r, t, b) = orient(&pipeline, rest.left.crop, rest.right.crop, rest.top.crop, rest.bottom.crop); + ( + Some(Sides { bottom: repair, ..Sides::default() }), + Some(Sides { + left: round_up(l, sx), + right: round_up(r, sx), + top: round_up(t, sy), + bottom: round_up(b, sy), + }), + ) + } else { + (None, None) + } + } else { + (None, None) + }; + + let (l, r, t, b) = orient(&pipeline, e.left, e.right, e.top, e.bottom); + let edges = EdgeFindings { left: l, right: r, top: t, bottom: b }; + let confidence = [l, r, t, b] + .iter() + .filter(|f| f.kind != EdgeKind::None) + .map(|f| f.confidence) + .fold(1.0, f64::min); + let found = analysis.inconclusive.is_none() && (!crop.is_zero() || has_bars); + let message = match &analysis.inconclusive { + Some(m) => Some(m.clone()), + None if !found => Some("No black borders, blanking or head-switching noise at the edges.".into()), + None => None, + }; + CropReport { + found, + width: w, + height: h, + interlaced, + frames, + usable_frames: analysis.usable_frames as i32, + crop, + crop_with_bars: has_bars.then_some(with_bars), + edge_repair, + crop_with_repair, + edges, + confidence, + step_x: sx, + step_y: sy, + message, + } +} + +/// Which frames to sample: `SAMPLE_FRAMES` spread through the clip or its +/// trim, leaving the first and last 5 % (titles, fades). +pub fn sample_frames(total: Option, start: Option, end: Option) -> Vec { + sample_frames_n(SAMPLE_FRAMES, total, start, end) +} + +/// [`sample_frames`] with `count` samples instead of `SAMPLE_FRAMES`. +pub fn sample_frames_n(count: usize, total: Option, start: Option, end: Option) -> Vec { + let first = start.unwrap_or(0); + let last = match (end, total) { + (Some(e), _) => e, + (None, Some(t)) if t > 0 => t - 1, + _ => return vec![first as i32], + }; + if last <= first { + return vec![first as i32]; + } + let span = (last - first) as f64; + let mut frames: Vec = (0..count) + .map(|i| (first as f64 + span * (0.05 + 0.9 * (i as f64 + 0.5) / count as f64)).round() as i32) + .collect(); + frames.dedup(); + frames +} + +/// Whether the source's lines alternate between two fields. +pub fn is_interlaced(job: &VideoJob) -> bool { + matches!( + job.detected_field_order, + Some(FieldOrder::TopFieldFirst) | Some(FieldOrder::BottomFieldFirst) + ) +} + +/// The job's probed frame size, or the error every detector reports. +pub fn frame_size(job: &VideoJob) -> Result<(i32, i32)> { + match (job.input_width, job.input_height) { + (Some(w), Some(h)) if w > 0 && h > 0 => Ok((w, h)), + _ => bail!("The video's size is unknown; re-add it to the queue."), + } +} + +/// Decode `count` consecutive frames from each of `frames` (source frame +/// numbers) as raw `pix_fmt`, a few decoders at a time (each seeks on its +/// own), and turn each decoder's output into a `T` with `parse` (`None` for a +/// short decode). A sample that fails is skipped as long as enough others +/// decode; the samples that did come back are returned in order. Shared by +/// the detectors (`--detect-crop`, `--detect-noise-halo`), so they sample and +/// decode the same way: the stored frame at its probed size, never resampled. +pub fn decode_samples( + deps: &DependencyLocator, + job: &VideoJob, + frames: &[i32], + count: i32, + pix_fmt: &str, + parse: impl Fn(&[u8]) -> Option + Sync, +) -> Result> { + let ffmpeg = deps.ffmpeg_path()?; + let env = deps.build_environment(); + let (width, height) = frame_size(job)?; + let fps = job.input_frame_rate.filter(|f| *f > 0.0).unwrap_or(29.97); + + let decode = |frame: i32| -> Result { + let seek = ((frame as f64 - 0.5) / fps).max(0.0); + let out = Command::new(&ffmpeg) + .args(source_decode::preview_decoder_args( + &job.input_path, seek, count, width, height, pix_fmt, "pipe:1", + )) + .envs(&env) + .stdin(Stdio::null()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .context("running ffmpeg")?; + if !out.status.success() { + let stderr = String::from_utf8_lossy(&out.stderr); + if let Some(msg) = source_decode::explain_decoder_failure(&stderr, width, height) { + bail!("{msg}"); + } + bail!("Could not decode frame {frame}: {stderr}"); + } + parse(&out.stdout).with_context(|| format!("short decode at frame {frame}")) + }; + let decode = &decode; + let mut decoded = Vec::new(); + for chunk in frames.chunks(4) { + let results: Vec> = std::thread::scope(|scope| { + let handles: Vec<_> = chunk.iter().map(|&f| scope.spawn(move || decode(f))).collect(); + handles.into_iter().map(|h| h.join().unwrap_or_else(|_| bail!("decoder thread panicked"))).collect() + }); + for r in results { + match r { + Ok(f) => decoded.push(f), + // A frame past a damaged stretch is not worth failing for, as + // long as enough others decode. + Err(e) if decoded.len() + frames.len() > 6 => eprintln!("detect: {e:#}"), + Err(e) => return Err(e), + } + } + } + if decoded.is_empty() { + bail!("No frames could be decoded."); + } + Ok(decoded) +} + +/// Decode the sample frames and measure them. +pub fn detect(deps: &DependencyLocator, job: &VideoJob) -> Result { + let (width, height) = frame_size(job)?; + let frames = sample_frames( + job.total_frames.map(|f| f.max(0) as u64), + job.start_frame.map(|f| f.max(0) as u64), + job.end_frame.map(|f| f.max(0) as u64), + ); + let decoded = decode_samples(deps, job, &frames, 1, "yuv444p", |bytes| { + Frame::from_yuv444p(bytes, width as usize, height as usize) + })?; + let interlaced = is_interlaced(job); + let analysis = analyse(&decoded, interlaced, job.color_metadata().is_full_range()); + Ok(report_from(&analysis, job, frames, interlaced)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::{GeometryParameters, QTGMCParameters}; + + /// A deterministic noise source, so fixtures are repeatable. + struct Lcg(u64); + impl Lcg { + fn next(&mut self) -> f64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + ((self.0 >> 33) as f64) / (1u64 << 31) as f64 + } + fn noise(&mut self, amp: f64) -> f64 { + (self.next() - 0.5) * 2.0 * amp + } + } + + /// What to draw into a fixture. + #[derive(Clone, Copy, Default)] + struct Spec { + w: usize, + h: usize, + /// Black borders / blanking per side. + left: usize, + right: usize, + top: usize, + bottom: usize, + /// A dim ramp of this many columns inside the left and right borders. + ramp: usize, + /// Noise in the blanking (levels). + blank_noise: f64, + /// Head-switching lines at the bottom (per frame, i.e. both fields). + band: usize, + /// A burnt-in subtitle in this fraction of frames, its bottom this + /// many lines above the frame's bottom. + subtitle_every: usize, + subtitle_gap: usize, + /// Every n-th frame is a fade to (near) black. + dark_every: usize, + /// The picture is interlaced: fields differ (motion). + interlaced: bool, + /// The picture's own edge columns are dark in most frames (a night scene). + dark_edges: bool, + } + + /// A frame of picture: gradients, texture and noise, moving with `t`. + fn frame(spec: &Spec, t: usize, rng: &mut Lcg) -> Frame { + let (w, h) = (spec.w, spec.h); + let mut y = vec![16u8; w * h]; + let mut u = vec![128u8; w * h]; + let mut v = vec![128u8; w * h]; + let dark = spec.dark_every > 0 && t % spec.dark_every == 0; + for yy in spec.top..h - spec.bottom { + for xx in spec.left..w - spec.right { + let field_shift = if spec.interlaced && yy % 2 == 1 { 3.0 } else { 0.0 }; + let xs = xx as f64 + t as f64 * 5.0 + field_shift; + let base = 60.0 + 100.0 * (0.5 + 0.5 * (xs / 37.0).sin() * (yy as f64 / 23.0).cos()); + let mut lum = base + rng.noise(6.0); + if spec.dark_edges && t % 4 != 0 { + let edge = (xx.min(w - 1 - xx) as f64 / (w as f64 * 0.15)).min(1.0); + lum = 16.0 + (lum - 16.0) * edge; + } + if dark { + lum = 18.0 + rng.noise(2.0); + } + let i = yy * w + xx; + y[i] = lum.clamp(16.0, 235.0) as u8; + u[i] = (128.0 + 20.0 * (xs / 61.0).sin() + rng.noise(2.0)) as u8; + v[i] = (128.0 + 20.0 * (yy as f64 / 41.0).cos() + rng.noise(2.0)) as u8; + } + } + // The ramp: columns inside each side border fade in. + for r in 0..spec.ramp { + let k = (r + 1) as f64 / (spec.ramp + 1) as f64; + for yy in 0..h { + for xx in [spec.left + r, w - 1 - spec.right - r] { + let i = yy * w + xx; + y[i] = (16.0 + (y[i] as f64 - 16.0) * k * 0.6) as u8; + } + } + } + // Noisy blanking. + if spec.blank_noise > 0.0 { + for yy in 0..h { + for xx in (0..spec.left).chain(w - spec.right..w) { + y[yy * w + xx] = (16.0 + rng.noise(spec.blank_noise).abs()) as u8; + } + } + } + // A subtitle: a short run of bright text-like blocks in the middle. + if spec.subtitle_every > 0 && t % spec.subtitle_every == 0 { + let bottom = h - spec.bottom - spec.subtitle_gap; + for yy in bottom - 18..bottom { + for xx in w * 3 / 8..w * 5 / 8 { + if (xx / 7 + yy / 5 + t) % 3 != 0 { + y[yy * w + xx] = 230; + } + } + } + } + // Head switching: the bottom lines torn sideways with smeared, noisy + // luma and wrecked chroma. + for o in 0..spec.band { + let yy = h - 1 - o; + let shift = 6 + (o * 3 + t) % 9; + let row: Vec = y[(yy - 8) * w..(yy - 7) * w].to_vec(); + for xx in 0..w { + let i = yy * w + xx; + let src = row[(xx + w - shift) % w] as f64; + y[i] = (src * 0.7 + 40.0 + rng.noise(25.0)).clamp(16.0, 235.0) as u8; + u[i] = (128.0 + rng.noise(40.0)) as u8; + v[i] = (128.0 + rng.noise(40.0)) as u8; + } + } + Frame { width: w, height: h, y, u, v } + } + + fn frames(spec: &Spec, n: usize) -> Vec { + let mut rng = Lcg(42); + (0..n).map(|t| frame(spec, t, &mut rng)).collect() + } + + fn sd(spec: Spec) -> Spec { + Spec { w: 720, h: 576, ..spec } + } + + fn near(actual: i32, expected: usize, what: &str) { + assert!( + (actual - expected as i32).abs() <= 2, + "{what}: measured {actual}, expected {expected} (within 2)" + ); + } + + #[test] + fn blanking_ramps_and_band_are_found_within_two_lines() { + let spec = sd(Spec { left: 12, right: 8, top: 4, bottom: 0, ramp: 0, band: 8, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + assert!(a.inconclusive.is_none()); + near(a.edges.left.crop, 12, "left"); + near(a.edges.right.crop, 8, "right"); + near(a.edges.top.crop, 4, "top"); + assert_eq!(a.edges.bottom.kind, EdgeKind::HeadSwitching); + near(a.edges.bottom.crop, 8, "bottom band"); + assert_eq!(a.edges.left.kind, EdgeKind::Blanking); + assert!(a.edges.left.confidence > 0.9); + } + + #[test] + fn a_ramp_is_followed_but_not_into_the_picture() { + let spec = sd(Spec { left: 10, right: 10, ramp: 3, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + near(a.edges.left.crop, 13, "left with ramp"); + near(a.edges.right.crop, 13, "right with ramp"); + assert!(a.edges.left.ramp >= 1, "the ramp is reported"); + } + + #[test] + fn noisy_blanking_is_still_blanking() { + let spec = sd(Spec { left: 14, right: 14, blank_noise: 14.0, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + near(a.edges.left.crop, 14, "noisy left"); + near(a.edges.right.crop, 14, "noisy right"); + } + + #[test] + fn one_line_of_true_black_counts_but_one_dim_line_does_not() { + let a = analyse(&frames(&sd(Spec { top: 1, ..Spec::default() }), 12), false, false); + assert_eq!(a.edges.top.crop, 1, "a capture card's black first line"); + // A single dark-but-not-black line is a hard edge in the picture. + let mut fs = frames(&sd(Spec::default()), 12); + for f in &mut fs { + for x in 0..f.width { + f.y[x] = 40; + } + } + let a = analyse(&fs, false, false); + assert_eq!(a.edges.top.crop, 0); + } + + #[test] + fn a_clean_frame_has_nothing_to_crop() { + let a = analyse(&frames(&sd(Spec::default()), 16), false, false); + assert_eq!(a.edges.left.crop + a.edges.right.crop + a.edges.top.crop + a.edges.bottom.crop, 0); + assert_eq!(a.edges.bottom.kind, EdgeKind::None); + } + + #[test] + fn letterbox_bars_are_reported_but_not_cropped_by_default() { + let spec = sd(Spec { top: 72, bottom: 72, left: 8, right: 8, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + assert_eq!(a.edges.top.kind, EdgeKind::Bar); + assert_eq!(a.edges.bottom.kind, EdgeKind::Bar); + assert_eq!(a.edges.top.crop, 0); + near(a.edges.top.crop_with_bars, 72, "letterbox top"); + near(a.edges.bottom.crop_with_bars, 72, "letterbox bottom"); + near(a.edges.left.crop, 8, "blanking beside a letterbox"); + } + + #[test] + fn a_band_under_a_letterbox_is_cropped_and_the_bar_kept() { + // Head switching appears at the bottom of the frame whatever the + // picture's framing, so it tears the bottom of the bar. + let spec = sd(Spec { top: 72, bottom: 72, band: 8, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + assert_eq!(a.edges.bottom.kind, EdgeKind::HeadSwitching); + near(a.edges.bottom.crop, 8, "band only"); + near(a.edges.bottom.crop_with_bars, 72, "band and bar"); + } + + #[test] + fn a_subtitle_near_the_bottom_is_not_head_switching() { + // In most frames, ending only 6 lines above the bottom. + let spec = sd(Spec { subtitle_every: 1, subtitle_gap: 6, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + assert_ne!(a.edges.bottom.kind, EdgeKind::HeadSwitching); + assert_eq!(a.edges.bottom.crop, 0); + // And with a real band beneath it, only the band is taken. + let spec = sd(Spec { subtitle_every: 1, subtitle_gap: 10, band: 6, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), false, false); + near(a.edges.bottom.crop, 6, "band under a subtitle"); + } + + #[test] + fn fades_and_dark_edges_do_not_widen_the_crop() { + let spec = sd(Spec { left: 10, right: 10, dark_every: 3, dark_edges: true, ..Spec::default() }); + let a = analyse(&frames(&spec, 24), false, false); + assert!(a.usable_frames < 24, "the fades are left out"); + near(a.edges.left.crop, 10, "left in a dark film"); + near(a.edges.right.crop, 10, "right in a dark film"); + } + + #[test] + fn an_all_black_sample_is_inconclusive() { + let spec = sd(Spec { dark_every: 1, ..Spec::default() }); + let a = analyse(&frames(&spec, 8), false, false); + assert!(a.inconclusive.is_some()); + } + + #[test] + fn interlaced_band_is_measured_per_field() { + let spec = sd(Spec { left: 8, right: 8, band: 12, interlaced: true, ..Spec::default() }); + let a = analyse(&frames(&spec, 16), true, false); + assert_eq!(a.edges.bottom.kind, EdgeKind::HeadSwitching); + near(a.edges.bottom.crop, 12, "interlaced band"); + near(a.edges.left.crop, 8, "interlaced left"); + } + + #[test] + fn hd_pillarbox_is_a_bar() { + let spec = Spec { w: 960, h: 540, left: 120, right: 120, ..Spec::default() }; + let a = analyse(&frames(&spec, 12), false, false); + assert_eq!(a.edges.left.kind, EdgeKind::Bar); + near(a.edges.left.crop_with_bars, 120, "pillarbox"); + assert_eq!(a.edges.left.crop, 0); + } + + fn job(w: i32, h: i32, pix: &str, field: Option, pipeline: ProcessingPipeline) -> VideoJob { + let mut j: VideoJob = serde_json::from_value(serde_json::json!({ + "id": "00000000-0000-0000-0000-000000000000", + "inputPath": "in.avi", + "outputPath": "out.mkv", + "qtgmcParameters": {}, + "encodingSettings": {}, + })) + .unwrap(); + j.input_width = Some(w); + j.input_height = Some(h); + j.input_pixel_format = Some(pix.into()); + j.detected_field_order = field; + j.processing_pipeline = Some(pipeline); + j + } + + fn analysis(left: i32, right: i32, top: i32, band: i32) -> Analysis { + let f = |crop: i32, kind| EdgeFinding { kind, blank: crop, crop, crop_with_bars: crop, confidence: 1.0, ..Default::default() }; + Analysis { + edges: EdgeFindings { + left: f(left, EdgeKind::Blanking), + right: f(right, EdgeKind::Blanking), + top: f(top, EdgeKind::Blanking), + bottom: EdgeFinding { + kind: EdgeKind::HeadSwitching, + band, + crop: band, + crop_with_bars: band, + confidence: 0.8, + ..Default::default() + }, + }, + usable_frames: 10, + inconclusive: None, + } + } + + #[test] + fn the_report_rounds_outward_to_steps_the_crop_pass_accepts() { + let progressive = ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + ..ProcessingPipeline::default() + }; + let r = report_from(&analysis(11, 9, 3, 7), &job(720, 576, "yuv420p", None, progressive.clone()), vec![1], false); + assert!(r.found); + assert_eq!(r.crop, Sides { left: 12, right: 10, top: 4, bottom: 8 }); + assert_eq!((r.step_x, r.step_y), (2, 2)); + assert_eq!(r.confidence, 0.8); + // The band can be repaired instead: Edge Repair at the bottom, crop the rest. + assert_eq!(r.edge_repair, Some(Sides { bottom: 8, ..Sides::default() })); + assert_eq!(r.crop_with_repair, Some(Sides { left: 12, right: 10, top: 4, bottom: 0 })); + // Still interlaced 4:2:0 at the crop: vertical in fours. + let r = report_from( + &analysis(11, 9, 3, 7), + &job(720, 576, "yuv420p", Some(FieldOrder::TopFieldFirst), progressive.clone()), + vec![1], + true, + ); + assert_eq!((r.crop.top, r.crop.bottom, r.step_y), (4, 8, 4)); + let r = report_from( + &analysis(0, 0, 5, 9), + &job(720, 576, "yuv420p", Some(FieldOrder::TopFieldFirst), progressive.clone()), + vec![1], + true, + ); + assert_eq!((r.crop.top, r.crop.bottom), (8, 12)); + // Deinterlaced before the crop: twos again. + let r = report_from( + &analysis(0, 0, 5, 9), + &job(720, 576, "yuv420p", Some(FieldOrder::TopFieldFirst), ProcessingPipeline::default()), + vec![1], + true, + ); + assert_eq!((r.crop.top, r.crop.bottom, r.step_y), (6, 10, 2)); + // 4:2:2 interlaced: twos. + let r = report_from( + &analysis(0, 0, 5, 9), + &job(720, 576, "yuv422p", Some(FieldOrder::TopFieldFirst), progressive), + vec![1], + true, + ); + assert_eq!(r.step_y, 2); + } + + #[test] + fn the_report_follows_rotate_and_flip() { + let turned = |rotation, flip_horizontal| ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + geometry: GeometryParameters { enabled: true, rotation, flip_horizontal, flip_vertical: false }, + ..ProcessingPipeline::default() + }; + let a = analysis(2, 4, 6, 8); + let crop = |p| report_from(&a, &job(720, 576, "yuv420p", None, p), vec![1], false).crop; + // Clockwise: left -> top, top -> right, right -> bottom, bottom -> left. + assert_eq!(crop(turned(Rotation::Cw90, false)), Sides { left: 8, right: 6, top: 2, bottom: 4 }); + assert_eq!(crop(turned(Rotation::Ccw90, false)), Sides { left: 6, right: 8, top: 4, bottom: 2 }); + assert_eq!(crop(turned(Rotation::Rotate180, false)), Sides { left: 4, right: 2, top: 8, bottom: 6 }); + assert_eq!(crop(turned(Rotation::None, true)), Sides { left: 4, right: 2, top: 6, bottom: 8 }); + // Edge Repair works before the turn, so it stays at the source's bottom. + let r = report_from(&a, &job(720, 576, "yuv420p", None, turned(Rotation::Cw90, false)), vec![1], false); + assert_eq!(r.edge_repair, Some(Sides { bottom: 8, ..Sides::default() })); + assert_eq!(r.crop_with_repair.unwrap().left, 0); + } + + #[test] + fn repairing_the_band_covers_the_black_lines_below_it() { + // Two black lines at the very bottom, then a 7-line band. + let mut a = analysis(4, 4, 0, 0); + a.edges.bottom = EdgeFinding { + kind: EdgeKind::HeadSwitching, + blank: 2, + band: 7, + crop: 9, + crop_with_bars: 9, + confidence: 1.0, + ..Default::default() + }; + let r = report_from(&a, &job(720, 576, "yuv422p", None, ProcessingPipeline::default()), vec![1], false); + assert_eq!(r.crop.bottom, 10); + assert_eq!(r.edge_repair, Some(Sides { bottom: 10, ..Sides::default() })); + assert_eq!(r.crop_with_repair.unwrap().bottom, 0); + // A band too tall for the Edge Repair panel is only ever cropped. + a.edges.bottom.band = 16; + a.edges.bottom.crop = 18; + let r = report_from(&a, &job(720, 576, "yuv422p", None, ProcessingPipeline::default()), vec![1], false); + assert!(r.edge_repair.is_none()); + } + + #[test] + fn nothing_found_says_so() { + let mut a = analysis(0, 0, 0, 0); + a.edges.bottom = EdgeFinding::default(); + let r = report_from(&a, &job(720, 576, "yuv420p", None, ProcessingPipeline::default()), vec![1], false); + assert!(!r.found); + assert!(r.message.is_some()); + assert!(r.edge_repair.is_none()); + } + + #[test] + fn samples_spread_through_the_clip_or_its_trim() { + let all = sample_frames(Some(1000), None, None); + assert_eq!(all.len(), SAMPLE_FRAMES); + assert!(all[0] >= 50 && *all.last().unwrap() <= 950); + let trim = sample_frames(Some(1000), Some(100), Some(199)); + assert!(trim.iter().all(|&f| (100..=199).contains(&f))); + assert_eq!(sample_frames(None, None, None), vec![0]); + assert!(sample_frames(Some(5), None, None).len() <= 5); + } + + #[test] + fn report_json_is_camel_case() { + let r = report_from(&analysis(2, 2, 2, 6), &job(720, 576, "yuv420p", None, ProcessingPipeline::default()), vec![3], false); + let v = serde_json::to_value(&r).unwrap(); + for key in ["found", "crop", "cropWithBars", "edgeRepair", "cropWithRepair", "edges", "usableFrames", "stepY"] { + assert!(v.get(key).is_some(), "{key}"); + } + assert_eq!(v["edges"]["bottom"]["kind"], "headSwitching"); + } +} diff --git a/worker/src/dependency_locator.rs b/worker/src/dependency_locator.rs index b511fe28..e5b55c07 100644 --- a/worker/src/dependency_locator.rs +++ b/worker/src/dependency_locator.rs @@ -12,9 +12,10 @@ use crate::gpu_plugin::{self, GpuPlugin, GpuPluginFiles, GpuPluginReason, GpuPlu /// Why the KNLMeans noise-reduction method can't run, as the app shows it. pub const KNLM_NO_DEVICE_MESSAGE: &str = "KNLMeans runs on the graphics card through OpenCL, \ and no OpenCL device that works with it was found on this computer. Updating the graphics \ -driver may fix this. BM3D does the same job on the CPU and needs nothing extra."; +driver may fix this. Or choose NLMeans (CPU): the same filter with the same settings, run \ +on the processor instead, so it needs nothing extra."; pub const KNLM_DISABLED_MESSAGE: &str = "KNLMeans is turned off on this computer \ -(VAPOURBOX_DISABLE_OPENCL is set). BM3D does the same job on the CPU."; +(VAPOURBOX_DISABLE_OPENCL is set). NLMeans (CPU) is the same filter run on the processor."; /// Platform-specific dependency locator. pub struct DependencyLocator { @@ -862,6 +863,85 @@ impl DependencyLocator { PathBuf::from("addons") } + /// A manifest add-on's directory (`addons//`, see `addons.rs`). + pub fn addon_dir(&self, id: &str) -> PathBuf { + crate::addons::addon_dir(&self.addons_dir(), id) + } + + /// Refuse unless every add-on in `ids` (and what they require) is + /// installed for this platform, naming them and Settings → Add-ons. + /// `feature` starts the message ("Face restoration needs …"). + pub fn require_addons(&self, ids: &[&str], feature: &str) -> Result<()> { + crate::addons::require(&self.addons_dir(), ids, feature, self.platform_suffix()) + } + + /// Whether the AI runtime add-on runs here, and on which provider (see + /// `ai_runtime.rs`). As for the GPU plugins, only a positive result is + /// cached for the boot — keyed by the add-on's version and the force-CPU + /// setting too, so reinstalling or upgrading it re-probes — because a + /// failure is what the user is about to fix. + pub fn ai_runtime_status(&self) -> crate::ai_runtime::AiRuntimeStatus { + use crate::ai_runtime::{self, AiRuntimeReason, AiRuntimeStatus}; + let addons = self.addons_dir(); + if let Some(status) = ai_runtime::precheck(&addons, self.platform_suffix()) { + return status; + } + let force_cpu = ai_runtime::force_cpu_requested(); + let version = crate::addons::manifest(crate::addons::AI_RUNTIME) + .map(|m| m.version.clone()) + .unwrap_or_default(); + let boot = Self::boot_token(); + let cache_file = self.platform_dir().join("ai-runtime-probe.json"); + if !boot.is_empty() { + if let Ok(txt) = std::fs::read_to_string(&cache_file) { + if let Ok(json) = serde_json::from_str::(&txt) { + if json.get("boot").and_then(|v| v.as_str()) == Some(boot.as_str()) + && json.get("addonVersion").and_then(|v| v.as_str()) == Some(version.as_str()) + && json.get("forceCpu").and_then(|v| v.as_bool()) == Some(force_cpu) + { + if let Some(status) = json + .get("status") + .and_then(|s| serde_json::from_value::(s.clone()).ok()) + .filter(|s| s.available) + { + return status; + } + } + } + } + } + let vspipe = match self.vspipe_path() { + Ok(p) => p, + Err(e) => return AiRuntimeStatus::unavailable(AiRuntimeReason::ProbeFailed, e.to_string()), + }; + let templates = match crate::script_generator::ScriptGenerator::pipe_source_dir() { + Ok(p) => p, + Err(e) => return AiRuntimeStatus::unavailable(AiRuntimeReason::ProbeFailed, e.to_string()), + }; + let status = ai_runtime::run_probe( + &vspipe, + &self.build_environment(), + &templates, + &self.addon_dir(crate::addons::AI_RUNTIME), + force_cpu, + ); + if status.available { + let _ = std::fs::write( + &cache_file, + serde_json::json!({ + "boot": boot, + "addonVersion": version, + "forceCpu": force_cpu, + "status": status, + }) + .to_string(), + ); + } else { + let _ = std::fs::remove_file(&cache_file); + } + status + } + /// AnimeJaNai's installed model directory under the add-ons, or the /// message explaining that the add-on is missing — what /// `ScriptGenerator::with_animejanai` takes. diff --git a/worker/src/gpu_plugin.rs b/worker/src/gpu_plugin.rs index d7a90e45..20cea0d8 100644 --- a/worker/src/gpu_plugin.rs +++ b/worker/src/gpu_plugin.rs @@ -19,6 +19,17 @@ //! (a missing runtime DLL, a driver that faults on load, AVX in a static //! initializer) can reach a job that did not ask for it. On macOS both resolve //! Vulkan from MoltenVK, which is preloaded from `vapoursynth/moltenvk/`. +//! +//! The plugins are in the deps bundle; their models are not. Every AI model is +//! an add-on (docs/ADDONS.md): RIFE's is `frame-rate-rife` ([`RIFE_ADDON`]), +//! the upscalers' are listed in `upscale_addons.rs`, and a job that needs one +//! is refused through `ProcessingPipeline::required_addons` like any other +//! add-on. So the probe must not need a model either: it runs each plugin on a +//! stand-in network of its own — for VSNCNN a 264-byte ONNX convolution +//! compiled into the worker ([`NCNN_SELFTEST_MODEL`]), for RIFE a two-layer +//! ncnn network written beside the probe script ([`RIFE_SELFTEST_PARAM`]) — +//! and checks the answer. That tells "this computer can't run it" apart from +//! "the add-on isn't installed" before the user has downloaded anything. use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -27,18 +38,47 @@ use std::path::{Path, PathBuf}; use std::process::{Command, Stdio}; use std::time::{Duration, Instant}; -/// RIFE's bundled model: 4.26, ncnn conversion, ensemble off. +/// The add-on holding RIFE's model (`app/assets/addons/frame-rate-rife.json`). +pub const RIFE_ADDON: &str = "frame-rate-rife"; + +/// RIFE's model directory inside the add-on: 4.26, ncnn conversion, ensemble +/// off. The name is load-bearing: the plugin takes the model's version — and +/// with it the padding, 64 for 4.26 against 32 for most — from substrings of +/// this path, so the add-on installs its files under exactly this name. pub const RIFE_MODEL_DIR_NAME: &str = "rife-v4.26_ensembleFalse"; -/// The upscaling models, as paths under `vapoursynth/ncnn/models/` — vs-mlrt's -/// ONNX conversions, kept under the names vs-mlrt publishes them by. -pub const UPSCALE_MODELS: &[&str] = &[ - "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx", - "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx", - "cugan/up2x-latest-no-denoise.onnx", - "cugan/up2x-latest-conservative.onnx", - "cugan/up2x-latest-denoise3x.onnx", -]; +/// Where a job's RIFE model is, under the add-ons directory. +pub fn rife_model_dir(addons_root: &Path) -> PathBuf { + crate::addons::addon_dir(addons_root, RIFE_ADDON).join(RIFE_MODEL_DIR_NAME) +} + +/// The upscaling probe's model: one 1x1 convolution that swaps the first and +/// third channels and adds 0.125 (`Scripts/models/make_ncnn_selftest.py`, +/// committed as `Tests/TestResources/ncnn_selftest.onnx`). Handed to VSNCNN as +/// bytes, so the probe needs no add-on and writes no model file. +pub const NCNN_SELFTEST_MODEL: &[u8] = include_bytes!("../../Tests/TestResources/ncnn_selftest.onnx"); + +/// The RIFE probe's stand-in for `flownet`: what RIFE's v4 path feeds the +/// network (`in0`, `in1`: the two frames; `in2`: the timestep) and extracts +/// (`out0`), with the average of the two frames in between. No weights, so +/// `flownet.bin` is empty. Run through the plugin's real pre- and +/// post-processing shaders, frame 1 of a doubled two-frame clip is exactly the +/// average of its two frames. +pub const RIFE_SELFTEST_PARAM: &str = "7767517\n\ + 5 6\n\ + Input in0 0 1 in0\n\ + Input in1 0 1 in1\n\ + Input in2 0 1 in2\n\ + BinaryOp sum 2 1 in0 in1 sum 0=0\n\ + BinaryOp out0 1 1 sum out0 0=2 1=1 2=5.000000e-01\n"; + +/// The stand-in's directory name: RIFE reads the model version from the path, +/// so it must say v4 (and 4.26, for the real model's padding). +const RIFE_SELFTEST_DIR_NAME: &str = "rife-v4.26_selftest"; + +/// How far the probe's answer may be from the exact one: the networks run in +/// fp16, whose step near 0.7 is about 0.0005. +const SELFTEST_TOLERANCE: f64 = 0.01; /// How long the probe may take. GPU initialisation and the first shader /// compile take a second or two; a driver that hangs must not hang the app. @@ -119,17 +159,14 @@ fn platform_ships(plugin: GpuPlugin, platform_suffix: &str) -> bool { } } -/// The files a plugin needs, as laid out under `deps//vapoursynth/`. +/// The files a plugin needs from the deps bundle, as laid out under +/// `deps//vapoursynth/`. Its models are add-ons, not listed here. #[derive(Debug, Clone, PartialEq, Eq)] pub struct GpuPluginFiles { pub plugin: PathBuf, - /// RIFE's model directory, or the upscaling models' root. - pub model_dir: PathBuf, /// A library to load into the process before the plugin: MoltenVK on /// macOS, where ncnn resolves Vulkan from whatever is already loaded. pub preload: Option, - /// The model files inside `model_dir` that must exist. - pub models: Vec, } impl GpuPluginFiles { @@ -148,19 +185,7 @@ impl GpuPluginFiles { let preload = platform_suffix .starts_with("macos") .then(|| vs_dir.join("moltenvk").join("libMoltenVK.dylib")); - let (model_dir, models) = match plugin { - GpuPlugin::Rife => { - let d = dir.join("models").join(RIFE_MODEL_DIR_NAME); - let m = vec![d.join("flownet.param"), d.join("flownet.bin")]; - (d, m) - } - GpuPlugin::Upscale => { - let d = dir.join("models"); - let m = UPSCALE_MODELS.iter().map(|m| d.join(m)).collect(); - (d, m) - } - }; - Self { plugin: dir.join(file), model_dir, preload, models } + Self { plugin: dir.join(file), preload } } /// Every file that must exist, in the order a user would want to hear @@ -168,7 +193,6 @@ impl GpuPluginFiles { pub fn required(&self) -> Vec { let mut files = vec![self.plugin.clone()]; files.extend(self.preload.iter().cloned()); - files.extend(self.models.iter().cloned()); files } @@ -193,6 +217,9 @@ pub enum GpuPluginReason { LoadFailed, /// The probe process crashed after the GPU was initialised. Crashed, + /// The plugin ran its self-test network on the GPU and got the wrong + /// answer: a driver that computes garbage, whose output can't be trusted. + WrongResult, /// The plugin used an instruction this CPU does not have (SIGILL, or /// Windows' 0xC000001D) — an x86 machine older than its build target. UnsupportedCpu, @@ -294,6 +321,11 @@ impl GpuPluginStatus { "{name} crashed while testing the GPU. Updating your graphics driver \ usually fixes this; restart VapourBox afterwards." ), + R::WrongResult => format!( + "{name} ran on the GPU but got a test calculation wrong, so its output \ + can't be trusted. Updating your graphics driver usually fixes this; \ + restart VapourBox afterwards." + ), R::UnsupportedCpu => format!( "{name}'s plugin needs a newer processor than this computer has (it uses \ instructions this CPU lacks). {}", @@ -362,9 +394,22 @@ pub fn py_str(path: &Path) -> String { out } -/// The probe: load the plugin, create the GPU instance, and render one frame — -/// the three places it has been seen to fail. -pub fn probe_script(plugin: GpuPlugin, files: &GpuPluginFiles, platform_suffix: &str) -> String { +/// Bytes as a Python `bytes.fromhex(...)` expression. +fn py_bytes(data: &[u8]) -> String { + let hex: String = data.iter().map(|b| format!("{b:02x}")).collect(); + format!("bytes.fromhex('{hex}')") +} + +/// The probe: load the plugin, create the GPU instance, and run its stand-in +/// network on one frame — the three places it has been seen to fail — then +/// check the answer. `selftest_dir` is where [`write_rife_selftest`] put RIFE's +/// stand-in model (unused for the upscaler, whose model is in the script). +pub fn probe_script( + plugin: GpuPlugin, + files: &GpuPluginFiles, + platform_suffix: &str, + selftest_dir: &Path, +) -> String { // Shared libraries whose absence makes the plugin fail to load with only a // generic "module not found". Checked first so the user is told which one. // VSNCNN's Windows build links its runtime statically. @@ -381,16 +426,27 @@ pub fn probe_script(plugin: GpuPlugin, files: &GpuPluginFiles, platform_suffix: }; let libs = runtime_libs.iter().map(|l| format!("'{l}'")).collect::>().join(", "); let preload = files.preload.as_deref().map(py_str).unwrap_or_else(|| "None".into()); - // The call that makes ncnn create its GPU instance and run a network. - let call = match plugin { - GpuPlugin::Rife => format!( - "clip = core.rife.RIFE(clip, model_path={}, fps_num=50, fps_den=1)", - py_str(&files.model_dir) + // The clip, the call that makes ncnn create its GPU instance and run the + // stand-in network, and the answer frame 1 must hold. + let (clip, call, expected) = match plugin { + // Two different frames, doubled: frame 1 is the network's average. + GpuPlugin::Rife => ( + "clip = core.std.BlankClip(width=64, height=64, format=vs.RGBS, length=1, fpsnum=25, fpsden=1, color=[0.2, 0.4, 0.6])\n\ + clip = clip + core.std.BlankClip(clip, color=[0.6, 0.4, 0.2])", + format!( + "clip = core.rife.RIFE(clip, model_path={}, fps_num=50, fps_den=1)", + py_str(selftest_dir) + ), + "[0.4, 0.4, 0.4]", ), - // The smallest bundled network. - GpuPlugin::Upscale => format!( - "clip = core.ncnn.Model(clip, network_path={}, fp16=True)", - py_str(&files.model_dir.join(UPSCALE_MODELS[0])) + // Swap the first and third channels, add 0.125. + GpuPlugin::Upscale => ( + "clip = core.std.BlankClip(width=64, height=64, format=vs.RGBS, length=2, fpsnum=25, fpsden=1, color=[0.2, 0.4, 0.6])", + format!( + "clip = core.ncnn.Model(clip, network_path={}, path_is_serialization=True, fp16=True)", + py_bytes(NCNN_SELFTEST_MODEL) + ), + "[0.725, 0.525, 0.325]", ), }; format!( @@ -404,16 +460,34 @@ pub fn probe_script(plugin: GpuPlugin, files: &GpuPluginFiles, platform_suffix: stage('load')\n\ if not hasattr(core, '{namespace}'):\n core.std.LoadPlugin({plugin_path})\n\ stage('init')\n\ - clip = core.std.BlankClip(width=64, height=64, format=vs.RGBS, length=2, fpsnum=25, fpsden=1)\n\ + {clip}\n\ {call}\n\ stage('render')\n\ + f = clip.get_frame(1)\n\ + got = [float(f[p][8, 8]) for p in range(3)]\n\ + del f\n\ + if any(abs(g - e) > {tol} for g, e in zip(got, {expected})):\n \ + stage('wrong')\n \ + raise vs.Error('self-test result ' + repr(got) + ', expected ' + repr({expected}))\n\ + stage('verified')\n\ clip.set_output()\n", marker = plugin.stage_marker(), namespace = plugin.namespace(), plugin_path = py_str(&files.plugin), + tol = SELFTEST_TOLERANCE, ) } +/// Write RIFE's stand-in model ([`RIFE_SELFTEST_PARAM`], an empty +/// `flownet.bin`) into a directory under `parent`, returning that directory. +pub fn write_rife_selftest(parent: &Path) -> std::io::Result { + let dir = parent.join(RIFE_SELFTEST_DIR_NAME); + std::fs::create_dir_all(&dir)?; + std::fs::write(dir.join("flownet.param"), RIFE_SELFTEST_PARAM)?; + std::fs::write(dir.join("flownet.bin"), b"")?; + Ok(dir) +} + /// How the probe process ended. #[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct ProbeExit { @@ -457,10 +531,13 @@ pub fn classify(plugin: GpuPlugin, exit: &ProbeExit, stderr: &str) -> GpuPluginS { return GpuPluginStatus::unavailable(R::NoVulkan, error); } + // The stand-in network did not load. Nothing the user installs fixes that + // (it is part of the probe, not an add-on), so it is a plugin fault. if stderr.contains("failed to load model") || stderr.contains("load param failed") { - return GpuPluginStatus::unavailable(R::NotInstalled, error); + return GpuPluginStatus::unavailable(R::LoadFailed, error); } match last { + Some("wrong") => GpuPluginStatus::unavailable(R::WrongResult, error), // The plugin would not load. A runtime library the pre-check could not // open is the likeliest cause, and the one the user can fix. Some("load") if !missing.is_empty() => GpuPluginStatus::unavailable( @@ -507,20 +584,28 @@ pub fn run_probe( files: &GpuPluginFiles, platform_suffix: &str, ) -> GpuPluginStatus { - let script = probe_script(plugin, files, platform_suffix); - let tmp = std::env::temp_dir().join(format!( - "vapourbox_{}_probe_{}.vpy", + let work = std::env::temp_dir().join(format!( + "vapourbox_{}_probe_{}", plugin.namespace(), std::process::id() )); - if let Err(e) = std::fs::write(&tmp, script) { + let tmp = work.join("probe.vpy"); + let written = std::fs::create_dir_all(&work).and_then(|_| { + let selftest = match plugin { + GpuPlugin::Rife => write_rife_selftest(&work)?, + GpuPlugin::Upscale => work.clone(), + }; + std::fs::write(&tmp, probe_script(plugin, files, platform_suffix, &selftest)) + }); + if let Err(e) = written { + let _ = std::fs::remove_dir_all(&work); return GpuPluginStatus::unavailable( GpuPluginReason::ProbeFailed, format!("could not write the probe script: {e}"), ); } let exit_and_stderr = run_with_timeout(vspipe, &tmp, env); - let _ = std::fs::remove_file(&tmp); + let _ = std::fs::remove_dir_all(&work); match exit_and_stderr { Ok((exit, stderr)) => classify(plugin, &exit, &stderr), Err(e) => GpuPluginStatus::unavailable( @@ -537,8 +622,21 @@ fn run_with_timeout( ) -> std::io::Result<(ProbeExit, String)> { // Frame 1: for RIFE's 2x output the first interpolated frame (frame 0 is a // copy and would never touch the GPU); for upscaling, any frame runs it. + run_vspipe_probe(vspipe, script, env, &["-s", "1", "-e", "1"], PROBE_TIMEOUT) +} + +/// Run a probe script through vspipe with `args` (the frame range), output +/// discarded, killing it after `timeout`; returns how it ended and its stderr. +/// Shared with the AI runtime's probe (`ai_runtime.rs`). +pub(crate) fn run_vspipe_probe( + vspipe: &Path, + script: &Path, + env: &HashMap, + args: &[&str], + timeout: Duration, +) -> std::io::Result<(ProbeExit, String)> { let mut child = Command::new(vspipe) - .args(["-s", "1", "-e", "1"]) + .args(args) .arg(script) .arg("-") .envs(env) @@ -559,7 +657,7 @@ fn run_with_timeout( if let Some(status) = child.try_wait()? { break Some(status); } - if start.elapsed() > PROBE_TIMEOUT { + if start.elapsed() > timeout { let _ = child.kill(); let _ = child.wait(); break None; @@ -593,15 +691,15 @@ mod tests { let mac = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/d/vapoursynth"), "macos-arm64"); assert_eq!(mac.plugin, Path::new("/d/vapoursynth/rife/librife.dylib")); assert_eq!(mac.preload.as_deref(), Some(Path::new("/d/vapoursynth/moltenvk/libMoltenVK.dylib"))); - assert!(mac.model_dir.ends_with("rife/models/rife-v4.26_ensembleFalse")); let win = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("D:/v"), "windows-x64"); assert!(win.plugin.ends_with("rife/librife.dll")); assert_eq!(win.preload, None); let linux = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/v"), "linux-x64"); assert!(linux.plugin.ends_with("rife/librife.so")); - // MoltenVK is required on macOS, and only there. - assert_eq!(mac.required().len(), 4); - assert_eq!(linux.required().len(), 3); + // MoltenVK is required on macOS, and only there. No model is: they + // are add-ons. + assert_eq!(mac.required(), [mac.plugin.clone(), mac.preload.clone().unwrap()]); + assert_eq!(linux.required(), [linux.plugin.clone()]); let mac = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("/d/vapoursynth"), "macos-arm64"); assert_eq!(mac.plugin, Path::new("/d/vapoursynth/ncnn/libvsncnn.dylib")); @@ -610,8 +708,13 @@ mod tests { assert!(win.plugin.ends_with("ncnn/vsncnn.dll")); let linux = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("/v"), "linux-x64"); assert!(linux.plugin.ends_with("ncnn/libvsncnn.so")); - assert_eq!(linux.required().len(), 1 + UPSCALE_MODELS.len()); - assert!(linux.models[0].ends_with("ncnn/models/RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx")); + assert_eq!(linux.required(), [linux.plugin.clone()]); + // A job's RIFE model is the add-on's, in the directory whose name + // tells the plugin which version (and padding) it is. + assert_eq!( + rife_model_dir(Path::new("/a")), + Path::new("/a/frame-rate-rife/rife-v4.26_ensembleFalse") + ); } #[test] @@ -640,6 +743,7 @@ mod tests { R::MissingLibrary, R::LoadFailed, R::Crashed, + R::WrongResult, R::UnsupportedCpu, R::Disabled, R::ProbeFailed, @@ -692,13 +796,21 @@ mod tests { assert_eq!(classify(Rife, &failed("exit code 1"), gpu).reason, Some(R::NoVulkan)); let dev = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: invalid GPU device\n"; assert_eq!(classify(Rife, &failed("exit code 1"), dev).reason, Some(R::NoVulkan)); + // The stand-in model is part of the probe: failing to load it is the + // plugin's fault, never a missing add-on. let model = "RIFE_PROBE:init\nvapoursynth.Error: RIFE: failed to load model\n"; - assert_eq!(classify(Rife, &failed("exit code 1"), model).reason, Some(R::NotInstalled)); + assert_eq!(classify(Rife, &failed("exit code 1"), model).reason, Some(R::LoadFailed)); // VSNCNN's words. let dev = "UPSCALE_PROBE:init\nvapoursynth.Error: ncnn.Model: get_gpu_device failed\n"; assert_eq!(classify(Upscale, &failed("exit code 1"), dev).reason, Some(R::NoVulkan)); let param = "UPSCALE_PROBE:init\nvapoursynth.Error: ncnn.Model: load param failed\n"; - assert_eq!(classify(Upscale, &failed("exit code 1"), param).reason, Some(R::NotInstalled)); + assert_eq!(classify(Upscale, &failed("exit code 1"), param).reason, Some(R::LoadFailed)); + // A self-test that ran but computed the wrong answer. + let wrong = "UPSCALE_PROBE:render\nUPSCALE_PROBE:wrong\n\ + vapoursynth.Error: self-test result [0.0, 0.0, 0.0], expected [0.725, 0.525, 0.325]\n"; + let s = classify(Upscale, &failed("exit code 1"), wrong); + assert_eq!(s.reason, Some(R::WrongResult)); + assert!(s.detail.unwrap().contains("expected")); } #[test] @@ -748,18 +860,90 @@ mod tests { #[test] fn probe_script_preloads_only_where_a_preload_exists() { + let selftest = Path::new("/t/rife-v4.26_selftest"); let mac = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/v"), "macos-arm64"); - let s = probe_script(Rife, &mac, "macos-arm64"); + let s = probe_script(Rife, &mac, "macos-arm64", selftest); let preload = py_str(mac.preload.as_deref().unwrap()); assert!(s.contains(&format!("ctypes.CDLL({preload}, mode=ctypes.RTLD_GLOBAL)"))); let win = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("C:/v"), "windows-x64"); - let s = probe_script(Rife, &win, "windows-x64"); + let s = probe_script(Rife, &win, "windows-x64", selftest); assert!(s.contains("if None:")); assert!(s.contains("'vcomp140.dll'")); // VSNCNN's Windows DLL needs no VC++ runtime, and loads by its own namespace. let win = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("C:/v"), "windows-x64"); - let s = probe_script(Upscale, &win, "windows-x64"); + let s = probe_script(Upscale, &win, "windows-x64", selftest); assert!(!s.contains("vcomp140")); assert!(s.contains("hasattr(core, 'ncnn')") && s.contains("core.ncnn.Model(")); } + + #[test] + fn probes_run_a_stand_in_model_never_an_addon() { + // The probes must work before any model add-on is installed: RIFE's + // runs the stand-in written beside the script, VSNCNN's the model + // embedded in it, and both check the answer. + let selftest = Path::new("/t/rife-v4.26_selftest"); + for platform in ["macos-arm64", "windows-x64", "linux-x64"] { + let files = GpuPluginFiles::in_vapoursynth_dir(Rife, Path::new("/v"), platform); + let s = probe_script(Rife, &files, platform, selftest); + assert!(s.contains(&format!("model_path={}", py_str(selftest))), "{s}"); + assert!(!s.contains(RIFE_MODEL_DIR_NAME) && !s.contains(RIFE_ADDON)); + assert!(s.contains("stage('wrong')") && s.contains("[0.4, 0.4, 0.4]")); + + let files = GpuPluginFiles::in_vapoursynth_dir(Upscale, Path::new("/v"), platform); + let s = probe_script(Upscale, &files, platform, selftest); + assert!(s.contains("path_is_serialization=True"), "{s}"); + assert!(s.contains(&py_bytes(NCNN_SELFTEST_MODEL))); + assert!(!s.contains(".onnx"), "no model file: {s}"); + assert!(s.contains("[0.725, 0.525, 0.325]")); + } + } + + #[test] + fn rife_stand_in_is_written_where_the_plugin_reads_v4() { + let tmp = std::env::temp_dir().join(format!("vb-rife-selftest-{}", std::process::id())); + let dir = write_rife_selftest(&tmp).unwrap(); + // The plugin decides "v4, padding 64" from the path. + assert!(dir.to_string_lossy().contains("rife-v4.26")); + assert_eq!(std::fs::read_to_string(dir.join("flownet.param")).unwrap(), RIFE_SELFTEST_PARAM); + assert_eq!(std::fs::read(dir.join("flownet.bin")).unwrap().len(), 0); + // An ncnn param file: magic, then the layer and blob counts it declares. + let lines: Vec<&str> = RIFE_SELFTEST_PARAM.lines().collect(); + assert_eq!(lines[0], "7767517"); + assert_eq!(lines[1], format!("{} 6", lines.len() - 2)); + for blob in ["in0", "in1", "in2", "out0"] { + assert!(RIFE_SELFTEST_PARAM.split_whitespace().any(|t| t == blob), "{blob}"); + } + std::fs::remove_dir_all(&tmp).unwrap(); + } + + #[test] + fn rife_addon_matches_the_plugin() { + let m = crate::addons::manifest(RIFE_ADDON).expect("frame-rate-rife.json"); + // Offered exactly where the plugin ships. + for p in crate::addons::PLATFORMS { + assert_eq!(m.supports(p), platform_ships_rife(p), "{p}"); + } + // Installed under the directory name the plugin reads its version from. + let mut paths = m.expected_paths("macos-arm64"); + paths.sort(); + assert_eq!( + paths, + [format!("{RIFE_MODEL_DIR_NAME}/flownet.bin"), format!("{RIFE_MODEL_DIR_NAME}/flownet.param")] + ); + // Run by the bundled plugin, not ONNX Runtime. + assert!(!crate::addons::needs_ai_runtime(&[RIFE_ADDON])); + } + + #[test] + fn ncnn_selftest_is_the_committed_model() { + // Scripts/models/make_ncnn_selftest.py writes this file; the worker + // embeds it. A 1x1 convolution: tiny, so the probe script stays small. + assert!(NCNN_SELFTEST_MODEL.len() < 1024, "{}", NCNN_SELFTEST_MODEL.len()); + let committed = std::fs::read( + Path::new(env!("CARGO_MANIFEST_DIR")).join("../Tests/TestResources/ncnn_selftest.onnx"), + ) + .unwrap(); + assert_eq!(committed, NCNN_SELFTEST_MODEL); + assert!(NCNN_SELFTEST_MODEL.windows(4).any(|w| w == b"Conv")); + } } diff --git a/worker/src/lib.rs b/worker/src/lib.rs index 98d8108d..6183a77e 100644 --- a/worker/src/lib.rs +++ b/worker/src/lib.rs @@ -4,6 +4,7 @@ pub mod models; pub mod comment_metadata; +pub mod crop_detect; pub mod cpu; pub mod dependency_locator; pub mod dvd_reader; @@ -14,10 +15,15 @@ pub mod pixel_format; pub mod source_decode; pub mod source_timing; pub mod progress_reporter; +pub mod addons; +pub mod audio_restoration; +pub mod ai_runtime; pub mod animejanai; +pub mod upscale_addons; pub mod gpu_plugin; pub mod hdr_metadata; pub mod native_res; +pub mod noise_halo; pub mod schema_script_generator; pub mod script_generator; pub mod subtitle_generator; diff --git a/worker/src/main.rs b/worker/src/main.rs index 6a83968d..420e3057 100644 --- a/worker/src/main.rs +++ b/worker/src/main.rs @@ -20,6 +20,7 @@ use std::sync::Arc; mod models; mod comment_metadata; +mod crop_detect; mod cpu; mod dependency_locator; mod dvd_reader; @@ -28,10 +29,15 @@ mod pixel_format; mod source_decode; mod source_timing; mod progress_reporter; +mod addons; +mod audio_restoration; +mod ai_runtime; mod animejanai; +mod upscale_addons; mod gpu_plugin; mod hdr_metadata; mod native_res; +mod noise_halo; mod script_generator; mod subtitle_generator; mod platform; @@ -120,6 +126,35 @@ struct Args { /// from (for the Descale pass); prints a JSON report. #[arg(long)] detect_native: bool, + + /// Measure the black borders, analog blanking and head-switching band at + /// the edges of the video in `--config` (for Crop & Resize's Detect); + /// prints a JSON report. + #[arg(long)] + detect_crop: bool, + + /// Measure how noisy the video in `--config` is and how strongly its + /// edges ring (for the pass list's suggestions); prints a JSON report. + #[arg(long)] + detect_noise_halo: bool, + + /// Probe whether the AI runtime add-on (ONNX Runtime) runs here: imports + /// it the way a job does, runs a tiny test model, and prints JSON + /// (`{"available", "reason", "detail", "message", "version", "providers", + /// "provider", "fallback"}`). `VAPOURBOX_AI_FORCE_CPU=1` tests the CPU path. + #[arg(long)] + probe_ai_runtime: bool, + + /// Print the audio restoration plan for the job in `--config` (the probes + /// it makes, then every ffmpeg / DeepFilterNet argument list it would run) + /// as JSON, and exit without rendering anything. + #[arg(long)] + audio_plan: bool, + + /// Print the add-on manifests compiled into this worker, as JSON + /// (`{"": "", …}`), and exit. + #[arg(long)] + list_addons: bool, } fn main() -> ExitCode { @@ -181,6 +216,31 @@ fn main() -> ExitCode { return run_detect_native(&args); } + if args.detect_crop { + return run_detect_crop(&args); + } + + if args.detect_noise_halo { + return run_detect_noise_halo(&args); + } + + if args.probe_ai_runtime { + return run_probe_ai_runtime(); + } + + if args.audio_plan { + return run_audio_plan(&args); + } + + if args.list_addons { + let list: serde_json::Map = addons::manifests() + .iter() + .map(|m| (m.id.clone(), serde_json::Value::String(m.version.clone()))) + .collect(); + println!("{}", serde_json::Value::Object(list)); + return ExitCode::SUCCESS; + } + // Preview mode outputs raw PNG to stdout - no JSON messages if args.preview { return run_preview_mode(&args); @@ -267,6 +327,49 @@ fn run_probe_opencl() -> ExitCode { } +/// Print the audio restoration plan for the job in `--config` as JSON +/// (`{"active": false}` when the pass does nothing for it). Log messages go to +/// stderr, so stdout is the one JSON line. +fn run_audio_plan(args: &Args) -> ExitCode { + let result = (|| -> anyhow::Result { + let config = args + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("--config is required with --audio-plan"))?; + let mut job: models::VideoJob = serde_json::from_str(&std::fs::read_to_string(config)?)?; + let deps = dependency_locator::DependencyLocator::new()?; + fill_total_frames(&mut job, Some(&deps), |_| {}); + let reporter = ProgressReporter::new_to_stderr(); + let Some((plan, source)) = audio_restoration::AudioRestorer::new(reporter, deps).plan(&job)? else { + return Ok(serde_json::json!({ "active": false })); + }; + let p = |s: &str| std::path::PathBuf::from(s); + let first = if plan.single_pass() { "restored.wav" } else { "restore.wav" }; + Ok(serde_json::json!({ + "active": true, + "sourceStreams": source.streams, + "stream": plan.stream, + "outRate": plan.out_rate, + "outSamples": plan.out_samples, + "restore": plan.restore_args(&job.input_path, &p(first)), + "deepFilter": plan.deep_filter_args(&p(first), &p("dfn")), + "measure": plan.measure_args(&p(first)), + "finish": (!plan.single_pass()).then(|| plan.finish_args(&p(first), &p("restored.wav"), None)), + "notes": plan.notes, + })) + })(); + match result { + Ok(v) => { + println!("{v}"); + ExitCode::SUCCESS + } + Err(e) => { + println!("{}", serde_json::json!({ "error": format!("{e:#}") })); + ExitCode::FAILURE + } + } +} + /// Detect a source's native resolution and kernel and print the report as one /// JSON line (`{"error": ...}` when it could not be analysed). fn run_detect_native(args: &Args) -> ExitCode { @@ -292,10 +395,73 @@ fn run_detect_native(args: &Args) -> ExitCode { } } +/// Measure a source's edges for the crop and print the report as one JSON +/// line (`{"error": ...}` when it could not be analysed). +fn run_detect_crop(args: &Args) -> ExitCode { + let result = (|| -> anyhow::Result { + let config = args + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("--config is required with --detect-crop"))?; + let mut job: models::VideoJob = serde_json::from_str(&std::fs::read_to_string(config)?)?; + let deps = dependency_locator::DependencyLocator::new()?; + fill_frame_size(&mut job, Some(&deps), |_| {}); + crop_detect::detect(&deps, &job) + })(); + match result { + Ok(report) => { + println!("{}", serde_json::to_string(&report).unwrap_or_default()); + ExitCode::SUCCESS + } + Err(e) => { + println!("{}", serde_json::json!({ "error": e.to_string() })); + ExitCode::FAILURE + } + } +} + +/// Measure a source's noise and halos and print the report as one JSON line +/// (`{"error": ...}` when it could not be analysed). +fn run_detect_noise_halo(args: &Args) -> ExitCode { + let result = (|| -> anyhow::Result { + let config = args + .config + .as_ref() + .ok_or_else(|| anyhow::anyhow!("--config is required with --detect-noise-halo"))?; + let mut job: models::VideoJob = serde_json::from_str(&std::fs::read_to_string(config)?)?; + let deps = dependency_locator::DependencyLocator::new()?; + fill_frame_size(&mut job, Some(&deps), |_| {}); + noise_halo::detect(&deps, &job) + })(); + match result { + Ok(report) => { + println!("{}", serde_json::to_string(&report).unwrap_or_default()); + ExitCode::SUCCESS + } + Err(e) => { + println!("{}", serde_json::json!({ "error": e.to_string() })); + ExitCode::FAILURE + } + } +} + /// Probe whether a GPU plugin (RIFE, AI upscaling) runs here and print the /// result as JSON to stdout. Deps that cannot be located are reported as "not /// installed" rather than as a failure of the probe: the remedy is the same, /// let the app download them. +fn run_probe_ai_runtime() -> ExitCode { + let json = match dependency_locator::DependencyLocator::new() { + Ok(deps) => deps.ai_runtime_status().to_json(deps.platform_suffix()), + Err(e) => ai_runtime::AiRuntimeStatus::unavailable( + ai_runtime::AiRuntimeReason::ProbeFailed, + e.to_string(), + ) + .to_json(""), + }; + println!("{json}"); + ExitCode::SUCCESS +} + fn run_probe_gpu_plugin(plugin: gpu_plugin::GpuPlugin) -> ExitCode { let json = match dependency_locator::DependencyLocator::new() { Ok(deps) => deps.gpu_plugin_status(plugin).to_json(plugin, deps.platform_suffix()), @@ -497,6 +663,33 @@ fn run_preview_mode(args: &Args) -> ExitCode { } } +/// Resolve the frame count if the caller didn't supply one. The Flutter app +/// probes and sets total_frames; direct callers (tests, CLI) may omit it, and +/// the script's pipe_source needs an exact length (it builds a fixed-size +/// clip). Without this the worker defaults to 1 frame and the whole output is +/// a single frame. total_frames must be the *post-trim* count the decoder +/// will actually pipe (trimming is decoder-side: -ss + -frames:v), matching +/// the decoder's own range logic in pipeline_executor. +fn fill_total_frames( + job: &mut VideoJob, + deps: Option<&dependency_locator::DependencyLocator>, + log: impl Fn(&str), +) { + if job.total_frames.is_none() { + if let Some(full) = deps.and_then(|d| d.probe_frame_count(&job.input_path)) { + let effective = match (job.start_frame, job.end_frame) { + (Some(s), Some(e)) => (e - s + 1).max(0), + (None, Some(e)) => e + 1, + (Some(s), None) => (full - s).max(0), + (None, None) => full, + }; + log(&format!("Probed input frame count: {} (effective after trim: {})", full, effective)); + job.total_frames = Some(effective); + } + } + +} + /// Fill a missing `input_width`/`input_height` from ffprobe. A job that carries /// both is never touched; if either is missing and the probe succeeds, both /// come from the probe; a failed probe leaves the old fallback in place. @@ -579,80 +772,9 @@ fn run_worker( // Resolve deps once for the pre-generation probes below. let deps = dependency_locator::DependencyLocator::new().ok(); - // ------------------------------------------------------------------ - // Pre-encode transcription - // ------------------------------------------------------------------ - // Whisper used to run only after the encode, which made burn-in - // impossible: the encoder needs the subtitle file while it is running, and - // a transcript that does not exist yet cannot be drawn into the picture. - // - // Transcribing the SOURCE instead means honouring the trim. The encoder - // seeks the audio input to the trim point, so the output's audio starts - // there — a transcript of the whole source would put every cue early by - // exactly the trimmed-off head. Nothing else retimes audio: IVTC and - // frame-rate conversion change the video timeline only. - let mut whisper_srt: Option = None; - if let Some(ref sub_settings) = job.subtitle_settings { - if sub_settings.enabled && job.burn_in_subtitle_path.is_none() { - let fps = job.input_frame_rate.unwrap_or(29.97); - let window = match (job.start_frame, job.end_frame) { - (Some(start), Some(end)) if end >= start => Some(( - start as f64 / fps, - Some((end - start + 1) as f64 / fps), - )), - (Some(start), None) => Some((start as f64 / fps, None)), - (None, Some(end)) => Some((0.0, Some((end + 1) as f64 / fps))), - _ => None, - }; - let srt_target = std::path::Path::new(&job.output_path).with_extension("srt"); - match dependency_locator::DependencyLocator::new().and_then(|d| { - SubtitleGenerator::new(reporter.clone(), d).transcribe( - &job.input_path, - sub_settings, - window, - &srt_target, - || cancelled.load(Ordering::SeqCst), - ) - }) { - Ok(Some(path)) => { - // Burn-in reads this during the encode; the mux pass below - // reads it afterwards. - if sub_settings.output.burns_in() { - job.burn_in_subtitle_path = Some(path.to_string_lossy().to_string()); - } - whisper_srt = Some(path); - } - Ok(None) => {} - Err(e) => reporter.send_log( - models::LogLevel::Warning, - &format!("Subtitle generation failed: {e}"), - ), - } - } - } - - // Resolve the frame count if the caller didn't supply one. The Flutter app - // probes and sets total_frames; direct callers (tests, CLI) may omit it, and - // the script's pipe_source needs an exact length (it builds a fixed-size - // clip). Without this the worker defaults to 1 frame and the whole output is - // a single frame. total_frames must be the *post-trim* count the decoder - // will actually pipe (trimming is decoder-side: -ss + -frames:v), matching - // the decoder's own range logic in pipeline_executor. - if job.total_frames.is_none() { - if let Some(full) = deps.as_ref().and_then(|d| d.probe_frame_count(&job.input_path)) { - let effective = match (job.start_frame, job.end_frame) { - (Some(s), Some(e)) => (e - s + 1).max(0), - (None, Some(e)) => e + 1, - (Some(s), None) => (full - s).max(0), - (None, None) => full, - }; - reporter.send_log( - models::LogLevel::Debug, - &format!("Probed input frame count: {} (effective after trim: {})", full, effective), - ); - job.total_frames = Some(effective); - } - } + fill_total_frames(&mut job, deps.as_ref(), |msg| { + reporter.send_log(models::LogLevel::Debug, msg) + }); // Same for the frame size: the app always sends ffprobe's width/height, but // a direct caller may not, and the old 720x480 fallback is now a decode @@ -719,6 +841,10 @@ fn run_worker( _ => Err(animejanai::MISSING_MESSAGE.to_string()), }) .with_timeline(timeline_window); + let script_generator = match deps.as_ref() { + Some(d) => script_generator.with_addons(d.addons_dir(), d.platform_suffix()), + None => script_generator, + }; let script_path = script_generator .generate(&job) .with_context(|| "Failed to generate VapourSynth script")?; @@ -728,6 +854,81 @@ fn run_worker( &format!("Script written to: {:?}", script_path), ); + // ------------------------------------------------------------------ + // Audio restoration + // ------------------------------------------------------------------ + // Rendered before the transcription and the encode, both of which then + // use the rendered track (see audio_restoration.rs). After the script, so + // a job the script generator refuses fails before any audio work. Held + // to the end of the job: its scratch directory goes when it is dropped. + let restored_audio = if job.effective_pipeline().audio.has_effect() { + let d = dependency_locator::DependencyLocator::new()?; + audio_restoration::AudioRestorer::new(reporter.clone(), d) + .render(&job, || cancelled.load(Ordering::SeqCst))? + } else { + None + }; + + // ------------------------------------------------------------------ + // Pre-encode transcription + // ------------------------------------------------------------------ + // Whisper used to run only after the encode, which made burn-in + // impossible: the encoder needs the subtitle file while it is running, and + // a transcript that does not exist yet cannot be drawn into the picture. + // + // Transcribing before the encode means transcribing exactly the audio the + // output will carry. With audio restoration that is the rendered track: + // it is the output's soundtrack sample for sample — trimmed, offset, + // drift-corrected and cleaned — so cues line up whatever was done to it, + // and Whisper hears the cleaned sound. Without it, it is the source's + // audio over the trim window: the encoder seeks the audio input to the + // trim point, so a transcript of the whole source would put every cue + // early by exactly the trimmed-off head. Nothing else retimes audio: IVTC + // and frame-rate conversion change the video timeline only. + let mut whisper_srt: Option = None; + if let Some(ref sub_settings) = job.subtitle_settings { + if sub_settings.enabled && job.burn_in_subtitle_path.is_none() { + let fps = job.input_frame_rate.unwrap_or(29.97); + let source_window = match (job.start_frame, job.end_frame) { + (Some(start), Some(end)) if end >= start => Some(( + start as f64 / fps, + Some((end - start + 1) as f64 / fps), + )), + (Some(start), None) => Some((start as f64 / fps, None)), + (None, Some(end)) => Some((0.0, Some((end + 1) as f64 / fps))), + _ => None, + }; + let (transcribe_from, window) = match &restored_audio { + Some(r) => (r.track.path.to_string_lossy().to_string(), None), + None => (job.input_path.clone(), source_window), + }; + let srt_target = std::path::Path::new(&job.output_path).with_extension("srt"); + match dependency_locator::DependencyLocator::new().and_then(|d| { + SubtitleGenerator::new(reporter.clone(), d).transcribe( + &transcribe_from, + sub_settings, + window, + &srt_target, + || cancelled.load(Ordering::SeqCst), + ) + }) { + Ok(Some(path)) => { + // Burn-in reads this during the encode; the mux pass below + // reads it afterwards. + if sub_settings.output.burns_in() { + job.burn_in_subtitle_path = Some(path.to_string_lossy().to_string()); + } + whisper_srt = Some(path); + } + Ok(None) => {} + Err(e) => reporter.send_log( + models::LogLevel::Warning, + &format!("Subtitle generation failed: {e}"), + ), + } + } + } + // Execute pipeline. // // Retry on the rare VapourSynth plugin-autoload skip (a needed plugin @@ -740,7 +941,9 @@ fn run_worker( const MAX_PIPELINE_ATTEMPTS: u32 = 3; let mut attempt = 1; loop { - let mut executor = PipelineExecutor::new(reporter.clone())?.with_timeline(timeline.clone()); + let mut executor = PipelineExecutor::new(reporter.clone())? + .with_timeline(timeline.clone()) + .with_restored_audio(restored_audio.as_ref().map(|r| r.track.clone())); match executor.execute(&script_path, &job, || cancelled.load(Ordering::SeqCst)) { Ok(()) => break, Err(e) diff --git a/worker/src/models/anti_alias_parameters.rs b/worker/src/models/anti_alias_parameters.rs index 3755edff..7611aa3e 100644 --- a/worker/src/models/anti_alias_parameters.rs +++ b/worker/src/models/anti_alias_parameters.rs @@ -5,8 +5,9 @@ //! the frame with an edge-directed kernel and take the smoother of the two //! results, so the edge is rebuilt rather than blurred. //! -//! Neither has a bit-depth limit — verified against the bundle at 8/10/12/16-bit -//! and 4:2:2, so no conversion guard is needed. +//! None has a bit-depth limit — verified against the bundle at 8/10/12/16-bit +//! and 4:2:2, so no conversion guard is needed. SangNom's `aa` is in 8-bit +//! units at every depth; the plugin scales it itself. use serde::{Deserialize, Serialize}; @@ -21,6 +22,10 @@ pub enum AntiAliasMethod { /// santiag: separate horizontal and vertical strength, so it can be aimed /// at one axis. Stronger, and slower. Santiag, + /// SangNom AA: supersample 2x, SangNom along each axis, back down. The + /// sharpest of the three; slower than daa at HD (app 1.7.0, deps 1.16.0). + #[serde(rename = "sangnom")] + SangNom, } /// Parameters for the anti-aliasing pass. @@ -46,13 +51,22 @@ pub struct AntiAliasParameters { pub santiag_strh: i32, /// Interpolator for the vertical pass: `nnedi3`, `eedi2` or `sangnom`. - /// Only `nnedi3` is bundled, so the others are not offered. + /// Pinned to `nnedi3` (see [`Self::effective_santiag_type`]). #[serde(default = "default_santiag_type")] pub santiag_type: String, + + // --- SangNom parameters --- + + /// SangNom's anti-aliasing strength, 0-128 in 8-bit units (the plugin's + /// default is 48). 0 still re-interpolates every other line; it only stops + /// SangNom smoothing the edge direction it finds. + #[serde(default = "default_sangnom_aa")] + pub sangnom_aa: i32, } fn default_strength() -> i32 { 1 } fn default_santiag_type() -> String { "nnedi3".to_string() } +fn default_sangnom_aa() -> i32 { 48 } impl Default for AntiAliasParameters { fn default() -> Self { @@ -62,17 +76,20 @@ impl Default for AntiAliasParameters { santiag_strv: default_strength(), santiag_strh: default_strength(), santiag_type: default_santiag_type(), + sangnom_aa: default_sangnom_aa(), } } } impl AntiAliasParameters { - /// santiag's interpolator name, restricted to what is bundled. + /// santiag's interpolator name, pinned to nnedi3. /// - /// havsfunc accepts `nnedi3`, `eedi2` and `sangnom`; only nnedi3 ships here, - /// and naming an absent one fails at script evaluation with a bare "no - /// attribute" error. Anything unrecognised falls back to nnedi3 rather than - /// reaching havsfunc. + /// havsfunc accepts `nnedi3`, `eedi2` and `sangnom`. eedi2 is not bundled + /// (naming it fails at script evaluation with a bare "no attribute"), and + /// `sangnom`, although SangNom ships from deps 1.16.0, is broken in + /// havsfunc: its strength-0 step calls `SangNom(order=0)`, which SangNom r45 + /// treats as double-rate and refuses without `_FieldBased` (measured). + /// SangNom is offered as its own method instead. pub fn effective_santiag_type(&self) -> &'static str { match self.santiag_type.to_ascii_lowercase().as_str() { "nnedi3" => "nnedi3", @@ -89,6 +106,12 @@ impl AntiAliasParameters { pub fn effective_strh(&self) -> i32 { self.santiag_strh.clamp(0, 3) } + + /// SangNom's `aa`, clamped to the 0-128 the plugin accepts (it rejects + /// anything outside with a script error). + pub fn effective_sangnom_aa(&self) -> i32 { + self.sangnom_aa.clamp(0, 128) + } } #[cfg(test)] @@ -106,7 +129,8 @@ mod tests { #[test] fn test_santiag_type_is_restricted_to_what_is_bundled() { - // eedi2 and sangnom are not in the deps bundle; naming one would fail at + // eedi2 is not in the deps bundle, and havsfunc's sangnom type calls + // SangNom(order=0), which SangNom r45 refuses; either would fail at // script evaluation rather than degrade. let with = |t: &str| AntiAliasParameters { santiag_type: t.to_string(), @@ -130,6 +154,17 @@ mod tests { assert_eq!(with(9).effective_strh(), 3); } + #[test] + fn test_sangnom_method_wire_name_and_strength() { + let p: AntiAliasParameters = + serde_json::from_str(r#"{"enabled":true,"method":"sangnom","sangnomAa":200}"#).unwrap(); + assert_eq!(p.method, AntiAliasMethod::SangNom); + assert_eq!(p.effective_sangnom_aa(), 128); + assert_eq!(AntiAliasParameters::default().sangnom_aa, 48); + let json = serde_json::to_string(&p).unwrap(); + assert!(json.contains("\"method\":\"sangnom\""), "{json}"); + } + #[test] fn test_serialization() { let json = serde_json::to_string(&AntiAliasParameters::default()).unwrap(); diff --git a/worker/src/models/audio_restoration_parameters.rs b/worker/src/models/audio_restoration_parameters.rs new file mode 100644 index 00000000..d164218d --- /dev/null +++ b/worker/src/models/audio_restoration_parameters.rs @@ -0,0 +1,235 @@ +//! Audio restoration — the one pass that works on the soundtrack. +//! +//! Everything here is applied by ffmpeg (and, for speech, the DeepFilterNet +//! add-on) in `audio_restoration.rs`, which renders the output's audio track +//! before the video encode; nothing reaches the VapourSynth script. See that +//! module for the order of the steps and why each sits where it does. + +use serde::{Deserialize, Serialize}; + +/// What to do about the two channels of a stereo source. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum ChannelFix { + /// Leave the channels alone. + #[default] + Off, + /// Measure both channels; if one is silent, copy the other into it. + /// Otherwise leave them alone. + Auto, + /// Both outputs from the left channel. + LeftOnly, + /// Both outputs from the right channel. + RightOnly, + /// Both outputs from the average of the two. + Mix, +} + +/// Which mains frequency to notch out. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum HumFrequency { + /// Measure the audio for a 50 or 60 Hz hum and notch whichever is there; + /// nothing when neither is. + #[default] + #[serde(rename = "auto")] + Auto, + #[serde(rename = "50")] + Hz50, + #[serde(rename = "60")] + Hz60, +} + +/// Parameters for the audio restoration pass. Off by default, and every step +/// inside it is off by default too: switching the pass on changes nothing until +/// a step is chosen. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct AudioRestorationParameters { + #[serde(default)] + pub enabled: bool, + + /// Which of the source's audio streams to restore, counted among its audio + /// streams from 0. The others are kept as they are. + #[serde(default)] + pub stream_index: u32, + + #[serde(default)] + pub channel_fix: ChannelFix, + + /// Move the sound against the picture, in milliseconds. Positive delays + /// the sound (it plays later), negative advances it. + #[serde(default)] + pub offset_ms: i32, + + /// Stretch or squeeze the sound to the picture's length when the two were + /// recorded on clocks that disagreed. + #[serde(default)] + pub drift_correction: bool, + + /// The largest drift corrected, as a percentage of the length. A bigger + /// mismatch is a different problem (the wrong track, a missing stretch of + /// audio), which a stretch would only disguise. + #[serde(default = "default_drift_max_percent")] + pub drift_max_percent: f64, + + /// Rebuild waveform peaks the recording flattened (`adeclip`). + #[serde(default)] + pub declip: bool, + + /// Remove clicks and crackle (`adeclick`). + #[serde(default)] + pub declick: bool, + + /// `adeclick`'s threshold: lower removes more (and risks transients). + #[serde(default = "default_declick_threshold")] + pub declick_threshold: f64, + + /// Notch out mains hum and its first two harmonics. + #[serde(default)] + pub hum_removal: bool, + + #[serde(default)] + pub hum_frequency: HumFrequency, + + /// High-pass out rumble (turntable, handling and wind noise). + #[serde(default)] + pub rumble_filter: bool, + + #[serde(default = "default_rumble_cutoff_hz")] + pub rumble_cutoff_hz: u32, + + /// Broadband noise reduction (`afftdn`): tape hiss, fan noise. + #[serde(default)] + pub denoise: bool, + + /// How much noise to remove, in dB. + #[serde(default = "default_denoise_strength")] + pub denoise_strength: f64, + + /// Speech enhancement with DeepFilterNet (an add-on). + #[serde(default)] + pub deep_filter: bool, + + /// DeepFilterNet's attenuation limit in dB: how far it may turn noise + /// down. 100 is no limit; lower keeps a little of the room. + #[serde(default = "default_deep_filter_attenuation")] + pub deep_filter_attenuation: u32, + + /// Normalise loudness to a target (EBU R128, two passes). + #[serde(default)] + pub loudness_normalize: bool, + + /// Integrated loudness target in LUFS. + #[serde(default = "default_loudness_target")] + pub loudness_target: f64, + + /// Keep peaks under a ceiling (`alimiter`). + #[serde(default)] + pub limiter: bool, + + /// The ceiling in dBFS. + #[serde(default = "default_limiter_ceiling_db")] + pub limiter_ceiling_db: f64, +} + +fn default_drift_max_percent() -> f64 { 0.5 } +fn default_declick_threshold() -> f64 { 2.0 } +fn default_rumble_cutoff_hz() -> u32 { 40 } +fn default_denoise_strength() -> f64 { 12.0 } +fn default_deep_filter_attenuation() -> u32 { 100 } +fn default_loudness_target() -> f64 { -16.0 } +fn default_limiter_ceiling_db() -> f64 { -1.0 } + +impl Default for AudioRestorationParameters { + fn default() -> Self { + Self { + enabled: false, + stream_index: 0, + channel_fix: ChannelFix::Off, + offset_ms: 0, + drift_correction: false, + drift_max_percent: default_drift_max_percent(), + declip: false, + declick: false, + declick_threshold: default_declick_threshold(), + hum_removal: false, + hum_frequency: HumFrequency::Auto, + rumble_filter: false, + rumble_cutoff_hz: default_rumble_cutoff_hz(), + denoise: false, + denoise_strength: default_denoise_strength(), + deep_filter: false, + deep_filter_attenuation: default_deep_filter_attenuation(), + loudness_normalize: false, + loudness_target: default_loudness_target(), + limiter: false, + limiter_ceiling_db: default_limiter_ceiling_db(), + } + } +} + +impl AudioRestorationParameters { + /// Whether the pass changes anything: on, with at least one step chosen. + /// A pass switched on with nothing in it leaves the audio exactly as + /// before — copied or converted by the encoder, not rendered. + pub fn has_effect(&self) -> bool { + self.enabled + && (self.channel_fix != ChannelFix::Off + || self.offset_ms != 0 + || self.drift_correction + || self.declip + || self.declick + || self.hum_removal + || self.rumble_filter + || self.denoise + || self.deep_filter + || self.loudness_normalize + || self.limiter) + } + + /// Whether the job needs the DeepFilterNet add-on. + pub fn uses_deep_filter(&self) -> bool { + self.enabled && self.deep_filter + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn json_names_match_the_app() { + let json = r#"{"enabled":true,"streamIndex":1,"channelFix":"leftOnly","offsetMs":-120, + "driftCorrection":true,"driftMaxPercent":0.3,"declip":true,"declick":true, + "declickThreshold":3,"humRemoval":true,"humFrequency":"60","rumbleFilter":true, + "rumbleCutoffHz":60,"denoise":true,"denoiseStrength":18,"deepFilter":true, + "deepFilterAttenuation":30,"loudnessNormalize":true,"loudnessTarget":-23, + "limiter":true,"limiterCeilingDb":-2}"#; + let p: AudioRestorationParameters = serde_json::from_str(json).unwrap(); + assert_eq!(p.stream_index, 1); + assert_eq!(p.channel_fix, ChannelFix::LeftOnly); + assert_eq!(p.offset_ms, -120); + assert_eq!(p.hum_frequency, HumFrequency::Hz60); + assert_eq!(p.deep_filter_attenuation, 30); + assert_eq!(p.loudness_target, -23.0); + let back = serde_json::to_value(&p).unwrap(); + assert_eq!(back["channelFix"], "leftOnly"); + assert_eq!(back["humFrequency"], "60"); + for fix in ["off", "auto", "leftOnly", "rightOnly", "mix"] { + let v: ChannelFix = serde_json::from_str(&format!("\"{fix}\"")).unwrap(); + assert_eq!(serde_json::to_value(v).unwrap(), fix); + } + } + + #[test] + fn an_empty_object_is_the_default_and_does_nothing() { + let p: AudioRestorationParameters = serde_json::from_str("{}").unwrap(); + assert_eq!(p, AudioRestorationParameters::default()); + assert!(!p.has_effect()); + // On, with nothing chosen, still does nothing. + let on = AudioRestorationParameters { enabled: true, ..Default::default() }; + assert!(!on.has_effect()); + assert!(AudioRestorationParameters { enabled: true, offset_ms: 10, ..Default::default() }.has_effect()); + assert!(!AudioRestorationParameters { enabled: false, denoise: true, ..Default::default() }.has_effect()); + } +} diff --git a/worker/src/models/color_metadata.rs b/worker/src/models/color_metadata.rs index 1860eb58..8f4459ba 100644 --- a/worker/src/models/color_metadata.rs +++ b/worker/src/models/color_metadata.rs @@ -24,9 +24,12 @@ //! input they ride on the frames and are written. //! //! So this is a *metadata* fix, and the input-side declaration is what keeps it -//! one. Nothing here should convert anything: the samples coming off the pipe -//! already carry whatever matrix the source used, and none of the passes -//! re-matrix them. See docs/ENGINEERING_NOTES.md (2026-09-26). +//! one. Nothing in ffmpeg should convert anything: the samples coming off the +//! pipe carry the source's matrix, except in one case the script itself +//! converts — a BT.601 source taken to BT.709 by Crop & Resize (app 1.7.0), +//! when the tags declared are `bt709_converted`'s and `VideoJob:: +//! output_color_metadata` is the one place that decides which. See +//! docs/ENGINEERING_NOTES.md (2026-09-26). //! //! Values are validated against what FFmpeg actually accepts rather than passed //! through, on the same principle as `parse_ratio`: a value we do not recognise @@ -173,6 +176,64 @@ impl ColorMetadata { } } + /// Whether the picture's colour is BT.601: tagged `bt470bg` (PAL) or + /// `smpte170m` (NTSC), or untagged below HD height, which is how the rest + /// of the app (`zimg_matrix`) and every player read an untagged SD file. + /// Never an HDR picture. + pub fn is_bt601_family(&self, source_height: Option) -> bool { + if self.is_hdr() { + return false; + } + match self.matrix.as_deref() { + Some("bt470bg") | Some("smpte170m") => true, + Some(_) => false, + None => self.zimg_matrix(source_height) == "170m", + } + } + + /// The primaries a BT.601 picture was made for: its tag, else PAL's + /// (EBU, `bt470bg`) for a 576- or 288-line picture or a `bt470bg` matrix, + /// else NTSC's (SMPTE C, `smpte170m`). + fn bt601_primaries(&self, source_height: Option) -> String { + if let Some(p) = &self.primaries { + return p.clone(); + } + let pal = self.matrix.as_deref() == Some("bt470bg") + || matches!(source_height, Some(576) | Some(288)); + if pal { "bt470bg" } else { "smpte170m" }.to_string() + } + + /// The tags of a BT.601 picture after its samples are converted to the + /// BT.709 matrix (the script's `COLOR_709` block, app 1.7.0). + /// + /// The matrix and transfer say 709. BT.601's transfer is the same curve as + /// BT.709's, and PAL's nominal gamma-2.8 tag has never described a real + /// camera, so relabelling it changes nothing about the samples. + /// + /// The **primaries keep the source's** (tagged, or PAL/NTSC by height), + /// because only the matrix is converted: the samples still describe the + /// source's primaries. Converting the gamut as well was measured and + /// rejected (docs/reports/2026-11-crop-detect-709/): SMPTE C to BT.709 + /// moves a 75% green from RGB (0,191,0) to (55,188,0) and a 75% magenta's + /// green from 0 to 47, which every player that does not colour-manage + /// (most of them) would show as a hue shift of its own, while a player + /// that does colour-manage reads the source's primaries from the tag. The + /// range is the source's (limited when untagged), declared explicitly. + pub fn bt709_converted(&self, source_height: Option) -> ColorMetadata { + let transfer = match self.transfer.as_deref() { + None | Some("bt709") | Some("smpte170m") | Some("gamma22") | Some("gamma28") => { + "bt709".to_string() + } + Some(other) => other.to_string(), + }; + ColorMetadata { + matrix: Some("bt709".to_string()), + primaries: Some(self.bt601_primaries(source_height)), + transfer: Some(transfer), + range: Some(self.range.clone().unwrap_or_else(|| "tv".to_string())), + } + } + /// Whether the transfer is HDR: PQ (HDR10, and the base layer of most /// Dolby Vision) or HLG. pub fn is_hdr(&self) -> bool { @@ -428,4 +489,47 @@ mod tests { ); assert!(ColorMetadata::default().to_ffmpeg_input_args().is_empty()); } + + #[test] + fn bt601_family_is_tagged_or_untagged_sd() { + let tagged = |m: &str| ColorMetadata::from_raw(Some(m), None, None, None); + assert!(tagged("bt470bg").is_bt601_family(Some(1080))); + assert!(tagged("smpte170m").is_bt601_family(Some(480))); + assert!(!tagged("bt709").is_bt601_family(Some(576))); + assert!(!tagged("bt2020nc").is_bt601_family(Some(576))); + let untagged = ColorMetadata::default(); + assert!(untagged.is_bt601_family(Some(576))); + assert!(!untagged.is_bt601_family(Some(720))); + // HDR is never 601, whatever its matrix says. + let hdr = ColorMetadata::from_raw(Some("bt470bg"), None, Some("smpte2084"), None); + assert!(!hdr.is_bt601_family(Some(576))); + } + + #[test] + fn bt709_converted_keeps_primaries_and_range() { + // PAL, fully tagged: matrix and transfer become 709, primaries stay. + let pal = ColorMetadata::from_raw(Some("bt470bg"), Some("bt470bg"), None, Some("tv")); + assert_eq!( + pal.bt709_converted(Some(576)).to_ffmpeg_output_args(), + vec![ + "-colorspace", "bt709", + "-color_primaries", "bt470bg", + "-color_trc", "bt709", + "-color_range", "tv", + ] + ); + // Untagged: primaries by height (576 is PAL, 480 NTSC), range limited. + let untagged = ColorMetadata::default(); + let pal = untagged.bt709_converted(Some(576)); + assert_eq!(pal.primaries.as_deref(), Some("bt470bg")); + assert_eq!(pal.range.as_deref(), Some("tv")); + let ntsc = untagged.bt709_converted(Some(480)); + assert_eq!(ntsc.primaries.as_deref(), Some("smpte170m")); + // A full-range source stays full range. + let full = ColorMetadata::from_raw(Some("smpte170m"), None, Some("smpte170m"), Some("pc")); + let out = full.bt709_converted(Some(480)); + assert_eq!(out.range.as_deref(), Some("pc")); + assert_eq!(out.transfer.as_deref(), Some("bt709")); + assert_eq!(out.zimg_matrix(Some(480)), "709"); + } } diff --git a/worker/src/models/colorize_parameters.rs b/worker/src/models/colorize_parameters.rs new file mode 100644 index 00000000..de6ee45b --- /dev/null +++ b/worker/src/models/colorize_parameters.rs @@ -0,0 +1,172 @@ +//! Colourisation: colour black-and-white footage with a neural network +//! (DDColor or DeOldify Video), keeping the source's luma bit-exact and +//! replacing only its chroma. Runs on ONNX Runtime from the ai-runtime add-on; +//! each model is an add-on of its own (`colorize-ddcolor`, +//! `colorize-deoldify`). See `worker/templates/colorize.py`. + +use serde::{Deserialize, Serialize}; + +pub const COLORIZE_DDCOLOR_ADDON: &str = "colorize-ddcolor"; +pub const COLORIZE_DEOLDIFY_ADDON: &str = "colorize-deoldify"; + +/// Which network predicts the colour. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +pub enum ColorizeModel { + /// DDColor-T (Apache-2.0): vivid, predicted per frame, so it leans on + /// temporal smoothing. + #[default] + #[serde(rename = "ddcolor")] + DdColor, + /// DeOldify's Video generator (MIT): muted, trained for steady colour on + /// video. + #[serde(rename = "deoldify")] + DeOldify, +} + +impl ColorizeModel { + /// The add-on holding this model's file. + pub fn addon(&self) -> &'static str { + match self { + ColorizeModel::DdColor => COLORIZE_DDCOLOR_ADDON, + ColorizeModel::DeOldify => COLORIZE_DEOLDIFY_ADDON, + } + } + + /// The model's file in that add-on. + pub fn file(&self) -> &'static str { + match self { + ColorizeModel::DdColor => "colorize-ddcolor-tiny.onnx", + ColorizeModel::DeOldify => "colorize-deoldify-video.onnx", + } + } + + /// The name `colorize.colorize(model=...)` takes. + pub fn script_name(&self) -> &'static str { + match self { + ColorizeModel::DdColor => "ddcolor", + ColorizeModel::DeOldify => "deoldify", + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ColorizeParameters { + #[serde(default)] + pub enabled: bool, + + #[serde(default)] + pub model: ColorizeModel, + + /// DDColor's input size (pixels, square; a multiple of 32). Upstream's + /// default is 512. + #[serde(default = "default_ddcolor_size")] + pub ddcolor_render_size: i32, + + /// DeOldify's `render_factor`: the model sees a square of 16 x this many + /// pixels. Upstream's default for video is 21 (336 pixels). + #[serde(default = "default_render_factor")] + pub deoldify_render_factor: i32, + + /// How much of the predicted colour is applied: 0 leaves the picture + /// grey, 1 is the model's colour, above 1 is more saturated. + #[serde(default = "default_strength")] + pub strength: f64, + + /// Frames either side each frame's colour is averaged with (weighted + /// down to nothing where the picture moved), so it does not flicker. + #[serde(default = "default_radius")] + pub temporal_radius: i32, +} + +fn default_ddcolor_size() -> i32 { 512 } +fn default_render_factor() -> i32 { 21 } +fn default_strength() -> f64 { 1.0 } +fn default_radius() -> i32 { 2 } + +impl Default for ColorizeParameters { + fn default() -> Self { + Self { + enabled: false, + model: ColorizeModel::default(), + ddcolor_render_size: default_ddcolor_size(), + deoldify_render_factor: default_render_factor(), + strength: default_strength(), + temporal_radius: default_radius(), + } + } +} + +impl ColorizeParameters { + /// The add-on a job with this pass needs: the chosen model's. + pub fn required_addons(&self) -> Vec<&'static str> { + vec![self.model.addon()] + } + + /// The square size (pixels) the model runs at: DDColor's size rounded + /// down to a multiple of 32 (its decoder needs one), DeOldify's render + /// factor x 16, each clamped to the range the panel offers. + pub fn render_size(&self) -> u32 { + match self.model { + ColorizeModel::DdColor => (self.ddcolor_render_size.clamp(256, 768) / 32 * 32) as u32, + ColorizeModel::DeOldify => (self.deoldify_render_factor.clamp(10, 40) * 16) as u32, + } + } + + pub fn clamped_strength(&self) -> f64 { self.strength.clamp(0.0, 1.5) } + + /// Frames read either side of the one being coloured (the preview's + /// temporal context). + pub fn temporal_radius(&self) -> u32 { self.temporal_radius.clamp(0, 4) as u32 } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_and_wire_names() { + let p: ColorizeParameters = serde_json::from_str("{}").unwrap(); + assert_eq!(p, ColorizeParameters::default()); + assert!(!p.enabled); + assert_eq!(p.model, ColorizeModel::DdColor); + let json = serde_json::to_value(ColorizeParameters { model: ColorizeModel::DeOldify, ..Default::default() }).unwrap(); + assert_eq!(json["model"], "deoldify"); + assert_eq!(json["ddcolorRenderSize"], 512); + assert_eq!(json["deoldifyRenderFactor"], 21); + assert_eq!(json["temporalRadius"], 2); + assert_eq!(serde_json::to_string(&ColorizeModel::DdColor).unwrap(), "\"ddcolor\""); + } + + #[test] + fn render_sizes_are_what_each_model_can_take() { + let dd = |s| ColorizeParameters { ddcolor_render_size: s, ..Default::default() }.render_size(); + assert_eq!(dd(512), 512); + assert_eq!(dd(500), 480, "rounded down to a multiple of 32"); + assert_eq!(dd(64), 256); + assert_eq!(dd(4096), 768); + let de = |f| ColorizeParameters { model: ColorizeModel::DeOldify, deoldify_render_factor: f, ..Default::default() } + .render_size(); + assert_eq!(de(21), 336); + assert_eq!(de(0), 160); + assert_eq!(de(99), 640); + } + + #[test] + fn required_addon_per_model() { + let with = |model| ColorizeParameters { model, ..Default::default() }.required_addons(); + assert_eq!(with(ColorizeModel::DdColor), vec!["colorize-ddcolor"]); + assert_eq!(with(ColorizeModel::DeOldify), vec!["colorize-deoldify"]); + } + + /// The files named here are the ones the manifests install. + #[test] + fn model_files_are_in_their_manifests() { + let repo = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + for m in [ColorizeModel::DdColor, ColorizeModel::DeOldify] { + let manifest = std::fs::read_to_string(repo.join("app/assets/addons").join(format!("{}.json", m.addon()))).unwrap(); + assert!(manifest.contains(&format!("\"name\": \"{}\"", m.file())), "{m:?}"); + assert!(manifest.contains("\"ai-runtime\""), "{m:?} must require the AI runtime"); + } + } +} diff --git a/worker/src/models/crop_resize_parameters.rs b/worker/src/models/crop_resize_parameters.rs index 129551c6..78706c95 100644 --- a/worker/src/models/crop_resize_parameters.rs +++ b/worker/src/models/crop_resize_parameters.rs @@ -68,13 +68,24 @@ pub enum UpscaleMethod { Eedi3Rpow2, Spline36, /// Real-ESRGAN's anime video model (BSD-3), on the GPU through VSNCNN. + /// Its models are an add-on (`upscale-anime`) — see `upscale_addons.rs`. RealEsrgan, - /// Real-CUGAN (MIT, bilibili), on the GPU through VSNCNN. + /// Real-CUGAN (MIT, bilibili), on the GPU through VSNCNN. Add-on, as above. RealCugan, /// AnimeJaNai (the-database, CC BY-NC-SA 4.0), on the GPU through VSNCNN. /// Its models are an optional download, not in the deps bundle — see /// `animejanai.rs`. AnimeJanai, + /// Real-ESRGAN's general model for real-world footage (`realesr-general-x4v3`, + /// BSD-3), 4x native; a 2x upscale runs it at 4x and resamples down. Its + /// model is an add-on — see `upscale_addons.rs`. + RealEsrganGeneral, + /// jcj83429's 2xLiveActionV1_SPAN (Apache-2.0), 2x native, for live + /// action; 4x runs it twice. Add-on, as above. + LiveAction, + /// Real-ESRGAN x4plus / x2plus (BSD-3), the large RRDB networks, chosen by + /// factor. Add-on, as above. + RealEsrganPlus, } impl UpscaleMethod { @@ -83,21 +94,55 @@ impl UpscaleMethod { pub fn is_ai(self) -> bool { matches!( self, - UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan | UpscaleMethod::AnimeJanai + UpscaleMethod::RealEsrgan + | UpscaleMethod::RealCugan + | UpscaleMethod::AnimeJanai + | UpscaleMethod::RealEsrganGeneral + | UpscaleMethod::LiveAction + | UpscaleMethod::RealEsrganPlus ) } + + /// The manifest add-on holding this method's models, and the feature name + /// the worker's refusal starts with when it is missing. `None` for the + /// methods that run no model (and for AnimeJaNai, whose hand-written + /// add-on is checked separately — see `animejanai.rs`). + pub fn addon(self) -> Option<(&'static str, &'static str)> { + use crate::upscale_addons::{ANIME_ADDON, ESRGAN_PLUS_ADDON, GENERAL_ADDON}; + match self { + UpscaleMethod::RealEsrgan => Some((ANIME_ADDON, "Real-ESRGAN upscaling")), + UpscaleMethod::RealCugan => Some((ANIME_ADDON, "Real-CUGAN upscaling")), + UpscaleMethod::RealEsrganGeneral => Some((GENERAL_ADDON, "Real-ESRGAN General upscaling")), + UpscaleMethod::LiveAction => Some((GENERAL_ADDON, "Live-action upscaling")), + UpscaleMethod::RealEsrganPlus => Some((ESRGAN_PLUS_ADDON, "Real-ESRGAN x4plus upscaling")), + _ => None, + } + } } -/// One network an AI upscale runs. +/// One network an AI upscale runs. None is in the deps bundle: every model +/// is an add-on (docs/ADDONS.md). #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum AiModel { - /// A model the deps bundle ships, as a path under `vapoursynth/ncnn/models/`. - Bundled(&'static str), /// An AnimeJaNai model pair under the add-on directory: the script runs /// `sd` on a picture at most `animejanai::SD_MAX_HEIGHT` tall, else `hd`. /// Decided in the script, per doubling, because only it knows the height /// after cropping (and after a first doubling). AnimeJanai { sd: &'static str, hd: &'static str }, + /// A model from a manifest add-on: `addons//` (see + /// `upscale_addons.rs`). + Addon { addon: &'static str, file: &'static str }, +} + +impl AiModel { + /// The network's own scale factor. AnimeJaNai's double; manifest + /// add-on models say (`upscale_addons::MODELS`). + pub fn scale(&self) -> u32 { + match self { + AiModel::AnimeJanai { .. } => 2, + AiModel::Addon { file, .. } => crate::upscale_addons::scale_of(file), + } + } } /// What to do with the pixel aspect ratio (SAR) on output. @@ -140,6 +185,34 @@ pub fn parse_hex_color(text: &str) -> Option<[u8; 3]> { Some([channel(0)?, channel(2)?, channel(4)?]) } +/// Whether an SD (BT.601) source is converted to BT.709 colour (app 1.7.0). +/// +/// An SD source upscaled to HD keeps its BT.601 matrix unless something +/// converts it. The tag on the file says 601, which is correct, but many +/// players and TVs ignore the tag on an HD-sized picture and decode it as 709, +/// shifting greens by up to 40 levels and reds by up to 25 (measured: 75% +/// green comes back as 161 instead of 191). Converting the samples and +/// declaring 709 makes every player agree. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum Bt709Conversion { + /// Convert a BT.601 source when the output is HD-sized + /// (`HD_MIN_HEIGHT` lines or `HD_MIN_WIDTH` columns). + #[default] + Auto, + /// Convert a BT.601 source whatever the output size. + On, + /// Never convert: the output keeps the source's matrix and tags. + Off, +} + +/// The smallest output that counts as HD for [`Bt709Conversion::Auto`]: 720 +/// lines, or 1280 columns for a wide picture cropped short (1280x536 scope). +/// The same height `ColorMetadata::zimg_matrix` guesses 709 from, and the +/// definition players use when they pick a matrix by size. +pub const HD_MIN_HEIGHT: i32 = 720; +pub const HD_MIN_WIDTH: i32 = 1280; + /// Crop/resize preset options. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] @@ -291,6 +364,18 @@ pub struct CropResizeParameters { #[serde(default)] pub animejanai_sharp: bool, + /// Real-ESRGAN General's denoise variant (`realesr-general-wdn-x4v3`), + /// which also removes noise as it upscales. + #[serde(default)] + pub general_denoise: bool, + + /// How much of an AI upscale to keep, 0.0-1.0: the result is blended with + /// a Spline36 upscale of the same frame (`std.Merge`) at the clip's own + /// format. 1.0 (the default) is the network's output alone, and leaves the + /// script exactly as it was before this setting existed. + #[serde(default = "default_ai_strength")] + pub ai_strength: f64, + // --- nnedi3 controls for the edge-directed upscale paths --- // With EEDI3 selected these shape the nnedi3 sclip that guides it. @@ -335,12 +420,18 @@ pub struct CropResizeParameters { /// Maximum search distance (EEDI3 `mdis`). #[serde(default)] pub upscale_mdis: Option, + + /// BT.601 -> BT.709 colour conversion (app 1.7.0). Absent in a job or + /// preset saved before it existed, which therefore gets `Auto`. + #[serde(default)] + pub convert_to_bt709: Bt709Conversion, } fn default_true() -> bool { true } fn default_upscale_factor() -> i32 { 2 } fn default_cugan_noise() -> String { "conservative".to_string() } fn default_animejanai_model() -> String { "compact".to_string() } +fn default_ai_strength() -> f64 { 1.0 } impl Default for CropResizeParameters { fn default() -> Self { @@ -375,6 +466,8 @@ impl Default for CropResizeParameters { cugan_noise: default_cugan_noise(), animejanai_model: default_animejanai_model(), animejanai_sharp: false, + general_denoise: false, + ai_strength: default_ai_strength(), upscale_nsize: None, upscale_neurons: None, upscale_qual: None, @@ -385,6 +478,7 @@ impl Default for CropResizeParameters { upscale_gamma: None, upscale_nrad: None, upscale_mdis: None, + convert_to_bt709: Bt709Conversion::default(), } } } @@ -441,16 +535,27 @@ impl CropResizeParameters { self.uses_ai_upscale() && self.upscale_method == UpscaleMethod::AnimeJanai } - /// The models an AI upscale runs, in order, as paths under - /// `vapoursynth/ncnn/models/`. Factors follow the edge-directed path's + /// The manifest add-on this pass's AI upscale needs, with the feature name + /// for the refusal (see `UpscaleMethod::addon`). + pub fn upscale_addon(&self) -> Option<(&'static str, &'static str)> { + if self.uses_ai_upscale() { self.upscale_method.addon() } else { None } + } + + /// The models an AI upscale runs, in order, each a file in an add-on's + /// directory. Factors follow the edge-directed path's /// rule (3x behaves as 2x). Real-ESRGAN has a 4x model; Real-CUGAN's - /// bundled models are 2x only, so 4x runs it twice, denoising on the first + /// models are 2x only, so 4x runs it twice, denoising on the first /// pass only — a second denoise pass would smooth the result twice. /// AnimeJaNai is 2x only, so 4x is two doublings; the script picks SD or /// HD for each, so a DVD runs the SD model and then an HD one. pub fn ai_upscale_models(&self) -> Vec { let four = self.upscale_factor >= 4; - let bundled = |names: Vec<&'static str>| names.into_iter().map(AiModel::Bundled).collect(); + let anime = |names: Vec<&'static str>| { + names + .into_iter() + .map(|file| AiModel::Addon { addon: crate::upscale_addons::ANIME_ADDON, file }) + .collect() + }; match self.upscale_method { UpscaleMethod::AnimeJanai => { let pair = AiModel::AnimeJanai { @@ -459,27 +564,91 @@ impl CropResizeParameters { }; vec![pair; if four { 2 } else { 1 }] } - UpscaleMethod::RealEsrgan => bundled(vec![if four { - "RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx" - } else { - "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx" - }]), + UpscaleMethod::RealEsrgan => { + use crate::upscale_addons::{ANIME_ESRGAN_X2, ANIME_ESRGAN_X4}; + anime(vec![if four { ANIME_ESRGAN_X4 } else { ANIME_ESRGAN_X2 }]) + } UpscaleMethod::RealCugan => { + use crate::upscale_addons::{CUGAN_CONSERVATIVE, CUGAN_DENOISE3X, CUGAN_NO_DENOISE}; let first = match self.cugan_noise.as_str() { - "none" => "cugan/up2x-latest-no-denoise.onnx", - "denoise3x" => "cugan/up2x-latest-denoise3x.onnx", - _ => "cugan/up2x-latest-conservative.onnx", + "none" => CUGAN_NO_DENOISE, + "denoise3x" => CUGAN_DENOISE3X, + _ => CUGAN_CONSERVATIVE, }; - bundled(if four { - vec![first, "cugan/up2x-latest-no-denoise.onnx"] - } else { - vec![first] - }) + anime(if four { vec![first, CUGAN_NO_DENOISE] } else { vec![first] }) + } + UpscaleMethod::RealEsrganGeneral => { + use crate::upscale_addons::{GENERAL_ADDON, GENERAL_WDN_X4, GENERAL_X4}; + let file = if self.general_denoise { GENERAL_WDN_X4 } else { GENERAL_X4 }; + // 4x only: a 2x upscale resamples its result down (see + // `ai_resample_to`). + vec![AiModel::Addon { addon: GENERAL_ADDON, file }] + } + UpscaleMethod::LiveAction => { + use crate::upscale_addons::{GENERAL_ADDON, LIVE_ACTION_X2}; + let m = AiModel::Addon { addon: GENERAL_ADDON, file: LIVE_ACTION_X2 }; + vec![m; if four { 2 } else { 1 }] + } + UpscaleMethod::RealEsrganPlus => { + use crate::upscale_addons::{ESRGAN_PLUS_ADDON, PLUS_X2, PLUS_X4}; + vec![AiModel::Addon { addon: ESRGAN_PLUS_ADDON, file: if four { PLUS_X4 } else { PLUS_X2 } }] } _ => Vec::new(), } } + /// The factor an AI upscale delivers: 4 for 4x, else 2 (3x behaves as 2x, + /// as on the edge-directed path). + pub fn ai_upscale_factor(&self) -> u32 { + if self.upscale_factor >= 4 { 4 } else { 2 } + } + + /// When the networks enlarge further than asked — Real-ESRGAN General is + /// 4x only, so a 2x job runs it at 4x — the factor to resample their + /// result down to (Spline36, in the networks' float RGB, before the + /// conversion back). `None` when the networks land on the factor exactly, + /// which leaves the script as it was. Upstream does the same for its own + /// `--outscale` (4x, then a Lanczos resize); see the report in + /// docs/reports/2026-11-ai-upscale-addons/ for the kernel comparison. + pub fn ai_resample_to(&self) -> Option { + let models = self.ai_upscale_models(); + if models.is_empty() { + return None; + } + let total: u32 = models.iter().map(AiModel::scale).product(); + (total != self.ai_upscale_factor()).then(|| self.ai_upscale_factor()) + } + + /// The AI strength as a blend weight for the network's output, or `None` + /// at full strength (1.0, the default), where no blend is emitted and the + /// script is unchanged. Clamped to 0..1; a non-finite value (a hand-edited + /// preset) is full strength. + pub fn ai_blend_weight(&self) -> Option { + let s = self.ai_strength; + if !s.is_finite() || s >= 1.0 { + None + } else { + Some(s.max(0.0)) + } + } + + /// When a frame runs in tiles, and their size: `(largest whole frame in + /// pixels, tile size)`. Above SD every model tiles at 512 — whole-frame + /// Real-CUGAN at 1080p ran out of GPU memory on a 16 GB Mac. Real-ESRGAN + /// x4plus (23 RRDB blocks, then two doublings at full width) is far larger: + /// whole-frame at 720x576 lost the Vulkan device (`VK_ERROR_DEVICE_LOST`, + /// then a segfault) and 512-pixel tiles thrashed at 29 s a frame, so it + /// runs in 384-pixel tiles (6.8 s a frame there; 256 took 10 s). x2plus + /// works at a quarter of the area until its last layers and runs a DVD + /// frame whole, so it keeps the common rule. + pub fn ai_tiling(&self) -> (&'static str, u32) { + if self.upscale_method == UpscaleMethod::RealEsrganPlus && self.ai_upscale_factor() == 4 { + ("384 * 384", 384) + } else { + ("720 * 576", 512) + } + } + /// Whether the pixel grid itself has to be resampled to square up the /// pixels. True even with no target size — "make this anamorphic source /// square" is a resize in its own right. @@ -574,6 +743,92 @@ impl CropResizeParameters { } } + /// The frame size this pass delivers from a `width` x `height` picture + /// whose samples are `sar` wide, mirroring the script: crop, then the + /// integer upscale, then the resize fitting (`RESIZE_STANDARD`), then the + /// bars. The script stays the authority on the exact size; this is what + /// the worker needs to decide things that must be fixed before the script + /// runs, such as the colour tags on both sides of the encoder. + pub fn output_size(&self, width: i32, height: i32, sar: f64) -> (i32, i32) { + if !self.enabled { + return (width, height); + } + let (mut w, mut h) = (width as f64, height as f64); + if self.crop_enabled { + w -= f64::from(self.crop_left.max(0) + self.crop_right.max(0)); + h -= f64::from(self.crop_top.max(0) + self.crop_bottom.max(0)); + } + let squares = self.squares_pixels(); + if self.resize_enabled || self.use_integer_upscale || squares { + if self.use_integer_upscale { + let factor = if self.upscale_method.is_ai() { + f64::from(self.ai_upscale_factor()) + } else if self.upscale_method == UpscaleMethod::Spline36 { + f64::from(self.upscale_factor.max(1)) + } else { + // Edge-directed: max(factor // 2, 1) doublings. + f64::from(1 << (self.upscale_factor / 2).max(1)) + }; + w *= factor; + h *= factor; + } + if self.resize_enabled || squares { + let even = |v: f64| ((v.round() as i32) & !1).max(2) as f64; + let display = self.display_aspect_ratio().unwrap_or(w * sar / h.max(1.0)); + let fit = if squares { display } else { w / h.max(1.0) }; + let mut tw = if self.resize_enabled { self.target_width.unwrap_or(-1) } else { -1 } as f64; + let mut th = if self.resize_enabled { self.target_height.unwrap_or(-1) } else { -1 } as f64; + if squares && tw <= 0.0 && th <= 0.0 { + th = h; + } + if self.maintain_aspect || (squares && tw <= 0.0 && th <= 0.0) { + if tw > 0.0 && th <= 0.0 { + th = even(tw / fit); + } else if th > 0.0 && tw <= 0.0 { + tw = even(th * fit); + } else if tw > 0.0 && th > 0.0 { + let scale = (tw / (h * fit)).min(th / h); + tw = even(h * fit * scale); + th = even(h * scale); + } + } + if tw > 0.0 { + w = tw; + } + if th > 0.0 { + h = th; + } + if self.pads_to_aspect() { + w = w.max(f64::from(self.target_width.unwrap_or(0))); + h = h.max(f64::from(self.target_height.unwrap_or(0))); + } + } + } + if let Some((cw, ch)) = self.pad_canvas() { + if cw > 0 { + w = w.max(f64::from(cw)); + } + if ch > 0 { + h = h.max(f64::from(ch)); + } + } + (w.round() as i32, h.round() as i32) + } + + /// Whether the samples are converted from BT.601 to BT.709, for a source + /// whose colour is BT.601 (`source_is_601`, see + /// `ColorMetadata::is_bt601_family`) delivered at `output` size. + pub fn converts_to_bt709(&self, source_is_601: bool, output: (i32, i32)) -> bool { + if !self.enabled || !source_is_601 { + return false; + } + match self.convert_to_bt709 { + Bt709Conversion::Off => false, + Bt709Conversion::On => true, + Bt709Conversion::Auto => output.1 >= HD_MIN_HEIGHT || output.0 >= HD_MIN_WIDTH, + } + } + /// Get total horizontal crop. pub fn total_horizontal_crop(&self) -> i32 { self.crop_left + self.crop_right @@ -781,27 +1036,28 @@ mod tests { cugan_noise: noise.into(), ..Default::default() }; - let b = |names: &[&'static str]| names.iter().map(|n| AiModel::Bundled(n)).collect::>(); - assert_eq!(p(UpscaleMethod::RealEsrgan, 2, "").ai_upscale_models(), - b(&["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"])); - assert_eq!(p(UpscaleMethod::RealEsrgan, 3, "").ai_upscale_models(), - b(&["RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"])); - assert_eq!(p(UpscaleMethod::RealEsrgan, 4, "").ai_upscale_models(), - b(&["RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx"])); - assert_eq!(p(UpscaleMethod::RealCugan, 2, "denoise3x").ai_upscale_models(), - b(&["cugan/up2x-latest-denoise3x.onnx"])); + use crate::upscale_addons::*; + let b = |names: &[&'static str]| { + names.iter().map(|&file| AiModel::Addon { addon: ANIME_ADDON, file }).collect::>() + }; + assert_eq!(p(UpscaleMethod::RealEsrgan, 2, "").ai_upscale_models(), b(&[ANIME_ESRGAN_X2])); + assert_eq!(p(UpscaleMethod::RealEsrgan, 3, "").ai_upscale_models(), b(&[ANIME_ESRGAN_X2])); + assert_eq!(p(UpscaleMethod::RealEsrgan, 4, "").ai_upscale_models(), b(&[ANIME_ESRGAN_X4])); + assert_eq!(p(UpscaleMethod::RealCugan, 2, "denoise3x").ai_upscale_models(), b(&[CUGAN_DENOISE3X])); assert_eq!(p(UpscaleMethod::RealCugan, 4, "none").ai_upscale_models(), - b(&["cugan/up2x-latest-no-denoise.onnx", "cugan/up2x-latest-no-denoise.onnx"])); + b(&[CUGAN_NO_DENOISE, CUGAN_NO_DENOISE])); // An unknown strength from a hand-edited preset is the default. - assert_eq!(p(UpscaleMethod::RealCugan, 2, "max").ai_upscale_models(), - b(&["cugan/up2x-latest-conservative.onnx"])); - // Every model named here is one the deps bundle ships. + assert_eq!(p(UpscaleMethod::RealCugan, 2, "max").ai_upscale_models(), b(&[CUGAN_CONSERVATIVE])); + // Every model named here is one the add-on installs, and the add-on + // is what the pass declares it needs. for m in [UpscaleMethod::RealEsrgan, UpscaleMethod::RealCugan] { for f in [2, 4] { for n in ["none", "conservative", "denoise3x"] { - for model in p(m, f, n).ai_upscale_models() { - let AiModel::Bundled(model) = model else { panic!("{model:?}") }; - assert!(crate::gpu_plugin::UPSCALE_MODELS.contains(&model), "{model}"); + let params = p(m, f, n); + assert_eq!(params.upscale_addon().map(|a| a.0), Some(ANIME_ADDON)); + for model in params.ai_upscale_models() { + let AiModel::Addon { addon, file } = model else { panic!("{model:?}") }; + assert!(MODELS.contains(&(addon, file, model.scale())), "{file}"); } } } @@ -836,4 +1092,155 @@ mod tests { let d = CropResizeParameters::default(); assert_eq!((d.animejanai_model.as_str(), d.animejanai_sharp), ("compact", false)); } + + #[test] + fn test_addon_upscalers_follow_their_factor_rules() { + use crate::upscale_addons::*; + let p = |method, factor, denoise| CropResizeParameters { + enabled: true, + use_integer_upscale: true, + upscale_method: method, + upscale_factor: factor, + general_denoise: denoise, + ..Default::default() + }; + let a = |addon, file| AiModel::Addon { addon, file }; + // General is 4x only: 2x (and 3x) run it at 4x and resample down. + for f in [2, 3] { + let g = p(UpscaleMethod::RealEsrganGeneral, f, false); + assert_eq!(g.ai_upscale_models(), [a(GENERAL_ADDON, GENERAL_X4)]); + assert_eq!(g.ai_resample_to(), Some(2)); + } + let g4 = p(UpscaleMethod::RealEsrganGeneral, 4, true); + assert_eq!(g4.ai_upscale_models(), [a(GENERAL_ADDON, GENERAL_WDN_X4)]); + assert_eq!(g4.ai_resample_to(), None); + // SPAN is 2x only: 4x runs it twice. + assert_eq!(p(UpscaleMethod::LiveAction, 2, false).ai_upscale_models(), [a(GENERAL_ADDON, LIVE_ACTION_X2)]); + assert_eq!(p(UpscaleMethod::LiveAction, 4, false).ai_upscale_models(), [a(GENERAL_ADDON, LIVE_ACTION_X2); 2]); + assert_eq!(p(UpscaleMethod::LiveAction, 4, false).ai_resample_to(), None); + // x2plus / x4plus by factor, the 4x one in its own tiles. + let plus2 = p(UpscaleMethod::RealEsrganPlus, 2, false); + let plus4 = p(UpscaleMethod::RealEsrganPlus, 4, false); + assert_eq!(plus2.ai_upscale_models(), [a(ESRGAN_PLUS_ADDON, PLUS_X2)]); + assert_eq!(plus4.ai_upscale_models(), [a(ESRGAN_PLUS_ADDON, PLUS_X4)]); + assert_eq!(plus2.ai_tiling(), ("720 * 576", 512)); + assert_eq!(plus4.ai_tiling(), ("384 * 384", 384)); + // The bundled models never resample, and keep the common tiles. + for m in [UpscaleMethod::RealEsrgan, UpscaleMethod::RealCugan, UpscaleMethod::AnimeJanai] { + for f in [2, 4] { + assert_eq!(p(m, f, false).ai_resample_to(), None, "{m:?} {f}"); + assert_eq!(p(m, f, false).ai_tiling(), ("720 * 576", 512)); + } + } + // The add-on each needs, only when it is what the pass runs. + assert_eq!(plus4.upscale_addon().map(|a| a.0), Some(ESRGAN_PLUS_ADDON)); + assert_eq!(p(UpscaleMethod::LiveAction, 2, false).upscale_addon().map(|a| a.0), Some(GENERAL_ADDON)); + assert_eq!(p(UpscaleMethod::RealEsrgan, 2, false).upscale_addon().map(|a| a.0), Some(ANIME_ADDON)); + assert_eq!(p(UpscaleMethod::RealCugan, 4, false).upscale_addon().map(|a| a.0), Some(ANIME_ADDON)); + assert_eq!(p(UpscaleMethod::AnimeJanai, 2, false).upscale_addon(), None); + let off = CropResizeParameters { use_integer_upscale: false, ..plus4.clone() }; + assert_eq!(off.upscale_addon(), None); + for m in [UpscaleMethod::RealEsrganGeneral, UpscaleMethod::LiveAction, UpscaleMethod::RealEsrganPlus] { + assert!(m.is_ai(), "{m:?}"); + } + } + + #[test] + fn test_ai_strength_defaults_to_full_and_clamps() { + let d = CropResizeParameters::default(); + assert_eq!(d.ai_strength, 1.0); + assert_eq!(d.ai_blend_weight(), None); + let with = |s| CropResizeParameters { ai_strength: s, ..Default::default() }.ai_blend_weight(); + assert_eq!(with(0.5), Some(0.5)); + assert_eq!(with(-0.2), Some(0.0)); + assert_eq!(with(1.5), None); + assert_eq!(with(f64::NAN), None); + // Absent from old JSON = full strength; the camel-case wire names. + let p: CropResizeParameters = serde_json::from_str(r#"{"enabled":true}"#).unwrap(); + assert_eq!((p.ai_strength, p.general_denoise), (1.0, false)); + let json = serde_json::to_string(&CropResizeParameters { + upscale_method: UpscaleMethod::RealEsrganGeneral, + ..Default::default() + }) + .unwrap(); + assert!(json.contains("\"aiStrength\":1.0"), "{json}"); + assert!(json.contains("\"generalDenoise\":false"), "{json}"); + assert!(json.contains("\"upscaleMethod\":\"realEsrganGeneral\""), "{json}"); + for (v, s) in [(UpscaleMethod::LiveAction, "liveAction"), (UpscaleMethod::RealEsrganPlus, "realEsrganPlus")] { + assert_eq!(serde_json::to_string(&v).unwrap(), format!("\"{s}\"")); + } + } + + #[test] + fn output_size_follows_the_script() { + let base = CropResizeParameters { enabled: true, ..Default::default() }; + // Nothing switched on: the source size. + assert_eq!(base.output_size(720, 576, 1.0), (720, 576)); + // Crop, then a 2x NNEDI3 doubling. + let p = CropResizeParameters { + crop_enabled: true, + crop_left: 8, + crop_right: 8, + crop_bottom: 8, + use_integer_upscale: true, + ..base.clone() + }; + assert_eq!(p.output_size(720, 576, 1.0), (1408, 1136)); + // 4x edge-directed is two doublings; Spline36 multiplies by the factor. + let p4 = CropResizeParameters { use_integer_upscale: true, upscale_factor: 4, ..base.clone() }; + assert_eq!(p4.output_size(720, 480, 1.0), (2880, 1920)); + // A 1920x1080 box with the storage aspect kept: 720x576 fits as 1350x1080. + let fit = CropResizeParameters { + resize_enabled: true, + target_width: Some(1920), + target_height: Some(1080), + ..base.clone() + }; + assert_eq!(fit.output_size(720, 576, 1.0), (1350, 1080)); + // Squared up from a 16:9 PAL DVD (SAR 64:45): fills the box exactly. + let square = CropResizeParameters { pixel_aspect: PixelAspectMode::Square, ..fit.clone() }; + assert_eq!(square.output_size(720, 576, 64.0 / 45.0), (1920, 1080)); + // Squared with no target size keeps the height. + let square_only = CropResizeParameters { pixel_aspect: PixelAspectMode::Square, ..base.clone() }; + assert_eq!(square_only.output_size(720, 576, 16.0 / 15.0).1, 576); + // A canvas only ever adds. + let canvas = CropResizeParameters { + pad_enabled: true, + pad_width: Some(1920), + pad_height: Some(1080), + ..base.clone() + }; + assert_eq!(canvas.output_size(720, 576, 1.0), (1920, 1080)); + // A disabled pass changes nothing. + let off = CropResizeParameters { enabled: false, ..fit }; + assert_eq!(off.output_size(720, 576, 1.0), (720, 576)); + } + + #[test] + fn bt709_conversion_auto_converts_only_bt601_to_hd() { + let p = CropResizeParameters { enabled: true, ..Default::default() }; + assert_eq!(p.convert_to_bt709, Bt709Conversion::Auto, "the default"); + assert!(p.converts_to_bt709(true, (1440, 1080))); + assert!(p.converts_to_bt709(true, (1280, 536)), "a wide picture cropped short is HD"); + assert!(p.converts_to_bt709(true, (960, 720))); + assert!(!p.converts_to_bt709(true, (1024, 576)), "SD-sized output stays 601"); + assert!(!p.converts_to_bt709(false, (1920, 1080)), "only a 601 source"); + let on = CropResizeParameters { convert_to_bt709: Bt709Conversion::On, ..p.clone() }; + assert!(on.converts_to_bt709(true, (720, 576))); + assert!(!on.converts_to_bt709(false, (720, 576))); + let off = CropResizeParameters { convert_to_bt709: Bt709Conversion::Off, ..p.clone() }; + assert!(!off.converts_to_bt709(true, (1920, 1080))); + let disabled = CropResizeParameters { enabled: false, convert_to_bt709: Bt709Conversion::On, ..p }; + assert!(!disabled.converts_to_bt709(true, (1920, 1080)), "a disabled pass converts nothing"); + } + + #[test] + fn bt709_conversion_wire_names_and_missing_field() { + for (v, s) in [(Bt709Conversion::Auto, "auto"), (Bt709Conversion::On, "on"), (Bt709Conversion::Off, "off")] { + assert_eq!(serde_json::to_string(&v).unwrap(), format!("\"{s}\"")); + } + // A preset saved before 1.7.0 has no field: Auto. + let p: CropResizeParameters = serde_json::from_str(r#"{"enabled": true}"#).unwrap(); + assert_eq!(p.convert_to_bt709, Bt709Conversion::Auto); + } } diff --git a/worker/src/models/deblock_parameters.rs b/worker/src/models/deblock_parameters.rs index 71a49124..10dc1fa3 100644 --- a/worker/src/models/deblock_parameters.rs +++ b/worker/src/models/deblock_parameters.rs @@ -12,6 +12,11 @@ pub enum DeblockMethod { /// ringing and mosquito noise rather than block edges. #[serde(rename = "DCTFilter")] DctFilter, + /// mpeg2stinx: the comb-like noise of a film DVD whose progressive frames + /// were encoded as interlaced MPEG-2 (vs-jetpack's mpeg2stinx, ported into + /// the templates). Temporal (one frame each side); never per field. + #[serde(rename = "mpeg2stinx")] + Mpeg2Stinx, } #[allow(dead_code)] @@ -21,6 +26,7 @@ impl DeblockMethod { DeblockMethod::DeblockQed => "Deblock_QED", DeblockMethod::Deblock => "Deblock", DeblockMethod::DctFilter => "DCTFilter", + DeblockMethod::Mpeg2Stinx => "mpeg2stinx", } } } diff --git a/worker/src/models/dehalo_parameters.rs b/worker/src/models/dehalo_parameters.rs index 3aa47376..90ef8c5e 100644 --- a/worker/src/models/dehalo_parameters.rs +++ b/worker/src/models/dehalo_parameters.rs @@ -27,6 +27,12 @@ pub enum DehaloMethod { /// dehalo's wider bright band. #[serde(rename = "HQDeringmod")] HqDeringmod, + /// vine: dehalo by non-local-means filtering of the error (vs-jetpack's + /// vine_dehalo, ported into the templates). Keeps the edge detail the halo + /// rides on rather than blurring the band (app 1.7.0, deps 1.16.0: needs + /// vszip's NLMeans). + #[serde(rename = "vine")] + Vine, } #[allow(dead_code)] @@ -41,6 +47,7 @@ impl DehaloMethod { DehaloMethod::Vinverse => "Vinverse", DehaloMethod::Vinverse2 => "Vinverse2", DehaloMethod::HqDeringmod => "HQDeringmod", + DehaloMethod::Vine => "vine", } } @@ -203,6 +210,20 @@ pub struct DehaloParameters { /// inside havsfunc, which is usually what you want. #[serde(default)] pub dering_darkthr: Option, + + // --- vine parameters --- + + /// NLMeans strength (vs-jetpack's `strength`, default 16). Depth-independent. + #[serde(default = "default_vine_strength")] + pub vine_strength: f64, + + /// How much of the supersampled pass to blend in (`sharp`, default 0.5). + #[serde(default = "default_vine_sharp")] + pub vine_sharp: f64, + + /// Gaussian sigma of the low/high frequency split (`sigma`, default 1.0). + #[serde(default = "default_vine_sigma")] + pub vine_sigma: f64, } fn default_rx() -> f64 { 2.0 } @@ -213,6 +234,9 @@ fn default_low_threshold() -> i32 { 80 } fn default_high_threshold() -> i32 { 128 } fn default_yahr_blur() -> i32 { 2 } fn default_yahr_depth() -> i32 { 32 } +fn default_vine_strength() -> f64 { 16.0 } +fn default_vine_sharp() -> f64 { 0.5 } +fn default_vine_sigma() -> f64 { 1.0 } impl Default for DehaloParameters { fn default() -> Self { @@ -248,6 +272,24 @@ impl Default for DehaloParameters { dering_mthr: None, dering_thr: None, dering_darkthr: None, + vine_strength: default_vine_strength(), + vine_sharp: default_vine_sharp(), + vine_sigma: default_vine_sigma(), } } } + +impl DehaloParameters { + /// vine's (strength, sharp, sigma), clamped to what the script can take: + /// NLMeans needs h > 0; vs-jetpack refuses sharp <= 0 and caps its effect + /// at 1; the Gaussian kernel is limited to 25 taps (sigma 4). A non-finite + /// value falls back to the default rather than reaching the script. + pub fn vine_effective(&self) -> (f64, f64, f64) { + let pick = |v: f64, d: f64, lo: f64, hi: f64| if v.is_finite() { v.clamp(lo, hi) } else { d }; + ( + pick(self.vine_strength, default_vine_strength(), 1.0, 64.0), + pick(self.vine_sharp, default_vine_sharp(), 0.05, 1.0), + pick(self.vine_sigma, default_vine_sigma(), 0.3, 4.0), + ) + } +} diff --git a/worker/src/models/face_restore_parameters.rs b/worker/src/models/face_restore_parameters.rs new file mode 100644 index 00000000..ee651c9f --- /dev/null +++ b/worker/src/models/face_restore_parameters.rs @@ -0,0 +1,206 @@ +//! Face Restoration: find the faces in each frame and restore them with a +//! neural network (RestoreFormer++, GFPGAN or CodeFormer), blended back into +//! the frame at the clip's own resolution. Runs on ONNX Runtime from the +//! ai-runtime add-on; the models are add-ons too (`face-restoration`, the +//! base with the detector, parser and RestoreFormer++; `face-gfpgan` and +//! `face-codeformer`, each non-commercial). See `worker/templates/face_restore.py`. + +use serde::{Deserialize, Serialize}; + +/// The base add-on: RetinaFace, BiSeNet and RestoreFormer++. +pub const FACE_RESTORATION_ADDON: &str = "face-restoration"; +pub const FACE_GFPGAN_ADDON: &str = "face-gfpgan"; +pub const FACE_CODEFORMER_ADDON: &str = "face-codeformer"; + +/// Model files, as the add-on manifests (`app/assets/addons/`) name them. +pub const FACE_DETECTOR_FILE: &str = "face-restoration-retinaface-r50.onnx"; +pub const FACE_PARSER_FILE: &str = "face-restoration-bisenet.onnx"; + +/// The temporal radius smoothing uses: landmarks smoothed over, and restored +/// faces blended with, one frame either side. Measured against 0 and 2 in +/// docs/reports/2026-11-face-restoration. +pub const FACE_SMOOTHING_RADIUS: u32 = 1; + +/// Which network restores the faces. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +pub enum FaceRestoreModel { + /// RestoreFormer++ (Apache-2.0): the default, in the base add-on. + #[default] + RestoreFormer, + /// GFPGAN v1.4 (non-commercial): its own add-on. + Gfpgan, + /// CodeFormer (S-Lab License, non-commercial): its own add-on, with a + /// fidelity control. + CodeFormer, +} + +impl FaceRestoreModel { + /// The add-on holding this model's file. + pub fn addon(&self) -> &'static str { + match self { + FaceRestoreModel::RestoreFormer => FACE_RESTORATION_ADDON, + FaceRestoreModel::Gfpgan => FACE_GFPGAN_ADDON, + FaceRestoreModel::CodeFormer => FACE_CODEFORMER_ADDON, + } + } + + /// The model's file in that add-on. + pub fn file(&self) -> &'static str { + match self { + FaceRestoreModel::RestoreFormer => "face-restoration-restoreformerpp.onnx", + FaceRestoreModel::Gfpgan => "face-gfpgan-v1.4.onnx", + FaceRestoreModel::CodeFormer => "face-codeformer-v0.1.0.onnx", + } + } + + /// The name `face_restore.restore_faces(model=...)` takes. + pub fn script_name(&self) -> &'static str { + match self { + FaceRestoreModel::RestoreFormer => "restoreformer", + FaceRestoreModel::Gfpgan => "gfpgan", + FaceRestoreModel::CodeFormer => "codeformer", + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct FaceRestoreParameters { + #[serde(default)] + pub enabled: bool, + + #[serde(default)] + pub model: FaceRestoreModel, + + /// How much of the restored face replaces the original (0-1). + #[serde(default = "default_strength")] + pub strength: f64, + + /// CodeFormer's fidelity weight `w`: 0 trusts its learned faces, 1 stays + /// closest to the picture. Ignored by the other models. + #[serde(default = "default_fidelity")] + pub code_former_fidelity: f64, + + /// The detector score (0-1) a face needs. + #[serde(default = "default_threshold")] + pub detection_threshold: f64, + + /// Faces smaller than this (pixels, the larger side of the detected box, + /// at the pass's resolution) are left alone: there is too little in them + /// to restore, and a restorer invents a face rather than recovering one. + #[serde(default = "default_min_face")] + pub min_face_size: i32, + + /// At most this many faces (the largest) are restored per frame. + #[serde(default = "default_max_faces")] + pub max_faces: i32, + + /// Steady each face over one frame either side: its landmarks smoothed, + /// one-frame detection gaps bridged, and the restored face averaged (in + /// the aligned face space, weighted down where the input moved) with its + /// neighbours', so it does not jitter, blink or shimmer. + #[serde(default = "default_true")] + pub temporal_smoothing: bool, +} + +fn default_strength() -> f64 { 0.8 } +fn default_fidelity() -> f64 { 0.7 } +fn default_threshold() -> f64 { 0.8 } +fn default_min_face() -> i32 { 32 } +fn default_max_faces() -> i32 { 8 } +fn default_true() -> bool { true } + +impl Default for FaceRestoreParameters { + fn default() -> Self { + Self { + enabled: false, + model: FaceRestoreModel::default(), + strength: default_strength(), + code_former_fidelity: default_fidelity(), + detection_threshold: default_threshold(), + min_face_size: default_min_face(), + max_faces: default_max_faces(), + temporal_smoothing: true, + } + } +} + +impl FaceRestoreParameters { + /// The add-ons a job with this pass needs: the base (detector and parser) + /// always, and the chosen model's own add-on. + pub fn required_addons(&self) -> Vec<&'static str> { + let mut ids = vec![FACE_RESTORATION_ADDON]; + if self.model.addon() != FACE_RESTORATION_ADDON { + ids.push(self.model.addon()); + } + ids + } + + /// The smoothing radius `restore_faces(temporal_radius=...)` takes: each + /// face's landmarks are smoothed, and each restored face blended, over + /// this many frames either side. 0 restores every frame on its own. + pub fn temporal_radius(&self) -> u32 { + if self.temporal_smoothing { FACE_SMOOTHING_RADIUS } else { 0 } + } + + /// Frames read either side of the one being restored (the preview's + /// temporal context): the blend reads `temporal_radius` neighbours, and + /// their landmarks are smoothed over `temporal_radius` more. + pub fn frames_read_either_side(&self) -> u32 { + self.temporal_radius() * 2 + } + + pub fn clamped_strength(&self) -> f64 { self.strength.clamp(0.0, 1.0) } + pub fn clamped_fidelity(&self) -> f64 { self.code_former_fidelity.clamp(0.0, 1.0) } + pub fn clamped_threshold(&self) -> f64 { self.detection_threshold.clamp(0.3, 0.99) } + pub fn clamped_min_face(&self) -> i32 { self.min_face_size.clamp(8, 1024) } + pub fn clamped_max_faces(&self) -> i32 { self.max_faces.clamp(1, 64) } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn defaults_and_wire_names() { + let p: FaceRestoreParameters = serde_json::from_str("{}").unwrap(); + assert_eq!(p, FaceRestoreParameters::default()); + assert!(!p.enabled); + assert_eq!(p.model, FaceRestoreModel::RestoreFormer); + let json = serde_json::to_value(FaceRestoreParameters { model: FaceRestoreModel::CodeFormer, ..Default::default() }).unwrap(); + assert_eq!(json["model"], "codeFormer"); + assert_eq!(json["codeFormerFidelity"], 0.7); + assert_eq!(json["temporalSmoothing"], true); + for (m, name) in [ + (FaceRestoreModel::RestoreFormer, "\"restoreFormer\""), + (FaceRestoreModel::Gfpgan, "\"gfpgan\""), + (FaceRestoreModel::CodeFormer, "\"codeFormer\""), + ] { + assert_eq!(serde_json::to_string(&m).unwrap(), name); + } + } + + /// Every model needs the base add-on (detector and parser); the + /// non-commercial ones need their own as well. + #[test] + fn required_addons_per_model() { + let with = |model| FaceRestoreParameters { model, ..Default::default() }.required_addons(); + assert_eq!(with(FaceRestoreModel::RestoreFormer), vec!["face-restoration"]); + assert_eq!(with(FaceRestoreModel::Gfpgan), vec!["face-restoration", "face-gfpgan"]); + assert_eq!(with(FaceRestoreModel::CodeFormer), vec!["face-restoration", "face-codeformer"]); + } + + /// The files named here are the ones the manifests install. + #[test] + fn model_files_are_in_their_manifests() { + let repo = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + let read = |id: &str| std::fs::read_to_string(repo.join("app/assets/addons").join(format!("{id}.json"))).unwrap(); + for m in [FaceRestoreModel::RestoreFormer, FaceRestoreModel::Gfpgan, FaceRestoreModel::CodeFormer] { + assert!(read(m.addon()).contains(&format!("\"name\": \"{}\"", m.file())), "{m:?}"); + } + let base = read(FACE_RESTORATION_ADDON); + assert!(base.contains(&format!("\"name\": \"{FACE_DETECTOR_FILE}\""))); + assert!(base.contains(&format!("\"name\": \"{FACE_PARSER_FILE}\""))); + } +} diff --git a/worker/src/models/mod.rs b/worker/src/models/mod.rs index 3300ac4d..b1928aaf 100644 --- a/worker/src/models/mod.rs +++ b/worker/src/models/mod.rs @@ -22,9 +22,12 @@ mod deblock_parameters; mod deband_parameters; mod descratch_parameters; mod descale_parameters; +mod face_restore_parameters; +mod colorize_parameters; mod spotless_parameters; mod stabilize_parameters; mod sharpen_parameters; +mod audio_restoration_parameters; mod processing_pipeline; mod dvd_info; @@ -49,8 +52,11 @@ pub use deblock_parameters::*; pub use deband_parameters::*; pub use descratch_parameters::*; pub use descale_parameters::*; +pub use face_restore_parameters::*; +pub use colorize_parameters::*; pub use stabilize_parameters::*; pub use spotless_parameters::*; pub use sharpen_parameters::*; +pub use audio_restoration_parameters::*; pub use processing_pipeline::*; pub use dvd_info::*; diff --git a/worker/src/models/noise_reduction_parameters.rs b/worker/src/models/noise_reduction_parameters.rs index 9e869475..e14f2b84 100644 --- a/worker/src/models/noise_reduction_parameters.rs +++ b/worker/src/models/noise_reduction_parameters.rs @@ -46,6 +46,11 @@ pub enum NoiseReductionMethod { /// works; refused with a reason where none does (see /// `DependencyLocator::knlm_device`). KnlMeans, + /// vszip's NLMeans: KNLMeansCL's algorithm, arguments and defaults on the + /// CPU, so it runs anywhere (app 1.7.0, deps 1.16.0). Shares the `knlm_*` + /// parameters with KnlMeans: the two take the same arguments and give the + /// same picture. + NlMeans, } /// Noise reduction preset levels. @@ -478,7 +483,7 @@ impl NoiseReductionParameters { let r = self.bm3d_effective_radius(); if self.bm3d_final { 2 * r } else { r } } - NoiseReductionMethod::KnlMeans => self.knlm_effective().1, + NoiseReductionMethod::KnlMeans | NoiseReductionMethod::NlMeans => self.knlm_effective().1, _ => 0, }; own.max(3) as u32 @@ -686,6 +691,7 @@ mod tests { (NoiseReductionMethod::Ctmf, "ctmf"), (NoiseReductionMethod::Bm3d, "bm3d"), (NoiseReductionMethod::KnlMeans, "knlMeans"), + (NoiseReductionMethod::NlMeans, "nlMeans"), ] { let json = name(method); assert!( diff --git a/worker/src/models/processing_pipeline.rs b/worker/src/models/processing_pipeline.rs index b10dd237..a4cecfb4 100644 --- a/worker/src/models/processing_pipeline.rs +++ b/worker/src/models/processing_pipeline.rs @@ -8,7 +8,7 @@ use super::{ AntiAliasParameters, GeometryParameters, GrainParameters, StabilizeParameters, ChromaDenoiseParameters, ChromaFixParameters, ColorCorrectionParameters, CropResizeParameters, DebandParameters, DeblockParameters, DehaloParameters, DeinterlaceMethod, DeScratchParameters, - SpotLessParameters, SharpenParameters, NoiseReductionParameters, QTGMCParameters, FrameRateParameters, FrameRateMethod, NoiseReductionMethod, DeflickerParameters, EdgeRepairParameters, GhostRemovalParameters, DescaleParameters,}; + SpotLessParameters, SharpenParameters, NoiseReductionParameters, QTGMCParameters, FrameRateParameters, FrameRateMethod, NoiseReductionMethod, DeflickerParameters, EdgeRepairParameters, GhostRemovalParameters, DescaleParameters, FaceRestoreParameters, ColorizeParameters, AudioRestorationParameters,}; /// Minimum temporal context (frames on each side of the target) a windowed /// preview decodes, regardless of which filters are enabled. Motion-compensated @@ -169,6 +169,10 @@ pub enum PassType { ColorCorrection, ChromaFixes, CropResize, + /// Restore faces with a neural network (an add-on). + FaceRestore, + /// Colour black-and-white footage with a neural network (an add-on). + Colorize, } #[allow(dead_code)] @@ -197,6 +201,8 @@ impl PassType { PassType::ColorCorrection => "Color Correction", PassType::ChromaFixes => "Chroma Fixes", PassType::CropResize => "Crop / Resize", + PassType::FaceRestore => "Face Restoration", + PassType::Colorize => "Colourisation", } } @@ -224,6 +230,8 @@ impl PassType { PassType::ColorCorrection => "Adjust brightness, contrast, and colors", PassType::ChromaFixes => "Fix chroma bleeding and crawl artifacts", PassType::CropResize => "Crop borders and resize output", + PassType::FaceRestore => "Restore detail in blurry, low-resolution faces with AI", + PassType::Colorize => "Colour black-and-white footage with AI", } } } @@ -313,6 +321,19 @@ pub struct ProcessingPipeline { /// Crop and resize pass parameters. #[serde(default)] pub crop_resize: CropResizeParameters, + + /// Face restoration (RestoreFormer++ / GFPGAN / CodeFormer). Off by default. + #[serde(default)] + pub face_restore: FaceRestoreParameters, + + /// Colourisation (DDColor / DeOldify Video). Off by default. + #[serde(default)] + pub colorize: ColorizeParameters, + /// Audio restoration. Not a VapourSynth pass — it is rendered by ffmpeg + /// before the encode (`audio_restoration.rs`) — so it is not in + /// `PassType` or `enabled_passes`, which describe the picture. + #[serde(default)] + pub audio: AudioRestorationParameters, } impl Default for ProcessingPipeline { @@ -339,6 +360,9 @@ impl Default for ProcessingPipeline { color_correction: ColorCorrectionParameters::default(), chroma_fixes: ChromaFixParameters::default(), crop_resize: CropResizeParameters::default(), + face_restore: FaceRestoreParameters::default(), + colorize: ColorizeParameters::default(), + audio: AudioRestorationParameters::default(), } } } @@ -369,6 +393,9 @@ impl ProcessingPipeline { color_correction: ColorCorrectionParameters { enabled: false, ..Default::default() }, chroma_fixes: ChromaFixParameters { enabled: false, ..Default::default() }, crop_resize: CropResizeParameters { enabled: false, ..Default::default() }, + face_restore: FaceRestoreParameters::default(), + colorize: ColorizeParameters::default(), + audio: AudioRestorationParameters::default(), } } @@ -377,8 +404,8 @@ impl ProcessingPipeline { let mut passes = Vec::new(); // Order: deinterlace, damage, noise, dehalo, deblock, deband, anti-alias, - // chroma, color, stabilize, rotate, crop + resize, then sharpen, grain, - // frame rate. + // chroma, colourisation, color, stabilize, rotate, crop + resize, face + // restoration, then sharpen, grain, frame rate. if self.deinterlace_enabled() { passes.push(PassType::Deinterlace); } @@ -436,6 +463,16 @@ impl ProcessingPipeline { if self.chroma_fixes.enabled { passes.push(PassType::ChromaFixes); } + // Colourisation follows the cleanup (a denoised, deblocked grey picture + // is what the model should judge, and its colour should not be + // denoised or deblocked again) and Chroma Fixes (which repair the + // source's chroma, which colourisation replaces), and precedes Colour + // Correction, so the result can be graded. Before the resize: the + // model sees a fixed-size copy either way, and the chroma is rebuilt + // at the source's size, so the resize then treats it as any chroma. + if self.colorize.enabled { + passes.push(PassType::Colorize); + } if self.color_correction.enabled { passes.push(PassType::ColorCorrection); } @@ -459,6 +496,15 @@ impl ProcessingPipeline { passes.push(PassType::CropResize); } + // Faces are restored at the delivered resolution, after the resize: + // an upscale gives the restorer more pixels per face, and a downscale + // after it would soften the detail it recovered. Before sharpening, + // which would otherwise sharpen the VHS face only to have it replaced, + // and before grain, which puts texture back over the smooth result. + if self.face_restore.enabled { + passes.push(PassType::FaceRestore); + } + // Sharpening follows the resize (issue #109), so it works on the // delivered pixels: sharpened earlier, a downscale softens the edges // again and an upscale enlarges the halos. It still follows @@ -504,6 +550,8 @@ impl ProcessingPipeline { if self.color_correction.enabled { count += 1; } if self.chroma_fixes.enabled { count += 1; } if self.crop_resize.enabled { count += 1; } + if self.face_restore.enabled { count += 1; } + if self.colorize.enabled { count += 1; } count } @@ -525,6 +573,45 @@ impl ProcessingPipeline { self.crop_resize.uses_animejanai() } + /// The manifest add-ons (`app/assets/addons/`, see `addons.rs`) each + /// enabled feature of this pipeline needs. `ScriptGenerator` refuses the + /// job, naming them and Settings → Add-ons, unless all are installed, and + /// emits the `{{#AI_RUNTIME}}` block when any of them needs the AI runtime. + /// + /// A pass that runs a model from an add-on adds an entry here when + /// enabled (a model add-on whose manifest `requires` `ai-runtime`). + /// + /// Picture passes only. Audio restoration's DeepFilterNet add-on is + /// checked by `audio_restoration` when the track is rendered: a preview + /// never touches the audio, so it must not be refused for it. + pub fn required_addons(&self) -> Vec { + let mut needs = Vec::new(); + if self.face_restore.enabled { + needs.push(crate::addons::AddonNeed { + feature: "Face restoration", + ids: self.face_restore.required_addons(), + }); + } + if self.colorize.enabled { + needs.push(crate::addons::AddonNeed { + feature: "Colourisation", + ids: self.colorize.required_addons(), + }); + } + // Crop & Resize's add-on upscalers (`upscale_addons.rs`). + if let Some((id, feature)) = self.crop_resize.upscale_addon() { + needs.push(crate::addons::AddonNeed { feature, ids: vec![id] }); + } + // RIFE's model (the plugin is in the deps bundle; see `gpu_plugin.rs`). + if self.uses_rife() { + needs.push(crate::addons::AddonNeed { + feature: "AI interpolation (RIFE)", + ids: vec![crate::gpu_plugin::RIFE_ADDON], + }); + } + needs + } + /// Whether this pipeline denoises with KNLMeans, which needs a working /// OpenCL device before its script can be generated. pub fn uses_knlm(&self) -> bool { @@ -561,6 +648,16 @@ impl ProcessingPipeline { PassType::EdgeRepair => FrameMap::Identity { radius: 0 }, PassType::GhostRemoval => FrameMap::Identity { radius: 0 }, PassType::Descale => FrameMap::Identity { radius: 0 }, + // mpeg2stinx limits each change by the neighbouring frames. + PassType::Deblock => FrameMap::Identity { + radius: if self.deblock.method == super::DeblockMethod::Mpeg2Stinx { 1 } else { 0 }, + }, + // Restored faces are blended with their neighbours', whose landmarks + // are smoothed over theirs: twice the smoothing radius. + PassType::FaceRestore => FrameMap::Identity { radius: self.face_restore.frames_read_either_side() }, + // Each frame's colour is averaged with the predictions r frames + // either side. + PassType::Colorize => FrameMap::Identity { radius: self.colorize.temporal_radius() }, // The one pass that emits a Retime. The ratio is known up front, so // the count is exact; `synthesizes` is true only for the // interpolating method, where an output frame has no single source. @@ -656,6 +753,29 @@ impl ProcessingPipeline { .fold(source_frames, |n, m| m.output_count(n)) } + /// How long the encoded picture is, in seconds, for `source_frames` at + /// `source_fps`: the output frame count over the output rate, both + /// composed from every pass's `FrameMap`. Double-rate deinterlacing and + /// IVTC change the count and the rate together, so the length stays the + /// source's; a frame-rate conversion's count is floored, so it can be up + /// to one output frame short. Audio restoration fits the sound to this. + pub fn output_duration_secs(&self, source_frames: u64, source_fps: f64) -> Option { + if !(source_fps > 0.0) { + return None; + } + let (mut num, mut den) = (1u64, 1u64); + for m in self.frame_maps() { + let (a, b) = m.rate_factor(); + num *= a; + den *= b; + let g = gcd(num, den); + num /= g; + den /= g; + } + let out_rate = source_fps * num as f64 / den as f64; + Some(self.output_count(source_frames) as f64 / out_rate) + } + /// Map a final output frame index back to the source frame span that /// produces it, composing every enabled pass's inverse in reverse order. pub fn invert(&self, output_frame: u64) -> SourceSpan { @@ -764,6 +884,8 @@ impl ProcessingPipeline { PassType::ColorCorrection => self.color_correction.enabled, PassType::ChromaFixes => self.chroma_fixes.enabled, PassType::CropResize => self.crop_resize.enabled, + PassType::FaceRestore => self.face_restore.enabled, + PassType::Colorize => self.colorize.enabled, } } } diff --git a/worker/src/models/qtgmc_parameters.rs b/worker/src/models/qtgmc_parameters.rs index b3ab1a85..2677b126 100644 --- a/worker/src/models/qtgmc_parameters.rs +++ b/worker/src/models/qtgmc_parameters.rs @@ -411,8 +411,22 @@ pub struct QTGMCParameters { /// VDecimate scene change threshold #[serde(skip_serializing_if = "Option::is_none")] pub ivtc_scthresh: Option, + + /// Fix fades added after telecine (vs-jetpack's FixInterlacedFades, run + /// right after field matching). Off by default: it only helps a fade that + /// combs, and touches every frame. + #[serde(default)] + pub ivtc_fix_fades: bool, + + /// How the two fields of a fading frame are reconciled: `average` (both + /// to their mean, upstream's AVERAGE) or `match` (the brighter to the + /// darker, MATCH). See [`Self::effective_ivtc_fade_mode`]. + #[serde(default = "default_ivtc_fade_mode")] + pub ivtc_fade_mode: String, } +fn default_ivtc_fade_mode() -> String { "average".to_string() } + // Default value functions fn default_true() -> bool { true } @@ -423,6 +437,16 @@ fn default_true() -> bool { true } const SUPPORTED_CHROMA_EDI: [&str; 2] = ["nnedi3", "bob"]; impl QTGMCParameters { + /// The fade fix's mode as the template spells it. Anything but `match` + /// is `average`, upstream's first and gentler mode: it keeps a fade's + /// overall level, where `match` darkens every frame to its darker field. + /// `match` is the one that also cleans the last frames of a fade that + /// reaches full black, where one field is already black and `average` + /// leaves the other at half. + pub fn effective_ivtc_fade_mode(&self) -> &'static str { + if self.ivtc_fade_mode.eq_ignore_ascii_case("match") { "match" } else { "average" } + } + /// Whether to upsample 4:2:0 chroma to 4:2:2 around the deinterlace pass. /// Defaults to disabled: it is the more correct way to handle interlaced /// 4:2:0 chroma, but it costs roughly 30% throughput, so it is offered as @@ -548,6 +572,8 @@ impl Default for QTGMCParameters { ivtc_cycle: None, ivtc_dupthresh: None, ivtc_scthresh: None, + ivtc_fix_fades: false, + ivtc_fade_mode: default_ivtc_fade_mode(), } } } diff --git a/worker/src/models/video_job.rs b/worker/src/models/video_job.rs index 93f00a18..1a427c9e 100644 --- a/worker/src/models/video_job.rs +++ b/worker/src/models/video_job.rs @@ -103,6 +103,54 @@ impl VideoJob { ) } + /// The frame size the pipeline delivers, as far as the worker can tell + /// before the script runs: the source, turned by Rotate / Flip, then + /// Crop & Resize (`CropResizeParameters::output_size`). `None` when the + /// source size is unknown. + pub fn estimated_output_size(&self) -> Option<(i32, i32)> { + let (w, h) = match (self.input_width, self.input_height) { + (Some(w), Some(h)) if w > 0 && h > 0 => (w, h), + _ => return None, + }; + let pipeline = self.effective_pipeline(); + let (w, h) = if pipeline.geometry.swaps_axes() { (h, w) } else { (w, h) }; + let sar = pipeline + .geometry + .adjusted_sar(self.input_sar.as_deref()) + .as_deref() + .and_then(super::parse_ratio) + .unwrap_or(1.0); + Some(pipeline.crop_resize.output_size(w, h, sar)) + } + + /// Whether the script converts the samples from BT.601 to BT.709 (Crop & + /// Resize's "Convert colour to BT.709", app 1.7.0). The ONE decision: the + /// script's `COLOR_709` block, the tags declared on both sides of the + /// encoder and the preview's display conversion all follow it, through + /// [`Self::output_color_metadata`]. + pub fn converts_to_bt709(&self) -> bool { + let Some(size) = self.estimated_output_size() else { + return false; + }; + let source_is_601 = self.color_metadata().is_bt601_family(self.input_height); + self.effective_pipeline().crop_resize.converts_to_bt709(source_is_601, size) + } + + /// The colour tags of the samples the script delivers: the source's, or + /// BT.709's after the conversion above. Everything that describes the + /// pipeline's OUTPUT — the encoder's input and output flags, the preview's + /// YUV-to-RGB step, and the passes that run after Crop & Resize (Face + /// Restoration, RIFE, the border fill) — must use this, never + /// [`Self::color_metadata`], or the conversion is undone as a relabel. + pub fn output_color_metadata(&self) -> ColorMetadata { + let source = self.color_metadata(); + if self.converts_to_bt709() { + source.bt709_converted(self.input_height) + } else { + source + } + } + /// The pixel format the encoder ffmpeg is handed on stdin. /// /// The output conversion is the last thing the `.vpy` does, so when one is diff --git a/worker/src/noise_halo.rs b/worker/src/noise_halo.rs new file mode 100644 index 00000000..b815b111 --- /dev/null +++ b/worker/src/noise_halo.rs @@ -0,0 +1,756 @@ +//! How noisy a source is, and how strongly its edges ring (app 1.7.0, +//! `--detect-noise-halo`). The app runs this in the background when a video +//! is loaded and badges Noise Reduction or Dehalo as recommended when a score +//! is clearly high; nothing here changes a setting. +//! +//! Frames are sampled across the source (or its trim) in pairs of consecutive +//! frames, decoded at their stored size and depth, and measured on luma in +//! plain Rust; no VapourSynth. VCG Deinterlacer v1's `analyze_noise_level` / +//! `analyze_halo_level` (MIT) were read for the idea; neither of their +//! measures separated our fixtures (ffmpeg signalstats TOUT does not respond +//! to added noise at all), so both measures here are our own, chosen and +//! calibrated in docs/reports/2026-11-noise-halo-scores/. +//! +//! **Noise** is the frame-to-frame difference after motion compensation. Each +//! 16x16 block is matched in the next frame (searched at half resolution +//! within +-10 px, refined to the pixel), the match chosen on box-smoothed +//! pictures so the choice barely depends on the noise being measured, and its +//! sigma is `std(block - match) / sqrt 2`. A low percentile over blocks keeps +//! the flattest, best-matched ones. Unlike a spatial high-pass this sees noise +//! at every spatial frequency, so tape noise (band-limited along the line) and +//! film grain (clumped) count in full, while texture that moved cancels. An +//! interlaced source is measured field by field, against the same field of the +//! next frame, so combing is never read as noise. Blocks near black or white +//! are left out (clipping hides noise there), which also leaves out letterbox +//! bars. A source whose frames repeat exactly (static digital video, cartoons +//! on twos, a codec's skipped blocks) reads low: the measure under-reports, +//! never over-reports. +//! +//! **Halo** is ringing beside isolated step edges. Edges are found along each +//! line (so interlacing cannot fake one) and followed three lines up and down +//! in the same field; the seven lines' profiles are averaged, and an edge is +//! kept when both sides settle to flat plateaus 6-10 px out. In units of the +//! step: overshoot past the bright plateau within 5 px, and undershoot below +//! the dark one. A halo rings on both sides; an outline (a dark line beside a +//! cartoon fill) is one-sided, so each edge scores the smaller of the two, and +//! the score is the mean over every edge in the sample. +//! +//! Both are in 8-bit units whatever the source's depth. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use crate::crop_detect; +use crate::dependency_locator::DependencyLocator; +use crate::models::VideoJob; +use crate::pixel_format; + +/// Noise sigma (8-bit levels) at and above which Noise Reduction is +/// recommended. Clean fixtures, including dense texture panned by a fraction +/// of a pixel each frame, measure at most 2.1; added white noise of sigma 4 +/// measures 3.7-3.8, tape-like noise of sigma 6 5.0-5.4. +pub const NOISE_HIGH: f64 = 3.5; + +/// Halo score (% of the step) at and above which Dehalo is recommended. Clean +/// fixtures measure at most 3.3 (a 1080p camera file); unsharp masks of +/// amount 1 on sharp-edged pictures measure 7-71. +pub const HALO_HIGH: f64 = 6.0; + +/// Fewer edges than this in the whole sample is too little to judge halos. +pub const HALO_MIN_EDGES: usize = 200; + +/// Pairs of frames sampled across the source; fewer above HD, where each +/// decode costs more. +pub const SAMPLE_PAIRS: usize = 24; +pub const SAMPLE_PAIRS_LARGE: usize = 12; +const LARGE_PIXELS: i64 = 1920 * 1088; + +/// The source's noise is this percentile of the pairs' estimates: noise is in +/// every shot, texture that defeats motion compensation (water, foliage in +/// wind) only in some. +const PAIR_PERCENTILE: f64 = 0.25; + +/// Motion compensation: block size, coarse search radius (at half +/// resolution, so twice this in pixels), the border kept clear for the +/// search, and the most blocks measured per field (a grid is thinned above +/// it, which bounds the cost on HD and 4K). +const BLOCK: usize = 16; +const COARSE: i32 = 5; +const MARGIN: usize = 2 * COARSE as usize + 1; +const MAX_BLOCKS: usize = 4096; + +/// A field's noise is this percentile of its blocks' estimates. +const BLOCK_PERCENTILE: f64 = 0.10; + +/// Blocks whose mean is within this many levels of black or white are left +/// out. +const CLIP_MARGIN: f64 = 4.0; + +/// Halo: plateaus `NEAR..=FAR` px from the edge, the near zone `1..NEAR`, and +/// lines followed this far up and down. +const NEAR: usize = 6; +const FAR: usize = 10; +const REACH: usize = 3; + +/// A plateau is flat when its standard deviation is within twice the noise +/// left after averaging the lines, 2 levels, or this fraction of the step, +/// whichever is most: a sharpened picture's plateaus carry sharpened texture. +const FLAT_REL: f64 = 0.03; + +/// One plane of luma in 8-bit units. +#[derive(Clone)] +pub struct Plane { + pub width: usize, + pub height: usize, + pub data: Vec, +} + +impl Plane { + /// The luma plane at the start of `bytes`: 8-bit, or little-endian 16-bit + /// samples holding `depth` bits, scaled to 8-bit units. + pub fn from_luma(bytes: &[u8], width: usize, height: usize, depth: u32) -> Option { + let n = width * height; + let data: Vec = if depth > 8 { + if bytes.len() < 2 * n { + return None; + } + let scale = (1u32 << (depth - 8)) as f32; + bytes[..2 * n] + .chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]]) as f32 / scale) + .collect() + } else { + if bytes.len() < n { + return None; + } + bytes[..n].iter().map(|&v| v as f32).collect() + }; + Some(Plane { width, height, data }) + } + + #[inline] + fn at(&self, x: usize, y: usize) -> f32 { + self.data[y * self.width + x] + } + + /// Every other line, from `parity`. + fn field(&self, parity: usize) -> Plane { + let lines: Vec = (parity..self.height).step_by(2).collect(); + let mut data = Vec::with_capacity(lines.len() * self.width); + for &y in &lines { + data.extend_from_slice(&self.data[y * self.width..(y + 1) * self.width]); + } + Plane { width: self.width, height: lines.len(), data } + } + + /// The fields of an interlaced frame, or the frame itself. + fn fields(&self, interlaced: bool) -> Vec { + if interlaced { + vec![self.field(0), self.field(1)] + } else { + vec![self.clone()] + } + } + + /// 3x3 box average, edges clamped. + fn box3(&self) -> Plane { + let (w, h) = (self.width, self.height); + let mut out = vec![0f32; w * h]; + for y in 0..h { + for x in 0..w { + let mut s = 0f32; + for dy in [-1i64, 0, 1] { + let yy = (y as i64 + dy).clamp(0, h as i64 - 1) as usize; + for dx in [-1i64, 0, 1] { + let xx = (x as i64 + dx).clamp(0, w as i64 - 1) as usize; + s += self.data[yy * w + xx]; + } + } + out[y * w + x] = s / 9.0; + } + } + Plane { width: w, height: h, data: out } + } + + /// Half size, each pixel the mean of a 2x2 block. + fn half(&self) -> Plane { + let (w, h) = (self.width / 2, self.height / 2); + let mut out = vec![0f32; w * h]; + for y in 0..h { + for x in 0..w { + let s = self.at(2 * x, 2 * y) + + self.at(2 * x + 1, 2 * y) + + self.at(2 * x, 2 * y + 1) + + self.at(2 * x + 1, 2 * y + 1); + out[y * w + x] = s / 4.0; + } + } + Plane { width: w, height: h, data: out } + } +} + +/// The value at `pct` of `values` (the lower of two neighbours, as NumPy's +/// `method='lower'`), or `None` for none. +fn percentile_lower(values: &mut [f64], pct: f64) -> Option { + if values.is_empty() { + return None; + } + values.sort_by(|a, b| a.total_cmp(b)); + let i = ((values.len() - 1) as f64 * pct).floor() as usize; + Some(values[i]) +} + +/// Sum of absolute differences between the `size`x`size` block of `a` at +/// (`ax`, `ay`) and that of `b` at (`bx`, `by`). +fn sad(a: &Plane, ax: usize, ay: usize, b: &Plane, bx: usize, by: usize, size: usize) -> f32 { + let mut s = 0f32; + for r in 0..size { + let ra = &a.data[(ay + r) * a.width + ax..(ay + r) * a.width + ax + size]; + let rb = &b.data[(by + r) * b.width + bx..(by + r) * b.width + bx + size]; + for (p, q) in ra.iter().zip(rb) { + s += (p - q).abs(); + } + } + s +} + +/// Motion-compensated noise sigma of field (or frame) `f` against the same +/// field of the next frame `g`, or `None` when too few blocks can be judged. +pub fn mc_sigma(f: &Plane, g: &Plane, black: f64, white: f64) -> Option { + let (w, h) = (f.width, f.height); + if g.width != w || g.height != h || w < 2 * MARGIN + 2 * BLOCK || h < 2 * MARGIN + 2 * BLOCK { + return None; + } + let by = (h - 2 * MARGIN) / BLOCK; + let bx = (w - 2 * MARGIN) / BLOCK; + if by < 2 || bx < 2 { + return None; + } + let mut stride = 1; + while (by / stride) * (bx / stride) > MAX_BLOCKS { + stride += 1; + } + let (sf, sg) = (f.box3(), g.box3()); + let (cf, cg) = (f.half().box3(), g.half().box3()); + let half = BLOCK / 2; + let mut sigmas = Vec::new(); + let mut total = 0usize; + for i in (0..by).step_by(stride) { + for j in (0..bx).step_by(stride) { + total += 1; + let (y, x) = (MARGIN + i * BLOCK, MARGIN + j * BLOCK); + // Coarse: the half-resolution block, +-COARSE. + let (cy, cx) = (y / 2, x / 2); + let mut best = f32::INFINITY; + let mut v = (0i32, 0i32); + for dy in -COARSE..=COARSE { + for dx in -COARSE..=COARSE { + let (ty, tx) = (cy as i32 + dy, cx as i32 + dx); + if ty < 0 || tx < 0 || ty as usize + half > cg.height || tx as usize + half > cg.width { + continue; + } + let s = sad(&cf, cx, cy, &cg, tx as usize, ty as usize, half); + if s < best { + best = s; + v = (dy, dx); + } + } + } + // Refine +-1 px at full resolution. + let mut best = f32::INFINITY; + let mut m = (0i32, 0i32); + for ry in -1..=1 { + for rx in -1..=1 { + let (dy, dx) = (2 * v.0 + ry, 2 * v.1 + rx); + let (ty, tx) = (y as i32 + dy, x as i32 + dx); + if ty < 0 || tx < 0 || ty as usize + BLOCK > h || tx as usize + BLOCK > w { + continue; + } + let s = sad(&sf, x, y, &sg, tx as usize, ty as usize, BLOCK); + if s < best { + best = s; + m = (dy, dx); + } + } + } + let mut mean = 0f64; + for r in 0..BLOCK { + for c in 0..BLOCK { + mean += f.at(x + c, y + r) as f64; + } + } + mean /= (BLOCK * BLOCK) as f64; + if !(mean > black + CLIP_MARGIN && mean < white - CLIP_MARGIN) { + continue; + } + let (ty, tx) = ((y as i32 + m.0) as usize, (x as i32 + m.1) as usize); + let (mut s1, mut s2) = (0f64, 0f64); + for r in 0..BLOCK { + for c in 0..BLOCK { + let d = (f.at(x + c, y + r) - g.at(tx + c, ty + r)) as f64; + s1 += d; + s2 += d * d; + } + } + let n = (BLOCK * BLOCK) as f64; + let var = (s2 / n - (s1 / n).powi(2)).max(0.0); + sigmas.push(var.sqrt() / std::f64::consts::SQRT_2); + } + } + if sigmas.len() < (total / 20).max(8) { + return None; + } + percentile_lower(&mut sigmas, BLOCK_PERCENTILE) +} + +/// For every isolated, vertically coherent step edge in `a`: the smaller of +/// its overshoot and undershoot, as a fraction of the step. `noise` (8-bit +/// levels) raises how strong and flat an edge must be to count. +pub fn edge_halos(a: &Plane, noise: f64, interlaced: bool) -> Vec { + let t_edge = (5.0 * noise).max(16.0) as f32; + let min_step = (8.0 * noise).max(24.0); + let flat = (2.0 * noise / ((2 * REACH + 1) as f64).sqrt()).max(2.0); + let mut out = Vec::new(); + for f in a.fields(interlaced) { + let (w, h) = (f.width, f.height); + if w < 2 * (FAR + REACH + 3) || h < 2 * REACH + 3 { + continue; + } + // The horizontal gradient a[x+1] - a[x-1]. + let mut g = vec![0f32; w * h]; + for y in 0..h { + for x in 1..w - 1 { + g[y * w + x] = f.at(x + 1, y) - f.at(x - 1, y); + } + } + let gi = |x: usize, y: usize| g[y * w + x]; + let sign = |v: f32| -> i8 { + if v > 0.0 { + 1 + } else if v < 0.0 { + -1 + } else { + 0 + } + }; + let lo_x = FAR + REACH + 2; + let hi_x = w - FAR - REACH - 2; + let mut prof = [0f64; 2 * FAR + 1]; + for y in REACH.max(1)..(h - REACH).min(h - 1) { + for x in lo_x..hi_x { + let v = gi(x, y).abs(); + if v < t_edge || v < gi(x - 1, y).abs() || v <= gi(x + 1, y).abs() { + continue; + } + let s = sign(gi(x, y)); + // Follow it up and down, a pixel of drift per line at most. + let mut ok = true; + let mut rows = [(y, x); 2 * REACH + 1]; + let mut n = 1; + for dir in [-1i64, 1] { + let mut px = x; + for k in 1..=REACH { + let yy = (y as i64 + dir * k as i64) as usize; + let mut best_v = f32::MIN; + let mut best_dx = 0i64; + for dx in [-1i64, 0, 1] { + let xx = (px as i64 + dx) as usize; + let gv = gi(xx, yy); + let cand = if sign(gv) == s { gv.abs() } else { 0.0 }; + if cand > best_v { + best_v = cand; + best_dx = dx; + } + } + if best_v < t_edge * 0.5 { + ok = false; + } + px = (px as i64 + best_dx) as usize; + rows[n] = (yy, px); + n += 1; + } + } + if !ok { + continue; + } + for (k, p) in prof.iter_mut().enumerate() { + let mut acc = 0f64; + for &(yy, xx) in &rows { + acc += f.at(xx + k - FAR, yy) as f64; + } + *p = acc / rows.len() as f64; + } + if s < 0 { + prof.reverse(); + } + let stats = |z: &[f64]| -> (f64, f64) { + let m = z.iter().sum::() / z.len() as f64; + let v = z.iter().map(|q| (q - m) * (q - m)).sum::() / z.len() as f64; + (m, v.sqrt()) + }; + let far_d = &prof[..=FAR - NEAR]; + let far_b = &prof[FAR + NEAR..]; + let (fd, sd) = stats(far_d); + let (fb, sb) = stats(far_b); + let step = fb - fd; + let tol = flat.max(FLAT_REL * step); + if sd > tol || sb > tol || step < min_step { + continue; + } + let max = |z: &[f64]| z.iter().cloned().fold(f64::MIN, f64::max); + let min = |z: &[f64]| z.iter().cloned().fold(f64::MAX, f64::min); + let near_b = &prof[FAR + 1..FAR + NEAR]; + let near_d = &prof[FAR + 1 - NEAR..FAR]; + let over = (max(near_b) - max(far_b)) / step; + let under = (min(far_d) - min(near_d)) / step; + out.push(over.min(under)); + } + } + } + out +} + +/// What one pair of frames gave. +#[derive(Debug, Clone, Default)] +pub struct PairResult { + pub noise: Option, + pub halos: Vec, +} + +/// Measure one pair of consecutive frames: noise from the two together, halos +/// in each. +pub fn measure_pair(a: &Plane, b: &Plane, interlaced: bool, full_range: bool) -> PairResult { + let (black, white) = if full_range { (0.0, 255.0) } else { (16.0, 235.0) }; + let fa = a.fields(interlaced); + let fb = b.fields(interlaced); + let sigmas: Vec> = fa.iter().zip(&fb).map(|(f, g)| mc_sigma(f, g, black, white)).collect(); + let noise = if sigmas.iter().all(|s| s.is_some()) { + Some(sigmas.iter().flatten().sum::() / sigmas.len() as f64) + } else { + None + }; + let mut halos = edge_halos(a, noise.unwrap_or(0.0), interlaced); + halos.extend(edge_halos(b, noise.unwrap_or(0.0), interlaced)); + PairResult { noise, halos } +} + +/// The source's scores from its pairs. +#[derive(Debug, Clone, PartialEq)] +pub struct Analysis { + pub noise: Option, + pub halo: Option, + pub halo_edges: usize, + pub usable_pairs: usize, +} + +pub fn combine(results: &[PairResult]) -> Analysis { + let mut sigmas: Vec = results.iter().filter_map(|r| r.noise).collect(); + let usable_pairs = sigmas.len(); + let noise = percentile_lower(&mut sigmas, PAIR_PERCENTILE); + let halo_edges: usize = results.iter().map(|r| r.halos.len()).sum(); + let halo = (halo_edges >= HALO_MIN_EDGES).then(|| { + results.iter().flat_map(|r| r.halos.iter()).sum::() / halo_edges as f64 * 100.0 + }); + Analysis { noise, halo, halo_edges, usable_pairs } +} + +/// What `--detect-noise-halo` prints. +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct NoiseHaloReport { + pub width: i32, + pub height: i32, + /// Whether each field was measured on its own. + pub interlaced: bool, + /// The first frame of each sampled pair, and how many pairs gave a noise + /// estimate. + pub frames: Vec, + pub usable_pairs: i32, + /// Noise sigma in 8-bit levels; `None` when it could not be measured. + pub noise: Option, + /// Halo score in % of the edge's step; `None` with too few edges. + pub halo: Option, + pub halo_edges: i32, + /// Whether each is high enough to recommend its pass. The thresholds are + /// decided here, once; the app only reads these. + pub noise_high: bool, + pub halo_high: bool, + pub noise_threshold: f64, + pub halo_threshold: f64, +} + +pub fn report_from(analysis: &Analysis, job: &VideoJob, frames: Vec, interlaced: bool) -> NoiseHaloReport { + NoiseHaloReport { + width: job.input_width.unwrap_or(0), + height: job.input_height.unwrap_or(0), + interlaced, + frames, + usable_pairs: analysis.usable_pairs as i32, + noise: analysis.noise, + halo: analysis.halo, + halo_edges: analysis.halo_edges as i32, + noise_high: analysis.noise.is_some_and(|n| n >= NOISE_HIGH), + halo_high: analysis.halo.is_some_and(|h| h >= HALO_HIGH), + noise_threshold: NOISE_HIGH, + halo_threshold: HALO_HIGH, + } +} + +/// Decode the sampled pairs and measure them. +pub fn detect(deps: &DependencyLocator, job: &VideoJob) -> Result { + let (width, height) = crop_detect::frame_size(job)?; + let count = if width as i64 * height as i64 > LARGE_PIXELS { SAMPLE_PAIRS_LARGE } else { SAMPLE_PAIRS }; + let frames = crop_detect::sample_frames_n( + count, + job.total_frames.map(|f| f.max(0) as u64), + job.start_frame.map(|f| f.max(0) as u64), + job.end_frame.map(|f| f.max(0) as u64), + ); + // The source's own format, so the luma plane comes through untouched (a + // conversion to gray would expand its range). + let pipe = pixel_format::decode_pixel_format(job.input_pixel_format.as_deref()); + let (_, depth) = pixel_format::chroma_and_depth(&pipe.name); + let (w, h) = (width as usize, height as usize); + let pairs = crop_detect::decode_samples(deps, job, &frames, 2, &pipe.name, |bytes| { + // Two frames of the same size back to back; one means the clip ended. + if bytes.len() % 2 != 0 { + return None; + } + let half = bytes.len() / 2; + let a = Plane::from_luma(&bytes[..half], w, h, depth)?; + let b = Plane::from_luma(&bytes[half..], w, h, depth)?; + Some((a, b)) + })?; + let interlaced = crop_detect::is_interlaced(job); + let full_range = job.color_metadata().is_full_range() || pipe.name.starts_with("yuvj"); + let results: Vec = std::thread::scope(|scope| { + let handles: Vec<_> = pairs + .iter() + .map(|(a, b)| scope.spawn(move || measure_pair(a, b, interlaced, full_range))) + .collect(); + handles.into_iter().map(|h| h.join().unwrap_or_default()).collect() + }); + Ok(report_from(&combine(&results), job, frames, interlaced)) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// A deterministic Gaussian source (Box-Muller over an LCG). + struct Rng(u64); + impl Rng { + fn uniform(&mut self) -> f64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407); + ((self.0 >> 11) as f64 + 0.5) / (1u64 << 53) as f64 + } + fn gauss(&mut self) -> f64 { + let (u, v) = (self.uniform(), self.uniform()); + (-2.0 * u.ln()).sqrt() * (std::f64::consts::TAU * v).cos() + } + } + + /// A textured picture at horizontal offset `shift`: smooth shading, + /// fine texture and a few hard-edged rectangles. + fn picture(w: usize, h: usize, shift: f64) -> Vec { + let mut p = vec![0f64; w * h]; + for y in 0..h { + for x in 0..w { + let xs = x as f64 + shift; + let mut v = 90.0 + 50.0 * (xs / 53.0).sin() * (y as f64 / 37.0).cos() + + 12.0 * (xs / 3.1).sin() * (y as f64 / 2.7).sin(); + // Rectangles with hard vertical edges. + let rx = (xs / 90.0).floor() as i64; + if (xs % 90.0) > 30.0 && (xs % 90.0) < 60.0 && (y / 40) % 2 == 0 { + v = 60.0 + 100.0 * ((rx % 2) as f64); + } + p[y * w + x] = v; + } + } + p + } + + fn plane(v: &[f64], w: usize, h: usize) -> Plane { + Plane { width: w, height: h, data: v.iter().map(|&x| x.round().clamp(0.0, 255.0) as f32).collect() } + } + + fn noisy(v: &[f64], sigma: f64, rng: &mut Rng) -> Vec { + v.iter().map(|&x| x + sigma * rng.gauss()).collect() + } + + /// Unsharp mask along the line: a + amount * (a - box5(a)). + fn sharpen(v: &[f64], w: usize, h: usize, amount: f64) -> Vec { + let mut out = v.to_vec(); + for y in 0..h { + for x in 2..w - 2 { + let b: f64 = (0..5).map(|k| v[y * w + x + k - 2]).sum::() / 5.0; + out[y * w + x] = v[y * w + x] + amount * (v[y * w + x] - b); + } + } + out + } + + fn pairs(n: usize, w: usize, h: usize, sigma: f64, amount: f64) -> Vec { + let mut rng = Rng(7); + (0..n) + .map(|t| { + let a = sharpen(&picture(w, h, 3.0 * t as f64), w, h, amount); + let b = sharpen(&picture(w, h, 3.0 * t as f64 + 3.0), w, h, amount); + measure_pair( + &plane(&noisy(&a, sigma, &mut rng), w, h), + &plane(&noisy(&b, sigma, &mut rng), w, h), + false, + false, + ) + }) + .collect() + } + + #[test] + fn noise_is_measured_close_to_what_was_added() { + for sigma in [0.0, 2.0, 5.0, 8.0] { + let a = combine(&pairs(4, 320, 240, sigma, 0.0)); + let n = a.noise.unwrap(); + assert!((n - sigma).abs() <= 0.15 * sigma + 0.6, "sigma {sigma}: measured {n}"); + } + } + + #[test] + fn texture_that_moves_is_not_noise_and_heavy_noise_is_high() { + let job = job_for(320, 240); + let clean = report_from(&combine(&pairs(4, 320, 240, 0.0, 0.0)), &job, vec![0], false); + assert!(!clean.noise_high); + let heavy = report_from(&combine(&pairs(4, 320, 240, 6.0, 0.0)), &job, vec![0], false); + assert!(heavy.noise_high, "{heavy:?}"); + } + + #[test] + fn interlaced_frames_are_measured_per_field() { + // Each frame's fields 4 px apart: combing on every edge. + let (w, h) = (320, 240); + let mut rng = Rng(3); + let weave = |t: f64, rng: &mut Rng, sigma: f64| { + let top = picture(w, h, t); + let bottom = picture(w, h, t + 4.0); + let mut v: Vec = (0..w * h).map(|i| if (i / w) % 2 == 0 { top[i] } else { bottom[i] }).collect(); + v = noisy(&v, sigma, rng); + plane(&v, w, h) + }; + for sigma in [0.0, 3.0] { + let r: Vec = (0..4) + .map(|t| { + let a = weave(8.0 * t as f64, &mut rng, sigma); + let b = weave(8.0 * t as f64 + 8.0, &mut rng, sigma); + measure_pair(&a, &b, true, false) + }) + .collect(); + let n = combine(&r).noise.unwrap(); + assert!((n - sigma).abs() <= 0.15 * sigma + 0.6, "interlaced sigma {sigma}: measured {n}"); + } + } + + #[test] + fn black_frames_cannot_be_measured() { + let (w, h) = (160, 120); + let black = Plane { width: w, height: h, data: vec![16.0; w * h] }; + let r = measure_pair(&black, &black, false, false); + assert!(r.noise.is_none()); + assert!(r.halos.is_empty()); + let a = combine(&[r]); + assert_eq!((a.noise, a.halo), (None, None)); + } + + #[test] + fn sharpening_reads_as_halo_and_a_clean_picture_does_not() { + let clean = combine(&pairs(4, 320, 240, 0.0, 0.0)); + let sharp = combine(&pairs(4, 320, 240, 0.0, 1.5)); + assert!(clean.halo_edges >= HALO_MIN_EDGES, "{clean:?}"); + assert!(clean.halo.unwrap() < 2.0, "{clean:?}"); + assert!(sharp.halo.unwrap() >= HALO_HIGH, "{sharp:?}"); + // Noise alone is not a halo. + let noisy = combine(&pairs(4, 320, 240, 5.0, 0.0)); + assert!(noisy.halo.map_or(true, |h| h < 2.0), "{noisy:?}"); + } + + #[test] + fn an_outline_is_not_a_halo() { + // Flat fills with a 3 px dark outline between them: the profile dips + // on one side only. + let (w, h) = (320, 120); + let mut v = vec![0f64; w * h]; + for y in 0..h { + for x in 0..w { + let cell = (x / 40) % 2; + let edge = x % 40; + v[y * w + x] = if edge < 3 { 30.0 } else if cell == 0 { 170.0 } else { 110.0 }; + } + } + let p = plane(&v, w, h); + let halos = edge_halos(&p, 0.0, false); + assert!(!halos.is_empty()); + let mean = halos.iter().sum::() / halos.len() as f64 * 100.0; + assert!(mean.abs() < 1.0, "outline read as halo: {mean}"); + } + + #[test] + fn ten_bit_luma_is_read_in_eight_bit_units() { + let raw: Vec = [64u16, 940, 512].iter().flat_map(|v| v.to_le_bytes()).collect(); + let p = Plane::from_luma(&raw, 3, 1, 10).unwrap(); + assert_eq!(p.data, vec![16.0, 235.0, 128.0]); + assert!(Plane::from_luma(&raw[..4], 3, 1, 10).is_none()); + let p8 = Plane::from_luma(&[16, 235, 128], 3, 1, 8).unwrap(); + assert_eq!(p8.data, vec![16.0, 235.0, 128.0]); + } + + #[test] + fn percentile_is_the_lower_neighbour() { + assert_eq!(percentile_lower(&mut [4.0, 1.0, 3.0, 2.0], 0.25), Some(1.0)); + assert_eq!(percentile_lower(&mut [5.0], 0.1), Some(5.0)); + assert_eq!(percentile_lower(&mut [], 0.5), None); + } + + fn job_for(w: i32, h: i32) -> VideoJob { + let mut j: VideoJob = serde_json::from_value(serde_json::json!({ + "id": "00000000-0000-0000-0000-000000000000", + "inputPath": "in.avi", + "outputPath": "out.mkv", + "qtgmcParameters": {}, + "encodingSettings": {}, + })) + .unwrap(); + j.input_width = Some(w); + j.input_height = Some(h); + j + } + + #[test] + fn report_says_high_only_at_the_thresholds() { + let job = job_for(720, 576); + let at = |noise, halo| { + report_from( + &Analysis { noise, halo, halo_edges: 500, usable_pairs: 10 }, + &job, + vec![1], + false, + ) + }; + let r = at(Some(NOISE_HIGH), Some(HALO_HIGH)); + assert!(r.noise_high && r.halo_high); + let r = at(Some(NOISE_HIGH - 0.01), Some(HALO_HIGH - 0.01)); + assert!(!r.noise_high && !r.halo_high); + let r = at(None, None); + assert!(!r.noise_high && !r.halo_high); + let v = serde_json::to_value(&r).unwrap(); + for key in ["noise", "halo", "noiseHigh", "haloHigh", "haloEdges", "usablePairs", "noiseThreshold"] { + assert!(v.get(key).is_some(), "{key}"); + } + } + + #[test] + fn too_few_edges_is_no_halo_score() { + let r = PairResult { noise: Some(1.0), halos: vec![0.5; HALO_MIN_EDGES - 1] }; + assert_eq!(combine(&[r]).halo, None); + let r = PairResult { noise: Some(1.0), halos: vec![0.5; HALO_MIN_EDGES] }; + assert_eq!(combine(&[r]).halo, Some(50.0)); + } +} diff --git a/worker/src/pipeline_executor.rs b/worker/src/pipeline_executor.rs index 82e9cf2d..42cd146c 100644 --- a/worker/src/pipeline_executor.rs +++ b/worker/src/pipeline_executor.rs @@ -162,11 +162,12 @@ fn preview_window(target: i32, radius: i32) -> (i32, i32, i32) { (first, local, num_frames) } +use crate::audio_restoration::{self, RestoredTrack}; use crate::comment_metadata::{self, CommentFile}; use crate::gpu_plugin::GpuPlugin; use crate::hdr_metadata::HdrMetadata; use crate::dependency_locator::DependencyLocator; -use crate::models::{AspectDeclaration, AudioMode, ContainerFormat, DeinterlaceMethod, EncoderFamily, LogLevel, ProgressInfo, SubtitleOutput, VideoCodec, VideoJob}; +use crate::models::{AspectDeclaration, ContainerFormat, DeinterlaceMethod, EncoderFamily, LogLevel, ProgressInfo, SubtitleOutput, VideoCodec, VideoJob}; use crate::pixel_format; use crate::source_decode; use crate::progress_reporter::ProgressReporter; @@ -184,6 +185,9 @@ pub struct PipelineExecutor { /// which the encode and preview decoders take their window. `None` for a /// constant-rate source. timeline: Option, + /// The track audio restoration rendered for this job, mapped in place of + /// the source's stream (see `audio_restoration`). `None` otherwise. + restored_audio: Option, } /// Watches for the pipeline making no progress at all. @@ -247,6 +251,7 @@ impl PipelineExecutor { vspipe_process: None, ffmpeg_process: None, timeline: None, + restored_audio: None, }) } @@ -256,6 +261,12 @@ impl PipelineExecutor { self } + /// Use a restored audio track (see `self.restored_audio`). + pub fn with_restored_audio(mut self, track: Option) -> Self { + self.restored_audio = track; + self + } + /// Execute the deinterlacing pipeline. pub fn execute(&mut self, script_path: &Path, job: &VideoJob, on_cancel: F) -> Result<()> where @@ -852,7 +863,7 @@ impl PipelineExecutor { // "unknown" as BT.601 and re-matrixes every pixel to the output's tag. // Declaring the same tags here leaves it nothing to convert. Must match // the output flags exactly — see ColorMetadata. - args.extend(job.color_metadata().to_ffmpeg_input_args()); + args.extend(job.output_color_metadata().to_ffmpeg_input_args()); args.extend(["-i".to_string(), "-".to_string()]); // Input 1: Original file for audio stream @@ -872,14 +883,18 @@ impl PipelineExecutor { // Input 2: the output's comment, as an FFMETADATA file (issue #138) args.extend(comment_file.input_args()); + // Input 3: the restored audio track, when audio restoration ran. + args.extend(audio_restoration::encode_input_args(self.restored_audio.as_ref())); + // Progress output to a temp file (avoids Windows pipe buffering delay) args.extend(["-progress".to_string(), progress_file.to_string_lossy().to_string()]); // Map streams: video from input 0 (processed), audio and subtitles from input 1 (original) args.extend(["-map".to_string(), "0:v".to_string()]); // Video from Y4M pipe - if settings.audio_mode != AudioMode::None { - args.extend(["-map".to_string(), "1:a?".to_string()]); // Audio from original (? = optional) - } + // Audio from the original (every stream, `1:a?`), or — when audio + // restoration rendered a track — that track (input 3) first, then the + // source's other audio streams. One decision site: audio_restoration. + args.extend(audio_restoration::encode_map_args(settings, self.restored_audio.as_ref(), 3)); // Preserve existing subtitle streams from source (? = optional, no error if none) let subtitle_embed_pending = job.subtitle_settings.as_ref() @@ -1010,24 +1025,11 @@ impl PipelineExecutor { // stops ffmpeg treating them as a conversion target. A Custom FFmpeg // Argument such as `-colorspace smpte170m` still wins (it comes later) // and is then a genuine conversion from the source's real matrix. - args.extend(job.color_metadata().to_ffmpeg_output_args()); + args.extend(job.output_color_metadata().to_ffmpeg_output_args()); - // Audio handling - match settings.audio_mode { - AudioMode::Passthrough => { - args.extend(["-c:a".to_string(), "copy".to_string()]); - } - AudioMode::Convert => { - args.extend(["-c:a".to_string(), settings.audio_codec.ffmpeg_name().to_string()]); - // Only add bitrate for lossy codecs - if !settings.audio_codec.is_lossless() { - args.extend(["-b:a".to_string(), format!("{}k", settings.audio_quality.bitrate())]); - } - } - AudioMode::None => { - args.push("-an".to_string()); - } - } + // Audio handling: copy / convert / none, and a restored track is + // always encoded (see audio_restoration::encode_codec_args). + args.extend(audio_restoration::encode_codec_args(settings, self.restored_audio.as_ref())); // Timestamp normalization for modes that change frame rate. // Fixes audio sync offset caused by mismatched initial PTS between the @@ -1339,6 +1341,7 @@ impl PipelineExecutor { Err("KNLMeans is not used by this job.".into()) }) .with_animejanai(self.deps.animejanai_runtime()) + .with_addons(self.deps.addons_dir(), self.deps.platform_suffix()) .with_timeline(window.clone()); let (fps_num, fps_den) = script_generator.frame_rate_to_rational(frame_rate); @@ -1438,7 +1441,11 @@ impl PipelineExecutor { // // An HDR source is tone-mapped for display instead (issue #133), with // the same chain the app uses for its "before" frame. - let color = job.color_metadata(); + // + // The OUTPUT's tags, not the source's: when Crop & Resize converts a + // BT.601 source to BT.709, the pipe carries 709 samples, and reading + // them as 601 would show a colour shift the encode does not have. + let color = job.output_color_metadata(); let scale_filter = color.display_tonemap_filter().unwrap_or_else(|| { let mut scale_opts = color.swscale_input_opts(); scale_opts.push("out_range=pc".to_string()); @@ -1558,7 +1565,7 @@ impl Drop for PipelineExecutor { #[cfg(test)] mod tests { use super::*; - use crate::models::{AudioCodec, AudioQuality, ChromaSubsampling, EncodingSettings, ProResQuantMat, QTGMCParameters, VideoCodec}; + use crate::models::{AudioCodec, AudioMode, AudioQuality, ChromaSubsampling, EncodingSettings, ProResQuantMat, QTGMCParameters, VideoCodec}; use uuid::Uuid; /// The two codes that have actually cost debugging time must be named in @@ -1856,6 +1863,26 @@ mod tests { ); } + /// The real builder takes every audio decision from audio_restoration, + /// with the restored track as input 3 (after the source and the metadata + /// file); the copy below cannot show that, having neither. + #[test] + fn test_real_ffmpeg_builder_delegates_audio_to_audio_restoration() { + let src = include_str!("pipeline_executor.rs").replace("\r\n", "\n"); + let start = src.find("fn build_ffmpeg_args(\n").expect("real builder"); + let body = &src[start..]; + let body = &body[..body.find("\n }\n").expect("end of builder")]; + let file = body.find("comment_file.input_args()").expect("metadata file input"); + let track = body + .find("audio_restoration::encode_input_args(self.restored_audio.as_ref())") + .expect("restored track input"); + assert!(file < track, "the restored track must be input 3, after the metadata file"); + assert!(body.contains("audio_restoration::encode_map_args(settings, self.restored_audio.as_ref(), 3)")); + assert!(body.contains("audio_restoration::encode_codec_args(settings, self.restored_audio.as_ref())")); + assert!(!body.contains("\"1:a?\""), "audio mapping belongs to audio_restoration"); + assert_eq!(body.matches("\"-af\"").count(), 0, "the encode never filters audio itself"); + } + /// Helper to build FFmpeg args without requiring a full PipelineExecutor. /// This mirrors the logic in PipelineExecutor::build_ffmpeg_args for testing. fn build_ffmpeg_args_for_test(job: &VideoJob) -> Vec { @@ -1865,7 +1892,7 @@ mod tests { // Input 0: Processed video from vspipe (Y4M pipe), with the source's // colour tags declared on it (mirrors build_ffmpeg_args). args.extend(["-f".to_string(), "yuv4mpegpipe".to_string()]); - args.extend(job.color_metadata().to_ffmpeg_input_args()); + args.extend(job.output_color_metadata().to_ffmpeg_input_args()); args.extend(["-i".to_string(), "-".to_string()]); // Input 1: Original file for audio stream @@ -1876,9 +1903,7 @@ mod tests { // Map streams: video from input 0 (processed), audio from input 1 (original) if not disabled args.extend(["-map".to_string(), "0:v".to_string()]); - if settings.audio_mode != AudioMode::None { - args.extend(["-map".to_string(), "1:a?".to_string()]); - } + args.extend(audio_restoration::encode_map_args(settings, None, 2)); // Video codec args.extend(["-c:v".to_string(), settings.codec.ffmpeg_codec().to_string()]); @@ -1910,24 +1935,10 @@ mod tests { // Colour tags. This helper duplicates build_ffmpeg_args rather than // calling it, so anything added there has to be added here too — the // SAR block was missed that way and is still absent below. - args.extend(job.color_metadata().to_ffmpeg_output_args()); + args.extend(job.output_color_metadata().to_ffmpeg_output_args()); - // Audio handling - match settings.audio_mode { - AudioMode::Passthrough => { - args.extend(["-c:a".to_string(), "copy".to_string()]); - } - AudioMode::Convert => { - args.extend(["-c:a".to_string(), settings.audio_codec.ffmpeg_name().to_string()]); - // Only add bitrate for lossy codecs - if !settings.audio_codec.is_lossless() { - args.extend(["-b:a".to_string(), format!("{}k", settings.audio_quality.bitrate())]); - } - } - AudioMode::None => { - args.push("-an".to_string()); - } - } + // Audio handling (mirrors build_ffmpeg_args; no restored track here). + args.extend(audio_restoration::encode_codec_args(settings, None)); // Custom arguments if !settings.custom_ffmpeg_args.is_empty() { @@ -2032,7 +2043,7 @@ mod tests { .flat_map(|w| [w[0].clone(), w[1].clone()]) .collect() }; - let expected = job.color_metadata().to_ffmpeg_output_args(); + let expected = job.output_color_metadata().to_ffmpeg_output_args(); assert_eq!(input_side, expected.as_slice(), "input declaration"); assert_eq!(output_side, expected, "output tags"); } @@ -2064,6 +2075,12 @@ mod tests { body.contains("to_ffmpeg_output_args()"), "build_ffmpeg_args must still tag the output" ); + // Both sides describe what the SCRIPT delivers, which is not the + // source's colour when it converts to BT.709. + assert!( + !body.contains("job.color_metadata().to_ffmpeg"), + "the encoder must be told the output's colour, not the source's" + ); } /// An untagged source must stay untagged rather than being guessed at. @@ -2076,6 +2093,51 @@ mod tests { } } + /// An SD BT.601 source resized to 1080 lines is converted to BT.709 by the + /// script (app 1.7.0), so both sides of the encoder must say 709 — the + /// source's 601 tags there would relabel the converted samples back. + #[test] + fn test_sd_to_hd_declares_the_converted_bt709_on_both_sides() { + let mut job = create_test_job("out.mkv"); + job.input_width = Some(720); + job.input_height = Some(576); + job.input_color_matrix = Some("bt470bg".to_string()); + job.input_color_primaries = Some("bt470bg".to_string()); + job.input_color_range = Some("tv".to_string()); + let mut pipeline = crate::models::ProcessingPipeline::default(); + pipeline.deinterlace.enabled = false; + pipeline.crop_resize.enabled = true; + pipeline.crop_resize.resize_enabled = true; + pipeline.crop_resize.target_width = Some(1920); + pipeline.crop_resize.target_height = Some(1080); + job.processing_pipeline = Some(pipeline.clone()); + let args = build_ffmpeg_args_for_test(&job); + let pairs: Vec<(String, String)> = args + .windows(2) + .filter(|w| w[0] == "-colorspace" || w[0] == "-color_primaries" || w[0] == "-color_trc") + .map(|w| (w[0].clone(), w[1].clone())) + .collect(); + let expected = |flag: &str, v: &str| (flag.to_string(), v.to_string()); + assert_eq!( + pairs, + vec![ + expected("-colorspace", "bt709"), + expected("-color_primaries", "bt470bg"), + expected("-color_trc", "bt709"), + expected("-colorspace", "bt709"), + expected("-color_primaries", "bt470bg"), + expected("-color_trc", "bt709"), + ] + ); + + // Off: the source's tags, exactly as before the option existed. + pipeline.crop_resize.convert_to_bt709 = crate::models::Bt709Conversion::Off; + job.processing_pipeline = Some(pipeline); + let args = build_ffmpeg_args_for_test(&job); + assert_eq!(args.iter().filter(|a| *a == "bt470bg").count(), 4); + assert!(!args.iter().any(|a| a == "bt709")); + } + /// ffprobe says "unknown" for an untagged stream; that must not be /// forwarded as if it were a value, or the encode fails on an argument the /// user can neither see nor fix. diff --git a/worker/src/progress_reporter.rs b/worker/src/progress_reporter.rs index b550ff29..1ac78939 100644 --- a/worker/src/progress_reporter.rs +++ b/worker/src/progress_reporter.rs @@ -13,6 +13,8 @@ pub struct ProgressReporter { struct ProgressReporterInner { output_lock: Mutex<()>, + /// Write to stderr instead: for a mode whose stdout is one JSON answer. + to_stderr: bool, } impl ProgressReporter { @@ -21,6 +23,18 @@ impl ProgressReporter { Self { inner: std::sync::Arc::new(ProgressReporterInner { output_lock: Mutex::new(()), + to_stderr: false, + }), + } + } + + /// A reporter that writes its messages to stderr, leaving stdout to a + /// mode that prints one JSON answer there (`--audio-plan`). + pub fn new_to_stderr() -> Self { + Self { + inner: std::sync::Arc::new(ProgressReporterInner { + output_lock: Mutex::new(()), + to_stderr: true, }), } } @@ -60,6 +74,9 @@ impl ProgressReporter { let _lock = self.inner.output_lock.lock().unwrap(); match serde_json::to_string(message) { + Ok(json) if self.inner.to_stderr => { + eprintln!("{}", json); + } Ok(json) => { let stdout = io::stdout(); let mut handle = stdout.lock(); diff --git a/worker/src/script_generator.rs b/worker/src/script_generator.rs index c8276e74..c86fa340 100644 --- a/worker/src/script_generator.rs +++ b/worker/src/script_generator.rs @@ -83,6 +83,10 @@ pub struct ScriptGenerator { /// addition to `upscale`. Defaults to "not checked"; callers set it from /// `DependencyLocator::animejanai_runtime()`. animejanai: std::result::Result, + /// The add-ons directory and this platform's id, for the manifest add-ons + /// a job's passes need (`ProcessingPipeline::required_addons`, see + /// `addons.rs`). `None` ("not checked") refuses any job that needs one. + addons: Option<(PathBuf, String)>, /// For a variable-frame-rate source, where the decoded window's stored /// frames fall on the grid (see `source_timing`). `None` for a /// constant-rate source, whose script is unchanged. @@ -125,10 +129,19 @@ impl ScriptGenerator { upscale: Err("AI upscaling was not checked for this job.".to_string()), knlm: Err("KNLMeansCL was not checked for this job.".to_string()), animejanai: Err("AnimeJaNai was not checked for this job.".to_string()), + addons: None, timeline: None, }) } + /// Set the add-ons directory and platform id the job's add-ons are + /// checked against and loaded from (`DependencyLocator::addons_dir`, + /// `platform_suffix`). + pub fn with_addons(mut self, addons_dir: PathBuf, platform: &str) -> Self { + self.addons = Some((addons_dir, platform.to_string())); + self + } + /// Set whether OpenCL is available (probe result from `DependencyLocator`). /// When false, the GPU NNEDI3CL path is suppressed and QTGMC uses CPU NNEDI3. pub fn with_opencl_available(mut self, available: bool) -> Self { @@ -212,9 +225,86 @@ impl ScriptGenerator { anyhow::bail!("{message}"); } } + for need in pipeline.required_addons() { + match &self.addons { + Some((dir, platform)) => crate::addons::require(dir, &need.ids, need.feature, platform)?, + None => anyhow::bail!("{} needs add-ons, which were not checked for this job.", need.feature), + } + } Ok(()) } + /// Emit or drop the `{{#AI_RUNTIME}}` block: emitted, with the add-on's + /// directory, when a pass needs an add-on that requires the AI runtime. + fn substitute_addons(&self, script: String, pipeline: &ProcessingPipeline) -> String { + let ids: Vec<&str> = pipeline.required_addons().iter().flat_map(|n| n.ids.clone()).collect(); + let dir = match &self.addons { + Some((root, _)) if crate::addons::needs_ai_runtime(&ids) => { + Some(crate::addons::addon_dir(root, crate::addons::AI_RUNTIME)) + } + _ => None, + }; + ai_runtime_block(script, dir.as_deref()) + } + + /// Emit or drop the `{{#FACE_RESTORE}}` block (both templates), with the + /// model paths from the add-ons directory. A job that gets here with the + /// pass on has passed `ensure_methods_ready`, so its add-ons are installed. + fn substitute_face_restore(&self, script: String, job: &VideoJob, pipeline: &ProcessingPipeline) -> String { + use crate::models::{FACE_DETECTOR_FILE, FACE_PARSER_FILE, FACE_RESTORATION_ADDON}; + let fr = &pipeline.face_restore; + if !fr.enabled { + return remove_block("{{#FACE_RESTORE}}", "{{/FACE_RESTORE}}", script); + } + let path = |id: &str, file: &str| match &self.addons { + Some((root, _)) => py_str(&crate::addons::addon_dir(root, id).join(file)), + None => "''".to_string(), + }; + // After Crop & Resize, so the samples are in the output's colour. + let color = job.output_color_metadata(); + script + .replace("{{#FACE_RESTORE}}", "") + .replace("{{/FACE_RESTORE}}", "") + .replace("{{FR_DETECTOR}}", &path(FACE_RESTORATION_ADDON, FACE_DETECTOR_FILE)) + .replace("{{FR_PARSER}}", &path(FACE_RESTORATION_ADDON, FACE_PARSER_FILE)) + .replace("{{FR_RESTORER}}", &path(fr.model.addon(), fr.model.file())) + .replace("{{FR_MODEL}}", fr.model.script_name()) + .replace("{{FR_STRENGTH}}", &format!("{:.3}", fr.clamped_strength())) + .replace("{{FR_FIDELITY}}", &format!("{:.3}", fr.clamped_fidelity())) + .replace("{{FR_THRESHOLD}}", &format!("{:.3}", fr.clamped_threshold())) + .replace("{{FR_MIN_FACE}}", &fr.clamped_min_face().to_string()) + .replace("{{FR_MAX_FACES}}", &fr.clamped_max_faces().to_string()) + .replace("{{FR_RADIUS}}", &fr.temporal_radius().to_string()) + .replace("{{FR_MATRIX}}", color.zimg_matrix(job.input_height)) + .replace("{{FR_RANGE}}", if color.is_full_range() { "full" } else { "limited" }) + } + + /// Emit or drop the `{{#COLORIZE}}` block (both templates), with the + /// chosen model's path from the add-ons directory. A job that gets here + /// with the pass on has passed `ensure_methods_ready`, so its add-on is + /// installed. + fn substitute_colorize(&self, script: String, job: &VideoJob, pipeline: &ProcessingPipeline) -> String { + let cz = &pipeline.colorize; + if !cz.enabled { + return remove_block("{{#COLORIZE}}", "{{/COLORIZE}}", script); + } + let model_path = match &self.addons { + Some((root, _)) => py_str(&crate::addons::addon_dir(root, cz.model.addon()).join(cz.model.file())), + None => "''".to_string(), + }; + let color = job.color_metadata(); + script + .replace("{{#COLORIZE}}", "") + .replace("{{/COLORIZE}}", "") + .replace("{{CZ_MODEL_PATH}}", &model_path) + .replace("{{CZ_MODEL}}", cz.model.script_name()) + .replace("{{CZ_RENDER_SIZE}}", &cz.render_size().to_string()) + .replace("{{CZ_STRENGTH}}", &format!("{:.3}", cz.clamped_strength())) + .replace("{{CZ_RADIUS}}", &cz.temporal_radius().to_string()) + .replace("{{CZ_MATRIX}}", color.zimg_matrix(job.input_height)) + .replace("{{CZ_RANGE}}", if color.is_full_range() { "full" } else { "limited" }) + } + /// Generate a .vpy script file for the given job. pub fn generate(&self, job: &VideoJob) -> Result { let pipeline = job.effective_pipeline(); @@ -466,7 +556,7 @@ impl ScriptGenerator { /// Substitute pipeline parameters on an already-prepared script. fn substitute_parameters_on(&self, script: &str, job: &VideoJob, pipeline: &ProcessingPipeline) -> String { - let mut script = script.to_string(); + let mut script = self.substitute_addons(script.to_string(), pipeline); let params = &pipeline.deinterlace; // ==================================================================== @@ -770,6 +860,15 @@ impl ScriptGenerator { }; script = script.replace("{{IVTC_ORDER}}", &order.to_string()); + // Fades added after telecine (vs-jetpack's FixInterlacedFades). + if params.ivtc_fix_fades { + script = script.replace("{{#DEINT_IVTC_FIX_FADES}}", ""); + script = script.replace("{{/DEINT_IVTC_FIX_FADES}}", ""); + script = script.replace("{{IVTC_FADE_MODE}}", params.effective_ivtc_fade_mode()); + } else { + script = remove_block("{{#DEINT_IVTC_FIX_FADES}}", "{{/DEINT_IVTC_FIX_FADES}}", script); + } + // VFM parameters script = process_optional_int("IVTC_MODE", params.ivtc_mode, script); script = process_optional_int("IVTC_CTHRESH", params.ivtc_cthresh, script); @@ -897,17 +996,20 @@ impl ScriptGenerator { script = script.replace("{{#NOISE_REDUCTION}}", ""); script = script.replace("{{/NOISE_REDUCTION}}", ""); - // BM3D and KNLMeans are removed here rather than in every other - // method's arm below. + // BM3D, KNLMeans and NLMeans are removed here rather than in every + // other method's arm below. if nr.method != NoiseReductionMethod::Bm3d { script = remove_block("{{#NR_BM3D}}", "{{/NR_BM3D}}", script); } if nr.method != NoiseReductionMethod::KnlMeans { script = remove_block("{{#NR_KNLM}}", "{{/NR_KNLM}}", script); } + if nr.method != NoiseReductionMethod::NlMeans { + script = remove_block("{{#NR_NLM}}", "{{/NR_NLM}}", script); + } match nr.method { - NoiseReductionMethod::Bm3d | NoiseReductionMethod::KnlMeans => { + NoiseReductionMethod::Bm3d | NoiseReductionMethod::KnlMeans | NoiseReductionMethod::NlMeans => { for other in [ "NR_SMDEGRAIN", "NR_MCTD", "NR_MCDEGRAINSHARP", "NR_CTMF", "NR_FLUXSMOOTH_T", "NR_FLUXSMOOTH_ST", "NR_STPRESSO", "NR_MCLEAN", @@ -924,6 +1026,17 @@ impl ScriptGenerator { script = script.replace("{{NR_BM3D_RADIUS}}", &nr.bm3d_effective_radius().to_string()); script = script.replace("{{NR_BM3D_FINAL}}", if nr.bm3d_final { "True" } else { "False" }); script = script.replace("{{NR_BM3D_PROFILE}}", nr.bm3d_effective_profile()); + } else if nr.method == NoiseReductionMethod::NlMeans { + // The same arguments as KNLMeansCL (vszip's port takes + // its argument names and defaults), on the CPU. + let (h, d, a, s) = nr.knlm_effective(); + script = script.replace("{{#NR_NLM}}", ""); + script = script.replace("{{/NR_NLM}}", ""); + script = script.replace("{{NR_NLM_H}}", &format!("{h:.2}")); + script = script.replace("{{NR_NLM_D}}", &d.to_string()); + script = script.replace("{{NR_NLM_A}}", &a.to_string()); + script = script.replace("{{NR_NLM_S}}", &s.to_string()); + script = script.replace("{{NR_NLM_CHROMA}}", if nr.knlm_chroma { "True" } else { "False" }); } else { let (h, d, a, s) = nr.knlm_effective(); // ensure_methods_ready has already refused a job without one. @@ -1396,7 +1509,7 @@ impl ScriptGenerator { // every block per arm is what the other passes do, but with seven // methods it stops being readable — so keep the block the method // selects and drop the others by difference. - const DEHALO_BLOCKS: [&str; 7] = [ + const DEHALO_BLOCKS: [&str; 8] = [ "DEHALO_DEHALO_ALPHA", "DEHALO_FINE_DEHALO", "DEHALO_FINE_DEHALO2", @@ -1404,6 +1517,7 @@ impl ScriptGenerator { "DEHALO_EDGE_CLEANER", "DEHALO_VINVERSE", "DEHALO_HQDERING", + "DEHALO_VINE", ]; let selected = match dehalo.method { @@ -1416,6 +1530,7 @@ impl ScriptGenerator { // name differs. DehaloMethod::Vinverse | DehaloMethod::Vinverse2 => "DEHALO_VINVERSE", DehaloMethod::HqDeringmod => "DEHALO_HQDERING", + DehaloMethod::Vine => "DEHALO_VINE", }; for block in DEHALO_BLOCKS { @@ -1469,6 +1584,12 @@ impl ScriptGenerator { script = process_optional_double("DEHALO_DERING_THR", dehalo.dering_thr, script); script = process_optional_double("DEHALO_DERING_DARKTHR", dehalo.dering_darkthr, script); } + DehaloMethod::Vine => { + let (strength, sharp, sigma) = dehalo.vine_effective(); + script = script.replace("{{DEHALO_VINE_STRENGTH}}", &format!("{strength:.3}")); + script = script.replace("{{DEHALO_VINE_SHARP}}", &format!("{sharp:.3}")); + script = script.replace("{{DEHALO_VINE_SIGMA}}", &format!("{sigma:.3}")); + } } // Radius and strength are shared by DeHalo_alpha and FineDehalo only. @@ -1492,8 +1613,14 @@ impl ScriptGenerator { // Still interlaced: deblock each field on its own, never the woven // frame (docs/reports/2026-11-deblock-order). Deinterlaced output - // is progressive and keeps the whole-frame path. - match Self::still_interlaced_tff(job, pipeline) { + // is progressive and keeps the whole-frame path. mpeg2stinx is a + // cross-field filter by design and always takes the whole frame. + let per_field = if deblock.method == DeblockMethod::Mpeg2Stinx { + None + } else { + Self::still_interlaced_tff(job, pipeline) + }; + match per_field { Some(tff) => { script = script.replace("{{#DEBLOCK_PER_FIELD}}", ""); script = script.replace("{{/DEBLOCK_PER_FIELD}}", ""); @@ -1505,7 +1632,20 @@ impl ScriptGenerator { } match deblock.method { + DeblockMethod::Mpeg2Stinx => { + for other in ["DEBLOCK_QED", "DEBLOCK_SIMPLE", "DEBLOCK_DCTFILTER"] { + script = remove_block(&format!("{{{{#{other}}}}}"), &format!("{{{{/{other}}}}}"), script); + } + script = script.replace("{{#DEBLOCK_MPEG2STINX}}", ""); + script = script.replace("{{/DEBLOCK_MPEG2STINX}}", ""); + // The field order the frames were ENCODED with: the + // deinterlace pass's (IVTC) when it runs, else what was + // detected, else top field first (DVD's usual order). + let tff = Self::field_based_for(job, pipeline).map_or(true, |fb| fb == 2); + script = script.replace("{{DEBLOCK_STINX_TFF}}", if tff { "True" } else { "False" }); + } DeblockMethod::DeblockQed => { + script = remove_block("{{#DEBLOCK_MPEG2STINX}}", "{{/DEBLOCK_MPEG2STINX}}", script); script = script.replace("{{#DEBLOCK_QED}}", ""); script = script.replace("{{/DEBLOCK_QED}}", ""); script = remove_block("{{#DEBLOCK_SIMPLE}}", "{{/DEBLOCK_SIMPLE}}", script); @@ -1517,6 +1657,7 @@ impl ScriptGenerator { script = process_optional_int("DEBLOCK_AOFFSET2", Some(deblock.a_offset2), script); } DeblockMethod::Deblock => { + script = remove_block("{{#DEBLOCK_MPEG2STINX}}", "{{/DEBLOCK_MPEG2STINX}}", script); script = remove_block("{{#DEBLOCK_QED}}", "{{/DEBLOCK_QED}}", script); script = script.replace("{{#DEBLOCK_SIMPLE}}", ""); script = script.replace("{{/DEBLOCK_SIMPLE}}", ""); @@ -1525,6 +1666,7 @@ impl ScriptGenerator { script = process_optional_int("DEBLOCK_QUANT1", Some(deblock.quant1), script); } DeblockMethod::DctFilter => { + script = remove_block("{{#DEBLOCK_MPEG2STINX}}", "{{/DEBLOCK_MPEG2STINX}}", script); script = remove_block("{{#DEBLOCK_QED}}", "{{/DEBLOCK_QED}}", script); script = remove_block("{{#DEBLOCK_SIMPLE}}", "{{/DEBLOCK_SIMPLE}}", script); script = script.replace("{{#DEBLOCK_DCTFILTER}}", ""); @@ -1596,9 +1738,18 @@ impl ScriptGenerator { script = script.replace("{{#AA_DAA}}", ""); script = script.replace("{{/AA_DAA}}", ""); script = remove_block("{{#AA_SANTIAG}}", "{{/AA_SANTIAG}}", script); + script = remove_block("{{#AA_SANGNOM}}", "{{/AA_SANGNOM}}", script); + } + AntiAliasMethod::SangNom => { + script = remove_block("{{#AA_DAA}}", "{{/AA_DAA}}", script); + script = remove_block("{{#AA_SANTIAG}}", "{{/AA_SANTIAG}}", script); + script = script.replace("{{#AA_SANGNOM}}", ""); + script = script.replace("{{/AA_SANGNOM}}", ""); + script = script.replace("{{AA_SANGNOM_AA}}", &anti_alias.effective_sangnom_aa().to_string()); } AntiAliasMethod::Santiag => { script = remove_block("{{#AA_DAA}}", "{{/AA_DAA}}", script); + script = remove_block("{{#AA_SANGNOM}}", "{{/AA_SANGNOM}}", script); script = script.replace("{{#AA_SANTIAG}}", ""); script = script.replace("{{/AA_SANTIAG}}", ""); @@ -1715,6 +1866,11 @@ impl ScriptGenerator { script = remove_block("{{#GEOMETRY}}", "{{/GEOMETRY}}", script); } + // ==================================================================== + // FACE RESTORATION PASS + // ==================================================================== + script = self.substitute_face_restore(script, job, pipeline); + // ==================================================================== // SHARPEN PASS // ==================================================================== @@ -1931,19 +2087,24 @@ impl ScriptGenerator { script = script.replace("{{#FRAME_RATE_RIFE}}", ""); script = script.replace("{{/FRAME_RATE_RIFE}}", ""); // generate()/generate_preview() refuse a RIFE job without - // files, so the fallbacks here are never reached by a job. - let (plugin, model, preload) = match &self.rife { + // the plugin or the model add-on, so the fallbacks here + // are never reached by a job. + let (plugin, preload) = match &self.rife { Ok(f) => ( py_str(&f.plugin), - py_str(&f.model_dir), f.preload.as_deref().map(py_str).unwrap_or_else(|| "None".into()), ), - Err(_) => ("''".into(), "''".into(), "None".into()), + Err(_) => ("''".into(), "None".into()), + }; + let model = match &self.addons { + Some((root, _)) => py_str(&crate::gpu_plugin::rife_model_dir(root)), + None => "''".into(), }; script = script.replace("{{RIFE_PLUGIN}}", &plugin); script = script.replace("{{RIFE_MODEL_DIR}}", &model); script = script.replace("{{RIFE_PRELOAD}}", &preload); - let color = job.color_metadata(); + // Frame Rate runs after Crop & Resize: the output's colour. + let color = job.output_color_metadata(); script = script.replace("{{RIFE_MATRIX}}", color.zimg_matrix(job.input_height)); script = script.replace( "{{RIFE_RANGE}}", @@ -1958,6 +2119,11 @@ impl ScriptGenerator { script = remove_block("{{#FRAME_RATE}}", "{{/FRAME_RATE}}", script); } + // ==================================================================== + // COLOURISATION PASS + // ==================================================================== + script = self.substitute_colorize(script, job, pipeline); + // ==================================================================== // COLOR CORRECTION PASS // ==================================================================== @@ -2139,12 +2305,18 @@ impl ScriptGenerator { .ai_upscale_models() .iter() .map(|m| match *m { - AiModel::Bundled(name) => py_str(&f.model_dir.join(name)), // A pair the script chooses from by height. AiModel::AnimeJanai { sd, hd } => { let dir = self.animejanai.as_deref().unwrap_or(Path::new("")); format!("({}, {})", py_str(&dir.join(sd)), py_str(&dir.join(hd))) } + // ensure_methods_ready has refused the job + // unless the add-on is installed here. + AiModel::Addon { addon, file } => { + let root = self.addons.as_ref().map(|(r, _)| r.as_path()); + let dir = crate::addons::addon_dir(root.unwrap_or(Path::new("")), addon); + py_str(&dir.join(file)) + } }) .collect::>() .join(", "), @@ -2158,6 +2330,27 @@ impl ScriptGenerator { "{{AI_SD_MAX_HEIGHT}}", &crate::animejanai::SD_MAX_HEIGHT.to_string(), ); + let (tile_above, tile) = resize.ai_tiling(); + script = script.replace("{{AI_TILE_ABOVE}}", tile_above); + script = script.replace("{{AI_TILE}}", &tile.to_string()); + match resize.ai_resample_to() { + Some(factor) => { + script = script.replace("{{#AI_RESAMPLE}}", ""); + script = script.replace("{{/AI_RESAMPLE}}", ""); + script = script.replace("{{AI_FACTOR}}", &factor.to_string()); + } + None => script = remove_block("{{#AI_RESAMPLE}}", "{{/AI_RESAMPLE}}", script), + } + // Full strength emits nothing: the script is exactly the + // one from before the setting existed. + match resize.ai_blend_weight() { + Some(weight) => { + script = script.replace("{{#AI_STRENGTH}}", ""); + script = script.replace("{{/AI_STRENGTH}}", ""); + script = script.replace("{{AI_STRENGTH}}", &format!("{weight:.3}")); + } + None => script = remove_block("{{#AI_STRENGTH}}", "{{/AI_STRENGTH}}", script), + } let color = job.color_metadata(); script = script.replace("{{AI_MATRIX}}", color.zimg_matrix(job.input_height)); script = script.replace( @@ -2197,7 +2390,10 @@ impl ScriptGenerator { UpscaleMethod::Spline36 | UpscaleMethod::RealEsrgan | UpscaleMethod::RealCugan - | UpscaleMethod::AnimeJanai => { + | UpscaleMethod::AnimeJanai + | UpscaleMethod::RealEsrganGeneral + | UpscaleMethod::LiveAction + | UpscaleMethod::RealEsrganPlus => { script = remove_block("{{#UPSCALE_NNEDI3}}", "{{/UPSCALE_NNEDI3}}", script); script = remove_block("{{#UPSCALE_EEDI3}}", "{{/UPSCALE_EEDI3}}", script); } @@ -2314,6 +2510,25 @@ impl ScriptGenerator { script = remove_block("{{#RESIZE}}", "{{/RESIZE}}", script); } + // ==================================================================== + // BT.601 -> BT.709 (app 1.7.0) + // ==================================================================== + // Decided once, by the job (`VideoJob::converts_to_bt709`), because the + // encoder's colour flags and the preview's display conversion have to + // follow the same answer. + if job.converts_to_bt709() { + let source = job.color_metadata(); + script = script.replace("{{#COLOR_709}}", ""); + script = script.replace("{{/COLOR_709}}", ""); + script = script.replace("{{C709_MATRIX_IN}}", source.zimg_matrix(job.input_height)); + script = script.replace( + "{{C709_RANGE}}", + if source.is_full_range() { "full" } else { "limited" }, + ); + } else { + script = remove_block("{{#COLOR_709}}", "{{/COLOR_709}}", script); + } + // ==================================================================== // OUTPUT CHROMA SUBSAMPLING CONVERSION // ==================================================================== @@ -2344,7 +2559,8 @@ impl ScriptGenerator { let [r, g, b] = resize.border_rgb(); script = script.replace("{{BORDER_RGB}}", &format!("({r}, {g}, {b})")); - let color = job.color_metadata(); + // The bars go on last, in the colour the samples are delivered in. + let color = job.output_color_metadata(); script = script.replace("{{BORDER_MATRIX}}", color.zimg_matrix(job.input_height)); script = script.replace( "{{BORDER_RANGE}}", @@ -2500,6 +2716,20 @@ const DEINT_METHOD_BLOCKS: [(DeinterlaceMethod, &str); 4] = [ (DeinterlaceMethod::Bwdif, "DEINT_BWDIF"), ]; +/// The `{{#AI_RUNTIME}}` block with `{{AI_RUNTIME_DIR}}` set to the AI runtime +/// add-on's directory, or removed when `dir` is `None`. The block imports +/// `ai_runtime.py` (beside `pipe_source.py`) and calls its `setup()`, after +/// which any later block can use `ai_runtime.session(...)` and friends. +pub(crate) fn ai_runtime_block(script: String, dir: Option<&Path>) -> String { + match dir { + Some(dir) => script + .replace("{{#AI_RUNTIME}}", "") + .replace("{{/AI_RUNTIME}}", "") + .replace("{{AI_RUNTIME_DIR}}", &py_str(dir)), + None => remove_block("{{#AI_RUNTIME}}", "{{/AI_RUNTIME}}", script), + } +} + /// Remove a block from start tag to end tag (including the line). fn remove_block(start_tag: &str, end_tag: &str, mut script: String) -> String { while let Some(start_pos) = script.find(start_tag) { @@ -2523,6 +2753,40 @@ fn remove_block(start_tag: &str, end_tag: &str, mut script: String) -> String { mod tests { use super::*; + /// Both templates carry the AI runtime block once, before any pass, and + /// it imports the helper and sets it up from the substituted directory. + #[test] + fn ai_runtime_block_in_both_templates() { + for name in ["pipeline_template.vpy", "preview_template.vpy"] { + let t = ScriptGenerator::load_template_by_name(name).unwrap(); + assert_eq!(t.matches("{{#AI_RUNTIME}}").count(), 1, "{name}"); + let block_at = t.find("{{#AI_RUNTIME}}").unwrap(); + assert!(block_at > t.find("from pipe_source import").unwrap(), "{name}: after the pipe source"); + assert!(block_at < t.find("{{#DEINTERLACE}}").unwrap(), "{name}: before the passes"); + + let on = ai_runtime_block(t.clone(), Some(Path::new("/x/addons/ai-runtime"))); + assert!(on.contains("import ai_runtime\n"), "{name}"); + assert!(on.contains("ai_runtime.setup('/x/addons/ai-runtime')"), "{name}"); + assert!(!on.contains("AI_RUNTIME}}"), "{name}"); + + let off = ai_runtime_block(t, None); + assert!(!off.contains("import ai_runtime"), "{name}"); + assert!(!off.contains("AI_RUNTIME"), "{name}"); + } + } + + /// A job whose passes need no add-on (the default pipeline) never has the + /// block, whatever the generator was told about add-ons. + #[test] + fn a_job_without_addon_passes_has_no_ai_runtime_block() { + let gen = ScriptGenerator::new().unwrap().with_addons(PathBuf::from("/nonexistent"), "macos-arm64"); + let pipeline = ProcessingPipeline::default(); + assert!(pipeline.required_addons().is_empty()); + gen.ensure_methods_ready(&pipeline).unwrap(); + let script = gen.substitute_addons(gen.template.clone(), &pipeline); + assert!(!script.contains("ai_runtime")); + } + /// A method missing from DEINT_METHOD_BLOCKS keeps every other method's /// block in its script (issue #108). The match has no catch-all, so a new /// variant fails to compile here until it is listed. diff --git a/worker/src/upscale_addons.rs b/worker/src/upscale_addons.rs new file mode 100644 index 00000000..5bd0acd3 --- /dev/null +++ b/worker/src/upscale_addons.rs @@ -0,0 +1,114 @@ +//! The Crop & Resize AI upscalers whose models are manifest add-ons +//! (`app/assets/addons/upscale-*.json`, see `addons.rs` and docs/ADDONS.md): +//! Real-ESRGAN's anime video model and Real-CUGAN, Real-ESRGAN's +//! general-purpose model, a live-action SPAN model, and Real-ESRGAN's large +//! x4plus/x2plus models. (AnimeJaNai's hand-written add-on is `animejanai.rs`.) +//! +//! They all run through the VSNCNN plugin and `{{#UPSCALE_AI}}` block, so +//! everything in `gpu_plugin.rs` (the Vulkan probe, never autoloading) applies +//! — none of them needs the ONNX Runtime add-on. This module only says which +//! add-on holds which file and what each network's own factor is. +//! +//! Every model is an add-on, never part of the deps bundle (docs/ADDONS.md). +//! The anime models are vs-mlrt's ONNX conversions, published only inside its +//! ~810 MB `models.v15.16.7z`, so they are hosted individually by this project, +//! byte for byte as vs-mlrt ships them. Real-ESRGAN publishes `.pth` +//! checkpoints only, so those are converted by +//! `Scripts/models/convert_realesrgan.py` and hosted the same way; the SPAN +//! model is upstream's own ONNX. + +/// Real-ESRGAN's anime video model (`realesr-animevideov3`) and Real-CUGAN: +/// the anime upscalers, deps-bundled until deps 1.16.0. +pub const ANIME_ADDON: &str = "upscale-anime"; + +/// Real-ESRGAN General (`realesr-general-x4v3` and its denoise variant) and +/// 2xLiveActionV1_SPAN: both small, both for real-world footage. +pub const GENERAL_ADDON: &str = "upscale-general"; + +/// Real-ESRGAN x4plus / x2plus: the original 23-block RRDB networks — the +/// most detailed of the Real-ESRGAN family, and about 40x slower. +pub const ESRGAN_PLUS_ADDON: &str = "upscale-esrgan-plus"; + +/// vs-mlrt's `RealESRGANv2-animevideo-xsx2` (SRVGGNetCompact, 2x). +pub const ANIME_ESRGAN_X2: &str = "upscale-RealESRGANv2-animevideo-xsx2.onnx"; +/// vs-mlrt's `RealESRGANv2-animevideo-xsx4` (4x). +pub const ANIME_ESRGAN_X4: &str = "upscale-RealESRGANv2-animevideo-xsx4.onnx"; +/// Real-CUGAN 2x without denoising (vs-mlrt's `up2x-latest-no-denoise`). +pub const CUGAN_NO_DENOISE: &str = "upscale-cugan-up2x-latest-no-denoise.onnx"; +/// Real-CUGAN 2x, conservative (the default strength). +pub const CUGAN_CONSERVATIVE: &str = "upscale-cugan-up2x-latest-conservative.onnx"; +/// Real-CUGAN 2x, strongest denoising. +pub const CUGAN_DENOISE3X: &str = "upscale-cugan-up2x-latest-denoise3x.onnx"; + +/// `realesr-general-x4v3` (SRVGGNetCompact, 4x). +pub const GENERAL_X4: &str = "upscale-realesr-general-x4v3.onnx"; +/// `realesr-general-wdn-x4v3`: the same network trained to remove noise too +/// ("wdn": with denoise). +pub const GENERAL_WDN_X4: &str = "upscale-realesr-general-wdn-x4v3.onnx"; +/// jcj83429's 2xLiveActionV1_SPAN (SPAN, 2x). +pub const LIVE_ACTION_X2: &str = "2xLiveActionV1_SPAN_490000.onnx"; +/// `RealESRGAN_x4plus` (RRDBNet, 4x). +pub const PLUS_X4: &str = "upscale-RealESRGAN_x4plus.onnx"; +/// `RealESRGAN_x2plus` (RRDBNet, 2x). +pub const PLUS_X2: &str = "upscale-RealESRGAN_x2plus.onnx"; + +/// Every model file, with the add-on that installs it and the network's own +/// scale factor. The manifests list the same files (asserted below). +pub const MODELS: &[(&str, &str, u32)] = &[ + (ANIME_ADDON, ANIME_ESRGAN_X2, 2), + (ANIME_ADDON, ANIME_ESRGAN_X4, 4), + (ANIME_ADDON, CUGAN_NO_DENOISE, 2), + (ANIME_ADDON, CUGAN_CONSERVATIVE, 2), + (ANIME_ADDON, CUGAN_DENOISE3X, 2), + (GENERAL_ADDON, GENERAL_X4, 4), + (GENERAL_ADDON, GENERAL_WDN_X4, 4), + (GENERAL_ADDON, LIVE_ACTION_X2, 2), + (ESRGAN_PLUS_ADDON, PLUS_X4, 4), + (ESRGAN_PLUS_ADDON, PLUS_X2, 2), +]; + +/// A model file's own scale factor (2 for an unknown name, which no job can +/// reach: every name comes from [`MODELS`]). +pub fn scale_of(file: &str) -> u32 { + MODELS.iter().find(|(_, f, _)| *f == file).map(|(_, _, s)| *s).unwrap_or(2) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn models_match_the_manifests() { + for addon in [ANIME_ADDON, GENERAL_ADDON, ESRGAN_PLUS_ADDON] { + let m = crate::addons::manifest(addon).unwrap_or_else(|| panic!("no manifest {addon}")); + let mut listed: Vec = m + .expected_paths("macos-arm64") + .into_iter() + .filter(|p| p.ends_with(".onnx")) + .collect(); + listed.sort(); + let mut ours: Vec = + MODELS.iter().filter(|(a, _, _)| *a == addon).map(|(_, f, _)| f.to_string()).collect(); + ours.sort(); + assert_eq!(listed, ours, "{addon}"); + // VSNCNN ships on exactly these platforms, so the models are + // offered on exactly these. + for p in crate::addons::PLATFORMS { + assert_eq!(m.supports(p), crate::gpu_plugin::platform_ships_upscale(p), "{addon} {p}"); + } + // They run on VSNCNN, not ONNX Runtime. + assert!(!crate::addons::needs_ai_runtime(&[addon]), "{addon}"); + } + } + + #[test] + fn scales() { + assert_eq!(scale_of(ANIME_ESRGAN_X2), 2); + assert_eq!(scale_of(ANIME_ESRGAN_X4), 4); + assert_eq!(scale_of(CUGAN_DENOISE3X), 2); + assert_eq!(scale_of(GENERAL_X4), 4); + assert_eq!(scale_of(LIVE_ACTION_X2), 2); + assert_eq!(scale_of(PLUS_X4), 4); + assert_eq!(scale_of(PLUS_X2), 2); + } +} diff --git a/worker/templates/ai_runtime.py b/worker/templates/ai_runtime.py new file mode 100644 index 00000000..89542fa7 --- /dev/null +++ b/worker/templates/ai_runtime.py @@ -0,0 +1,409 @@ +""" +ONNX Runtime for VapourSynth scripts, from the ai-runtime add-on. + +The ai-runtime add-on (app/assets/addons/ai-runtime.json) unpacks ONNX Runtime +and NumPy wheels into /ai-runtime/site-packages/. Passes that run a +model the VSNCNN plugin can't (per-crop inference, models with more than one +input, anything needing NumPy around the network) use them through this +module, which ships beside pipe_source.py in the templates directory. + + import ai_runtime + ai_runtime.setup(r"/ai-runtime") # the {{#AI_RUNTIME}} block + sess = ai_runtime.session(r"/my-model/model.onnx") + clip = ai_runtime.map_frames(clip, lambda x: sess.run(None, {"input": x})[0], + width=clip.width * 2, height=clip.height * 2) + +setup() puts the add-on's packages on sys.path itself rather than relying on +PYTHONPATH, because Windows' embeddable Python ignores PYTHONPATH (its ._pth +file fixes sys.path), and registers onnxruntime's DLL directory there. + +Execution providers are chosen in this order, from those the installed wheel +offers: TensorRT, CUDA (neither is in today's wheels; a future NVIDIA add-on +can add them by passing its own directory to setup() first), DirectML +(Windows), Core ML (macOS), then the CPU. VAPOURBOX_AI_FORCE_CPU=1, or +force_cpu=True, uses the CPU only. A session that fails to create on a GPU +provider is retried on the CPU, and the reason is kept in last_fallback. + +`vapourbox-worker --probe-ai-runtime` calls self_test() through vspipe. +""" + +import base64 +import os +import sys +import threading + +ADDON_ID = "ai-runtime" +FORCE_CPU_ENV = "VAPOURBOX_AI_FORCE_CPU" + +# Most-preferred first. Only those the installed onnxruntime reports as +# available are used; the CPU is always last. +GPU_PROVIDERS = [ + "TensorrtExecutionProvider", + "CUDAExecutionProvider", + "DmlExecutionProvider", + "CoreMLExecutionProvider", +] +CPU_PROVIDER = "CPUExecutionProvider" + +# Options per provider. Core ML's MLProgram format supports more operators +# than the older NeuralNetwork format and runs on the GPU and Neural Engine. +PROVIDER_OPTIONS = { + "CoreMLExecutionProvider": {"ModelFormat": "MLProgram", "MLComputeUnits": "ALL"}, +} + +# y = x * 2 + 1 over a float32 (1, 3, h, w) tensor, IR 8 / opset 13. Built by +# Scripts/models/make_ai_runtime_selftest.py; the same bytes are committed as +# Tests/TestResources/ai_runtime_selftest.onnx (a test asserts they agree). +_SELFTEST_MODEL_B64 = ( + "CAgSCXZhcG91cmJveDqWAQoRCgF4CgN0d28SAngyIgNNdWwKEQoCeDIKA29uZRIBeSIDQWRkEhJ2YXBvdXJib3hfc2VsZnRlc3Qq" + "DRABIgQAAABAQgN0d28qDRABIgQAAIA/QgNvbmVaHQoBeBIYChYIARISCgIIAQoCCAMKAxIBaAoDEgF3Yh0KAXkSGAoWCAESEgoC" + "CAEKAggDCgMSAWgKAxIBd0IECgAQDQ==" +) + +_lock = threading.Lock() +_sessions = {} +_dll_dirs = [] +_ort = None + +# Why the last session() call fell back to the CPU, or None. +last_fallback = None + + +def setup(*addon_dirs): + """Make the add-on's packages importable. Pass the ai-runtime add-on's + directory (and, first, any add-on that should take precedence). Safe to + call more than once.""" + for addon_dir in reversed(addon_dirs): + site = os.path.join(addon_dir, "site-packages") + if not os.path.isdir(site): + raise RuntimeError( + "The AI runtime add-on is not installed (no %s). Install it in " + "Settings -> Add-ons." % site + ) + if site not in sys.path: + sys.path.insert(0, site) + capi = os.path.join(site, "onnxruntime", "capi") + if os.name == "nt" and os.path.isdir(capi) and hasattr(os, "add_dll_directory"): + # onnxruntime_pybind11_state.pyd links onnxruntime.dll and + # DirectML.dll beside it. Keep the handle: closing it removes the + # directory from the search path again. + _dll_dirs.append(os.add_dll_directory(capi)) + + +def ort(): + """The onnxruntime module (after setup()).""" + global _ort + if _ort is None: + import onnxruntime # noqa: deferred until setup() has run + + _ort = onnxruntime + return _ort + + +def force_cpu_requested(): + return os.environ.get(FORCE_CPU_ENV, "").strip().lower() in ("1", "true", "yes") + + +def available_providers(): + return list(ort().get_available_providers()) + + +def preferred_providers(force_cpu=False): + """The providers a session should try, most preferred first.""" + if force_cpu or force_cpu_requested(): + return [CPU_PROVIDER] + available = set(available_providers()) + return [p for p in GPU_PROVIDERS if p in available] + [CPU_PROVIDER] + + +def _session_options(providers, dims=None): + rt = ort() + opts = rt.SessionOptions() + opts.log_severity_level = 3 # errors only: Core ML warns per unsupported node + if "DmlExecutionProvider" in providers: + # DirectML requires both (onnxruntime's DirectML documentation). + opts.enable_mem_pattern = False + opts.execution_mode = rt.ExecutionMode.ORT_SEQUENTIAL + for name, value in sorted((dims or {}).items()): + opts.add_free_dimension_override_by_name(name, int(value)) + return opts + + +def _create(model, providers, dims=None): + rt = ort() + with_options = [(p, PROVIDER_OPTIONS[p]) if p in PROVIDER_OPTIONS else p for p in providers] + try: + return rt.InferenceSession(model, sess_options=_session_options(providers, dims), providers=with_options) + except Exception: + if with_options == providers: + raise + # An older runtime that doesn't know an option: the provider's defaults. + return rt.InferenceSession(model, sess_options=_session_options(providers, dims), providers=providers) + + +def session(model, force_cpu=False, providers=None, dims=None): + """An InferenceSession for `model` (a path, or the model's bytes), on the + preferred providers unless `providers` is given. Cached per model, + provider list and `dims`, so a pass calling this from a frame callback + creates it once. Falls back to the CPU if a GPU provider can't create it. + + `dims` ({"height": 512, "width": 512}) fixes a model's named dynamic + dimensions for this session (ONNX Runtime's free-dimension overrides): + one exported file serves several input sizes, and each session still has + the static shapes Core ML needs to take the whole graph.""" + global last_fallback + chosen = list(providers) if providers else preferred_providers(force_cpu) + key = (model if isinstance(model, str) else hash(bytes(model)), tuple(chosen), + tuple(sorted((dims or {}).items()))) + with _lock: + cached = _sessions.get(key) + if cached is not None: + return cached + try: + sess = _create(model, chosen, dims) + last_fallback = None + except Exception as e: + if chosen == [CPU_PROVIDER]: + raise + last_fallback = "%s: %s" % (chosen[0], e) + sess = _create(model, [CPU_PROVIDER], dims) + _sessions[key] = sess + return sess + + +def session_provider(sess): + """The provider a session prefers (its first registered one).""" + return sess.get_providers()[0] + + +def frame_to_nchw(frame): + """A planar RGBS VideoFrame as a float32 (1, 3, height, width) array.""" + import numpy as np + + return np.stack([np.asarray(frame[p]) for p in range(3)])[None].astype(np.float32, copy=False) + + +def nchw_to_frame(array, frame): + """Write a (1, 3, height, width) array into a writable copy of `frame` + (planar RGBS, of the array's size) and return the copy.""" + import numpy as np + + out = frame.copy() + data = np.asarray(array, dtype=np.float32) + if data.ndim == 4: + data = data[0] + for p in range(3): + np.copyto(np.asarray(out[p]), data[p]) + return out + + +def map_frames(clip, fn, width=None, height=None): + """Apply `fn` (a float32 (1, 3, h, w) array -> a (1, 3, height, width) + array) to every frame of a planar RGBS clip. The output is `width` x + `height` (default: the input's) and keeps each source frame's properties.""" + import vapoursynth as vs + + core = vs.core + if clip.format is None or clip.format.id != vs.RGBS: + raise ValueError("ai_runtime.map_frames needs a constant-format RGBS clip") + width = width or clip.width + height = height or clip.height + template = clip + if (width, height) != (clip.width, clip.height): + template = core.std.BlankClip(clip, width=width, height=height, keep=True) + + def _apply(n, f): + src, dst = f[0], f[1] + out = nchw_to_frame(fn(frame_to_nchw(src)), dst) + for k in list(out.props.keys()): + del out.props[k] + out.props.update(src.props) + return out + + return core.std.ModifyFrame(template, [clip, template], _apply) + + +def selftest_model(): + """The self-test model's bytes.""" + return base64.b64decode(_SELFTEST_MODEL_B64) + + +def frame_test(force_cpu=False): + """Run the self-test model over a VapourSynth frame through map_frames + (the path a pass takes: RGBS frame -> NumPy -> session -> frame) and check + the pixels. Returns the three output plane values.""" + import numpy as np + import vapoursynth as vs + + sess = session(selftest_model(), force_cpu=force_cpu) + name = sess.get_inputs()[0].name + clip = vs.core.std.BlankClip(width=24, height=16, format=vs.RGBS, color=[0.25, 0.5, 0.75], length=1) + clip = map_frames(clip, lambda x: sess.run(None, {name: x})[0], width=24, height=16) + frame = clip.get_frame(0) + got = [float(np.asarray(frame[p]).mean()) for p in range(3)] + want = [1.5, 2.0, 2.5] + if max(abs(a - b) for a, b in zip(got, want)) > 1e-2: + raise RuntimeError("the self-test model gave the wrong pixels through VapourSynth: %r" % got) + return got + + +def self_test(force_cpu=False): + """Create a session for the self-test model on the preferred provider, + run it, and check the answer. Returns a dict for the probe: + {version, providers (available), requested, provider, fallback, maxError}. + Raises if the session can't be created or the answer is wrong.""" + import numpy as np + + rt = ort() + requested = preferred_providers(force_cpu) + sess = session(selftest_model(), force_cpu=force_cpu) + x = np.linspace(0.0, 1.0, 3 * 16 * 16, dtype=np.float32).reshape(1, 3, 16, 16) + y = sess.run(None, {sess.get_inputs()[0].name: x})[0] + err = float(np.abs(y - (x * 2.0 + 1.0)).max()) + if not err < 1e-2: + raise RuntimeError("the self-test model computed the wrong answer (max error %g)" % err) + return { + "version": rt.__version__, + "providers": available_providers(), + "requested": requested, + "provider": session_provider(sess), + "sessionProviders": list(sess.get_providers()), + "fallback": last_fallback, + "maxError": err, + } + + +# ---------------------------------------------------------------------------- +# Helpers for passes that run a model on part of a frame (face restoration, +# and anything else that crops, warps or masks around a network). They keep a +# pass a pure function of its input frames, which VapourSynth needs: it asks +# for frames in parallel and out of order, so nothing may depend on the order +# frames were computed in. +# ---------------------------------------------------------------------------- + + +def to_rgbs(clip, matrix, range_): + """`clip` (YUV, GRAY or RGB, any depth) as planar float RGB, using the + source's matrix and range (zimg names, e.g. "709" / "limited") for YUV.""" + import vapoursynth as vs + + core = vs.core + family = clip.format.color_family + if family == vs.YUV: + return core.resize.Spline36(clip, format=vs.RGBS, matrix_in_s=matrix, range_in_s=range_) + if family == vs.GRAY: + gray = core.resize.Point(clip, format=vs.GRAYS, range_in_s=range_) + return core.std.ShufflePlanes(gray, [0, 0, 0], vs.RGB) + return core.resize.Point(clip, format=vs.RGBS) + + +def from_rgbs(rgbs, fmt, matrix, range_): + """The inverse of to_rgbs: back to `fmt` (a VideoFormat), dithered. A GRAY + source takes the green plane, as the AI upscaling block does.""" + import vapoursynth as vs + + core = vs.core + if fmt.color_family == vs.YUV: + return core.resize.Spline36(rgbs, format=fmt.id, matrix_s=matrix, range_s=range_, + dither_type="error_diffusion") + if fmt.color_family == vs.GRAY: + return core.resize.Point(core.std.ShufflePlanes(rgbs, 1, vs.GRAY), format=fmt.id, range_s=range_, + dither_type="error_diffusion") + return core.resize.Point(rgbs, format=fmt.id, dither_type="error_diffusion") + + +def shifted(clip, offset): + """`clip` moved by `offset` frames, clamped at both ends: frame n of the + result is frame clamp(n + offset) of `clip`. Passing these to + std.ModifyFrame alongside the clip gives a callback its neighbours' + frames without calling get_frame() from inside a frame request.""" + n = clip.num_frames + if offset == 0 or n < 2: + return clip + offset = max(-(n - 1), min(n - 1, offset)) + if offset > 0: + return clip[offset:] + clip[-1] * offset + return clip[0] * (-offset) + clip[: n + offset] + + +class FrameMemo: + """A bounded cache of a pure per-frame computation, computed once even + when several frame requests (several nodes reading the same result, or + neighbours sharing a detection) ask for the same key at the same time. + Losing an entry only costs recomputing it, which gives the same answer.""" + + def __init__(self, capacity=64): + import collections + + self._capacity = capacity + self._lock = threading.Lock() + self._done = collections.OrderedDict() + self._pending = {} + + def get(self, key, compute): + with self._lock: + if key in self._done: + self._done.move_to_end(key) + return self._done[key] + event = self._pending.get(key) + owner = event is None + if owner: + event = self._pending[key] = threading.Event() + if not owner: + event.wait() + with self._lock: + if key in self._done: + return self._done[key] + return compute() # evicted already (or the owner failed): recompute + try: + value = compute() + with self._lock: + self._done[key] = value + while len(self._done) > self._capacity: + self._done.popitem(last=False) + return value + finally: + with self._lock: + self._pending.pop(key, None) + event.set() + + +class Runner: + """One model's session for a pass that calls it from frame requests. + + Runs are serialised: VapourSynth asks for several frames at once, and on + a GPU provider concurrent runs each allocate their own working memory + without running any faster (Core ML ran out of GPU memory on a 16 GB Mac + with two jobs' worth). A run that fails on a GPU provider (out of memory, + an unsupported shape at run time) is retried on the CPU, which this + runner then keeps using; `fallback` says why. It runs like an + InferenceSession (`run(output_names, feeds)`, `get_inputs()`), so code + written against a session takes either.""" + + def __init__(self, model, force_cpu=False, lock=None, dims=None): + """`lock`: share one between the runners of a pass to serialise all + of its models' runs, not just each model's. `dims`: fixed sizes for + the model's named dynamic dimensions (see session()).""" + self.model = model + self.dims = dims + self.session = session(model, force_cpu=force_cpu, dims=dims) + self.fallback = last_fallback + self._lock = lock or threading.Lock() + + @property + def provider(self): + return session_provider(self.session) + + def get_inputs(self): + return self.session.get_inputs() + + def run(self, output_names, feeds): + with self._lock: + try: + return self.session.run(output_names, feeds) + except Exception as e: + if self.provider == CPU_PROVIDER: + raise + self.fallback = "%s failed at run time: %s" % (self.provider, e) + self.session = session(self.model, force_cpu=True, dims=self.dims) + return self.session.run(output_names, feeds) diff --git a/worker/templates/colorize.py b/worker/templates/colorize.py new file mode 100644 index 00000000..28c99477 --- /dev/null +++ b/worker/templates/colorize.py @@ -0,0 +1,269 @@ +""" +Colourisation for VapourSynth, on ONNX Runtime from the ai-runtime add-on. + + import colorize + clip = colorize.colorize(clip, model_path=..., model="ddcolor", render_size=512, + strength=1.0, temporal_radius=2, matrix="709", + color_range="limited") + +Two models predict colour from a grey picture: + +- **DDColor** (piddnad/DDColor, the DDColor-T checkpoint) predicts CIELAB's a + and b. As upstream's ColorizationPipeline does, they are scaled up to the + frame and recombined with the frame's own lightness, then converted to RGB + (sRGB, D65, OpenCV's constants), here with a chain of std.Expr stages at + the luma's resolution. +- **DeOldify Video** (jantic/DeOldify, ColorizeVideo_gen) predicts an RGB + picture. As upstream's ColorizerFilter does, it is scaled up to the frame + and only its chroma is kept. + +Both see the luma stretched to a square of `render_size` (upstream's +`input_size` / `render_factor x 16`), as one channel in [0, 1]. + +**The source's luma is kept bit-exact.** The predicted picture is converted +to YUV with the source's matrix, its chroma scaled by `strength` (0 is +neutral grey), converted to the source's format (bit depth and subsampling, +dithered), and only its chroma planes are taken: the Y plane of the output +is the input's, untouched. A source that already has colour is coloured +from its luma all the same: its own chroma is replaced. + +**Temporal smoothing** (`temporal_radius` r > 0): a per-frame colouriser +guesses colour afresh for every frame, and its guesses flicker (DDColor's +especially). Each frame's prediction is averaged, at the render size, with +the predictions of the r frames either side, weighted by distance and down +to nothing where the grey picture itself differs there (MOTION_LIMIT, over +MOTION_BLOCK-pixel blocks) — so colour follows a moving object rather than +smearing behind it, and a cut takes nothing from the shot before it. The +pass is a pure function of frames n-r..n+r, as VapourSynth requires +(neighbours via ai_runtime.shifted, predictions shared through a FrameMemo), +so a preview is the encode's frame. + +Model runs are serialised with a CPU retry (ai_runtime.Runner); each session +has the render size fixed (ai_runtime's `dims`), so Core ML gets static +shapes from one exported file. +""" + +import numpy as np + +import ai_runtime + +MODELS = ("ddcolor", "deoldify") + +# Models a preview (preview=True) runs on the CPU. Core ML compiles DDColor's +# transformer for 50-80 s when a session is created, which happens in every +# vspipe process, and a preview is a process of its own that colours only +# 2r+1 frames (0.8 s each on the CPU at 512). DeOldify compiles in about 5 s +# and is 6x slower on the CPU, so it keeps the preferred provider. The two +# providers' outputs differ by under 0.001 of a CIELAB unit. Measured in +# docs/reports/2026-11-colourisation. +PREVIEW_ON_CPU = {"ddcolor": True, "deoldify": False} + +# Temporal smoothing: neighbours are compared on the render-size luma, +# averaged over MOTION_BLOCK x MOTION_BLOCK blocks; where a neighbour's block +# differs from this frame's by MOTION_LIMIT (8-bit levels: 10) it contributes +# nothing. Measured in docs/reports/2026-11-colourisation. +MOTION_BLOCK = 8 +MOTION_LIMIT = 10.0 / 255.0 + +# OpenCV's float CIELAB <-> sRGB (D65) constants, as DDColor's pipeline uses +# them (cv2.COLOR_LAB2BGR on float32). +XN, ZN = 0.950456, 1.088754 +XYZ_TO_RGB = ( + (3.240479, -1.53715, -0.498535), + (-0.969256, 1.875991, 0.041556), + (0.055648, -0.204043, 1.057311), +) +LAB_T = 0.008856 # cube-root threshold on Y +LAB_K = 7.787 +LAB_F = 0.206893 # 6/29: the threshold on f + +# The runner of the most recent colorize() call, for diagnostics. +last_runner = None + + +# See the identical helper in the two .vpy templates: VapourSynth's std.Expr +# JIT is x86-only, so on ARM every expression is walked once per pixel. akarin +# has a real LLVM JIT that works on aarch64 (and is in every bundle but macOS +# x64). The fallback names core.std.Expr explicitly -- calling _expr() would +# recurse forever. The expressions here use only the subset both share. +def _expr(clips, expr, **kwargs): + import vapoursynth as vs + core = vs.core + _akarin_expr = getattr(getattr(core, "akarin", None), "Expr", None) + if _akarin_expr is not None: + return _akarin_expr(clips, expr, **kwargs) + return core.std.Expr(clips, expr, **kwargs) + + +def lab_stages(core, y, a, b): + """CIELAB with the lightness of `y` (GRAYS: the frame's luma, as a grey + sRGB value in [0, 1]) and the colour of `a`, `b` (GRAYS, the same size), + as an RGBS clip: OpenCV's Lab -> sRGB, out-of-gamut values clipped. In + stages, so no expression repeats another's work. + + A grey pixel's L is the lightness of its own value, so the chain starts + from the linear luminance: cv2 would take L back to the same Y.""" + expr = _expr + ylin = expr([y], "x 0.04045 <= x 12.92 / x 0.055 + 1.055 / 2.4 pow ?") + fy = expr([ylin], f"x {LAB_T} > x 0.0 max 0.3333333333 pow x {LAB_K} * 0.1379310345 + ?") + + def finv(f): + return f"{f} {LAB_F} > {f} 3 pow {f} 0.1379310345 - {LAB_K} / ?" + + x = expr([fy, a], finv("x y 500 / +") + f" {XN} *") + z = expr([fy, b], finv("x y 200 / -") + f" {ZN} *") + + def channel(row): + c0, c1, c2 = row + lin = f"x {c0} * y {c1} * + z {c2} * + 0 max 1 min" + # sRGB encoding (OpenCV's gamma for float Lab conversions). + return expr([x, ylin, z], f"{lin} 0.0031308 <= {lin} 12.92 * {lin} 0.4166666667 pow 1.055 * 0.055 - ?") + + r, g, bl = (channel(row) for row in XYZ_TO_RGB) + import vapoursynth as vs + + return core.std.ShufflePlanes([r, g, bl], [0, 0, 0], vs.RGB) + + +def motion_crop(luma): + """The render-size luma averaged over MOTION_BLOCK blocks (edges padded).""" + b = MOTION_BLOCK + h, w = luma.shape + ph, pw = -(-h // b) * b, -(-w // b) * b + if (ph, pw) != (h, w): + luma = np.pad(luma, ((0, ph - h), (0, pw - w)), mode="edge") + return luma.reshape(ph // b, b, pw // b, b).mean((1, 3)).astype(np.float32) + + +def motion_weight(a, b, shape, limit=MOTION_LIMIT): + """Per-pixel weight (`shape`) for blending a neighbour whose motion crop + is `b` into a frame whose motion crop is `a`: 1 where they agree, falling + to 0 at `limit` (3x3-block mean of the difference, so one noisy block + is not motion).""" + d = np.abs(a - b) + p = np.pad(d, 1, mode="edge") + d = sum(p[i:i + d.shape[0], j:j + d.shape[1]] for i in range(3) for j in range(3)) / 9.0 + w = np.clip(1.0 - d / limit, 0.0, 1.0) + w = np.repeat(np.repeat(w, MOTION_BLOCK, 0), MOTION_BLOCK, 1) + return w[: shape[0], : shape[1]] + + +def colorize( + clip, + model_path, + model="ddcolor", + render_size=512, + strength=1.0, + temporal_radius=2, + matrix="709", + color_range="limited", + preview=False, + motion_limit=MOTION_LIMIT, +): + """Colour `clip` (YUV, any depth and subsampling) and return it in its own + format, its Y plane bit-exact. + + `model` is "ddcolor" (`model_path`: colorize-ddcolor-tiny.onnx, render + size a multiple of 32) or "deoldify" (colorize-deoldify-video.onnx, a + multiple of 16). `strength` scales the predicted chroma (0 grey, 1 as + predicted, above 1 more saturated). `temporal_radius` is how many frames + either side each frame's colour is averaged with (0: none). `matrix` / + `color_range` are the source's (zimg names). `preview`: the preview + script's call, which runs the models in PREVIEW_ON_CPU on the CPU. + `motion_limit`: the neighbour-blend cut-off (the report compares it). + """ + global last_runner + limit = float(motion_limit) + import sys + + import vapoursynth as vs + + core = vs.core + fmt = clip.format + if fmt is None or fmt.color_family != vs.YUV: + raise ValueError("colorize needs a constant-format YUV clip") + if model not in MODELS: + raise ValueError("colorize: unknown model %r" % model) + step = 32 if model == "ddcolor" else 16 + size = max(step, int(render_size) // step * step) + r = max(0, int(temporal_radius)) + strength = max(0.0, float(strength)) + + force_cpu = bool(preview) and PREVIEW_ON_CPU[model] + if not force_cpu: + sys.stderr.write("Colourisation: preparing %s (the first frame can take a minute or two while the " + "model is compiled for this machine)\n" % model) + runner = ai_runtime.Runner(model_path, force_cpu=force_cpu, dims={"height": size, "width": size}) + last_runner = runner + in_name = runner.get_inputs()[0].name + sys.stderr.write("Colourisation: %s at %dx%d on %s%s\n" % ( + model, size, size, runner.provider, "" if not runner.fallback else " (%s)" % runner.fallback)) + + # The luma as a grey picture in [0, 1], and the model's square copy of it. + luma = core.std.ShufflePlanes(clip, 0, vs.GRAY) + yf = core.resize.Point(luma, format=vs.GRAYS, range_in_s=color_range) + small = core.resize.Bilinear(yf, width=size, height=size) + last = clip.num_frames - 1 + + def clamp(i): + return min(max(i, 0), last) + + offsets = list(range(-r, r + 1)) + inputs = [ai_runtime.shifted(small, k) for k in offsets] + predictions = ai_runtime.FrameMemo(capacity=4 * r + 8) + results = ai_runtime.FrameMemo(capacity=8) + + def predict(index, frame): + def run(): + g = np.asarray(frame[0]).astype(np.float32) + out = runner.run(None, {in_name: np.clip(g, 0.0, 1.0)[None, None]})[0][0] + return out.astype(np.float32), motion_crop(g) + + return predictions.get(index, run) + + def smoothed(n, frames): + def run(): + pred, crop = predict(n, frames[r]) + if not r: + return pred + total = pred * float(r + 1) + weight = np.full(pred.shape[1:], float(r + 1), np.float32) + for i, k in enumerate(offsets): + index = clamp(n + k) + if k == 0 or index == n: + continue + p, c = predict(index, frames[i]) + w = (r + 1 - abs(k)) * motion_weight(crop, c, pred.shape[1:], limit) + total += w * p + weight += w + return total / weight + + return results.get(n, run) + + def write(n, f): + out = f[0].copy() + pred = smoothed(n, f[1:]) + for p in range(3): + np.asarray(out[p])[:] = pred[p] if p < len(pred) else 0.0 + return out + + carrier = core.std.BlankClip(small, format=vs.RGBS, keep=True) + pred = core.std.ModifyFrame(carrier, [carrier] + inputs, write) + + if model == "ddcolor": + # a and b, scaled to the frame (bilinear: upstream's nearest-neighbour + # interpolate leaves render-size blocks of colour at 1080p). + a = core.resize.Bilinear(core.std.ShufflePlanes(pred, 0, vs.GRAY), width=clip.width, height=clip.height) + b = core.resize.Bilinear(core.std.ShufflePlanes(pred, 1, vs.GRAY), width=clip.width, height=clip.height) + rgb = lab_stages(core, yf, a, b) + else: + rgb = core.resize.Bilinear(pred, width=clip.width, height=clip.height) + + yuv = core.resize.Bilinear(rgb, format=vs.YUV444PS, matrix_in_s="rgb", matrix_s=matrix) + s = "%.6f" % strength + yuv = _expr(yuv, ["", "x %s *" % s, "x %s *" % s]) + yuv = core.std.ShufflePlanes([yf, yuv, yuv], [0, 1, 2], vs.YUV) + out = core.resize.Bicubic(yuv, format=fmt.id, matrix_in_s=matrix, matrix_s=matrix, range_s=color_range, + dither_type="error_diffusion") + out = core.std.ShufflePlanes([clip, out, out], [0, 1, 2], vs.YUV) + return core.std.CopyFrameProps(out, clip) diff --git a/worker/templates/face_restore.py b/worker/templates/face_restore.py new file mode 100644 index 00000000..ff416824 --- /dev/null +++ b/worker/templates/face_restore.py @@ -0,0 +1,617 @@ +""" +Face restoration for VapourSynth, on ONNX Runtime from the ai-runtime add-on. + + import face_restore + clip = face_restore.restore_faces(clip, detector=..., parser=..., restorer=..., + model="restoreformer", ...) + +For every frame: RetinaFace finds the faces and their five landmarks (eyes, +nose, mouth corners); each face is aligned to the FFHQ 512x512 template by a +similarity transform, restored by RestoreFormer++, GFPGAN or CodeFormer, and +parsed by BiSeNet into a mask of the face itself (skin, brows, eyes, glasses, +ears, nose, mouth and lips; not hair, neck or clothes), which is feathered, +warped back and used to blend the restored face into the frame at the clip's +own resolution. + +The pass is a pure function of its input frames, as VapourSynth requires +(frames are requested in parallel and out of order): + +- **Temporal smoothing** reads the detections of the `temporal_radius` frames + either side (std.ModifyFrame over clips shifted by -r..r, never get_frame() + from inside a request). Each face's landmarks are averaged with the faces it + matches in its neighbours (triangular weights), a face detected just under + the threshold is kept when a neighbour has it above the threshold + (hysteresis), and a face missing from one frame between two that have it is + filled in. Per-frame detections jitter by a pixel or two, and a restorer + turns that jitter into flicker; smoothing removes most of it. +- **Bit-exact outside the faces.** The restored faces are composited with + std.MaskedMerge in the source's own format, so every pixel the face mask + does not reach (and every frame with no face) is the input, unchanged. + +Detection runs once per frame (shared through a FrameMemo), on a copy scaled +to fit 640x640 and padded, so every model has a static input shape — which is +what lets Core ML take the whole graph. + +Pure NumPy: OpenCV is not in the deps bundle or the ai-runtime add-on, and +nothing here needs it. +""" + +import threading + +import numpy as np + +import ai_runtime + +# FFHQ's five-point template at 512x512 (facexlib's FaceRestoreHelper, the +# alignment every one of these restorers was trained on). +FFHQ_TEMPLATE = np.array( + [ + [192.98138, 239.94708], + [318.90277, 240.1936], + [256.63416, 314.01935], + [201.26117, 371.41043], + [313.08905, 371.15118], + ], + np.float64, +) +FACE_SIZE = 512 +# What facexlib fills outside the frame when it crops a face near an edge. +BORDER_RGB = np.array([132.0, 133.0, 135.0], np.float32) / 255.0 + +# RetinaFace (biubug6/Pytorch_Retinaface's configuration for both backbones). +DET_SIZE = 640 +DET_MEAN_BGR = np.array([104.0, 117.0, 123.0], np.float32) +DET_MIN_SIZES = [[16, 32], [64, 128], [256, 512]] +DET_STEPS = [8, 16, 32] +DET_VARIANCE = (0.1, 0.2) +DET_NMS = 0.4 + +# BiSeNet (CelebAMask-HQ labels): 1 skin, 2-3 brows, 4-5 eyes, 6 glasses, +# 7-9 ears and earrings, 10 nose, 11 mouth, 12-13 lips. Not 14 neck, +# 15 necklace, 16 cloth, 17 hair, 18 hat. +FACE_CLASSES = list(range(1, 14)) +PARSE_MEAN = np.array([0.485, 0.456, 0.406], np.float32)[:, None, None] +PARSE_STD = np.array([0.229, 0.224, 0.225], np.float32)[:, None, None] +# facexlib feathers its parse mask with two 101-tap Gaussian blurs of sigma 11 +# and clears a 10-pixel border; one blur of sigma 11 * sqrt(2) is the same. +MASK_SIGMA = 11.0 * np.sqrt(2.0) +MASK_BORDER = 10 +# Mask values below this count as zero (so the blend's reach is finite). +MASK_FLOOR = 1.0 / 1024.0 + +MODELS = ("restoreformer", "gfpgan", "codeformer") + +# The runners of the most recent restore_faces() call ({"detector", "parser", +# "restorer"} -> ai_runtime.Runner), for diagnostics: which provider each +# model runs on and any fallback to the CPU. +last_restorer = {} + + +# ---------------------------------------------------------------------------- +# Detection +# ---------------------------------------------------------------------------- + +_priors = None + + +def priors(): + """RetinaFace's anchors for a DET_SIZE square input, (N, 4) as + (cx, cy, w, h) in [0, 1], in the network's output order.""" + global _priors + if _priors is None: + rows = [] + for k, step in enumerate(DET_STEPS): + f = -(-DET_SIZE // step) + cy, cx = np.meshgrid((np.arange(f) + 0.5) * step / DET_SIZE, (np.arange(f) + 0.5) * step / DET_SIZE, + indexing="ij") + for size in DET_MIN_SIZES[k]: + s = np.full_like(cx, size / DET_SIZE) + rows.append(np.stack([cx, cy, s, s], -1)) + # Interleave the two sizes per cell, as PriorBox does. + a, b = rows[-2], rows[-1] + rows[-2:] = [np.stack([a, b], 2).reshape(-1, 4)] + _priors = np.concatenate(rows).astype(np.float32) + return _priors + + +def nms(boxes, scores, threshold): + """Greedy non-maximum suppression (Pytorch_Retinaface's py_cpu_nms).""" + x1, y1, x2, y2 = boxes.T + areas = (x2 - x1 + 1) * (y2 - y1 + 1) + order = scores.argsort()[::-1] + keep = [] + while order.size: + i = order[0] + keep.append(i) + xx1 = np.maximum(x1[i], x1[order[1:]]) + yy1 = np.maximum(y1[i], y1[order[1:]]) + xx2 = np.minimum(x2[i], x2[order[1:]]) + yy2 = np.minimum(y2[i], y2[order[1:]]) + inter = np.maximum(0.0, xx2 - xx1 + 1) * np.maximum(0.0, yy2 - yy1 + 1) + iou = inter / (areas[i] + areas[order[1:]] - inter) + order = order[1:][iou <= threshold] + return np.array(keep, np.int64) + + +def detect(session, rgb, scale, floor): + """Faces in one frame (`session`: an InferenceSession or ai_runtime.Runner). `rgb` is the detection copy, (3, h, w) float RGB in + [0, 1] with h, w <= DET_SIZE; `scale` (x, y) maps its pixels to the + clip's (clip = (det + 0.5) / scale - 0.5). Returns (scores (k,), boxes (k, 4) + x1 y1 x2 y2, landmarks (k, 5, 2)) in clip pixels, every face scoring at + least `floor`, best first.""" + _, h, w = rgb.shape + x = np.zeros((1, 3, DET_SIZE, DET_SIZE), np.float32) + # BGR, 0-255, minus the mean; the padding is the mean (zero after it). + x[0, :, :h, :w] = rgb[::-1] * 255.0 - DET_MEAN_BGR[:, None, None] + loc, conf, landm = session.run(None, {session.get_inputs()[0].name: x}) + loc, scores, landm = loc[0], conf[0][:, 1], landm[0] + keep = scores >= floor + if not keep.any(): + return np.zeros(0, np.float32), np.zeros((0, 4)), np.zeros((0, 5, 2)) + p, loc, scores, landm = priors()[keep], loc[keep], scores[keep], landm[keep] + v0, v1 = DET_VARIANCE + centre = p[:, :2] + loc[:, :2] * v0 * p[:, 2:] + size = p[:, 2:] * np.exp(loc[:, 2:] * v1) + boxes = np.concatenate([centre - size / 2, centre + size / 2], 1) * DET_SIZE + marks = (p[:, None, :2] + landm.reshape(-1, 5, 2) * v0 * p[:, None, 2:]) * DET_SIZE + k = nms(boxes, scores, DET_NMS) + scale = np.asarray(scale, np.float64) + boxes = (boxes[k] + 0.5) / np.tile(scale, 2) - 0.5 + marks = (marks[k] + 0.5) / scale - 0.5 + return scores[k], boxes.astype(np.float64), marks.astype(np.float64) + + +# ---------------------------------------------------------------------------- +# Temporal smoothing: a pure function of the detections in frames n-r..n+r +# ---------------------------------------------------------------------------- + + +def _match(box, boxes): + """Index of the box in `boxes` overlapping `box` most (IoU > 0.3), or None.""" + if not len(boxes): + return None + x1 = np.maximum(box[0], boxes[:, 0]) + y1 = np.maximum(box[1], boxes[:, 1]) + x2 = np.minimum(box[2], boxes[:, 2]) + y2 = np.minimum(box[3], boxes[:, 3]) + inter = np.maximum(0, x2 - x1) * np.maximum(0, y2 - y1) + area = (box[2] - box[0]) * (box[3] - box[1]) + areas = (boxes[:, 2] - boxes[:, 0]) * (boxes[:, 3] - boxes[:, 1]) + iou = inter / np.maximum(area + areas - inter, 1e-6) + i = int(iou.argmax()) + return i if iou[i] > 0.3 else None + + +def select_faces(dets, radius, threshold, min_face, max_faces): + """The faces to restore in the centre frame of `dets` (a list of 2r+1 + detections, frame n-r first), as (box, landmarks) with smoothed + landmarks. `radius` 0 is per-frame detection, unsmoothed.""" + centre = dets[radius] + low = threshold * 0.75 + faces = [] + + def confirmed(box, score): + """Above the threshold here, or near it here and above it in a neighbour.""" + if score >= threshold: + return True + if score < low: + return False + for d in (dets[radius - 1], dets[radius + 1]) if radius else (): + j = _match(box, d[1]) + if j is not None and d[0][j] >= threshold: + return True + return False + + for i in range(len(centre[0])): + if confirmed(centre[1][i], centre[0][i]): + faces.append((centre[1][i], centre[2][i])) + + if radius: + # A face in the frames either side but not this one: a dropout. + prev, nxt = dets[radius - 1], dets[radius + 1] + for i in range(len(prev[0])): + if prev[0][i] < threshold: + continue + j = _match(prev[1][i], nxt[1]) + if j is None or nxt[0][j] < threshold: + continue + box = (prev[1][i] + nxt[1][j]) / 2 + if any(_match(box, np.array([f[0]])) is not None for f in faces): + continue + faces.append((box, (prev[2][i] + nxt[2][j]) / 2)) + + smoothed = [] + for box, marks in faces: + total, wsum = np.zeros((5, 2)), 0.0 + for k, d in enumerate(dets): + weight = radius + 1 - abs(k - radius) + if k == radius: + total += weight * marks + wsum += weight + continue + j = _match(box, d[1]) + if j is not None and d[0][j] >= low: + total += weight * d[2][j] + wsum += weight + smoothed.append((box, total / wsum)) + faces = smoothed + + faces = [f for f in faces if max(f[0][2] - f[0][0], f[0][3] - f[0][1]) >= min_face] + faces.sort(key=lambda f: -(f[0][2] - f[0][0]) * (f[0][3] - f[0][1])) + return faces[:max_faces] + + +# ---------------------------------------------------------------------------- +# Alignment and warping +# ---------------------------------------------------------------------------- + + +def similarity(src, dst): + """The least-squares similarity transform (rotation, uniform scale, + translation; Umeyama) taking points `src` to `dst`, as a 2x3 matrix.""" + mu_s, mu_d = src.mean(0), dst.mean(0) + s, d = src - mu_s, dst - mu_d + cov = d.T @ s / len(src) + u, sv, vt = np.linalg.svd(cov) + sign = np.eye(2) + if np.linalg.det(u) * np.linalg.det(vt) < 0: + sign[1, 1] = -1 + rot = u @ sign @ vt + scale = np.trace(np.diag(sv) @ sign) / ((s ** 2).sum() / len(src)) + t = mu_d - scale * rot @ mu_s + return np.hstack([scale * rot, t[:, None]]) + + +def invert(m): + a = np.vstack([m, [0.0, 0.0, 1.0]]) + return np.linalg.inv(a)[:2] + + +def bilinear(img, xs, ys, fill): + """Sample `img` (c, h, w) at float pixel coordinates (pixel centres at + integers, as OpenCV's warpAffine), `fill` (c,) outside it.""" + c, h, w = img.shape + x0 = np.floor(xs).astype(np.int64) + y0 = np.floor(ys).astype(np.int64) + fx = (xs - x0).astype(np.float32) + fy = (ys - y0).astype(np.float32) + out = np.zeros((c,) + xs.shape, np.float32) + for dy, wy in ((0, 1 - fy), (1, fy)): + for dx, wx in ((0, 1 - fx), (1, fx)): + xi, yi = x0 + dx, y0 + dy + inside = (xi >= 0) & (xi < w) & (yi >= 0) & (yi < h) + v = img[:, np.clip(yi, 0, h - 1), np.clip(xi, 0, w - 1)] + v = np.where(inside, v, fill[:, None, None] if v.ndim == 3 else fill[:, None]) + out += v * (wx * wy) + return out + + +_grid = None + + +def face_grid(): + global _grid + if _grid is None: + ys, xs = np.mgrid[0:FACE_SIZE, 0:FACE_SIZE].astype(np.float64) + _grid = (xs, ys) + return _grid + + +def align(rgb, m): + """The FACE_SIZE square face `m` (frame -> face) picks out of `rgb`.""" + xs, ys = face_grid() + inv = invert(m) + sx = inv[0, 0] * xs + inv[0, 1] * ys + inv[0, 2] + sy = inv[1, 0] * xs + inv[1, 1] * ys + inv[1, 2] + return bilinear(rgb, sx, sy, BORDER_RGB) + + +_blur = None + + +def _blur_matrix(): + """A FACE_SIZE x FACE_SIZE matrix applying a Gaussian of MASK_SIGMA with + reflected edges (OpenCV's default border), so blur(M) = G @ M @ G.T.""" + global _blur + if _blur is None: + r = int(np.ceil(3 * MASK_SIGMA)) + taps = np.exp(-0.5 * (np.arange(-r, r + 1) / MASK_SIGMA) ** 2) + taps /= taps.sum() + g = np.zeros((FACE_SIZE, FACE_SIZE)) + for i in range(FACE_SIZE): + for k, t in enumerate(taps): + j = i + k - r + if j < 0: + j = -j + if j >= FACE_SIZE: + j = 2 * (FACE_SIZE - 1) - j + g[i, j] += t + _blur = g.astype(np.float32) + return _blur + + +def face_mask(parsing): + """The feathered blend mask (FACE_SIZE square, [0, 1]) for BiSeNet's + logits (1, 19, FACE_SIZE, FACE_SIZE).""" + labels = parsing[0].argmax(0) + mask = np.isin(labels, FACE_CLASSES).astype(np.float32) + g = _blur_matrix() + mask = g @ mask @ g.T + b = MASK_BORDER + mask[:b] = 0 + mask[-b:] = 0 + mask[:, :b] = 0 + mask[:, -b:] = 0 + mask[mask < MASK_FLOOR] = 0 + return mask + + +# ---------------------------------------------------------------------------- +# The pass +# ---------------------------------------------------------------------------- + + +class FaceRestorer: + """The models for one restore_faces() call.""" + + def __init__(self, detector, parser, restorer, model, fidelity): + if model not in MODELS: + raise ValueError("face_restore: unknown model %r" % model) + # One lock for all three models: their runs are serialised (one GPU, + # and concurrent Core ML predictions ran it out of memory), with a CPU + # retry (see ai_runtime.Runner). The NumPy work around them, and the + # rest of the script, stay parallel. + lock = threading.Lock() + self.det = ai_runtime.Runner(detector, lock=lock) + self.parse = ai_runtime.Runner(parser, lock=lock) + self.restore = ai_runtime.Runner(restorer, lock=lock) + self.model = model + self.fidelity = float(fidelity) + self.restore_input = self.restore.get_inputs()[0].name + self.parse_input = self.parse.get_inputs()[0].name + + def restore_face(self, face): + """A (3, 512, 512) RGB face in [0, 1], restored.""" + feeds = {self.restore_input: (face * 2.0 - 1.0)[None].astype(np.float32)} + if self.model == "codeformer": + feeds["w"] = np.array([self.fidelity], np.float32) + out = self.restore.run(None, feeds)[0][0] + return np.clip((out + 1.0) * 0.5, 0.0, 1.0) + + def mask_for(self, face): + x = ((face - PARSE_MEAN) / PARSE_STD)[None].astype(np.float32) + return face_mask(self.parse.run(None, {self.parse_input: x})[0]) + + +def paste(rgb, out, alpha, restored, mask, m): + """Write `restored`/`mask` (face space, `m` frame -> face) into `out` / + `alpha` (frame space) where this face's mask is the strongest so far.""" + _, h, w = rgb.shape + inv = invert(m) + corners = inv @ np.array([[0, 0, 1], [FACE_SIZE, 0, 1], [0, FACE_SIZE, 1], [FACE_SIZE, FACE_SIZE, 1]], np.float64).T + x0 = max(int(np.floor(corners[0].min())) - 1, 0) + x1 = min(int(np.ceil(corners[0].max())) + 2, w) + y0 = max(int(np.floor(corners[1].min())) - 1, 0) + y1 = min(int(np.ceil(corners[1].max())) + 2, h) + if x1 <= x0 or y1 <= y0: + return + ys, xs = np.mgrid[y0:y1, x0:x1].astype(np.float64) + fx = m[0, 0] * xs + m[0, 1] * ys + m[0, 2] + fy = m[1, 0] * xs + m[1, 1] * ys + m[1, 2] + a = bilinear(mask[None], fx, fy, np.zeros(1, np.float32))[0] + a[a < MASK_FLOOR] = 0 + take = a > alpha[y0:y1, x0:x1] + if not take.any(): + return + face = bilinear(restored, fx, fy, BORDER_RGB) + region = out[:, y0:y1, x0:x1] + region[:, take] = face[:, take] + alpha[y0:y1, x0:x1][take] = a[take] + + +# Temporal blending of the restored faces (see restore_faces). Motion is +# judged on the aligned *input* crops, luma averaged over 4x4 blocks: where a +# neighbour's crop differs from this frame's by MOTION_LIMIT (8-bit levels: 15) +# the neighbour contributes nothing, so a moving mouth or eye is not ghosted. +MOTION_BLOCK = 4 +MOTION_LIMIT = 15.0 / 255.0 + + +def motion_crop(face): + """A face crop's luma at 1/MOTION_BLOCK size, for comparing neighbours.""" + y = 0.299 * face[0] + 0.587 * face[1] + 0.114 * face[2] + b = MOTION_BLOCK + return y.reshape(FACE_SIZE // b, b, FACE_SIZE // b, b).mean((1, 3)).astype(np.float32) + + +def motion_weight(a, b): + """Per-pixel weight (FACE_SIZE square) for blending a neighbour whose + motion crop is `b` into a face whose motion crop is `a`.""" + d = np.abs(a - b) + # 3x3 box blur, edges repeated: one noisy block is not motion. + p = np.pad(d, 1, mode="edge") + d = sum(p[i:i + d.shape[0], j:j + d.shape[1]] for i in range(3) for j in range(3)) / 9.0 + w = np.clip(1.0 - d / MOTION_LIMIT, 0.0, 1.0) + return np.repeat(np.repeat(w, MOTION_BLOCK, 0), MOTION_BLOCK, 1) + + +def restore_faces( + clip, + detector, + parser, + restorer, + model="restoreformer", + strength=1.0, + fidelity=0.7, + threshold=0.8, + min_face=32, + max_faces=8, + temporal_radius=1, + temporal_blend=True, + matrix="709", + color_range="limited", +): + """Restore the faces in `clip` (any format) and return it in its own + format. + + `strength` (0-1) scales the blend; `fidelity` is CodeFormer's w. + `threshold` is the detector score a face needs, `min_face` the size (in + pixels, the larger side of its box) below which a face is left alone, + `max_faces` how many of the largest are restored per frame. `matrix` / + `color_range` are the source's (zimg names), for the conversion to RGB and + back. + + `temporal_radius` r > 0 stabilises the faces over time, in two steps: each + face's landmarks are smoothed over the detections r frames either side + (select_faces), and each restored face is averaged, in the aligned 512x512 + space where neighbouring frames' faces line up, with the restored faces + it matches in the r frames either side — weighted down to nothing where + the input itself moved (motion_weight). A restorer re-invents fine + texture for every frame from noisy input; the average keeps what the + frames agree on. `temporal_blend=False` keeps only the landmark smoothing. + Each output frame reads 2r frames either side (r for the blend, and r + more for those frames' landmark smoothing).""" + import vapoursynth as vs + + core = vs.core + fmt = clip.format + if fmt is None: + raise ValueError("face_restore needs a constant-format clip") + r = max(0, int(temporal_radius)) + blend_r = r if temporal_blend else 0 + strength = min(max(float(strength), 0.0), 1.0) + models = FaceRestorer(detector, parser, restorer, model, fidelity) + last_restorer.clear() + last_restorer.update(detector=models.det, parser=models.parse, restorer=models.restore) + import sys + + sys.stderr.write("Face restoration: %s on %s%s\n" % ( + model, models.restore.provider, + "" if not models.restore.fallback else " (%s)" % models.restore.fallback)) + + rgbs = ai_runtime.to_rgbs(clip, matrix, color_range) + scale = min(1.0, DET_SIZE / max(clip.width, clip.height)) + dw, dh = max(1, round(clip.width * scale)), max(1, round(clip.height * scale)) + small = rgbs if (dw, dh) == (clip.width, clip.height) else core.resize.Bicubic(rgbs, width=dw, height=dh) + det_scale = (dw / clip.width, dh / clip.height) + floor = threshold * 0.75 + last = clip.num_frames - 1 + + def clamp(i): + return min(max(i, 0), last) + + # The frames each request reads: the RGB frames the blend needs (n-b..n+b) + # and the detection copies their landmark smoothing needs (n-b-r..n+b+r). + det_reach = blend_r + r + rgb_offsets = list(range(-blend_r, blend_r + 1)) + det_offsets = list(range(-det_reach, det_reach + 1)) + inputs = [ai_runtime.shifted(rgbs, k) for k in rgb_offsets] + [ai_runtime.shifted(small, k) for k in det_offsets] + + detections = ai_runtime.FrameMemo(capacity=64) + faces_memo = ai_runtime.FrameMemo(capacity=48) + results = ai_runtime.FrameMemo(capacity=16) + + def rgb_frame(frames, n, k): + return frames[rgb_offsets.index(k)] + + def det_frame(frames, n, k): + return frames[len(rgb_offsets) + det_offsets.index(k)] + + def detections_for(index, frame): + def run(): + rgb = np.stack([np.asarray(frame[p]) for p in range(3)]) + return detect(models.det, rgb, det_scale, floor) + + return detections.get(index, run) + + def faces_for(n, frames, k): + """Stage 1 for frame clamp(n + k): its faces, each aligned, restored + and parsed, from that frame and the detections r either side of it.""" + index = clamp(n + k) + + def run(): + dets = [detections_for(clamp(index + j), det_frame(frames, n, k + j)) for j in range(-r, r + 1)] + chosen = select_faces(dets, r, threshold, min_face, max_faces) + src = rgb_frame(frames, n, k) + rgb = np.stack([np.asarray(src[p]) for p in range(3)]).astype(np.float32) + out = [] + for box, marks in chosen: + m = similarity(marks, FFHQ_TEMPLATE) + face = align(rgb, m) + restored = models.restore_face(face) + out.append({ + "box": box, + "m": m, + "motion": motion_crop(face), + "restored": restored.astype(np.float16), + "mask": models.mask_for(restored).astype(np.float16), + }) + return out + + return faces_memo.get(index, run) + + def compute(n, frames): + def run(): + here = faces_for(n, frames, 0) + src = rgb_frame(frames, n, 0) + rgb = np.stack([np.asarray(src[p]) for p in range(3)]).astype(np.float32) + out = rgb.copy() + alpha = np.zeros(rgb.shape[1:], np.float32) + neighbours = [(k, faces_for(n, frames, k)) for k in rgb_offsets if k != 0 and clamp(n + k) != n] + for face in here: + restored = face["restored"].astype(np.float32) + if neighbours: + total = restored * float(blend_r + 1) + weight = np.full(restored.shape[1:], float(blend_r + 1), np.float32) + for k, faces in neighbours: + j = _match(face["box"], np.array([f["box"] for f in faces])) if faces else None + if j is None: + continue + w = (blend_r + 1 - abs(k)) * motion_weight(face["motion"], faces[j]["motion"]) + total += w * faces[j]["restored"].astype(np.float32) + weight += w + restored = total / weight + paste(rgb, out, alpha, restored, face["mask"].astype(np.float32), face["m"]) + # Keep only the region the faces reach: a whole 1080p RGB frame + # is 25 MB, and the memo holds several. + ys, xs = np.nonzero(alpha) + if not len(ys): + return None, None, None, len(here) + y0, y1, x0, x1 = ys.min(), ys.max() + 1, xs.min(), xs.max() + 1 + return (y0, y1, x0, x1), out[:, y0:y1, x0:x1].copy(), alpha[y0:y1, x0:x1] * strength, len(here) + + return results.get(n, run) + + def restored_frame(n, f): + region, patch, _alpha, count = compute(n, f[1:]) + dst = f[0].copy() + if region is not None: + y0, y1, x0, x1 = region + for p in range(3): + np.asarray(dst[p])[y0:y1, x0:x1] = patch[p] + dst.props["FaceRestoreFaces"] = count + return dst + + def mask_frame(n, f): + region, _patch, alpha, _count = compute(n, f[1:]) + dst = f[0].copy() # a blank (zero) frame + if region is not None: + y0, y1, x0, x1 = region + plane = np.asarray(dst[0]) + if mask_fmt.sample_type == vs.FLOAT: + plane[y0:y1, x0:x1] = alpha + else: + peak = (1 << mask_fmt.bits_per_sample) - 1 + plane[y0:y1, x0:x1] = np.rint(alpha * peak).astype(plane.dtype) + return dst + + mask_fmt = core.query_video_format(vs.GRAY, fmt.sample_type, fmt.bits_per_sample, 0, 0) + blank = core.std.BlankClip(clip, format=mask_fmt.id, keep=True) + restored = core.std.ModifyFrame(rgbs, [rgbs] + inputs, restored_frame) + mask = core.std.ModifyFrame(blank, [blank] + inputs, mask_frame) + restored = ai_runtime.from_rgbs(restored, fmt, matrix, color_range) + # Composited in the source's format, so the mask's zeros leave the input + # bit-exact. The mask is luma-sized; MaskedMerge resamples it for + # subsampled chroma. + merged = core.std.MaskedMerge(clip, restored, mask, first_plane=True) + return core.std.CopyFrameProps(merged, clip) diff --git a/worker/templates/pipeline_template.vpy b/worker/templates/pipeline_template.vpy index df6f311c..a60f3ac9 100644 --- a/worker/templates/pipeline_template.vpy +++ b/worker/templates/pipeline_template.vpy @@ -24,6 +24,15 @@ import os sys.path.insert(0, r"{{PIPE_SOURCE_DIR}}") from pipe_source import create_pipe_clip +{{#AI_RUNTIME}} +# ONNX Runtime from the ai-runtime add-on (Settings -> Add-ons), for passes +# that run a model through ai_runtime.session() / ai_runtime.map_frames(). +# ai_runtime.py sits beside pipe_source.py and puts the add-on's packages on +# sys.path itself: Windows' embeddable Python ignores PYTHONPATH. +import ai_runtime +ai_runtime.setup({{AI_RUNTIME_DIR}}) +{{/AI_RUNTIME}} + clip = create_pipe_clip( width={{INPUT_WIDTH}}, height={{INPUT_HEIGHT}}, @@ -448,6 +457,75 @@ clip = core.vivtc.VFM(_ivtc_metrics, order={{IVTC_ORDER}}, blocky={{IVTC_BLOCK_Y}}, {{/IVTC_BLOCK_Y}} clip2=_ivtc_src) +{{#DEINT_IVTC_FIX_FADES}} +# Fades added after telecine (vs-jetpack's FixInterlacedFades, MIT, ported from +# release v3.0.0; upstream calls it an improved Fix-Telecined-Fades). A fade +# applied to the telecined video changes brightness between the two fields of +# a frame, so no field match is clean and the fade combs. Each field is scaled +# about black (neutral for chroma) so its mean distance from it equals the +# {{IVTC_FADE_MODE}} of the two fields': exact for fades to or from black, and +# blur-free. On frames that are not fading the two fields' means agree and the +# change is negligible (measured ~0.1 levels). Float, as upstream requires. +# +# The per-frame gains are stamped on each field as frame properties by +# ModifyFrame and applied by ONE akarin expression that reads them. Never +# create an akarin node per frame inside a FrameEval here: with two such +# FrameEvals running concurrently under vspipe, the job aborted (SIGABRT) or +# spun one core indefinitely (a 97-minute hang on 30 frames, 2026-11-08), and +# ran fine single-threaded. macOS x64 has no akarin (so no frame-property +# Expr); there std.Expr nodes are made per frame instead, which does not hang. +_fade_fmt = clip.format +_fade_f = core.resize.Point(clip, format=_fade_fmt.replace(sample_type=vs.FLOAT, bits_per_sample=32).id) +_fade_fields = core.std.SeparateFields(core.std.SetFieldBased(_fade_f, 0), tff=True) +_fade_dist = _expr([_fade_fields], 'x abs') +for _fade_i in range(_fade_fmt.num_planes): + _fade_dist = core.std.PlaneStats(_fade_dist, plane=_fade_i, prop=f'P{_fade_i}') + + +def _fade_gains(own, other): + gains = [] + for i in range(_fade_fmt.num_planes): + a, b = own.props[f'P{i}Average'], other.props[f'P{i}Average'] + target = (a + b) / 2 if '{{IVTC_FADE_MODE}}' == 'average' else min(a, b) + gains.append(target / a if a > 1e-6 else 1.0) + return gains + + +def _fade_stamp(n, f, own, other): + out = f[0].copy() + for i, g in enumerate(_fade_gains(f[own], f[other])): + out.props[f'FadeGain{i}'] = g + return out + + +def _fade_scale(n, f, c, own, other): + gains = _fade_gains(f[own], f[other]) + if all(abs(g - 1.0) < 1e-6 for g in gains): + return c + return _expr([c], [f'x {g} *' for g in gains]) + + +_fade_top, _fade_bot = _fade_fields[::2], _fade_fields[1::2] +_fade_props = [_fade_dist[::2], _fade_dist[1::2]] +if _akarin_expr is not None: + _fade_top = core.std.ModifyFrame(_fade_top, [_fade_top] + _fade_props, + functools.partial(_fade_stamp, own=1, other=2)) + _fade_bot = core.std.ModifyFrame(_fade_bot, [_fade_bot] + _fade_props, + functools.partial(_fade_stamp, own=2, other=1)) + _fade_apply = [f'x x.FadeGain{i} *' for i in range(_fade_fmt.num_planes)] + _fade_top = _expr([_fade_top], _fade_apply) + _fade_bot = _expr([_fade_bot], _fade_apply) +else: + _fade_top = core.std.FrameEval(_fade_top, functools.partial(_fade_scale, c=_fade_top, own=0, other=1), + prop_src=_fade_props) + _fade_bot = core.std.FrameEval(_fade_bot, functools.partial(_fade_scale, c=_fade_bot, own=1, other=0), + prop_src=_fade_props) +clip = core.std.DoubleWeave(core.std.Interleave([_fade_top, _fade_bot]), tff=True)[::2] +# DoubleWeave flags its output interlaced; field-matched frames are +# progressive, and a later resize would otherwise treat them as fields. +clip = core.std.SetFieldBased(clip, 0) +clip = core.resize.Point(clip, format=_fade_fmt.id, dither_type="error_diffusion") +{{/DEINT_IVTC_FIX_FADES}} # VFM output is full-depth again; VDecimate accepts 8..16 bit natively. clip = core.vivtc.VDecimate(clip, {{#IVTC_CYCLE}} @@ -972,6 +1050,20 @@ clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_ if {{NR_KNLM_CHROMA}} and clip.format.color_family == vs.YUV: clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_S}}, h={{NR_KNLM_H}}, channels="UV", device_type="{{NR_KNLM_DEVICE}}") {{/NR_KNLM}} +{{#NR_NLM}} +# NLMeans on the CPU (vszip >= 24.0.0): a port of KNLMeansCL with its +# arguments and defaults, so it needs no OpenCL device. Same output as +# KNLMeansCL for the same arguments (measured: identical to the level on the +# same input). Every depth natively; h is depth-independent (an 8-bit and a +# 10-bit run agree to ~0.2 levels). Chroma in its own pass, as for KNLMeansCL: +# channels="YUV" needs 4:4:4. +if clip.format.color_family == vs.YUV: + clip = core.vszip.NLMeans(clip, d={{NR_NLM_D}}, a={{NR_NLM_A}}, s={{NR_NLM_S}}, h={{NR_NLM_H}}, channels="Y") + if {{NR_NLM_CHROMA}}: + clip = core.vszip.NLMeans(clip, d={{NR_NLM_D}}, a={{NR_NLM_A}}, s={{NR_NLM_S}}, h={{NR_NLM_H}}, channels="UV") +else: + clip = core.vszip.NLMeans(clip, d={{NR_NLM_D}}, a={{NR_NLM_A}}, s={{NR_NLM_S}}, h={{NR_NLM_H}}) +{{/NR_NLM}} {{#NR_MCLEAN}} # mClean — denoise, then put detail and grain back, so the result does not look # plastic. The only method here that is a goal rather than a mechanism. @@ -1238,6 +1330,53 @@ clip = haf.HQDeringmod( {{/DEHALO_DERING_DARKTHR}} ) {{/DEHALO_HQDERING}} +{{#DEHALO_VINE}} +# vine: dehalo by non-local-means filtering of the error (vs-jetpack's +# vine_dehalo, MIT, ported from release v3.0.0). Luma only, as upstream. +# 1. NLMeans (single-pixel similarity, s=0) on a 2x NNEDI3 supersample and on +# the picture itself, blended by a weight derived from `sharp`; +# 2. keep the source's low frequencies and the smoothed picture's high ones +# (a Gaussian split at `sigma`); +# 3. NLMeans the difference between the source and that, guided by it, and +# add it back: the halo goes, the edge detail it was riding on returns. +# The constants are upstream's. NLMeans' h is depth-independent, so nothing is +# rescaled for >8-bit sources (8-bit and 10-bit runs agree to within rounding). +import math as _vine_math + + +def _vine_gauss(c, sigma): + r = max(1, min(12, int(_vine_math.ceil(sigma * 3)))) + k = [_vine_math.exp(-(x * x) / (2.0 * sigma * sigma)) for x in range(-r, r + 1)] + return core.std.Convolution(core.std.Convolution(c, matrix=k, mode='h'), matrix=k, mode='v') + + +_vine_strength = {{DEHALO_VINE_STRENGTH}} +_vine_sharp = {{DEHALO_VINE_SHARP}} +_vine_sigma = {{DEHALO_VINE_SIGMA}} +_vine_c0 = 0.3926327792690057290863679493724 * min(_vine_sharp, 1.0) +_vine_weight = _vine_c0 * _vine_math.log(1 + 1 / _vine_c0) +_vine_h_refine = 18.880334973195822973214959957208 * ( + _vine_strength / 18.880334973195822973214959957208) ** 0.5862453661304626725671053478676 +_vine_y = core.std.ShufflePlanes(clip, 0, vs.GRAY) if clip.format.num_planes > 1 else clip +# 2x supersample: NNEDI3 keeping the source lines on the even rows (field=1), +# then the half-pixel shift that leaves is folded into the Catrom downscale. +_vine_ss = _nnedi3(_vine_y, field=1, dh=True) +_vine_ss = core.std.Transpose(_nnedi3(core.std.Transpose(_vine_ss), field=1, dh=True)) +_vine_ss = core.vszip.NLMeans(_vine_ss, d=0, a=2, s=0, h=_vine_strength) +_vine_ss = core.resize.Bicubic(_vine_ss, _vine_y.width, _vine_y.height, filter_param_a=0, + filter_param_b=0.5, src_left=-0.5, src_top=-0.5) +_vine_sm = core.vszip.NLMeans(_vine_y, d=0, a=2, s=0, h=_vine_strength) +_vine_sm = core.std.Merge(_vine_ss, _vine_sm, _vine_weight) +_vine_high = core.std.MergeDiff(_vine_gauss(_vine_y, _vine_sigma), + core.std.MakeDiff(_vine_sm, _vine_gauss(_vine_sm, _vine_sigma))) +_vine_ref = core.vszip.NLMeans(core.std.MakeDiff(_vine_y, _vine_high), d=0, a=2, s=4, + h=_vine_h_refine, rclip=_vine_high) +_vine_y = core.std.MergeDiff(_vine_high, _vine_ref) +if clip.format.num_planes > 1: + clip = core.std.ShufflePlanes([_vine_y, clip], [0, 1, 2], clip.format.color_family) +else: + clip = _vine_y +{{/DEHALO_VINE}} {{/DEHALO}} # ============================================================================ @@ -1300,6 +1439,64 @@ clip = core.dctf.DCTFilter( planes={{DEBLOCK_DCT_PLANES}}, ) {{/DEBLOCK_DCTFILTER}} +{{#DEBLOCK_MPEG2STINX}} +# mpeg2stinx: removes the comb-like noise a film DVD picks up when its +# progressive frames were encoded as interlaced MPEG-2 (vs-jetpack's +# mpeg2stinx, MIT, ported from release v3.0.0, with its defaults: NNEDI3 bob +# motion-masked by BWDIF, radius 2, limit 1.0). Each field is clamped to the +# local range of a bob of the other field; that is done twice, each time +# limited to how much the frame differs from its neighbours, and the two +# results averaged. It works on the whole frame by design (it is a cross-field +# filter), so it never takes the per-field path. Every operation is a min/max +# or a ratio, so nothing needs rescaling for >8-bit sources. +_stx_tff = {{DEBLOCK_STINX_TFF}} +_stx_field = 3 if _stx_tff else 2 + + +def _stx_morpho(c, rx, ry, grow): + # vsmasktools' Morpho.expand/inpand: a (2rx+1)x(2ry+1) rectangle. + f = core.std.Maximum if grow else core.std.Minimum + for i in range(max(rx, ry)): + h, v = i < rx, i < ry + coords = [1] * 8 if (h and v) else ([0, 0, 0, 1, 1, 0, 0, 0] if h else [0, 1, 0, 0, 0, 0, 1, 0]) + c = f(c, coordinates=coords) + return c + + +def _stx_weave(c): + return core.std.DoubleWeave(c, tff=_stx_tff)[::2] + + +def _stx_fields(c): + # _FieldBased=0 so SeparateFields honours tff (a set 1/2 overrides it). + return core.std.SeparateFields(core.std.SetFieldBased(c, 0), tff=_stx_tff) + + +def _stx_bob(c): + c = core.std.SetFieldBased(c, 2 if _stx_tff else 1) + return core.bwdif.Bwdif(c, field=_stx_field, edeint=_nnedi3(c, field=_stx_field)) + + +def _stx_crossfield(c, bobbed): + lo, hi = _stx_morpho(bobbed, 2, 2, False), _stx_morpho(bobbed, 2, 2, True) + rep = _expr([core.std.Interleave([c, c]), lo, hi], 'x y max z min') + return _stx_weave(core.std.SelectEvery(_stx_fields(rep), 4, (2, 1))) + + +def _stx_limit(src, flt, adj): + d = _expr([core.std.Interleave([src, src]), adj], 'x y - abs') + d = _stx_fields(d) + d = _expr([core.std.SelectEvery(d, 4, (0, 1)), core.std.SelectEvery(d, 4, (2, 3))], 'x y min') + d = _stx_weave(_stx_morpho(d, 2, 1, True)) + return _expr([flt, src, d], 'x y z - max y z + min') + + +clip = core.std.SetFieldBased(clip, 0) +_stx_adj = core.std.Interleave([clip[0] + clip[:-1], clip[1:] + clip[-1]]) +_stx_1 = _stx_limit(clip, _stx_crossfield(clip, _stx_bob(clip)), _stx_adj) +_stx_2 = _stx_limit(_stx_1, _stx_crossfield(_stx_1, _stx_bob(_stx_1)), _stx_adj) +clip = core.std.SetFieldBased(core.std.Merge(_stx_1, _stx_2), 0) +{{/DEBLOCK_MPEG2STINX}} {{#DEBLOCK_PER_FIELD}} if _deblock_per_field: clip = core.std.DoubleWeave(clip, tff={{DEBLOCK_TFF}})[::2] @@ -1369,9 +1566,11 @@ clip = haf.daa(clip) {{#AA_SANTIAG}} # santiag - stronger anti-aliasing with independent horizontal and vertical -# strength, so it can be aimed at one axis. `type` is pinned to nnedi3: havsfunc -# also accepts eedi2 and sangnom, neither of which is in the deps bundle, and -# naming an absent one fails at script evaluation with a bare "no attribute". +# strength, so it can be aimed at one axis. `type` is pinned to nnedi3: eedi2 is +# not in the deps bundle, and havsfunc's `sangnom` type calls SangNom with +# order=0, which SangNom r45 treats as double-rate (it refuses the clip without +# _FieldBased, and would double the frame count with it). SangNom has its own +# method below. clip = haf.santiag( clip, strh={{AA_STRH}}, @@ -1379,6 +1578,21 @@ clip = haf.santiag( type="{{AA_TYPE}}", ) {{/AA_SANTIAG}} +{{#AA_SANGNOM}} +# SangNom AA: supersample 2x, run SangNom along each axis keeping the even +# lines (order=1: same rate, no shift) and come back down. The sharpest of +# the three; at 1080p the slowest (8.6 fps against daa's 15.9, M1 Pro), since it +# works at twice the size. `aa` +# is in 8-bit units at every depth: the plugin scales it itself (an 8-bit and +# a 10-bit run agree to 0.05 levels). SangNom flags its output progressive, so +# the field order set above is put back. +_aa_w, _aa_h = clip.width, clip.height +_aa = core.resize.Spline36(clip, _aa_w * 2, _aa_h * 2) +_aa = core.sangnom.SangNom(_aa, order=1, aa={{AA_SANGNOM_AA}}) +_aa = core.sangnom.SangNom(core.std.Transpose(_aa), order=1, aa={{AA_SANGNOM_AA}}) +clip = core.resize.Spline36(core.std.Transpose(_aa), _aa_w, _aa_h) +clip = core.std.SetFieldBased(clip, {{AA_FIELD_BASED_VALUE}}) +{{/AA_SANGNOM}} {{/ANTI_ALIAS}} # ============================================================================ @@ -1570,6 +1784,31 @@ clip = haf.Vinverse( {{/CHROMA_VINVERSE}} {{/CHROMA_FIXES}} +# ============================================================================ +# PASS: COLOURISATION +# ============================================================================ +# DDColor or DeOldify Video predicts colour from the luma, at a fixed square +# size; the colour is scaled up to the frame, converted to this clip's YUV with +# the source's matrix, and only its chroma is kept: the Y plane is the input's, +# bit-exact (any colour the source had is replaced). After the cleanup and +# Chroma Fixes, before Colour Correction, so the result can be graded. ONNX +# Runtime comes from the ai-runtime add-on (the AI runtime block above) and the +# models are add-ons; colorize.py sits beside pipe_source.py. Each frame's +# colour is averaged with its neighbours' where the picture did not move. +{{#COLORIZE}} +import colorize as _colorize +clip = _colorize.colorize( + clip, + model_path={{CZ_MODEL_PATH}}, + model="{{CZ_MODEL}}", + render_size={{CZ_RENDER_SIZE}}, + strength={{CZ_STRENGTH}}, + temporal_radius={{CZ_RADIUS}}, + matrix="{{CZ_MATRIX}}", + color_range="{{CZ_RANGE}}", +) +{{/COLORIZE}} + # ============================================================================ # PASS 9: COLOR CORRECTION # ============================================================================ @@ -2049,14 +2288,16 @@ clip = _upscale_fix_shift(clip, _upscale_shift) clip = core.resize.Spline36(clip, width=clip.width * {{UPSCALE_FACTOR}}, height=clip.height * {{UPSCALE_FACTOR}}) {{/UPSCALE_SPLINE36}} {{#UPSCALE_AI}} -# AI upscaling: Real-ESRGAN / Real-CUGAN through vs-mlrt's VSNCNN (ncnn on -# Vulkan), loaded explicitly from vapoursynth/ncnn/ (never autoloaded) after -# MoltenVK on macOS, exactly as RIFE is. The networks take float RGB, so the -# source's matrix and range convert there and back, and the result is dithered -# to the source format. Above SD the frame runs in overlapping 512-pixel tiles: -# whole-frame Real-CUGAN at 1080p ran out of GPU memory on a 16 GB Mac. An odd -# size is padded to even by repeating the edge (CUGAN works in 2-pixel steps) -# and cropped after. fp16 differs from fp32 by at most 0.3/255 and is ~3x faster. +# AI upscaling: Real-ESRGAN / Real-CUGAN (and the add-on models) through +# vs-mlrt's VSNCNN (ncnn on Vulkan), loaded explicitly from vapoursynth/ncnn/ +# (never autoloaded) after MoltenVK on macOS, exactly as RIFE is. The networks +# take float RGB, so the source's matrix and range convert there and back, and +# the result is dithered to the source format. Above a size the frame runs in +# overlapping tiles: whole-frame Real-CUGAN at 1080p ran out of GPU memory on a +# 16 GB Mac (512-pixel tiles above SD), and the large Real-ESRGAN x4plus lost +# the device at SD (384-pixel tiles at 4x, any size). An odd size is padded to even by +# repeating the edge (CUGAN works in 2-pixel steps) and cropped after. fp16 +# differs from fp32 by at most 0.3/255 and is ~3x faster. import ctypes as _ai_ctypes _ai_preload = {{AI_PRELOAD}} if _ai_preload: @@ -2084,17 +2325,30 @@ for _ai_model in [{{AI_MODELS}}]: # AnimeJaNai: its SD model for a picture at most {{AI_SD_MAX_HEIGHT}} tall, else # the HD one (trained on 1080p anime), chosen per doubling. _ai_model = _ai_model[0] if _ai_rgb.height <= {{AI_SD_MAX_HEIGHT}} else _ai_model[1] - _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= 720 * 576 else dict(tilesize=[512, 512], overlap=[16, 16]) + # A tile larger than the frame in either direction segfaults VSNCNN (a + # wide, short frame can pass the area test), so it never exceeds the frame. + _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= {{AI_TILE_ABOVE}} else dict( + tilesize=[min({{AI_TILE}}, _ai_rgb.width), min({{AI_TILE}}, _ai_rgb.height)], overlap=[16, 16]) _ai_rgb = core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True, num_streams=2, **_ai_tiles) _ai_scale = _ai_rgb.width // _ai_in_w if _ai_pad_w or _ai_pad_h: _ai_rgb = core.std.Crop(_ai_rgb, right=_ai_pad_w * _ai_scale, bottom=_ai_pad_h * _ai_scale) +{{#AI_RESAMPLE}} +# The network enlarges further than asked (Real-ESRGAN General is 4x only), so +# resample to {{AI_FACTOR}}x here, in float RGB, before the conversion back. +_ai_rgb = core.resize.Spline36(_ai_rgb, width=_ai_rgb.width * {{AI_FACTOR}} // _ai_scale, height=_ai_rgb.height * {{AI_FACTOR}} // _ai_scale) +{{/AI_RESAMPLE}} if _ai_fmt.color_family == vs.YUV: clip = core.resize.Spline36(_ai_rgb, format=_ai_fmt.id, matrix_s="{{AI_MATRIX}}", range_s="{{AI_RANGE}}", dither_type="error_diffusion") elif _ai_fmt.color_family == vs.GRAY: clip = core.resize.Point(core.std.ShufflePlanes(_ai_rgb, 1, vs.GRAY), format=_ai_fmt.id, range_s="{{AI_RANGE}}", dither_type="error_diffusion") else: clip = core.resize.Point(_ai_rgb, format=_ai_fmt.id, dither_type="error_diffusion") +{{#AI_STRENGTH}} +# AI strength: keep {{AI_STRENGTH}} of the network's picture and blend in a Spline36 +# upscale of the same frame for the rest, at the clip's own format. +clip = core.std.Merge(core.resize.Spline36(_ai_src, width=clip.width, height=clip.height), clip, weight={{AI_STRENGTH}}) +{{/AI_STRENGTH}} # The network drops frame properties; field order, matrix and the rest are the source's. clip = core.std.CopyFrameProps(clip, _ai_src) {{/UPSCALE_AI}} @@ -2174,6 +2428,69 @@ _aspect_box_h = _even(_box_h) if _box_h > 0 else 0 {{/RESIZE_STANDARD}} {{/RESIZE}} +# ============================================================================ +# BT.601 -> BT.709 (Crop & Resize, app 1.7.0) +# ============================================================================ +{{#COLOR_709}} +# An SD source keeps its BT.601 matrix through an upscale unless something +# converts it, and many players and TVs ignore the tag on an HD-sized picture +# and decode it as BT.709 - a 75% green then shows as 161 instead of 191. So +# the samples are converted here, and the worker declares BT.709 on both sides +# of the encoder to match (VideoJob::output_color_metadata, the one decision). +# +# The matrix only: the primaries keep the source's, and so does its tag (see +# ColorMetadata::bt709_converted for the measurement behind that). zimg works +# through 4:4:4 float and the result is dithered back to the clip's own +# format; _FieldBased keeps the chroma resampling per field on a picture that +# is still interlaced. +# +# VapourSynth R78's resizers let a frame's own _Matrix / _Range properties +# override matrix_in_s / range_in_s, so they are removed first: the worker's +# reading of the source's tags is what decides the conversion. +if clip.format.color_family == vs.YUV: + clip = core.std.RemoveFrameProps(clip, props=["_Matrix", "_Range", "_ColorRange"]) + clip = core.resize.Spline36( + clip, + format=clip.format.id, + matrix_in_s="{{C709_MATRIX_IN}}", + matrix_s="709", + range_in_s="{{C709_RANGE}}", + range_s="{{C709_RANGE}}", + dither_type="error_diffusion", + ) +{{/COLOR_709}} + +# ============================================================================ +# PASS: FACE RESTORATION +# ============================================================================ +# Each face is found (RetinaFace), aligned to the FFHQ template, restored +# (RestoreFormer++, GFPGAN or CodeFormer), masked to the face itself (BiSeNet) +# and blended back at this resolution: after the resize, so an upscale gives the +# restorer more pixels and no later resample softens what it recovered; before +# sharpening and grain. ONNX Runtime comes from the ai-runtime add-on (the AI +# runtime block above) and the models are add-ons; face_restore.py sits beside +# pipe_source.py. Landmarks are smoothed over the detections either side, and +# the faces are merged in the source format, so every pixel outside them is the +# input, bit-exact. +{{#FACE_RESTORE}} +import face_restore as _face_restore +clip = _face_restore.restore_faces( + clip, + detector={{FR_DETECTOR}}, + parser={{FR_PARSER}}, + restorer={{FR_RESTORER}}, + model="{{FR_MODEL}}", + strength={{FR_STRENGTH}}, + fidelity={{FR_FIDELITY}}, + threshold={{FR_THRESHOLD}}, + min_face={{FR_MIN_FACE}}, + max_faces={{FR_MAX_FACES}}, + temporal_radius={{FR_RADIUS}}, + matrix="{{FR_MATRIX}}", + color_range="{{FR_RANGE}}", +) +{{/FACE_RESTORE}} + # ============================================================================ # PASS: SHARPEN # ============================================================================ diff --git a/worker/templates/preview_template.vpy b/worker/templates/preview_template.vpy index 69bef234..e5764690 100644 --- a/worker/templates/preview_template.vpy +++ b/worker/templates/preview_template.vpy @@ -22,6 +22,15 @@ core.max_cache_size = {{MAX_CACHE_SIZE}} sys.path.insert(0, r"{{PIPE_SOURCE_DIR}}") from pipe_source import create_pipe_clip +{{#AI_RUNTIME}} +# ONNX Runtime from the ai-runtime add-on (Settings -> Add-ons), for passes +# that run a model through ai_runtime.session() / ai_runtime.map_frames(). +# ai_runtime.py sits beside pipe_source.py and puts the add-on's packages on +# sys.path itself: Windows' embeddable Python ignores PYTHONPATH. +import ai_runtime +ai_runtime.setup({{AI_RUNTIME_DIR}}) +{{/AI_RUNTIME}} + clip = create_pipe_clip( width={{INPUT_WIDTH}}, height={{INPUT_HEIGHT}}, num_frames={{TOTAL_FRAMES}}, fps_num={{FPS_NUM}}, @@ -417,6 +426,75 @@ clip = core.vivtc.VFM(_ivtc_metrics, order={{IVTC_ORDER}}, blocky={{IVTC_BLOCK_Y}}, {{/IVTC_BLOCK_Y}} clip2=_ivtc_src) +{{#DEINT_IVTC_FIX_FADES}} +# Fades added after telecine (vs-jetpack's FixInterlacedFades, MIT, ported from +# release v3.0.0; upstream calls it an improved Fix-Telecined-Fades). A fade +# applied to the telecined video changes brightness between the two fields of +# a frame, so no field match is clean and the fade combs. Each field is scaled +# about black (neutral for chroma) so its mean distance from it equals the +# {{IVTC_FADE_MODE}} of the two fields': exact for fades to or from black, and +# blur-free. On frames that are not fading the two fields' means agree and the +# change is negligible (measured ~0.1 levels). Float, as upstream requires. +# +# The per-frame gains are stamped on each field as frame properties by +# ModifyFrame and applied by ONE akarin expression that reads them. Never +# create an akarin node per frame inside a FrameEval here: with two such +# FrameEvals running concurrently under vspipe, the job aborted (SIGABRT) or +# spun one core indefinitely (a 97-minute hang on 30 frames, 2026-11-08), and +# ran fine single-threaded. macOS x64 has no akarin (so no frame-property +# Expr); there std.Expr nodes are made per frame instead, which does not hang. +_fade_fmt = clip.format +_fade_f = core.resize.Point(clip, format=_fade_fmt.replace(sample_type=vs.FLOAT, bits_per_sample=32).id) +_fade_fields = core.std.SeparateFields(core.std.SetFieldBased(_fade_f, 0), tff=True) +_fade_dist = _expr([_fade_fields], 'x abs') +for _fade_i in range(_fade_fmt.num_planes): + _fade_dist = core.std.PlaneStats(_fade_dist, plane=_fade_i, prop=f'P{_fade_i}') + + +def _fade_gains(own, other): + gains = [] + for i in range(_fade_fmt.num_planes): + a, b = own.props[f'P{i}Average'], other.props[f'P{i}Average'] + target = (a + b) / 2 if '{{IVTC_FADE_MODE}}' == 'average' else min(a, b) + gains.append(target / a if a > 1e-6 else 1.0) + return gains + + +def _fade_stamp(n, f, own, other): + out = f[0].copy() + for i, g in enumerate(_fade_gains(f[own], f[other])): + out.props[f'FadeGain{i}'] = g + return out + + +def _fade_scale(n, f, c, own, other): + gains = _fade_gains(f[own], f[other]) + if all(abs(g - 1.0) < 1e-6 for g in gains): + return c + return _expr([c], [f'x {g} *' for g in gains]) + + +_fade_top, _fade_bot = _fade_fields[::2], _fade_fields[1::2] +_fade_props = [_fade_dist[::2], _fade_dist[1::2]] +if _akarin_expr is not None: + _fade_top = core.std.ModifyFrame(_fade_top, [_fade_top] + _fade_props, + functools.partial(_fade_stamp, own=1, other=2)) + _fade_bot = core.std.ModifyFrame(_fade_bot, [_fade_bot] + _fade_props, + functools.partial(_fade_stamp, own=2, other=1)) + _fade_apply = [f'x x.FadeGain{i} *' for i in range(_fade_fmt.num_planes)] + _fade_top = _expr([_fade_top], _fade_apply) + _fade_bot = _expr([_fade_bot], _fade_apply) +else: + _fade_top = core.std.FrameEval(_fade_top, functools.partial(_fade_scale, c=_fade_top, own=0, other=1), + prop_src=_fade_props) + _fade_bot = core.std.FrameEval(_fade_bot, functools.partial(_fade_scale, c=_fade_bot, own=1, other=0), + prop_src=_fade_props) +clip = core.std.DoubleWeave(core.std.Interleave([_fade_top, _fade_bot]), tff=True)[::2] +# DoubleWeave flags its output interlaced; field-matched frames are +# progressive, and a later resize would otherwise treat them as fields. +clip = core.std.SetFieldBased(clip, 0) +clip = core.resize.Point(clip, format=_fade_fmt.id, dither_type="error_diffusion") +{{/DEINT_IVTC_FIX_FADES}} {{/DEINT_IVTC}} {{/DEINTERLACE}} @@ -915,6 +993,20 @@ clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_ if {{NR_KNLM_CHROMA}} and clip.format.color_family == vs.YUV: clip = core.knlm.KNLMeansCL(clip, d={{NR_KNLM_D}}, a={{NR_KNLM_A}}, s={{NR_KNLM_S}}, h={{NR_KNLM_H}}, channels="UV", device_type="{{NR_KNLM_DEVICE}}") {{/NR_KNLM}} +{{#NR_NLM}} +# NLMeans on the CPU (vszip >= 24.0.0): a port of KNLMeansCL with its +# arguments and defaults, so it needs no OpenCL device. Same output as +# KNLMeansCL for the same arguments (measured: identical to the level on the +# same input). Every depth natively; h is depth-independent (an 8-bit and a +# 10-bit run agree to ~0.2 levels). Chroma in its own pass, as for KNLMeansCL: +# channels="YUV" needs 4:4:4. +if clip.format.color_family == vs.YUV: + clip = core.vszip.NLMeans(clip, d={{NR_NLM_D}}, a={{NR_NLM_A}}, s={{NR_NLM_S}}, h={{NR_NLM_H}}, channels="Y") + if {{NR_NLM_CHROMA}}: + clip = core.vszip.NLMeans(clip, d={{NR_NLM_D}}, a={{NR_NLM_A}}, s={{NR_NLM_S}}, h={{NR_NLM_H}}, channels="UV") +else: + clip = core.vszip.NLMeans(clip, d={{NR_NLM_D}}, a={{NR_NLM_A}}, s={{NR_NLM_S}}, h={{NR_NLM_H}}) +{{/NR_NLM}} {{#NR_MCLEAN}} # mClean — denoise, then put detail and grain back, so the result does not look # plastic. The only method here that is a goal rather than a mechanism. @@ -1181,6 +1273,53 @@ clip = haf.HQDeringmod( {{/DEHALO_DERING_DARKTHR}} ) {{/DEHALO_HQDERING}} +{{#DEHALO_VINE}} +# vine: dehalo by non-local-means filtering of the error (vs-jetpack's +# vine_dehalo, MIT, ported from release v3.0.0). Luma only, as upstream. +# 1. NLMeans (single-pixel similarity, s=0) on a 2x NNEDI3 supersample and on +# the picture itself, blended by a weight derived from `sharp`; +# 2. keep the source's low frequencies and the smoothed picture's high ones +# (a Gaussian split at `sigma`); +# 3. NLMeans the difference between the source and that, guided by it, and +# add it back: the halo goes, the edge detail it was riding on returns. +# The constants are upstream's. NLMeans' h is depth-independent, so nothing is +# rescaled for >8-bit sources (8-bit and 10-bit runs agree to within rounding). +import math as _vine_math + + +def _vine_gauss(c, sigma): + r = max(1, min(12, int(_vine_math.ceil(sigma * 3)))) + k = [_vine_math.exp(-(x * x) / (2.0 * sigma * sigma)) for x in range(-r, r + 1)] + return core.std.Convolution(core.std.Convolution(c, matrix=k, mode='h'), matrix=k, mode='v') + + +_vine_strength = {{DEHALO_VINE_STRENGTH}} +_vine_sharp = {{DEHALO_VINE_SHARP}} +_vine_sigma = {{DEHALO_VINE_SIGMA}} +_vine_c0 = 0.3926327792690057290863679493724 * min(_vine_sharp, 1.0) +_vine_weight = _vine_c0 * _vine_math.log(1 + 1 / _vine_c0) +_vine_h_refine = 18.880334973195822973214959957208 * ( + _vine_strength / 18.880334973195822973214959957208) ** 0.5862453661304626725671053478676 +_vine_y = core.std.ShufflePlanes(clip, 0, vs.GRAY) if clip.format.num_planes > 1 else clip +# 2x supersample: NNEDI3 keeping the source lines on the even rows (field=1), +# then the half-pixel shift that leaves is folded into the Catrom downscale. +_vine_ss = _nnedi3(_vine_y, field=1, dh=True) +_vine_ss = core.std.Transpose(_nnedi3(core.std.Transpose(_vine_ss), field=1, dh=True)) +_vine_ss = core.vszip.NLMeans(_vine_ss, d=0, a=2, s=0, h=_vine_strength) +_vine_ss = core.resize.Bicubic(_vine_ss, _vine_y.width, _vine_y.height, filter_param_a=0, + filter_param_b=0.5, src_left=-0.5, src_top=-0.5) +_vine_sm = core.vszip.NLMeans(_vine_y, d=0, a=2, s=0, h=_vine_strength) +_vine_sm = core.std.Merge(_vine_ss, _vine_sm, _vine_weight) +_vine_high = core.std.MergeDiff(_vine_gauss(_vine_y, _vine_sigma), + core.std.MakeDiff(_vine_sm, _vine_gauss(_vine_sm, _vine_sigma))) +_vine_ref = core.vszip.NLMeans(core.std.MakeDiff(_vine_y, _vine_high), d=0, a=2, s=4, + h=_vine_h_refine, rclip=_vine_high) +_vine_y = core.std.MergeDiff(_vine_high, _vine_ref) +if clip.format.num_planes > 1: + clip = core.std.ShufflePlanes([_vine_y, clip], [0, 1, 2], clip.format.color_family) +else: + clip = _vine_y +{{/DEHALO_VINE}} {{/DEHALO}} # ============================================================================ @@ -1243,6 +1382,64 @@ clip = core.dctf.DCTFilter( planes={{DEBLOCK_DCT_PLANES}}, ) {{/DEBLOCK_DCTFILTER}} +{{#DEBLOCK_MPEG2STINX}} +# mpeg2stinx: removes the comb-like noise a film DVD picks up when its +# progressive frames were encoded as interlaced MPEG-2 (vs-jetpack's +# mpeg2stinx, MIT, ported from release v3.0.0, with its defaults: NNEDI3 bob +# motion-masked by BWDIF, radius 2, limit 1.0). Each field is clamped to the +# local range of a bob of the other field; that is done twice, each time +# limited to how much the frame differs from its neighbours, and the two +# results averaged. It works on the whole frame by design (it is a cross-field +# filter), so it never takes the per-field path. Every operation is a min/max +# or a ratio, so nothing needs rescaling for >8-bit sources. +_stx_tff = {{DEBLOCK_STINX_TFF}} +_stx_field = 3 if _stx_tff else 2 + + +def _stx_morpho(c, rx, ry, grow): + # vsmasktools' Morpho.expand/inpand: a (2rx+1)x(2ry+1) rectangle. + f = core.std.Maximum if grow else core.std.Minimum + for i in range(max(rx, ry)): + h, v = i < rx, i < ry + coords = [1] * 8 if (h and v) else ([0, 0, 0, 1, 1, 0, 0, 0] if h else [0, 1, 0, 0, 0, 0, 1, 0]) + c = f(c, coordinates=coords) + return c + + +def _stx_weave(c): + return core.std.DoubleWeave(c, tff=_stx_tff)[::2] + + +def _stx_fields(c): + # _FieldBased=0 so SeparateFields honours tff (a set 1/2 overrides it). + return core.std.SeparateFields(core.std.SetFieldBased(c, 0), tff=_stx_tff) + + +def _stx_bob(c): + c = core.std.SetFieldBased(c, 2 if _stx_tff else 1) + return core.bwdif.Bwdif(c, field=_stx_field, edeint=_nnedi3(c, field=_stx_field)) + + +def _stx_crossfield(c, bobbed): + lo, hi = _stx_morpho(bobbed, 2, 2, False), _stx_morpho(bobbed, 2, 2, True) + rep = _expr([core.std.Interleave([c, c]), lo, hi], 'x y max z min') + return _stx_weave(core.std.SelectEvery(_stx_fields(rep), 4, (2, 1))) + + +def _stx_limit(src, flt, adj): + d = _expr([core.std.Interleave([src, src]), adj], 'x y - abs') + d = _stx_fields(d) + d = _expr([core.std.SelectEvery(d, 4, (0, 1)), core.std.SelectEvery(d, 4, (2, 3))], 'x y min') + d = _stx_weave(_stx_morpho(d, 2, 1, True)) + return _expr([flt, src, d], 'x y z - max y z + min') + + +clip = core.std.SetFieldBased(clip, 0) +_stx_adj = core.std.Interleave([clip[0] + clip[:-1], clip[1:] + clip[-1]]) +_stx_1 = _stx_limit(clip, _stx_crossfield(clip, _stx_bob(clip)), _stx_adj) +_stx_2 = _stx_limit(_stx_1, _stx_crossfield(_stx_1, _stx_bob(_stx_1)), _stx_adj) +clip = core.std.SetFieldBased(core.std.Merge(_stx_1, _stx_2), 0) +{{/DEBLOCK_MPEG2STINX}} {{#DEBLOCK_PER_FIELD}} if _deblock_per_field: clip = core.std.DoubleWeave(clip, tff={{DEBLOCK_TFF}})[::2] @@ -1301,9 +1498,11 @@ clip = haf.daa(clip) {{#AA_SANTIAG}} # santiag - stronger anti-aliasing with independent horizontal and vertical -# strength, so it can be aimed at one axis. `type` is pinned to nnedi3: havsfunc -# also accepts eedi2 and sangnom, neither of which is in the deps bundle, and -# naming an absent one fails at script evaluation with a bare "no attribute". +# strength, so it can be aimed at one axis. `type` is pinned to nnedi3: eedi2 is +# not in the deps bundle, and havsfunc's `sangnom` type calls SangNom with +# order=0, which SangNom r45 treats as double-rate (it refuses the clip without +# _FieldBased, and would double the frame count with it). SangNom has its own +# method below. clip = haf.santiag( clip, strh={{AA_STRH}}, @@ -1311,6 +1510,21 @@ clip = haf.santiag( type="{{AA_TYPE}}", ) {{/AA_SANTIAG}} +{{#AA_SANGNOM}} +# SangNom AA: supersample 2x, run SangNom along each axis keeping the even +# lines (order=1: same rate, no shift) and come back down. The sharpest of +# the three; at 1080p the slowest (8.6 fps against daa's 15.9, M1 Pro), since it +# works at twice the size. `aa` +# is in 8-bit units at every depth: the plugin scales it itself (an 8-bit and +# a 10-bit run agree to 0.05 levels). SangNom flags its output progressive, so +# the field order set above is put back. +_aa_w, _aa_h = clip.width, clip.height +_aa = core.resize.Spline36(clip, _aa_w * 2, _aa_h * 2) +_aa = core.sangnom.SangNom(_aa, order=1, aa={{AA_SANGNOM_AA}}) +_aa = core.sangnom.SangNom(core.std.Transpose(_aa), order=1, aa={{AA_SANGNOM_AA}}) +clip = core.resize.Spline36(core.std.Transpose(_aa), _aa_w, _aa_h) +clip = core.std.SetFieldBased(clip, {{AA_FIELD_BASED_VALUE}}) +{{/AA_SANGNOM}} {{/ANTI_ALIAS}} # ============================================================================ @@ -1502,6 +1716,34 @@ clip = haf.Vinverse( {{/CHROMA_VINVERSE}} {{/CHROMA_FIXES}} +# ============================================================================ +# PASS: COLOURISATION +# ============================================================================ +# DDColor or DeOldify Video predicts colour from the luma, at a fixed square +# size; the colour is scaled up to the frame, converted to this clip's YUV with +# the source's matrix, and only its chroma is kept: the Y plane is the input's, +# bit-exact (any colour the source had is replaced). After the cleanup and +# Chroma Fixes, before Colour Correction, so the result can be graded. ONNX +# Runtime comes from the ai-runtime add-on (the AI runtime block above) and the +# models are add-ons; colorize.py sits beside pipe_source.py. Each frame's +# colour is averaged with its neighbours' where the picture did not move. +# preview=True: see colorize.PREVIEW_ON_CPU (Core ML compiles DDColor for over +# a minute in every process, longer than a preview's few frames take on the CPU). +{{#COLORIZE}} +import colorize as _colorize +clip = _colorize.colorize( + clip, + model_path={{CZ_MODEL_PATH}}, + model="{{CZ_MODEL}}", + render_size={{CZ_RENDER_SIZE}}, + strength={{CZ_STRENGTH}}, + temporal_radius={{CZ_RADIUS}}, + matrix="{{CZ_MATRIX}}", + color_range="{{CZ_RANGE}}", + preview=True, +) +{{/COLORIZE}} + # ============================================================================ # PASS 9: COLOR CORRECTION # ============================================================================ @@ -1981,14 +2223,16 @@ clip = _upscale_fix_shift(clip, _upscale_shift) clip = core.resize.Spline36(clip, width=clip.width * {{UPSCALE_FACTOR}}, height=clip.height * {{UPSCALE_FACTOR}}) {{/UPSCALE_SPLINE36}} {{#UPSCALE_AI}} -# AI upscaling: Real-ESRGAN / Real-CUGAN through vs-mlrt's VSNCNN (ncnn on -# Vulkan), loaded explicitly from vapoursynth/ncnn/ (never autoloaded) after -# MoltenVK on macOS, exactly as RIFE is. The networks take float RGB, so the -# source's matrix and range convert there and back, and the result is dithered -# to the source format. Above SD the frame runs in overlapping 512-pixel tiles: -# whole-frame Real-CUGAN at 1080p ran out of GPU memory on a 16 GB Mac. An odd -# size is padded to even by repeating the edge (CUGAN works in 2-pixel steps) -# and cropped after. fp16 differs from fp32 by at most 0.3/255 and is ~3x faster. +# AI upscaling: Real-ESRGAN / Real-CUGAN (and the add-on models) through +# vs-mlrt's VSNCNN (ncnn on Vulkan), loaded explicitly from vapoursynth/ncnn/ +# (never autoloaded) after MoltenVK on macOS, exactly as RIFE is. The networks +# take float RGB, so the source's matrix and range convert there and back, and +# the result is dithered to the source format. Above a size the frame runs in +# overlapping tiles: whole-frame Real-CUGAN at 1080p ran out of GPU memory on a +# 16 GB Mac (512-pixel tiles above SD), and the large Real-ESRGAN x4plus lost +# the device at SD (384-pixel tiles at 4x, any size). An odd size is padded to even by +# repeating the edge (CUGAN works in 2-pixel steps) and cropped after. fp16 +# differs from fp32 by at most 0.3/255 and is ~3x faster. import ctypes as _ai_ctypes _ai_preload = {{AI_PRELOAD}} if _ai_preload: @@ -2016,17 +2260,30 @@ for _ai_model in [{{AI_MODELS}}]: # AnimeJaNai: its SD model for a picture at most {{AI_SD_MAX_HEIGHT}} tall, else # the HD one (trained on 1080p anime), chosen per doubling. _ai_model = _ai_model[0] if _ai_rgb.height <= {{AI_SD_MAX_HEIGHT}} else _ai_model[1] - _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= 720 * 576 else dict(tilesize=[512, 512], overlap=[16, 16]) + # A tile larger than the frame in either direction segfaults VSNCNN (a + # wide, short frame can pass the area test), so it never exceeds the frame. + _ai_tiles = {} if _ai_rgb.width * _ai_rgb.height <= {{AI_TILE_ABOVE}} else dict( + tilesize=[min({{AI_TILE}}, _ai_rgb.width), min({{AI_TILE}}, _ai_rgb.height)], overlap=[16, 16]) _ai_rgb = core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True, num_streams=2, **_ai_tiles) _ai_scale = _ai_rgb.width // _ai_in_w if _ai_pad_w or _ai_pad_h: _ai_rgb = core.std.Crop(_ai_rgb, right=_ai_pad_w * _ai_scale, bottom=_ai_pad_h * _ai_scale) +{{#AI_RESAMPLE}} +# The network enlarges further than asked (Real-ESRGAN General is 4x only), so +# resample to {{AI_FACTOR}}x here, in float RGB, before the conversion back. +_ai_rgb = core.resize.Spline36(_ai_rgb, width=_ai_rgb.width * {{AI_FACTOR}} // _ai_scale, height=_ai_rgb.height * {{AI_FACTOR}} // _ai_scale) +{{/AI_RESAMPLE}} if _ai_fmt.color_family == vs.YUV: clip = core.resize.Spline36(_ai_rgb, format=_ai_fmt.id, matrix_s="{{AI_MATRIX}}", range_s="{{AI_RANGE}}", dither_type="error_diffusion") elif _ai_fmt.color_family == vs.GRAY: clip = core.resize.Point(core.std.ShufflePlanes(_ai_rgb, 1, vs.GRAY), format=_ai_fmt.id, range_s="{{AI_RANGE}}", dither_type="error_diffusion") else: clip = core.resize.Point(_ai_rgb, format=_ai_fmt.id, dither_type="error_diffusion") +{{#AI_STRENGTH}} +# AI strength: keep {{AI_STRENGTH}} of the network's picture and blend in a Spline36 +# upscale of the same frame for the rest, at the clip's own format. +clip = core.std.Merge(core.resize.Spline36(_ai_src, width=clip.width, height=clip.height), clip, weight={{AI_STRENGTH}}) +{{/AI_STRENGTH}} # The network drops frame properties; field order, matrix and the rest are the source's. clip = core.std.CopyFrameProps(clip, _ai_src) {{/UPSCALE_AI}} @@ -2106,6 +2363,69 @@ _aspect_box_h = _even(_box_h) if _box_h > 0 else 0 {{/RESIZE_STANDARD}} {{/RESIZE}} +# ============================================================================ +# BT.601 -> BT.709 (Crop & Resize, app 1.7.0) +# ============================================================================ +{{#COLOR_709}} +# An SD source keeps its BT.601 matrix through an upscale unless something +# converts it, and many players and TVs ignore the tag on an HD-sized picture +# and decode it as BT.709 - a 75% green then shows as 161 instead of 191. So +# the samples are converted here, and the worker declares BT.709 on both sides +# of the encoder to match (VideoJob::output_color_metadata, the one decision). +# +# The matrix only: the primaries keep the source's, and so does its tag (see +# ColorMetadata::bt709_converted for the measurement behind that). zimg works +# through 4:4:4 float and the result is dithered back to the clip's own +# format; _FieldBased keeps the chroma resampling per field on a picture that +# is still interlaced. +# +# VapourSynth R78's resizers let a frame's own _Matrix / _Range properties +# override matrix_in_s / range_in_s, so they are removed first: the worker's +# reading of the source's tags is what decides the conversion. +if clip.format.color_family == vs.YUV: + clip = core.std.RemoveFrameProps(clip, props=["_Matrix", "_Range", "_ColorRange"]) + clip = core.resize.Spline36( + clip, + format=clip.format.id, + matrix_in_s="{{C709_MATRIX_IN}}", + matrix_s="709", + range_in_s="{{C709_RANGE}}", + range_s="{{C709_RANGE}}", + dither_type="error_diffusion", + ) +{{/COLOR_709}} + +# ============================================================================ +# PASS: FACE RESTORATION +# ============================================================================ +# Each face is found (RetinaFace), aligned to the FFHQ template, restored +# (RestoreFormer++, GFPGAN or CodeFormer), masked to the face itself (BiSeNet) +# and blended back at this resolution: after the resize, so an upscale gives the +# restorer more pixels and no later resample softens what it recovered; before +# sharpening and grain. ONNX Runtime comes from the ai-runtime add-on (the AI +# runtime block above) and the models are add-ons; face_restore.py sits beside +# pipe_source.py. Landmarks are smoothed over the detections either side, and +# the faces are merged in the source format, so every pixel outside them is the +# input, bit-exact. +{{#FACE_RESTORE}} +import face_restore as _face_restore +clip = _face_restore.restore_faces( + clip, + detector={{FR_DETECTOR}}, + parser={{FR_PARSER}}, + restorer={{FR_RESTORER}}, + model="{{FR_MODEL}}", + strength={{FR_STRENGTH}}, + fidelity={{FR_FIDELITY}}, + threshold={{FR_THRESHOLD}}, + min_face={{FR_MIN_FACE}}, + max_faces={{FR_MAX_FACES}}, + temporal_radius={{FR_RADIUS}}, + matrix="{{FR_MATRIX}}", + color_range="{{FR_RANGE}}", +) +{{/FACE_RESTORE}} + # ============================================================================ # PASS: SHARPEN # ============================================================================ diff --git a/worker/tests/filter_integration_test.rs b/worker/tests/filter_integration_test.rs index 89c94cb9..91bc743c 100644 --- a/worker/tests/filter_integration_test.rs +++ b/worker/tests/filter_integration_test.rs @@ -4022,8 +4022,9 @@ fn test_105_anti_alias_santiag() { #[test] fn test_106_santiag_type_cannot_name_an_unbundled_interpolator() { - // havsfunc accepts eedi2 and sangnom; neither is in the deps bundle, and - // naming one fails at script evaluation rather than degrading. + // havsfunc accepts eedi2 and sangnom. eedi2 is not in the deps bundle, and + // havsfunc's sangnom type calls SangNom(order=0), which SangNom r45 refuses + // without _FieldBased; either fails at script evaluation. create_output_dir(); let mut job = create_base_job("test_106_aa_santiag_type"); job.qtgmc_parameters.enabled = false; @@ -4325,6 +4326,7 @@ fn test_115_every_noise_method_still_emits_only_its_own_filter() { ("stpresso", "haf.STPresso"), ("bm3d", "core.bm3d."), ("knlm", "core.knlm.KNLMeansCL("), + ("nlm", "core.vszip.NLMeans("), ]; let methods = [ @@ -4338,6 +4340,7 @@ fn test_115_every_noise_method_still_emits_only_its_own_filter() { (NoiseReductionMethod::StPresso, "stpresso"), (NoiseReductionMethod::Bm3d, "bm3d"), (NoiseReductionMethod::KnlMeans, "knlm"), + (NoiseReductionMethod::NlMeans, "nlm"), ]; for (method, key) in methods { @@ -5467,6 +5470,7 @@ fn test_149_every_noise_reduction_method_emits_its_filter() { (NoiseReductionMethod::TemporalDegrain2, "_TemporalDegrain2("), (NoiseReductionMethod::Bm3d, "core.bm3d.VBasic("), (NoiseReductionMethod::KnlMeans, "core.knlm.KNLMeansCL("), + (NoiseReductionMethod::NlMeans, "core.vszip.NLMeans("), ]; // The list above is hand-written; this match is not. A new method fails to // compile here until it is added both here and to the list. @@ -5475,11 +5479,11 @@ fn test_149_every_noise_reduction_method_emits_its_filter() { match m { SmDegrain | McTemporalDenoise | McDegrainSharp | DfTtest | Fft3dFilter | TTempSmooth | FluxSmoothT | FluxSmoothSt | StPresso | Ctmf | MClean - | TemporalDegrain2 | Bm3d | KnlMeans => true, + | TemporalDegrain2 | Bm3d | KnlMeans | NlMeans => true, QtgmcBuiltin => false, // denoises inside the QTGMC pass } } - assert_eq!(expected.len(), 14); + assert_eq!(expected.len(), 15); assert!(expected.iter().all(|(m, _)| listed(*m))); for (method, call) in expected { @@ -6165,6 +6169,16 @@ fn rife_job(name: &str) -> VideoJob { job } +/// A throwaway add-ons directory with the GPU plugins' model add-ons (RIFE's +/// and the anime upscalers') installed as far as the worker can tell. +fn model_addons() -> PathBuf { + fake_addons_with_requirements( + "gpu-models", + &[vapourbox_worker::gpu_plugin::RIFE_ADDON, vapourbox_worker::upscale_addons::ANIME_ADDON], + "macos-arm64", + ) +} + fn rife_files() -> vapourbox_worker::gpu_plugin::GpuPluginFiles { vapourbox_worker::gpu_plugin::GpuPluginFiles::in_vapoursynth_dir( vapourbox_worker::gpu_plugin::GpuPlugin::Rife, @@ -6192,9 +6206,18 @@ fn test_166_rife_frame_rate_emits_explicit_load_and_real_frame_passthrough() { let files = rife_files(); // Paths as the generator writes them: Path::join uses `\` on Windows. let py = |p: &std::path::Path| vapourbox_worker::gpu_plugin::py_str(p); - let (preload, plugin, model) = - (py(files.preload.as_deref().unwrap()), py(&files.plugin), py(&files.model_dir)); - let generator = ScriptGenerator::new().unwrap().with_rife(Ok(files)); + // The model is the add-on's, never the deps bundle's. + let root = model_addons(); + let (preload, plugin, model) = ( + py(files.preload.as_deref().unwrap()), + py(&files.plugin), + py(&vapourbox_worker::gpu_plugin::rife_model_dir(&root)), + ); + assert!(model.contains("frame-rate-rife") && model.contains("rife-v4.26_ensembleFalse"), "{model}"); + let generator = ScriptGenerator::new() + .unwrap() + .with_rife(Ok(files)) + .with_addons(root.clone(), "macos-arm64"); let (encode, preview) = both_scripts_with(&generator, &rife_job("test_166_rife")).unwrap(); for (name, script) in [("encode", &encode), ("preview", &preview)] { assert!(script.contains(&format!("_rife_preload = {preload}")), @@ -6406,10 +6429,16 @@ fn test_172_ai_upscale_emits_explicit_load_and_the_right_model() { create_output_dir(); let files = upscale_files(); let py = |p: &std::path::Path| vapourbox_worker::gpu_plugin::py_str(p); - let generator = ScriptGenerator::new().unwrap().with_upscale(Ok(files.clone())); + use vapourbox_worker::upscale_addons::{ANIME_ADDON, ANIME_ESRGAN_X2, CUGAN_DENOISE3X}; + let root = model_addons(); + let dir = vapourbox_worker::addons::addon_dir(&root, ANIME_ADDON); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(files.clone())) + .with_addons(root.clone(), "macos-arm64"); for (method, model) in [ - (UpscaleMethod::RealEsrgan, "RealESRGANv2/RealESRGANv2-animevideo-xsx2.onnx"), - (UpscaleMethod::RealCugan, "cugan/up2x-latest-denoise3x.onnx"), + (UpscaleMethod::RealEsrgan, ANIME_ESRGAN_X2), + (UpscaleMethod::RealCugan, CUGAN_DENOISE3X), ] { let job = ai_upscale_job("test_172_ai_upscale", method, 2, "denoise3x"); let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); @@ -6419,7 +6448,7 @@ fn test_172_ai_upscale_emits_explicit_load_and_the_right_model() { "{ctx}: MoltenVK not preloaded"); assert!(script.contains(&format!("core.std.LoadPlugin({})", py(&files.plugin))), "{ctx}: VSNCNN not loaded explicitly"); - assert!(script.contains(&format!("for _ai_model in [{}]:", py(&files.model_dir.join(model)))), + assert!(script.contains(&format!("for _ai_model in [{}]:", py(&dir.join(model)))), "{ctx}: wrong model"); assert!(script.contains("core.ncnn.Model(_ai_rgb, network_path=_ai_model, fp16=True"), "{ctx}: no ncnn.Model call"); @@ -6442,14 +6471,19 @@ fn test_172_ai_upscale_emits_explicit_load_and_the_right_model() { fn test_173_ai_upscale_4x_models() { create_output_dir(); let files = upscale_files(); - let py = |m: &str| vapourbox_worker::gpu_plugin::py_str(&files.model_dir.join(m)); - let generator = ScriptGenerator::new().unwrap().with_upscale(Ok(files.clone())); + use vapourbox_worker::upscale_addons::*; + let root = model_addons(); + let dir = vapourbox_worker::addons::addon_dir(&root, ANIME_ADDON); + let py = |m: &str| vapourbox_worker::gpu_plugin::py_str(&dir.join(m)); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(files.clone())) + .with_addons(root.clone(), "macos-arm64"); let esrgan = script_text_with(&generator, &ai_upscale_job("test_173_esrgan", UpscaleMethod::RealEsrgan, 4, "")); - assert!(esrgan.contains(&format!("for _ai_model in [{}]:", - py("RealESRGANv2/RealESRGANv2-animevideo-xsx4.onnx")))); + assert!(esrgan.contains(&format!("for _ai_model in [{}]:", py(ANIME_ESRGAN_X4)))); let cugan = script_text_with(&generator, &ai_upscale_job("test_173_cugan", UpscaleMethod::RealCugan, 4, "conservative")); assert!(cugan.contains(&format!("for _ai_model in [{}, {}]:", - py("cugan/up2x-latest-conservative.onnx"), py("cugan/up2x-latest-no-denoise.onnx")))); + py(CUGAN_CONSERVATIVE), py(CUGAN_NO_DENOISE)))); } /// Without a working Vulkan device an AI upscale job is refused, before any @@ -6602,13 +6636,14 @@ fn test_179_animejanai_is_refused_without_its_models() { let generator = ScriptGenerator::new() .unwrap() .with_upscale(Ok(upscale_files())) - .with_animejanai(Err(vapourbox_worker::animejanai::MISSING_MESSAGE.into())); + .with_animejanai(Err(vapourbox_worker::animejanai::MISSING_MESSAGE.into())) + .with_addons(model_addons(), "macos-arm64"); let job = ai_upscale_job("test_179_refused", UpscaleMethod::AnimeJanai, 2, ""); let err = both_scripts_with(&generator, &job).unwrap_err().to_string(); assert!(err.contains("AnimeJaNai's models are not installed"), "{err}"); assert!(err.contains("non-commercial"), "{err}"); let esrgan = ai_upscale_job("test_179_esrgan", UpscaleMethod::RealEsrgan, 2, ""); - both_scripts_with(&generator, &esrgan).expect("Real-ESRGAN needs no add-on"); + both_scripts_with(&generator, &esrgan).expect("Real-ESRGAN needs only its own add-on"); } /// A phone-style VFR source: 30 fps for 30 frames, then every third frame @@ -6647,6 +6682,7 @@ fn test_180_vfr_source_timeline_fills_gaps() { let generator = ScriptGenerator::new() .unwrap() .with_rife(Ok(rife_files())) + .with_addons(model_addons(), "macos-arm64") .with_timeline(Some(window.clone())); for method in [FrameRateMethod::FlowFps, FrameRateMethod::Rife] { @@ -6718,7 +6754,8 @@ fn test_180_vfr_source_timeline_fills_gaps() { #[test] fn test_181_custom_frame_rate_target() { create_output_dir(); - let generator = ScriptGenerator::new().unwrap().with_rife(Ok(rife_files())); + let generator = ScriptGenerator::new().unwrap().with_rife(Ok(rife_files())) + .with_addons(model_addons(), "macos-arm64"); let custom = |name: &str, method: FrameRateMethod, fps: f64, src: (u32, u32)| { let mut job = rife_job(name); let fr = &mut job.processing_pipeline.as_mut().unwrap().frame_rate; @@ -6765,7 +6802,8 @@ fn test_181_custom_frame_rate_target() { #[test] fn test_182_preview_retime_padding() { create_output_dir(); - let generator = ScriptGenerator::new().unwrap().with_rife(Ok(rife_files())); + let generator = ScriptGenerator::new().unwrap().with_rife(Ok(rife_files())) + .with_addons(model_addons(), "macos-arm64"); for method in [FrameRateMethod::FlowFps, FrameRateMethod::Rife, FrameRateMethod::Duplicate] { let mut job = rife_job("test_182_pad"); let fr = &mut job.processing_pipeline.as_mut().unwrap().frame_rate; @@ -6818,6 +6856,8 @@ fn test_183_deblock_per_field_when_still_interlaced() { DeblockMethod::DeblockQed => "haf.Deblock_QED(", DeblockMethod::Deblock => "core.deblock.Deblock(", DeblockMethod::DctFilter => "core.dctf.DCTFilter(", + // Never per field; test_203's. + DeblockMethod::Mpeg2Stinx => unreachable!("mpeg2stinx takes the whole frame"), }; for method in &[DeblockMethod::DeblockQed, DeblockMethod::Deblock, DeblockMethod::DctFilter] { @@ -6862,7 +6902,8 @@ fn test_183_deblock_per_field_when_still_interlaced() { #[test] fn test_184_flowfps_length_is_set_by_the_script() { create_output_dir(); - let generator = ScriptGenerator::new().unwrap().with_rife(Ok(rife_files())); + let generator = ScriptGenerator::new().unwrap().with_rife(Ok(rife_files())) + .with_addons(model_addons(), "macos-arm64"); let mut job = rife_job("test_184_flowfps_length"); let fr = &mut job.processing_pipeline.as_mut().unwrap().frame_rate; fr.method = FrameRateMethod::FlowFps; @@ -6879,3 +6920,901 @@ fn test_184_flowfps_length_is_set_by_the_script() { assert!(call < trim && call < pad, "{name}: length set before FlowFPS"); } } + + +// =========================================================================== +// deps 1.16.0: SangNom, vszip NLMeans, and the vs-jetpack ports (app 1.7.0) +// =========================================================================== + +/// SangNom AA: its own block, `aa` clamped to the plugin's 0-128, the field +/// order put back afterwards, and neither of the other methods emitted. +#[test] +fn test_200_anti_alias_sangnom() { + create_output_dir(); + let job_with = |aa: i32| { + let mut job = create_base_job("test_200_aa_sangnom"); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + anti_alias: AntiAliasParameters { + enabled: true, + method: AntiAliasMethod::SangNom, + sangnom_aa: aa, + ..Default::default() + }, + ..ProcessingPipeline::default() + }); + job + }; + let (encode, preview) = generate_both_scripts(&job_with(64)); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains("_aa = core.sangnom.SangNom(_aa, order=1, aa=64)"), "{name}: first axis"); + assert!( + script.contains("_aa = core.sangnom.SangNom(core.std.Transpose(_aa), order=1, aa=64)"), + "{name}: second axis" + ); + assert!(script.contains("core.resize.Spline36(clip, _aa_w * 2, _aa_h * 2)"), "{name}: supersample"); + let sangnom = script.find("core.sangnom.SangNom(").unwrap(); + let restore = script.rfind("clip = core.std.SetFieldBased(clip, ").unwrap(); + assert!(restore > sangnom, "{name}: field order not put back after SangNom"); + assert!(!script.contains("haf.daa("), "{name}: daa emitted too"); + assert!(!script.contains("haf.santiag("), "{name}: santiag emitted too"); + assert!(!script.contains("{{AA_"), "{name}: placeholder left"); + } + assert!(script_text(&job_with(500)).contains("order=1, aa=128)")); + assert!(script_text(&job_with(-3)).contains("order=1, aa=0)")); +} + +/// NLMeans (CPU): vszip's NLMeans with the knlm_* values, chroma in its own +/// pass, and it runs where KNLMeansCL is refused for want of a device. +#[test] +fn test_201_noise_reduction_nlmeans_cpu() { + create_output_dir(); + let job = nr_job("test_201_nlm", NoiseReductionParameters { + method: NoiseReductionMethod::NlMeans, + knlm_h: 2.5, + knlm_d: 2, + knlm_a: 3, + knlm_s: 1, + knlm_chroma: true, + ..Default::default() + }); + // No OpenCL device at all: NLMeans must not care. + let message = vapourbox_worker::dependency_locator::KNLM_NO_DEVICE_MESSAGE; + let generator = ScriptGenerator::new().unwrap().with_knlm(Err(message.into())); + let (encode, preview) = both_scripts_with(&generator, &job).expect("NLMeans needs no OpenCL"); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains( + "clip = core.vszip.NLMeans(clip, d=2, a=3, s=1, h=2.50, channels=\"Y\")"), "{name}: luma"); + assert!(script.contains( + "clip = core.vszip.NLMeans(clip, d=2, a=3, s=1, h=2.50, channels=\"UV\")"), "{name}: chroma"); + assert!(script.contains(" if True:"), "{name}: chroma switch"); + assert!(!script.contains("core.knlm.KNLMeansCL("), "{name}: KNLMeansCL emitted"); + assert!(!script.contains("core.bm3d."), "{name}: BM3D emitted"); + assert!(!script.contains("{{NR_NLM"), "{name}: placeholder left"); + } + // The refusal names the CPU method as the way out. + assert!(message.contains("NLMeans (CPU)"), "{message}"); + // Same preview radius as KNLMeans: d frames each side, at least 3. + let mut nr = NoiseReductionParameters { method: NoiseReductionMethod::NlMeans, knlm_d: 4, ..Default::default() }; + assert_eq!(nr.temporal_radius(), 4); + nr.knlm_d = 1; + assert_eq!(nr.temporal_radius(), 3); +} + +/// vine: the ported block with the clamped values, through the helpers. +#[test] +fn test_202_dehalo_vine() { + create_output_dir(); + let job_with = |strength: f64, sharp: f64, sigma: f64| { + let mut job = create_base_job("test_202_vine"); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + dehalo: DehaloParameters { + enabled: true, + method: DehaloMethod::Vine, + vine_strength: strength, + vine_sharp: sharp, + vine_sigma: sigma, + ..Default::default() + }, + ..ProcessingPipeline::default() + }); + job + }; + let (encode, preview) = generate_both_scripts(&job_with(24.0, 0.25, 1.5)); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains("_vine_strength = 24.000"), "{name}"); + assert!(script.contains("_vine_sharp = 0.250"), "{name}"); + assert!(script.contains("_vine_sigma = 1.500"), "{name}"); + assert!(script.contains("_nnedi3(_vine_y, field=1, dh=True)"), "{name}: supersample"); + assert!(script.contains("core.vszip.NLMeans(_vine_ss, d=0, a=2, s=0, h=_vine_strength)"), "{name}"); + assert!(script.contains("rclip=_vine_high"), "{name}: refine pass"); + assert!(!script.contains("haf.DeHalo_alpha("), "{name}: other method emitted"); + assert!(!script.contains("{{DEHALO_VINE"), "{name}: placeholder left"); + } + // NLMeans refuses h <= 0; vs-jetpack refuses sharp <= 0; 25 taps max. + let clamped = script_text(&job_with(0.0, -1.0, 99.0)); + assert!(clamped.contains("_vine_strength = 1.000")); + assert!(clamped.contains("_vine_sharp = 0.050")); + assert!(clamped.contains("_vine_sigma = 4.000")); + let nan = script_text(&job_with(f64::NAN, f64::INFINITY, f64::NAN)); + assert!(nan.contains("_vine_strength = 16.000") && nan.contains("_vine_sharp = 0.500")); +} + +/// mpeg2stinx: always the whole frame (it is a cross-field filter), with the +/// encode's field order, and a preview window of one frame each side. +#[test] +fn test_203_deblock_mpeg2stinx() { + create_output_dir(); + let job_for = |deinterlace: bool, detected: Option| { + let mut job = create_base_job("test_203_mpeg2stinx"); + job.detected_field_order = detected; + job.qtgmc_parameters.enabled = deinterlace; + job.qtgmc_parameters.tff = Some(false); + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: job.qtgmc_parameters.clone(), + deblock: DeblockParameters { enabled: true, method: DeblockMethod::Mpeg2Stinx, ..Default::default() }, + ..ProcessingPipeline::default() + }); + job + }; + for (job, tff) in [ + (job_for(false, Some(FieldOrder::BottomFieldFirst)), "False"), + (job_for(false, Some(FieldOrder::TopFieldFirst)), "True"), + (job_for(false, None), "True"), + (job_for(true, Some(FieldOrder::TopFieldFirst)), "False"), // the deinterlace pass's order wins + ] { + let (encode, preview) = generate_both_scripts(&job); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains(&format!("_stx_tff = {tff}")), "{name}: field order {tff}"); + assert!(script.contains("core.bwdif.Bwdif(c, field=_stx_field, edeint=_nnedi3(c, field=_stx_field))")); + assert!(script.contains("clip = core.std.SetFieldBased(core.std.Merge(_stx_1, _stx_2), 0)")); + assert!(!script.contains("_deblock_per_field"), "{name}: mpeg2stinx must not run per field"); + assert!(!script.contains("haf.Deblock_QED(") && !script.contains("core.dctf.DCTFilter(")); + assert!(!script.contains("{{DEBLOCK"), "{name}: placeholder left"); + } + } + let p = job_for(false, None).processing_pipeline.unwrap(); + let deblock_map = p + .enabled_passes() + .into_iter() + .zip(p.frame_maps()) + .find(|(pass, _)| *pass == PassType::Deblock) + .map(|(_, m)| m) + .unwrap(); + assert_eq!(deblock_map, FrameMap::Identity { radius: 1 }); +} + +/// The IVTC fade fix: right after VFM in both scripts when ticked, absent +/// otherwise and for every other deinterlace method. +#[test] +fn test_204_ivtc_fix_fades() { + create_output_dir(); + let job_for = |method: DeinterlaceMethod, fix: bool, mode: &str| { + let mut job = create_base_job("test_204_ivtc_fades"); + job.qtgmc_parameters = QTGMCParameters { + enabled: true, + method, + tff: Some(true), + ivtc_fix_fades: fix, + ivtc_fade_mode: mode.to_string(), + ..Default::default() + }; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: job.qtgmc_parameters.clone(), + ..ProcessingPipeline::default() + }); + job + }; + for (mode, expect) in [("average", "average"), ("match", "match"), ("MATCH", "match"), ("bogus", "average")] { + let (encode, preview) = generate_both_scripts(&job_for(DeinterlaceMethod::Ivtc, true, mode)); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + let vfm = script.find("core.vivtc.VFM(").expect("VFM"); + let fix = script.find("_fade_fields = core.std.SeparateFields(").unwrap_or_else(|| panic!("{name}: no fix")); + assert!(fix > vfm, "{name}: the fix must follow field matching"); + assert!(script.contains(&format!("if '{expect}' == 'average' else min(a, b)")), "{name}: mode {mode}"); + // Where akarin exists the gains are frame properties read by ONE + // expression: an akarin node made per frame inside FrameEval + // crashed or hung vspipe under concurrency. + assert!(script.contains("core.std.ModifyFrame(_fade_top, [_fade_top] + _fade_props,"), "{name}"); + assert!(script.contains("_fade_apply = [f'x x.FadeGain{i} *' for i in range(_fade_fmt.num_planes)]"), + "{name}"); + if let Some(decimate) = script.find("core.vivtc.VDecimate(") { + assert!(fix < decimate, "{name}: before decimation"); + } + assert!(!script.contains("{{IVTC_FADE_MODE}}") && !script.contains("DEINT_IVTC_FIX_FADES")); + } + } + for job in [ + job_for(DeinterlaceMethod::Ivtc, false, "average"), + job_for(DeinterlaceMethod::Qtgmc, true, "average"), + ] { + let (encode, preview) = generate_both_scripts(&job); + for script in [&encode, &preview] { + assert!(!script.contains("_fade_fields"), "fade fix emitted when not asked for"); + assert!(!script.contains("DEINT_IVTC_FIX_FADES"), "marker left"); + } + } + let p: QTGMCParameters = serde_json::from_str("{}").unwrap(); + assert!(!p.ivtc_fix_fades); + assert_eq!(p.effective_ivtc_fade_mode(), "average"); +} + +/// Every block added for deps 1.16.0 goes through the templates' helpers +/// (test_92 / test_93 enforce the rule template-wide; this pins it to the new +/// code paths by generating each one) and leaves no placeholder behind. +#[test] +fn test_205_new_blocks_use_the_helpers() { + create_output_dir(); + let base = || { + let mut job = create_base_job("test_205_helpers"); + job.qtgmc_parameters.enabled = false; + job + }; + let pipelines = [ + ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + anti_alias: AntiAliasParameters { enabled: true, method: AntiAliasMethod::SangNom, ..Default::default() }, + dehalo: DehaloParameters { enabled: true, method: DehaloMethod::Vine, ..Default::default() }, + deblock: DeblockParameters { enabled: true, method: DeblockMethod::Mpeg2Stinx, ..Default::default() }, + noise_reduction: NoiseReductionParameters { + enabled: true, + method: NoiseReductionMethod::NlMeans, + ..Default::default() + }, + ..ProcessingPipeline::default() + }, + ProcessingPipeline { + deinterlace: QTGMCParameters { + enabled: true, + method: DeinterlaceMethod::Ivtc, + ivtc_fix_fades: true, + ..Default::default() + }, + ..ProcessingPipeline::default() + }, + ]; + for pipeline in pipelines { + let mut job = base(); + job.qtgmc_parameters = pipeline.deinterlace.clone(); + job.processing_pipeline = Some(pipeline); + let (encode, preview) = generate_both_scripts(&job); + for script in [&encode, &preview] { + for banned in ["core.std.Expr(", "core.znedi3.nnedi3(", "core.nnedi3.nnedi3("] { + // The helper's own body is the only call site. + let uses = script.matches(banned).count(); + assert!(uses <= 1, "{banned} called outside its helper ({uses} uses)"); + } + for marker in ["{{AA_", "NR_NLM", "DEHALO_VINE", "DEBLOCK_STINX", "DEBLOCK_MPEG2STINX", "IVTC_FADE", + "DEINT_IVTC_FIX_FADES"] { + assert!(!script.contains(marker), "{marker} left in a generated script"); + } + } + } +} + +/// An add-ons directory with `ids` installed for macOS arm64, as the app's +/// installer leaves it: every expected file plus the marker, written last. +fn fake_addons(tag: &str, ids: &[&str]) -> PathBuf { + let root = std::env::temp_dir().join(format!("vb-upscale-addons-{tag}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&root); + std::fs::create_dir_all(&root).unwrap(); + for id in ids { + let m = vapourbox_worker::addons::manifest(id).expect("manifest"); + let dir = vapourbox_worker::addons::addon_dir(&root, id); + std::fs::create_dir_all(&dir).unwrap(); + for p in m.expected_paths("macos-arm64") { + let path = dir.join(p); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(path, b"x").unwrap(); + } + std::fs::write( + dir.join(vapourbox_worker::addons::MARKER), + format!(r#"{{"id":"{id}","version":"{}","platform":"macos-arm64"}}"#, m.version), + ) + .unwrap(); + } + root +} + +/// A throwaway add-ons directory with `ids` (and what they require) installed +/// as far as the worker can tell: each manifest's expected paths present +/// (empty files will do for script generation) and its marker written. +fn fake_addons_with_requirements(name: &str, ids: &[&str], platform: &str) -> PathBuf { + use vapourbox_worker::addons; + let root = std::env::temp_dir().join(format!("vb-addons-{name}-{}", Uuid::new_v4())); + for id in addons::with_requirements(ids) { + let m = addons::manifest(&id).expect("manifest"); + let dir = addons::addon_dir(&root, &id); + for p in m.expected_paths(platform) { + let path = dir.join(&p); + if p.ends_with("site-packages") { + std::fs::create_dir_all(&path).unwrap(); + } else { + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + std::fs::write(&path, b"").unwrap(); + } + } + std::fs::write( + dir.join(addons::MARKER), + serde_json::json!({"id": id, "version": m.version, "platform": platform}).to_string(), + ) + .unwrap(); + } + root +} + +fn face_restore_job(name: &str, fr: FaceRestoreParameters) -> VideoJob { + let mut job = create_base_job(name); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + crop_resize: CropResizeParameters { + enabled: true, + resize_enabled: true, + target_width: Some(1280), + target_height: Some(960), + ..Default::default() + }, + sharpen: SharpenParameters { enabled: true, ..Default::default() }, + face_restore: FaceRestoreParameters { enabled: true, ..fr }, + ..ProcessingPipeline::default() + }); + job +} + +/// Face Restoration runs face_restore.restore_faces on ONNX Runtime from the +/// add-ons, in both scripts: after the resize and before sharpening, with the +/// AI runtime set up first and every model path taken from its add-on. +#[test] +fn test_189_face_restore_script_and_models() { + create_output_dir(); + let platform = "macos-arm64"; + let root = fake_addons_with_requirements("189", &["face-restoration", "face-codeformer"], platform); + let generator = ScriptGenerator::new().unwrap().with_addons(root.clone(), platform); + let job = face_restore_job("test_189_face_restore", FaceRestoreParameters { + model: FaceRestoreModel::CodeFormer, + strength: 0.6, + code_former_fidelity: 0.5, + detection_threshold: 0.7, + min_face_size: 40, + max_faces: 3, + temporal_smoothing: true, + ..Default::default() + }); + let py = |id: &str, f: &str| vapourbox_worker::gpu_plugin::py_str(&root.join(id).join(f)); + let runtime = vapourbox_worker::gpu_plugin::py_str(&root.join("ai-runtime")); + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + let setup = script + .find(&format!("ai_runtime.setup({runtime})")) + .unwrap_or_else(|| panic!("{name}: no AI runtime")); + let call = script.find("_face_restore.restore_faces(").unwrap_or_else(|| panic!("{name}: no call")); + for want in [ + format!("detector={}", py("face-restoration", "face-restoration-retinaface-r50.onnx")), + format!("parser={}", py("face-restoration", "face-restoration-bisenet.onnx")), + format!("restorer={}", py("face-codeformer", "face-codeformer-v0.1.0.onnx")), + "model=\"codeformer\"".into(), + "strength=0.600".into(), + "fidelity=0.500".into(), + "threshold=0.700".into(), + "min_face=40".into(), + "max_faces=3".into(), + "temporal_radius=1".into(), + "color_range=\"limited\"".into(), + ] { + assert!(script.contains(&want), "{name}: missing {want}"); + } + assert!(setup < call, "{name}: runtime set up after the pass"); + let resize = script.find("target_w = 1280").expect("resize"); + let sharpen = script.find("haf.LSFmod(").expect("sharpen"); + assert!(resize < call && call < sharpen, "{name}: after the resize, before sharpening"); + assert!(!script.contains("{{FR_") && !script.contains("FACE_RESTORE}}"), "{name}: unsubstituted"); + } + // Pass order and the preview's temporal context agree with the script. + let pipeline = job.effective_pipeline(); + let passes = pipeline.enabled_passes(); + let at = |p| passes.iter().position(|x| *x == p).unwrap(); + assert!(at(PassType::CropResize) < at(PassType::FaceRestore)); + assert!(at(PassType::FaceRestore) < at(PassType::Sharpen)); + assert_eq!(pipeline.output_count(100), 100); + // The blend reads one neighbour each side, whose landmarks read one more. + let face_map = pipeline.frame_maps()[at(PassType::FaceRestore)].clone(); + assert_eq!(face_map.radius(), 2); + let mut off = job.clone(); + off.processing_pipeline.as_mut().unwrap().face_restore.temporal_smoothing = false; + assert!(script_text_with(&generator, &off).contains("temporal_radius=0")); + let _ = std::fs::remove_dir_all(&root); +} + +/// Without its add-ons a Face Restoration job is refused, naming them and +/// Settings → Add-ons; with the pass off nothing needs them and the script +/// has neither the runtime nor the call. +#[test] +fn test_190_face_restore_refused_without_addons() { + create_output_dir(); + let platform = "macos-arm64"; + // The base installed but not GFPGAN's add-on. + let root = fake_addons_with_requirements("190", &["face-restoration"], platform); + let generator = ScriptGenerator::new().unwrap().with_addons(root.clone(), platform); + let gfpgan = face_restore_job("test_190_gfpgan", FaceRestoreParameters { + model: FaceRestoreModel::Gfpgan, + ..Default::default() + }); + let err = both_scripts_with(&generator, &gfpgan).unwrap_err().to_string(); + assert!(err.contains("Face restoration"), "{err}"); + assert!(err.contains("Settings → Add-ons"), "{err}"); + assert!(err.to_lowercase().contains("gfpgan"), "{err}"); + // RestoreFormer++ needs only the base. + let rf = face_restore_job("test_190_rf", FaceRestoreParameters::default()); + let (encode, _) = both_scripts_with(&generator, &rf).expect("the base add-on is enough"); + assert!(encode.contains("model=\"restoreformer\"")); + assert!(encode.contains("strength=0.800")); + // Nothing installed at all. + let empty = std::env::temp_dir().join(format!("vb-addons-190-empty-{}", Uuid::new_v4())); + let bare = ScriptGenerator::new().unwrap().with_addons(empty, platform); + assert!(both_scripts_with(&bare, &rf).is_err()); + // Pass off: no add-on needed, no runtime, no call. + let mut off = rf.clone(); + off.processing_pipeline.as_mut().unwrap().face_restore.enabled = false; + let (encode, preview) = both_scripts_with(&bare, &off).unwrap(); + for script in [&encode, &preview] { + assert!(!script.contains("import ai_runtime")); + assert!(!script.contains("restore_faces(")); + } + let _ = std::fs::remove_dir_all(&root); +} + +fn colorize_job(name: &str, cz: ColorizeParameters) -> VideoJob { + let mut job = create_base_job(name); + job.qtgmc_parameters.enabled = false; + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + chroma_fixes: ChromaFixParameters { + enabled: true, + apply_chroma_bleeding_fix: true, + ..ChromaFixParameters::default() + }, + color_correction: ColorCorrectionParameters { + enabled: true, + saturation: 1.2, + ..ColorCorrectionParameters::default() + }, + colorize: ColorizeParameters { enabled: true, ..cz }, + ..ProcessingPipeline::default() + }); + job +} + +/// Colourisation runs colorize.colorize on ONNX Runtime from the add-ons, in +/// both scripts: after Chroma Fixes and before Colour Correction, with the AI +/// runtime set up first, the chosen model's path from its add-on, and each +/// model's render size (DDColor's in pixels, DeOldify's render factor x 16). +#[test] +fn test_207_colorize_script_and_model() { + create_output_dir(); + let platform = "macos-arm64"; + let root = fake_addons_with_requirements("207", &["colorize-ddcolor", "colorize-deoldify"], platform); + let generator = ScriptGenerator::new().unwrap().with_addons(root.clone(), platform); + let job = colorize_job("test_207_colorize", ColorizeParameters { + model: ColorizeModel::DeOldify, + deoldify_render_factor: 25, + strength: 0.7, + temporal_radius: 3, + ..Default::default() + }); + let py = |id: &str, f: &str| vapourbox_worker::gpu_plugin::py_str(&root.join(id).join(f)); + let runtime = vapourbox_worker::gpu_plugin::py_str(&root.join("ai-runtime")); + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + let setup = script + .find(&format!("ai_runtime.setup({runtime})")) + .unwrap_or_else(|| panic!("{name}: no AI runtime")); + let call = script.find("_colorize.colorize(").unwrap_or_else(|| panic!("{name}: no call")); + for want in [ + format!("model_path={}", py("colorize-deoldify", "colorize-deoldify-video.onnx")), + "model=\"deoldify\"".into(), + "render_size=400".into(), + "strength=0.700".into(), + "temporal_radius=3".into(), + "matrix=\"".into(), + "color_range=\"limited\"".into(), + ] { + assert!(script.contains(&want), "{name}: missing {want}"); + } + assert!(setup < call, "{name}: runtime set up after the pass"); + let chroma = script.find("# PASS 8: CHROMA FIXES").expect("chroma fixes"); + let color = script.find("clip = adjust.Tweak(").expect("colour correction"); + assert!(chroma < call && call < color, "{name}: after Chroma Fixes, before Colour Correction"); + assert!(!script.contains("{{CZ_") && !script.contains("COLORIZE}}"), "{name}: unsubstituted"); + } + // Only the preview says it is one (DDColor previews on the CPU, see + // colorize.PREVIEW_ON_CPU). + assert!(preview.contains(" preview=True,")); + assert!(!encode.contains("preview=True")); + // Pass order and the preview's temporal context agree with the script. + let pipeline = job.effective_pipeline(); + let passes = pipeline.enabled_passes(); + let at = |p| passes.iter().position(|x| *x == p).unwrap(); + assert!(at(PassType::ChromaFixes) < at(PassType::Colorize)); + assert!(at(PassType::Colorize) < at(PassType::ColorCorrection)); + assert_eq!(pipeline.output_count(100), 100); + assert_eq!(pipeline.frame_maps()[at(PassType::Colorize)].radius(), 3); + // DDColor's defaults: 512 pixels, two frames either side. + let ddcolor = colorize_job("test_207_ddcolor", ColorizeParameters::default()); + let script = script_text_with(&generator, &ddcolor); + for want in [ + format!("model_path={}", py("colorize-ddcolor", "colorize-ddcolor-tiny.onnx")), + "model=\"ddcolor\"".into(), + "render_size=512".into(), + "strength=1.000".into(), + "temporal_radius=2".into(), + ] { + assert!(script.contains(&want), "ddcolor: missing {want}"); + } + let _ = std::fs::remove_dir_all(&root); +} + +/// Without the chosen model's add-on a Colourisation job is refused, naming it +/// and Settings → Add-ons; with the pass off nothing needs it and the script +/// has neither the runtime nor the call. +#[test] +fn test_208_colorize_refused_without_addon() { + create_output_dir(); + let platform = "macos-arm64"; + let root = fake_addons_with_requirements("208", &["colorize-ddcolor"], platform); + let generator = ScriptGenerator::new().unwrap().with_addons(root.clone(), platform); + let deoldify = colorize_job("test_208_deoldify", ColorizeParameters { + model: ColorizeModel::DeOldify, + ..Default::default() + }); + let err = both_scripts_with(&generator, &deoldify).unwrap_err().to_string(); + assert!(err.contains("Colourisation"), "{err}"); + assert!(err.contains("Settings → Add-ons"), "{err}"); + assert!(err.to_lowercase().contains("deoldify"), "{err}"); + let ddcolor = colorize_job("test_208_ddcolor", ColorizeParameters::default()); + both_scripts_with(&generator, &ddcolor).expect("DDColor's add-on is installed"); + let empty = std::env::temp_dir().join(format!("vb-addons-208-empty-{}", Uuid::new_v4())); + let bare = ScriptGenerator::new().unwrap().with_addons(empty, platform); + assert!(both_scripts_with(&bare, &ddcolor).is_err()); + let mut off = ddcolor.clone(); + off.processing_pipeline.as_mut().unwrap().colorize.enabled = false; + let (encode, preview) = both_scripts_with(&bare, &off).unwrap(); + for script in [&encode, &preview] { + assert!(!script.contains("import ai_runtime")); + assert!(!script.contains("_colorize.colorize(")); + } + let _ = std::fs::remove_dir_all(&root); +} + +/// The add-on upscalers (Real-ESRGAN General, live-action SPAN, Real-ESRGAN +/// x4plus/x2plus): the same VSNCNN block, each model from its add-on's +/// directory, with the factor rules — General is 4x only, so 2x resamples its +/// result down; SPAN is 2x only, so 4x runs it twice; x4plus/x2plus by factor, +/// in 384-pixel tiles at 4x — in both scripts. +#[test] +fn test_186_upscale_addon_models_and_factors() { + use vapourbox_worker::upscale_addons::*; + create_output_dir(); + let root = fake_addons("186", &[GENERAL_ADDON, ESRGAN_PLUS_ADDON]); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(upscale_files())) + .with_addons(root.clone(), "macos-arm64"); + let py = |addon: &str, f: &str| { + vapourbox_worker::gpu_plugin::py_str(&vapourbox_worker::addons::addon_dir(&root, addon).join(f)) + }; + let cases: Vec<(UpscaleMethod, i32, bool, Vec, Option)> = vec![ + (UpscaleMethod::RealEsrganGeneral, 2, false, vec![py(GENERAL_ADDON, GENERAL_X4)], Some(2)), + (UpscaleMethod::RealEsrganGeneral, 4, false, vec![py(GENERAL_ADDON, GENERAL_X4)], None), + (UpscaleMethod::RealEsrganGeneral, 2, true, vec![py(GENERAL_ADDON, GENERAL_WDN_X4)], Some(2)), + (UpscaleMethod::LiveAction, 2, false, vec![py(GENERAL_ADDON, LIVE_ACTION_X2)], None), + (UpscaleMethod::LiveAction, 4, false, vec![py(GENERAL_ADDON, LIVE_ACTION_X2); 2], None), + (UpscaleMethod::RealEsrganPlus, 2, false, vec![py(ESRGAN_PLUS_ADDON, PLUS_X2)], None), + (UpscaleMethod::RealEsrganPlus, 4, false, vec![py(ESRGAN_PLUS_ADDON, PLUS_X4)], None), + ]; + for (method, factor, denoise, models, resample) in cases { + let label = format!("{method:?} {factor}x denoise={denoise}"); + let mut job = ai_upscale_job("test_186", method, factor, ""); + job.processing_pipeline.as_mut().unwrap().crop_resize.general_denoise = denoise; + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains(&format!("for _ai_model in [{}]:", models.join(", "))), + "{label} {name}: wrong models"); + assert!(script.contains("core.ncnn.Model(_ai_rgb, network_path=_ai_model"), "{label} {name}"); + match resample { + Some(f) => assert!(script.contains(&format!( + "_ai_rgb = core.resize.Spline36(_ai_rgb, width=_ai_rgb.width * {f} // _ai_scale, \ + height=_ai_rgb.height * {f} // _ai_scale)")), "{label} {name}: no resample"), + None => assert!(!script.contains("// _ai_scale"), "{label} {name}: stray resample"), + } + let tiles = if method == UpscaleMethod::RealEsrganPlus && factor == 4 { + "<= 384 * 384 else dict(\n tilesize=[min(384, _ai_rgb.width), min(384, _ai_rgb.height)]" + } else { + "<= 720 * 576 else dict(\n tilesize=[min(512, _ai_rgb.width), min(512, _ai_rgb.height)]" + }; + assert!(script.contains(tiles), "{label} {name}: tiles"); + assert!(!script.contains("AI_RESAMPLE") && !script.contains("AI_STRENGTH") && !script.contains("{{AI_"), "{label} {name}: unsubstituted placeholder"); + assert!(!script.contains("core.std.Merge(core.resize.Spline36(_ai_src"), "{label} {name}"); + } + } + let _ = std::fs::remove_dir_all(&root); +} + +/// Without its add-on each add-on upscaler is refused, naming the add-on and +/// Settings → Add-ons; with the other add-on installed it is still refused, +/// and the bundled models need none. +#[test] +fn test_187_upscale_addons_are_refused_when_missing() { + use vapourbox_worker::upscale_addons::*; + create_output_dir(); + let root = fake_addons("187", &[ESRGAN_PLUS_ADDON]); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(upscale_files())) + .with_addons(root.clone(), "macos-arm64"); + for (method, feature) in [ + (UpscaleMethod::RealEsrganGeneral, "Real-ESRGAN General upscaling"), + (UpscaleMethod::LiveAction, "Live-action upscaling"), + ] { + let job = ai_upscale_job("test_187", method, 2, ""); + let err = both_scripts_with(&generator, &job).unwrap_err().to_string(); + assert!(err.starts_with(feature), "{err}"); + assert!(err.contains("General & live-action upscaling models"), "{err}"); + assert!(err.contains("Settings → Add-ons"), "{err}"); + } + both_scripts_with(&generator, &ai_upscale_job("test_187_plus", UpscaleMethod::RealEsrganPlus, 4, "")) + .expect("x4plus is installed"); + let missing = fake_addons("187b", &[]); + let bare = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(upscale_files())) + .with_addons(missing.clone(), "macos-arm64"); + let err = both_scripts_with(&bare, &ai_upscale_job("test_187_plus", UpscaleMethod::RealEsrganPlus, 2, "")) + .unwrap_err() + .to_string(); + assert!(err.contains("Real-ESRGAN x4plus / x2plus models"), "{err}"); + // Real-ESRGAN's anime model is an add-on too (deps 1.16.0 on). + let err = both_scripts_with(&bare, &ai_upscale_job("test_187_esrgan", UpscaleMethod::RealEsrgan, 2, "")) + .unwrap_err() + .to_string(); + assert!(err.contains("Real-ESRGAN anime & Real-CUGAN models"), "{err}"); + // On a platform VSNCNN doesn't ship on, the add-on is unavailable too. + let mac_x64 = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(upscale_files())) + .with_addons(root.clone(), "macos-x64"); + let err = both_scripts_with(&mac_x64, &ai_upscale_job("test_187_x64", UpscaleMethod::LiveAction, 2, "")) + .unwrap_err() + .to_string(); + assert!(err.contains("isn't available on this platform"), "{err}"); + let _ = std::fs::remove_dir_all(&root); + let _ = std::fs::remove_dir_all(&missing); +} + +/// AI strength blends the network's result with a Spline36 upscale of the same +/// frame in one std.Merge at the clip's format, for every AI method, in both +/// scripts. At full strength (the default) nothing is emitted: the script is +/// byte-for-byte the one a job without the setting gets. +#[test] +fn test_188_ai_strength_blends_with_spline36() { + use vapourbox_worker::upscale_addons::*; + create_output_dir(); + let root = fake_addons("188", &[GENERAL_ADDON, ANIME_ADDON]); + let generator = ScriptGenerator::new() + .unwrap() + .with_upscale(Ok(upscale_files())) + .with_addons(root.clone(), "macos-arm64"); + let merge = "clip = core.std.Merge(core.resize.Spline36(_ai_src, width=clip.width, \ + height=clip.height), clip, weight=0.650)"; + for method in [UpscaleMethod::RealEsrgan, UpscaleMethod::RealCugan, UpscaleMethod::LiveAction, + UpscaleMethod::RealEsrganGeneral] { + let mut job = ai_upscale_job("test_188", method, 2, "conservative"); + job.processing_pipeline.as_mut().unwrap().crop_resize.ai_strength = 0.65; + let (encode, preview) = both_scripts_with(&generator, &job).unwrap(); + for (name, script) in [("encode", &encode), ("preview", &preview)] { + assert!(script.contains(merge), "{method:?} {name}: no blend"); + let at = script.find(merge).unwrap(); + assert!(at > script.find("for _ai_model in").unwrap(), "{method:?} {name}: blend before the network"); + assert!(at < script.find("clip = core.std.CopyFrameProps(clip, _ai_src)").unwrap(), + "{method:?} {name}: blend after the props copy"); + } + } + // Full strength: byte-identical to a job without the field at all. + let full = ai_upscale_job("test_188_full", UpscaleMethod::RealEsrgan, 2, ""); + assert_eq!(full.processing_pipeline.as_ref().unwrap().crop_resize.ai_strength, 1.0); + let mut json = serde_json::to_value(&full).unwrap(); + json["processingPipeline"]["cropResize"].as_object_mut().unwrap().remove("aiStrength"); + json["id"] = serde_json::Value::String(Uuid::new_v4().to_string()); + let legacy: VideoJob = serde_json::from_value(json).unwrap(); + let a = script_text_with(&generator, &full); + let b = script_text_with(&generator, &legacy); + assert_eq!(a.replace(&full.id.to_string(), "ID"), b.replace(&legacy.id.to_string(), "ID")); + assert!(!a.contains("core.std.Merge(core.resize.Spline36(_ai_src")); + assert!(!a.contains("AI_STRENGTH")); + // Out-of-range values clamp; 0 is all Spline36. + let mut zero = ai_upscale_job("test_188_zero", UpscaleMethod::RealEsrgan, 2, ""); + zero.processing_pipeline.as_mut().unwrap().crop_resize.ai_strength = -1.0; + assert!(script_text_with(&generator, &zero).contains("clip, weight=0.000)")); + let _ = std::fs::remove_dir_all(&root); +} + +/// Audio restoration is rendered by ffmpeg before the encode, never by the +/// VapourSynth script: every step switched on — DeepFilterNet included, which +/// is not installed here — leaves both scripts byte-for-byte as they are with +/// the pass off. (A preview must never be refused for an audio add-on.) +#[test] +fn test_185_audio_restoration_never_touches_the_scripts() { + create_output_dir(); + let generator = ScriptGenerator::new() + .unwrap() + .with_addons(std::env::temp_dir().join("vb-no-addons-here"), "macos-arm64"); + let mut job = create_base_job("test_185_audio_restoration"); + job.processing_pipeline = Some(ProcessingPipeline::default()); + let (encode_off, preview_off) = both_scripts_with(&generator, &job).unwrap(); + + job.processing_pipeline = Some(ProcessingPipeline { + audio: AudioRestorationParameters { + enabled: true, + channel_fix: ChannelFix::Auto, + offset_ms: -250, + drift_correction: true, + declip: true, + declick: true, + hum_removal: true, + rumble_filter: true, + denoise: true, + deep_filter: true, + loudness_normalize: true, + limiter: true, + ..Default::default() + }, + ..ProcessingPipeline::default() + }); + let (encode_on, preview_on) = both_scripts_with(&generator, &job).unwrap(); + assert_eq!(encode_on, encode_off, "the encode script must not change"); + assert_eq!(preview_on, preview_off, "the preview script must not change"); + for script in [&encode_on, &preview_on] { + for word in ["afftdn", "loudnorm", "deep-filter", "atempo"] { + assert!(!script.contains(word), "{word} reached the script"); + } + } +} + +/// BT.601 -> BT.709 on SD -> HD (app 1.7.0). The conversion block appears in +/// BOTH scripts exactly when the job decides to convert, converts from the +/// source's own matrix and range, and everything after it (the border fill +/// here) works in BT.709. Off, or an SD-sized output, leaves no block at all. +#[test] +fn test_206_bt601_to_bt709_on_sd_to_hd() { + create_output_dir(); + let job_for = |mode: Bt709Conversion, target_h: i32, matrix: Option<&str>, range: &str| { + let mut job = create_base_job("test_206_bt709"); + job.qtgmc_parameters.enabled = false; + job.input_width = Some(720); + job.input_height = Some(576); + job.input_pixel_format = Some("yuv420p".to_string()); + job.input_color_matrix = matrix.map(str::to_string); + job.input_color_range = Some(range.to_string()); + job.processing_pipeline = Some(ProcessingPipeline { + deinterlace: QTGMCParameters { enabled: false, ..Default::default() }, + crop_resize: CropResizeParameters { + enabled: true, + resize_enabled: true, + target_width: Some(target_h * 4 / 3), + target_height: Some(target_h), + pad_enabled: true, + pad_width: Some(target_h * 16 / 9), + pad_height: Some(target_h), + convert_to_bt709: mode, + ..CropResizeParameters::default() + }, + ..ProcessingPipeline::default() + }); + job + }; + + // PAL, tagged: converts from 470bg, limited range, in both scripts. + let job = job_for(Bt709Conversion::Auto, 1080, Some("bt470bg"), "tv"); + assert!(job.converts_to_bt709()); + let (encode, preview) = generate_both_scripts(&job); + for script in [&encode, &preview] { + assert!(script.contains(r#"matrix_in_s="470bg""#), "converts from the source's matrix"); + assert!(script.contains(r#"matrix_s="709""#)); + assert!(script.contains(r#"range_in_s="limited""#)); + assert!(script.contains(r#"RemoveFrameProps(clip, props=["_Matrix", "_Range", "_ColorRange"])"#)); + assert!(!script.contains("{{C709") && !script.contains("{{#COLOR_709"), "unsubstituted placeholder"); + // The bars go on after the conversion, so their colour is 709's. + assert!(script.contains(r#"matrix_s="709", range_s="limited")"#), "border fill in 709"); + // After the resize and before Face Restoration. + let at = script.find(r#"matrix_in_s="470bg""#).unwrap(); + assert!(script.find("PASS: FACE RESTORATION").unwrap() > at); + } + run_job_and_verify(&job, "BT.709 conversion", &[r#"matrix_s="709""#]).unwrap(); + + // Untagged full-range SD: read as 601 (170m), range kept full. + let (encode, _) = generate_both_scripts(&job_for(Bt709Conversion::Auto, 1080, None, "pc")); + assert!(encode.contains(r#"matrix_in_s="170m""#)); + assert!(encode.contains(r#"range_in_s="full""#)); + + // Off, an SD-sized output under Auto, or an HD source: no conversion. + for job in [ + job_for(Bt709Conversion::Off, 1080, Some("bt470bg"), "tv"), + job_for(Bt709Conversion::Auto, 576, Some("bt470bg"), "tv"), + job_for(Bt709Conversion::Auto, 1080, Some("bt709"), "tv"), + ] { + assert!(!job.converts_to_bt709()); + let (encode, preview) = generate_both_scripts(&job); + for script in [&encode, &preview] { + assert!(!script.contains(r#"RemoveFrameProps(clip, props=["_Matrix""#), "no conversion expected"); + } + } + // On converts whatever the size. + assert!(job_for(Bt709Conversion::On, 576, Some("smpte170m"), "tv").converts_to_bt709()); +} + +/// The GPU plugins' models are add-ons (deps 1.16.0 on): a RIFE, Real-ESRGAN +/// or Real-CUGAN job on a machine where the plugin runs is still refused +/// without its add-on, naming it and Settings → Add-ons, in both scripts — and +/// with it installed the script reads the model from the add-on's directory, +/// never from the deps bundle's `vapoursynth/rife/models` or `ncnn/models`. +#[test] +fn test_209_gpu_plugin_models_are_addons() { + use vapourbox_worker::gpu_plugin::{rife_model_dir, RIFE_ADDON}; + use vapourbox_worker::upscale_addons::ANIME_ADDON; + create_output_dir(); + let empty = std::env::temp_dir().join(format!("vb-addons-209-empty-{}", Uuid::new_v4())); + let bare = ScriptGenerator::new() + .unwrap() + .with_rife(Ok(rife_files())) + .with_upscale(Ok(upscale_files())) + .with_addons(empty, "macos-arm64"); + let cases = [ + (rife_job("test_209_rife"), "AI interpolation (RIFE)", "RIFE 4.26 interpolation model"), + (ai_upscale_job("test_209_esrgan", UpscaleMethod::RealEsrgan, 2, ""), "Real-ESRGAN upscaling", + "Real-ESRGAN anime & Real-CUGAN models"), + (ai_upscale_job("test_209_cugan", UpscaleMethod::RealCugan, 4, "none"), "Real-CUGAN upscaling", + "Real-ESRGAN anime & Real-CUGAN models"), + ]; + let params = PreviewParams { + rate_pad: 0, + width: 720, height: 576, pix_fmt: "yuv420p".into(), num_frames: 11, + fps_num: 25, fps_den: 1, output_index: 5, + }; + for (job, feature, addon) in &cases { + for err in [ + bare.generate(job).unwrap_err().to_string(), + bare.generate_preview(job, ¶ms).unwrap_err().to_string(), + ] { + assert!(err.starts_with(feature), "{err}"); + assert!(err.contains(addon) && err.contains("Settings → Add-ons"), "{err}"); + } + } + let root = model_addons(); + let installed = ScriptGenerator::new() + .unwrap() + .with_rife(Ok(rife_files())) + .with_upscale(Ok(upscale_files())) + .with_addons(root.clone(), "macos-arm64"); + for (job, _, _) in &cases { + let (encode, preview) = both_scripts_with(&installed, job).unwrap(); + for script in [&encode, &preview] { + assert!(!script.contains("/rife/models") && !script.contains("/ncnn/models"), + "a bundled model path: {script}"); + } + } + let (rife, _) = both_scripts_with(&installed, &cases[0].0).unwrap(); + assert!(rife.contains(&vapourbox_worker::gpu_plugin::py_str(&rife_model_dir(&root)))); + let (esrgan, _) = both_scripts_with(&installed, &cases[1].0).unwrap(); + assert!(esrgan.contains(&vapourbox_worker::gpu_plugin::py_str( + &vapourbox_worker::addons::addon_dir(&root, ANIME_ADDON) + .join(vapourbox_worker::upscale_addons::ANIME_ESRGAN_X2)))); + // RIFE's add-on is required exactly when RIFE is what the pass runs. + let mut flow = rife_job("test_209_flow"); + flow.processing_pipeline.as_mut().unwrap().frame_rate.method = FrameRateMethod::FlowFps; + assert!(flow.effective_pipeline().required_addons().is_empty()); + assert!(cases[0].0.effective_pipeline().required_addons().iter().any(|n| n.ids == [RIFE_ADDON])); + let _ = std::fs::remove_dir_all(&root); +}