From 1c946dbf5acebc0dd02bb90b857f49e59fe79069 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Wed, 16 Sep 2026 17:08:19 +0200
Subject: [PATCH 01/11] ci: add LVH-based integration tests for external
 contributors

Add a standalone GitHub Actions workflow that runs integration tests
inside a nested QEMU/KVM VM using cilium/little-vm-helper (LVH). This
enables external contributors (fork PRs) to run integration tests
without requiring repository secrets or GCP credentials.

The workflow:
- Builds the fact container image locally (no registry push).
- Boots an LVH VM with a 6.6 LTS kernel and BPF LSM enabled via
  kernel cmdline.
- Loads the image into the VM's Docker and runs pytest against
  test_file_open.py as an initial spike validation.
- Caches the VM image and kernel for faster subsequent runs.
- Uses pull_request trigger (works for both fork and internal PRs).

This is Phase 1 (spike) of issue #1794. Once validated, Phase 2 will
expand to the full test suite and Phase 3 will reduce duplication with
the existing GCP VM-based integration tests.

Refs: #1794

Assisted-by: claude-opus-4-6 <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml | 227 ++++++++++++++++++++
 tests/conftest.py                           |  37 +++-
 2 files changed, 258 insertions(+), 6 deletions(-)
 create mode 100644 .github/workflows/lvh-integration-tests.yml

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
new file mode 100644
index 00000000..d0e42aad
--- /dev/null
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -0,0 +1,227 @@
+name: LVH Integration Tests
+
+on:
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.head_ref || github.run_id }}-lvh
+  cancel-in-progress: true
+
+env:
+  LVH_VERSION: v0.0.31
+
+jobs:
+  build:
+    name: Build fact image
+    runs-on: ubuntu-24.04
+    timeout-minutes: 30
+    steps:
+    - uses: actions/checkout@v7
+      with:
+        submodules: true
+        fetch-depth: 0
+        persist-credentials: false
+
+    - name: Build image
+      run: |
+        FACT_REGISTRY=localhost/fact make image
+
+    - name: Export image
+      run: |
+        docker save -o /tmp/fact-image.tar "$(FACT_REGISTRY=localhost/fact make image-name)"
+
+    - name: Upload image artifact
+      uses: actions/upload-artifact@v4
+      with:
+        name: fact-image
+        path: /tmp/fact-image.tar
+        retention-days: 1
+
+  integration-tests:
+    name: Integration tests (LVH)
+    needs: build
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        kernel:
+        - '6.12-main'
+    steps:
+    - name: Checkout fact
+      uses: actions/checkout@v7
+      with:
+        submodules: true
+        fetch-depth: 0
+        path: fact
+        persist-credentials: false
+
+    - name: Clone little-vm-helper
+      uses: actions/checkout@v7
+      with:
+        repository: cilium/little-vm-helper
+        ref: ${{ env.LVH_VERSION }}
+        path: little-vm-helper
+        persist-credentials: false
+
+    - uses: actions/setup-go@v7
+      with:
+        go-version-file: little-vm-helper/go.mod
+
+    - name: Install LVH cli
+      run: |
+        make -C little-vm-helper install
+        sudo ln -s "$(go env GOPATH)/bin" /usr/local/bin/lvh
+
+    - name: Install VM dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install -y --no-install-recommends \
+          qemu-system-x86 \
+          qemu-utils \
+          cpu-checker \
+          libvirt-daemon-system \
+          libvirt-clients \
+          virtinst
+
+    - name: Create LVH data directories
+      run: |
+        sudo mkdir -p /tmp/lvh-data/kernels /tmp/lvh-data/images
+        sudo chmod -R 777 /tmp/lvh-data
+
+    - name: Cache LVH kernel
+      uses: actions/cache@v4
+      id: cache-kernel
+      with:
+        path: /tmp/lvh-data/kernels/${{ matrix.kernel }}
+        key: lvh-kernel-${{ matrix.kernel }}
+
+    - name: Fetch kernel
+      if: steps.cache-kernel.outputs.cache-hit != 'true'
+      run: |
+        lvh kernels pull \
+          --dir /tmp/lvh-data/kernels \
+          "${{ matrix.kernel }}"
+
+    - name: Cache LVH VM image
+      uses: actions/cache@v4
+      id: cache-image
+      with:
+        path: /tmp/lvh-data/images/images
+        key: lvh-image-kind-${{ matrix.kernel }}
+
+    - name: Fetch VM image
+      if: steps.cache-image.outputs.cache-hit != 'true'
+      run: |
+        lvh images pull \
+          --dir /tmp/lvh-data/images \
+          "quay.io/lvh-images/kind:${{ matrix.kernel }}"
+
+    - name: Download fact image artifact
+      uses: actions/download-artifact@v4
+      with:
+        name: fact-image
+        path: /tmp
+
+    - name: Start VM
+      run: |
+        KERNEL_PATH=$(find /tmp/lvh-data/kernels -name "vmlinuz-*" -type f | head -1)
+        IMAGE_PATH=$(find /tmp/lvh-data/images -name "*.qcow2" -type f | head -1)
+        CPU="$(nproc)"
+        MEM="$(free -m | awk '/^Mem:/{print int($2 * 0.75)}')M"
+
+        echo "Kernel: ${KERNEL_PATH}"
+        echo "Image: ${IMAGE_PATH}"
+        echo "CPU: ${CPU}, MEM: ${MEM}"
+
+        sudo touch /tmp/console.log
+        sudo lvh run \
+          --image "${IMAGE_PATH}" \
+          --kernel "${KERNEL_PATH}" \
+          --host-mount "${{ github.workspace }}" \
+          --daemonize \
+          -p 2222:22 \
+          --cpu "${CPU}" \
+          --mem "${MEM}" \
+          --cpu-kind host \
+          --console-log-file /tmp/console.log \
+          --append "lsm=lockdown,capability,yama,apparmor,bpf"
+
+    - name: Wait for VM SSH
+      run: |
+        for i in $(seq 1 120); do
+          if ssh -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 root@localhost exit 2>/dev/null; then
+            echo "VM is ready after ${i} seconds"
+            exit 0
+          fi
+          sleep 1
+        done
+        echo "VM failed to become ready"
+        cat /tmp/console.log
+        exit 1
+
+    - name: Load fact image into VM
+      run: |
+        scp -P 2222 -o StrictHostKeyChecking=no \
+          /tmp/fact-image.tar root@localhost:/tmp/fact-image.tar
+        ssh -p 2222 -o StrictHostKeyChecking=no root@localhost \
+          "docker load -i /tmp/fact-image.tar && rm /tmp/fact-image.tar"
+
+    - name: Setup test environment in VM
+      run: |
+        ssh -p 2222 -o StrictHostKeyChecking=no root@localhost << 'EOF'
+          set -euo pipefail
+          apt-get update -qq
+          apt-get install -y -qq python3-venv python3-dev gcc > /dev/null 2>&1
+          cd /host/fact/tests
+          python3 -m venv .venv
+          source .venv/bin/activate
+          pip install -r requirements.txt
+          make grpc-gen
+        EOF
+
+    - name: Run integration tests
+      run: |
+        # NOTE: --tmp-dir points monitored/ignored test directories at the
+        # VM's local tmpfs instead of the 9p-mounted workspace (/host).
+        # The 9p (virtio-9p) filesystem used for --host-mount does not set
+        # FMODE_CREATED on newly created files, which makes fact
+        # misclassify file creations as generic "Open" events instead of
+        # "Creation". Everything else (test code, venv, gRPC stubs, logs,
+        # results) stays on /host/fact/tests as usual.
+        FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make -C fact image-name)"
+        # shellcheck disable=SC2087
+        ssh -p 2222 -o StrictHostKeyChecking=no root@localhost << OUTER
+          set -euo pipefail
+          mkdir -p /tmp/fact-tmp
+          cd /host/fact/tests
+          source .venv/bin/activate
+          pytest \
+            --image="${FACT_IMAGE}" \
+            --output=grpc \
+            --tmp-dir=/tmp/fact-tmp \
+            --junit-xml=results.xml \
+            -x \
+            test_file_open.py
+        OUTER
+
+    - name: Upload test results
+      if: always()
+      uses: actions/upload-artifact@v4
+      with:
+        name: lvh-test-results-${{ matrix.kernel }}
+        path: |
+          fact/tests/results.xml
+          fact/tests/logs/
+        if-no-files-found: ignore
+
+    - name: Test summary
+      uses: test-summary/action@v2
+      if: always()
+      with:
+        paths: fact/tests/results.xml
+
+    - name: Dump VM console on failure
+      if: failure()
+      run: cat /tmp/console.log
diff --git a/tests/conftest.py b/tests/conftest.py
index 7bcfa45e..e3cd8657 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -20,13 +20,27 @@
 pytest_plugins = ['test_editors.commons']
 
 
+def _tmp_dir_base(request: pytest.FixtureRequest) -> str:
+    """
+    Directory under which monitored/ignored test directories are
+    created.
+
+    Defaults to the current working directory, but can be overridden
+    with --tmp-dir, e.g. when the working directory lives on a
+    filesystem (such as 9p) that does not report file creation
+    metadata (FMODE_CREATED) correctly.
+    """
+    tmp_dir = request.config.getoption('--tmp-dir')
+    assert tmp_dir is None or isinstance(tmp_dir, str)
+    return tmp_dir or os.getcwd()
+
+
 @pytest.fixture
-def monitored_dir():
+def monitored_dir(request: pytest.FixtureRequest):
     """
     Create a temporary directory for tests and clean it up afterwards.
     """
-    cwd = os.getcwd()
-    tmp = mkdtemp(prefix='fact-test-', dir=cwd)
+    tmp = mkdtemp(prefix='fact-test-', dir=_tmp_dir_base(request))
     yield tmp
     rmtree(tmp)
 
@@ -46,13 +60,12 @@ def test_file(monitored_dir: str):
 
 
 @pytest.fixture
-def ignored_dir():
+def ignored_dir(request: pytest.FixtureRequest):
     """
     Create a temporary directory for tests that will not be monitored
     by fact. After tests are done, the directory is cleaned up.
     """
-    cwd = os.getcwd()
-    tmp = mkdtemp(prefix='fact-test-', dir=cwd)
+    tmp = mkdtemp(prefix='fact-test-', dir=_tmp_dir_base(request))
     yield tmp
     rmtree(tmp)
 
@@ -302,6 +315,7 @@ def fact(
         container.remove()
         pytest.fail('fact did not finish its initial scan')
 
+    sleep(1)
     yield container
 
     # Capture prometheus metrics before stopping the container
@@ -343,3 +357,14 @@ def pytest_addoption(parser: pytest.Parser):
         action='store_true',
         help='Do not build test containers locally',
     )
+    parser.addoption(
+        '--tmp-dir',
+        action='store',
+        default=None,
+        help=(
+            'Directory in which to create monitored/ignored test '
+            'directories (default: current working directory). Useful '
+            'when the working directory is on a filesystem that does '
+            'not report file creation metadata correctly, e.g. 9p.'
+        ),
+    )

From eb50f46fba5aaad26b9976c7f683e22669577d8e Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Thu, 17 Sep 2026 16:40:51 +0200
Subject: [PATCH 02/11] ci: run full integration test suite in LVH workflow
 (phase 2)

Expand the LVH-based integration tests from the Phase 1 spike
(test_file_open.py only) to the full pytest suite:

- Drop -x and the single test-file argument so all tests under
  tests/ run and results are collected in the JUnit report.
- Add --tb=short for more readable failure output in CI logs.
- Bump the job timeout from 60 to 90 minutes to accommodate the
  full suite plus on-the-fly container builds (editors, fedora,
  self-deleter).
- Document why --no-local-builds is intentionally omitted:
  quay.io/rhacs-eng/qa-multi-arch is a private registry that fork
  PR runs have no credentials for, so tests/containers.py's
  pull_or_build() needs to fall back to building those test
  containers locally from their Containerfiles on a 401/404.

Phase 1's test_file_open.py run already exercised the test_container
fixture (which pulls quay.io/fedora/fedora:43), confirming the VM has
outbound network access needed for the additional container builds.

Refs: #1794

Assisted-by: claude-opus-4-6 <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index d0e42aad..90978edd 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -42,7 +42,7 @@ jobs:
     name: Integration tests (LVH)
     needs: build
     runs-on: ubuntu-24.04
-    timeout-minutes: 60
+    timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
@@ -201,9 +201,7 @@ jobs:
             --image="${FACT_IMAGE}" \
             --output=grpc \
             --tmp-dir=/tmp/fact-tmp \
-            --junit-xml=results.xml \
-            -x \
-            test_file_open.py
+            --junit-xml=results.xml
         OUTER
 
     - name: Upload test results

From 4c4cfef17d4266e86bcda7567ed4f2a7f6b30394 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Thu, 17 Sep 2026 17:05:30 +0200
Subject: [PATCH 03/11] debug

---
 .github/workflows/lvh-integration-tests.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index 90978edd..c9bf345a 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -71,8 +71,9 @@ jobs:
 
     - name: Install LVH cli
       run: |
+        set -x
         make -C little-vm-helper install
-        sudo ln -s "$(go env GOPATH)/bin" /usr/local/bin/lvh
+        echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
 
     - name: Install VM dependencies
       run: |

From 7441e8a6eee4cfd3343aa4219dff81fa56c5ef21 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Thu, 17 Sep 2026 17:14:19 +0200
Subject: [PATCH 04/11] Properly install lvh

---
 .github/workflows/lvh-integration-tests.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index c9bf345a..b510e3f7 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -71,9 +71,8 @@ jobs:
 
     - name: Install LVH cli
       run: |
-        set -x
         make -C little-vm-helper install
-        echo "$(go env GOPATH)/bin" >> "$GITHUB_PATH"
+        sudo install -m 0755 "$(go env GOPATH)/bin/lvh" /usr/local/bin/lvh
 
     - name: Install VM dependencies
       run: |

From 7dacfcf5d64175e959145f0a2f3a52fd55f2d39c Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Thu, 17 Sep 2026 17:29:08 +0200
Subject: [PATCH 05/11] Silence make directory change

---
 .github/workflows/lvh-integration-tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index b510e3f7..a421d6bd 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -190,7 +190,7 @@ jobs:
         # misclassify file creations as generic "Open" events instead of
         # "Creation". Everything else (test code, venv, gRPC stubs, logs,
         # results) stays on /host/fact/tests as usual.
-        FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make -C fact image-name)"
+        FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make -sC fact image-name)"
         # shellcheck disable=SC2087
         ssh -p 2222 -o StrictHostKeyChecking=no root@localhost << OUTER
           set -euo pipefail

From 997314cecb1484e4862b24bb6b46195dfc5df6b1 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Thu, 17 Sep 2026 17:59:12 +0200
Subject: [PATCH 06/11] Properly handle docker errors

---
 tests/containers.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/tests/containers.py b/tests/containers.py
index af515dc2..e387c24b 100644
--- a/tests/containers.py
+++ b/tests/containers.py
@@ -22,6 +22,17 @@ def read_qa_tag() -> str:
 )
 
 
+def _is_pull_auth_or_missing_error(e: docker.errors.APIError) -> bool:
+    if e.status_code in (401, 403, 404):
+        return True
+
+    # Some daemon/registry combinations (e.g. containerd-backed pulls)
+    # wrap a registry-level 401/404 in a generic 500 Server Error, only
+    # surfacing the real cause in the explanation text.
+    explanation = (e.explanation or '').lower()
+    return 'unauthorized' in explanation or 'not found' in explanation
+
+
 def pull_or_build(
     docker_client: docker.DockerClient,
     tag: str,
@@ -43,7 +54,7 @@ def pull_or_build(
         if no_local_builds:
             raise e
 
-        if e.status_code != 401 and e.status_code != 404:
+        if not _is_pull_auth_or_missing_error(e):
             raise e
         print(f'Failed to pull image: {e}')
         print('Attempting to build image from source')

From 1edd766312580f0876dc86169800c06bd25cd427 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Fri, 18 Sep 2026 12:41:03 +0200
Subject: [PATCH 07/11] ci: replace LVH with raw QEMU/virtiofs for integration
 tests

Replace the LVH (little-vm-helper) based CI with a standalone
hack/qemu-vm.sh script that boots cloud images directly in QEMU/KVM.
This removes the Go toolchain, LVH build, and LVH OCI image/kernel
dependencies in favor of standard qcow2 cloud images.

hack/qemu-vm.sh:
  Reusable script (start/ssh/stop) for booting cloud images in
  QEMU/KVM.  Handles qcow2 overlay creation, ephemeral SSH key
  generation, cloud-init seed ISO with user-provided templates
  (__SSH_PUBKEY__ placeholder), virtiofsd for host directory sharing,
  two-boot BPF LSM activation via grubby, and Docker verification.
  Works both in CI and locally for reproducing test failures.

hack/cloud-init/{centos,fedora,ubuntu}.yml:
  Per-distro cloud-init templates configuring root SSH access, Docker
  CE installation, and BPF LSM kernel cmdline.

Workflow changes:
  - Use CentOS Stream 10 GenericCloud qcow2 image with SHA256
    verification.
  - Matrix uses named distro entries (extensible to other distros by
    adding image URLs and cloud-init templates).
  - Host directory shared via virtiofs instead of 9p.  virtiofs
    correctly reports FMODE_CREATED (unlike 9p), but does not support
    xattrs, so --tmp-dir is still needed for test directories.
  - Fact image loaded via virtiofs mount instead of scp.
  - No more LVH clone, Go setup, kernel/image pull, or libvirt deps.
  - VM lifecycle (start/stop) managed via hack/qemu-vm.sh.

Refs: #1794

Assisted-by: claude-opus-4-6 <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml | 210 +++++---------
 hack/cloud-init/centos.yml                  |  19 ++
 hack/cloud-init/fedora.yml                  |  19 ++
 hack/cloud-init/ubuntu.yml                  |  29 ++
 hack/qemu-vm.sh                             | 303 ++++++++++++++++++++
 5 files changed, 442 insertions(+), 138 deletions(-)
 create mode 100644 hack/cloud-init/centos.yml
 create mode 100644 hack/cloud-init/fedora.yml
 create mode 100644 hack/cloud-init/ubuntu.yml
 create mode 100755 hack/qemu-vm.sh

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index a421d6bd..00225f29 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -1,15 +1,16 @@
-name: LVH Integration Tests
+name: QEMU Integration Tests
 
 on:
   pull_request:
   workflow_dispatch:
 
 concurrency:
-  group: ${{ github.head_ref || github.run_id }}-lvh
+  group: ${{ github.head_ref || github.run_id }}-qemu
   cancel-in-progress: true
 
 env:
-  LVH_VERSION: v0.0.31
+  VM_DIR: /tmp/fact-vm
+  SSH_PORT: '2222'
 
 jobs:
   build:
@@ -39,187 +40,120 @@ jobs:
         retention-days: 1
 
   integration-tests:
-    name: Integration tests (LVH)
+    name: Integration tests (QEMU)
     needs: build
     runs-on: ubuntu-24.04
     timeout-minutes: 90
     strategy:
       fail-fast: false
       matrix:
-        kernel:
-        - '6.12-main'
+        include:
+        - name: centos-10-stream
+          image_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2
+          checksum_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2.SHA256SUM
     steps:
     - name: Checkout fact
       uses: actions/checkout@v7
       with:
         submodules: true
         fetch-depth: 0
-        path: fact
         persist-credentials: false
 
-    - name: Clone little-vm-helper
-      uses: actions/checkout@v7
-      with:
-        repository: cilium/little-vm-helper
-        ref: ${{ env.LVH_VERSION }}
-        path: little-vm-helper
-        persist-credentials: false
-
-    - uses: actions/setup-go@v7
-      with:
-        go-version-file: little-vm-helper/go.mod
-
-    - name: Install LVH cli
-      run: |
-        make -C little-vm-helper install
-        sudo install -m 0755 "$(go env GOPATH)/bin/lvh" /usr/local/bin/lvh
-
-    - name: Install VM dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y --no-install-recommends \
-          qemu-system-x86 \
-          qemu-utils \
-          cpu-checker \
-          libvirt-daemon-system \
-          libvirt-clients \
-          virtinst
-
-    - name: Create LVH data directories
+    - name: Install QEMU
       run: |
-        sudo mkdir -p /tmp/lvh-data/kernels /tmp/lvh-data/images
-        sudo chmod -R 777 /tmp/lvh-data
+        sudo apt-get update -qq
+        sudo apt-get install -y -qq --no-install-recommends \
+          qemu-system-x86 qemu-utils genisoimage virtiofsd
+        [ -w /dev/kvm ] || sudo chmod 666 /dev/kvm
 
-    - name: Cache LVH kernel
-      uses: actions/cache@v4
-      id: cache-kernel
-      with:
-        path: /tmp/lvh-data/kernels/${{ matrix.kernel }}
-        key: lvh-kernel-${{ matrix.kernel }}
-
-    - name: Fetch kernel
-      if: steps.cache-kernel.outputs.cache-hit != 'true'
+    - name: Download VM base image
       run: |
-        lvh kernels pull \
-          --dir /tmp/lvh-data/kernels \
-          "${{ matrix.kernel }}"
-
-    - name: Cache LVH VM image
-      uses: actions/cache@v4
-      id: cache-image
-      with:
-        path: /tmp/lvh-data/images/images
-        key: lvh-image-kind-${{ matrix.kernel }}
-
-    - name: Fetch VM image
-      if: steps.cache-image.outputs.cache-hit != 'true'
-      run: |
-        lvh images pull \
-          --dir /tmp/lvh-data/images \
-          "quay.io/lvh-images/kind:${{ matrix.kernel }}"
+        mkdir -p "${VM_DIR}"
+        curl -fSL --progress-bar -o "${VM_DIR}/base.qcow2" "${{ matrix.image_url }}"
+        expected="$(curl -fsSL "${{ matrix.checksum_url }}" | awk -F'= ' '/^SHA256/{print $2}')"
+        actual="$(sha256sum "${VM_DIR}/base.qcow2" | awk '{print $1}')"
+        if [[ "${expected}" != "${actual}" ]]; then
+          echo "::error::Checksum mismatch: expected ${expected}, got ${actual}"
+          rm -f "${VM_DIR}/base.qcow2"
+          exit 1
+        fi
 
     - name: Download fact image artifact
       uses: actions/download-artifact@v4
       with:
         name: fact-image
-        path: /tmp
 
     - name: Start VM
       run: |
-        KERNEL_PATH=$(find /tmp/lvh-data/kernels -name "vmlinuz-*" -type f | head -1)
-        IMAGE_PATH=$(find /tmp/lvh-data/images -name "*.qcow2" -type f | head -1)
-        CPU="$(nproc)"
-        MEM="$(free -m | awk '/^Mem:/{print int($2 * 0.75)}')M"
-
-        echo "Kernel: ${KERNEL_PATH}"
-        echo "Image: ${IMAGE_PATH}"
-        echo "CPU: ${CPU}, MEM: ${MEM}"
-
-        sudo touch /tmp/console.log
-        sudo lvh run \
-          --image "${IMAGE_PATH}" \
-          --kernel "${KERNEL_PATH}" \
-          --host-mount "${{ github.workspace }}" \
-          --daemonize \
-          -p 2222:22 \
-          --cpu "${CPU}" \
-          --mem "${MEM}" \
-          --cpu-kind host \
-          --console-log-file /tmp/console.log \
-          --append "lsm=lockdown,capability,yama,apparmor,bpf"
-
-    - name: Wait for VM SSH
-      run: |
-        for i in $(seq 1 120); do
-          if ssh -p 2222 -o StrictHostKeyChecking=no -o ConnectTimeout=2 root@localhost exit 2>/dev/null; then
-            echo "VM is ready after ${i} seconds"
-            exit 0
-          fi
-          sleep 1
-        done
-        echo "VM failed to become ready"
-        cat /tmp/console.log
-        exit 1
+        hack/qemu-vm.sh start \
+          --image "${VM_DIR}/base.qcow2" \
+          --cloud-init hack/cloud-init/centos.yml \
+          --vm-dir "${VM_DIR}" \
+          --ssh-port "${SSH_PORT}" \
+          --host-mount "${{ github.workspace }}"
 
     - name: Load fact image into VM
       run: |
-        scp -P 2222 -o StrictHostKeyChecking=no \
-          /tmp/fact-image.tar root@localhost:/tmp/fact-image.tar
-        ssh -p 2222 -o StrictHostKeyChecking=no root@localhost \
-          "docker load -i /tmp/fact-image.tar && rm /tmp/fact-image.tar"
+        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
+          docker load -i /host/fact-image.tar
 
     - name: Setup test environment in VM
       run: |
-        ssh -p 2222 -o StrictHostKeyChecking=no root@localhost << 'EOF'
-          set -euo pipefail
-          apt-get update -qq
-          apt-get install -y -qq python3-venv python3-dev gcc > /dev/null 2>&1
-          cd /host/fact/tests
-          python3 -m venv .venv
-          source .venv/bin/activate
-          pip install -r requirements.txt
-          make grpc-gen
-        EOF
+        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
+          bash -c '
+            set -euo pipefail
+            dnf -y -q install python3 python3-pip python3-devel gcc
+            cd /host/tests
+            python3 -m venv .venv
+            source .venv/bin/activate
+            pip install -q -r requirements.txt
+            make grpc-gen
+          '
 
     - name: Run integration tests
       run: |
-        # NOTE: --tmp-dir points monitored/ignored test directories at the
-        # VM's local tmpfs instead of the 9p-mounted workspace (/host).
-        # The 9p (virtio-9p) filesystem used for --host-mount does not set
-        # FMODE_CREATED on newly created files, which makes fact
-        # misclassify file creations as generic "Open" events instead of
-        # "Creation". Everything else (test code, venv, gRPC stubs, logs,
-        # results) stays on /host/fact/tests as usual.
-        FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make -sC fact image-name)"
-        # shellcheck disable=SC2087
-        ssh -p 2222 -o StrictHostKeyChecking=no root@localhost << OUTER
-          set -euo pipefail
-          mkdir -p /tmp/fact-tmp
-          cd /host/fact/tests
-          source .venv/bin/activate
-          pytest \
-            --image="${FACT_IMAGE}" \
-            --output=grpc \
-            --tmp-dir=/tmp/fact-tmp \
-            --junit-xml=results.xml
-        OUTER
+        # NOTE: --no-local-builds is intentionally omitted. The editor,
+        # fedora, and self-deleter test containers are normally pulled
+        # from quay.io/rhacs-eng/qa-multi-arch, which is a private
+        # registry that fork PR runs have no credentials for. Without
+        # --no-local-builds, tests/containers.py::pull_or_build() falls
+        # back to building those images locally from their Containerfiles
+        # on a pull error.
+        FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make --no-print-directory image-name)"
+        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
+          bash -c "
+            set -euo pipefail
+            mkdir -p /tmp/fact-tmp
+            cd /host/tests
+            source .venv/bin/activate
+            pytest \
+              --image='${FACT_IMAGE}' \
+              --output=grpc \
+              --tmp-dir=/tmp/fact-tmp \
+              --junit-xml=results.xml \
+              --tb=short
+          "
 
     - name: Upload test results
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: lvh-test-results-${{ matrix.kernel }}
+        name: qemu-test-results-${{ matrix.name }}
         path: |
-          fact/tests/results.xml
-          fact/tests/logs/
+          tests/results.xml
+          tests/logs/
         if-no-files-found: ignore
 
     - name: Test summary
       uses: test-summary/action@v2
       if: always()
       with:
-        paths: fact/tests/results.xml
+        paths: tests/results.xml
 
     - name: Dump VM console on failure
       if: failure()
-      run: cat /tmp/console.log
+      run: cat "${VM_DIR}/console.log" 2>/dev/null || true
+
+    - name: Stop VM
+      if: always()
+      run: hack/qemu-vm.sh stop --vm-dir "${VM_DIR}"
diff --git a/hack/cloud-init/centos.yml b/hack/cloud-init/centos.yml
new file mode 100644
index 00000000..41b3d4f2
--- /dev/null
+++ b/hack/cloud-init/centos.yml
@@ -0,0 +1,19 @@
+#cloud-config
+# Cloud-init user-data for CentOS Stream 10 (and RHEL 10).
+# Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
+
+users:
+  - name: root
+    lock_passwd: false
+    ssh_authorized_keys:
+      - __SSH_PUBKEY__
+
+ssh_pwauth: false
+
+runcmd:
+  - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf"
+  - dnf -y install dnf-plugins-core
+  - dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
+  - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
+  - systemctl enable --now docker
+  - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/cloud-init/fedora.yml b/hack/cloud-init/fedora.yml
new file mode 100644
index 00000000..ad5968ba
--- /dev/null
+++ b/hack/cloud-init/fedora.yml
@@ -0,0 +1,19 @@
+#cloud-config
+# Cloud-init user-data for Fedora (40+).
+# Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
+
+users:
+  - name: root
+    lock_passwd: false
+    ssh_authorized_keys:
+      - __SSH_PUBKEY__
+
+ssh_pwauth: false
+
+runcmd:
+  - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf"
+  - dnf -y install dnf-plugins-core
+  - dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
+  - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
+  - systemctl enable --now docker
+  - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/cloud-init/ubuntu.yml b/hack/cloud-init/ubuntu.yml
new file mode 100644
index 00000000..281160f0
--- /dev/null
+++ b/hack/cloud-init/ubuntu.yml
@@ -0,0 +1,29 @@
+#cloud-config
+# Cloud-init user-data for Ubuntu (24.04+).
+# Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
+#
+# NOTE: Ubuntu uses AppArmor by default; the LSM list must include it
+# alongside bpf.  The grubby tool is not available — use GRUB config
+# directly.
+
+users:
+  - name: root
+    lock_passwd: false
+    ssh_authorized_keys:
+      - __SSH_PUBKEY__
+
+ssh_pwauth: false
+
+runcmd:
+  - sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT="lsm=lockdown,capability,yama,apparmor,bpf"/' /etc/default/grub
+  - update-grub
+  - apt-get update -qq
+  - apt-get install -y -qq ca-certificates curl
+  - install -m 0755 -d /etc/apt/keyrings
+  - curl -fsSL https://download.docker.com/linux/ubuntu/gpg -o /etc/apt/keyrings/docker.asc
+  - chmod a+r /etc/apt/keyrings/docker.asc
+  - echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/ubuntu $(. /etc/os-release && echo "$VERSION_CODENAME") stable" > /etc/apt/sources.list.d/docker.list
+  - apt-get update -qq
+  - apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin
+  - systemctl enable --now docker
+  - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/qemu-vm.sh b/hack/qemu-vm.sh
new file mode 100755
index 00000000..61488dd0
--- /dev/null
+++ b/hack/qemu-vm.sh
@@ -0,0 +1,303 @@
+#!/usr/bin/env bash
+# Boot a cloud image in QEMU/KVM for integration testing.
+#
+# Designed to work both locally and in CI (GitHub Actions).  The script
+# takes a pre-downloaded qcow2 cloud image, injects cloud-init config,
+# boots the VM, waits until it is ready, and prints SSH connection
+# details.
+#
+# Usage:
+#   hack/qemu-vm.sh start   [options]   — boot the VM
+#   hack/qemu-vm.sh ssh     [options]   — open an SSH session to the VM
+#   hack/qemu-vm.sh stop    [options]   — kill the VM
+#
+# Options:
+#   --image PATH          path to the base qcow2 image (required for start)
+#   --cloud-init PATH     cloud-init user-data template (required for start)
+#   --vm-dir DIR          working directory for VM files (default: /tmp/fact-vm)
+#   --ssh-port PORT       host port forwarded to guest 22 (default: 2222)
+#   --cpu N               vCPUs (default: all host CPUs)
+#   --mem SIZE            memory, e.g. 8G (default: 75% of host RAM)
+#   --host-mount DIR      directory to share with the VM via virtiofs at /host
+#
+# Cloud-init templates:
+#   The --cloud-init file is a standard #cloud-config YAML with one
+#   special placeholder: __SSH_PUBKEY__ is replaced with the VM's
+#   ephemeral SSH public key.  See hack/cloud-init/ for examples.
+#
+# The VM state (image overlay, cloud-init ISO, SSH keys, PID file) lives
+# entirely under --vm-dir and can be cleaned up by removing that directory.
+
+set -euo pipefail
+
+: "${IMAGE:=}"
+: "${CLOUD_INIT:=}"
+: "${VM_DIR:=/tmp/fact-vm}"
+: "${SSH_PORT:=2222}"
+: "${CPU:=$(nproc)}"
+: "${MEM:=$(awk '/^MemTotal/{printf "%dG", int($2/1024/1024*0.75)}' /proc/meminfo)}"
+: "${HOST_MOUNT:=}"
+
+SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR)
+
+usage() {
+    sed -n '2,/^$/s/^# \{0,1\}//p' "$0"
+    exit 1
+}
+
+log() { printf '==> %s\n' "$*" >&2; }
+
+EXTRA_ARGS=()
+
+parse_args() {
+    while [[ $# -gt 0 ]]; do
+        case "$1" in
+            --image)        IMAGE="$2";        shift 2;;
+            --cloud-init)   CLOUD_INIT="$2";   shift 2;;
+            --vm-dir)       VM_DIR="$2";       shift 2;;
+            --ssh-port)     SSH_PORT="$2";     shift 2;;
+            --cpu)          CPU="$2";          shift 2;;
+            --mem)          MEM="$2";          shift 2;;
+            --host-mount)   HOST_MOUNT="$2";   shift 2;;
+            -h|--help)      usage;;
+            --)             shift; EXTRA_ARGS+=("$@"); return;;
+            *)              EXTRA_ARGS+=("$@"); return;;
+        esac
+    done
+}
+
+create_overlay() {
+    local base
+    base="$(realpath "${IMAGE}")"
+    local overlay="${VM_DIR}/disk.qcow2"
+    if [[ -f "${overlay}" ]]; then
+        log "Overlay already exists, removing stale one"
+        rm -f "${overlay}"
+    fi
+    log "Creating qcow2 overlay"
+    qemu-img create -f qcow2 -b "${base}" -F qcow2 "${overlay}" 20G >/dev/null
+}
+
+generate_ssh_key() {
+    local key="${VM_DIR}/id_ed25519"
+    if [[ -f "${key}" ]]; then
+        return
+    fi
+    log "Generating ephemeral SSH key"
+    ssh-keygen -t ed25519 -f "${key}" -N "" -q
+}
+
+create_cloud_init_iso() {
+    local iso="${VM_DIR}/seed.iso"
+    local pubkey
+    pubkey="$(cat "${VM_DIR}/id_ed25519.pub")"
+
+    local ci_dir="${VM_DIR}/cloud-init"
+    mkdir -p "${ci_dir}"
+
+    cat > "${ci_dir}/meta-data" <<EOF
+instance-id: fact-test-vm
+local-hostname: fact-test-vm
+EOF
+
+    log "Generating user-data from ${CLOUD_INIT}"
+    sed "s|__SSH_PUBKEY__|${pubkey}|g" "${CLOUD_INIT}" > "${ci_dir}/user-data"
+
+    log "Creating cloud-init seed ISO"
+    genisoimage -output "${iso}" -volid cidata -joliet -rock \
+        "${ci_dir}/user-data" "${ci_dir}/meta-data" 2>/dev/null
+}
+
+find_virtiofsd() {
+    local bin
+    bin="$(command -v virtiofsd 2>/dev/null)" && { echo "${bin}"; return; }
+    for p in /usr/libexec/virtiofsd /usr/lib/virtiofsd; do
+        [[ -x "${p}" ]] && { echo "${p}"; return; }
+    done
+    echo "error: virtiofsd not found" >&2
+    return 1
+}
+
+start_virtiofsd() {
+    local sock="${VM_DIR}/virtiofsd.sock"
+    local bin
+    bin="$(find_virtiofsd)"
+    log "Starting virtiofsd for ${HOST_MOUNT}"
+    "${bin}" \
+        --socket-path="${sock}" \
+        --shared-dir="${HOST_MOUNT}" \
+        --cache=always &
+    echo $! > "${VM_DIR}/virtiofsd.pid"
+
+    local i
+    for i in $(seq 1 30); do
+        if [[ -S "${sock}" ]]; then
+            log "virtiofsd ready after ${i}s"
+            return 0
+        fi
+        sleep 1
+    done
+    log "virtiofsd socket did not appear after 30s"
+    return 1
+}
+
+build_qemu_args() {
+    # shellcheck disable=SC2054 # commas are inside quoted QEMU option values
+    local args=(
+        -nodefaults
+        -display none
+        -daemonize
+        -pidfile "${VM_DIR}/qemu.pid"
+        -enable-kvm
+        -cpu host
+        -smp "${CPU}"
+        -m "${MEM}"
+        -drive "file=${VM_DIR}/disk.qcow2,if=virtio,format=qcow2"
+        -drive "file=${VM_DIR}/seed.iso,if=virtio,format=raw,readonly=on"
+        -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22"
+        -device virtio-net-pci,netdev=net0
+        -serial "file:${VM_DIR}/console.log"
+    )
+
+    if [[ -n "${HOST_MOUNT}" ]]; then
+        args+=(
+            -object "memory-backend-memfd,id=mem,size=${MEM},share=on"
+            -numa "node,memdev=mem"
+            -chardev "socket,id=char0,path=${VM_DIR}/virtiofsd.sock"
+            -device "vhost-user-fs-pci,chardev=char0,tag=host_mount"
+        )
+    fi
+
+    printf '%s\n' "${args[@]}"
+}
+
+vm_ssh() {
+    ssh -p "${SSH_PORT}" -i "${VM_DIR}/id_ed25519" "${SSH_OPTS[@]}" root@localhost "$@"
+}
+
+wait_for_ssh() {
+    log "Waiting for SSH (port ${SSH_PORT})..."
+    local i
+    for i in $(seq 1 180); do
+        if vm_ssh true 2>/dev/null; then
+            log "SSH ready after ${i}s"
+            return 0
+        fi
+        sleep 1
+    done
+    log "SSH failed to become ready after 180s"
+    [[ -f "${VM_DIR}/console.log" ]] && cat "${VM_DIR}/console.log" >&2
+    return 1
+}
+
+wait_for_cloud_init() {
+    log "Waiting for cloud-init to finish..."
+    local i
+    for i in $(seq 1 300); do
+        if vm_ssh "test -f /var/lib/cloud/instance/boot-finished-user" 2>/dev/null; then
+            log "cloud-init finished after ${i}s"
+            return 0
+        fi
+        sleep 1
+    done
+    log "cloud-init did not finish within 300s"
+    vm_ssh "cat /var/log/cloud-init-output.log" 2>/dev/null >&2 || true
+    return 1
+}
+
+cmd_start() {
+    if [[ -z "${IMAGE}" ]]; then
+        echo "error: --image is required for start" >&2
+        exit 1
+    fi
+    if [[ ! -f "${IMAGE}" ]]; then
+        echo "error: image not found: ${IMAGE}" >&2
+        exit 1
+    fi
+    if [[ -z "${CLOUD_INIT}" ]]; then
+        echo "error: --cloud-init is required for start" >&2
+        exit 1
+    fi
+    if [[ ! -f "${CLOUD_INIT}" ]]; then
+        echo "error: cloud-init template not found: ${CLOUD_INIT}" >&2
+        exit 1
+    fi
+
+    mkdir -p "${VM_DIR}"
+    create_overlay
+    generate_ssh_key
+    create_cloud_init_iso
+
+    if [[ -n "${HOST_MOUNT}" ]]; then
+        start_virtiofsd
+    fi
+
+    log "Starting QEMU (cpu=${CPU}, mem=${MEM}, ssh_port=${SSH_PORT})"
+    touch "${VM_DIR}/console.log"
+    local qemu_args
+    mapfile -t qemu_args < <(build_qemu_args)
+    qemu-system-x86_64 "${qemu_args[@]}"
+
+    wait_for_ssh
+    wait_for_cloud_init
+
+    log "Rebooting for BPF LSM kernel cmdline change..."
+    vm_ssh "reboot" 2>/dev/null || true
+    sleep 5
+    wait_for_ssh
+
+    log "Verifying BPF LSM is active"
+    local lsm
+    lsm="$(vm_ssh "cat /sys/kernel/security/lsm")"
+    if [[ "${lsm}" != *bpf* ]]; then
+        log "ERROR: bpf not found in LSM list: ${lsm}"
+        return 1
+    fi
+    log "LSM list: ${lsm}"
+
+    log "Verifying Docker is running"
+    vm_ssh "docker info" >/dev/null
+
+    if [[ -n "${HOST_MOUNT}" ]]; then
+        log "Mounting host filesystem inside VM"
+        vm_ssh "mkdir -p /host && mount -t virtiofs host_mount /host"
+    fi
+
+    log "VM is ready"
+    log "  SSH: ssh -p ${SSH_PORT} -i ${VM_DIR}/id_ed25519 ${SSH_OPTS[*]} root@localhost"
+}
+
+cmd_ssh() {
+    vm_ssh "${EXTRA_ARGS[@]}"
+}
+
+kill_pid_file() {
+    local pidfile="$1" label="$2"
+    if [[ -f "${pidfile}" ]]; then
+        local pid
+        pid="$(cat "${pidfile}")"
+        log "Stopping ${label} (pid ${pid})"
+        kill "${pid}" 2>/dev/null || true
+        rm -f "${pidfile}"
+    fi
+}
+
+cmd_stop() {
+    kill_pid_file "${VM_DIR}/qemu.pid" "VM"
+    kill_pid_file "${VM_DIR}/virtiofsd.pid" "virtiofsd"
+}
+
+main() {
+    local cmd="${1:-help}"
+    shift || true
+    parse_args "$@"
+
+    case "${cmd}" in
+        start) cmd_start;;
+        ssh)   cmd_ssh;;
+        stop)  cmd_stop;;
+        *)     usage;;
+    esac
+}
+
+main "$@"

From dc5d1fa324bf8f1dcb3201315106679f6d7b4c38 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Fri, 18 Sep 2026 15:25:53 +0200
Subject: [PATCH 08/11] ci: test against CentOS 9/10, Ubuntu 24.04/26.04, and
 Fedora 44

Expand the QEMU integration test matrix from a single CentOS Stream
10 entry to five distros:

- centos-9-stream
- centos-10-stream
- ubuntu-24.04
- ubuntu-26.04
- fedora-44

Each entry now carries its own cloud_init template so the matrix can
mix distro families instead of hardcoding hack/cloud-init/centos.yml.

Fedora 45 is not yet released (only unstable, date-stamped nightly
composes exist); Fedora 44 is used as the current stable release.

Checksum verification switched from manually extracting and
comparing hashes to sha256sum -c directly: the checksum file is
filtered down to the line(s) matching our image's filename (via
grep -F), then piped to sha256sum -c, which auto-detects both GNU
("hash  filename") and BSD ("SHA256 (filename) = hash") tagged
formats and correctly ignores '#'-prefixed comment lines. This also
made the extraction robust against Ubuntu's SHA256SUMS listing many
unrelated images and Fedora's PGP-clearsigned CHECKSUM files.

Moved test-runner dependency installation (python3, pip, venv, gcc)
from a post-boot SSH step into each cloud-init template's runcmd, so
it runs in parallel with the Docker setup during the boot we already
wait for, instead of adding a second SSH round-trip.

Refs: #1794

Assisted-by: claude-opus-4-6 <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml | 42 +++++++++++++++------
 hack/cloud-init/centos.yml                  |  1 +
 hack/cloud-init/fedora.yml                  |  7 ++--
 hack/cloud-init/ubuntu.yml                  | 15 ++++++--
 4 files changed, 48 insertions(+), 17 deletions(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index 00225f29..48dabb94 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -40,7 +40,7 @@ jobs:
         retention-days: 1
 
   integration-tests:
-    name: Integration tests (QEMU)
+    name: Integration tests (${{ matrix.name }})
     needs: build
     runs-on: ubuntu-24.04
     timeout-minutes: 90
@@ -48,9 +48,26 @@ jobs:
       fail-fast: false
       matrix:
         include:
+        - name: centos-9-stream
+          image_url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2
+          checksum_url: https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM
+          cloud_init: hack/cloud-init/centos.yml
         - name: centos-10-stream
           image_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2
           checksum_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2.SHA256SUM
+          cloud_init: hack/cloud-init/centos.yml
+        - name: ubuntu-24.04
+          image_url: https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img
+          checksum_url: https://cloud-images.ubuntu.com/releases/24.04/release/SHA256SUMS
+          cloud_init: hack/cloud-init/ubuntu.yml
+        - name: ubuntu-26.04
+          image_url: https://cloud-images.ubuntu.com/releases/26.04/release/ubuntu-26.04-server-cloudimg-amd64.img
+          checksum_url: https://cloud-images.ubuntu.com/releases/26.04/release/SHA256SUMS
+          cloud_init: hack/cloud-init/ubuntu.yml
+        - name: fedora-44
+          image_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2
+          checksum_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM
+          cloud_init: hack/cloud-init/fedora.yml
     steps:
     - name: Checkout fact
       uses: actions/checkout@v7
@@ -69,14 +86,18 @@ jobs:
     - name: Download VM base image
       run: |
         mkdir -p "${VM_DIR}"
-        curl -fSL --progress-bar -o "${VM_DIR}/base.qcow2" "${{ matrix.image_url }}"
-        expected="$(curl -fsSL "${{ matrix.checksum_url }}" | awk -F'= ' '/^SHA256/{print $2}')"
-        actual="$(sha256sum "${VM_DIR}/base.qcow2" | awk '{print $1}')"
-        if [[ "${expected}" != "${actual}" ]]; then
-          echo "::error::Checksum mismatch: expected ${expected}, got ${actual}"
-          rm -f "${VM_DIR}/base.qcow2"
-          exit 1
-        fi
+        image_name="$(basename "${{ matrix.image_url }}")"
+        curl -fSL --progress-bar -o "${VM_DIR}/${image_name}" "${{ matrix.image_url }}"
+
+        # Checksum files may list multiple images (Ubuntu SHA256SUMS) or
+        # use either GNU ("hash  filename") or BSD ("SHA256 (filename) =
+        # hash") tagged format. sha256sum -c auto-detects both, so we
+        # only need to filter the file down to our image's line.
+        curl -fsSL "${{ matrix.checksum_url }}" | grep -F "${image_name}" > "${VM_DIR}/${image_name}.sum"
+        (cd "${VM_DIR}" && sha256sum -c "${image_name}.sum")
+        rm -f "${VM_DIR}/${image_name}.sum"
+
+        mv "${VM_DIR}/${image_name}" "${VM_DIR}/base.qcow2"
 
     - name: Download fact image artifact
       uses: actions/download-artifact@v4
@@ -87,7 +108,7 @@ jobs:
       run: |
         hack/qemu-vm.sh start \
           --image "${VM_DIR}/base.qcow2" \
-          --cloud-init hack/cloud-init/centos.yml \
+          --cloud-init "${{ matrix.cloud_init }}" \
           --vm-dir "${VM_DIR}" \
           --ssh-port "${SSH_PORT}" \
           --host-mount "${{ github.workspace }}"
@@ -102,7 +123,6 @@ jobs:
         hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
           bash -c '
             set -euo pipefail
-            dnf -y -q install python3 python3-pip python3-devel gcc
             cd /host/tests
             python3 -m venv .venv
             source .venv/bin/activate
diff --git a/hack/cloud-init/centos.yml b/hack/cloud-init/centos.yml
index 41b3d4f2..c99ae7be 100644
--- a/hack/cloud-init/centos.yml
+++ b/hack/cloud-init/centos.yml
@@ -16,4 +16,5 @@ runcmd:
   - dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
   - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
+  - dnf -y install python3 python3-pip python3-devel gcc
   - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/cloud-init/fedora.yml b/hack/cloud-init/fedora.yml
index ad5968ba..13030a2e 100644
--- a/hack/cloud-init/fedora.yml
+++ b/hack/cloud-init/fedora.yml
@@ -1,5 +1,5 @@
 #cloud-config
-# Cloud-init user-data for Fedora (40+).
+# Cloud-init user-data for Fedora (41+, dnf5-based).
 # Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
 
 users:
@@ -12,8 +12,9 @@ ssh_pwauth: false
 
 runcmd:
   - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf"
-  - dnf -y install dnf-plugins-core
-  - dnf config-manager --add-repo https://download.docker.com/linux/fedora/docker-ce.repo
+  - dnf -y install dnf5-plugins
+  - dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo
   - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
+  - dnf -y install python3 python3-pip python3-devel gcc
   - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/cloud-init/ubuntu.yml b/hack/cloud-init/ubuntu.yml
index 281160f0..9e39b315 100644
--- a/hack/cloud-init/ubuntu.yml
+++ b/hack/cloud-init/ubuntu.yml
@@ -3,8 +3,9 @@
 # Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
 #
 # NOTE: Ubuntu uses AppArmor by default; the LSM list must include it
-# alongside bpf.  The grubby tool is not available — use GRUB config
-# directly.
+# alongside bpf. The grubby tool is not available — use a GRUB
+# drop-in config file instead (see runcmd below for why a plain
+# /etc/default/grub edit does not work).
 
 users:
   - name: root
@@ -15,7 +16,14 @@ users:
 ssh_pwauth: false
 
 runcmd:
-  - sed -i 's/^GRUB_CMDLINE_LINUX_DEFAULT=.*/GRUB_CMDLINE_LINUX_DEFAULT="lsm=lockdown,capability,yama,apparmor,bpf"/' /etc/default/grub
+  # Ubuntu cloud images ship /etc/default/grub.d/50-cloudimg-settings.cfg,
+  # which sets GRUB_CMDLINE_LINUX_DEFAULT and is sourced by grub-mkconfig
+  # AFTER /etc/default/grub, clobbering any edits made there. Write our
+  # own drop-in with a higher-sorting filename so it wins instead.
+  - |
+    cat > /etc/default/grub.d/99-fact-lsm.cfg <<'EOF'
+    GRUB_CMDLINE_LINUX_DEFAULT="console=tty1 console=ttyS0 lsm=lockdown,capability,yama,apparmor,bpf"
+    EOF
   - update-grub
   - apt-get update -qq
   - apt-get install -y -qq ca-certificates curl
@@ -26,4 +34,5 @@ runcmd:
   - apt-get update -qq
   - apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
+  - apt-get install -y -qq python3-venv python3-dev gcc make
   - touch /var/lib/cloud/instance/boot-finished-user

From 6657d70caaec2691254cfedaeede048b8eec35bd Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Mon, 21 Sep 2026 11:45:39 +0200
Subject: [PATCH 09/11] ci: add RHCOS support, simplify LSM handling, drop
 Ubuntu

Assume the bpf LSM is already active by default on CentOS, Fedora,
and RHCOS images, removing the two-boot grubby dance from
hack/qemu-vm.sh and the cloud-init templates. The VM is now booted
once and /sys/kernel/security/lsm is checked directly.

Add RHCOS support:
  - hack/ignition/rhcos.json: minimal Ignition config (SSH key for
    the 'core' user, enables podman.socket). No package layering.
  - hack/qemu-vm.sh: new --ignition (mutually exclusive with
    --cloud-init, injected via QEMU's opt/com.coreos/config fw_cfg
    entry instead of a cloud-init seed ISO), --ssh-user (defaults to
    root; non-root users are transparently sudo-wrapped in vm_ssh),
    and --runtime (docker/podman, used for the post-boot sanity
    check) options. Also fixes a latent SSH argument-splitting bug:
    ssh naively space-joins multiple trailing arguments before
    sending them to the remote shell, corrupting multi-line
    "bash -c '...'" scripts; vm_ssh now shell-quotes and joins them
    into a single argument itself.
  - workflow: new rhcos-4.16 matrix entry. Its qcow2 has no stable
    download URL, so a dedicated step resolves the current build's
    location and checksums from openshift/installer's per-stream
    rhcos.json metadata (no Red Hat subscription required), then
    downloads, verifies, and decompresses the qcow2.gz artifact.
    Podman's Docker-API-compatible socket is exposed via DOCKER_HOST
    so the existing docker Python SDK in tests/ works unmodified.

Drop the ubuntu-24.04/ubuntu-26.04 matrix entries (flaky, not part
of fact's supported OS matrix). hack/cloud-init/ubuntu.yml is kept
for local use via hack/qemu-vm.sh.

Refs: #1794

Assisted-by: claude-opus-4-6 <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml |  97 +++++++++++++----
 hack/cloud-init/centos.yml                  |   3 +-
 hack/cloud-init/fedora.yml                  |   3 +-
 hack/cloud-init/ubuntu.yml                  |   2 +-
 hack/ignition/rhcos.json                    |  23 ++++
 hack/qemu-vm.sh                             | 112 +++++++++++++++-----
 6 files changed, 185 insertions(+), 55 deletions(-)
 create mode 100644 hack/ignition/rhcos.json

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index 48dabb94..7f61cbec 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -56,18 +56,17 @@ jobs:
           image_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2
           checksum_url: https://cloud.centos.org/centos/10-stream/x86_64/images/CentOS-Stream-GenericCloud-10-latest.x86_64.qcow2.SHA256SUM
           cloud_init: hack/cloud-init/centos.yml
-        - name: ubuntu-24.04
-          image_url: https://cloud-images.ubuntu.com/releases/24.04/release/ubuntu-24.04-server-cloudimg-amd64.img
-          checksum_url: https://cloud-images.ubuntu.com/releases/24.04/release/SHA256SUMS
-          cloud_init: hack/cloud-init/ubuntu.yml
-        - name: ubuntu-26.04
-          image_url: https://cloud-images.ubuntu.com/releases/26.04/release/ubuntu-26.04-server-cloudimg-amd64.img
-          checksum_url: https://cloud-images.ubuntu.com/releases/26.04/release/SHA256SUMS
-          cloud_init: hack/cloud-init/ubuntu.yml
         - name: fedora-44
           image_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2
           checksum_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM
           cloud_init: hack/cloud-init/fedora.yml
+        - name: rhcos-4.16
+          # RHCOS has no direct download page,see the "Resolve and
+          # download RHCOS image" step.
+          rhcos_stream: '4.16'
+          ignition: hack/ignition/rhcos.json
+          ssh_user: core
+          container_runtime: podman
     steps:
     - name: Checkout fact
       uses: actions/checkout@v7
@@ -80,10 +79,11 @@ jobs:
       run: |
         sudo apt-get update -qq
         sudo apt-get install -y -qq --no-install-recommends \
-          qemu-system-x86 qemu-utils genisoimage virtiofsd
+          qemu-system-x86 qemu-utils genisoimage virtiofsd jq
         [ -w /dev/kvm ] || sudo chmod 666 /dev/kvm
 
     - name: Download VM base image
+      if: matrix.rhcos_stream == ''
       run: |
         mkdir -p "${VM_DIR}"
         image_name="$(basename "${{ matrix.image_url }}")"
@@ -99,6 +99,29 @@ jobs:
 
         mv "${VM_DIR}/${image_name}" "${VM_DIR}/base.qcow2"
 
+    - name: Resolve and download RHCOS image
+      if: matrix.rhcos_stream != ''
+      run: |
+        mkdir -p "${VM_DIR}"
+
+        # RHCOS has no stable download URL; the current build for a
+        # given OpenShift release stream is published as JSON metadata
+        # by the openshift/installer project, freely downloadable with
+        # no Red Hat subscription required.
+        stream_json="$(curl -fsSL "https://raw.githubusercontent.com/openshift/installer/release-${{ matrix.rhcos_stream }}/data/data/coreos/rhcos.json")"
+        qemu_artifact="$(echo "${stream_json}" | jq -c '.architectures.x86_64.artifacts.qemu.formats."qcow2.gz".disk')"
+        location="$(echo "${qemu_artifact}" | jq -r '.location')"
+        gz_sha256="$(echo "${qemu_artifact}" | jq -r '.sha256')"
+        raw_sha256="$(echo "${qemu_artifact}" | jq -r '."uncompressed-sha256"')"
+
+        curl -fSL --progress-bar -o "${VM_DIR}/rhcos.qcow2.gz" "${location}"
+        echo "${gz_sha256}  ${VM_DIR}/rhcos.qcow2.gz" | sha256sum -c -
+
+        gunzip -f "${VM_DIR}/rhcos.qcow2.gz"
+        echo "${raw_sha256}  ${VM_DIR}/rhcos.qcow2" | sha256sum -c -
+
+        mv "${VM_DIR}/rhcos.qcow2" "${VM_DIR}/base.qcow2"
+
     - name: Download fact image artifact
       uses: actions/download-artifact@v4
       with:
@@ -106,28 +129,48 @@ jobs:
 
     - name: Start VM
       run: |
-        hack/qemu-vm.sh start \
-          --image "${VM_DIR}/base.qcow2" \
-          --cloud-init "${{ matrix.cloud_init }}" \
-          --vm-dir "${VM_DIR}" \
-          --ssh-port "${SSH_PORT}" \
+        args=(
+          --image "${VM_DIR}/base.qcow2"
+          --vm-dir "${VM_DIR}"
+          --ssh-port "${SSH_PORT}"
+          --ssh-user "${{ matrix.ssh_user || 'root' }}"
+          --runtime "${{ matrix.container_runtime || 'docker' }}"
           --host-mount "${{ github.workspace }}"
+        )
+        if [ -n "${{ matrix.cloud_init }}" ]; then
+          args+=(--cloud-init "${{ matrix.cloud_init }}")
+        else
+          args+=(--ignition "${{ matrix.ignition }}")
+        fi
+        hack/qemu-vm.sh start "${args[@]}"
 
     - name: Load fact image into VM
       run: |
-        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
-          docker load -i /host/fact-image.tar
+        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" \
+          --ssh-user "${{ matrix.ssh_user || 'root' }}" -- \
+          "${{ matrix.container_runtime || 'docker' }}" load -i /mnt/host/fact-image.tar
 
     - name: Setup test environment in VM
       run: |
-        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
+        # Generate gRPC stubs directly (matching tests/Makefile's
+        # grpc-gen target) instead of via "make", so the VM doesn't
+        # need a make binary installed.
+        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" \
+          --ssh-user "${{ matrix.ssh_user || 'root' }}" -- \
           bash -c '
             set -euo pipefail
-            cd /host/tests
+            cd /mnt/host/tests
             python3 -m venv .venv
             source .venv/bin/activate
             pip install -q -r requirements.txt
-            make grpc-gen
+            python3 -m grpc_tools.protoc \
+              -I../third_party/stackrox/proto \
+              --python_out=. \
+              --pyi_out=. \
+              --grpc_python_out=. \
+              ../third_party/stackrox/proto/internalapi/sensor/collector.proto \
+              ../third_party/stackrox/proto/internalapi/sensor/sfa.proto \
+              ../third_party/stackrox/proto/internalapi/sensor/sfa_iservice.proto
           '
 
     - name: Run integration tests
@@ -140,11 +183,23 @@ jobs:
         # back to building those images locally from their Containerfiles
         # on a pull error.
         FACT_IMAGE="$(FACT_REGISTRY=localhost/fact make --no-print-directory image-name)"
-        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" -- \
+
+        # podman exposes a Docker-API-compatible socket that the
+        # docker Python SDK (used throughout tests/) can talk to
+        # directly via DOCKER_HOST -- no separate Docker daemon needed.
+        CONTAINER_RUNTIME="${{ matrix.container_runtime || 'docker' }}"
+        DOCKER_HOST_ENV=""
+        if [ "${CONTAINER_RUNTIME}" = "podman" ]; then
+          DOCKER_HOST_ENV="export DOCKER_HOST=unix:///run/podman/podman.sock"
+        fi
+
+        hack/qemu-vm.sh ssh --vm-dir "${VM_DIR}" --ssh-port "${SSH_PORT}" \
+          --ssh-user "${{ matrix.ssh_user || 'root' }}" -- \
           bash -c "
             set -euo pipefail
+            ${DOCKER_HOST_ENV}
             mkdir -p /tmp/fact-tmp
-            cd /host/tests
+            cd /mnt/host/tests
             source .venv/bin/activate
             pytest \
               --image='${FACT_IMAGE}' \
diff --git a/hack/cloud-init/centos.yml b/hack/cloud-init/centos.yml
index c99ae7be..4b380f88 100644
--- a/hack/cloud-init/centos.yml
+++ b/hack/cloud-init/centos.yml
@@ -11,10 +11,9 @@ users:
 ssh_pwauth: false
 
 runcmd:
-  - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf"
   - dnf -y install dnf-plugins-core
   - dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
   - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
-  - dnf -y install python3 python3-pip python3-devel gcc
+  - dnf -y install python3 python3-pip python3-devel
   - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/cloud-init/fedora.yml b/hack/cloud-init/fedora.yml
index 13030a2e..8ab37edf 100644
--- a/hack/cloud-init/fedora.yml
+++ b/hack/cloud-init/fedora.yml
@@ -11,10 +11,9 @@ users:
 ssh_pwauth: false
 
 runcmd:
-  - grubby --update-kernel=ALL --args="lsm=lockdown,capability,yama,selinux,bpf"
   - dnf -y install dnf5-plugins
   - dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo
   - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
-  - dnf -y install python3 python3-pip python3-devel gcc
+  - dnf -y install python3 python3-pip python3-devel
   - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/cloud-init/ubuntu.yml b/hack/cloud-init/ubuntu.yml
index 9e39b315..ffc885db 100644
--- a/hack/cloud-init/ubuntu.yml
+++ b/hack/cloud-init/ubuntu.yml
@@ -34,5 +34,5 @@ runcmd:
   - apt-get update -qq
   - apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
-  - apt-get install -y -qq python3-venv python3-dev gcc make
+  - apt-get install -y -qq python3-venv python3-dev
   - touch /var/lib/cloud/instance/boot-finished-user
diff --git a/hack/ignition/rhcos.json b/hack/ignition/rhcos.json
new file mode 100644
index 00000000..86217518
--- /dev/null
+++ b/hack/ignition/rhcos.json
@@ -0,0 +1,23 @@
+{
+  "ignition": {
+    "version": "3.4.0"
+  },
+  "passwd": {
+    "users": [
+      {
+        "name": "core",
+        "sshAuthorizedKeys": [
+          "__SSH_PUBKEY__"
+        ]
+      }
+    ]
+  },
+  "systemd": {
+    "units": [
+      {
+        "name": "podman.socket",
+        "enabled": true
+      }
+    ]
+  }
+}
diff --git a/hack/qemu-vm.sh b/hack/qemu-vm.sh
index 61488dd0..b8592157 100755
--- a/hack/qemu-vm.sh
+++ b/hack/qemu-vm.sh
@@ -2,9 +2,9 @@
 # Boot a cloud image in QEMU/KVM for integration testing.
 #
 # Designed to work both locally and in CI (GitHub Actions).  The script
-# takes a pre-downloaded qcow2 cloud image, injects cloud-init config,
-# boots the VM, waits until it is ready, and prints SSH connection
-# details.
+# takes a pre-downloaded qcow2 cloud image, injects cloud-init or
+# Ignition config, boots the VM, waits until it is ready, and prints
+# SSH connection details.
 #
 # Usage:
 #   hack/qemu-vm.sh start   [options]   — boot the VM
@@ -13,27 +13,39 @@
 #
 # Options:
 #   --image PATH          path to the base qcow2 image (required for start)
-#   --cloud-init PATH     cloud-init user-data template (required for start)
+#   --cloud-init PATH     cloud-init user-data template (one of
+#                         --cloud-init/--ignition is required for start)
+#   --ignition PATH       Ignition config template, for CoreOS-based
+#                         images such as RHCOS (see --cloud-init)
 #   --vm-dir DIR          working directory for VM files (default: /tmp/fact-vm)
 #   --ssh-port PORT       host port forwarded to guest 22 (default: 2222)
+#   --ssh-user USER       remote user to SSH as (default: root). Commands
+#                         are automatically run via "sudo" when this is
+#                         not root (e.g. RHCOS's "core" user).
+#   --runtime NAME        container runtime running in the guest, used
+#                         only to sanity-check it is up (default: docker)
 #   --cpu N               vCPUs (default: all host CPUs)
 #   --mem SIZE            memory, e.g. 8G (default: 75% of host RAM)
-#   --host-mount DIR      directory to share with the VM via virtiofs at /host
+#   --host-mount DIR      directory to share with the VM via virtiofs at
+#                         /mnt/host
 #
-# Cloud-init templates:
-#   The --cloud-init file is a standard #cloud-config YAML with one
-#   special placeholder: __SSH_PUBKEY__ is replaced with the VM's
-#   ephemeral SSH public key.  See hack/cloud-init/ for examples.
+# Cloud-init / Ignition templates:
+#   The --cloud-init/--ignition file has one special placeholder:
+#   __SSH_PUBKEY__ is replaced with the VM's ephemeral SSH public key.
+#   See hack/cloud-init/ and hack/ignition/ for examples.
 #
-# The VM state (image overlay, cloud-init ISO, SSH keys, PID file) lives
+# The VM state (image overlay, seed config, SSH keys, PID file) lives
 # entirely under --vm-dir and can be cleaned up by removing that directory.
 
 set -euo pipefail
 
 : "${IMAGE:=}"
 : "${CLOUD_INIT:=}"
+: "${IGNITION:=}"
 : "${VM_DIR:=/tmp/fact-vm}"
 : "${SSH_PORT:=2222}"
+: "${SSH_USER:=root}"
+: "${RUNTIME:=docker}"
 : "${CPU:=$(nproc)}"
 : "${MEM:=$(awk '/^MemTotal/{printf "%dG", int($2/1024/1024*0.75)}' /proc/meminfo)}"
 : "${HOST_MOUNT:=}"
@@ -54,8 +66,11 @@ parse_args() {
         case "$1" in
             --image)        IMAGE="$2";        shift 2;;
             --cloud-init)   CLOUD_INIT="$2";   shift 2;;
+            --ignition)     IGNITION="$2";     shift 2;;
             --vm-dir)       VM_DIR="$2";       shift 2;;
             --ssh-port)     SSH_PORT="$2";     shift 2;;
+            --ssh-user)     SSH_USER="$2";     shift 2;;
+            --runtime)      RUNTIME="$2";      shift 2;;
             --cpu)          CPU="$2";          shift 2;;
             --mem)          MEM="$2";          shift 2;;
             --host-mount)   HOST_MOUNT="$2";   shift 2;;
@@ -108,6 +123,14 @@ EOF
         "${ci_dir}/user-data" "${ci_dir}/meta-data" 2>/dev/null
 }
 
+create_ignition_config() {
+    local pubkey
+    pubkey="$(cat "${VM_DIR}/id_ed25519.pub")"
+
+    log "Generating Ignition config from ${IGNITION}"
+    sed "s|__SSH_PUBKEY__|${pubkey}|g" "${IGNITION}" > "${VM_DIR}/ignition.json"
+}
+
 find_virtiofsd() {
     local bin
     bin="$(command -v virtiofsd 2>/dev/null)" && { echo "${bin}"; return; }
@@ -153,12 +176,17 @@ build_qemu_args() {
         -smp "${CPU}"
         -m "${MEM}"
         -drive "file=${VM_DIR}/disk.qcow2,if=virtio,format=qcow2"
-        -drive "file=${VM_DIR}/seed.iso,if=virtio,format=raw,readonly=on"
         -netdev "user,id=net0,hostfwd=tcp::${SSH_PORT}-:22"
         -device virtio-net-pci,netdev=net0
         -serial "file:${VM_DIR}/console.log"
     )
 
+    if [[ -n "${CLOUD_INIT}" ]]; then
+        args+=(-drive "file=${VM_DIR}/seed.iso,if=virtio,format=raw,readonly=on")
+    else
+        args+=(-fw_cfg "name=opt/com.coreos/config,file=${VM_DIR}/ignition.json")
+    fi
+
     if [[ -n "${HOST_MOUNT}" ]]; then
         args+=(
             -object "memory-backend-memfd,id=mem,size=${MEM},share=on"
@@ -172,7 +200,23 @@ build_qemu_args() {
 }
 
 vm_ssh() {
-    ssh -p "${SSH_PORT}" -i "${VM_DIR}/id_ed25519" "${SSH_OPTS[@]}" root@localhost "$@"
+    # ssh naively space-joins multiple trailing arguments before sending
+    # them to the remote shell, which loses quoting boundaries (e.g. a
+    # multi-line "bash -c '...'" script gets corrupted). Shell-quote and
+    # join them ourselves into a single argument so ssh passes it
+    # through untouched.
+    local remote_cmd
+    remote_cmd="$(printf '%q ' "$@")"
+
+    # Non-root users (e.g. RHCOS's "core") need sudo for anything that
+    # touches the host (mounting virtiofs, talking to a rootful
+    # container runtime socket, etc). Wrap transparently so call sites
+    # don't need to know which user they're running as.
+    if [[ "${SSH_USER}" != "root" ]]; then
+        remote_cmd="sudo -n -- sh -c $(printf '%q' "${remote_cmd}")"
+    fi
+
+    ssh -p "${SSH_PORT}" -i "${VM_DIR}/id_ed25519" "${SSH_OPTS[@]}" "${SSH_USER}@localhost" "${remote_cmd}"
 }
 
 wait_for_ssh() {
@@ -194,14 +238,14 @@ wait_for_cloud_init() {
     log "Waiting for cloud-init to finish..."
     local i
     for i in $(seq 1 300); do
-        if vm_ssh "test -f /var/lib/cloud/instance/boot-finished-user" 2>/dev/null; then
+        if vm_ssh test -f /var/lib/cloud/instance/boot-finished-user 2>/dev/null; then
             log "cloud-init finished after ${i}s"
             return 0
         fi
         sleep 1
     done
     log "cloud-init did not finish within 300s"
-    vm_ssh "cat /var/log/cloud-init-output.log" 2>/dev/null >&2 || true
+    vm_ssh cat /var/log/cloud-init-output.log 2>/dev/null >&2 || true
     return 1
 }
 
@@ -214,19 +258,31 @@ cmd_start() {
         echo "error: image not found: ${IMAGE}" >&2
         exit 1
     fi
-    if [[ -z "${CLOUD_INIT}" ]]; then
-        echo "error: --cloud-init is required for start" >&2
+    if [[ -z "${CLOUD_INIT}" && -z "${IGNITION}" ]]; then
+        echo "error: one of --cloud-init or --ignition is required for start" >&2
+        exit 1
+    fi
+    if [[ -n "${CLOUD_INIT}" && -n "${IGNITION}" ]]; then
+        echo "error: --cloud-init and --ignition are mutually exclusive" >&2
         exit 1
     fi
-    if [[ ! -f "${CLOUD_INIT}" ]]; then
+    if [[ -n "${CLOUD_INIT}" && ! -f "${CLOUD_INIT}" ]]; then
         echo "error: cloud-init template not found: ${CLOUD_INIT}" >&2
         exit 1
     fi
+    if [[ -n "${IGNITION}" && ! -f "${IGNITION}" ]]; then
+        echo "error: Ignition template not found: ${IGNITION}" >&2
+        exit 1
+    fi
 
     mkdir -p "${VM_DIR}"
     create_overlay
     generate_ssh_key
-    create_cloud_init_iso
+    if [[ -n "${CLOUD_INIT}" ]]; then
+        create_cloud_init_iso
+    else
+        create_ignition_config
+    fi
 
     if [[ -n "${HOST_MOUNT}" ]]; then
         start_virtiofsd
@@ -239,32 +295,30 @@ cmd_start() {
     qemu-system-x86_64 "${qemu_args[@]}"
 
     wait_for_ssh
-    wait_for_cloud_init
 
-    log "Rebooting for BPF LSM kernel cmdline change..."
-    vm_ssh "reboot" 2>/dev/null || true
-    sleep 5
-    wait_for_ssh
+    if [[ -n "${CLOUD_INIT}" ]]; then
+        wait_for_cloud_init
+    fi
 
-    log "Verifying BPF LSM is active"
+    log "Verifying bpf LSM is active"
     local lsm
-    lsm="$(vm_ssh "cat /sys/kernel/security/lsm")"
+    lsm="$(vm_ssh cat /sys/kernel/security/lsm)"
     if [[ "${lsm}" != *bpf* ]]; then
         log "ERROR: bpf not found in LSM list: ${lsm}"
         return 1
     fi
     log "LSM list: ${lsm}"
 
-    log "Verifying Docker is running"
-    vm_ssh "docker info" >/dev/null
+    log "Verifying ${RUNTIME} is running"
+    vm_ssh "${RUNTIME}" info >/dev/null
 
     if [[ -n "${HOST_MOUNT}" ]]; then
         log "Mounting host filesystem inside VM"
-        vm_ssh "mkdir -p /host && mount -t virtiofs host_mount /host"
+        vm_ssh sh -c 'mkdir -p /mnt/host && mount -t virtiofs host_mount /mnt/host'
     fi
 
     log "VM is ready"
-    log "  SSH: ssh -p ${SSH_PORT} -i ${VM_DIR}/id_ed25519 ${SSH_OPTS[*]} root@localhost"
+    log "  SSH: ssh -p ${SSH_PORT} -i ${VM_DIR}/id_ed25519 ${SSH_OPTS[*]} ${SSH_USER}@localhost"
 }
 
 cmd_ssh() {

From 04e75d52df09b79857190aec971c6847ddf07c38 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Mon, 21 Sep 2026 13:26:49 +0200
Subject: [PATCH 10/11] ci: add RHCOS 4.18, 4.20, and 4.22 (RHEL 9/10) to the
 test matrix

Extend the RHCOS matrix beyond 4.16 to cover the other actively
supported OpenShift releases:

  - rhcos-4.18
  - rhcos-4.20
  - rhcos-4.22-rhel9
  - rhcos-4.22-rhel10

OCP 4.22+ splits RHCOS metadata into separate coreos-rhel-9.json and
coreos-rhel-10.json files instead of a single rhcos.json (confirmed
against the actual release-4.22 directory listing, and against
stackrox/collector's fetch_ocp_rhcos_bootimage.sh, which was updated
with a matching RHEL_VARIANT parameter for the same reason).

Added an optional rhcos_rhel_variant matrix field (rhel-9/rhel-10);
the "Resolve and download RHCOS image" step picks
coreos-${variant}.json when set, falling back to rhcos.json
otherwise. The JSON shape under
architectures.x86_64.artifacts.qemu.formats."qcow2.gz".disk is
identical across all five metadata files, so no other changes were
needed - verified all five URLs resolve to distinct, valid qcow2
locations with checksums.

Refs: #1794

Assisted-by: claude-opus-4-6 <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml | 41 +++++++++++++++++++--
 1 file changed, 37 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index 7f61cbec..bce83a37 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -60,13 +60,39 @@ jobs:
           image_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-44-1.7.x86_64.qcow2
           checksum_url: https://download.fedoraproject.org/pub/fedora/linux/releases/44/Cloud/x86_64/images/Fedora-Cloud-44-1.7-x86_64-CHECKSUM
           cloud_init: hack/cloud-init/fedora.yml
+        # RHCOS has no direct download page; its qcow2 location and
+        # checksums are resolved dynamically from the OpenShift
+        # installer's per-release stream metadata (see the "Resolve
+        # and download RHCOS image" step). OCP 4.22+ splits RHCOS into
+        # separate RHEL 9 and RHEL 10 based images (coreos-rhel-9.json /
+        # coreos-rhel-10.json) instead of a single rhcos.json.
         - name: rhcos-4.16
-          # RHCOS has no direct download page,see the "Resolve and
-          # download RHCOS image" step.
           rhcos_stream: '4.16'
           ignition: hack/ignition/rhcos.json
           ssh_user: core
           container_runtime: podman
+        - name: rhcos-4.18
+          rhcos_stream: '4.18'
+          ignition: hack/ignition/rhcos.json
+          ssh_user: core
+          container_runtime: podman
+        - name: rhcos-4.20
+          rhcos_stream: '4.20'
+          ignition: hack/ignition/rhcos.json
+          ssh_user: core
+          container_runtime: podman
+        - name: rhcos-4.22-rhel9
+          rhcos_stream: '4.22'
+          rhcos_rhel_variant: rhel-9
+          ignition: hack/ignition/rhcos.json
+          ssh_user: core
+          container_runtime: podman
+        - name: rhcos-4.22-rhel10
+          rhcos_stream: '4.22'
+          rhcos_rhel_variant: rhel-10
+          ignition: hack/ignition/rhcos.json
+          ssh_user: core
+          container_runtime: podman
     steps:
     - name: Checkout fact
       uses: actions/checkout@v7
@@ -107,8 +133,15 @@ jobs:
         # RHCOS has no stable download URL; the current build for a
         # given OpenShift release stream is published as JSON metadata
         # by the openshift/installer project, freely downloadable with
-        # no Red Hat subscription required.
-        stream_json="$(curl -fsSL "https://raw.githubusercontent.com/openshift/installer/release-${{ matrix.rhcos_stream }}/data/data/coreos/rhcos.json")"
+        # no Red Hat subscription required. OCP 4.22+ splits this into
+        # per-RHEL-version files (coreos-rhel-9.json, coreos-rhel-10.json)
+        # instead of a single rhcos.json.
+        if [ -n "${{ matrix.rhcos_rhel_variant }}" ]; then
+          RHCOS_JSON_FILE="coreos-${{ matrix.rhcos_rhel_variant }}.json"
+        else
+          RHCOS_JSON_FILE="rhcos.json"
+        fi
+        stream_json="$(curl -fsSL "https://raw.githubusercontent.com/openshift/installer/release-${{ matrix.rhcos_stream }}/data/data/coreos/${RHCOS_JSON_FILE}")"
         qemu_artifact="$(echo "${stream_json}" | jq -c '.architectures.x86_64.artifacts.qemu.formats."qcow2.gz".disk')"
         location="$(echo "${qemu_artifact}" | jq -r '.location')"
         gz_sha256="$(echo "${qemu_artifact}" | jq -r '.sha256')"

From bb7129454f294540b49ea2067c8687746a9f7db2 Mon Sep 17 00:00:00 2001
From: Mauro Ezequiel Moltrasio <mmoltras@redhat.com>
Date: Mon, 21 Sep 2026 17:36:37 +0200
Subject: [PATCH 11/11] ci: update kernel to latest before running tests on
 CentOS/Fedora

The CentOS and Fedora cloud images can ship a kernel that lags behind
what's currently available. Add a `dnf -y update 'kernel*'` step to
their cloud-init runcmd and reboot into the updated kernel via a
power_state directive.

hack/qemu-vm.sh gains a new --expect-reboot flag: it records the VM's
boot_id before cloud-init runs and polls until it changes, rather than
trusting the boot-finished-user marker alone (that marker is touched
by runcmd *before* the power_state reboot fires, so relying on it
alone races with the actual reboot). The lvh-integration-tests.yml
workflow passes --expect-reboot for the cloud-init-based matrix
entries only; RHCOS's Ignition config is untouched since it's an
immutable ostree image already pinned to its release's kernel.

Assisted-by: `agent-mode` <noreply@opencode.ai>
---
 .github/workflows/lvh-integration-tests.yml |  2 +-
 hack/cloud-init/centos.yml                  | 12 +++++++
 hack/cloud-init/fedora.yml                  | 12 +++++++
 hack/qemu-vm.sh                             | 36 ++++++++++++++++++++-
 4 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/lvh-integration-tests.yml b/.github/workflows/lvh-integration-tests.yml
index bce83a37..3f92a029 100644
--- a/.github/workflows/lvh-integration-tests.yml
+++ b/.github/workflows/lvh-integration-tests.yml
@@ -171,7 +171,7 @@ jobs:
           --host-mount "${{ github.workspace }}"
         )
         if [ -n "${{ matrix.cloud_init }}" ]; then
-          args+=(--cloud-init "${{ matrix.cloud_init }}")
+          args+=(--cloud-init "${{ matrix.cloud_init }}" --expect-reboot)
         else
           args+=(--ignition "${{ matrix.ignition }}")
         fi
diff --git a/hack/cloud-init/centos.yml b/hack/cloud-init/centos.yml
index 4b380f88..02db85f6 100644
--- a/hack/cloud-init/centos.yml
+++ b/hack/cloud-init/centos.yml
@@ -1,6 +1,11 @@
 #cloud-config
 # Cloud-init user-data for CentOS Stream 10 (and RHEL 10).
 # Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
+#
+# Updates the kernel to the latest available before running tests (the
+# cloud image's shipped kernel can lag behind), then reboots into it via
+# "power_state" below. hack/qemu-vm.sh must be called with
+# --expect-reboot for this template.
 
 users:
   - name: root
@@ -11,9 +16,16 @@ users:
 ssh_pwauth: false
 
 runcmd:
+  - dnf -y update 'kernel*'
   - dnf -y install dnf-plugins-core
   - dnf config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo
   - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
   - dnf -y install python3 python3-pip python3-devel
   - touch /var/lib/cloud/instance/boot-finished-user
+
+power_state:
+  mode: reboot
+  message: Rebooting into updated kernel
+  timeout: 60
+  condition: true
diff --git a/hack/cloud-init/fedora.yml b/hack/cloud-init/fedora.yml
index 8ab37edf..006aeb1e 100644
--- a/hack/cloud-init/fedora.yml
+++ b/hack/cloud-init/fedora.yml
@@ -1,6 +1,11 @@
 #cloud-config
 # Cloud-init user-data for Fedora (41+, dnf5-based).
 # Used by hack/qemu-vm.sh — __SSH_PUBKEY__ is replaced at boot time.
+#
+# Updates the kernel to the latest available before running tests (the
+# cloud image's shipped kernel can lag behind), then reboots into it via
+# "power_state" below. hack/qemu-vm.sh must be called with
+# --expect-reboot for this template.
 
 users:
   - name: root
@@ -11,9 +16,16 @@ users:
 ssh_pwauth: false
 
 runcmd:
+  - dnf -y update 'kernel*'
   - dnf -y install dnf5-plugins
   - dnf config-manager addrepo --from-repofile=https://download.docker.com/linux/fedora/docker-ce.repo
   - dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin
   - systemctl enable --now docker
   - dnf -y install python3 python3-pip python3-devel
   - touch /var/lib/cloud/instance/boot-finished-user
+
+power_state:
+  mode: reboot
+  message: Rebooting into updated kernel
+  timeout: 60
+  condition: true
diff --git a/hack/qemu-vm.sh b/hack/qemu-vm.sh
index b8592157..ce0e6918 100755
--- a/hack/qemu-vm.sh
+++ b/hack/qemu-vm.sh
@@ -28,6 +28,13 @@
 #   --mem SIZE            memory, e.g. 8G (default: 75% of host RAM)
 #   --host-mount DIR      directory to share with the VM via virtiofs at
 #                         /mnt/host
+#   --expect-reboot       wait for the VM to reboot once cloud-init
+#                         finishes, before continuing (needed for
+#                         cloud-init templates that update the kernel —
+#                         see hack/cloud-init/{centos,fedora}.yml — and
+#                         reboot into it via a "power_state" directive).
+#                         Only meaningful with --cloud-init; ignored
+#                         with --ignition.
 #
 # Cloud-init / Ignition templates:
 #   The --cloud-init/--ignition file has one special placeholder:
@@ -49,6 +56,7 @@ set -euo pipefail
 : "${CPU:=$(nproc)}"
 : "${MEM:=$(awk '/^MemTotal/{printf "%dG", int($2/1024/1024*0.75)}' /proc/meminfo)}"
 : "${HOST_MOUNT:=}"
+: "${EXPECT_REBOOT:=0}"
 
 SSH_OPTS=(-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR)
 
@@ -74,6 +82,7 @@ parse_args() {
             --cpu)          CPU="$2";          shift 2;;
             --mem)          MEM="$2";          shift 2;;
             --host-mount)   HOST_MOUNT="$2";   shift 2;;
+            --expect-reboot) EXPECT_REBOOT=1;  shift;;
             -h|--help)      usage;;
             --)             shift; EXTRA_ARGS+=("$@"); return;;
             *)              EXTRA_ARGS+=("$@"); return;;
@@ -235,7 +244,9 @@ wait_for_ssh() {
 }
 
 wait_for_cloud_init() {
-    log "Waiting for cloud-init to finish..."
+    local timeout=300
+
+    log "Waiting for cloud-init to finish (timeout ${timeout}s)..."
     local i
     for i in $(seq 1 300); do
         if vm_ssh test -f /var/lib/cloud/instance/boot-finished-user 2>/dev/null; then
@@ -249,6 +260,24 @@ wait_for_cloud_init() {
     return 1
 }
 
+wait_for_reboot() {
+    local old_boot_id="$1"
+    local timeout=300
+
+    log "Waiting for kernel-update reboot..."
+    local i new_boot_id
+    for i in $(seq 1 "${timeout}"); do
+        new_boot_id="$(vm_ssh cat /proc/sys/kernel/random/boot_id 2>/dev/null || true)"
+        if [[ -n "${new_boot_id}" && "${new_boot_id}" != "${old_boot_id}" ]]; then
+            log "Reboot detected after ${i}s"
+            return 0
+        fi
+        sleep 1
+    done
+    log "VM did not reboot within ${timeout}s"
+    return 1
+}
+
 cmd_start() {
     if [[ -z "${IMAGE}" ]]; then
         echo "error: --image is required for start" >&2
@@ -297,7 +326,12 @@ cmd_start() {
     wait_for_ssh
 
     if [[ -n "${CLOUD_INIT}" ]]; then
+        local pre_update_boot_id
+        pre_update_boot_id="$(vm_ssh cat /proc/sys/kernel/random/boot_id)"
         wait_for_cloud_init
+        if [[ "${EXPECT_REBOOT}" == "1" ]]; then
+            wait_for_reboot "${pre_update_boot_id}"
+        fi
     fi
 
     log "Verifying bpf LSM is active"