Update qemu to v10.2.0 and fix CI (#3646)

* update to qemu v10.2.0

* GuestAddr is now mapping to vaddr, since qemu uses this type also in usermode to refer to addresses in general.

* many fixes for the CI to finally run again

Author: rmalmain

.github/workflows/asan-prepare/action.yml

@@ -55,7 +55,7 @@ runs:
     - name: Install nextest
       shell: bash
       run: |
-        cargo binstall --no-confirm cargo-nextest
+        cargo binstall --no-confirm --locked cargo-nextest
     - name: Install Rust Targets
       shell: bash
       run: |

.github/workflows/build_and_test.yml

@@ -4,9 +4,9 @@ permissions:
 
 on:
   push:
-    branches: [ main, "pr/**" ]
+    branches: [main, "pr/**"]
   pull_request:
-    branches: [ main ]
+    branches: [main]
     types: ["labeled", "opened", "synchronize", "reopened"]
   workflow_dispatch:
   merge_group:
@@ -24,7 +24,7 @@ jobs:
   common:
     strategy:
       matrix:
-        os: [ ubuntu-24.04, windows-latest, macOS-latest ]
+        os: [ubuntu-24.04, windows-latest, macOS-latest]
     runs-on: ${{ matrix.os }}
     steps:
       - name: Install mimetype
@@ -84,7 +84,6 @@ jobs:
         if: runner.os != 'Windows'
         run: just test-serial
 
-
   ubuntu-doc-build:
     runs-on: ubuntu-24.04
     steps:
@@ -100,10 +99,15 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
-      - uses: Swatinem/rust-cache@v2
+      - name: install mdbook
+        uses: baptiste0928/cargo-install@v3
+        with:
+          crate: mdbook
       # ---- doc check ----
       - name: Test Docs
         run: just test-docs
+      # DO NOT move the cache pull before this point, as it causes mdbook to fail due to duplicated deps
+      - uses: Swatinem/rust-cache@v2
 
   ubuntu-miri:
     runs-on: ubuntu-24.04
@@ -112,7 +116,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: ./.github/workflows/ubuntu-prepare
         with:
-          use_nightly: 'true'
+          use_nightly: "true"
       - name: Add miri
         run: rustup +nightly component add miri
       - uses: Swatinem/rust-cache@v2
@@ -168,7 +172,27 @@ jobs:
     needs: ubuntu
     strategy:
       matrix:
-        instance_idx: [ "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "17" ]
+        instance_idx:
+          [
+            "0",
+            "1",
+            "2",
+            "3",
+            "4",
+            "5",
+            "6",
+            "7",
+            "8",
+            "9",
+            "10",
+            "11",
+            "12",
+            "13",
+            "14",
+            "15",
+            "16",
+            "17",
+          ]
       fail-fast: false
     steps:
       - uses: actions/checkout@v4
@@ -234,8 +258,8 @@ jobs:
       - name: Run linkspector
         uses: umbrelladocs/action-linkspector@v1
         with:
-          fail_on_error: 'true'
-          config_file: '.github/.linkspector.yml'
+          fail_on_error: "true"
+          config_file: ".github/.linkspector.yml"
 
   msrv:
     runs-on: ubuntu-latest
@@ -342,8 +366,8 @@ jobs:
 
     runs-on: ubuntu-24.04
     steps:
-        # Get the name of the fuzzer so that we can use it as the key for a cache
-        # of the built artefacts. The key cannot have any special characters.
+      # Get the name of the fuzzer so that we can use it as the key for a cache
+      # of the built artefacts. The key cannot have any special characters.
       - name: Get fuzzer name
         id: fuzzer_name
         run: |
@@ -416,7 +440,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ ubuntu-24.04 ]
+        os: [ubuntu-24.04]
         fuzzer:
           - fuzzers/full_system/unicorn
     steps:
@@ -441,17 +465,17 @@ jobs:
     strategy:
       matrix:
         fuzzer:
-            - binary_only/qemu_cmin
-            - binary_only/qemu_tmin
-            - binary_only/qemu_coverage
-            - binary_only/qemu_launcher
-            # - full_system/qemu_linux_kernel
-            # - full_system/qemu_linux_process
+          - binary_only/qemu_cmin
+          - binary_only/qemu_tmin
+          - binary_only/qemu_coverage
+          - binary_only/qemu_launcher
+          # - full_system/qemu_linux_kernel
+          # - full_system/qemu_linux_process
     runs-on: ubuntu-24.04
     # container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest
     steps:
-        # Get the name of the fuzzer so that we can use it as the key for a cache
-        # of the built artefacts. The key cannot have any special characters.
+      # Get the name of the fuzzer so that we can use it as the key for a cache
+      # of the built artefacts. The key cannot have any special characters.
       - name: Get fuzzer name
         id: fuzzer_name
         run: |
@@ -464,7 +488,7 @@ jobs:
       - name: Configure Cache
         uses: Swatinem/rust-cache@v2
         with:
-          # We will have each of these fuzzers have it's own cache since these
+          # We will have each of these fuzzers have its own cache since these
           # are some of the heaviest fuzzers to build.
           shared-key: qemu-${{ steps.fuzzer_name.outputs.fuzzer_name }}-x86_64
           # We want to include the commit hash to ensure the cache is replaced
@@ -483,7 +507,6 @@ jobs:
           LLVM_CONFIG=llvm-config-${{env.MAIN_LLVM_VERSION}} \
           just test-fuzzer fuzzers/${{ matrix.fuzzer }}
 
-
   # Job to build and test the QEMU fuzzers
   qemu-system:
     name: 🤖 ${{ matrix.fuzzer }}
@@ -494,14 +517,14 @@ jobs:
     strategy:
       matrix:
         fuzzer:
-            - full_system/qemu_baremetal
-            - full_system/qemu_linux_kernel
-            # - full_system/qemu_linux_process
+          - full_system/qemu_baremetal
+          - full_system/qemu_linux_kernel
+          # - full_system/qemu_linux_process
     runs-on: ubuntu-24.04
     # container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest
     steps:
-        # Get the name of the fuzzer so that we can use it as the key for a cache
-        # of the built artefacts. The key cannot have any special characters.
+      # Get the name of the fuzzer so that we can use it as the key for a cache
+      # of the built artefacts. The key cannot have any special characters.
       - name: Get fuzzer name
         id: fuzzer_name
         run: |
@@ -514,7 +537,7 @@ jobs:
       - name: Configure Cache
         uses: Swatinem/rust-cache@v2
         with:
-          # We will have each of these fuzzers have it's own cache since these
+          # We will have each of these fuzzers have its own cache since these
           # are some of the heaviest fuzzers to build.
           shared-key: qemu-${{ steps.fuzzer_name.outputs.fuzzer_name }}-arm
           # We want to include the commit hash to ensure the cache is replaced
@@ -543,12 +566,12 @@ jobs:
     strategy:
       matrix:
         fuzzer:
-            - full_system/qemu_baremetal
+          - full_system/qemu_baremetal
     runs-on: ubuntu-24.04
     # container: registry.gitlab.com/qemu-project/qemu/qemu/ubuntu2204:latest
     steps:
-        # Get the name of the fuzzer so that we can use it as the key for a cache
-        # of the built artefacts. The key cannot have any special characters.
+      # Get the name of the fuzzer so that we can use it as the key for a cache
+      # of the built artefacts. The key cannot have any special characters.
       - name: Get fuzzer name
         id: fuzzer_name
         run: |
@@ -602,6 +625,62 @@ jobs:
         shell: bash
         run: just -d utils/${{ matrix.util }} --justfile utils/${{ matrix.util }}/Justfile test
 
+  libafl_qemu-full:
+    if: contains(github.event.pull_request.labels.*.name, 'qemu-full')
+    runs-on: ubuntu-24.04
+    strategy:
+      matrix:
+        mode:
+          - usermode
+          - systemmode
+        arch:
+          - aarch64
+          - arm
+          - i386
+          - mips
+          - ppc
+          - riscv32
+          - riscv64
+          - x86_64
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Prepare
+        uses: ./.github/workflows/qemu-fuzzer-tester-prepare
+      - name: Configure Cache
+        uses: Swatinem/rust-cache@v2
+      # ---- build normal and examples ----
+      - name: Build LibAFL QEMU (${{ matrix.mode }} - ${{ matrix.arch }})
+        env:
+          LLVM_CONFIG: llvm-config-${{env.MAIN_LLVM_VERSION}}
+        run: cargo build --verbose --package libafl_qemu --no-default-features --features ${{ matrix.mode }},${{ matrix.arch }}
+      - name: Test LibAFL QEMU (${{ matrix.mode }} - ${{ matrix.arch }})
+        env:
+          LLVM_CONFIG: llvm-config-${{env.MAIN_LLVM_VERSION}}
+        run: cargo test --package libafl_qemu --no-default-features --features ${{ matrix.mode }},${{ matrix.arch }}
+
+  # hexagon systemmode is not available yet in qemu upstream.
+  # in the meantime, we only test usermode
+  libafl_qemu-hexagon:
+    if: contains(github.event.pull_request.labels.*.name, 'qemu-full')
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Prepare
+        uses: ./.github/workflows/qemu-fuzzer-tester-prepare
+      - name: Configure Cache
+        uses: Swatinem/rust-cache@v2
+      # ---- build normal and examples ----
+      - name: Build LibAFL QEMU (usermode - hexagon)
+        env:
+          LLVM_CONFIG: llvm-config-${{env.MAIN_LLVM_VERSION}}
+        run: cargo build --verbose --package libafl_qemu --no-default-features --features usermode,hexagon
+      - name: Test LibAFL QEMU (usermode - hexagon)
+        env:
+          LLVM_CONFIG: llvm-config-${{env.MAIN_LLVM_VERSION}}
+        run: cargo test --package libafl_qemu --no-default-features --features usermode,hexagon
+
   libafl_asan:
     name: 🔧 libafl_asan
     runs-on: ubuntu-24.04
@@ -640,7 +719,7 @@ jobs:
   libafl_qemu_asan:
     name: 🔧 libafl_qemu_asan
     needs:
-        - qemu-changes
+      - qemu-changes
     if: ${{ needs.qemu-changes.outputs.qemu == 'true' }}
     runs-on: ubuntu-24.04
     steps:
@@ -693,7 +772,7 @@ jobs:
         run: just test-no-std
       - name: Build aarch64-unknown-none
         run: just build-aarch64-unknown-none
-  
+
   x86-i686-build:
     runs-on: ubuntu-24.04
 
@@ -712,7 +791,6 @@ jobs:
           shared-key: x86-i686
       - name: test
         run: just check-i686
-        
 
   nostd-clippy:
     runs-on: ubuntu-24.04
@@ -797,8 +875,7 @@ jobs:
           file: ./Dockerfile
           push: ${{ github.ref == 'refs/heads/main' }}
           load: ${{ github.ref != 'refs/heads/main' }}
-          tags:
-            ghcr.io/aflplusplus/libafl:latest
+          tags: ghcr.io/aflplusplus/libafl:latest
           cache-from: |
             type=registry,ref=ghcr.io/aflplusplus/libafl:cache
             type=local,src=/tmp/.buildx-cache
@@ -823,20 +900,20 @@ jobs:
     strategy:
       matrix:
         fuzzer:
-            - binary_only/frida_libpng
-            - inprocess/libafl_libfuzzer_windows
-            - inprocess/libfuzzer_stb_image
-            - binary_only/frida_windows_gdiplus
-            - binary_only/tinyinst_simple
-            # - inprocess/libfuzzer_windows_asan
+          - binary_only/frida_libpng
+          - inprocess/libafl_libfuzzer_windows
+          - inprocess/libfuzzer_stb_image
+          - binary_only/frida_windows_gdiplus
+          - binary_only/tinyinst_simple
+          # - inprocess/libfuzzer_windows_asan
       fail-fast: false
     runs-on: windows-latest
     needs:
       - fuzzers-preflight
       - common
     steps:
-        # Get the name of the fuzzer so that we can use it as the key for a cache
-        # of the built artefacts. The key cannot have any special characters.
+      # Get the name of the fuzzer so that we can use it as the key for a cache
+      # of the built artefacts. The key cannot have any special characters.
       - name: Get fuzzer name
         id: fuzzer_name
         shell: pwsh
@@ -850,6 +927,10 @@ jobs:
       - name: install cxx bridge
         # Needs to match version in tinyinst-rs
         run: cargo install cxxbridge-cmd@=1.0.190
+      - name: Install vcpkg dependencies
+        run: vcpkg install zlib:x64-windows-static-md
+        env:
+          VCPKG_ROOT: C:\vcpkg
       - name: Configure Cache
         uses: Swatinem/rust-cache@v2
         with:
@@ -866,6 +947,8 @@ jobs:
             fuzzers/${{ matrix.fuzzer }}
       - name: Test
         run: cd fuzzers/${{ matrix.fuzzer }} && just test
+        env:
+          VCPKG_ROOT: C:\vcpkg
 
   windows-clippy:
     runs-on: windows-latest
@@ -893,6 +976,8 @@ jobs:
       - uses: dtolnay/rust-toolchain@stable
         with:
           components: clippy
+      - name: Install nightly
+        run: rustup toolchain install nightly --component clippy
       - name: Install deps
         run: brew install z3 gtk+3 python llvm@${{env.MAIN_LLVM_VERSION}}
       - name: Install cxxbridge

.github/workflows/ubuntu-prepare/action.yml

@@ -23,8 +23,7 @@ runs:
         sudo apt-get update
         sudo apt-get install -y \
           curl lsb-release wget software-properties-common gnupg shellcheck pax-utils \
-          libsqlite3-dev libpixman-1-dev libc6-dev gcc g++ build-essential libglib2.0-dev
-        echo "export LLVM_CONFIG_PATH=$(which llvm-config-${{ env.MAIN_LLVM_VERSION }})" | sudo tee -a /etc/environment
+          libsqlite3-dev libpixman-1-dev libc6-dev gcc g++ build-essential libglib2.0-dev meson
 
     # ---------- toolchain selection ----------
     - name: Install Rust (stable)
@@ -52,6 +51,8 @@ runs:
         chmod +x llvm.sh
         sudo ./llvm.sh ${{ env.MAIN_LLVM_VERSION }} all
         sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.MAIN_LLVM_VERSION }} 200
+        echo "LLVM_CONFIG_PATH=$(which llvm-config-${{ env.MAIN_LLVM_VERSION }})" >> "$GITHUB_ENV"
+        echo "LIBCLANG_PATH=/usr/lib/llvm-${{ env.MAIN_LLVM_VERSION }}/lib" >> "$GITHUB_ENV"
 
     - name: Symlink headers
       shell: bash

Cargo.toml

@@ -142,8 +142,8 @@ paste = "1.0.15"
 postcard = { version = "1.1.3", features = [
   "alloc",
 ], default-features = false } # no_std compatible serde serialization format
-pyo3 = { version = "0.26.0", features = ["auto-initialize"] }
-pyo3-build-config = "0.27.2"
+pyo3 = { version = "0.28.3", features = ["auto-initialize"] }
+pyo3-build-config = "0.28.3"
 rangemap = "1.7.1"
 regex = "1.12.2"
 rustversion = "1.0.22"