Collection of CI patches for 2.3.7

.github/workflows/scripts/generate-ci-type.py

@@ -3,13 +3,16 @@
 """
 Determine the CI type based on the change list and commit message.
 
-Prints "quick" if (explicity required by user):
+Output format: "<type> <source>" where source is "manual" (from
+ZFS-CI-Type commit tag) or "auto" (from file change heuristics).
+
+Prints "quick manual" if:
 - the *last* commit message contains 'ZFS-CI-Type: quick'
-or if (heuristics):
+or "quick auto" if (heuristics):
 - the files changed are not in the list of specified directories, and
 - all commit messages do not contain 'ZFS-CI-Type: (full|linux|freebsd)'
 
-Otherwise prints "full".
+Otherwise prints "full auto" (or "<type> manual" if explicitly requested).
 """
 
 import sys
@@ -58,9 +61,10 @@
 
     head, base = sys.argv[1:3]
 
-    def output_type(type, reason):
-        print(f'{prog}: will run {type} CI: {reason}', file=sys.stderr)
-        print(type)
+    def output_type(type, source, reason):
+        print(f'{prog}: will run {type} CI ({source}): {reason}',
+              file=sys.stderr)
+        print(f'{type} {source}')
         sys.exit(0)
 
     # check last (HEAD) commit message
@@ -70,7 +74,8 @@ def output_type(type, reason):
 
     for line in last_commit_message_raw.stdout.decode().splitlines():
         if line.strip().lower() == 'zfs-ci-type: quick':
-            output_type('quick', f'requested by HEAD commit {head}')
+            output_type('quick', 'manual',
+                        f'requested by HEAD commit {head}')
 
     # check all commit messages
     all_commit_message_raw = subprocess.run([
@@ -84,11 +89,14 @@ def output_type(type, reason):
         if line.startswith('ZFS-CI-Commit:'):
             commit_ref = line.lstrip('ZFS-CI-Commit:').rstrip()
         if line.strip().lower() == 'zfs-ci-type: freebsd':
-            output_type('freebsd', f'requested by commit {commit_ref}')
+            output_type('freebsd', 'manual',
+                        f'requested by commit {commit_ref}')
         if line.strip().lower() == 'zfs-ci-type: linux':
-            output_type('linux', f'requested by commit {commit_ref}')
+            output_type('linux', 'manual',
+                        f'requested by commit {commit_ref}')
         if line.strip().lower() == 'zfs-ci-type: full':
-            output_type('full', f'requested by commit {commit_ref}')
+            output_type('full', 'manual',
+                        f'requested by commit {commit_ref}')
 
     # check changed files
     changed_files_raw = subprocess.run([
@@ -104,9 +112,10 @@ def output_type(type, reason):
             for r in FULL_RUN_REGEX:
                 if r.match(f):
                     output_type(
-                        'full',
+                        'full', 'auto',
                         f'changed file "{f}" matches pattern "{r.pattern}"'
                         )
 
     # catch-all
-    output_type('quick', 'no changed file matches full CI patterns')
+    output_type('quick', 'auto',
+                'no changed file matches full CI patterns')

.github/workflows/scripts/qemu-1-setup.sh

@@ -6,6 +6,27 @@
 
 set -eu
 
+# The default runner has a bunch of development tools and other things
+# that we do not need.  Remove them here to free up a total of 35GB.
+#
+# First remove packages - this frees up ~10GB
+echo "Disk space before purge:"
+df -h /
+sudo docker image prune --all --force
+sudo docker builder prune -a
+unneeded="microsoft-edge-stable|azure-cli|google-cloud|google-chrome-stable|"\
+"temurin|llvm|firefox|mysql-server|snapd|android|dotnet|haskell|ghcup|"\
+"powershell|julia|swift|miniconda|chromium"
+sudo apt-get -y remove $(dpkg-query -f '${binary:Package}\n' -W | grep -E "'$unneeded'")
+sudo apt-get -y autoremove
+
+# Next, remove unneeded files in /usr.  This frees up an additional 25GB.
+sudo rm -fr /usr/local/lib/android /usr/share/dotnet /usr/local/.ghcup \
+        /usr/share/swift /usr/local/share/powershell /usr/local/julia* \
+        /usr/share/miniconda /usr/local/share/chromium
+echo "Disk space after:"
+df -h /
+
 # The default 'azure.archive.ubuntu.com' mirrors can be really slow.
 # Prioritize the official Ubuntu mirrors.
 #

.github/workflows/scripts/qemu-2-start.sh

@@ -78,11 +78,6 @@ case "$OS" in
     OPTS[0]="--boot"
     OPTS[1]="uefi=on"
     ;;
-  fedora41)
-    OSNAME="Fedora 41"
-    OSv="fedora-unknown"
-    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/41/Cloud/x86_64/images/Fedora-Cloud-Base-Generic-41-1.4.x86_64.qcow2"
-    ;;
   fedora42)
     OSNAME="Fedora 42"
     OSv="fedora-unknown"
@@ -108,8 +103,8 @@ case "$OS" in
     URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz"
     KSRC="$FREEBSD_REL/../amd64/$FreeBSD/src.txz"
     ;;
-  freebsd14-3r)
-    FreeBSD="14.3-RELEASE"
+  freebsd14-4r)
+    FreeBSD="14.4-RELEASE"
     OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
     URLxz="$FREEBSD_REL/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI.raw.xz"
@@ -123,8 +118,8 @@ case "$OS" in
     KSRC="$FREEBSD_SNAP/../amd64/$FreeBSD/src.txz"
     NIC="rtl8139"
     ;;
-  freebsd14-3s)
-    FreeBSD="14.3-STABLE"
+  freebsd14-4s)
+    FreeBSD="14.4-STABLE"
     OSNAME="FreeBSD $FreeBSD"
     OSv="freebsd14.0"
     URLxz="$FREEBSD_SNAP/$FreeBSD/amd64/Latest/FreeBSD-$FreeBSD-amd64-BASIC-CI-ufs.raw.xz"

.github/workflows/scripts/qemu-3-deps-vm.sh

@@ -3,8 +3,11 @@
 ######################################################################
 # 3) install dependencies for compiling and loading
 #
-# $1: OS name (like 'fedora41')
-# $2: (optional) Experimental Fedora kernel version, like "6.14" to
+# qemu-3-deps-vm.sh [--poweroff] OS_NAME [FEDORA_VERSION]
+#
+# --poweroff: Power off the VM after installing dependencies
+# OS_NAME: OS name (like 'fedora41')
+# FEDORA_VERSION: (optional) Experimental Fedora kernel version, like "6.14" to
 #     install instead of Fedora defaults.
 ######################################################################
 
@@ -153,6 +156,12 @@ function install_fedora_experimental_kernel {
   sudo dnf -y copr disable @kernel-vanilla/mainline
 }
 
+POWEROFF=""
+if [ "$1" == "--poweroff" ] ; then
+        POWEROFF=1
+        shift
+fi
+
 # Install dependencies
 case "$1" in
   almalinux8)
@@ -212,6 +221,11 @@ case "$1" in
     sudo apt-get install -yq linux-tools-common libtirpc-dev \
       linux-modules-extra-$(uname -r)
     sudo apt-get install -yq dh-sequence-dkms
+
+    # Need 'build-essential' explicitly for ARM builder
+    # https://github.com/actions/runner-images/issues/9946
+    sudo apt-get install -yq build-essential
+
     echo "##[endgroup]"
     echo "##[group]Delete Ubuntu OpenZFS modules"
     for i in $(find /lib/modules -name zfs -type d); do sudo rm -rvf $i; done
@@ -306,5 +320,7 @@ esac
 
 # reset cloud-init configuration and poweroff
 sudo cloud-init clean --logs
-sleep 2 && sudo poweroff &
+if [ "$POWEROFF" == "1" ] ; then
+        sleep 2 && sudo poweroff &
+fi
 exit 0

.github/workflows/scripts/qemu-4-build-vm.sh

@@ -350,7 +350,16 @@ fi
 # save some sysinfo
 uname -a > /var/tmp/uname.txt
 
-cd $HOME/zfs
+# Check if we're running this script from within a VM or on the runner itself.
+# Most of the time we will be running in a VM, but the ARM builder actually
+# runs this script on the runner.  If we happen to be running on the ARM
+# runner, we will start in the ZFS source directory.  If we're running on a VM
+# then we'll just start in our home directory, and will need to 'cd' into our
+# source directory.
+if [ ! -e META ] ; then
+  cd $HOME/zfs
+fi
+
 export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
 
 extra=""

.github/workflows/zfs-arm.yml

@@ -0,0 +1,40 @@
+name: zfs-arm
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  zfs-arm:
+    name: ZFS ARM build
+    runs-on: ubuntu-24.04-arm
+    steps:
+    - uses: actions/checkout@v6
+      with:
+        fetch-depth: 0
+        ref: ${{ github.event.pull_request.head.sha }}
+    - name: Install dependencies
+      timeout-minutes: 20
+      run: |
+        sudo apt-get -y remove firefox || true
+        .github/workflows/scripts/qemu-3-deps-vm.sh ubuntu24
+
+        # We're running the VM scripts locally on the runner, so need to fix
+        # up hostnames to make it work.
+        for ((i=0; i<=3; i++)); do
+          echo "127.0.0.1 vm$i" | sudo tee -a /etc/hosts
+        done
+    - name: Build modules
+      timeout-minutes: 30
+      run: |
+        .github/workflows/scripts/qemu-4-build-vm.sh --enable-debug ubuntu24
+
+        # Quick sanity test since we're not running the full ZTS
+        sudo modprobe zfs
+        sudo dmesg | grep -i zfs
+        truncate -s 100M file
+        sudo zpool create tank ./file
+        zpool status
+
+        echo "Built ZFS successfully on ARM"

.github/workflows/zfs-qemu-packages.yml

@@ -58,7 +58,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: ['almalinux8', 'almalinux9', 'almalinux10', 'fedora41', 'fedora42', 'fedora43']
+        os: ['almalinux8', 'almalinux9', 'almalinux10', 'fedora42', 'fedora43']
     runs-on: ubuntu-24.04
     steps:
     - uses: actions/checkout@v4

.github/workflows/zfs-qemu.yml

@@ -35,30 +35,44 @@ jobs:
         id: os
         run: |
           ci_type="default"
+          ci_source="auto"
 
           # determine CI type when running on PR
           if ${{ github.event_name == 'pull_request' }}; then
             head=${{ github.event.pull_request.head.sha }}
             base=${{ github.event.pull_request.base.sha }}
-            ci_type=$(python3 .github/workflows/scripts/generate-ci-type.py $head $base)
+            read ci_type ci_source <<< "$(python3 .github/workflows/scripts/generate-ci-type.py $head $base)"
           fi
 
           case "$ci_type" in
           quick)
             os_selection='["almalinux8", "almalinux9", "almalinux10", "debian12", "fedora42", "freebsd15-0s", "ubuntu24"]'
             ;;
           linux)
-            os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian11", "debian12", "debian13", "fedora41", "fedora42", "fedora43", "ubuntu22", "ubuntu24"]'
+            os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian11", "debian12", "debian13", "fedora42", "fedora43", "ubuntu22", "ubuntu24"]'
             ;;
           freebsd)
-            os_selection='["freebsd13-5r", "freebsd14-2r", "freebsd14-3r", "freebsd13-5s", "freebsd14-3s", "freebsd15-0s", "freebsd16-0c"]'
+            os_selection='["freebsd13-5r", "freebsd14-2r", "freebsd14-4r", "freebsd13-5s", "freebsd14-4s", "freebsd15-0s", "freebsd16-0c"]'
             ;;
           *)
             # default list
-            os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian12", "debian13", "fedora42", "fedora43", "freebsd14-3r", "freebsd15-0s", "freebsd16-0c", "ubuntu22", "ubuntu24"]'
+            os_selection='["almalinux8", "almalinux9", "almalinux10", "centos-stream9", "centos-stream10", "debian12", "debian13", "fedora42", "fedora43", "freebsd14-4r", "freebsd15-0s", "freebsd16-0c", "ubuntu22", "ubuntu24"]'
             ;;
           esac
 
+          # Repository-level override for OS selection.
+          # Set vars.ZTS_OS_OVERRIDE in repo settings to restrict targets
+          # (e.g. '["debian13"]' or '["debian13", "fedora42"]').
+          # Manual ZFS-CI-Type in commit messages bypasses the override.
+          if [ -n "${{ vars.ZTS_OS_OVERRIDE }}" ] && [ "$ci_source" != "manual" ]; then
+            override='${{ vars.ZTS_OS_OVERRIDE }}'
+            if echo "$override" | jq -e 'type == "array"' >/dev/null 2>&1; then
+              os_selection="$override"
+            else
+              echo "::warning::Invalid ZTS_OS_OVERRIDE, using default"
+            fi
+          fi
+
           if ${{ github.event.inputs.fedora_kernel_ver != '' }}; then
               # They specified a custom kernel version for Fedora.
               # Use only Fedora runners.
@@ -84,8 +98,8 @@ jobs:
         # debian:  debian12, debian13, ubuntu22, ubuntu24
         # misc:    archlinux, tumbleweed
         # FreeBSD variants of november 2025:
-        # FreeBSD Release: freebsd13-5r, freebsd14-2r, freebsd14-3r
-        # FreeBSD Stable:  freebsd13-5s, freebsd14-3s, freebsd15-0s
+        # FreeBSD Release: freebsd13-5r, freebsd14-2r, freebsd14-4r
+        # FreeBSD Stable:  freebsd13-5s, freebsd14-4s, freebsd15-0s
         # FreeBSD Current: freebsd16-0c
         os: ${{ fromJson(needs.test-config.outputs.test_os) }}
     runs-on: ubuntu-24.04
@@ -104,7 +118,7 @@ jobs:
 
     - name: Install dependencies
       timeout-minutes: 60
-      run: .github/workflows/scripts/qemu-3-deps.sh ${{ matrix.os }} ${{ github.event.inputs.fedora_kernel_ver }}
+      run: .github/workflows/scripts/qemu-3-deps.sh --poweroff ${{ matrix.os }} ${{ github.event.inputs.fedora_kernel_ver }}
 
     - name: Build modules
       timeout-minutes: 30