mkosi: Run integration tests in CI

We do the image build and run the tests in a btrfs loopback so we
can make use of btrfs subvolumes and COW to keep the disk space
requirements to a minimum and speed up the ephemeral copies we make
of the image to run the tests.

We also switch to building debug packages and publishing the built
packages as artifacts.

.github/workflows/mkosi.yml

@@ -69,9 +69,6 @@ jobs:
           - distro: centos
             release: "9"
 
-    env:
-      SYSTEMD_LOG_LEVEL: debug
-
     steps:
     - uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633
     - uses: systemd/mkosi@8cbde8a4ed20a078ad5c70fe38c0dd2294a68bb1
@@ -85,6 +82,18 @@ jobs:
         sudo mv /opt/hostedtoolcache /opt/hostedtoolcache.trash
         sudo systemd-run rm -rf /usr/local.trash /opt/hostedtoolcache.trash
 
+    - name: Btrfs
+      run: |
+        truncate --size=100G btrfs.raw
+        mkfs.btrfs btrfs.raw
+        sudo mkdir /mnt/mkosi
+        LOOP="$(sudo losetup --find --show --direct-io=on btrfs.raw)"
+        sudo mount "$LOOP" /mnt/mkosi --options compress=zstd:1,user_subvol_rm_allowed,noatime,discard=async,space_cache=v2
+        sudo chown "$(id -u):$(id -g)" /mnt/mkosi
+        mkdir /mnt/mkosi/tmp
+        echo "TMPDIR=/mnt/mkosi/tmp" >>"$GITHUB_ENV"
+        ln -s /mnt/mkosi/build build
+
     - name: Configure
       run: |
         tee mkosi.local.conf <<EOF
@@ -95,6 +104,19 @@ jobs:
         [Output]
         # Build a disk image in CI as this logic is much more prone to breakage.
         Format=disk
+        UseSubvolumes=yes
+
+        WorkspaceDirectory=$TMPDIR
+        PackageCacheDirectory=$TMPDIR/cache
+
+        [Content]
+        Environment=
+                # mkfs.erofs is extremely noisy when not connected to a tty.
+                SYSTEMD_REPART_MKFS_OPTIONS_EROFS="--quiet"
+                # Build debuginfo packages since we'll be publishing the packages as artifacts.
+                WITH_DEBUG=1
+                # Enabling optimizations significantly speeds up integration tests.
+                OPTIMIZATION=g
 
         [Host]
         ToolsTree=default
@@ -103,47 +125,82 @@ jobs:
         QemuKvm=yes
         # TODO: Drop once https://bugs.launchpad.net/ubuntu/+source/linux/+bug/2038777 is fixed in Github Actions
         QemuFirmware=uefi
-        Ephemeral=yes
         EOF
 
         # These should override the options from mkosi.conf so we put them in a dropin that's ordered later
         # instead.
         tee mkosi.conf.d/99-ci.conf <<EOF
         [Host]
-        KernelCommandLineExtra=systemd.unit=mkosi-check-and-shutdown.service
+        KernelCommandLineExtra=
-                               systemd.log_level=debug
+                # udev's debug log output is very verbose, so up it to info in CI.
-                               systemd.journald.max_level_console=debug
+                udev.log_level=info
-                               # udev's debug log output is very verbose, so up it to info in CI.
+                # Root device can take a long time to appear, so let's bump the timeout.
-                               udev.log_level=info
+                systemd.default_device_timeout_sec=180
-                               # Root device can take a long time to appear, so let's bump the timeout.
-                               systemd.default_device_timeout_sec=180
         EOF
 
-        # The emergency shell is not useful in the CI, as it just blocks for a long time before the job
-        # eventually times out. Override it to just shutdown immediately.
-        mkdir -p mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/
-        mkdir -p mkosi.images/system/mkosi.extra/usr/lib/systemd/system/emergency.service.d/
-        tee mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf <<EOF
-        [Unit]
-        FailureAction=exit
-        [Service]
-        ExecStartPre=
-        ExecStart=
-        ExecStart=false
-        EOF
-        cp mkosi.images/initrd/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf mkosi.images/system/mkosi.extra/usr/lib/systemd/system/emergency.service.d/poweroff.conf
-
     - name: Generate secure boot key
       run: mkosi --debug genkey
 
     - name: Show image summary
       run: mkosi summary
 
-    - name: Build
+    - name: Install build dependencies
-      run: mkosi --debug
+      run: |
+        sudo apt-get install \
+            meson \
+            gperf \
+            libfdisk-dev \
+            libtss2-dev \
+            libblkid-dev \
+            libmicrohttpd-dev \
+            libcap-dev \
+            libcurl4-openssl-dev \
+            libcryptsetup-dev \
+            erofs-utils \
+            dosfstools \
+            python3-pefile \
+            sbsigntool \
+            mtools
 
-    - name: Boot systemd-nspawn
+    - name: Configure meson
-      run: test "$(sudo mkosi --debug boot 1>&2; echo $?)" -eq 123
+      run: |
+        meson setup build \
+            --buildtype=debugoptimized \
+            -Dintegration-tests=true \
+            -Dremote=enabled \
+            -Dopenssl=enabled \
+            -Dblkid=enabled \
+            -Dtpm2=enabled \
+            -Dlibcryptsetup=enabled \
+            -Dlibcurl=enabled \
+            -Drepart=enabled \
+            -Dfirstboot=true \
+            -Dsysusers=true \
+            -Dtmpfiles=true \
+            -Dhwdb=true \
+            -Dvmspawn=enabled
 
-    - name: Boot QEMU
+    - name: Build image
-      run: timeout -k 30 10m test "$(mkosi --debug qemu 1>&2; echo $?)" -eq 123
+      run: meson compile -C build mkosi
+
+    - name: Run integration tests
+      run: meson test -C build --no-rebuild --suite integration-tests --print-errorlogs --no-stdsplit
+
+    - name: Archive failed test journals
+      uses: actions/upload-artifact@v4
+      if: failure()
+      with:
+        name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-failed-test-journals
+        path: |
+          build/test/journal/*.journal
+
+    - name: Archive packages
+      uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: ci-mkosi-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.distro }}-${{ matrix.release }}-packages
+        path: |
+          build/mkosi.output/*.rpm
+          build/mkosi.output/*.deb
+          build/mkosi.output/*.ddeb
+          build/mkosi.output/*.tar.pkg

mkosi.images/system/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh

@@ -1,20 +0,0 @@
-#!/bin/bash -eux
-# SPDX-License-Identifier: LGPL-2.1-or-later
-
-systemctl --failed --no-legend | tee /failed-services
-
-# Check that secure boot keys were properly enrolled.
-if ! systemd-detect-virt --container && \
-   cmp /sys/firmware/efi/efivars/SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c <(printf '\6\0\0\0\1')
-then
-    cmp /sys/firmware/efi/efivars/SetupMode-8be4df61-93ca-11d2-aa0d-00e098032b8c <(printf '\6\0\0\0\0')
-
-    if command -v sbsign &>/dev/null; then
-        cat /proc/cmdline
-        grep -q this_should_be_here /proc/cmdline
-        (! grep -q this_should_not_be_here /proc/cmdline)
-    fi
-fi
-
-# Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
-[[ ! -s /failed-services ]]

mkosi.images/system/mkosi.extra/usr/lib/systemd/system/mkosi-check-and-shutdown.service

@@ -1,15 +0,0 @@
-# SPDX-License-Identifier: LGPL-2.1-or-later
-[Unit]
-Description=Check if any service failed and then shutdown the machine
-After=multi-user.target network-online.target
-Requires=multi-user.target
-Wants=systemd-resolved.service systemd-networkd.service network-online.target
-SuccessAction=exit
-FailureAction=exit
-# On success, exit with 123 so that we can check that we receive the actual exit code from the script on the
-# host.
-SuccessActionExitStatus=123
-
-[Service]
-Type=oneshot
-ExecStart=/usr/lib/systemd/mkosi-check-and-shutdown.sh