diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml
index 1d71471a90..2b5f204f7a 100644
--- a/.github/workflows/mkosi.yml
+++ b/.github/workflows/mkosi.yml
@@ -121,11 +121,7 @@ jobs:
       run: mkosi --debug
 
     - name: Boot ${{ matrix.distro }} systemd-nspawn
-      run: sudo mkosi --debug boot
+      run: test "$(sudo mkosi --debug boot 1>&2; echo $?)" -eq 123
 
     - name: Boot ${{ matrix.distro }} QEMU
-      run: timeout -k 30 10m mkosi --debug qemu
-
-    # vsock in Github Actions with qemu is broken so for now we check for failures manually.
-    - name: Check ${{ matrix.distro }} QEMU
-      run: sudo mkosi shell bash -c "[[ -e /testok ]] || { cat /failed-services; exit 1; }"
+      run: timeout -k 30 10m test "$(mkosi --debug qemu 1>&2; echo $?)" -eq 123
diff --git a/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh b/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
index b77d52d828..30fb30fde7 100755
--- a/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
+++ b/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
@@ -1,8 +1,6 @@
 #!/bin/bash -eux
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-rm -f /testok
-
 # TODO: Figure out why this is failing
 systemctl reset-failed systemd-vconsole-setup.service
 
@@ -20,4 +18,6 @@ fi
 # Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
 [[ ! -s /failed-services ]]
 
-touch /testok
+# On success, exit with 123 so that we can check that we receive the actual exit code from the script on the
+# host.
+exit 123