diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml
index 908c36b3f44..886f0bc91d4 100644
--- a/.github/workflows/mkosi.yml
+++ b/.github/workflows/mkosi.yml
@@ -124,3 +124,7 @@ jobs:
 
     - name: Boot ${{ matrix.distro }} QEMU
       run: timeout -k 30 10m mkosi --debug qemu
+
+    # vsock in Github Actions with qemu is broken so for now we check for failures manually.
+    - name: Check ${{ matrix.distro }} QEMU
+      run: sudo mkosi shell bash -c "[[ -e /testok ]] || { cat /failed-services; exit 1; }"
diff --git a/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh b/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
index 9bb246263e4..b77d52d8282 100755
--- a/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
+++ b/mkosi.presets/20-final/mkosi.extra/usr/lib/systemd/mkosi-check-and-shutdown.sh
@@ -1,6 +1,8 @@
 #!/bin/bash -eux
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
+rm -f /testok
+
 # TODO: Figure out why this is failing
 systemctl reset-failed systemd-vconsole-setup.service
 
@@ -17,3 +19,5 @@ fi
 
 # Exit with non-zero EC if the /failed-services file is not empty (we have -e set)
 [[ ! -s /failed-services ]]
+
+touch /testok