From 4cfcde024f34b3e5f682364d4e0c6185ef07d467 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 11:16:21 +0100 Subject: [PATCH 01/11] mkosi: enable unprivileged user ns for integration tests Ubuntu disables them by default in Noble, ship a sysctl to turn them back on so that tests can use them --- .../usr/lib/sysctl.d/99-apparmor-unpriv-userns.conf | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 mkosi.images/system/mkosi.extra/usr/lib/sysctl.d/99-apparmor-unpriv-userns.conf diff --git a/mkosi.images/system/mkosi.extra/usr/lib/sysctl.d/99-apparmor-unpriv-userns.conf b/mkosi.images/system/mkosi.extra/usr/lib/sysctl.d/99-apparmor-unpriv-userns.conf new file mode 100644 index 00000000000..657ac72f8de --- /dev/null +++ b/mkosi.images/system/mkosi.extra/usr/lib/sysctl.d/99-apparmor-unpriv-userns.conf @@ -0,0 +1,4 @@ +# Ubuntu since Noble disables unprivileged user namespaces by default, re-enable them as they are needed +# for integration tests +kernel.apparmor_restrict_unprivileged_unconfined = 0 +kernel.apparmor_restrict_unprivileged_userns = 0 From c01cb8cbff8512b65b7903b55f78c8d12661b8d7 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 15:28:56 +0100 Subject: [PATCH 02/11] mkosi: use ports.ubuntu.com for non-x86 backports Follow-up for 46368556afee7a1f3a1685609942438ef2d9d6c1 --- mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf | 3 --- .../mkosi.conf.d/10-ubuntu/mkosi.conf.d/non-x86.conf | 9 +++++++++ .../system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/x86.conf | 9 +++++++++ .../mkosi.conf.d/10-ubuntu/noble-backports-ports.sources | 6 ++++++ 4 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/non-x86.conf create mode 100644 mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/x86.conf create mode 100644 mkosi.images/system/mkosi.conf.d/10-ubuntu/noble-backports-ports.sources diff --git a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf index 25957b1e921..86f9736ed97 100644 --- a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf @@ -3,9 +3,6 @@ [Match] Distribution=ubuntu -[Distribution] -PackageManagerTrees=noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources - [Content] Packages= linux-image-generic diff --git a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/non-x86.conf b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/non-x86.conf new file mode 100644 index 00000000000..0ec4807822f --- /dev/null +++ b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/non-x86.conf @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# The ports Ubuntu archive is for non i386/amd64 repositories + +[Match] +Architecture=!x86-64 +Architecture=!x86 + +[Distribution] +PackageManagerTrees=noble-backports-ports.sources:/etc/apt/sources.list.d/noble-backports-ports.sources diff --git a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/x86.conf b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/x86.conf new file mode 100644 index 00000000000..c08eeac3376 --- /dev/null +++ b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf.d/x86.conf @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# The main Ubuntu archive is only for i386/amd64 repositories + +[Match] +Architecture=|x86-64 +Architecture=|x86 + +[Distribution] +PackageManagerTrees=noble-backports.sources:/etc/apt/sources.list.d/noble-backports.sources diff --git a/mkosi.images/system/mkosi.conf.d/10-ubuntu/noble-backports-ports.sources b/mkosi.images/system/mkosi.conf.d/10-ubuntu/noble-backports-ports.sources new file mode 100644 index 00000000000..5b96dc544dd --- /dev/null +++ b/mkosi.images/system/mkosi.conf.d/10-ubuntu/noble-backports-ports.sources @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +Types: deb +URIs: http://ports.ubuntu.com +Suites: noble-backports +Components: main universe +Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg From 47fe3f29b4ba1b44ae71a7e67c579c4883731dd4 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 20:42:12 +0100 Subject: [PATCH 03/11] mkosi: install EFI packages only on EFI architectures sbsigntool, systemd-boot and systemd-boot-efi do not exist on other architectures --- .../mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 3 --- .../10-debian-ubuntu/mkosi.conf.d/efi.conf | 16 ++++++++++++++++ 2 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/efi.conf diff --git a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index ae014fa9664..ecac78049d4 100644 --- a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -20,8 +20,6 @@ VolatilePackages= libsystemd-dev libudev-dev systemd - systemd-boot - systemd-boot-efi systemd-container systemd-coredump systemd-dev @@ -74,7 +72,6 @@ Packages= python3-pexpect python3-psutil quota - sbsigntool softhsm2 squashfs-tools stress diff --git a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/efi.conf b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/efi.conf new file mode 100644 index 00000000000..781670a775e --- /dev/null +++ b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf.d/efi.conf @@ -0,0 +1,16 @@ +# SPDX-License-Identifier: LGPL-2.1-or-later +# sbsigntool exists only on UEFI architectures + +[Match] +Architecture=|x86 +Architecture=|x86-64 +Architecture=|arm +Architecture=|arm64 +Architecture=|riscv32 +Architecture=|riscv64 + +[Content] +Packages= + sbsigntool + systemd-boot + systemd-boot-efi From e1daedb4be6d8180790e0b303872fb1c87ddc7fc Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 13:41:50 +0100 Subject: [PATCH 04/11] test: check the skip condition before installing additional files --- test/units/TEST-43-PRIVATEUSER-UNPRIV.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/units/TEST-43-PRIVATEUSER-UNPRIV.sh b/test/units/TEST-43-PRIVATEUSER-UNPRIV.sh index 165af47f152..f8a2a624676 100755 --- a/test/units/TEST-43-PRIVATEUSER-UNPRIV.sh +++ b/test/units/TEST-43-PRIVATEUSER-UNPRIV.sh @@ -6,13 +6,13 @@ set -o pipefail # shellcheck source=test/units/util.sh . "$(dirname "$0")"/util.sh -install_extension_images - if [[ "$(sysctl -ne kernel.apparmor_restrict_unprivileged_userns)" -eq 1 ]]; then echo "Cannot create unprivileged user namespaces" >/skipped exit 77 fi +install_extension_images + systemd-analyze log-level debug runas testuser systemd-run --wait --user --unit=test-private-users \ From f44fc531c95e37c83203375c411189009a01b482 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 16:08:57 +0100 Subject: [PATCH 05/11] test: drop unneeded firmware: uefi setting These tests no longer need this, as they are running in nspawn, drop it --- test/TEST-09-REBOOT/meson.build | 2 -- test/TEST-18-FAILUREACTION/meson.build | 2 -- 2 files changed, 4 deletions(-) diff --git a/test/TEST-09-REBOOT/meson.build b/test/TEST-09-REBOOT/meson.build index c4b41bc97b5..b7556189f51 100644 --- a/test/TEST-09-REBOOT/meson.build +++ b/test/TEST-09-REBOOT/meson.build @@ -4,7 +4,5 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), 'storage' : 'persistent', - # FIXME; Figure out why reboot sometimes hangs with 'linux' firmware. - 'firmware' : 'uefi', }, ] diff --git a/test/TEST-18-FAILUREACTION/meson.build b/test/TEST-18-FAILUREACTION/meson.build index 5edfbcad1fb..8dec5f37e73 100644 --- a/test/TEST-18-FAILUREACTION/meson.build +++ b/test/TEST-18-FAILUREACTION/meson.build @@ -3,7 +3,5 @@ integration_tests += [ integration_test_template + { 'name' : fs.name(meson.current_source_dir()), - # FIXME; Figure out why reboot sometimes hangs with 'linux' firmware. - 'firmware' : 'uefi', }, ] From 626518ecd5e7b0c0c708ba53d7eb62934506ed54 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 19:21:32 +0100 Subject: [PATCH 06/11] test: drop obsolete comment We want to keep various logic here instead of mkosi, so drop the temporary comment --- test/integration-test-wrapper.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 5b098a3e01f..1e015e7d47e 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -2,10 +2,6 @@ # SPDX-License-Identifier: LGPL-2.1-or-later '''Test wrapper command for driving integration tests. - -Note: This is deliberately rough and only intended to drive existing tests -with the expectation that as part of formally defining the API it will be tidy. - ''' import argparse From 7d2701e7d1d0a7194026dd371071df6e63f59a82 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Sun, 16 Jun 2024 19:15:24 +0100 Subject: [PATCH 07/11] test: support TEST_NO_KVM The shell integration suite allows to manually deselect KVM, so suppor the same env var for the same purpose in python. --- test/integration-test-wrapper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 1e015e7d47e..15b1ce10553 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -124,6 +124,7 @@ def main(): *args.mkosi_args, '--append', '--qemu-firmware', args.firmware, + '--qemu-kvm', "auto" if not bool(int(os.getenv("TEST_NO_KVM", "0"))) else "no", '--kernel-command-line-extra', ' '.join([ 'systemd.hostname=H', From 464d182b3e470e4163ca376145539a537a6e43a2 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Mon, 17 Jun 2024 14:09:40 +0100 Subject: [PATCH 08/11] test: support TEST_NO_QEMU in mkosi integration wrapper Same as the old integration test suite, allow skipping tests that require qemu. ppc64el's vsock support doesn't appear to work, so we'll skip it, as it is already done in the legacy framework. --- test/integration-test-wrapper.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 15b1ce10553..b6a16aa3efe 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -57,6 +57,10 @@ def main(): print(f"SYSTEMD_SLOW_TESTS=1 not found in environment, skipping {args.name}", file=sys.stderr) exit(77) + if args.vm and bool(int(os.getenv("TEST_NO_QEMU", "0"))): + print(f"TEST_NO_QEMU=1, skipping {args.name}", file=sys.stderr) + exit(77) + name = args.name + (f"-{i}" if (i := os.getenv("MESON_TEST_ITERATION")) else "") dropin = textwrap.dedent( From 80468db8fa21ffd07dc2f28c656eeaf8f0292367 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Mon, 17 Jun 2024 15:37:43 +0100 Subject: [PATCH 09/11] test: use 'auto' instead of 'uefi' for automated fallback mkosi will prefer UEFI if the architecture supports it, but fallback to 'linux' if it doesn't. --- test/TEST-06-SELINUX/meson.build | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/TEST-06-SELINUX/meson.build b/test/TEST-06-SELINUX/meson.build index 7a850beb816..9261a49c49c 100644 --- a/test/TEST-06-SELINUX/meson.build +++ b/test/TEST-06-SELINUX/meson.build @@ -5,7 +5,8 @@ integration_tests += [ 'name' : fs.name(meson.current_source_dir()), 'cmdline' : integration_test_template['cmdline'] + ['selinux=1', 'lsm=selinux'], # FIXME; Figure out why reboot sometimes hangs with 'linux' firmware. - 'firmware' : 'uefi', + # Use 'auto' to automatically fallback on non-uefi architectures. + 'firmware' : 'auto', 'vm' : true, }, ] From bdd0b45bfd7190bb8eb50c71ff6f50a80d6e6e52 Mon Sep 17 00:00:00 2001 From: Luca Boccassi Date: Mon, 17 Jun 2024 17:40:28 +0100 Subject: [PATCH 10/11] CI: disable secure boot in mkosi GHA runs Booting a guest with secure boot is broken in Azure due to a hypervisor bug. Disable it for now. Given there's no option, need to edit the configuration on the fly. --- .github/workflows/mkosi.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index 425d737b62f..62efd367cbd 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -117,6 +117,8 @@ jobs: - name: Configure run: | + # XXX: drop after the HyperV bug that breaks secure boot KVM guests is solved + sed -i "s/'firmware'\s*:\s*'auto'/'firmware' : 'uefi'/g" test/*/meson.build tee mkosi.local.conf < Date: Mon, 17 Jun 2024 15:40:10 +0100 Subject: [PATCH 11/11] mkosi: bump to latest --- .github/workflows/mkosi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mkosi.yml b/.github/workflows/mkosi.yml index 62efd367cbd..3a8dabd95c7 100644 --- a/.github/workflows/mkosi.yml +++ b/.github/workflows/mkosi.yml @@ -92,7 +92,7 @@ jobs: steps: - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 - - uses: systemd/mkosi@0081ea66faf56a35353d6aeadfe42f9679c7d1cf + - uses: systemd/mkosi@6972f9efba5c8472d990be3783b7e7dbf76e109e # Freeing up disk space with rm -rf can take multiple minutes. Since we don't need the extra free space # immediately, we remove the files in the background. However, we first move them to a different location