From ba592dc715eed67b0c939ded94b4c31bc5ba1070 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 16:24:42 +0200 Subject: [PATCH 1/5] mkosi: Replace submodules with our own thing Unfortunately, git submodules break in all sorts of ways: - Various github workflows (dependabot, github pages) try to do a shallow clone of git submodules which does not work at all when the git repository is hosted on pagure (https://pagure.io/pagure/issue/5453, https://github.com/dependabot/dependabot-core/issues/9391). - If the git forge hosting the git repository uses SHA256, then it breaks our usage of it as a submodule as SHA256 repositories cannot be used as submodules in SHA1 repositories (src.opensuse.org moved to SHA256 which broke our usage of opensuse's systemd spec as a submodule). - git submodules completely break usage of git worktrees. - ... Let's avoid all these issues by just doing our own home grown implementation of git submodules. We lose the automatic dependabot updates this way but since dependabot fails to run more often that not with submodules we don't really lose anything. --- .github/dependabot.yml | 5 ----- .gitmodules | 20 ------------------ docs/HACKING.md | 9 -------- .../system/mkosi.conf.d/10-arch/mkosi.conf | 5 +++++ .../system/mkosi.conf.d/10-centos/mkosi.conf | 5 +++++ .../mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 5 +++++ .../system/mkosi.conf.d/10-fedora/mkosi.conf | 5 +++++ .../mkosi.conf.d/10-opensuse/mkosi.conf | 5 +++++ mkosi.images/system/mkosi.sync | 21 +++++++++++++++---- pkg/arch | 1 - pkg/centos | 1 - pkg/debian | 1 - pkg/fedora | 1 - pkg/opensuse | 1 - tools/git-setup.sh | 18 ---------------- tools/git-submodule-update-hook.sh | 4 ---- 16 files changed, 42 insertions(+), 65 deletions(-) delete mode 160000 pkg/arch delete mode 160000 pkg/centos delete mode 160000 pkg/debian delete mode 160000 pkg/fedora delete mode 160000 pkg/opensuse delete mode 100755 tools/git-submodule-update-hook.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 54593c50f7b..3e067c176fd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,8 +17,3 @@ updates: schedule: interval: "monthly" open-pull-requests-limit: 2 - - package-ecosystem: "gitsubmodule" - directory: "/" - schedule: - interval: "daily" - open-pull-requests-limit: 2 diff --git a/.gitmodules b/.gitmodules index 7ae6ace6802..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,20 +0,0 @@ -[submodule "pkg/fedora"] - path = pkg/fedora - url = https://src.fedoraproject.org/rpms/systemd.git - branch = rawhide -[submodule "pkg/opensuse"] - path = pkg/opensuse - url = https://code.opensuse.org/package/systemd.git - branch = master -[submodule "pkg/debian"] - path = pkg/debian - url = https://salsa.debian.org/systemd-team/systemd.git - branch = debian/master -[submodule "pkg/centos"] - path = pkg/centos - url = https://git.centos.org/rpms/systemd.git - branch = c9s-sig-hyperscale -[submodule "pkg/arch"] - path = pkg/arch - url = https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git - branch = main diff --git a/docs/HACKING.md b/docs/HACKING.md index 980a45929eb..51499d7f798 100644 --- a/docs/HACKING.md +++ b/docs/HACKING.md @@ -22,15 +22,6 @@ If adding a new source file, consider adding a matching test executable. For features at a higher level, tests in `src/test/` are very strongly recommended. If that is not possible, integration tests in `test/` are encouraged. -```shell -$ git config submodule.recurse true -$ git config fetch.recurseSubmodules on-demand -$ git config push.recurseSubmodules no -$ cp .git/hooks/pre-commit.sample .git/hooks/pre-commit -$ cp tools/git-submodule-update-hook.sh .git/hooks/post-rewrite -$ cp tools/git-submodule-update-hook.sh .git/hooks/post-checkout -``` - Please always test your work before submitting a PR. For many of the components of systemd testing is straightforward as you can simply compile systemd and run the relevant tool from the build directory. diff --git a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf index 942db4cb967..9c7d09d6562 100644 --- a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf @@ -4,6 +4,11 @@ Distribution=arch [Content] +Environment= + GIT_URL=https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git + GIT_BRANCH=main + GIT_COMMIT=dc6c099e0785753c1c88b4adcbcbfc209a8d12e3 + VolatilePackages= systemd systemd-libs diff --git a/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf index 15f888168dc..395a22b56a8 100644 --- a/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf @@ -4,6 +4,11 @@ Distribution=centos [Content] +Environment= + GIT_URL=https://git.centos.org/rpms/systemd.git + GIT_BRANCH=c9s-sig-hyperscale + GIT_COMMIT=8cf2aed0181920611421384f7374720db269d6c7 + Packages= kernel-modules # For squashfs rpmautospec-rpm-macros diff --git a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index 7e870ace870..047dbc1dc3d 100644 --- a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -5,6 +5,11 @@ Distribution=|debian Distribution=|ubuntu [Content] +Environment= + GIT_URL=https://salsa.debian.org/systemd-team/systemd.git + GIT_BRANCH=debian/master + GIT_COMMIT=18201fa98d74172fa1a17242326e3275995cde13 + VolatilePackages= libnss-myhostname libnss-mymachines diff --git a/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf index 3acec1092c3..7b122e3c699 100644 --- a/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf @@ -4,6 +4,11 @@ Distribution=fedora [Content] +Environment= + GIT_URL=https://src.fedoraproject.org/rpms/systemd.git + GIT_BRANCH=rawhide + GIT_COMMIT=74810c5bc4fe7d872e54c253447ffd61bbc8839f + Packages= btrfs-progs compsize diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf index b3172bfc143..90844d4db40 100644 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf @@ -7,6 +7,11 @@ Distribution=opensuse InitrdInclude=initrd/ [Content] +Environment= + GIT_URL=https://src.opensuse.org/rpm/systemd + GIT_BRANCH=factory + GIT_COMMIT=28c9bef337dd47c88adc1d948cedfdc79c788e2955f17a534bd76da6419722dd + VolatilePackages= systemd systemd-boot diff --git a/mkosi.images/system/mkosi.sync b/mkosi.images/system/mkosi.sync index 3e42a09b2c6..a4f0ab94ec1 100755 --- a/mkosi.images/system/mkosi.sync +++ b/mkosi.images/system/mkosi.sync @@ -2,8 +2,21 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ -z "$(ls --almost-all "pkg/$DISTRIBUTION")" ] || [ -f "pkg/$DISTRIBUTION/.git" ]; then - PKG_SUBDIR="$(realpath "pkg/$DISTRIBUTION" --relative-to "$PWD")" - git submodule sync "$PKG_SUBDIR" - git submodule update --init "$PKG_SUBDIR" +if ((NO_SYNC)); then + exit 0 fi + +PKG_SUBDIR="$(realpath --canonicalize-missing "pkg/$DISTRIBUTION" --relative-to "$PWD")" + +if [[ -d "$PKG_SUBDIR/.git" ]] && [[ "$(git -C "$PKG_SUBDIR" rev-parse HEAD)" == "$GIT_COMMIT" ]]; then + exit 0 +fi + +if [[ ! -e "$PKG_SUBDIR" ]] || [[ -z "$(ls --almost-all "$PKG_SUBDIR")" ]]; then + git clone "$GIT_URL" --branch "$GIT_BRANCH" "$PKG_SUBDIR" +else + git -C "$PKG_SUBDIR" remote set-url origin "$GIT_URL" + git -C "$PKG_SUBDIR" fetch origin "$GIT_BRANCH" +fi + +git -C "$PKG_SUBDIR" -c advice.detachedHead=false checkout "$GIT_COMMIT" diff --git a/pkg/arch b/pkg/arch deleted file mode 160000 index b578e90bb68..00000000000 --- a/pkg/arch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b578e90bb68deaae28300300c57b6a99c916b201 diff --git a/pkg/centos b/pkg/centos deleted file mode 160000 index 61bb5f47014..00000000000 --- a/pkg/centos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 61bb5f47014707a927bc033a21e8dbf685c5d2c5 diff --git a/pkg/debian b/pkg/debian deleted file mode 160000 index 824c52fafc3..00000000000 --- a/pkg/debian +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 824c52fafc3afda4c8ad3892596e8243c3b45c91 diff --git a/pkg/fedora b/pkg/fedora deleted file mode 160000 index 8fe1f037d21..00000000000 --- a/pkg/fedora +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8fe1f037d21c9d68d96728843f22e5036d769521 diff --git a/pkg/opensuse b/pkg/opensuse deleted file mode 160000 index c3399411ac7..00000000000 --- a/pkg/opensuse +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3399411ac70a440ca9080d6df391eb897bba4e9 diff --git a/tools/git-setup.sh b/tools/git-setup.sh index 8cc1bfdfc2a..b5903b4ca10 100755 --- a/tools/git-setup.sh +++ b/tools/git-setup.sh @@ -4,12 +4,6 @@ set -eu cd "${MESON_SOURCE_ROOT:?}" -if [ -e .git ]; then - git config submodule.recurse true - git config fetch.recurseSubmodules on-demand - git config push.recurseSubmodules no -fi - ret=2 if [ -f .git/hooks/pre-commit.sample ] && [ ! -f .git/hooks/pre-commit ]; then @@ -19,16 +13,4 @@ if [ -f .git/hooks/pre-commit.sample ] && [ ! -f .git/hooks/pre-commit ]; then ret=0 fi -if [ ! -f .git/hooks/post-rewrite ]; then - cp -p tools/git-submodule-update-hook.sh .git/hooks/post-rewrite - echo 'Activated post-rewrite hook' - ret=0 -fi - -if [ ! -f .git/hooks/post-checkout ]; then - cp -p tools/git-submodule-update-hook.sh .git/hooks/post-checkout - echo 'Activated post-checkout hook' - ret=0 -fi - exit $ret diff --git a/tools/git-submodule-update-hook.sh b/tools/git-submodule-update-hook.sh deleted file mode 100755 index 78feb9dbbae..00000000000 --- a/tools/git-submodule-update-hook.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: LGPL-2.1-or-later - -exec git submodule update From 8c335463d9fe6c2e2f3d9584d7c26871800065e2 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 20:54:06 +0200 Subject: [PATCH 2/5] mkosi: Don't install wireguard-tools on Debian Package was removed from testing. This can be reverted next week once it's added back to testing. --- mkosi.images/system/mkosi.conf | 1 - mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf | 1 + mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf | 1 + mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf | 1 + mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf | 1 + 5 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mkosi.images/system/mkosi.conf b/mkosi.images/system/mkosi.conf index f543b6410f3..6455b0477e2 100644 --- a/mkosi.images/system/mkosi.conf +++ b/mkosi.images/system/mkosi.conf @@ -73,7 +73,6 @@ Packages= util-linux valgrind which - wireguard-tools xfsprogs zsh zstd diff --git a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf index 9c7d09d6562..51f28903b38 100644 --- a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf @@ -58,6 +58,7 @@ Packages= tpm2-tools tpm2-tss vim + wireguard-tools InitrdPackages= btrfs-progs diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf index 799de75e539..65bf20e8560 100644 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf @@ -65,6 +65,7 @@ Packages= util-linux veritysetup vim-common + wireguard-tools InitrdPackages= tpm2-tools diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf index 90844d4db40..188f548643b 100644 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf @@ -85,6 +85,7 @@ Packages= user(root) veritysetup vim + wireguard-tools xz InitrdPackages= diff --git a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf index 86f9736ed97..c52621d41aa 100644 --- a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf @@ -8,3 +8,4 @@ Packages= linux-image-generic linux-tools-common linux-tools-virtual + wireguard-tools From 3b046883addad621550aa74e873256e6fa18373c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 22:16:13 +0200 Subject: [PATCH 3/5] mkosi: Skip debuginfo workaround on newer rpm The corresponding bug was fixed in rpm 4.20 (of which the alpha is in rawhide as rpm 4.19.91) so skip the workaround when we detect a newer rpm version. --- .../mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 8 +++++--- .../system/mkosi.conf.d/10-opensuse/mkosi.build.chroot | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot index a8e4e7e90fb..6d9380c87bd 100755 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot @@ -20,13 +20,15 @@ else TS="${SOURCE_DATE_EPOCH:-$(date +%s)}" fi -# Fix the %install override so debuginfo packages are generated even when --build-in-place is used. -# See https://github.com/rpm-software-management/rpm/issues/3042. -tee --append /usr/lib/rpm/redhat/macros <<'EOF' +if systemd-analyze compare-versions "$(rpm --version | cut -d ' ' -f3)" lt "4.19.91"; then + # Fix the %install override so debuginfo packages are generated even when --build-in-place is used. + # See https://github.com/rpm-software-management/rpm/issues/3042. + tee --append /usr/lib/rpm/redhat/macros <<'EOF' %install %{?_enable_debug_packages:%{debug_package}}\ %%install\ %{nil} EOF +fi VERSION="$(cat meson.version)" RELEASE="$(date "+%Y%m%d%H%M%S" --date "@$TS")" diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot index 8b6fc7ab13c..e7f586cf8b8 100755 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot @@ -26,12 +26,14 @@ fi # extension. find "pkg/$ID" -name "files.*" -exec sed --in-place 's/\.gz$//' {} \; -# Fix the %install override so debuginfo packages are generated. -tee --append /usr/lib/rpm/suse/macros <<'EOF' +if systemd-analyze compare-versions "$(rpm --version | cut -d ' ' -f3)" lt "4.20"; then + # Fix the %install override so debuginfo packages are generated. + tee --append /usr/lib/rpm/suse/macros <<'EOF' %install %{debug_package}\ %%install\ %{nil} EOF +fi VERSION="$(cat meson.version)" RELEASE="$(date "+%Y%m%d%H%M%S" --date "@$TS")" From e74a994ebf7c813979b9e045f37907e6a5064f23 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 22:17:29 +0200 Subject: [PATCH 4/5] mkosi: Disable add-determinism on Fedora This was added for reproducible builds which we don't really care about when building rpms in mkosi, so disable it from running as it's broken and breaks our rpm builds. --- .../system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 1 + 1 file changed, 1 insertion(+) diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot index 6d9380c87bd..3a09f00c25a 100755 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot @@ -72,6 +72,7 @@ ANNOBIN="no-active-checks" rpmbuild \ --define "__brp_strip_comment_note %{nil}" \ --define "__brp_strip_static_archive %{nil}" \ --define "__brp_check_rpaths %{nil}" \ + --undefine __brp_add_determinism \ --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ --define "__script_requires %{nil}" \ --define "_find_debuginfo_dwz_opts %{nil}" \ From 1baea8628775da6b7713863a30381ddb0101e36d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 22:19:00 +0200 Subject: [PATCH 5/5] mkosi: Override rpm's _fixperms script to be a noop Since rpm 4.20 it tries to fix the permissions of all source files in the sources. This takes forever and fails in our case with a permission error. Since we don't want rpm touching our source files, override the script to be a noop. --- .../system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 1 + mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot | 1 + 2 files changed, 2 insertions(+) diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot index 3a09f00c25a..7618f07839e 100755 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot @@ -77,6 +77,7 @@ ANNOBIN="no-active-checks" rpmbuild \ --define "__script_requires %{nil}" \ --define "_find_debuginfo_dwz_opts %{nil}" \ --define "_fortify_level 0" \ + --define "_fixperms true" \ --undefine _lto_cflags \ --undefine _package_note_flags \ --noclean \ diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot index e7f586cf8b8..b2c56fda776 100755 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot @@ -76,6 +76,7 @@ build() { --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ --define "__script_requires %{nil}" \ --define "_find_debuginfo_dwz_opts %{nil}" \ + --define "_fixperms true" \ --noclean \ "$@" \ "pkg/$ID/systemd.spec"