Merge pull request #26518 from DaanDeMeyer/mkosi-stuff

mkosi: Drop build script workarounds
This commit is contained in:
Daan De Meyer 2023-02-21 18:23:06 +01:00 committed by GitHub
commit 6c53840958
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 203 additions and 284 deletions

View file

@ -20,7 +20,7 @@ indent_style = space
indent_size = 8
max_line_length = 109
[*.sh]
[*.sh,mkosi.build,mkosi.prepare,mkosi.postinst]
indent_style = space
indent_size = 4

View file

@ -79,7 +79,7 @@ jobs:
steps:
- uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
- uses: systemd/mkosi@4be912b0fa4931403fddf649aa242cd4406471c4
- uses: systemd/mkosi@1d131062066fe7b5a83b87319b4464b186adbb1c
- name: Configure
run: |

View file

@ -5,21 +5,6 @@ set -e
# This is a build script for OS image generation using mkosi (https://github.com/systemd/mkosi).
# Simply invoke "mkosi" in the project directory to build an OS image.
ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1
UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
# On Fedora "ld" is (unfortunately — if you ask me) managed via
# "alternatives". Since we'd like to support building images in environments
# with only /usr/ around (e.g. mkosi's UsrOnly=1 option), we have the problem
# that /usr/bin/ld is a symlink that points to a non-existing file in
# /etc/alternative/ in this mode. Let's work around this for now by manually
# redirect "ld" to "ld.bfd", i.e. circumventing the /usr/bin/ld symlink.
if [ ! -x /usr/bin/ld ] && [ -x /usr/bin/ld.bfd ]; then
mkdir -p "$HOME"/bin
ln -s /usr/bin/ld.bfd "$HOME"/bin/ld
PATH="$HOME/bin:$PATH"
fi
# If mkosi.builddir/ exists mkosi will set $BUILDDIR to it, let's then use it
# as out-of-tree build dir. Otherwise, let's make up our own builddir.
[ -z "$BUILDDIR" ] && BUILDDIR="$PWD"/build
@ -28,282 +13,177 @@ fi
PATH="$BUILDDIR:$PATH"
export PATH
# Meson uses Python 3 and requires a locale with an UTF-8 character map.
# Not running under UTF-8 makes the `ninja test` step break with a CodecError.
# So let's ensure we're running under UTF-8.
#
# If our current locale already is UTF-8, then we don't need to do anything:
if [ "$(locale charmap 2>/dev/null)" != "UTF-8" ] ; then
# Try using C.UTF-8 locale, if available. This locale is not shipped
# by upstream glibc, so it's not available in all distros.
# (In particular, it's not available in Arch Linux.)
if locale -a | grep -q -E "C.UTF-8|C.utf8"; then
export LC_CTYPE=C.UTF-8
# Finally, try something like en_US.UTF-8, which should be
# available in Arch Linux, but is not present in Debian's
# minimal image in our mkosi config.
elif locale -a | grep -q en_US.utf8; then
export LC_CTYPE=en_US.UTF-8
else
# If nothing works, fail early.
echo "*** Could not find a valid locale that supports UTF-8. ***" >&2
exit 1
fi
fi
# The bpftool script shipped by Ubuntu tries to find the actual program to run via querying `uname -r` and
# using the current kernel version. This obviously doesn't work in containers. As a workaround, we override
# the ubuntu script with a symlink to the first bpftool program we can find.
for bpftool in /usr/lib/linux-tools/*/bpftool; do
[ -x "$bpftool" ] || continue
ln -sf "$bpftool" "$BUILDDIR"/bpftool
break
[ -x "$bpftool" ] || continue
ln -sf "$bpftool" "$BUILDDIR"/bpftool
break
done
# CentOS Stream 8 includes bpftool 4.18.0 which is lower than what we need. However, they've backported the
# specific feature we need ("gen skeleton") to this version, so we replace bpftool with a script that reports
# version 5.6.0 to satisfy meson which makes bpf work on CentOS Stream 8 as well.
if [ "$(grep '^ID=' /etc/os-release)" = "ID=\"centos\"" ] && [ "$(grep '^VERSION=' /etc/os-release)" = "VERSION=\"8\"" ]; then
cat >"$BUILDDIR"/bpftool <<EOF
cat >"$BUILDDIR"/bpftool <<EOF
#!/bin/sh
if [ "\$1" = --version ]; then
echo 5.6.0
echo 5.6.0
else
exec /usr/sbin/bpftool \$@
exec /usr/sbin/bpftool \$@
fi
EOF
chmod +x "$BUILDDIR"/bpftool
chmod +x "$BUILDDIR"/bpftool
fi
if [ ! -f "$BUILDDIR"/build.ninja ] ; then
sysvinit_path=$(realpath /etc/init.d)
sysvinit_path=$(realpath /etc/init.d)
init_path=$(realpath /sbin/init 2>/dev/null)
if [ -z "$init_path" ] ; then
rootprefix=""
else
rootprefix=${init_path%/lib/systemd/systemd}
rootprefix=/${rootprefix#/}
fi
init_path=$(realpath /sbin/init 2>/dev/null)
if [ -z "$init_path" ] ; then
rootprefix=""
else
rootprefix=${init_path%/lib/systemd/systemd}
rootprefix=/${rootprefix#/}
fi
# On debian-like systems the library directory is not /usr/lib64 but /usr/lib/<arch-triplet>/.
# It is important to use the right one especially for cryptsetup plugins, otherwise they will be
# installed in the wrong directory and not be found by cryptsetup. Assume native build.
if grep -q -e "ID=debian" -e "ID_LIKE=debian" /etc/os-release && command -v dpkg 2>/dev/null; then
LIBDIR="-Drootlibdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)"
PAMDIR="-Dpamlibdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security"
fi
# On debian-like systems the library directory is not /usr/lib64 but /usr/lib/<arch-triplet>/.
# It is important to use the right one especially for cryptsetup plugins, otherwise they will be
# installed in the wrong directory and not be found by cryptsetup. Assume native build.
if grep -q -e "ID=debian" -e "ID_LIKE=debian" /etc/os-release && command -v dpkg 2>/dev/null; then
LIBDIR="-Drootlibdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)"
PAMDIR="-Dpamlibdir=/usr/lib/$(dpkg-architecture -qDEB_HOST_MULTIARCH)/security"
fi
# Cannot quote $LIBDIR and $PAMDIR, because they may be empty, and meson will fail.
# shellcheck disable=SC2086
meson setup "$BUILDDIR" \
${LIBDIR:-} \
${PAMDIR:-} \
-D "sysvinit-path=$sysvinit_path" \
-D "rootprefix=$rootprefix" \
-D man=false \
-D translations=false \
-D version-tag="${VERSION_TAG}" \
-D mode=developer \
-D b_sanitize="${SANITIZERS:-none}" \
-D install-tests=true \
-D tests=unsafe \
-D slow-tests=true \
-D utmp=true \
-D hibernate=true \
-D ldconfig=true \
-D resolve=true \
-D efi=true \
-D tpm=true \
-D environment-d=true \
-D binfmt=true \
-D repart=true \
-D sysupdate=true \
-D coredump=true \
-D pstore=true \
-D oomd=true \
-D logind=true \
-D hostnamed=true \
-D localed=true \
-D machined=true \
-D portabled=true \
-D sysext=true \
-D userdb=true \
-D homed=true \
-D networkd=true \
-D timedated=true \
-D timesyncd=true \
-D remote=true \
-D nss-myhostname=true \
-D nss-mymachines=true \
-D nss-resolve=true \
-D nss-systemd=true \
-D firstboot=true \
-D randomseed=true \
-D backlight=true \
-D vconsole=true \
-D quotacheck=true \
-D sysusers=true \
-D tmpfiles=true \
-D importd=true \
-D hwdb=true \
-D rfkill=true \
-D xdg-autostart=true \
-D translations=true \
-D polkit=true \
-D acl=true \
-D audit=true \
-D blkid=true \
-D fdisk=true \
-D kmod=true \
-D pam=true \
-D pwquality=true \
-D microhttpd=true \
-D libcryptsetup=true \
-D libcurl=true \
-D idn=true \
-D libidn2=true \
-D qrencode=true \
-D gcrypt=true \
-D gnutls=true \
-D openssl=true \
-D cryptolib=openssl \
-D p11kit=true \
-D libfido2=true \
-D tpm2=true \
-D elfutils=true \
-D zstd=true \
-D xkbcommon=true \
-D pcre2=true \
-D glib=true \
-D dbus=true \
-D gnu-efi=true \
-D kernel-install=true \
-D analyze=true \
-D bpf-framework=true \
-D ukify=true
# Cannot quote $LIBDIR and $PAMDIR, because they may be empty, and meson will fail.
# shellcheck disable=SC2086
meson setup "$BUILDDIR" \
${LIBDIR:-} \
${PAMDIR:-} \
-D "sysvinit-path=$sysvinit_path" \
-D "rootprefix=$rootprefix" \
-D man=false \
-D translations=false \
-D version-tag="${VERSION_TAG}" \
-D mode=developer \
-D b_sanitize="${SANITIZERS:-none}" \
-D install-tests=true \
-D tests=unsafe \
-D slow-tests=true \
-D utmp=true \
-D hibernate=true \
-D ldconfig=true \
-D resolve=true \
-D efi=true \
-D tpm=true \
-D environment-d=true \
-D binfmt=true \
-D repart=true \
-D sysupdate=true \
-D coredump=true \
-D pstore=true \
-D oomd=true \
-D logind=true \
-D hostnamed=true \
-D localed=true \
-D machined=true \
-D portabled=true \
-D sysext=true \
-D userdb=true \
-D homed=true \
-D networkd=true \
-D timedated=true \
-D timesyncd=true \
-D remote=true \
-D nss-myhostname=true \
-D nss-mymachines=true \
-D nss-resolve=true \
-D nss-systemd=true \
-D firstboot=true \
-D randomseed=true \
-D backlight=true \
-D vconsole=true \
-D quotacheck=true \
-D sysusers=true \
-D tmpfiles=true \
-D importd=true \
-D hwdb=true \
-D rfkill=true \
-D xdg-autostart=true \
-D translations=true \
-D polkit=true \
-D acl=true \
-D audit=true \
-D blkid=true \
-D fdisk=true \
-D kmod=true \
-D pam=true \
-D pwquality=true \
-D microhttpd=true \
-D libcryptsetup=true \
-D libcurl=true \
-D idn=true \
-D libidn2=true \
-D qrencode=true \
-D gcrypt=true \
-D gnutls=true \
-D openssl=true \
-D cryptolib=openssl \
-D p11kit=true \
-D libfido2=true \
-D tpm2=true \
-D elfutils=true \
-D zstd=true \
-D xkbcommon=true \
-D pcre2=true \
-D glib=true \
-D dbus=true \
-D gnu-efi=true \
-D kernel-install=true \
-D analyze=true \
-D bpf-framework=true \
-D ukify=true
fi
cd "$BUILDDIR"
ninja "$@"
ninja -C "$BUILDDIR" "$@"
if [ "$WITH_TESTS" = 1 ] ; then
if [ -n "$SANITIZERS" ]; then
export ASAN_OPTIONS="$ASAN_OPTIONS"
export UBSAN_OPTIONS="$UBSAN_OPTIONS"
TIMEOUT_MULTIPLIER=3
else
TIMEOUT_MULTIPLIER=1
fi
if [ -n "$SANITIZERS" ]; then
export ASAN_OPTIONS="$MKOSI_ASAN_OPTIONS"
export UBSAN_OPTIONS="$MKOSI_UBSAN_OPTIONS"
TIMEOUT_MULTIPLIER=3
else
TIMEOUT_MULTIPLIER=1
fi
meson test --print-errorlogs --timeout-multiplier=$TIMEOUT_MULTIPLIER
meson test -C "$BUILDDIR" --print-errorlogs --timeout-multiplier=$TIMEOUT_MULTIPLIER
fi
cd "$SRCDIR"
meson install -C "$BUILDDIR" --quiet --no-rebuild --only-changed
mkdir -p "$DESTDIR"/etc
cat >"$DESTDIR"/etc/issue <<EOF
\S (built from systemd tree)
Kernel \r on an \m (\l)
EOF
if [ -n "$IMAGE_ID" ] ; then
mkdir -p "$DESTDIR"/usr/lib
sed -n \
-e '/^IMAGE_ID=/!p' \
-e "\$aIMAGE_ID=$IMAGE_ID" <"/usr/lib/os-release" >"${DESTDIR}/usr/lib/os-release"
OSRELEASEFILE="$DESTDIR"/usr/lib/os-release
else
OSRELEASEFILE=/usr/lib/os-release
fi
if [ -n "$IMAGE_VERSION" ] ; then
mkdir -p "$DESTDIR"/usr/lib
sed -n \
-e '/^IMAGE_VERSION=/!p' \
-e "\$aIMAGE_VERSION=$IMAGE_VERSION" <$OSRELEASEFILE >"/tmp/os-release.tmp"
cat /tmp/os-release.tmp >"$DESTDIR"/usr/lib/os-release
rm /tmp/os-release.tmp
fi
# If $CI_BUILD is set, copy over the CI service which executes a service check
# after boot and then shuts down the machine
if [ -n "$CI_BUILD" ]; then
mkdir -p "$DESTDIR/usr/lib/systemd/system"
cp -v "$SRCDIR/test/mkosi-check-and-shutdown.service" "$DESTDIR/usr/lib/systemd/system/mkosi-check-and-shutdown.service"
cp -v "$SRCDIR/test/mkosi-check-and-shutdown.sh" "$DESTDIR/usr/lib/systemd/mkosi-check-and-shutdown.sh"
chmod +x "$DESTDIR/usr/lib/systemd/mkosi-check-and-shutdown.sh"
fi
if [ -n "$SANITIZERS" ]; then
LD_PRELOAD=$(ldd "$BUILDDIR"/systemd | grep libasan.so | awk '{print $3}')
mkdir -p "$DESTDIR/etc/systemd/system.conf.d"
cat >"$DESTDIR/etc/systemd/system.conf.d/10-asan.conf" <<EOF
[Manager]
ManagerEnvironment=ASAN_OPTIONS=$ASAN_OPTIONS\\
UBSAN_OPTIONS=$UBSAN_OPTIONS\\
LD_PRELOAD=$LD_PRELOAD
DefaultEnvironment=ASAN_OPTIONS=$ASAN_OPTIONS\\
UBSAN_OPTIONS=$UBSAN_OPTIONS\\
LD_PRELOAD=$LD_PRELOAD
EOF
# ASAN logs to stderr by default. However, journald's stderr is connected to /dev/null, so we lose
# all the ASAN logs. To rectify that, let's connect journald's stdout to the console so that any
# sanitizer failures appear directly on the user's console.
mkdir -p "$DESTDIR/etc/systemd/system/systemd-journald.service.d"
cat >"$DESTDIR/etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf" <<EOF
[Service]
StandardOutput=tty
EOF
# Both systemd and util-linux's login call vhangup() on /dev/console which disconnects all users.
# This means systemd-journald can't log to /dev/console even if we configure `StandardOutput=tty`. As
# a workaround, we modify console-getty.service to disable systemd's vhangup() and disallow login
# from calling vhangup() so that journald's ASAN logs correctly end up in the console.
mkdir -p "$DESTDIR/etc/systemd/system/console-getty.service.d"
cat >"$DESTDIR/etc/systemd/system/console-getty.service.d/10-no-vhangup.conf" <<EOF
[Service]
TTYVHangup=no
CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
EOF
fi
# Make sure services aren't enabled by default on Debian/Ubuntu.
mkdir -p "$DESTDIR/etc/systemd/system-preset"
echo "disable *" >"$DESTDIR/etc/systemd/system-preset/99-mkosi.preset"
if [ -d mkosi.kernel/ ]; then
cd "$SRCDIR/mkosi.kernel"
mkdir -p "$BUILDDIR/mkosi.kernel"
SRCDIR="$SRCDIR/mkosi.kernel"
BUILDDIR="$BUILDDIR/mkosi.kernel"
cd "$SRCDIR"
mkdir -p "$BUILDDIR"
# Ensure fast incremental builds by fixating these values which usually change for each build.
export KBUILD_BUILD_TIMESTAMP="Fri Jun 5 15:58:00 CEST 2015"
export KBUILD_BUILD_HOST="mkosi"
# Ensure fast incremental builds by fixating these values which usually change for each build.
export KBUILD_BUILD_TIMESTAMP="Fri Jun 5 15:58:00 CEST 2015"
export KBUILD_BUILD_HOST="mkosi"
scripts/kconfig/merge_config.sh -O "$BUILDDIR/mkosi.kernel" \
../mkosi.kernel.config \
tools/testing/selftests/bpf/config.x86_64 \
tools/testing/selftests/bpf/config
scripts/kconfig/merge_config.sh -O "$BUILDDIR" \
../mkosi.kernel.config \
tools/testing/selftests/bpf/config.x86_64 \
tools/testing/selftests/bpf/config
make O="$BUILDDIR/mkosi.kernel" -j "$(nproc)"
make O="$BUILDDIR" -j "$(nproc)"
KERNEL_RELEASE=$(make O="$BUILDDIR"/mkosi.kernel -s kernelrelease)
mkdir -p "$DESTDIR/usr/lib/modules/$KERNEL_RELEASE"
make O="$BUILDDIR/mkosi.kernel" INSTALL_MOD_PATH="$DESTDIR/usr" modules_install
make O="$BUILDDIR/mkosi.kernel" INSTALL_PATH="$DESTDIR/usr/lib/modules/$KERNEL_RELEASE" install
mkdir -p "$DESTDIR/usr/lib/kernel/selftests"
make -C tools/testing/selftests -j "$(nproc)" O="$BUILDDIR/mkosi.kernel" KSFT_INSTALL_PATH="$DESTDIR/usr/lib/kernel/selftests" SKIP_TARGETS="" install
KERNEL_RELEASE=$(make O="$BUILDDIR" -s kernelrelease)
mkdir -p "$DESTDIR/usr/lib/modules/$KERNEL_RELEASE"
make O="$BUILDDIR" INSTALL_MOD_PATH="$DESTDIR/usr" modules_install
make O="$BUILDDIR" INSTALL_PATH="$DESTDIR/usr/lib/modules/$KERNEL_RELEASE" install
mkdir -p "$DESTDIR/usr/lib/kernel/selftests"
make -C tools/testing/selftests -j "$(nproc)" O="$BUILDDIR" KSFT_INSTALL_PATH="$DESTDIR/usr/lib/kernel/selftests" SKIP_TARGETS="" install
ln -sf /usr/lib/kernel/selftests/bpf/bpftool "$DESTDIR/usr/bin/bpftool"
ln -sf /usr/lib/kernel/selftests/bpf/bpftool "$DESTDIR/usr/bin/bpftool"
fi

View file

@ -4,8 +4,10 @@
[Output]
Bootable=yes
# Prevent ASAN warnings when building the image
# Prevent ASAN warnings when building the image and ship the real ASAN options prefixed with MKOSI_.
Environment=ASAN_OPTIONS=verify_asan_link_order=false
MKOSI_ASAN_OPTIONS=strict_string_checks=1:detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1:disable_coredump=0:use_madv_dontdump=1
MKOSI_UBSAN_OPTIONS=print_stacktrace=1:print_summary=1:halt_on_error=1
OutputDirectory=mkosi.output
[Content]

2
mkosi.extra/etc/issue Normal file
View file

@ -0,0 +1,2 @@
\S (built from systemd tree)
Kernel \r on an \m (\l)

View file

@ -0,0 +1,2 @@
set debuginfod enabled off
set build-id-verbose 0

View file

@ -1,34 +1,67 @@
#!/bin/sh
# SPDX-License-Identifier: LGPL-2.1-or-later
if [ "$1" = "final" ]; then
if command -v bootctl >/dev/null && [ -d "/efi" ]; then
bootctl install
fi
if [ "$1" = "build" ]; then
exit 0
fi
cat >>/root/.gdbinit <<EOF
set debuginfod enabled off
set build-id-verbose 0
if [ -n "$SANITIZERS" ]; then
LD_PRELOAD=$(ldd /usr/lib/systemd/systemd | grep libasan.so | awk '{print $3}')
mkdir -p /etc/systemd/system.conf.d
cat >/etc/systemd/system.conf.d/10-asan.conf <<EOF
[Manager]
ManagerEnvironment=ASAN_OPTIONS=$MKOSI_ASAN_OPTIONS\\
UBSAN_OPTIONS=$MKOSI_UBSAN_OPTIONS\\
LD_PRELOAD=$LD_PRELOAD
DefaultEnvironment=ASAN_OPTIONS=$MKOSI_ASAN_OPTIONS\\
UBSAN_OPTIONS=$MKOSI_UBSAN_OPTIONS\\
LD_PRELOAD=$LD_PRELOAD
EOF
if [ -n "$SANITIZERS" ]; then
# ASAN and syscall filters aren't compatible with each other.
find / -name '*.service' -type f -exec sed -i 's/^\(MemoryDeny\|SystemCall\)/# \1/' {} +
# ASAN logs to stderr by default. However, journald's stderr is connected to /dev/null, so we lose
# all the ASAN logs. To rectify that, let's connect journald's stdout to the console so that any
# sanitizer failures appear directly on the user's console.
mkdir -p /etc/systemd/system/systemd-journald.service.d
cat >/etc/systemd/system/systemd-journald.service.d/10-stdout-tty.conf <<EOF
[Service]
StandardOutput=tty
EOF
# `systemd-hwdb update` takes > 50s when built with sanitizers so let's not run it by default.
systemctl mask systemd-hwdb-update.service
fi
# Both systemd and util-linux's login call vhangup() on /dev/console which disconnects all users.
# This means systemd-journald can't log to /dev/console even if we configure `StandardOutput=tty`. As
# a workaround, we modify console-getty.service to disable systemd's vhangup() and disallow login
# from calling vhangup() so that journald's ASAN logs correctly end up in the console.
# Make sure dnsmasq.service doesn't start on boot on Debian/Ubuntu.
rm -f /etc/systemd/system/multi-user.target.wants/dnsmasq.service
mkdir -p /etc/systemd/system/console-getty.service.d
cat >/etc/systemd/system/console-getty.service.d/10-no-vhangup.conf <<EOF
[Service]
TTYVHangup=no
CapabilityBoundingSet=~CAP_SYS_TTY_CONFIG
EOF
# ASAN and syscall filters aren't compatible with each other.
find / -name '*.service' -type f -exec sed -i 's/^\(MemoryDeny\|SystemCall\)/# \1/' {} +
# `systemd-hwdb update` takes > 50s when built with sanitizers so let's not run it by default.
systemctl mask systemd-hwdb-update.service
fi
# Temporary workaround until https://github.com/openSUSE/suse-module-tools/commit/158643414ddb8d8208016a5f03a4484d58944d7a
# gets into OpenSUSE repos
if [ "$1" = "final" ] && grep -q openSUSE /etc/os-release; then
if [ -e "/usr/lib/systemd/system/boot-sysctl.service" ] && \
! grep -F -q 'ConditionPathExists=/boot/sysctl.conf' "/usr/lib/systemd/system/boot-sysctl.service"; then
mkdir -p "/etc/systemd/system/boot-sysctl.service.d/"
printf '[Unit]\nConditionPathExists=/boot/sysctl.conf-%%v' >"/etc/systemd/system/boot-sysctl.service.d/99-temporary-workaround.conf"
fi
# Make sure dnsmasq.service doesn't start on boot on Debian/Ubuntu.
rm -f /etc/systemd/system/multi-user.target.wants/dnsmasq.service
if [ -n "$IMAGE_ID" ] ; then
sed -n \
-i \
-e '/^IMAGE_ID=/!p' \
-e "\$aIMAGE_ID=$IMAGE_ID" \
/usr/lib/os-release
fi
if [ -n "$IMAGE_VERSION" ] ; then
sed -n \
-i \
-e '/^IMAGE_VERSION=/!p' \
-e "\$aIMAGE_VERSION=$IMAGE_VERSION" \
/usr/lib/os-release
fi