mirror of
https://github.com/SerenityOS/serenity
synced 2024-07-21 18:15:58 +00:00
01b31d9858
Besides a version bump, the following changes have been made to our toolchain infrastructure: - LLVM/Clang is now built with -march=native if the host compiler supports it. An exception to this is CI, as the toolchain cache is shared among many different machines there. - The LLVM tarball is not re-extracted if the hash of the applied patches doesn't differ. - The patches have been split up into atomic chunks. - Port-specific patches have been integrated into the main patches, which will aid in the work towards self-hosting. - <sysroot>/usr/local/lib is now appended to the linker's search path by default. - --pack-dyn-relocs=relr is appended to the linker command line by default, meaning ports take advantage of RELR relocations without any patches or additional compiler flags. The formatting of LLVM port's package.sh has been bothering me, so I also indented the arguments to the CMake invocation.
343 lines
10 KiB
Bash
Executable file
343 lines
10 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
set -eo pipefail
|
|
|
|
# === CONFIGURATION AND SETUP ===
|
|
|
|
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
|
|
|
echo "$DIR"
|
|
|
|
PREFIX="$DIR/Local/clang/"
|
|
BUILD="$DIR/../Build/"
|
|
USERLAND_ARCHS="i686 x86_64"
|
|
ARCHS="$USERLAND_ARCHS aarch64"
|
|
|
|
MD5SUM="md5sum"
|
|
REALPATH="realpath"
|
|
NPROC="nproc"
|
|
INSTALL="install"
|
|
SED="sed"
|
|
|
|
SYSTEM_NAME="$(uname -s)"
|
|
|
|
if [ "$SYSTEM_NAME" = "OpenBSD" ]; then
|
|
MD5SUM="md5 -q"
|
|
REALPATH="readlink -f"
|
|
NPROC="sysctl -n hw.ncpuonline"
|
|
export CC=egcc
|
|
export CXX=eg++
|
|
export LDFLAGS=-Wl,-z,notext
|
|
elif [ "$SYSTEM_NAME" = "FreeBSD" ]; then
|
|
MD5SUM="md5 -q"
|
|
NPROC="sysctl -n hw.ncpu"
|
|
elif [ "$SYSTEM_NAME" = "Darwin" ]; then
|
|
MD5SUM="md5 -q"
|
|
NPROC="sysctl -n hw.ncpu"
|
|
REALPATH="grealpath" # GNU coreutils
|
|
INSTALL="ginstall" # GNU coreutils
|
|
SED="gsed" # GNU sed
|
|
fi
|
|
|
|
if [ -z "$MAKEJOBS" ]; then
|
|
MAKEJOBS=$($NPROC)
|
|
fi
|
|
|
|
|
|
if [ ! -d "$BUILD" ]; then
|
|
mkdir -p "$BUILD"
|
|
fi
|
|
BUILD=$($REALPATH "$BUILD")
|
|
|
|
dev=
|
|
ci=
|
|
|
|
while [ "$1" != "" ]; do
|
|
case $1 in
|
|
--dev ) dev=1
|
|
;;
|
|
--ci ) ci=1
|
|
;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
if [ "$dev" = "1" ] && [ "$ci" = "1" ]; then
|
|
echo "Please only set one of --dev or --ci."
|
|
exit 1
|
|
fi
|
|
|
|
echo PREFIX is "$PREFIX"
|
|
|
|
mkdir -p "$DIR/Tarballs"
|
|
|
|
LLVM_VERSION="14.0.1"
|
|
LLVM_MD5SUM="47a50c31659488a6ae562475b41d2c32"
|
|
LLVM_NAME="llvm-project-$LLVM_VERSION.src"
|
|
LLVM_PKG="$LLVM_NAME.tar.xz"
|
|
LLVM_URL="https://github.com/llvm/llvm-project/releases/download/llvmorg-$LLVM_VERSION/$LLVM_PKG"
|
|
|
|
buildstep() {
|
|
NAME=$1
|
|
shift
|
|
"$@" 2>&1 | "$SED" $'s|^|\e[34m['"${NAME}"$']\e[39m |'
|
|
}
|
|
|
|
buildstep_ninja() {
|
|
# When ninja writes to a pipe, it strips ANSI escape codes and prints one line per buildstep.
|
|
# Instead, use NINJA_STATUS so that we get colored output from LLVM's build and fewer lines of output when running in a tty.
|
|
# ANSI escape codes in NINJA_STATUS are slightly janky (ninja thinks that "\e[34m" needs 5 output characters instead of 5, so
|
|
# its middle elision is slightly off; also it would happily elide the "\e39m" which messes up the output if the terminal is too
|
|
# narrow), but it's still working better than the alternative.
|
|
NAME=$1
|
|
shift
|
|
env NINJA_STATUS=$'\e[34m['"${NAME}"$']\e[39m [%f/%t] ' "$@"
|
|
}
|
|
|
|
# === DEPENDENCIES ===
|
|
|
|
buildstep dependencies echo "Checking whether Ninja is available..."
|
|
if ! command -v ninja >/dev/null; then
|
|
buildstep dependencies echo "Please make sure to install Ninja."
|
|
exit 1
|
|
fi
|
|
|
|
buildstep dependencies echo "Checking whether CMake is available..."
|
|
if ! command -v cmake >/dev/null; then
|
|
buildstep dependencies echo "Please make sure to install CMake."
|
|
exit 1
|
|
fi
|
|
|
|
buildstep dependencies echo "Checking whether 'patch' is available..."
|
|
if ! command -v patch >/dev/null; then
|
|
buildstep dependencies echo "Please make sure to install GNU patch."
|
|
fi
|
|
|
|
buildstep dependencies echo "Checking whether your C compiler works..."
|
|
if ! ${CC:-cc} -o /dev/null -xc - >/dev/null <<'PROGRAM'
|
|
int main() {}
|
|
PROGRAM
|
|
then
|
|
buildstep dependencies echo "Please make sure to install a working C compiler."
|
|
exit 1
|
|
fi
|
|
|
|
buildstep dependencies echo "Checking whether your C++ compiler works..."
|
|
if ! ${CXX:-c++} -o /dev/null -xc - >/dev/null <<'PROGRAM'
|
|
int main() {}
|
|
PROGRAM
|
|
then
|
|
buildstep dependencies echo "Please make sure to install a working C++ compiler."
|
|
exit 1
|
|
fi
|
|
|
|
link_lld=
|
|
buildstep dependencies echo "Checking whether the LLD linker is available..."
|
|
if ${CXX:-c++} -o /dev/null -fuse-ld=lld -xc - >/dev/null 2>/dev/null << 'PROGRAM'
|
|
int main() {}
|
|
PROGRAM
|
|
then
|
|
link_lld=1
|
|
buildstep dependencies echo "Using LLD for linking LLVM."
|
|
else
|
|
buildstep dependencies echo "LLD not found. Using the default linker."
|
|
fi
|
|
|
|
buildstep setup echo "Determining if LLVM should be built with -march=native..."
|
|
if [ "$ci" = "1" ]; then
|
|
# The toolchain cache is shared among all runners, which might have different CPUs.
|
|
buildstep setup echo "On a CI runner. Using the default compiler settings."
|
|
elif [ -z "${CFLAGS+x}" ] && [ -z "${CXXFLAGS+x}" ]; then
|
|
if ${CXX:-c++} -o /dev/null -march=native -xc - >/dev/null 2>/dev/null << 'PROGRAM'
|
|
int main() {}
|
|
PROGRAM
|
|
then
|
|
export CFLAGS="-march=native"
|
|
export CXXFLAGS="-march=native"
|
|
buildstep setup echo "Using -march=native for compiling LLVM."
|
|
else
|
|
buildstep setup echo "-march=native is not supported by the compiler. Using the default settings."
|
|
fi
|
|
else
|
|
buildstep setup echo "Using user-provided CFLAGS/CXXFLAGS."
|
|
fi
|
|
|
|
# === CHECK CACHE AND REUSE ===
|
|
|
|
pushd "$DIR"
|
|
if [ "$TRY_USE_LOCAL_TOOLCHAIN" = "y" ]; then
|
|
mkdir -p Cache
|
|
echo "Cache (before):"
|
|
ls -l Cache
|
|
CACHED_TOOLCHAIN_ARCHIVE="Cache/ToolchainBinariesGithubActions.tar.gz"
|
|
if [ -r "${CACHED_TOOLCHAIN_ARCHIVE}" ] ; then
|
|
echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} exists!"
|
|
echo "Extracting toolchain from cache:"
|
|
if tar xzf "${CACHED_TOOLCHAIN_ARCHIVE}" ; then
|
|
echo "Done 'building' the toolchain."
|
|
echo "Cache unchanged."
|
|
exit 0
|
|
else
|
|
echo
|
|
echo
|
|
echo
|
|
echo "Could not extract cached toolchain archive."
|
|
echo "This means the cache is broken and *should be removed*!"
|
|
echo "As Github Actions cannot update a cache, this will unnecessarily"
|
|
echo "slow down all future builds for this hash, until someone"
|
|
echo "resets the cache."
|
|
echo
|
|
echo
|
|
echo
|
|
rm -f "${CACHED_TOOLCHAIN_ARCHIVE}"
|
|
fi
|
|
else
|
|
echo "Cache at ${CACHED_TOOLCHAIN_ARCHIVE} does not exist."
|
|
echo "Will rebuild toolchain from scratch, and save the result."
|
|
fi
|
|
fi
|
|
echo "::group::Actually building Toolchain"
|
|
popd
|
|
|
|
# === DOWNLOAD AND PATCH ===
|
|
|
|
pushd "$DIR/Tarballs"
|
|
md5=""
|
|
if [ -e "$LLVM_PKG" ]; then
|
|
md5="$($MD5SUM ${LLVM_PKG} | cut -f1 -d' ')"
|
|
echo "llvm md5='$md5'"
|
|
fi
|
|
|
|
if [ "$md5" != "$LLVM_MD5SUM" ] ; then
|
|
rm -f "$LLVM_PKG"
|
|
curl -LO "$LLVM_URL"
|
|
else
|
|
echo "Skipped downloading LLVM"
|
|
fi
|
|
|
|
patch_md5="$($MD5SUM "$DIR"/Patches/llvm/*.patch)"
|
|
|
|
if [ ! -d "$LLVM_NAME" ] || [ "$(cat $LLVM_NAME/.patch.applied)" != "$patch_md5" ]; then
|
|
if [ -d "$LLVM_NAME" ]; then
|
|
# Drop the previously patched extracted dir
|
|
rm -rf "${LLVM_NAME}"
|
|
fi
|
|
|
|
rm -rf "$DIR/Build/clang"
|
|
|
|
echo "Extracting LLVM..."
|
|
tar -xJf "$LLVM_PKG"
|
|
|
|
pushd "$LLVM_NAME"
|
|
if [ "$dev" = "1" ]; then
|
|
git init > /dev/null
|
|
git add . > /dev/null
|
|
git commit -am "BASE" > /dev/null
|
|
git am "$DIR"/Patches/llvm/*.patch > /dev/null
|
|
else
|
|
for patch in "$DIR"/Patches/llvm/*.patch; do
|
|
patch -p1 < "$patch" > /dev/null
|
|
done
|
|
fi
|
|
echo "$patch_md5" > .patch.applied
|
|
popd
|
|
else
|
|
echo "Using existing LLVM source directory"
|
|
fi
|
|
popd
|
|
|
|
# === COPY HEADERS ===
|
|
|
|
SRC_ROOT=$($REALPATH "$DIR"/..)
|
|
FILES=$(find "$SRC_ROOT"/Kernel/API "$SRC_ROOT"/Userland/Libraries/LibC "$SRC_ROOT"/Userland/Libraries/LibM "$SRC_ROOT"/Userland/Libraries/LibPthread "$SRC_ROOT"/Userland/Libraries/LibDl -name '*.h' -print)
|
|
|
|
for arch in $ARCHS; do
|
|
mkdir -p "$BUILD/${arch}clang"
|
|
pushd "$BUILD/${arch}clang"
|
|
mkdir -p Root/usr/include/
|
|
for header in $FILES; do
|
|
target=$(echo "$header" | "$SED" -e "s@$SRC_ROOT/Userland/Libraries/LibC@@" -e "s@$SRC_ROOT/Userland/Libraries/LibM@@" -e "s@$SRC_ROOT/Userland/Libraries/LibPthread@@" -e "s@$SRC_ROOT/Userland/Libraries/LibDl@@" -e "s@$SRC_ROOT/Kernel/@Kernel/@")
|
|
buildstep "system_headers" "$INSTALL" -D "$header" "Root/usr/include/$target"
|
|
done
|
|
popd
|
|
done
|
|
unset SRC_ROOT
|
|
|
|
# === COPY LIBRARY STUBS ===
|
|
|
|
for arch in $USERLAND_ARCHS; do
|
|
pushd "$BUILD/${arch}clang"
|
|
mkdir -p Root/usr/lib/
|
|
for lib in "$DIR/Stubs/${arch}clang/"*".so"; do
|
|
lib_name=$(basename "$lib")
|
|
[ ! -f "Root/usr/lib/${lib_name}" ] && cp "$lib" "Root/usr/lib/${lib_name}"
|
|
done
|
|
popd
|
|
done
|
|
|
|
# === COMPILE AND INSTALL ===
|
|
|
|
rm -rf "$PREFIX"
|
|
mkdir -p "$PREFIX"
|
|
|
|
mkdir -p "$DIR/Build/clang"
|
|
|
|
pushd "$DIR/Build/clang"
|
|
mkdir -p llvm
|
|
pushd llvm
|
|
buildstep "llvm/configure" cmake "$DIR/Tarballs/$LLVM_NAME/llvm" \
|
|
-G Ninja \
|
|
-DSERENITY_i686-pc-serenity_SYSROOT="$BUILD/i686clang/Root" \
|
|
-DSERENITY_x86_64-pc-serenity_SYSROOT="$BUILD/x86_64clang/Root" \
|
|
-DSERENITY_aarch64-pc-serenity_SYSROOT="$BUILD/aarch64clang/Root" \
|
|
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
|
|
-DSERENITY_MODULE_PATH="$DIR/CMake" \
|
|
-C "$DIR/CMake/LLVMConfig.cmake" \
|
|
${link_lld:+"-DLLVM_ENABLE_LLD=ON"} \
|
|
${dev:+"-DLLVM_CCACHE_BUILD=ON"} \
|
|
${ci:+"-DLLVM_CCACHE_BUILD=ON"} \
|
|
${ci:+"-DLLVM_CCACHE_DIR=$LLVM_CCACHE_DIR"} \
|
|
${ci:+"-DLLVM_CCACHE_MAXSIZE=$LLVM_CCACHE_MAXSIZE"}
|
|
|
|
buildstep_ninja "llvm/build" ninja -j "$MAKEJOBS"
|
|
buildstep_ninja "llvm/install" ninja install/strip
|
|
popd
|
|
|
|
for arch in $ARCHS; do
|
|
mkdir -p runtimes/"$arch"
|
|
pushd runtimes/"$arch"
|
|
buildstep "runtimes/$arch/configure" cmake "$DIR/Tarballs/$LLVM_NAME/runtimes" \
|
|
-G Ninja \
|
|
-DSERENITY_TOOLCHAIN_ARCH="$arch" \
|
|
-DSERENITY_TOOLCHAIN_ROOT="$PREFIX" \
|
|
-DSERENITY_BUILD_DIR="$BUILD/${arch}clang/" \
|
|
-DSERENITY_MODULE_PATH="$DIR/CMake" \
|
|
-DCMAKE_INSTALL_PREFIX="$PREFIX" \
|
|
-C "$DIR/CMake/LLVMRuntimesConfig.cmake"
|
|
|
|
buildstep_ninja "runtimes/$arch/build" ninja -j "$MAKEJOBS"
|
|
buildstep_ninja "runtimes/$arch/install" ninja install
|
|
popd
|
|
done
|
|
popd
|
|
|
|
pushd "$DIR/Local/clang/bin/"
|
|
buildstep "mold_symlink" ln -s ../../mold/bin/mold ld.mold
|
|
popd
|
|
|
|
# === SAVE TO CACHE ===
|
|
|
|
pushd "$DIR"
|
|
if [ "$TRY_USE_LOCAL_TOOLCHAIN" = "y" ]; then
|
|
echo "::endgroup::"
|
|
echo "Building cache tar:"
|
|
|
|
echo "Building cache tar:"
|
|
|
|
rm -f "${CACHED_TOOLCHAIN_ARCHIVE}" # Just in case
|
|
|
|
tar czf "${CACHED_TOOLCHAIN_ARCHIVE}" Local/
|
|
echo "Cache (after):"
|
|
ls -l Cache
|
|
fi
|
|
popd
|