Meta: Move UCD/CLDR/TZDB downloaded artifacts to Build/caches

They currently reside under Build/<arch>, meaning that they would be redownloaded for each architecture/toolchain build combo. Move them to a location that can be re-used for all builds.
2024-07-21 18:15:58 +00:00 · 2022-11-25 21:03:42 -05:00 · 2022-11-25 21:03:42 -05:00 · 2334b4cebd
parent 186accb81d
commit 2334b4cebd
13 changed files with 47 additions and 43 deletions
--- a/.github/workflows/cmake.yml
+++ b/.github/workflows/cmake.yml
@ -130,26 +130,27 @@ jobs:
          ccache -s
      - name: Create build directory
        run: |
-          mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}/TZDB
-          mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}/UCD
-          mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}/CLDR
+          mkdir -p ${{ github.workspace }}/Build/${{ matrix.arch }}
+          mkdir -p ${{ github.workspace }}/Build/caches/TZDB
+          mkdir -p ${{ github.workspace }}/Build/caches/UCD
+          mkdir -p ${{ github.workspace }}/Build/caches/CLDR
      - name: TimeZoneData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ matrix.arch }}/TZDB
+          path: ${{ github.workspace }}/Build/caches/TZDB
          key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}
      - name: UnicodeData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ matrix.arch }}/UCD
+          path: ${{ github.workspace }}/Build/caches/UCD
          key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}
      - name: UnicodeLocale Cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ matrix.arch }}/CLDR
+          path: ${{ github.workspace }}/Build/caches/CLDR
          key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }}
      - name: Create build environment with extra debug options
        # Build the entire project with all available debug options turned on, to prevent code rot.
--- a/.github/workflows/pvs-studio-static-analysis.yml
+++ b/.github/workflows/pvs-studio-static-analysis.yml
@ -64,27 +64,28 @@ jobs:

      - name: Create build directory
        run: |
-          mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/TZDB
-          mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/UCD
-          mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/CLDR
+          mkdir -p ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}
+          mkdir -p ${{ github.workspace }}/Build/caches/TZDB
+          mkdir -p ${{ github.workspace }}/Build/caches/UCD
+          mkdir -p ${{ github.workspace }}/Build/caches/CLDR

      - name: TimeZoneData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/TZDB
+          path: ${{ github.workspace }}/Build/caches/TZDB
          key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}
      - name: UnicodeData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/UCD
+          path: ${{ github.workspace }}/Build/caches/UCD
          key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}
      - name: UnicodeLocale Cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ env.PVS_STUDIO_ANALYSIS_ARCH }}/CLDR
+          path: ${{ github.workspace }}/Build/caches/CLDR
          key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }}

      - name: Create build environment
--- a/.github/workflows/sonar-cloud-static-analysis.yml
+++ b/.github/workflows/sonar-cloud-static-analysis.yml
@ -97,27 +97,28 @@ jobs:

      - name: Create build directory
        run: |
-          mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/TZDB
-          mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/UCD
-          mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/CLDR
+          mkdir -p ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}
+          mkdir -p ${{ github.workspace }}/Build/caches/TZDB
+          mkdir -p ${{ github.workspace }}/Build/caches/UCD
+          mkdir -p ${{ github.workspace }}/Build/caches/CLDR

      - name: TimeZoneData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/TZDB
+          path: ${{ github.workspace }}/Build/caches/TZDB
          key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}
      - name: UnicodeData cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/UCD
+          path: ${{ github.workspace }}/Build/caches/UCD
          key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}
      - name: UnicodeLocale Cache
        # TODO: Change the version to the released version when https://github.com/actions/cache/pull/489 (or 571) is merged.
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/${{ env.SONAR_ANALYSIS_ARCH }}/CLDR
+          path: ${{ github.workspace }}/Build/caches/CLDR
          key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }}

      - name: Create build environment
--- a/.github/workflows/wasm.yml
+++ b/.github/workflows/wasm.yml
@ -29,26 +29,23 @@ jobs:
          gcc --version
      - name: "Create build directories"
        run: |
-          mkdir -p ${{ github.workspace }}/Build/lagom-tools/TZDB
-          mkdir -p ${{ github.workspace }}/Build/wasm/UCD
-          mkdir -p ${{ github.workspace }}/Build/wasm/CLDR
+          mkdir -p ${{ github.workspace }}/Build/caches/TZDB
+          mkdir -p ${{ github.workspace }}/Build/caches/UCD
+          mkdir -p ${{ github.workspace }}/Build/caches/CLDR
      - name: "TimeZoneData cache"
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/lagom-tools/TZDB
+          path: ${{ github.workspace }}/Build/caches/TZDB
          key: TimeZoneData-${{ hashFiles('Meta/CMake/time_zone_data.cmake') }}
-      - name: "Copy over TZDB cache"
-        run: |
-          cp -r ${{ github.workspace }}/Build/lagom-tools/TZDB ${{ github.workspace }}/Build/wasm/TZDB
      - name: "UnicodeData cache"
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/wasm/UCD
+          path: ${{ github.workspace }}/Build/caches/UCD
          key: UnicodeData-${{ hashFiles('Meta/CMake/unicode_data.cmake') }}
      - name: "UnicodeLocale cache"
        uses: actions/cache@03e00da99d75a2204924908e1cca7902cafce66b
        with:
-          path: ${{ github.workspace }}/Build/wasm/CLDR
+          path: ${{ github.workspace }}/Build/caches/CLDR
          key: UnicodeLocale-${{ hashFiles('Meta/CMake/locale_data.cmake') }}
      - name: "Build host lagom tools"
        run: |
@ -56,11 +53,12 @@ jobs:
            -B ${{ github.workspace }}/Build/lagom-tools \
            -S ${{ github.workspace }}/Meta/Lagom \
            -DBUILD_LAGOM=OFF \
+            -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches \
            -DCMAKE_C_COMPILER=gcc-12 \
            -DCMAKE_CXX_COMPILER=g++-12 \
            -DCMAKE_INSTALL_PREFIX=${{ github.workspace }}/Build/lagom-tools \
            -Dpackage=LagomTools
-          
+
          ninja -C ${{ github.workspace }}/Build/lagom-tools install
      - name: "Create wasm build environment"
        run: |
@ -69,6 +67,7 @@ jobs:
            -S ${{ github.workspace }}/Meta/Lagom \
            -DLagomTools_DIR=${{ github.workspace }}/Build/lagom-tools/share/LagomTools \
            -DBUILD_LAGOM=ON \
+            -DSERENITY_CACHE_DIR=${{ github.workspace }}/Build/caches \
            -DBUILD_SHARED_LIBS=OFF
      - name: "Build libjs.{js,wasm}"
        run: |
--- a/Meta/Azure/Caches.yml
+++ b/Meta/Azure/Caches.yml
@ -3,7 +3,7 @@ parameters:
  arch: 'i686'
  toolchain: 'gcc'
  coverage: 'OFF'
-  build_directory: ''
+  download_cache_path: ''
  ccache_version: 1 # Increment this number if CI has trouble with ccache.
  serenity_ccache_path: ''
  serenity_ccache_size: '5G'
@ -64,7 +64,7 @@ steps:
        key: '"time_zone_data" | Meta/CMake/time_zone_data.cmake | "$(timestamp)"'
        restoreKeys: |
            "time_zone_data" | Meta/CMake/time_zone_data.cmake
-        path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/TZDB
+        path: $(Build.SourcesDirectory)/${{ parameters.download_cache_path }}/TZDB
      displayName: 'TimeZoneData Cache'

    - task: Cache@2
@ -72,7 +72,7 @@ steps:
        key: '"unicode_data" | Meta/CMake/unicode_data.cmake | "$(timestamp)"'
        restoreKeys: |
            "unicode_data" | Meta/CMake/unicode_data.cmake
-        path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/UCD
+        path: $(Build.SourcesDirectory)/${{ parameters.download_cache_path }}/UCD
      displayName: 'UnicodeData Cache'

    - task: Cache@2
@ -80,5 +80,5 @@ steps:
        key: '"unicode_locale" | Meta/CMake/locale_data.cmake | "$(timestamp)"'
        restoreKeys: |
            "unicode_locale" | Meta/CMake/locale_data.cmake
-        path: $(Build.SourcesDirectory)/${{ parameters.build_directory }}/CLDR
+        path: $(Build.SourcesDirectory)/${{ parameters.download_cache_path }}/CLDR
      displayName: 'UnicodeLocale Cache'
--- a/Meta/Azure/Lagom.yml
+++ b/Meta/Azure/Lagom.yml
@ -39,7 +39,7 @@ jobs:
        os: '${{ parameters.os }}'
        arch: 'Lagom'
        toolchain: '$(toolchain)'
-        build_directory: 'Meta/Lagom/Build'
+        download_cache_path: 'Meta/Lagom/Build'
        serenity_ccache_path: '$(SERENITY_CCACHE_DIR)'
        with_remote_data_caches: true
        ${{ if eq(parameters.os, 'macOS') }}:
--- a/Meta/Azure/Serenity.yml
+++ b/Meta/Azure/Serenity.yml
@ -27,7 +27,7 @@ jobs:
        arch: '${{ parameters.arch }}'
        coverage: '${{ parameters.coverage }}'
        toolchain: 'clang'
-        build_directory: 'Build/${{ parameters.arch }}clang'
+        download_cache_path: 'Build/caches'
        toolchain_ccache_path: '$(LLVM_CCACHE_DIR)'
        toolchain_ccache_size: '$(LLVM_CCACHE_MAXSIZE)'
        serenity_ccache_path: '$(SERENITY_CCACHE_DIR)'
--- a/Meta/CMake/Superbuild/CMakeLists.txt
+++ b/Meta/CMake/Superbuild/CMakeLists.txt
@ -44,6 +44,9 @@ if(NOT SERENITY_TOOLCHAIN STREQUAL "GNU")
 endif()
 set(SERENITY_BUILD_DIR "${PROJECT_BINARY_DIR}/../${SERENITY_ARCH}${SERENITY_BUILD_DIR_SUFFIX}")

+# Location to cache artifacts downloaded during the build.
+file(REAL_PATH "${PROJECT_BINARY_DIR}/../caches" SERENITY_CACHE_DIR)
+
 # Pkgconf incorrectly discards a sysroot if it doesn't match the start of the path to the
 # library file. To avoid that, resolve our sysroot into an absolute and canonical path
 # that matches pkgconf's result for resolving the library file.
@ -97,9 +100,7 @@ ExternalProject_Add(
    EXCLUDE_FROM_ALL YES
    CMAKE_CACHE_ARGS
    "-DCMAKE_INSTALL_PREFIX:STRING=<INSTALL_DIR>"
-    "-DTZDB_PATH:STRING=${SERENITY_BUILD_DIR}/TZDB"
-    "-DUCD_PATH:STRING=${SERENITY_BUILD_DIR}/UCD"
-    "-DCLDR_PATH:STRING=${SERENITY_BUILD_DIR}/CLDR"
+    "-DSERENITY_CACHE_DIR:STRING=${SERENITY_CACHE_DIR}"
    ${lagom_options}
    # Always call the build step of tools, so keeping things up-to-date is easy
    BUILD_ALWAYS YES
@ -126,9 +127,7 @@ ExternalProject_Add(
    # Tell the find_package(Lagom REQUIRED) command call where to find
    # the CMake package
    "-DCMAKE_PREFIX_PATH:STRING=${PROJECT_BINARY_DIR}/../lagom-install"
-    "-DTZDB_PATH:STRING=${SERENITY_BUILD_DIR}/TZDB"
-    "-DUCD_PATH:STRING=${SERENITY_BUILD_DIR}/UCD"
-    "-DCLDR_PATH:STRING=${SERENITY_BUILD_DIR}/CLDR"
+    "-DSERENITY_CACHE_DIR:STRING=${SERENITY_CACHE_DIR}"
    "-DSERENITY_ARCH:STRING=${SERENITY_ARCH}"
    "${SERENITY_TOOLCHAIN_FILE_ARG}"
    ${serenity_options}
--- a/Meta/CMake/locale_data.cmake
+++ b/Meta/CMake/locale_data.cmake
@ -1,7 +1,7 @@
 include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake)

 set(CLDR_VERSION 42.0.0)
-set(CLDR_PATH "${CMAKE_BINARY_DIR}/CLDR" CACHE PATH "Download location for CLDR files")
+set(CLDR_PATH "${SERENITY_CACHE_DIR}/CLDR" CACHE PATH "Download location for CLDR files")
 set(CLDR_VERSION_FILE "${CLDR_PATH}/version.txt")

 set(CLDR_ZIP_URL "https://github.com/unicode-org/cldr-json/releases/download/${CLDR_VERSION}/cldr-${CLDR_VERSION}-json-modern.zip")
--- a/Meta/CMake/time_zone_data.cmake
+++ b/Meta/CMake/time_zone_data.cmake
@ -1,6 +1,6 @@
 include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake)

-set(TZDB_PATH "${CMAKE_BINARY_DIR}/TZDB" CACHE PATH "Download location for TZDB files")
+set(TZDB_PATH "${SERENITY_CACHE_DIR}/TZDB" CACHE PATH "Download location for TZDB files")

 set(TZDB_VERSION 2022g)
 set(TZDB_VERSION_FILE "${TZDB_PATH}/version.txt")
--- a/Meta/CMake/unicode_data.cmake
+++ b/Meta/CMake/unicode_data.cmake
@ -1,7 +1,7 @@
 include(${CMAKE_CURRENT_LIST_DIR}/utils.cmake)

 set(UCD_VERSION 15.0.0)
-set(UCD_PATH "${CMAKE_BINARY_DIR}/UCD" CACHE PATH "Download location for UCD files")
+set(UCD_PATH "${SERENITY_CACHE_DIR}/UCD" CACHE PATH "Download location for UCD files")
 set(UCD_VERSION_FILE "${UCD_PATH}/version.txt")

 set(UCD_ZIP_URL "https://www.unicode.org/Public/${UCD_VERSION}/ucd/UCD.zip")
--- a/Meta/Lagom/CMakeLists.txt
+++ b/Meta/Lagom/CMakeLists.txt
@ -25,6 +25,8 @@ get_filename_component(
 )
 set(SerenityOS_SOURCE_DIR "${SERENITY_PROJECT_ROOT}" CACHE STRING "")

+set(SERENITY_CACHE_DIR "${PROJECT_BINARY_DIR}" CACHE STRING "")
+
 list(APPEND CMAKE_MODULE_PATH "${SERENITY_PROJECT_ROOT}/Meta/CMake")

 if(NOT COMMAND serenity_option)
--- a/Meta/serenity.sh
+++ b/Meta/serenity.sh
@ -238,6 +238,7 @@ cmd_with_target() {
    else
        SUPER_BUILD_DIR="$BUILD_DIR"
        CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$SERENITY_SOURCE_DIR/Build/lagom-install")
+        CMAKE_ARGS+=("-DSERENITY_CACHE_DIR=${SERENITY_SOURCE_DIR}/Build/caches")
    fi
    export PATH="$SERENITY_SOURCE_DIR/Toolchain/Local/cmake/bin":$PATH
 }