Auto merge of #112407 - tgross35:ci-docs-publish, r=Mark-Simulacrum

Publish docs as github artifacts during CI Discussed here: https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/Building.20docs.20for.20PR.20CI The goal is to make docs available for download after CI runs on PRs, for easy review of API changes. Notes: - Currently this only captures library documentation (`core`, `alloc`, `std`, `test`, `proc_macro`) - You can't see artifacts until the entire workflow run has completed https://github.com/actions/upload-artifact/issues/53 - There is currently a generic file name `ci-artifacts`. No way to customize this based on contained files unfortunately https://github.com/actions/upload-artifact/issues/349 You can find the results at the bottom of the CI "summary" page: <img width="379" alt="image" src="https://github.com/rust-lang/rust/assets/13724985/d3748e59-242c-40f8-9f54-82177b9b481b">
2024-09-15 22:50:55 +00:00 · 2023-06-17 03:28:53 +00:00 · 2023-06-17 03:28:53 +00:00 · 1d7d824726
parent 0cc541e4b2 696b0dd472
commit 1d7d824726
4 changed files with 97 additions and 0 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@ -38,6 +38,7 @@ jobs:
    env:
      CI_JOB_NAME: "${{ matrix.name }}"
      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
      SCCACHE_BUCKET: rust-lang-ci-sccache2
      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
      CACHE_DOMAIN: ci-caches.rust-lang.org
@ -143,6 +144,17 @@ jobs:
          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
        if: success() && !env.SKIP_JOB
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        if: success() && !env.SKIP_JOB
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        if: success() && !env.SKIP_JOB
      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
@ -156,6 +168,7 @@ jobs:
    env:
      CI_JOB_NAME: "${{ matrix.name }}"
      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
      SCCACHE_BUCKET: rust-lang-ci-sccache2
      DEPLOY_BUCKET: rust-lang-ci2
      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@ -557,6 +570,17 @@ jobs:
          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
        if: success() && !env.SKIP_JOB
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        if: success() && !env.SKIP_JOB
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        if: success() && !env.SKIP_JOB
      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
@ -571,6 +595,7 @@ jobs:
      DIST_TRY_BUILD: 1
      CI_JOB_NAME: "${{ matrix.name }}"
      CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+      HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
      SCCACHE_BUCKET: rust-lang-ci-sccache2
      DEPLOY_BUCKET: rust-lang-ci2
      TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
@ -672,6 +697,17 @@ jobs:
          AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
          TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
        if: success() && !env.SKIP_JOB
+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        if: success() && !env.SKIP_JOB
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          name: "${{ env.DOC_ARTIFACT_NAME }}"
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        if: success() && !env.SKIP_JOB
      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
--- a/src/ci/docker/host-x86_64/mingw-check/Dockerfile
+++ b/src/ci/docker/host-x86_64/mingw-check/Dockerfile
@ -45,6 +45,9 @@ ENV SCRIPT python3 ../x.py --stage 2 test src/tools/expand-yaml-anchors && \
           python3 ../x.py test --stage 0 src/tools/compiletest && \
           python3 ../x.py test --stage 0 core alloc std test proc_macro && \
           # Build both public and internal documentation.
+           RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library && \
+           mkdir -p /checkout/obj/staging/doc && \
+           cp -r build/x86_64-unknown-linux-gnu/doc /checkout/obj/staging && \
           RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 compiler && \
           RUSTDOCFLAGS=\"--document-private-items --document-hidden-items\" python3 ../x.py doc --stage 0 library/test && \
           /scripts/validate-toolstate.sh && \
--- a/src/ci/github-actions/ci.yml
+++ b/src/ci/github-actions/ci.yml
@ -34,6 +34,8 @@ x--expand-yaml-anchors--remove:
  - &shared-ci-variables
    CI_JOB_NAME: ${{ matrix.name }}
    CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
+    # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
+    HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}

  - &public-variables
    SCCACHE_BUCKET: rust-lang-ci-sccache2
@ -229,6 +231,20 @@ x--expand-yaml-anchors--remove:
          TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}
        <<: *step

+      - name: create github artifacts
+        run: src/ci/scripts/create-doc-artifacts.sh
+        <<: *step
+
+      - name: upload artifacts to github
+        uses: actions/upload-artifact@v3
+        with:
+          # name is set in previous step
+          name: ${{ env.DOC_ARTIFACT_NAME }}
+          path: obj/artifacts/doc
+          if-no-files-found: ignore
+          retention-days: 5
+        <<: *step
+
      - name: upload artifacts to S3
        run: src/ci/scripts/upload-artifacts.sh
        env:
--- a/src/ci/scripts/create-doc-artifacts.sh
+++ b/src/ci/scripts/create-doc-artifacts.sh
@ -0,0 +1,42 @@
+#!/bin/bash
+# Compress doc artifacts and name them based on the commit, or the date if
+# commit is not available.
+
+set -euox pipefail
+
+# Try to get short commit hash, fallback to date
+if [ -n "$HEAD_SHA" ]; then
+    short_rev=$(echo "${HEAD_SHA}" | cut -c1-8)
+else
+    short_rev=$(git rev-parse --short HEAD || date -u +'%Y-%m-%dT%H%M%SZ')
+fi
+
+# Try to get branch, fallback to none
+branch=$(git branch --show-current || echo)
+
+if [ -n "$branch" ]; then
+    branch="${branch}-"
+fi
+
+if [ "${GITHUB_EVENT_NAME:=none}" = "pull_request" ]; then
+    pr_num=$(echo "$GITHUB_REF_NAME" | cut -d'/' -f1)
+    name="doc-${pr_num}-${short_rev}"
+else
+    name="doc-${branch}${short_rev}"
+fi
+
+
+if [ -d "obj/staging/doc" ]; then
+    mkdir -p obj/artifacts/doc
+
+    # Level 12 seems to give a good tradeoff of time vs. space savings
+    ZSTD_CLEVEL=12 ZSTD_NBTHREADS=4 \
+    tar --zstd -cf "obj/artifacts/doc/${name}.tar.zst" -C obj/staging/doc .
+
+    ls -lh obj/artifacts/doc
+fi
+
+# Set this environment variable for future use if running in CI
+if [ -n "$GITHUB_ENV" ]; then
+    echo "DOC_ARTIFACT_NAME=${name}" >> "$GITHUB_ENV"
+fi