Merge pull request #3879 from cevich/zip_to_msi

Automate production of MSI release files

.cirrus.yml

@@ -351,8 +351,14 @@ testing_task:
     integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
     system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
     build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
+    # For PRs this confirms uploading releases after merge, is functional.
     upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}'
 
+    # When examining a particular run, provide convenient access to release files.
+    tar_artifacts:
+        path: "*.tar.gz"
+        type: "application/x-tar"
+
     on_failure:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
 
@@ -480,6 +486,15 @@ special_testing_cross_task:
     on_failure:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
 
+    # When examining a particular run, provide convenient access to release files.
+    zip_artifacts:
+        path: "*.zip"
+        type: "application/zip"
+
+    msi_artifacts:
+        path: "*.msi"
+        type: "application/octet-stream"
+
 
 special_testing_cgroupv2_task:
 

contrib/cirrus/build_release.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -e
+
 source $(dirname $0)/lib.sh
 
 req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC
@@ -13,18 +15,20 @@ fi
 
 if [[ -n "$CROSS_PLATFORM" ]]
 then
+    # Will fail if $CROSS_PLATFORM is unsupported cross-compile $GOOS value
+    make podman-remote-${CROSS_PLATFORM}-release
+
     echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
-    case "$CROSS_PLATFORM" in
-        linux) ;&
-        windows) ;&
-        darwin)
-            make podman-remote-${CROSS_PLATFORM}-release
-            ;;
-        *)
-            die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'"
-            ;;
-    esac
+    if [[ "$CROSS_PLATFORM" == "windows" ]]
+    then
+        # TODO: Remove next line, part of VM images next time they're built.
+        dnf install -y libmsi1 msitools pandoc
+        make podman.msi
+    fi
 else
     echo "Compiling release archive for $OS_RELEASE_ID"
     make podman-release
 fi
+
+echo "Preserving build details for later use."
+mv -v release.txt actual_release.txt  # Another 'make' during testing could overwrite it

contrib/cirrus/packer/fedora_setup.sh

@@ -53,6 +53,7 @@ ooe.sh sudo dnf install -y \
     jq \
     libassuan-devel \
     libcap-devel \
+    libmsi1 \
     libnet \
     libnet-devel \
     libnl3-devel \
@@ -62,9 +63,11 @@ ooe.sh sudo dnf install -y \
     libvarlink-util \
     lsof \
     make \
+    msitools \
     nmap-ncat \
     ostree \
     ostree-devel \
+    pandoc \
     podman \
     procps-ng \
     protobuf \

contrib/cirrus/upload_release_archive.sh

@@ -9,6 +9,7 @@ req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNA
 [[ "$CI" == "true" ]] || \
     die 56 "$0 must be run under Cirrus-CI to function"
 
+# We store "releases" for each PR, mostly to validate the process is functional
 unset PR_OR_BRANCH BUCKET
 if [[ -n "$CIRRUS_PR" ]]
 then
@@ -22,31 +23,76 @@ else
     die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
 fi
 
-# Functional local podman required for uploading a release
+echo "Parsing actual_release.txt contents: $(< actual_release.txt)"
 cd $GOSRC
+RELEASETXT=$(<actual_release.txt)  # see build_release.sh
+[[ -n "$RELEASETXT" ]] || \
+    die 3 "Could not obtain metadata from actual_release.txt"
+RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
+if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
+then
+    die 4 "Metadata is empty or invalid: '$RELEASETXT'"
+fi
+# Format specified in Makefile
+# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
+# or   libpod-remote v1.3.1-166-g60df124e windows - amd64
+FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
+read $FIELDS <<< $RELEASE_INFO
+req_env_var $FIELDS
+
+# Functional local podman required for uploading
+echo "Verifying a local, functional podman, building one if necessary."
 [[ -n "$(type -P podman)" ]] || \
-    make install || \
+    make install PREFIX=/usr || \
     die 57 "$0 requires working podman binary on path to function"
 
 TMPF=$(mktemp -p '' $(basename $0)_XXXX.json)
 trap "rm -f $TMPF" EXIT
 set +x
 echo "$RELEASE_GCPJSON" > "$TMPF"
+[[ "$OS_RELEASE_ID" == "ubuntu" ]] || \
+    chcon -t container_file_t "$TMPF"
 unset RELEASE_GCPJSON
 
 cd $GOSRC
-for filename in $(ls -1 *.tar.gz *.zip)
+for filename in $(ls -1 *.tar.gz *.zip *.msi)
 do
-    echo "Running podman ... $UPLDREL_IMAGE $filename"
+    unset EXT
+    EXT=$(echo "$filename" | sed -r -e 's/.+\.(.+$)/\1/g')
+    if [[ -z "$EXT" ]] || [[ "$EXT" == "$filename" ]]
+    then
+        echo "Warning: Not processing $filename (invalid extension '$EXT')"
+        continue
+    fi
+
+    [[ "$OS_RELEASE_ID" == "ubuntu" ]] || \
+        chcon -t container_file_t "$filename"
+    # Form the generic "latest" file for this branch or pr
+    TO_PREFIX="${RELEASE_BASENAME}-latest-${PR_OR_BRANCH}-${RELEASE_DIST}"
+    # Form the fully-versioned filename for historical sake
+    ALSO_PREFIX="${RELEASE_BASENAME}-${RELEASE_VERSION}-${PR_OR_BRANCH}-${RELEASE_DIST}"
+    TO_SUFFIX="${RELEASE_ARCH}.${EXT}"
+    if [[ "$RELEASE_DIST" == "windows" ]] || [[ "$RELEASE_DIST" == "darwin" ]]
+    then
+        TO_FILENAME="${TO_PREFIX}-${TO_SUFFIX}"
+        ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}"
+    else
+        TO_FILENAME="${TO_PREFIX}-${RELEASE_DIST_VER}-${TO_SUFFIX}"
+        ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}"
+    fi
+
+    echo "Running podman ... $UPLDREL_IMAGE for $filename -> $TO_FILENAME"
+    echo "Warning: upload failures are completely ignored, avoiding any needless holdup of PRs."
     podman run -i --rm \
         -e "GCPNAME=$RELEASE_GCPNAME" \
         -e "GCPPROJECT=$RELEASE_GCPROJECT" \
         -e "GCPJSON_FILEPATH=$TMPF" \
-        -e "REL_ARC_FILEPATH=/tmp/$filename" \
+        -e "FROM_FILEPATH=/tmp/$filename" \
+        -e "TO_FILENAME=$TO_FILENAME" \
+        -e "ALSO_FILENAME=$ALSO_FILENAME" \
         -e "PR_OR_BRANCH=$PR_OR_BRANCH" \
         -e "BUCKET=$BUCKET" \
-        --security-opt label=disable \
         -v "$TMPF:$TMPF:ro" \
-        -v "$GOSRC/$filename:/tmp/$filename:ro" \
-        $UPLDREL_IMAGE
+        -v "$(realpath $GOSRC/$filename):/tmp/$filename:ro" \
+        $UPLDREL_IMAGE || true
 done

contrib/upldrel/entrypoint.sh

@@ -4,59 +4,22 @@ set -e
 
 source /usr/local/bin/lib_entrypoint.sh
 
-req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT REL_ARC_FILEPATH PR_OR_BRANCH BUCKET
+req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT BUCKET FROM_FILEPATH TO_FILENAME ALSO_FILENAME
 
-[[ -r "$REL_ARC_FILEPATH" ]] || \
+[[ -r "$FROM_FILEPATH" ]] || \
     die 2 ERROR Cannot read release archive file: "$REL_ARC_FILEPATH"
 
 [[ -r "$GCPJSON_FILEPATH" ]] || \
     die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH"
 
-cd $TMPDIR
-echo "Attempting to extract release.txt from tar or zip $REL_ARC_FILEPATH"
-unset SFX
-if tar xzf "$REL_ARC_FILEPATH" "./release.txt"
-then
-    echo "It's a tarball"
-    SFX="tar.gz"
-elif unzip "$REL_ARC_FILEPATH" release.txt
-then
-    echo "It's a zip"
-    SFX="zip"
-else
-    die 5 ERROR Could not extract release.txt from $REL_ARC_FILEPATH
-fi
-
-echo "Parsing release.txt contents"
-RELEASETXT=$(<release.txt)
-cd -
-[[ -n "$RELEASETXT" ]] || \
-    die 3 ERROR Could not obtain metadata from release.txt in $REL_ARC_FILEPATH
-
-RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
-if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
-then
-    die 4 ERROR Metadata is empty or invalid: '$RELEASETXT'
-fi
-
-# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
-# or   libpod v1.3.1-166-g60df124e   amd64
-FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
-read $FIELDS <<< $RELEASE_INFO
-for f in $FIELDS
-do
-    [[ -n "${!f}" ]] || \
-        die 5 ERROR Expecting $f to be non-empty in metadata: '$RELEASE_INFO'
-done
-
+echo "Authenticating to google cloud for upload"
 gcloud_init "$GCPJSON_FILEPATH"
 
-# Drop version number to enable "latest" representation
-# (version available w/in zip-file comment)
-RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.${SFX}"
+echo "Uploading archive as $TO_FILENAME"
+gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$TO_FILENAME"
+gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$ALSO_FILENAME"
 
-echo "Uploading archive as $RELEASE_ARCHIVE_NAME"
-gsutil cp "$REL_ARC_FILEPATH" "gs://$BUCKET/$RELEASE_ARCHIVE_NAME"
-
-echo "Release now available at:"
-echo "    https://storage.cloud.google.com/$BUCKET/$RELEASE_ARCHIVE_NAME"
+echo "."
+echo "Release now available for download at:"
+echo "    https://storage.cloud.google.com/$BUCKET/$TO_FILENAME"
+echo "    https://storage.cloud.google.com/$BUCKET/$ALSO_FILENAME"

hack/get_release_info.sh

@@ -33,7 +33,7 @@ case "$1" in
         OUTPUT="${GOARCH:-$(go env GOARCH 2> /dev/null)}"
         ;;
     BASENAME*)
-        OUTPUT="${CIRRUS_REPO_NAME:-$(basename $(git rev-parse --show-toplevel))}"
+        OUTPUT="podman"
         ;;
     REMOTENAME*)
         OUTPUT="$($0 BASENAME)-remote"