mirror of
https://github.com/containers/podman
synced 2024-10-20 17:23:30 +00:00
Migrate container images to automation_images
Signed-off-by: Chris Evich <cevich@redhat.com>
This commit is contained in:
Chris Evich
parent
5d22eb02f9
commit
7c9c159e93
21
.cirrus.yml
21
.cirrus.yml
|
|
@ -34,16 +34,16 @@ env:
|
|||
####
|
||||
#### Cache-image names to test with (double-quotes around names are critical)
|
||||
###
|
||||
FEDORA_NAME: "fedora"
|
||||
PRIOR_FEDORA_NAME: "prior-fedora"
|
||||
UBUNTU_NAME: "ubuntu"
|
||||
PRIOR_UBUNTU_NAME: "prior-ubuntu"
|
||||
FEDORA_NAME: "fedora-32"
|
||||
PRIOR_FEDORA_NAME: "fedora-31"
|
||||
UBUNTU_NAME: "ubuntu-20"
|
||||
PRIOR_UBUNTU_NAME: "ubuntu-19"
|
||||
|
||||
_BUILT_IMAGE_SUFFIX: "c6110627968057344"
|
||||
FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
PRIOR_UBUNTU_CACHE_IMAGE_NAME: "${PRIOR_UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
|
||||
_BUILT_IMAGE_SUFFIX: "c4948709391728640"
|
||||
FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}"
|
||||
PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}"
|
||||
UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}"
|
||||
PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${_BUILT_IMAGE_SUFFIX}"
|
||||
|
||||
####
|
||||
#### Default to NOT operating in any special-case testing mode
|
||||
|
|
@ -290,7 +290,7 @@ build_without_cgo_task:
|
|||
meta_task:
|
||||
|
||||
container:
|
||||
image: "quay.io/libpod/imgts:master" # see contrib/imgts
|
||||
image: "quay.io/libpod/imgts:${_BUILT_IMAGE_SUFFIX}"
|
||||
cpu: 1
|
||||
memory: 1
|
||||
|
||||
|
|
@ -301,7 +301,6 @@ meta_task:
|
|||
${PRIOR_FEDORA_CACHE_IMAGE_NAME}
|
||||
${UBUNTU_CACHE_IMAGE_NAME}
|
||||
${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
|
||||
${IMAGE_BUILDER_CACHE_IMAGE_NAME}
|
||||
BUILDID: "${CIRRUS_BUILD_ID}"
|
||||
REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
|
||||
GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4]
|
||||
|
|
|
|||
|
|
@ -1,7 +0,0 @@
|
|||
FROM quay.io/libpod/imgts:latest
|
||||
|
||||
RUN yum -y update && \
|
||||
yum clean all
|
||||
|
||||
COPY /contrib/imgprune/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod 755 /usr/local/bin/entrypoint.sh
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
|
||||
|
||||
A container image for maintaining the collection of
|
||||
VM images used by CI/CD on this project and several others.
|
||||
Acts upon metadata maintained by the imgts container.
|
||||
|
||||
Example build (from repository root):
|
||||
|
||||
```bash
|
||||
sudo podman build -t $IMAGE_NAME -f contrib/imgprune/Dockerfile .
|
||||
```
|
||||
|
|
@ -1,106 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
source /usr/local/bin/lib_entrypoint.sh
|
||||
|
||||
req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES
|
||||
|
||||
unset BASE_IMAGES
|
||||
# When executing under Cirrus-CI, script have access to current source
|
||||
LIB="$CIRRUS_WORKING_DIR/$SCRIPT_BASE/lib.sh"
|
||||
if [[ "$CI" == "true" ]] && [[ -r "$LIB" ]]
|
||||
then
|
||||
# Avoid importing anything that might conflict
|
||||
for env in $(sed -ne 's/^[^#]\+_BASE_IMAGE=/img=/p' "$LIB")
|
||||
do
|
||||
eval $env
|
||||
BASE_IMAGES="$BASE_IMAGES $img"
|
||||
done
|
||||
else
|
||||
# metadata labeling may have broken for some reason in the future
|
||||
echo "Warning: Running outside of Cirrus-CI, very minor-risk of base-image deletion."
|
||||
fi
|
||||
|
||||
gcloud_init
|
||||
|
||||
# For safety's sake + limit nr background processes
|
||||
PRUNE_LIMIT=5
|
||||
THEFUTURE=$(date --date='+1 hour' +%s)
|
||||
TOO_OLD='30 days ago'
|
||||
THRESHOLD=$(date --date="$TOO_OLD" +%s)
|
||||
# Format Ref: https://cloud.google.com/sdk/gcloud/reference/topic/formats
|
||||
FORMAT='value[quote](name,selfLink,creationTimestamp,labels)'
|
||||
PROJRE="/v1/projects/$GCPPROJECT/global/"
|
||||
RECENTLY=$(date --date='3 days ago' --iso-8601=date)
|
||||
# Filter Ref: https://cloud.google.com/sdk/gcloud/reference/topic/filters
|
||||
FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($IMGNAMES $BASE_IMAGES)"
|
||||
TODELETE=$(mktemp -p '' todelete.XXXXXX)
|
||||
IMGCOUNT=$(mktemp -p '' imgcount.XXXXXX)
|
||||
|
||||
# Search-loop runs in a sub-process, must store count in file
|
||||
echo "0" > "$IMGCOUNT"
|
||||
count_image() {
|
||||
local count
|
||||
count=$(<"$IMGCOUNT")
|
||||
let 'count+=1'
|
||||
echo "$count" > "$IMGCOUNT"
|
||||
}
|
||||
|
||||
echo "Using filter: $FILTER"
|
||||
echo "Searching images for pruning candidates older than $TOO_OLD ($(date --date="$TOO_OLD" --iso-8601=date)):"
|
||||
$GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \
|
||||
while read name selfLink creationTimestamp labels
|
||||
do
|
||||
count_image
|
||||
created_ymd=$(date --date=$creationTimestamp --iso-8601=date)
|
||||
last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true)
|
||||
markmsgpfx="Marking $name (created $created_ymd) for deletion"
|
||||
if [[ -z "$last_used" ]]
|
||||
then # image pre-dates addition of tracking labels
|
||||
echo "$markmsgpfx: Missing 'last-used' metadata, labels: '$labels'"
|
||||
echo "$name" >> $TODELETE
|
||||
continue
|
||||
fi
|
||||
|
||||
last_used_timestamp=$(date --date=@$(cut -d= -f2 <<< $last_used || true) +%s || true)
|
||||
last_used_ymd=$(date --date=@$last_used_timestamp --iso-8601=date)
|
||||
if [[ -z "$last_used_timestamp" ]] || [[ "$last_used_timestamp" -ge "$THEFUTURE" ]]
|
||||
then
|
||||
echo "$markmsgpfx: Missing or invalid last-used timestamp: '$last_used_timestamp'"
|
||||
echo "$name" >> $TODELETE
|
||||
continue
|
||||
fi
|
||||
|
||||
if [[ "$last_used_timestamp" -le "$THRESHOLD" ]]
|
||||
then
|
||||
echo "$markmsgpfx: Used over $TOO_OLD on $last_used_ymd"
|
||||
echo "$name" >> $TODELETE
|
||||
continue
|
||||
fi
|
||||
done
|
||||
|
||||
COUNT=$(<"$IMGCOUNT")
|
||||
echo "########################################################################"
|
||||
echo "Deleting up to $PRUNE_LIMIT images marked ($(wc -l < $TODELETE)) of all searched ($COUNT):"
|
||||
|
||||
# Require a minimum number of images to exist
|
||||
NEED="$[$PRUNE_LIMIT*2]"
|
||||
if [[ "$COUNT" -lt "$NEED" ]]
|
||||
then
|
||||
die 0 Safety-net Insufficient images \($COUNT\) to process deletions \($NEED\)
|
||||
exit 0
|
||||
fi
|
||||
|
||||
for image_name in $(sort --random-sort $TODELETE | tail -$PRUNE_LIMIT)
|
||||
do
|
||||
if echo "$IMGNAMES $BASE_IMAGES" | grep -q "$image_name"
|
||||
then
|
||||
# double-verify in-use images were filtered out in search loop above
|
||||
die 8 FATAL ATTEMPT TO DELETE IN-USE IMAGE \'$image_name\' - THIS SHOULD NEVER HAPPEN
|
||||
fi
|
||||
echo "Deleting $image_name in parallel..."
|
||||
$GCLOUD compute images delete $image_name &
|
||||
done
|
||||
|
||||
wait || true # Nothing to delete: No background jobs
|
||||
|
|
@ -1,20 +0,0 @@
|
|||
FROM centos:7
|
||||
|
||||
# Only needed for installing build-time dependencies
|
||||
COPY /contrib/imgts/google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo
|
||||
RUN yum -y update && \
|
||||
yum -y install epel-release && \
|
||||
yum -y install google-cloud-sdk && \
|
||||
yum clean all
|
||||
|
||||
ENV GCPJSON="__unknown__" \
|
||||
GCPNAME="__unknown__" \
|
||||
GCPPROJECT="__unknown__" \
|
||||
IMGNAMES="__unknown__" \
|
||||
BUILDID="__unknown__" \
|
||||
REPOREF="__unknown__"
|
||||
|
||||
COPY ["/contrib/imgts/entrypoint.sh", "/contrib/imgts/lib_entrypoint.sh", "/usr/local/bin/"]
|
||||
RUN chmod 755 /usr/local/bin/entrypoint.sh
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
|
||||
|
||||
A container image for tracking automation metadata.
|
||||
Currently this is used to update last-used timestamps on
|
||||
VM images.
|
||||
|
||||
Example build (from repository root):
|
||||
|
||||
```bash
|
||||
sudo podman build -t $IMAGE_NAME -f contrib/imgts/Dockerfile .
|
||||
```
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
source /usr/local/bin/lib_entrypoint.sh
|
||||
|
||||
req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES BUILDID REPOREF
|
||||
|
||||
gcloud_init
|
||||
|
||||
ARGS="
|
||||
--update-labels=last-used=$(date +%s)
|
||||
--update-labels=build-id=$BUILDID
|
||||
--update-labels=repo-ref=$REPOREF
|
||||
--update-labels=project=$GCPPROJECT
|
||||
"
|
||||
|
||||
for image in $IMGNAMES
|
||||
do
|
||||
$GCLOUD compute images update "$image" $ARGS &
|
||||
done
|
||||
|
||||
wait || echo "Warning: No \$IMGNAMES were specified."
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
[google-cloud-sdk]
|
||||
name=Google Cloud SDK
|
||||
baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64
|
||||
enabled=1
|
||||
gpgcheck=1
|
||||
repo_gpgcheck=1
|
||||
gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
|
||||
https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
|
||||
|
|
@ -1,49 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
RED="\e[1;36;41m"
|
||||
YEL="\e[1;33;44m"
|
||||
NOR="\e[0m"
|
||||
SENTINEL="__unknown__" # default set in dockerfile
|
||||
# Disable all input prompts
|
||||
# https://cloud.google.com/sdk/docs/scripting-gcloud
|
||||
GCLOUD="gcloud --quiet"
|
||||
|
||||
die() {
|
||||
EXIT=$1
|
||||
PFX=$2
|
||||
shift 2
|
||||
MSG="$@"
|
||||
echo -e "${RED}${PFX}:${NOR} ${YEL}$MSG${NOR}"
|
||||
[[ "$EXIT" -eq "0" ]] || exit "$EXIT"
|
||||
}
|
||||
|
||||
# Pass in a list of one or more envariable names; exit non-zero with
|
||||
# helpful error message if any value is empty
|
||||
req_env_var() {
|
||||
for i; do
|
||||
if [[ -z "${!i}" ]]
|
||||
then
|
||||
die 1 FATAL entrypoint.sh requires \$$i to be non-empty.
|
||||
elif [[ "${!i}" == "$SENTINEL" ]]
|
||||
then
|
||||
die 2 FATAL entrypoint.sh requires \$$i to be explicitly set.
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
gcloud_init() {
|
||||
set +xe
|
||||
if [[ -n "$1" ]] && [[ -r "$1" ]]
|
||||
then
|
||||
TMPF="$1"
|
||||
else
|
||||
TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json)
|
||||
trap "rm -f $TMPF &> /dev/null" EXIT
|
||||
echo "$GCPJSON" > $TMPF
|
||||
fi
|
||||
$GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \
|
||||
die 5 FATAL auth
|
||||
rm -f $TMPF &> /dev/null || true # ignore any read-only error
|
||||
}
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
FROM quay.io/libpod/imgts:latest
|
||||
|
||||
RUN yum -y update && \
|
||||
yum -y install unzip && \
|
||||
rpm -V unzip && \
|
||||
yum clean all
|
||||
|
||||
COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod 755 /usr/local/bin/entrypoint.sh
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
|
||||
|
||||
A container image for canonical-naming and uploading of
|
||||
libpod and remote-client archives. Only intended to ever
|
||||
be used by CI/CD, and depends heavily on an embedded
|
||||
`release.txt` file produced by `make`.
|
||||
|
||||
Build script: [../cirrus/build_release.sh](../cirrus/build_release.sh)
|
||||
Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh)
|
||||
|
|
@ -1,27 +0,0 @@
|
|||
#!/usr/bin/env bash
|
||||
|
||||
set -e
|
||||
|
||||
source /usr/local/bin/lib_entrypoint.sh
|
||||
|
||||
req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT BUCKET FROM_FILEPATH TO_FILENAME
|
||||
|
||||
[[ -r "$FROM_FILEPATH" ]] || \
|
||||
die 2 ERROR Cannot read release archive file: "$FROM_FILEPATH"
|
||||
|
||||
[[ -r "$GCPJSON_FILEPATH" ]] || \
|
||||
die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH"
|
||||
|
||||
echo "Authenticating to google cloud for upload"
|
||||
gcloud_init "$GCPJSON_FILEPATH"
|
||||
|
||||
echo "Uploading archive as $TO_FILENAME"
|
||||
gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$TO_FILENAME"
|
||||
[[ -z "$ALSO_FILENAME" ]] || \
|
||||
gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$ALSO_FILENAME"
|
||||
|
||||
echo "."
|
||||
echo "Release now available for download at:"
|
||||
echo " https://storage.googleapis.com/$BUCKET/$TO_FILENAME"
|
||||
[[ -z "$ALSO_FILENAME" ]] || \
|
||||
echo " https://storage.googleapis.com/$BUCKET/$ALSO_FILENAME"
|
||||
Loading…
Reference in a new issue