system tests: refactor registry code

The podman-login tests have accumulated much cruft over the
years, because that's the only place where we run a local
registry, and the process was crufty: we actually start/stopped
the registry as the first & last tests of the file. Meaning,
you couldn't do 'hack/bats 150:just-one-test' because that
would skip the registry start. And just now, a completely
unrelated test has had to be shoved into the login file.

This PR revamps the whole thing, by adding a new registry helper
module that can be used anywhere. And, once the registry is
started, it just stays running until the end of tests. (This
requires BATS 1.7 or greater).

Signed-off-by: Ed Santiago <santiago@redhat.com>
This commit is contained in:
Ed Santiago 2023-06-29 14:26:46 -06:00
parent 5c302db506
commit ba1355b230
5 changed files with 181 additions and 117 deletions

View file

@ -242,7 +242,12 @@ function do_stop() {
podman rm -f registry
# Use straight podman, not our alias function, to avoid 'overlay: EBUSY'
${PODMAN} unshare rm -rf ${PODMAN_REGISTRY_WORKDIR}
cmd="rm -rf ${PODMAN_REGISTRY_WORKDIR}"
if [[ $(id -u) -eq 0 ]]; then
$cmd
else
${PODMAN} unshare $cmd
fi
}

View file

@ -5,31 +5,8 @@
load helpers
load helpers.network
load helpers.registry
###############################################################################
# BEGIN one-time envariable setup
# Create a scratch directory; our podman registry will run from here. We
# also use it for other temporary files like authfiles.
if [ -z "${PODMAN_LOGIN_WORKDIR}" ]; then
export PODMAN_LOGIN_WORKDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} podman_bats_login.XXXXXX)
fi
# Randomly-generated username and password
if [ -z "${PODMAN_LOGIN_USER}" ]; then
export PODMAN_LOGIN_USER="user$(random_string 4)"
export PODMAN_LOGIN_PASS=$(random_string 15)
fi
# Randomly-assigned port in the 5xxx range
if [ -z "${PODMAN_LOGIN_REGISTRY_PORT}" ]; then
export PODMAN_LOGIN_REGISTRY_PORT=$(random_free_port)
fi
# Override any user-set path to an auth file
unset REGISTRY_AUTH_FILE
# END one-time envariable setup
###############################################################################
# BEGIN filtering - none of these tests will work with podman-remote
@ -37,73 +14,11 @@ function setup() {
skip_if_remote "none of these tests work with podman-remote"
basic_setup
start_registry
}
# END filtering - none of these tests will work with podman-remote
###############################################################################
# BEGIN first "test" - start a registry for use by other tests
#
# This isn't really a test: it's a helper that starts a local registry.
# Note that we're careful to use a root/runroot separate from our tests,
# so setup/teardown don't clobber our registry image.
#
@test "podman login [start registry]" {
AUTHDIR=${PODMAN_LOGIN_WORKDIR}/auth
mkdir -p $AUTHDIR
# Registry image; copy of docker.io, but on our own registry
local REGISTRY_IMAGE="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/registry:2.8"
# Pull registry image, but into a separate container storage
mkdir -p ${PODMAN_LOGIN_WORKDIR}/root
mkdir -p ${PODMAN_LOGIN_WORKDIR}/runroot
PODMAN_LOGIN_ARGS="--storage-driver=vfs --root ${PODMAN_LOGIN_WORKDIR}/root --runroot ${PODMAN_LOGIN_WORKDIR}/runroot"
# Give it three tries, to compensate for flakes
run_podman ${PODMAN_LOGIN_ARGS} pull $REGISTRY_IMAGE ||
run_podman ${PODMAN_LOGIN_ARGS} pull $REGISTRY_IMAGE ||
run_podman ${PODMAN_LOGIN_ARGS} pull $REGISTRY_IMAGE
# Registry image needs a cert. Self-signed is good enough.
CERT=$AUTHDIR/domain.crt
if [ ! -e $CERT ]; then
openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout $AUTHDIR/domain.key -x509 -days 2 \
-out $AUTHDIR/domain.crt \
-subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=localhost" \
-addext "subjectAltName=DNS:localhost"
fi
# Copy a cert to another directory for --cert-dir option tests
mkdir -p ${PODMAN_LOGIN_WORKDIR}/trusted-registry-cert-dir
cp $CERT ${PODMAN_LOGIN_WORKDIR}/trusted-registry-cert-dir
# Store credentials where container will see them
if [ ! -e $AUTHDIR/htpasswd ]; then
htpasswd -Bbn ${PODMAN_LOGIN_USER} ${PODMAN_LOGIN_PASS} \
> $AUTHDIR/htpasswd
# In case $PODMAN_TEST_KEEP_LOGIN_REGISTRY is set, for testing later
echo "${PODMAN_LOGIN_USER}:${PODMAN_LOGIN_PASS}" \
> $AUTHDIR/htpasswd-plaintext
fi
# Run the registry container.
run_podman '?' ${PODMAN_LOGIN_ARGS} rm -t 0 -f registry
run_podman ${PODMAN_LOGIN_ARGS} run -d \
-p ${PODMAN_LOGIN_REGISTRY_PORT}:5000 \
--name registry \
-v $AUTHDIR:/auth:Z \
-e "REGISTRY_AUTH=htpasswd" \
-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt \
-e REGISTRY_HTTP_TLS_KEY=/auth/domain.key \
$REGISTRY_IMAGE
}
# END first "test" - start a registry for use by other tests
###############################################################################
# BEGIN actual tests
# BEGIN primary podman login/push/pull tests
@ -358,33 +273,5 @@ function _test_skopeo_credential_sharing() {
# END cooperation with skopeo
# END actual tests
###############################################################################
# BEGIN teardown (remove the registry container)
@test "podman login [stop registry, clean up]" {
# For manual debugging; user may request keeping the registry running
if [ -n "${PODMAN_TEST_KEEP_LOGIN_REGISTRY}" ]; then
skip "[leaving registry running by request]"
fi
run_podman --storage-driver=vfs --root ${PODMAN_LOGIN_WORKDIR}/root \
--runroot ${PODMAN_LOGIN_WORKDIR}/runroot \
rm -f registry
run_podman --storage-driver=vfs --root ${PODMAN_LOGIN_WORKDIR}/root \
--runroot ${PODMAN_LOGIN_WORKDIR}/runroot \
rmi -a
# By default, clean up
if [ -z "${PODMAN_TEST_KEEP_LOGIN_WORKDIR}" ]; then
rm -rf ${PODMAN_LOGIN_WORKDIR}
fi
# Make sure socket is closed
if tcp_port_probe $PODMAN_LOGIN_REGISTRY_PORT; then
die "Socket still seems open"
fi
}
# END teardown (remove the registry container)
###############################################################################
# vim: filetype=sh

View file

@ -78,9 +78,36 @@ function _prefetch() {
fi
fi
# Kludge alert.
# Skopeo has no --storage-driver, --root, or --runroot flags; those
# need to be expressed in the destination string inside [brackets].
# See containers-transports(5). So if we see those options in
# _PODMAN_TEST_OPTS, transmogrify $want into skopeo form.
skopeo_opts=''
driver="$(expr "$_PODMAN_TEST_OPTS" : ".*--storage-driver \([^ ]\+\)" || true)"
if [[ -n "$driver" ]]; then
skopeo_opts+="$driver@"
fi
altroot="$(expr "$_PODMAN_TEST_OPTS" : ".*--root \([^ ]\+\)" || true)"
if [[ -n "$altroot" ]] && [[ -d "$altroot" ]]; then
skopeo_opts+="$altroot"
altrunroot="$(expr "$_PODMAN_TEST_OPTS" : ".*--runroot \([^ ]\+\)" || true)"
if [[ -n "$altrunroot" ]] && [[ -d "$altrunroot" ]]; then
skopeo_opts+="+$altrunroot"
fi
fi
if [[ -n "$skopeo_opts" ]]; then
want="[$skopeo_opts]$want"
fi
# Cached image is now guaranteed to exist. Be sure to load it
# with skopeo, not podman, in order to preserve metadata
skopeo copy --all oci-archive:$cachepath containers-storage:$want
cmd="skopeo copy --all oci-archive:$cachepath containers-storage:$want"
echo "$_LOG_PROMPT $cmd"
$cmd
}
# END tools for fetching & caching test images

View file

@ -0,0 +1,116 @@
# -*- bash -*-
#
# helpers for starting/stopping a local registry.
#
# Used primarily in 150-login.bats
#
###############################################################################
# BEGIN one-time envariable setup
# Override any user-set path to an auth file
unset REGISTRY_AUTH_FILE
# END one-time envariable setup
###############################################################################
# Start a local registry. Only needed on demand (e.g. by 150-login.bats)
# and then only once: if we start, leave it running until final teardown.
function start_registry() {
if [[ -d "$PODMAN_LOGIN_WORKDIR/auth" ]]; then
# Already started
return
fi
AUTHDIR=${PODMAN_LOGIN_WORKDIR}/auth
mkdir -p $AUTHDIR
# Registry image; copy of docker.io, but on our own registry
local REGISTRY_IMAGE="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/registry:2.8"
# Pull registry image, but into a separate container storage
mkdir ${PODMAN_LOGIN_WORKDIR}/root
mkdir ${PODMAN_LOGIN_WORKDIR}/runroot
PODMAN_LOGIN_ARGS="--storage-driver vfs --root ${PODMAN_LOGIN_WORKDIR}/root --runroot ${PODMAN_LOGIN_WORKDIR}/runroot"
# _prefetch() will retry twice on network error, and will also use
# a pre-cached image if present (helpful on dev workstation, not in CI).
_PODMAN_TEST_OPTS="${PODMAN_LOGIN_ARGS}" _prefetch $REGISTRY_IMAGE
# Registry image needs a cert. Self-signed is good enough.
CERT=$AUTHDIR/domain.crt
if [ ! -e $CERT ]; then
openssl req -newkey rsa:4096 -nodes -sha256 \
-keyout $AUTHDIR/domain.key -x509 -days 2 \
-out $AUTHDIR/domain.crt \
-subj "/C=US/ST=Foo/L=Bar/O=Red Hat, Inc./CN=localhost" \
-addext "subjectAltName=DNS:localhost"
fi
# Copy a cert to another directory for --cert-dir option tests
mkdir -p ${PODMAN_LOGIN_WORKDIR}/trusted-registry-cert-dir
cp $CERT ${PODMAN_LOGIN_WORKDIR}/trusted-registry-cert-dir
# Store credentials where container will see them
htpasswd -Bbn ${PODMAN_LOGIN_USER} ${PODMAN_LOGIN_PASS} > $AUTHDIR/htpasswd
# In case $PODMAN_TEST_KEEP_LOGIN_REGISTRY is set, for testing later
echo "${PODMAN_LOGIN_USER}:${PODMAN_LOGIN_PASS}" > $AUTHDIR/htpasswd-plaintext
# Run the registry container.
run_podman ${PODMAN_LOGIN_ARGS} run -d \
-p 127.0.0.1:${PODMAN_LOGIN_REGISTRY_PORT}:5000 \
--name registry \
-v $AUTHDIR:/auth:Z \
-e "REGISTRY_AUTH=htpasswd" \
-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-e REGISTRY_HTTP_TLS_CERTIFICATE=/auth/domain.crt \
-e REGISTRY_HTTP_TLS_KEY=/auth/domain.key \
$REGISTRY_IMAGE
cid="$output"
# wait_for_port isn't enough: that just checks that podman has mapped the port...
wait_for_port 127.0.0.1 ${PODMAN_LOGIN_REGISTRY_PORT}
# ...so we look in container logs for confirmation that registry is running.
_PODMAN_TEST_OPTS="${PODMAN_LOGIN_ARGS}" wait_for_output "listening on .::.:5000" $cid
}
function stop_registry() {
if [[ ! -d "$PODMAN_LOGIN_WORKDIR/auth" ]]; then
# No registry running
return
fi
# For manual debugging; user may request keeping the registry running
if [ -n "${PODMAN_TEST_KEEP_LOGIN_REGISTRY}" ]; then
skip "[leaving registry running by request]"
fi
run_podman --storage-driver vfs \
--root ${PODMAN_LOGIN_WORKDIR}/root \
--runroot ${PODMAN_LOGIN_WORKDIR}/runroot \
rm -f -t0 registry
run_podman --storage-driver vfs \
--root ${PODMAN_LOGIN_WORKDIR}/root \
--runroot ${PODMAN_LOGIN_WORKDIR}/runroot \
rmi -a -f
# By default, clean up
if [ -z "${PODMAN_TEST_KEEP_LOGIN_WORKDIR}" ]; then
# FIXME: why is this necessary??? If we don't do this, we can't
# rm -rf the workdir, because ..../overlay is mounted
mount | grep ${PODMAN_LOGIN_WORKDIR} | awk '{print $3}' | xargs --no-run-if-empty umount
if [[ $(id -u) -eq 0 ]]; then
rm -rf ${PODMAN_LOGIN_WORKDIR}
else
# rootless image data is owned by a subuid
run_podman unshare rm -rf ${PODMAN_LOGIN_WORKDIR}
fi
fi
# Make sure socket is closed
if tcp_port_probe $PODMAN_LOGIN_REGISTRY_PORT; then
die "Socket still seems open"
fi
}

View file

@ -0,0 +1,29 @@
# -*- bash -*-
#
# global setup/teardown for the entire system test suite
#
bats_require_minimum_version 1.8.0
load helpers
load helpers.network
load helpers.registry
# Create common environment just in case we end up needing a registry.
# These environment variables will be available to all tests.
function setup_suite() {
# Can't use $BATS_SUITE_TMPDIR because podman barfs:
# Error: the specified runroot is longer than 50 characters
export PODMAN_LOGIN_WORKDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} podman-bats-registry.XXXXXX)
export PODMAN_LOGIN_USER="user$(random_string 4)"
export PODMAN_LOGIN_PASS="pw$(random_string 15)"
# FIXME: racy! It could be many minutes between now and when we start it.
# To mitigate, we use a range not used anywhere else in system tests.
export PODMAN_LOGIN_REGISTRY_PORT=$(random_free_port 42000-42999)
}
# Run at the very end of all tests. Useful for cleanup of non-BATS tmpdirs.
function teardown_suite() {
stop_registry
}