mirror of
https://github.com/containers/podman
synced 2024-10-19 08:44:11 +00:00
System tests: tighten 'is' operator
Fix day-one sloppiness: when I first wrote this framework it compared strings using 'expr', not '=', to be more forgiving of extra cruft in output. This was a bad decision. It means that warnings or additional text are ignored: is "all is ok, NOT!" "all is ok" <-- this would pass Solution: tighten up the 'is' check. Use '=' (direct compare) first. If it fails, look for wild cards ('*') or character classes ('[') in the expect string. If so, and only then, use 'expr'. And, thanks to a clever suggestion from Luap99, include '(using expr)' in the error message when we do so; this could make it easier for a developer to understand a string mismatch. This change exposes a lot of instances in which we weren't doing proper comparisons. Fix those. Thankfully, there weren't as many as I'd feared. Also, and completely unrelated, add '-T' flag to bats helper, for showing timing results. (I will open this as a separate PR if requested. I too find it offensive to jumble together unrelated commits.) Signed-off-by: Ed Santiago <santiago@redhat.com>
This commit is contained in:
parent
ba72b17d28
commit
bf94ebf423
|
@ -19,6 +19,8 @@ $0 is a wrapper for invoking podman system tests.
|
|||
version of bats installed, runs with '--filter pattern'
|
||||
which runs only subtests that match 'pattern'
|
||||
|
||||
-T Passed on to bats, which will then show timing data
|
||||
|
||||
--help display usage message
|
||||
|
||||
By default, tests ./bin/podman. To test a different podman, do:
|
||||
|
@ -60,6 +62,8 @@ REMOTE=
|
|||
ROOT_ONLY=
|
||||
ROOTLESS_ONLY=
|
||||
|
||||
declare -a bats_opts=()
|
||||
|
||||
declare -a bats_filter=()
|
||||
|
||||
for i;do
|
||||
|
@ -69,6 +73,7 @@ for i;do
|
|||
--root) ROOT_ONLY=1 ;;
|
||||
--rootless) ROOTLESS_ONLY=1 ;;
|
||||
--remote) REMOTE=remote; echo "--remote is TBI"; exit 1;;
|
||||
--ts|-T) bats_opts+=("-T") ;;
|
||||
*/*.bats) TESTS=$i ;;
|
||||
*)
|
||||
if [[ $i =~ : ]]; then
|
||||
|
@ -94,7 +99,7 @@ if [ -z "$ROOTLESS_ONLY" ]; then
|
|||
sudo --preserve-env=PODMAN \
|
||||
--preserve-env=PODMAN_TEST_DEBUG \
|
||||
--preserve-env=OCI_RUNTIME \
|
||||
bats "${bats_filter[@]}" $TESTS
|
||||
bats "${bats_opts[@]}" "${bats_filter[@]}" $TESTS
|
||||
rc=$?
|
||||
fi
|
||||
|
||||
|
@ -102,7 +107,7 @@ fi
|
|||
echo "--------------------------------------------------"
|
||||
if [ -z "$ROOT_ONLY" ]; then
|
||||
echo "\$ bats ${bats_filter[@]} $TESTS"
|
||||
bats "${bats_filter[@]}" $TESTS
|
||||
bats "${bats_opts[@]}" "${bats_filter[@]}" $TESTS
|
||||
rc=$((rc | $?))
|
||||
fi
|
||||
|
||||
|
|
|
@ -101,7 +101,9 @@ function setup() {
|
|||
fi
|
||||
|
||||
run_podman 125 --remote
|
||||
is "$output" "Error: missing command 'podman COMMAND'" "podman remote show usage message without running endpoint"
|
||||
is "$output" "Error: missing command 'podman COMMAND'
|
||||
Try 'podman --help' for more information." \
|
||||
"podman --remote show usage message without running endpoint"
|
||||
}
|
||||
|
||||
# This is for development only; it's intended to make sure our timeout
|
||||
|
|
|
@ -37,7 +37,7 @@ cgroupVersion: v[12]
|
|||
# FIXME: if we're ever able to get package versions on Debian,
|
||||
# add '-[0-9]' to all '*.package' queries below.
|
||||
tests="
|
||||
host.buildahVersion | [0-9.]
|
||||
host.buildahVersion | [1-9][0-9]*\.[0-9.]\\\+.*
|
||||
host.conmon.path | $expr_path
|
||||
host.conmon.package | .*conmon.*
|
||||
host.cgroupManager | \\\(systemd\\\|cgroupfs\\\)
|
||||
|
|
|
@ -19,8 +19,8 @@ load helpers
|
|||
|
||||
@test "podman images - custom formats" {
|
||||
tests="
|
||||
{{.ID}} | [0-9a-f]\\\{12\\\}
|
||||
{{.ID| upper}} | [0-9A-F]\\\{12\\\}
|
||||
{{.ID}} | [0-9a-f]\\\{12\\\}\\\$
|
||||
{{.ID| upper}} | [0-9A-F]\\\{12\\\}\\\$
|
||||
{{.Repository}}:{{.Tag}} | $PODMAN_TEST_IMAGE_FQN
|
||||
{{.Labels.created_by}} | test/system/build-testimage
|
||||
{{.Labels.created_at}} | 20[0-9-]\\\+T[0-9:]\\\+Z
|
||||
|
@ -28,7 +28,7 @@ load helpers
|
|||
|
||||
parse_table "$tests" | while read fmt expect; do
|
||||
run_podman images --format "$fmt"
|
||||
is "$output" "$expect\$" "podman images $fmt"
|
||||
is "$output" "$expect" "podman images --format '$fmt'"
|
||||
done
|
||||
|
||||
run_podman images --format "{{.ID}}" --no-trunc
|
||||
|
|
|
@ -517,7 +517,8 @@ json-file | f
|
|||
cid="$output"
|
||||
|
||||
run_podman inspect --format "{{.ImageName}}" $cid
|
||||
is "$output" "$newtag" "container .ImageName is the container-create name"
|
||||
is "$output" "$newtag:latest" \
|
||||
"container .ImageName is the container-create name"
|
||||
|
||||
# Same thing, but now with a :tag, and making sure it works with --name
|
||||
newtag2="${newtag}:$(random_string 6|tr A-Z a-z)"
|
||||
|
@ -526,7 +527,8 @@ json-file | f
|
|||
cname="$(random_string 14|tr A-Z a-z)"
|
||||
run_podman create --name $cname $newtag2
|
||||
run_podman inspect --format "{{.ImageName}}" $cname
|
||||
is "$output" "$newtag2" "container .ImageName is the container-create name"
|
||||
is "$output" "$newtag2" \
|
||||
"container .ImageName is the container-create name, with :tag"
|
||||
|
||||
# Clean up.
|
||||
run_podman rm $cid $cname
|
||||
|
@ -718,7 +720,7 @@ EOF
|
|||
run_podman 125 run --device-cgroup-rule="b 7:2" --rm $IMAGE
|
||||
is "$output" 'Error: invalid device cgroup rule requires type, major:Minor, and access rules: "b 7:2"'
|
||||
run_podman 125 run --device-cgroup-rule="x 7:* rmw" --rm $IMAGE
|
||||
is "$output" "Error: invalid device type in device-access-add:"
|
||||
is "$output" "Error: invalid device type in device-access-add: x"
|
||||
run_podman 125 run --device-cgroup-rule="a a:* rmw" --rm $IMAGE
|
||||
is "$output" "Error: strconv.ParseInt: parsing \"a\": invalid syntax"
|
||||
}
|
||||
|
|
|
@ -77,7 +77,7 @@ load helpers
|
|||
is "${lines[1]}" "3000+0 records out" "dd: number of records out"
|
||||
# Verify sha. '% *' strips off the path, keeping only the SHA
|
||||
run_podman exec $cid sha512sum /tmp/bigfile
|
||||
is "${output% *}" "$expect" "SHA of file in container"
|
||||
is "${output% *}" "$expect " "SHA of file in container"
|
||||
|
||||
# Clean up
|
||||
run_podman exec $cid touch /stop
|
||||
|
|
|
@ -11,7 +11,7 @@ load helpers
|
|||
# initialize, read image ID and name
|
||||
get_iid_and_name() {
|
||||
run_podman images -a --format '{{.ID}} {{.Repository}}:{{.Tag}}'
|
||||
read iid img_name < <(echo "$output")
|
||||
read iid img_name <<<"$output"
|
||||
|
||||
archive=$PODMAN_TMPDIR/myimage-$(random_string 8).tar
|
||||
}
|
||||
|
@ -62,7 +62,7 @@ verify_iid_and_name() {
|
|||
|
||||
# FIXME: cannot compare IID, see #7371, so we check only the tag
|
||||
run_podman images $fqin --format '{{.Repository}}:{{.Tag}}'
|
||||
is "$output" "$fqin" "image preserves name across save/load"
|
||||
is "${lines[0]}" "$fqin" "image preserves name across save/load"
|
||||
|
||||
# Load with a new tag
|
||||
local new_name=x1$(random_string 14 | tr A-Z a-z)
|
||||
|
|
|
@ -20,26 +20,26 @@ load helpers
|
|||
# Simple import
|
||||
run_podman import -q $archive
|
||||
iid="$output"
|
||||
run_podman run -t --rm $iid cat /random.txt
|
||||
run_podman run --rm $iid cat /random.txt
|
||||
is "$output" "$random_content" "simple import"
|
||||
run_podman rmi -f $iid
|
||||
|
||||
# Simple import via stdin
|
||||
run_podman import -q - < <(cat $archive)
|
||||
iid="$output"
|
||||
run_podman run -t --rm $iid cat /random.txt
|
||||
run_podman run --rm $iid cat /random.txt
|
||||
is "$output" "$random_content" "simple import via stdin"
|
||||
run_podman rmi -f $iid
|
||||
|
||||
# Tagged import
|
||||
run_podman import -q $archive $fqin
|
||||
run_podman run -t --rm $fqin cat /random.txt
|
||||
run_podman run --rm $fqin cat /random.txt
|
||||
is "$output" "$random_content" "tagged import"
|
||||
run_podman rmi -f $fqin
|
||||
|
||||
# Tagged import via stdin
|
||||
run_podman import -q - $fqin < <(cat $archive)
|
||||
run_podman run -t --rm $fqin cat /random.txt
|
||||
run_podman run --rm $fqin cat /random.txt
|
||||
is "$output" "$random_content" "tagged import via stdin"
|
||||
run_podman rmi -f $fqin
|
||||
}
|
||||
|
@ -100,7 +100,7 @@ EOF
|
|||
|
||||
# Confirm exit within timeout
|
||||
run_podman ps -a --filter name=$a_cnt --format '{{.Status}}'
|
||||
is "$output" "Exited (33)" "Exit by non-TERM/KILL"
|
||||
is "$output" "Exited (33) .*" "Exit by non-TERM/KILL"
|
||||
|
||||
run_podman rm -f $a_cnt
|
||||
run_podman rmi $b_img $a_img
|
||||
|
|
|
@ -245,7 +245,7 @@ function _test_skopeo_credential_sharing() {
|
|||
is "$status" "0" "skopeo inspect - exit status"
|
||||
|
||||
got_name=$(jq -r .Name <<<"$output")
|
||||
is "$got_name" "$registry/$dest_name" "skopeo inspect -> Name"
|
||||
is "$got_name" "$registry/$destname" "skopeo inspect -> Name"
|
||||
|
||||
# Now try without a valid login; it should fail
|
||||
run_podman logout "$@" $registry
|
||||
|
|
|
@ -261,7 +261,8 @@ EOF
|
|||
|
||||
# prune should remove v4
|
||||
run_podman volume prune --force
|
||||
is "$output" "${v[4]}" "volume prune, with 1, 2, 3 in use, deletes only 4"
|
||||
is "$(echo $(sort <<<$output))" "${v[4]} ${v[5]} ${v[6]}" \
|
||||
"volume prune, with 1, 2, 3 in use, deletes only 4, 5, 6"
|
||||
|
||||
# Remove the container using v2 and v3. Prune should now remove those.
|
||||
# The 'echo sort' is to get the output sorted and in one line.
|
||||
|
|
|
@ -74,7 +74,7 @@ function _corrupt_image_test() {
|
|||
# Corruptify, and confirm that 'podman images' throws an error
|
||||
rm -v ${PODMAN_CORRUPT_TEST_WORKDIR}/root/*-images/$id/${rm_path}
|
||||
run_podman 125 images
|
||||
is "$output" "Error: error retrieving label for image \"$id\": you may need to remove the image to resolve the error"
|
||||
is "$output" "Error: error retrieving label for image \"$id\": you may need to remove the image to resolve the error.*"
|
||||
|
||||
# Run the requested command. Confirm it succeeds, with suitable warnings
|
||||
run_podman $*
|
||||
|
|
|
@ -29,7 +29,7 @@ function check_label() {
|
|||
if [ -n "$2" ]; then
|
||||
# e.g. from the above example -> "s0:c45,c745"
|
||||
range=$(cut -d: -f4,5 <<<"$context")
|
||||
is "$range" "$2" "SELinux range"
|
||||
is "$range" "$2^@" "SELinux range"
|
||||
fi
|
||||
}
|
||||
|
||||
|
@ -101,7 +101,7 @@ function check_label() {
|
|||
--security-opt label=level:s0 \
|
||||
$IMAGE sh -c 'while test ! -e /stop; do sleep 0.1; done'
|
||||
run_podman inspect --format='{{ .HostConfig.SecurityOpt }}' myc
|
||||
is "$output" "\[label=type:spc_t,label=level:s0 seccomp=unconfined]" \
|
||||
is "$output" "[label=type:spc_t,label=level:s0 seccomp=unconfined]" \
|
||||
"'podman inspect' preserves all --security-opts"
|
||||
|
||||
run_podman exec myc touch /stop
|
||||
|
@ -117,6 +117,10 @@ function check_label() {
|
|||
# must use a pid namespace and not join an existing one.
|
||||
skip_if_rootless_cgroupsv1
|
||||
|
||||
if [[ $(podman_runtime) == "runc" ]]; then
|
||||
skip "some sort of runc bug, not worth fixing (#11784)"
|
||||
fi
|
||||
|
||||
run_podman run -d --name myctr $IMAGE top
|
||||
run_podman exec myctr cat -v /proc/self/attr/current
|
||||
context_c1="$output"
|
||||
|
@ -225,24 +229,25 @@ function check_label() {
|
|||
|
||||
run_podman run -v $tmpdir:/test $IMAGE cat /proc/self/attr/current
|
||||
run ls -dZ ${tmpdir}
|
||||
is "$output" ${LABEL} "No Relabel Correctly"
|
||||
is "$output" "${LABEL} ${tmpdir}" "No Relabel Correctly"
|
||||
|
||||
run_podman run -v $tmpdir:/test:z --security-opt label=disable $IMAGE cat /proc/self/attr/current
|
||||
run ls -dZ $tmpdir
|
||||
is "$output" ${RELABEL} "Privileged Relabel Correctly"
|
||||
is "$output" "${RELABEL} $tmpdir" "Privileged Relabel Correctly"
|
||||
|
||||
run_podman run -v $tmpdir:/test:z --privileged $IMAGE cat /proc/self/attr/current
|
||||
run ls -dZ $tmpdir
|
||||
is "$output" ${RELABEL} "Privileged Relabel Correctly"
|
||||
is "$output" "${RELABEL} $tmpdir" "Privileged Relabel Correctly"
|
||||
|
||||
run_podman run -v $tmpdir:/test:Z $IMAGE cat /proc/self/attr/current
|
||||
level=$(secon -l $output)
|
||||
run ls -dZ $tmpdir
|
||||
is "$output" "system_u:object_r:container_file_t:$level" "Confined Relabel Correctly"
|
||||
is "$output" "system_u:object_r:container_file_t:$level $tmpdir" \
|
||||
"Confined Relabel Correctly"
|
||||
|
||||
run_podman run -v $tmpdir:/test:z $IMAGE cat /proc/self/attr/current
|
||||
run ls -dZ $tmpdir
|
||||
is "$output" ${RELABEL} "Shared Relabel Correctly"
|
||||
is "$output" "${RELABEL} $tmpdir" "Shared Relabel Correctly"
|
||||
}
|
||||
|
||||
# vim: filetype=sh
|
||||
|
|
|
@ -55,9 +55,11 @@ function teardown() {
|
|||
cols=$(( 15 + RANDOM % 60 & 126 ))
|
||||
stty rows $rows cols $cols <$PODMAN_TEST_PTY
|
||||
|
||||
CR=$'\r'
|
||||
|
||||
# ...and make sure stty under podman reads that.
|
||||
run_podman run -it --name mystty $IMAGE stty size <$PODMAN_TEST_PTY
|
||||
is "$output" "$rows $cols" "stty under podman run reads the correct dimensions"
|
||||
is "$output" "$rows $cols$CR" "stty under podman run reads the correct dimensions"
|
||||
|
||||
run_podman rm -f mystty
|
||||
|
||||
|
@ -75,7 +77,7 @@ function teardown() {
|
|||
@test "podman load - will not read from tty" {
|
||||
run_podman 125 load <$PODMAN_TEST_PTY
|
||||
is "$output" \
|
||||
"Error: cannot read from terminal. Use command-line redirection" \
|
||||
"Error: cannot read from terminal. Use command-line redirection or the --input flag." \
|
||||
"Diagnostic from 'podman load' without redirection or -i"
|
||||
}
|
||||
|
||||
|
@ -84,14 +86,15 @@ function teardown() {
|
|||
run_podman run --tty -i --rm $IMAGE echo hello < /dev/null
|
||||
is "$output" ".*The input device is not a TTY.*" "-it _without_ a tty"
|
||||
|
||||
CR=$'\r'
|
||||
run_podman run --tty -i --rm $IMAGE echo hello <$PODMAN_TEST_PTY
|
||||
is "$output" "hello" "-it _with_ a pty"
|
||||
is "$output" "hello$CR" "-it _with_ a pty"
|
||||
|
||||
run_podman run --tty=false -i --rm $IMAGE echo hello < /dev/null
|
||||
is "$output" "hello" "-tty=false: no warning"
|
||||
|
||||
run_podman run --tty -i=false --rm $IMAGE echo hello < /dev/null
|
||||
is "$output" "hello" "-i=false: no warning"
|
||||
is "$output" "hello$CR" "-i=false: no warning"
|
||||
}
|
||||
|
||||
# vim: filetype=sh
|
||||
|
|
|
@ -417,7 +417,7 @@ load helpers
|
|||
|
||||
# check network alias for container short id
|
||||
run_podman inspect $cid --format "{{(index .NetworkSettings.Networks \"$netname\").Aliases}}"
|
||||
is "$output" "\[${cid:0:12}\]" "short container id in network aliases"
|
||||
is "$output" "[${cid:0:12}]" "short container id in network aliases"
|
||||
|
||||
run_podman network disconnect $netname $cid
|
||||
|
||||
|
@ -449,7 +449,7 @@ load helpers
|
|||
|
||||
# check network2 alias for container short id
|
||||
run_podman inspect $cid --format "{{(index .NetworkSettings.Networks \"$netname2\").Aliases}}"
|
||||
is "$output" "\[${cid:0:12}\]" "short container id in network aliases"
|
||||
is "$output" "[${cid:0:12}]" "short container id in network aliases"
|
||||
|
||||
# curl should work
|
||||
run curl --max-time 3 -s $SERVER/index.txt
|
||||
|
|
|
@ -30,7 +30,7 @@ spec:
|
|||
containers:
|
||||
- command:
|
||||
- sleep
|
||||
- "100"
|
||||
- \"100\"
|
||||
env:
|
||||
- name: PATH
|
||||
value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
|
||||
|
@ -49,7 +49,7 @@ spec:
|
|||
capabilities: {}
|
||||
privileged: false
|
||||
seLinuxOptions:
|
||||
level: "s0:c1,c2"
|
||||
level: \"s0:c1,c2\"
|
||||
readOnlyRootFilesystem: false
|
||||
volumeMounts:
|
||||
- mountPath: /testdir:z
|
||||
|
@ -73,7 +73,7 @@ RELABEL="system_u:object_r:container_file_t:s0"
|
|||
run_podman play kube - < $PODMAN_TMPDIR/test.yaml
|
||||
if [ -e /usr/sbin/selinuxenabled -a /usr/sbin/selinuxenabled ]; then
|
||||
run ls -Zd $TESTDIR
|
||||
is "$output" ${RELABEL} "selinux relabel should have happened"
|
||||
is "$output" "${RELABEL} $TESTDIR" "selinux relabel should have happened"
|
||||
fi
|
||||
|
||||
run_podman stop -a -t 0
|
||||
|
@ -88,7 +88,7 @@ RELABEL="system_u:object_r:container_file_t:s0"
|
|||
run_podman play kube $PODMAN_TMPDIR/test.yaml
|
||||
if [ -e /usr/sbin/selinuxenabled -a /usr/sbin/selinuxenabled ]; then
|
||||
run ls -Zd $TESTDIR
|
||||
is "$output" ${RELABEL} "selinux relabel should have happened"
|
||||
is "$output" "${RELABEL} $TESTDIR" "selinux relabel should have happened"
|
||||
fi
|
||||
|
||||
run_podman stop -a -t 0
|
||||
|
|
|
@ -478,13 +478,30 @@ function is() {
|
|||
local expect="$2"
|
||||
local testname="${3:-${MOST_RECENT_PODMAN_COMMAND:-[no test name given]}}"
|
||||
|
||||
local is_expr=
|
||||
if [ -z "$expect" ]; then
|
||||
if [ -z "$actual" ]; then
|
||||
# Both strings are empty.
|
||||
return
|
||||
fi
|
||||
expect='[no output]'
|
||||
elif expr "$actual" : "$expect" >/dev/null; then
|
||||
elif [[ "$actual" = "$expect" ]]; then
|
||||
# Strings are identical.
|
||||
return
|
||||
else
|
||||
# Strings are not identical. Are there wild cards in our expect string?
|
||||
if expr "$expect" : ".*[^\\][\*\[]" >/dev/null; then
|
||||
# There is a '[' or '*' without a preceding backslash.
|
||||
is_expr=' (using expr)'
|
||||
elif [[ "${expect:0:1}" = '[' ]]; then
|
||||
# String starts with '[', e.g. checking seconds like '[345]'
|
||||
is_expr=' (using expr)'
|
||||
fi
|
||||
if [[ -n "$is_expr" ]]; then
|
||||
if expr "$actual" : "$expect" >/dev/null; then
|
||||
return
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# This is a multi-line message, which may in turn contain multi-line
|
||||
|
@ -493,7 +510,7 @@ function is() {
|
|||
readarray -t actual_split <<<"$actual"
|
||||
printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2
|
||||
printf "#| FAIL: $testname\n" >&2
|
||||
printf "#| expected: '%s'\n" "$expect" >&2
|
||||
printf "#| expected: '%s'%s\n" "$expect" "$is_expr" >&2
|
||||
printf "#| actual: '%s'\n" "${actual_split[0]}" >&2
|
||||
local line
|
||||
for line in "${actual_split[@]:1}"; do
|
||||
|
|
Loading…
Reference in a new issue