Reenable remote system tests

podman-remote is in better shape now. Let's see what needs
to be done to reenable remote system tests.

 - logs test: skip multilog, it doesn't work remote

 - diff test: use -l only when local, not with remote

 - many other tests: skip_if_remote, with 'FIXME: pending #xxxx'
   where xxxx is a filed issue.

Unrelated: added new helper to skip_if_remote and _if_rootless,
where we check if the source message includes "remote"/"rootless"
and insert it if missing. This is a minor usability enhancement
to make it easier to understand at-a-glance why a skip triggers.

Signed-off-by: Ed Santiago <santiago@redhat.com>

Makefile

@@ -353,21 +353,19 @@ remotesystem:
 	# Start podman server using tmp socket; loop-wait for it;
 	# test podman-remote; kill server, clean up tmp socket file.
 	# podman server spews copious unhelpful output; ignore it.
-	# FIXME FIXME FIXME: remove 'exit 0' after #6538 and #6539 are fixed
-	exit 0;\
 	rc=0;\
 	if timeout -v 1 true; then \
 		SOCK_FILE=$(shell mktemp --dry-run --tmpdir podman.XXXXXX);\
 		export PODMAN_SOCKET=unix:$$SOCK_FILE; \
-		./bin/podman system service --timeout=0 $$PODMAN_SOCKET &> $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) & \
+		./bin/podman system service --timeout=0 $$PODMAN_SOCKET > $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) 2>&1 & \
 		retry=5;\
-		while [[ $$retry -ge 0 ]]; do\
+		while [ $$retry -ge 0 ]; do\
 			echo Waiting for server...;\
 			sleep 1;\
-			./bin/podman-remote --url $$PODMAN_SOCKET info &>/dev/null && break;\
+			./bin/podman-remote --url $$PODMAN_SOCKET info >/dev/null 2>&1 && break;\
 			retry=$$(expr $$retry - 1);\
 		done;\
-		if [[ $$retry -lt 0 ]]; then\
+		if [ $$retry -lt 0 ]; then\
 			echo "Error: ./bin/podman system service did not come up on $$SOCK_FILE" >&2;\
 			exit 1;\
 		fi;\

test/system/030-run.bats

@@ -96,6 +96,8 @@ echo $rand        |   0 | $rand
     # Believe it or not, 'sh -c' resulted in different behavior
     run_podman 0 run --rm $IMAGE sh -c /bin/true
     run_podman 1 run --rm $IMAGE sh -c /bin/false
+
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
 }
 
 @test "podman run --name" {
@@ -202,6 +204,8 @@ echo $rand        |   0 | $rand
 }
 
 @test "podman run docker-archive" {
+    skip_if_remote "FIXME: pending #7116"
+
     # Create an image that, when run, outputs a random magic string
     expect=$(random_string 20)
     run_podman run --name myc --entrypoint="[\"/bin/echo\",\"$expect\"]" $IMAGE
@@ -247,6 +251,8 @@ echo $rand        |   0 | $rand
 # symptom only manifests on a fedora container image -- we have no
 # reproducer on alpine. Checking directory ownership is good enough.
 @test "podman run : user namespace preserved root ownership" {
+    skip_if_remote "FIXME: pending #7195"
+
     for priv in "" "--privileged"; do
         for user in "--user=0" "--user=100"; do
             for keepid in "" "--userns=keep-id"; do
@@ -260,10 +266,14 @@ echo $rand        |   0 | $rand
             done
         done
     done
+
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
 }
 
 # #6829 : add username to /etc/passwd inside container if --userns=keep-id
 @test "podman run : add username to /etc/passwd if --userns=keep-id" {
+    skip_if_remote "FIXME: pending #7195"
+
     # Default: always run as root
     run_podman run --rm $IMAGE id -un
     is "$output" "root" "id -un on regular container"
@@ -282,10 +292,14 @@ echo $rand        |   0 | $rand
     run_podman run --rm --privileged --userns=keep-id --user=0 $IMAGE id -un
     remove_same_dev_warning      # grumble
     is "$output" "root" "--user=0 overrides keep-id"
+
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
 }
 
 # #6991 : /etc/passwd is modifiable
 @test "podman run : --userns=keep-id: passwd file is modifiable" {
+    skip_if_remote "FIXME: pending #7195"
+
     run_podman run -d --userns=keep-id $IMAGE sh -c 'while ! test -e /stop; do sleep 0.1; done'
     cid="$output"
 

test/system/035-logs.bats

@@ -25,6 +25,8 @@ load helpers
 }
 
 @test "podman logs - multi" {
+    skip_if_remote "logs does not support multiple containers when run remotely"
+
     # Simple helper to make the container starts, below, easier to read
     local -a cid
     doit() {

test/system/050-stop.bats

@@ -12,9 +12,12 @@ load helpers
     run_podman stop $cid
     t1=$SECONDS
 
-    # Confirm that container is stopped
+    # Confirm that container is stopped. Podman-remote unfortunately
+    # cannot tell the difference between "stopped" and "exited", and
+    # spits them out interchangeably, so we need to recognize either.
     run_podman inspect --format '{{.State.Status}} {{.State.ExitCode}}' $cid
-    is "$output" "exited \+137" "Status and exit code of stopped container"
+    is "$output" "\\(stopped\|exited\\) \+137" \
+       "Status and exit code of stopped container"
 
     # The initial SIGTERM is ignored, so this operation should take
     # exactly 10 seconds. Give it some leeway.

test/system/055-rm.bats

@@ -44,6 +44,8 @@ load helpers
 #
 # See https://github.com/containers/podman/issues/3795
 @test "podman rm -f" {
+    skip_if_remote "FIXME: pending #7117"
+
     rand=$(random_string 30)
     ( sleep 3; run_podman rm -f $rand ) &
     run_podman 137 run --name $rand $IMAGE sleep 30

test/system/070-build.bats

@@ -6,9 +6,7 @@
 load helpers
 
 @test "podman build - basic test" {
-    if is_remote && is_rootless; then
-        skip "unreliable with podman-remote and rootless; #2972"
-    fi
+    skip_if_remote "FIXME: pending #7136"
 
     rand_filename=$(random_string 20)
     rand_content=$(random_string 50)
@@ -34,6 +32,7 @@ EOF
 
 # Regression from v1.5.0. This test passes fine in v1.5.0, fails in 1.6
 @test "podman build - cache (#3920)" {
+    skip_if_remote "FIXME: pending #7136"
     if is_remote && is_rootless; then
         skip "unreliable with podman-remote and rootless; #2972"
     fi
@@ -81,6 +80,8 @@ EOF
 }
 
 @test "podman build - URLs" {
+    skip_if_remote "FIXME: pending #7137"
+
     tmpdir=$PODMAN_TMPDIR/build-test
     mkdir -p $tmpdir
 
@@ -90,6 +91,7 @@ ADD https://github.com/containers/podman/blob/master/README.md /tmp/
 EOF
     run_podman build -t add_url $tmpdir
     run_podman run --rm add_url stat /tmp/README.md
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman rmi -f add_url
 
     # Now test COPY. That should fail.
@@ -100,6 +102,8 @@ EOF
 
 
 @test "podman build - workdir, cmd, env, label" {
+    skip_if_remote "FIXME: pending #7137"
+
     tmpdir=$PODMAN_TMPDIR/build-test
     mkdir -p $tmpdir
 
@@ -234,19 +238,19 @@ Labels.$label_name | $label_value
 }
 
 @test "podman build - stdin test" {
-    if is_remote && is_rootless; then
-        skip "unreliable with podman-remote and rootless; #2972"
-    fi
+    skip_if_remote "FIXME: pending #7136"
 
-    # Random workdir, and multiple random strings to verify command & env
+    # Random workdir, and random string to verify build output
     workdir=/$(random_string 10)
+    random_echo=$(random_string 15)
     PODMAN_TIMEOUT=240 run_podman build -t build_test - << EOF
 FROM  $IMAGE
 RUN mkdir $workdir
 WORKDIR $workdir
-RUN /bin/echo 'Test'
+RUN /bin/echo $random_echo
 EOF
     is "$output" ".*STEP 5: COMMIT" "COMMIT seen in log"
+    is "$output" ".*STEP .: RUN /bin/echo $random_echo"
 
     run_podman run --rm build_test pwd
     is "$output" "$workdir" "pwd command in container"

test/system/110-history.bats

@@ -3,6 +3,8 @@
 load helpers
 
 @test "podman history - basic tests" {
+    skip_if_remote "FIXME: pending #7122"
+
     tests="
                                  | .*[0-9a-f]\\\{12\\\} .* CMD .* LABEL
 --format '{{.ID}} {{.Created}}'  | .*[0-9a-f]\\\{12\\\} .* ago

test/system/120-load.bats

@@ -28,6 +28,8 @@ verify_iid_and_name() {
 
 
 @test "podman load - by image ID" {
+    skip_if_remote "FIXME: pending #7123"
+
     # FIXME: how to build a simple archive instead?
     get_iid_and_name
 
@@ -74,7 +76,9 @@ verify_iid_and_name() {
     verify_iid_and_name $img_name
 }
 
-@test "podman load - NAME and NAME:TAG arguments work (requires: #2674)" {
+@test "podman load - NAME and NAME:TAG arguments work" {
+    skip_if_remote "FIXME: pending #7124"
+
     get_iid_and_name
     run_podman save $iid -o $archive
     run_podman rmi $iid

test/system/130-kill.bats

@@ -6,6 +6,8 @@
 load helpers
 
 @test "podman kill - test signal handling in containers" {
+    skip_if_remote "FIXME: pending #7135"
+
     # podman-remote and crun interact poorly in f31: crun seems to gobble up
     # some signals.
     # Workaround: run 'env --default-signal sh' instead of just 'sh' in

test/system/140-diff.bats

@@ -6,9 +6,16 @@
 load helpers
 
 @test "podman diff" {
+    n=$(random_string 10)          # container name
     rand_file=$(random_string 10)
-    run_podman run $IMAGE sh -c "touch /$rand_file;rm /etc/services"
-    run_podman diff --format json -l
+    run_podman run --name $n $IMAGE sh -c "touch /$rand_file;rm /etc/services"
+
+    # If running local, test `-l` (latest) option. This can't work with remote.
+    if ! is_remote; then
+        n=-l
+    fi
+
+    run_podman diff --format json $n
 
     # Expected results for each type of diff
     declare -A expect=(
@@ -22,7 +29,7 @@ load helpers
         is "$result" "${expect[$field]}" "$field"
     done
 
-    run_podman rm -l
+    run_podman rm $n
 }
 
 # vim: filetype=sh

test/system/160-volumes.bats

@@ -93,6 +93,7 @@ Labels.l       | $mylabel
     is "$(<$mountpoint/myfile)" "$rand" "we see content created in container"
 
     # Clean up
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman volume rm $myvolume
 }
 
@@ -134,12 +135,14 @@ EOF
     is "$output" "got here -$rand-" "script in volume is runnable with default (exec)"
 
     # Clean up
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman volume rm $myvolume
 }
 
 
 # Anonymous temporary volumes, and persistent autocreated named ones
 @test "podman volume, implicit creation with run" {
+    skip_if_remote "FIXME: pending #7128"
 
     # No hostdir arg: create anonymous container with random name
     rand=$(random_string)
@@ -172,6 +175,7 @@ EOF
     run_podman run --rm -v $myvol:/myvol:z $IMAGE \
                sh -c "cp /myvol/myfile /myvol/myfile2"
 
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman volume rm $myvol
 
     # Autocreated volumes should also work with keep-id
@@ -180,6 +184,7 @@ EOF
     run_podman run --rm -v $myvol:/myvol:z --userns=keep-id $IMAGE \
                touch /myvol/myfile
 
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman volume rm $myvol
 }
 
@@ -187,6 +192,7 @@ EOF
 # Confirm that container sees the correct id
 @test "podman volume with --userns=keep-id" {
     is_rootless || skip "only meaningful when run rootless"
+    skip_if_remote "FIXME: pending #7195"
 
     myvoldir=${PODMAN_TMPDIR}/volume_$(random_string)
     mkdir $myvoldir

test/system/200-pod.bats

@@ -18,7 +18,9 @@ function teardown() {
 
 
 @test "podman pod top - containers in different PID namespaces" {
-    skip_if_remote "podman-pod does not work with podman-remote"
+    if is_remote && is_rootless; then
+        skip "FIXME: pending #7139"
+    fi
 
     # With infra=false, we don't get a /pause container (we also
     # don't pull k8s.gcr.io/pause )
@@ -53,7 +55,9 @@ function teardown() {
 
 
 @test "podman pod - communicating between pods" {
-    skip_if_remote "podman-pod does not work with podman-remote"
+    if is_remote && is_rootless; then
+        skip "FIXME: pending #7139"
+    fi
 
     podname=pod$(random_string)
     run_podman 1 pod exists $podname
@@ -77,7 +81,7 @@ function teardown() {
     run_podman ps --format '{{.Pod}}'
     newline="
 "
-    is "$output" "${podid:0:12}${newline}${podid:0:12}" "sdfdsf"
+    is "$output" "${podid:0:12}${newline}${podid:0:12}" "ps shows 2 pod IDs"
 
     # Talker: send the message via common port on localhost
     message=$(random_string 15)
@@ -89,6 +93,7 @@ function teardown() {
     is "$output" "$message" "message sent from one container to another"
 
     # Clean up. First the nc -l container...
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman rm $cid1
 
     # ...then, from pause container, find the image ID of the pause image...
@@ -99,6 +104,7 @@ function teardown() {
     pause_iid="$output"
 
     # ...then rm the pod, then rmi the pause image so we don't leave strays.
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman pod rm $podname
     run_podman rmi $pause_iid
 
@@ -135,6 +141,10 @@ function random_ip() {
 }
 
 @test "podman pod create - hashtag AllTheOptions" {
+    if is_remote && is_rootless; then
+        skip "FIXME: pending #7139"
+    fi
+
     mac=$(random_mac)
     add_host_ip=$(random_ip)
     add_host_n=$(random_string | tr A-Z a-z).$(random_string | tr A-Z a-z).xyz
@@ -205,6 +215,7 @@ function random_ip() {
     is "$output" ".*options $dns_opt"        "--dns-opt was added"
 
     # pod inspect
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
     run_podman pod inspect --format '{{.Name}}: {{.ID}} : {{.NumContainers}} : {{.Labels}}' mypod
     is "$output" "mypod: $pod_id : 1 : map\[${labelname}:${labelvalue}]" \
        "pod inspect --format ..."

test/system/220-healthcheck.bats

@@ -25,6 +25,7 @@ function _check_health {
 
 
 @test "podman healthcheck" {
+    skip_if_remote "FIXME: pending #7137"
 
     # Create an image with a healthcheck script; said script will
     # pass until the file /uh-oh gets created (by us, via exec)

test/system/300-cli-parsing.bats

@@ -10,6 +10,8 @@ load helpers
     #   Error: invalid argument "true=\"false\"" for "-l, --label" \
     #      flag: parse error on line 1, column 5: bare " in non-quoted-field
     run_podman run --rm --label 'true="false"' $IMAGE true
+
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
 }
 
 # vim: filetype=sh

test/system/400-unprivileged-access.bats

@@ -101,6 +101,11 @@ EOF
 
 # #6957 - mask out /proc/acpi, /sys/dev, and other sensitive system files
 @test "sensitive mount points are masked without --privileged" {
+    # Weird error, maybe a flake?
+    #   can only attach to created or running containers: container state improper
+    # https://github.com/containers/podman/pull/7111#issuecomment-666858715
+    skip_if_remote "FIXME: Weird flake"
+
     # FIXME: this should match the list in pkg/specgen/generate/config_linux.go
     local -a mps=(
         /proc/acpi
@@ -160,6 +165,8 @@ EOF
             die "$path: Unknown file type '$type'"
         fi
     done
+
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
 }
 
 # vim: filetype=sh

test/system/410-selinux.bats

@@ -16,6 +16,7 @@ function check_label() {
     # FIXME: it'd be nice to specify the command to run, e.g. 'ls -dZ /',
     # but alpine ls (from busybox) doesn't support -Z
     run_podman run --rm $args $IMAGE cat -v /proc/self/attr/current
+    if is_remote; then sleep 2;fi   # FIXME: pending #7119
 
     # FIXME: on some CI systems, 'run --privileged' emits a spurious
     # warning line about dup devices. Ignore it.

test/system/helpers.bash

@@ -240,12 +240,29 @@ function is_remote() {
     [[ "$PODMAN" =~ -remote ]]
 }
 
+###########################
+#  _add_label_if_missing  #  make sure skip messages include rootless/remote
+###########################
+function _add_label_if_missing() {
+    local msg="$1"
+    local want="$2"
+
+    if [ -z "$msg" ]; then
+        echo
+    elif expr "$msg" : ".*$want" &>/dev/null; then
+        echo "$msg"
+    else
+        echo "[$want] $msg"
+    fi
+}
+
 ######################
 #  skip_if_rootless  #  ...with an optional message
 ######################
 function skip_if_rootless() {
     if is_rootless; then
-        skip "${1:-not applicable under rootless podman}"
+        local msg=$(_add_label_if_missing "$1" "rootless")
+        skip "${msg:-not applicable under rootless podman}"
     fi
 }
 
@@ -254,7 +271,8 @@ function skip_if_rootless() {
 ####################
 function skip_if_remote() {
     if is_remote; then
-        skip "${1:-test does not work with podman-remote}"
+        local msg=$(_add_label_if_missing "$1" "remote")
+        skip "${msg:-test does not work with podman-remote}"
     fi
 }