test: Rework TEST-02-UNITTESTS

Currently, A large amount of unit test output is logged directly
to the console instead of to the per test log file as any subprocesses
executed by a test manager will detect that stderr is not connected
to the journal and log directly to /dev/console instead.

To solve this issue, let's make sure all tests are connected directly
to the journal by running them with systemd-run. We also simplify the
entire test script by getting rid of the custom queue and replicating
it with xargs instead. By using bash's function export feature, we can
make our run_test() function available to the bash subprocess spawned
by xargs.

Once a test is finished, we read its logs from the journal and put them
in the appropriate file if needed.

test/units/end.sh

@@ -4,7 +4,7 @@
 set -eux
 set -o pipefail
 
-(! journalctl -q -o short-monotonic --grep "didn't pass validation" >>/failed)
+(! journalctl -q -o short-monotonic --grep "didn't pass validation" | grep -v "test-varlink-idl" >>/failed)
 
 # Here, the redundant '[ ]' in the pattern is required in order not to match the logged command itself.
 (! journalctl -q -o short-monotonic --grep 'Warning: cannot close sd-bus connection[ ].*after fork' >>/failed)

test/units/testsuite-02.sh

@@ -20,10 +20,10 @@ fi
 
 NPROC=$(nproc)
 MAX_QUEUE_SIZE=${NPROC:-2}
-mapfile -t TEST_LIST < <(find /usr/lib/systemd/tests/unit-tests/ -maxdepth 1 -type f -name "${TESTS_GLOB}")
 
 # Reset state
 rm -fv /failed /skipped /testok
+touch /lock
 
 if ! systemd-detect-virt -qc; then
     # Make sure ping works for unprivileged users (for test-bpf-firewall)
@@ -34,21 +34,28 @@ fi
 # Arguments:
 #   $1: test path
 #   $2: test exit code
-report_result() {
-    if [[ $# -ne 2 ]]; then
-        echo >&2 "check_result: missing arguments"
+run_test() {
+    if [[ $# -ne 1 ]]; then
+        echo >&2 "run_test: missing arguments"
         exit 1
     fi
 
-    local name="${1##*/}"
-    local ret=$2
+    local test="$1"
+    local name="${test##*/}"
+
+    echo "Executing test $name as unit $name.service"
+
+    systemd-run --quiet --property Delegate=1 --unit="$name" --wait "$test" && ret=0 || ret=$?
+
+    exec {LOCK_FD}> /lock
+    flock --exclusive ${LOCK_FD}
 
     if [[ $ret -ne 0 && $ret != 77 && $ret != 127 ]]; then
         echo "$name failed with $ret"
         echo "$name" >>/failed-tests
         {
             echo "--- $name begin ---"
-            cat "/$name.log"
+            journalctl --unit="$name" --no-hostname -o short-monotonic
             echo "--- $name end ---"
         } >>/failed
     elif [[ $ret == 77 || $ret == 127 ]]; then
@@ -56,55 +63,21 @@ report_result() {
         echo "$name" >>/skipped-tests
         {
             echo "--- $name begin ---"
-            cat "/$name.log"
+            journalctl --unit="$name" --no-hostname -o short-monotonic
             echo "--- $name end ---"
         } >>/skipped
     else
         echo "$name OK"
         echo "$name" >>/testok
     fi
+
+    exec {LOCK_FD}<&-
 }
 
-set +x
-# Associative array for running tasks, where running[test-path]=PID
-declare -A running=()
-for task in "${TEST_LIST[@]}"; do
-    # If there's MAX_QUEUE_SIZE running tasks, keep checking the running queue
-    # until one of the tasks finishes, so we can replace it.
-    while [[ ${#running[@]} -ge $MAX_QUEUE_SIZE ]]; do
-        for key in "${!running[@]}"; do
-            if ! kill -0 "${running[$key]}" &>/dev/null; then
-                # Task has finished, report its result and drop it from the queue
-                wait "${running[$key]}" && ec=0 || ec=$?
-                report_result "$key" "$ec"
-                unset "running[$key]"
-                # Break from inner for loop and outer while loop to skip
-                # the sleep below when we find a free slot in the queue
-                break 2
-            fi
-        done
+export -f run_test
 
-        # Precisely* calculated constant to keep the spinlock from burning the CPU(s)
-        sleep 0.01
-    done
-
-    if [[ -x $task ]]; then
-        echo "Executing test '$task'"
-        log_file="/${task##*/}.log"
-        $task &>"$log_file" &
-        running[$task]=$!
-    fi
-done
-
-# Wait for remaining running tasks
-for key in "${!running[@]}"; do
-    echo "Waiting for test '$key' to finish"
-    wait "${running[$key]}" && ec=0 || ec=$?
-    report_result "$key" "$ec"
-    unset "running[$key]"
-done
-
-set -x
+find /usr/lib/systemd/tests/unit-tests/ -maxdepth 1 -type f -name "${TESTS_GLOB}" -print0 |
+    xargs -0 -I {} --max-procs="$MAX_QUEUE_SIZE" bash -ec "run_test {}"
 
 # Test logs are sometimes lost, as the system shuts down immediately after
 journalctl --sync