test: give the container time to properly shut down on exception

Otherwise the `terminate()` method sends SIGKILL rather quickly (~0.3s),
which then leaves a dangling scope on the host system, breaking further
test executions.

test/test-shutdown.py

@@ -5,6 +5,7 @@
 import argparse
 import logging
 import sys
+import time
 
 import pexpect
 
@@ -90,7 +91,23 @@ def run(args):
     except Exception as e:
         logger.error(e)
         logger.info("killing child pid %d", console.pid)
-        console.terminate(force=True)
+        # We can't use console.terminate(force=True) right away, since
+        # the internal delay between sending a signal and checking the process
+        # is just 0.1s [0], which means we'd get SIGKILLed pretty quickly.
+        # Let's send SIGHUP/SIGINT first, wait a bit, and then follow-up with
+        # SIGHUP/SIGINT/SIGKILL if the process is still alive.
+        # [0] https://github.com/pexpect/pexpect/blob/acb017a97332c19a9295660fe87316926a8adc55/pexpect/spawnbase.py#L71
+        console.terminate()
+        for _ in range(10):
+            if not console.isalive():
+                break
+
+            time.sleep(1)
+        else:
+            # We haven't exited the loop early, so check if the process is
+            # still alive - if so, force-kill it.
+            if console.isalive():
+                console.terminate(force=True)
 
     return ret