torture: Add kvm-again.sh to rerun a previous torture-test

This commit adds a kvm-again.sh script that, given the results directory
of a torture-test run, re-runs that test.  This means that the kernels
need not be rebuilt, but it also is a step towards running torture tests
on remote systems.

This commit also adds a kvm-test-1-run-batch.sh script that runs one
batch out of the torture test.  The idea is to copy a results directory
tree to remote systems, then use kvm-test-1-run-batch.sh to run batches
on these systems.

Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
This commit is contained in:
Paul E. McKenney 2021-02-19 17:49:58 -08:00
parent d6100d764c
commit 7cf86c0b62
2 changed files with 238 additions and 0 deletions

View file

@ -0,0 +1,171 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0+
#
# Rerun a series of tests under KVM.
#
# Usage: kvm-again.sh /path/to/old/run [ options ]
#
# Copyright (C) 2021 Facebook, Inc.
#
# Authors: Paul E. McKenney <paulmck@kernel.org>
scriptname=$0
args="$*"
T=${TMPDIR-/tmp}/kvm-again.sh.$$
trap 'rm -rf $T' 0
mkdir $T
if ! test -d tools/testing/selftests/rcutorture/bin
then
echo $scriptname must be run from top-level directory of kernel source tree.
exit 1
fi
oldrun=$1
shift
if ! test -d "$oldrun"
then
echo "Usage: $scriptname /path/to/old/run [ options ]"
exit 1
fi
if ! cp "$oldrun/batches" $T/batches.oldrun
then
# Later on, can reconstitute this from console.log files.
echo Prior run batches file does not exist: $oldrun/batches
exit 1
fi
if test -f "$oldrun/torture_suite"
then
torture_suite="`cat $oldrun/torture_suite`"
elif test -f "$oldrun/TORTURE_SUITE"
then
torture_suite="`cat $oldrun/TORTURE_SUITE`"
else
echo "Prior run torture_suite file does not exist: $oldrun/{torture_suite,TORTURE_SUITE}"
exit 1
fi
KVM="`pwd`/tools/testing/selftests/rcutorture"; export KVM
PATH=${KVM}/bin:$PATH; export PATH
. functions.sh
dryrun=
default_link="cp -R"
rundir="`pwd`/tools/testing/selftests/rcutorture/res/`date +%Y.%m.%d-%H.%M.%S-again`"
startdate="`date`"
starttime="`get_starttime`"
usage () {
echo "Usage: $scriptname $oldrun [ arguments ]:"
echo " --dryrun"
echo " --link hard|soft|copy"
echo " --remote"
echo " --rundir /new/res/path"
exit 1
}
while test $# -gt 0
do
case "$1" in
--dryrun)
dryrun=1
;;
--link)
checkarg --link "hard|soft|copy" "$#" "$2" 'hard\|soft\|copy' '^--'
case "$2" in
copy)
arg_link="cp -R"
;;
hard)
arg_link="cp -Rl"
;;
soft)
arg_link="cp -Rs"
;;
esac
shift
;;
--remote)
arg_remote=1
default_link="cp -as"
;;
--rundir)
checkarg --rundir "(absolute pathname)" "$#" "$2" '^/' '^error'
rundir=$2
if test -e "$rundir"
then
echo "--rundir $2: Already exists."
usage
fi
shift
;;
*)
echo Unknown argument $1
usage
;;
esac
shift
done
if test -z "$arg_link"
then
arg_link="$default_link"
fi
echo ---- Re-run results directory: $rundir
# Copy old run directory tree over and adjust.
mkdir -p "`dirname "$rundir"`"
if ! $arg_link "$oldrun" "$rundir"
then
echo "Cannot copy from $oldrun to $rundir."
usage
fi
rm -f "$rundir"/*/{console.log,console.log.diags,qemu_pid,qemu-retval,Warnings,kvm-test-1-run.sh.out,kvm-test-1-run-qemu.sh.out,vmlinux} "$rundir"/log
echo $oldrun > "$rundir/re-run"
if ! test -d "$rundir/../../bin"
then
$arg_link "$oldrun/../../bin" "$rundir/../.."
fi
for i in $rundir/*/qemu-cmd
do
cp "$i" $T
qemu_cmd_dir="`dirname "$i"`"
kernel_dir="`echo $qemu_cmd_dir | sed -e 's/\.[0-9]\+$//'`"
kvm-transform.sh $kernel_dir/bzImage $qemu_cmd_dir/console.log < $T/qemu-cmd > $i
echo "# TORTURE_KCONFIG_GDB_ARG=''" >> $i
done
grep -v '^#' $T/batches.oldrun | awk '
BEGIN {
oldbatch = 1;
}
{
if (oldbatch != $1) {
print "kvm-test-1-run-batch.sh" curbatch;
curbatch = "";
oldbatch = $1;
}
curbatch = curbatch " " $2;
}
END {
print "kvm-test-1-run-batch.sh" curbatch
}' > $T/runbatches.sh
if test -n "$dryrun"
then
echo ---- Dryrun complete, directory: $rundir | tee -a "$rundir/log"
else
( cd "$rundir"; sh $T/runbatches.sh )
kcsan-collapse.sh "$rundir" | tee -a "$rundir/log"
echo | tee -a "$rundir/log"
echo ---- Results directory: $rundir | tee -a "$rundir/log"
kvm-recheck.sh "$rundir" > $T/kvm-recheck.sh.out 2>&1
ret=$?
cat $T/kvm-recheck.sh.out | tee -a "$rundir/log"
echo " --- Done at `date` (`get_starttime_duration $starttime`) exitcode $ret" | tee -a "$rundir/log"
exit $ret
fi

View file

@ -0,0 +1,67 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0+
#
# Carry out a kvm-based run for the specified batch of scenarios, which
# might have been built by --build-only kvm.sh run.
#
# Usage: kvm-test-1-run-batch.sh SCENARIO [ SCENARIO ... ]
#
# Each SCENARIO is the name of a directory in the current directory
# containing a ready-to-run qemu-cmd file.
#
# Copyright (C) 2021 Facebook, Inc.
#
# Authors: Paul E. McKenney <paulmck@kernel.org>
T=${TMPDIR-/tmp}/kvm-test-1-run-batch.sh.$$
trap 'rm -rf $T' 0
mkdir $T
echo ---- Running batch $*
# Check arguments
runfiles=
for i in "$@"
do
if ! echo $i | grep -q '^[^/.a-z]\+\(\.[0-9]\+\)\?$'
then
echo Bad scenario name: \"$i\" 1>&2
exit 1
fi
if ! test -d "$i"
then
echo Scenario name not a directory: \"$i\" 1>&2
exit 2
fi
if ! test -f "$i/qemu-cmd"
then
echo Scenario lacks a command file: \"$i/qemu-cmd\" 1>&2
exit 3
fi
rm -f $i/build.*
touch $i/build.run
runfiles="$runfiles $i/build.run"
done
# Extract settings from the qemu-cmd file.
grep '^#' $1/qemu-cmd | sed -e 's/^# //' > $T/qemu-cmd-settings
. $T/qemu-cmd-settings
# Start up jitter, start each scenario, wait, end jitter.
echo ---- System running test: `uname -a`
echo ---- Starting kernels. `date` | tee -a log
$TORTURE_JITTER_START
for i in "$@"
do
echo ---- System running test: `uname -a` > $i/kvm-test-1-run-qemu.sh.out
echo > $i/kvm-test-1-run-qemu.sh.out
kvm-test-1-run-qemu.sh $i >> $i/kvm-test-1-run-qemu.sh.out 2>&1 &
done
for i in $runfiles
do
while ls $i > /dev/null 2>&1
do
:
done
done
echo ---- All kernel runs complete. `date` | tee -a log
$TORTURE_JITTER_STOP