mirror of
https://gitlab.com/qemu-project/qemu
synced 2024-09-06 18:05:37 +00:00
![Paolo Bonzini](/assets/img/avatar_default.png)
Add an option to check if upload is permitted without actually attempting a build. This can be useful to add a third outcome beyond success and failure---namely, a CI job can self-cancel if the uploading quota has been reached. There is a small change here in that a failure to do the upload check changes the exit code from 1 to 99. 99 was chosen because it is what Autotools and Meson use to represent a problem in the setup (as opposed to a failure in the test). Reviewed-by: Peter Maydell <peter.maydell@linaro.org> Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>
459 lines
15 KiB
Bash
Executable file
459 lines
15 KiB
Bash
Executable file
#!/bin/sh -e
|
|
|
|
# Upload a created tarball to Coverity Scan, as per
|
|
# https://scan.coverity.com/projects/qemu/builds/new
|
|
|
|
# This work is licensed under the terms of the GNU GPL version 2,
|
|
# or (at your option) any later version.
|
|
# See the COPYING file in the top-level directory.
|
|
#
|
|
# Copyright (c) 2017-2020 Linaro Limited
|
|
# Written by Peter Maydell
|
|
|
|
# Note that this script will automatically download and
|
|
# run the (closed-source) coverity build tools, so don't
|
|
# use it if you don't trust them!
|
|
|
|
# This script assumes that you're running it from a QEMU source
|
|
# tree, and that tree is a fresh clean one, because we do an in-tree
|
|
# build. (This is necessary so that the filenames that the Coverity
|
|
# Scan server sees are relative paths that match up with the component
|
|
# regular expressions it uses; an out-of-tree build won't work for this.)
|
|
# The host machine should have as many of QEMU's dependencies
|
|
# installed as possible, for maximum coverity coverage.
|
|
|
|
# To do an upload you need to be a maintainer in the Coverity online
|
|
# service, and you will need to know the "Coverity token", which is a
|
|
# secret 8 digit hex string. You can find that from the web UI in the
|
|
# project settings, if you have maintainer access there.
|
|
|
|
# Command line options:
|
|
# --check-upload-only : return success if upload is possible
|
|
# --dry-run : run the tools, but don't actually do the upload
|
|
# --docker : create and work inside a container
|
|
# --docker-engine : specify the container engine to use (docker/podman/auto);
|
|
# implies --docker
|
|
# --update-tools-only : update the cached copy of the tools, but don't run them
|
|
# --no-update-tools : do not update the cached copy of the tools
|
|
# --tokenfile : file to read Coverity token from
|
|
# --version ver : specify version being analyzed (default: ask git)
|
|
# --description desc : specify description of this version (default: ask git)
|
|
# --srcdir : QEMU source tree to analyze (default: current working dir)
|
|
# --results-tarball : path to copy the results tarball to (default: don't
|
|
# copy it anywhere, just upload it)
|
|
# --src-tarball : tarball to untar into src dir (default: none); this
|
|
# is intended mainly for internal use by the Docker support
|
|
#
|
|
# User-specifiable environment variables:
|
|
# COVERITY_TOKEN -- Coverity token (default: looks at your
|
|
# coverity.token config)
|
|
# COVERITY_EMAIL -- the email address to use for uploads (default:
|
|
# looks at your git coverity.email or user.email config)
|
|
# COVERITY_BUILD_CMD -- make command (default: 'make -jN' where N is
|
|
# number of CPUs as determined by 'nproc')
|
|
# COVERITY_TOOL_BASE -- set to directory to put coverity tools
|
|
# (default: /tmp/coverity-tools)
|
|
#
|
|
# You must specify the token, either by environment variable or by
|
|
# putting it in a file and using --tokenfile. Everything else has
|
|
# a reasonable default if this is run from a git tree.
|
|
|
|
upload_permitted() {
|
|
# Check whether we can do an upload to the server; will exit *the script*
|
|
# with status 99 if the check failed (usually a bad token);
|
|
# will return from the function with status 1 if the check indicated
|
|
# that we can't upload yet (ie we are at quota)
|
|
# Assumes that COVERITY_TOKEN and PROJNAME have been initialized.
|
|
|
|
echo "Checking upload permissions..."
|
|
|
|
if ! up_perm="$(wget https://scan.coverity.com/api/upload_permitted --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -q -O -)"; then
|
|
echo "Coverity Scan API access denied: bad token?"
|
|
exit 99
|
|
fi
|
|
|
|
# Really up_perm is a JSON response with either
|
|
# {upload_permitted:true} or {next_upload_permitted_at:<date>}
|
|
# We do some hacky string parsing instead of properly parsing it.
|
|
case "$up_perm" in
|
|
*upload_permitted*true*)
|
|
return 0
|
|
;;
|
|
*next_upload_permitted_at*)
|
|
return 1
|
|
;;
|
|
*)
|
|
echo "Coverity Scan upload check: unexpected result $up_perm"
|
|
exit 99
|
|
;;
|
|
esac
|
|
}
|
|
|
|
|
|
check_upload_permissions() {
|
|
# Check whether we can do an upload to the server; will exit the script
|
|
# with status 99 if the check failed (usually a bad token);
|
|
# will exit the script with status 0 if the check indicated that we
|
|
# can't upload yet (ie we are at quota)
|
|
# Assumes that COVERITY_TOKEN, PROJNAME and DRYRUN have been initialized.
|
|
|
|
if upload_permitted; then
|
|
echo "Coverity Scan: upload permitted"
|
|
else
|
|
if [ "$DRYRUN" = yes ]; then
|
|
echo "Coverity Scan: upload quota reached, continuing dry run"
|
|
else
|
|
echo "Coverity Scan: upload quota reached; stopping here"
|
|
# Exit success as this isn't a build error.
|
|
exit 0
|
|
fi
|
|
fi
|
|
}
|
|
|
|
|
|
build_docker_image() {
|
|
# build docker container including the coverity-scan tools
|
|
echo "Building docker container..."
|
|
# TODO: This re-unpacks the tools every time, rather than caching
|
|
# and reusing the image produced by the COPY of the .tgz file.
|
|
# Not sure why.
|
|
tests/docker/docker.py --engine ${DOCKER_ENGINE} build \
|
|
-t coverity-scanner -f scripts/coverity-scan/coverity-scan.docker \
|
|
--extra-files scripts/coverity-scan/run-coverity-scan \
|
|
"$COVERITY_TOOL_BASE"/coverity_tool.tgz
|
|
}
|
|
|
|
update_coverity_tools () {
|
|
# Check for whether we need to download the Coverity tools
|
|
# (either because we don't have a copy, or because it's out of date)
|
|
# Assumes that COVERITY_TOOL_BASE, COVERITY_TOKEN and PROJNAME are set.
|
|
|
|
mkdir -p "$COVERITY_TOOL_BASE"
|
|
cd "$COVERITY_TOOL_BASE"
|
|
|
|
echo "Checking for new version of coverity build tools..."
|
|
wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME&md5=1" -O coverity_tool.md5.new
|
|
|
|
if ! cmp -s coverity_tool.md5 coverity_tool.md5.new; then
|
|
# out of date md5 or no md5: download new build tool
|
|
# blow away the old build tool
|
|
echo "Downloading coverity build tools..."
|
|
rm -rf coverity_tool coverity_tool.tgz
|
|
wget https://scan.coverity.com/download/cxx/linux64 --post-data "token=$COVERITY_TOKEN&project=$PROJNAME" -O coverity_tool.tgz
|
|
if ! (cat coverity_tool.md5.new; echo " coverity_tool.tgz") | md5sum -c --status; then
|
|
echo "Downloaded tarball didn't match md5sum!"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$DOCKER" != yes ]; then
|
|
# extract the new one, keeping it corralled in a 'coverity_tool' directory
|
|
echo "Unpacking coverity build tools..."
|
|
mkdir -p coverity_tool
|
|
cd coverity_tool
|
|
tar xf ../coverity_tool.tgz
|
|
cd ..
|
|
mv coverity_tool.md5.new coverity_tool.md5
|
|
fi
|
|
fi
|
|
rm -f coverity_tool.md5.new
|
|
cd "$SRCDIR"
|
|
|
|
if [ "$DOCKER" = yes ]; then
|
|
build_docker_image
|
|
fi
|
|
}
|
|
|
|
|
|
# Check user-provided environment variables and arguments
|
|
DRYRUN=no
|
|
UPDATE=yes
|
|
DOCKER=no
|
|
PROJNAME=QEMU
|
|
|
|
while [ "$#" -ge 1 ]; do
|
|
case "$1" in
|
|
--check-upload-only)
|
|
shift
|
|
DRYRUN=check
|
|
;;
|
|
--dry-run)
|
|
shift
|
|
DRYRUN=yes
|
|
;;
|
|
--no-update-tools)
|
|
shift
|
|
UPDATE=no
|
|
;;
|
|
--update-tools-only)
|
|
shift
|
|
UPDATE=only
|
|
;;
|
|
--version)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--version needs an argument"
|
|
exit 1
|
|
fi
|
|
VERSION="$1"
|
|
shift
|
|
;;
|
|
--description)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--description needs an argument"
|
|
exit 1
|
|
fi
|
|
DESCRIPTION="$1"
|
|
shift
|
|
;;
|
|
--tokenfile)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--tokenfile needs an argument"
|
|
exit 1
|
|
fi
|
|
COVERITY_TOKEN="$(cat "$1")"
|
|
shift
|
|
;;
|
|
--srcdir)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--srcdir needs an argument"
|
|
exit 1
|
|
fi
|
|
SRCDIR="$1"
|
|
shift
|
|
;;
|
|
--results-tarball)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--results-tarball needs an argument"
|
|
exit 1
|
|
fi
|
|
RESULTSTARBALL="$1"
|
|
shift
|
|
;;
|
|
--src-tarball)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--src-tarball needs an argument"
|
|
exit 1
|
|
fi
|
|
SRCTARBALL="$1"
|
|
shift
|
|
;;
|
|
--docker)
|
|
DOCKER=yes
|
|
DOCKER_ENGINE=auto
|
|
shift
|
|
;;
|
|
--docker-engine)
|
|
shift
|
|
if [ $# -eq 0 ]; then
|
|
echo "--docker-engine needs an argument"
|
|
exit 1
|
|
fi
|
|
DOCKER=yes
|
|
DOCKER_ENGINE="$1"
|
|
shift
|
|
;;
|
|
*)
|
|
echo "Unexpected argument '$1'"
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ -z "$COVERITY_TOKEN" ]; then
|
|
COVERITY_TOKEN="$(git config coverity.token)"
|
|
fi
|
|
if [ -z "$COVERITY_TOKEN" ]; then
|
|
echo "COVERITY_TOKEN environment variable not set"
|
|
exit 1
|
|
fi
|
|
|
|
if [ "$DRYRUN" = check ]; then
|
|
upload_permitted
|
|
exit $?
|
|
fi
|
|
|
|
if [ -z "$COVERITY_BUILD_CMD" ]; then
|
|
NPROC=$(nproc)
|
|
COVERITY_BUILD_CMD="make -j$NPROC"
|
|
echo "COVERITY_BUILD_CMD: using default '$COVERITY_BUILD_CMD'"
|
|
fi
|
|
|
|
if [ -z "$COVERITY_TOOL_BASE" ]; then
|
|
echo "COVERITY_TOOL_BASE: using default /tmp/coverity-tools"
|
|
COVERITY_TOOL_BASE=/tmp/coverity-tools
|
|
fi
|
|
|
|
if [ -z "$SRCDIR" ]; then
|
|
SRCDIR="$PWD"
|
|
fi
|
|
|
|
TARBALL=cov-int.tar.xz
|
|
|
|
if [ "$UPDATE" = only ]; then
|
|
# Just do the tools update; we don't need to check whether
|
|
# we are in a source tree or have upload rights for this,
|
|
# so do it before some of the command line and source tree checks.
|
|
|
|
if [ "$DOCKER" = yes ] && [ ! -z "$SRCTARBALL" ]; then
|
|
echo --update-tools-only --docker is incompatible with --src-tarball.
|
|
exit 1
|
|
fi
|
|
|
|
update_coverity_tools
|
|
exit 0
|
|
fi
|
|
|
|
if [ ! -e "$SRCDIR" ]; then
|
|
mkdir "$SRCDIR"
|
|
fi
|
|
|
|
cd "$SRCDIR"
|
|
|
|
if [ ! -z "$SRCTARBALL" ]; then
|
|
echo "Untarring source tarball into $SRCDIR..."
|
|
tar xvf "$SRCTARBALL"
|
|
fi
|
|
|
|
echo "Checking this is a QEMU source tree..."
|
|
if ! [ -e "$SRCDIR/VERSION" ]; then
|
|
echo "Not in a QEMU source tree?"
|
|
exit 1
|
|
fi
|
|
|
|
# Fill in defaults used by the non-update-only process
|
|
if [ -z "$VERSION" ]; then
|
|
VERSION="$(git describe --always HEAD)"
|
|
fi
|
|
|
|
if [ -z "$DESCRIPTION" ]; then
|
|
DESCRIPTION="$(git rev-parse HEAD)"
|
|
fi
|
|
|
|
if [ -z "$COVERITY_EMAIL" ]; then
|
|
COVERITY_EMAIL="$(git config coverity.email)"
|
|
fi
|
|
if [ -z "$COVERITY_EMAIL" ]; then
|
|
COVERITY_EMAIL="$(git config user.email)"
|
|
fi
|
|
|
|
# Otherwise, continue with the full build and upload process.
|
|
|
|
check_upload_permissions
|
|
|
|
if [ "$UPDATE" != no ]; then
|
|
update_coverity_tools
|
|
fi
|
|
|
|
# Run ourselves inside docker if that's what the user wants
|
|
if [ "$DOCKER" = yes ]; then
|
|
# Put the Coverity token into a temporary file that only
|
|
# we have read access to, and then pass it to docker build
|
|
# using a volume. A volume is enough for the token not to
|
|
# leak into the Docker image.
|
|
umask 077
|
|
SECRETDIR=$(mktemp -d)
|
|
if [ -z "$SECRETDIR" ]; then
|
|
echo "Failed to create temporary directory"
|
|
exit 1
|
|
fi
|
|
trap 'rm -rf "$SECRETDIR"' INT TERM EXIT
|
|
echo "Created temporary directory $SECRETDIR"
|
|
SECRET="$SECRETDIR/token"
|
|
echo "$COVERITY_TOKEN" > "$SECRET"
|
|
echo "Archiving sources to be analyzed..."
|
|
./scripts/archive-source.sh "$SECRETDIR/qemu-sources.tgz"
|
|
ARGS="--no-update-tools"
|
|
if [ "$DRYRUN" = yes ]; then
|
|
ARGS="$ARGS --dry-run"
|
|
fi
|
|
echo "Running scanner..."
|
|
# If we need to capture the output tarball, get the inner run to
|
|
# save it to the secrets directory so we can copy it out before the
|
|
# directory is cleaned up.
|
|
if [ ! -z "$RESULTSTARBALL" ]; then
|
|
ARGS="$ARGS --results-tarball /work/cov-int.tar.xz"
|
|
fi
|
|
# Arrange for this docker run to get access to the sources with -v.
|
|
# We pass through all the configuration from the outer script to the inner.
|
|
export COVERITY_EMAIL COVERITY_BUILD_CMD
|
|
tests/docker/docker.py run -it --env COVERITY_EMAIL --env COVERITY_BUILD_CMD \
|
|
-v "$SECRETDIR:/work" coverity-scanner \
|
|
./run-coverity-scan --version "$VERSION" \
|
|
--description "$DESCRIPTION" $ARGS --tokenfile /work/token \
|
|
--srcdir /qemu --src-tarball /work/qemu-sources.tgz
|
|
if [ ! -z "$RESULTSTARBALL" ]; then
|
|
echo "Copying results tarball to $RESULTSTARBALL..."
|
|
cp "$SECRETDIR/cov-int.tar.xz" "$RESULTSTARBALL"
|
|
fi
|
|
echo "Docker work complete."
|
|
exit 0
|
|
fi
|
|
|
|
TOOLBIN="$(cd "$COVERITY_TOOL_BASE" && echo $PWD/coverity_tool/cov-analysis-*/bin)"
|
|
|
|
if ! test -x "$TOOLBIN/cov-build"; then
|
|
echo "Couldn't find cov-build in the coverity build-tool directory??"
|
|
exit 1
|
|
fi
|
|
|
|
export PATH="$TOOLBIN:$PATH"
|
|
|
|
cd "$SRCDIR"
|
|
|
|
echo "Nuking build directory..."
|
|
rm -rf +build
|
|
mkdir +build
|
|
cd +build
|
|
|
|
echo "Configuring..."
|
|
# We configure with a fixed set of enables here to ensure that we don't
|
|
# accidentally reduce the scope of the analysis by doing the build on
|
|
# the system that's missing a dependency that we need to build part of
|
|
# the codebase.
|
|
../configure --disable-modules --enable-sdl --enable-gtk \
|
|
--enable-opengl --enable-vte --enable-gnutls \
|
|
--enable-nettle --enable-curses --enable-curl \
|
|
--audio-drv-list=oss,alsa,sdl,pa --enable-virtfs \
|
|
--enable-vnc --enable-vnc-sasl --enable-vnc-jpeg --enable-png \
|
|
--enable-xen --enable-brlapi \
|
|
--enable-linux-aio --enable-attr \
|
|
--enable-cap-ng --enable-trace-backends=log --enable-spice --enable-rbd \
|
|
--enable-libusb --enable-usb-redir \
|
|
--enable-libiscsi --enable-libnfs --enable-seccomp \
|
|
--enable-tpm --enable-libssh --enable-lzo --enable-snappy --enable-bzip2 \
|
|
--enable-numa --enable-rdma --enable-smartcard --enable-virglrenderer \
|
|
--enable-mpath --enable-glusterfs \
|
|
--enable-virtfs --enable-zstd
|
|
|
|
echo "Running cov-build..."
|
|
rm -rf cov-int
|
|
mkdir cov-int
|
|
cov-build --dir cov-int $COVERITY_BUILD_CMD
|
|
|
|
echo "Creating results tarball..."
|
|
tar cvf - cov-int | xz > "$TARBALL"
|
|
|
|
if [ ! -z "$RESULTSTARBALL" ]; then
|
|
echo "Copying results tarball to $RESULTSTARBALL..."
|
|
cp "$TARBALL" "$RESULTSTARBALL"
|
|
fi
|
|
|
|
echo "Uploading results tarball..."
|
|
|
|
if [ "$DRYRUN" = yes ]; then
|
|
echo "Dry run only, not uploading $TARBALL"
|
|
exit 0
|
|
fi
|
|
|
|
curl --form token="$COVERITY_TOKEN" --form email="$COVERITY_EMAIL" \
|
|
--form file=@"$TARBALL" --form version="$VERSION" \
|
|
--form description="$DESCRIPTION" \
|
|
https://scan.coverity.com/builds?project="$PROJNAME"
|
|
|
|
echo "Done."
|