run-tests.sh: Use igt_runner instead of piglit by default

The old behaviour of running everything with piglit with the script is
now behind the flag -p.

When using igt_runner and its pals, -b now takes a blacklist file á la
tests/intel-ci/blacklist.txt.

Additional useful command line flags for igt_runner, like the various
forms of --abort-on-monitored-error or --dmesg-warning-level are not
supported. The main supported form of running tests is igt_runner
directly.

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Tomi Sarvela <tomi.p.sarvela@intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>

1 files changed, 87 insertions, 23 deletions

diff --git a/scripts/run-tests.sh b/scripts/run-tests.sh
index 4209dd8c..615b7da5 100755
--- a/scripts/run-tests.sh
+++ b/scripts/run-tests.sh
@@ -27,17 +27,37 @@ ROOT="`readlink -f $ROOT/..`"
 IGT_CONFIG_PATH="`readlink -f ${IGT_CONFIG_PATH:-$HOME/.igtrc}`"
 RESULTS="$ROOT/results"
 PIGLIT=`which piglit 2> /dev/null`
+IGT_RUNNER=
+IGT_RESUME=
+USE_PIGLIT=0
+RUNNER=
+RESUME=
+BLACKLIST=
+
+function find_file # basename <possible paths>
+{
+	base=$1
+	shift
 
-if [ -z "$IGT_TEST_ROOT" ]; then
-	paths=("$ROOT/build/tests/test-list.txt"
-	       "$ROOT/tests/test-list.txt")
-	for p in "${paths[@]}"; do
-		if [ -f "$p" ]; then
-			echo "Found test list: \"$p\""
-			IGT_TEST_ROOT=$(dirname "$p")
-			break
+	while [ -n "$1" ]; do
+		if [ -f "$1/$base" ]; then
+			echo "$1/$base";
+			return 0
 		fi
+		shift
 	done
+
+	return 1
+}
+
+if [ -z "$IGT_TEST_ROOT" ]; then
+	p=$(find_file test-list.txt \
+		    "$ROOT/build/tests" \
+		    "$ROOT/tests" )
+	if [ -f "$p" ]; then
+		echo "Found test list: \"$p\"" >&2
+		IGT_TEST_ROOT=$(dirname "$p")
+	fi
 fi
 
 if [ -z "$IGT_TEST_ROOT" ]; then
@@ -48,14 +68,22 @@ fi
 
 IGT_TEST_ROOT="`readlink -f ${IGT_TEST_ROOT}`"
 
+function find_runner_binaries
+{
+	IGT_RUNNER=$(find_file igt_runner "$ROOT/build/runner" "$ROOT/runner")
+	IGT_RESUME=$(find_file igt_resume "$ROOT/build/runner" "$ROOT/runner")
+}
+
 function download_piglit {
 	git clone https://anongit.freedesktop.org/git/piglit.git "$ROOT/piglit"
 }
 
-function run_piglit # as-root <args>
+function execute_runner # as-root <runner> <args>
 {
 	local need_root=$1
 	shift
+	local runner=$1
+	shift
 	local sudo
 
 	export IGT_TEST_ROOT IGT_CONFIG_PATH
@@ -64,7 +92,7 @@ function run_piglit # as-root <args>
 		sudo="sudo --preserve-env=IGT_TEST_ROOT,IGT_CONFIG_PATH"
 	fi
 
-	$sudo $PIGLIT "$@"
+	$sudo $runner "$@"
 }
 
 function print_help {
@@ -79,19 +107,24 @@ function print_help {
 	echo "  -t <regex>      only include tests that match the regular expression"
 	echo "                  (can be used more than once)"
 	echo "  -T <filename>   run tests listed in testlist"
-	echo "                  (overrides -t and -x)"
+	echo "                  (overrides -t and -x when running with piglit)"
 	echo "  -v              enable verbose mode"
 	echo "  -x <regex>      exclude tests that match the regular expression"
 	echo "                  (can be used more than once)"
+	echo "  -b              blacklist file to use for filtering"
+	echo "                  (can be used more than once)"
+	echo "                  (not supported by Piglit)"
 	echo "  -R              resume interrupted test where the partial results"
 	echo "                  are in the directory given by -r"
 	echo "  -n              do not retry incomplete tests when resuming a"
 	echo "                  test run with -R"
+	echo "                  (only valid for Piglit)"
+	echo "  -p              use Piglit instead of igt_runner"
 	echo ""
 	echo "Useful patterns for test filtering are described in the API documentation."
 }
 
-while getopts ":dhlr:st:T:vx:Rn" opt; do
+while getopts ":dhlr:st:T:vx:Rnpb:" opt; do
 	case $opt in
 		d) download_piglit; exit ;;
 		h) print_help; exit ;;
@@ -100,10 +133,12 @@ while getopts ":dhlr:st:T:vx:Rn" opt; do
 		s) SUMMARY="html" ;;
 		t) FILTER="$FILTER -t $OPTARG" ;;
 		T) FILTER="$FILTER --test-list $OPTARG" ;;
-		v) VERBOSE="-v" ;;
+		v) VERBOSE="-l verbose" ;;
 		x) EXCLUDE="$EXCLUDE -x $OPTARG" ;;
-		R) RESUME="true" ;;
+		R) RESUME_RUN="true" ;;
 		n) NORETRY="--no-retry" ;;
+		p) USE_PIGLIT=1 ;;
+		b) BLACKLIST="$BLACKLIST -b $OPTARG" ;;
 		:)
 			echo "Option -$OPTARG requires an argument."
 			exit 1
@@ -127,25 +162,54 @@ if [ "x$PIGLIT" == "x" ]; then
 	PIGLIT="$ROOT/piglit/piglit"
 fi
 
-if [ ! -x "$PIGLIT" ]; then
-	echo "Could not find Piglit."
-	echo "Please install Piglit or use -d to download Piglit locally."
-	exit 1
+RUN_ARGS=
+RESUME_ARGS=
+LIST_ARGS=
+if [ "$USE_PIGLIT" -eq "1" ]; then
+	if [ ! -x "$PIGLIT" ]; then
+		echo "Could not find Piglit."
+		echo "Please install Piglit or use -d to download Piglit locally."
+		exit 1
+	fi
+
+	RUNNER=$PIGLIT
+	RESUME=$PIGLIT
+	RUN_ARGS="run igt --ignore-missing"
+	RESUME_ARGS="resume $NORETRY"
+	LIST_ARGS="print-cmd igt --format {name}"
+else
+	find_runner_binaries
+	if [ ! -x "$IGT_RUNNER" -o ! -x "$IGT_RESUME" ]; then
+		echo "Could not find igt_runner binaries."
+		echo "Please build the runner, or use Piglit with the -p flag."
+		exit 1
+	fi
+
+	RUNNER=$IGT_RUNNER
+	RESUME=$IGT_RESUME
+	RUN_ARGS="$BLACKLIST"
+	LIST_ARGS="-L $BLACKLIST"
 fi
 
 if [ "x$LIST_TESTS" != "x" ]; then
-	run_piglit 0 print-cmd --format "{name}" igt
+	execute_runner 0 $RUNNER $LIST_ARGS
 	exit
 fi
 
-if [ "x$RESUME" != "x" ]; then
-	run_piglit 1 resume "$RESULTS" $NORETRY
+if [ "x$RESUME_RUN" != "x" ]; then
+	execute_runner 1 $RESUME $RESUME_ARGS "$RESULTS"
 else
 	mkdir -p "$RESULTS"
-	run_piglit 1 run igt --ignore-missing -o "$RESULTS" -s $VERBOSE $EXCLUDE $FILTER
+	execute_runner 1 $RUNNER $RUN_ARGS -o -s "$RESULTS" $VERBOSE $EXCLUDE $FILTER
 fi
 
 if [ "$SUMMARY" == "html" ]; then
-	run_piglit 0 summary html --overwrite "$RESULTS/html" "$RESULTS"
+	if [ ! -x "$PIGLIT" ]; then
+		echo "Could not find Piglit, required for HTML generation."
+		echo "Please install Piglit or use -d to download Piglit locally."
+		exit 1
+	fi
+
+	execute_runner 0 $PIGLIT summary html --overwrite "$RESULTS/html" "$RESULTS"
 	echo "HTML summary has been written to $RESULTS/html/index.html"
 fi