runner: New test runner

This is a new test runner to replace piglit. Piglit has been very
useful as a test runner, but certain improvements have been very
difficult if possible at all in a generic test running framework.

Important improvements over piglit:

- Faster to launch. Being able to make assumptions about what we're
  executing makes it possible to save significant amounts of time. For
  example, a testlist file's line "igt@somebinary@somesubtest" already
  has all the information we need to construct the correct command
  line to execute that particular subtest, instead of listing all
  subtests of all test binaries and mapping them to command
  lines. Same goes for the regexp filters command line flags -t and
  -x; If we use -x somebinaryname, we don't need to list subtests from
  somebinaryname, we already know none of them will get executed.

- Logs of incomplete tests. Piglit collects test output to memory and
  dumps them to a file when the test is complete. The new runner
  writes all output to disk immediately.

- Ability to execute multiple subtests in one binary execution. This
  was possible with piglit, but its semantics made it very hard to
  implement in practice. For example, having a testlist file not only
  selected a subset of tests to run, but also mandated that they be
  executed in the same order.

- Flexible timeout support. Instead of mandating a time tests cannot
  exceed, the new runner has a timeout on inactivity. Activity is
  any output on the test's stdout or stderr, or kernel activity via
  /dev/kmsg.

The runner is fairly piglit compatible. The command line is very
similar, with a few additions. IGT_TEST_ROOT environment flag is still
supported, but can also be set via command line (in place of igt.py in
piglit command line).

The results are a set of log files, processed into a piglit-compatible
results.json file (BZ2 compression TODO). There are some new fields in
the json for extra information:

- "igt-version" contains the IGT version line. In
  multiple-subtests-mode the version information is only printed once,
  so it needs to be duplicated to all subtest results this way.
- "dmesg-warnings" contains the dmesg lines that triggered a
  dmesg-warn/dmesg-fail state.
- Runtime information will be different. Piglit takes a timestamp at
  the beginning and at the end of execution for runtime. The new
  runner uses the subtest output text. The binary execution time will
  also be included; The key "igt@somebinary" will have the runtime of
  the binary "somebinary", whereas "igt@somebinary@a" etc will have
  the runtime of the subtests. Substracting the subtest runtimes from
  the binary runtime yields the total time spent doing setup in
  igt_fixture blocks.

v2:
 - use clock handling from igt_core instead of copypaste
 - install results binary
 - less magic numbers
 - scanf doesn't give empty strings after all
 - use designated array initialization with _F_JOURNAL and pals
 - add more comments to dump_dmesg
 - use signal in kill_child instead of bool
 - use more 'usual' return values for execute_entry
 - use signal number instead of magic integers
 - use IGT_EXIT_INVALID instead of magic 79
 - properly remove files in clear_test_result_directory()
 - remove magic numbers
 - warn if results directory contains extra files
 - fix naming in matches_any
 - construct command line in a cleaner way in add_subtests()
 - clarify error in filtered_job_list
 - replace single string fprintfs with fputs
 - use getline() more sanely
 - refactor string constants to a shared header
 - explain non-nul-terminated string handling in resultgen
 - saner line parsing
 - rename gen_igt_name to generate_piglit_name
 - clean up parse_result_string
 - explain what we're parsing in resultgen
 - explain the runtime accumulation in add_runtime
 - refactor result overriding
 - stop passing needle sizes to find_line functions
 - refactor stdout/stderr parsing
 - fix regex whitelist compiling
 - add TODO for suppressions.txt
 - refactor dmesg parsing
 - fill_from_journal returns void
 - explain missing result fields with TODO comments
 - log_level parsing with typeof
 - pass stdout/stderr to usage() instead of a bool
 - fix absolute_path overflow
 - refactor settings serialization
 - remove maybe_strdup function
 - refactor job list serialization
 - refactor resuming, add new resume binary
 - catch mmap failure correctly

v3:
 - rename runner to igt_runner, etc
 - add meson option for building the runner
 - use UPPER_CASE names for string constants
 - add TODO comments for future refactoring
 - add a midding close()
 - const correctness where applicable
 - also build with autotools

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>

1 files changed, 502 insertions, 0 deletions

diff --git a/runner/settings.c b/runner/settings.c
new file mode 100644
index 00000000..31754a12
--- /dev/null
+++ b/runner/settings.c
@@ -0,0 +1,502 @@
+#include "settings.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <getopt.h>
+#include <libgen.h>
+#include <limits.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+enum {
+	OPT_ABORT_ON_ERROR,
+	OPT_TEST_LIST,
+	OPT_IGNORE_MISSING,
+	OPT_HELP = 'h',
+	OPT_NAME = 'n',
+	OPT_DRY_RUN = 'd',
+	OPT_INCLUDE = 't',
+	OPT_EXCLUDE = 'x',
+	OPT_SYNC = 's',
+	OPT_LOG_LEVEL = 'l',
+	OPT_OVERWRITE = 'o',
+	OPT_MULTIPLE = 'm',
+	OPT_TIMEOUT = 'c',
+	OPT_WATCHDOG = 'g',
+};
+
+static struct {
+	int level;
+	const char *name;
+} log_levels[] = {
+	{ LOG_LEVEL_NORMAL, "normal" },
+	{ LOG_LEVEL_QUIET, "quiet" },
+	{ LOG_LEVEL_VERBOSE, "verbose" },
+	{ 0, 0 },
+};
+
+static bool set_log_level(struct settings* settings, const char *level)
+{
+	typeof(*log_levels) *it;
+
+	for (it = log_levels; it->name; it++) {
+		if (!strcmp(level, it->name)) {
+			settings->log_level = it->level;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static const char *usage_str =
+	"usage: runner [options] [test_root] results-path\n\n"
+	"Options:\n"
+	" Piglit compatible:\n"
+	"  -h, --help            Show this help message and exit\n"
+	"  -n <test name>, --name <test name>\n"
+	"                        Name of this test run\n"
+	"  -d, --dry-run         Do not execute the tests\n"
+	"  -t <regex>, --include-tests <regex>\n"
+	"                        Run only matching tests (can be used more than once)\n"
+	"  -x <regex>, --exclude-tests <regex>\n"
+	"                        Exclude matching tests (can be used more than once)\n"
+	"  --abort-on-monitored-error\n"
+	"                        Abort execution when a fatal condition is detected.\n"
+	"                        <TODO>\n"
+	"  -s, --sync            Sync results to disk after every test\n"
+	"  -l {quiet,verbose,dummy}, --log-level {quiet,verbose,dummy}\n"
+	"                        Set the logger verbosity level\n"
+	"  --test-list TEST_LIST\n"
+	"                        A file containing a list of tests to run\n"
+	"  -o, --overwrite       If the results-path already exists, delete it\n"
+	"  --ignore-missing      Ignored but accepted, for piglit compatibility\n"
+	"\n"
+	" Incompatible options:\n"
+	"  -m, --multiple-mode   Run multiple subtests in the same binary execution.\n"
+	"                        If a testlist file is given, consecutive subtests are\n"
+	"                        run in the same execution if they are from the same\n"
+	"                        binary. Note that in that case relative ordering of the\n"
+	"                        subtest execution is dictated by the test binary, not\n"
+	"                        the testlist\n"
+	"  --inactivity-timeout <seconds>\n"
+	"                        Kill the running test after <seconds> of inactivity in\n"
+	"                        the test's stdout, stderr, or dmesg\n"
+	"  --use-watchdog        Use hardware watchdog for lethal enforcement of the\n"
+	"                        above timeout. Killing the test process is still\n"
+	"                        attempted at timeout trigger.\n"
+	"  [test_root]           Directory that contains the IGT tests. The environment\n"
+	"                        variable IGT_TEST_ROOT will be used if set, overriding\n"
+	"                        this option if given.\n"
+	;
+
+static void usage(const char *extra_message, FILE *f)
+{
+	if (extra_message)
+		fprintf(f, "%s\n\n", extra_message);
+
+	fputs(usage_str, f);
+}
+
+static bool add_regex(struct regex_list *list, char *new)
+{
+	regex_t *regex;
+	size_t buflen;
+	char *buf;
+	int s;
+
+	regex = malloc(sizeof(*regex));
+
+	if ((s = regcomp(regex, new,
+			 REG_EXTENDED | REG_NOSUB)) != 0) {
+		buflen = regerror(s, regex, NULL, 0);
+		buf = malloc(buflen);
+		regerror(s, regex, buf, buflen);
+		usage(buf, stderr);
+
+		free(buf);
+		regfree(regex);
+		free(regex);
+		return false;
+	}
+
+	list->regexes = realloc(list->regexes,
+				(list->size + 1) * sizeof(*list->regexes));
+	list->regex_strings = realloc(list->regex_strings,
+				      (list->size + 1) * sizeof(*list->regex_strings));
+	list->regexes[list->size] = regex;
+	list->regex_strings[list->size] = new;
+	list->size++;
+
+	return true;
+}
+
+static void free_regexes(struct regex_list *regexes)
+{
+	size_t i;
+
+	for (i = 0; i < regexes->size; i++) {
+		free(regexes->regex_strings[i]);
+		regfree(regexes->regexes[i]);
+		free(regexes->regexes[i]);
+	}
+	free(regexes->regex_strings);
+	free(regexes->regexes);
+}
+
+static bool readable_file(char *filename)
+{
+	return !access(filename, R_OK);
+}
+
+void init_settings(struct settings *settings)
+{
+	memset(settings, 0, sizeof(*settings));
+}
+
+void free_settings(struct settings *settings)
+{
+	free(settings->test_list);
+	free(settings->name);
+	free(settings->test_root);
+	free(settings->results_path);
+
+	free_regexes(&settings->include_regexes);
+	free_regexes(&settings->exclude_regexes);
+
+	init_settings(settings);
+}
+
+bool parse_options(int argc, char **argv,
+		   struct settings *settings)
+{
+	int c;
+	char *env_test_root;
+
+	static struct option long_options[] = {
+		{"help", no_argument, NULL, OPT_HELP},
+		{"name", required_argument, NULL, OPT_NAME},
+		{"dry-run", no_argument, NULL, OPT_DRY_RUN},
+		{"include-tests", required_argument, NULL, OPT_INCLUDE},
+		{"exclude-tests", required_argument, NULL, OPT_EXCLUDE},
+		{"abort-on-monitored-error", no_argument, NULL, OPT_ABORT_ON_ERROR},
+		{"sync", no_argument, NULL, OPT_SYNC},
+		{"log-level", required_argument, NULL, OPT_LOG_LEVEL},
+		{"test-list", required_argument, NULL, OPT_TEST_LIST},
+		{"overwrite", no_argument, NULL, OPT_OVERWRITE},
+		{"ignore-missing", no_argument, NULL, OPT_IGNORE_MISSING},
+		{"multiple-mode", no_argument, NULL, OPT_MULTIPLE},
+		{"inactivity-timeout", required_argument, NULL, OPT_TIMEOUT},
+		{"use-watchdog", no_argument, NULL, OPT_WATCHDOG},
+		{ 0, 0, 0, 0},
+	};
+
+	free_settings(settings);
+
+	optind = 1;
+
+	while ((c = getopt_long(argc, argv, "hn:dt:x:sl:om", long_options, NULL)) != -1) {
+		switch (c) {
+		case OPT_HELP:
+			usage(NULL, stdout);
+			goto error;
+		case OPT_NAME:
+			settings->name = strdup(optarg);
+			break;
+		case OPT_DRY_RUN:
+			settings->dry_run = true;
+			break;
+		case OPT_INCLUDE:
+			if (!add_regex(&settings->include_regexes, strdup(optarg)))
+				goto error;
+			break;
+		case OPT_EXCLUDE:
+			if (!add_regex(&settings->exclude_regexes, strdup(optarg)))
+				goto error;
+			break;
+		case OPT_ABORT_ON_ERROR:
+			settings->abort_on_error = true;
+			break;
+		case OPT_SYNC:
+			settings->sync = true;
+			break;
+		case OPT_LOG_LEVEL:
+			if (!set_log_level(settings, optarg)) {
+				usage("Cannot parse log level", stderr);
+				goto error;
+			}
+			break;
+		case OPT_TEST_LIST:
+			settings->test_list = absolute_path(optarg);
+			break;
+		case OPT_OVERWRITE:
+			settings->overwrite = true;
+			break;
+		case OPT_IGNORE_MISSING:
+			/* Ignored, piglit compatibility */
+			break;
+		case OPT_MULTIPLE:
+			settings->multiple_mode = true;
+			break;
+		case OPT_TIMEOUT:
+			settings->inactivity_timeout = atoi(optarg);
+			break;
+		case OPT_WATCHDOG:
+			settings->use_watchdog = true;
+			break;
+		case '?':
+			usage(NULL, stderr);
+			goto error;
+		default:
+			usage("Cannot parse options", stderr);
+			goto error;
+		}
+	}
+
+	switch (argc - optind) {
+	case 2:
+		settings->test_root = absolute_path(argv[optind]);
+		++optind;
+		/* fallthrough */
+	case 1:
+		settings->results_path = absolute_path(argv[optind]);
+		break;
+	case 0:
+		usage("Results-path missing", stderr);
+		goto error;
+	default:
+		usage("Extra arguments after results-path", stderr);
+		goto error;
+	}
+
+	if ((env_test_root = getenv("IGT_TEST_ROOT")) != NULL) {
+		free(settings->test_root);
+		settings->test_root = absolute_path(env_test_root);
+	}
+
+	if (!settings->test_root) {
+		usage("Test root not set", stderr);
+		goto error;
+	}
+
+	if (!settings->name) {
+		char *name = strdup(settings->results_path);
+		settings->name = strdup(basename(name));
+		free(name);
+	}
+
+	return true;
+
+ error:
+	free_settings(settings);
+	return false;
+}
+
+bool validate_settings(struct settings *settings)
+{
+	int dirfd, fd;
+
+	if (settings->test_list && !readable_file(settings->test_list)) {
+		usage("Cannot open test-list file", stderr);
+		return false;
+	}
+
+	if (!settings->results_path) {
+		usage("No results-path set; this shouldn't happen", stderr);
+		return false;
+	}
+
+	if (!settings->test_root) {
+		usage("No test root set; this shouldn't happen", stderr);
+		return false;
+	}
+
+	dirfd = open(settings->test_root, O_DIRECTORY | O_RDONLY);
+	if (dirfd < 0) {
+		fprintf(stderr, "Test directory %s cannot be opened\n", settings->test_root);
+		return false;
+	}
+
+	fd = openat(dirfd, "test-list.txt", O_RDONLY);
+	if (fd < 0) {
+		fprintf(stderr, "Cannot open %s/test-list.txt\n", settings->test_root);
+		close(dirfd);
+		return false;
+	}
+
+	close(fd);
+	close(dirfd);
+
+	return true;
+}
+
+char *absolute_path(char *path)
+{
+	char *result = NULL;
+	char *tmppath, *tmpname;
+
+	result = realpath(path, NULL);
+	if (result != NULL)
+		return result;
+
+	tmppath = strdup(path);
+	tmpname = dirname(tmppath);
+	free(result);
+	result = realpath(tmpname, NULL);
+	free(tmppath);
+
+	if (result != NULL) {
+		char *ret;
+
+		tmppath = strdup(path);
+		tmpname = basename(tmppath);
+
+		asprintf(&ret, "%s/%s", result, tmpname);
+		free(result);
+		free(tmppath);
+		return ret;
+	}
+
+	free(result);
+	return NULL;
+}
+
+static char settings_filename[] = "metadata.txt";
+bool serialize_settings(struct settings *settings)
+{
+#define SERIALIZE_LINE(f, s, name, format) fprintf(f, "%s : " format "\n", #name, s->name)
+
+	int dirfd, fd;
+	FILE *f;
+
+	if (!settings->results_path) {
+		usage("No results-path set; this shouldn't happen", stderr);
+		return false;
+	}
+
+	if ((dirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) {
+		mkdir(settings->results_path, 0755);
+		if ((dirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) {
+			usage("Creating results-path failed", stderr);
+			return false;
+		}
+	}
+
+	if (!settings->overwrite &&
+	    faccessat(dirfd, settings_filename, F_OK, 0) == 0) {
+		usage("Settings metadata already exists and not overwriting", stderr);
+		return false;
+	}
+
+	if (settings->overwrite &&
+	    unlinkat(dirfd, settings_filename, 0) != 0 &&
+	    errno != ENOENT) {
+		usage("Error removing old settings metadata", stderr);
+		return false;
+	}
+
+	if ((fd = openat(dirfd, settings_filename, O_CREAT | O_EXCL | O_WRONLY, 0666)) < 0) {
+		char *msg;
+
+		asprintf(&msg, "Creating settings serialization file failed: %s", strerror(errno));
+		usage(msg, stderr);
+
+		free(msg);
+		close(dirfd);
+		return false;
+	}
+
+	f = fdopen(fd, "w");
+	if (!f) {
+		close(fd);
+		close(dirfd);
+		return false;
+	}
+
+	SERIALIZE_LINE(f, settings, abort_on_error, "%d");
+	if (settings->test_list)
+		SERIALIZE_LINE(f, settings, test_list, "%s");
+	if (settings->name)
+		SERIALIZE_LINE(f, settings, name, "%s");
+	SERIALIZE_LINE(f, settings, dry_run, "%d");
+	SERIALIZE_LINE(f, settings, sync, "%d");
+	SERIALIZE_LINE(f, settings, log_level, "%d");
+	SERIALIZE_LINE(f, settings, overwrite, "%d");
+	SERIALIZE_LINE(f, settings, multiple_mode, "%d");
+	SERIALIZE_LINE(f, settings, inactivity_timeout, "%d");
+	SERIALIZE_LINE(f, settings, use_watchdog, "%d");
+	SERIALIZE_LINE(f, settings, test_root, "%s");
+	SERIALIZE_LINE(f, settings, results_path, "%s");
+
+	if (settings->sync) {
+		fsync(fd);
+		fsync(dirfd);
+	}
+
+	fclose(f);
+	close(dirfd);
+	return true;
+
+#undef SERIALIZE_LINE
+}
+
+bool read_settings(struct settings *settings, int dirfd)
+{
+#define PARSE_LINE(s, name, val, field, write) \
+	if (!strcmp(name, #field)) {	       \
+		s->field = write;	       \
+		free(name);		       \
+		free(val);		       \
+		name = val = NULL;	       \
+		continue;		       \
+	}
+
+	int fd;
+	FILE *f;
+	char *name = NULL, *val = NULL;
+
+	free_settings(settings);
+
+	if ((fd = openat(dirfd, settings_filename, O_RDONLY)) < 0)
+		return false;
+
+	f = fdopen(fd, "r");
+	if (!f) {
+		close(fd);
+		return false;
+	}
+
+	while (fscanf(f, "%ms : %ms", &name, &val) == 2) {
+		int numval = atoi(val);
+		PARSE_LINE(settings, name, val, abort_on_error, numval);
+		PARSE_LINE(settings, name, val, test_list, val ? strdup(val) : NULL);
+		PARSE_LINE(settings, name, val, name, val ? strdup(val) : NULL);
+		PARSE_LINE(settings, name, val, dry_run, numval);
+		PARSE_LINE(settings, name, val, sync, numval);
+		PARSE_LINE(settings, name, val, log_level, numval);
+		PARSE_LINE(settings, name, val, overwrite, numval);
+		PARSE_LINE(settings, name, val, multiple_mode, numval);
+		PARSE_LINE(settings, name, val, inactivity_timeout, numval);
+		PARSE_LINE(settings, name, val, use_watchdog, numval);
+		PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL);
+		PARSE_LINE(settings, name, val, results_path, val ? strdup(val) : NULL);
+
+		printf("Warning: Unknown field in settings file: %s = %s\n",
+		       name, val);
+		free(name);
+		free(val);
+		name = val = NULL;
+	}
+
+	free(name);
+	free(val);
+	fclose(f);
+
+	return true;
+
+#undef PARSE_LINE
+}