runner: Unit tests for the runner

TODO: Unit tests for the results.json file contents.

v2:
 - Avoid writing the nul character to mock files
 - Properly set up tmpdirs

v3:
 - Restore the resume-related changes that were lost in rebase

v4:
 - Better teardown for temporary directories
 - Build with autotools

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>

10 files changed, 1070 insertions, 0 deletions

diff --git a/configure.ac b/configure.ac
index b73cc9f0..72f35994 100644
--- a/configure.ac
+++ b/configure.ac
@@ -421,6 +421,7 @@ AC_CONFIG_FILES([
 		 assembler/intel-gen4asm.pc
 		 overlay/Makefile
 		 runner/Makefile
+		 runner/testdata/Makefile
 		 ])
 
 AC_CONFIG_FILES([tools/intel_aubdump], [chmod +x tools/intel_aubdump])
diff --git a/runner/.gitignore b/runner/.gitignore
index e3919fa7..c5030e66 100644
--- a/runner/.gitignore
+++ b/runner/.gitignore
@@ -1,3 +1,4 @@
 igt_runner
 igt_resume
 igt_results
+runner_test
diff --git a/runner/Makefile.am b/runner/Makefile.am
index 9c13a83c..ddcbf86a 100644
--- a/runner/Makefile.am
+++ b/runner/Makefile.am
@@ -1,6 +1,8 @@
 
 if BUILD_RUNNER
 
+SUBDIRS = testdata
+
 runnerlib = librunner.la
 noinst_LTLIBRARIES = $(runnerlib)
 librunner_la_SOURCES =	\
@@ -28,4 +30,14 @@ AM_CFLAGS = $(JSONC_CFLAGS) \
 	-I$(srcdir)/../lib \
 	-D_GNU_SOURCE
 
+TESTS = runner_test
+check_PROGRAMS = runner_test
+runner_test_SOURCES = runner_tests.c
+runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \
+	-I$(top_srcdir)/include/drm-uapi \
+	$(DRM_CFLAGS) $(CAIRO_CFLAGS) $(LIBUDEV_CFLAGS) \
+	-I$(srcdir)/.. \
+	-I$(srcdir)/../lib \
+	-D_GNU_SOURCE
+
 endif
diff --git a/runner/meson.build b/runner/meson.build
index 9dafb312..bea1d897 100644
--- a/runner/meson.build
+++ b/runner/meson.build
@@ -9,6 +9,7 @@ runnerlib_sources = [ 'settings.c',
 runner_sources = [ 'runner.c' ]
 resume_sources = [ 'resume.c' ]
 results_sources = [ 'results.c' ]
+runner_test_sources = [ 'runner_tests.c' ]
 
 if _build_runner and jsonc.found()
 	subdir('testdata')
@@ -35,6 +36,13 @@ if _build_runner and jsonc.found()
 			     install_dir : bindir,
 			     dependencies : igt_deps)
 
+	runner_test = executable('runner_test', runner_test_sources,
+				 c_args : '-DTESTDATA_DIRECTORY="@0@"'.format(testdata_dir),
+				 link_with : runnerlib,
+				 install : false,
+				 dependencies : igt_deps)
+	test('runner', runner_test)
+
 	build_info += 'Build test runner: Yes'
 else
 	build_info += 'Build test runner: No'
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
new file mode 100644
index 00000000..7c662acc
--- /dev/null
+++ b/runner/runner_tests.c
@@ -0,0 +1,973 @@
+#include <dirent.h>
+#include <fcntl.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <unistd.h>
+
+#include "igt.h"
+
+#include "settings.h"
+#include "job_list.h"
+#include "executor.h"
+
+static char testdatadir[] = TESTDATA_DIRECTORY;
+
+static void igt_assert_eqstr(char *one, char *two)
+{
+	if (one == NULL && two == NULL)
+		return;
+
+	igt_assert_f(one != NULL && two != NULL, "Strings differ (one is NULL): %s vs %s\n", one, two);
+
+	igt_assert_f(!strcmp(one, two), "Strings differ: '%s' vs '%s'\n", one, two);
+}
+
+static void debug_print_executions(struct job_list *list)
+{
+	size_t i;
+	int k;
+
+	igt_debug("Executions:\n");
+	for (i = 0; i < list->size; i++) {
+		struct job_list_entry *entry = &list->entries[i];
+		igt_debug(" %s\n", entry->binary);
+		for (k = 0; k < entry->subtest_count; ++k) {
+			igt_debug("  %s\n", entry->subtests[k]);
+		}
+	}
+
+}
+
+static char *dump_file(int dirfd, char *name)
+{
+	int fd = openat(dirfd, name, O_RDONLY);
+	ssize_t s;
+	char *buf = malloc(256);
+
+	if (fd < 0) {
+		free(buf);
+		return NULL;
+	}
+
+	s = read(fd, buf, 255);
+	if (s < 0) {
+		free(buf);
+		return NULL;
+	}
+
+	buf[s] = '\0';
+	return buf;
+}
+
+static void job_list_filter_test(char *name, char *filterarg1, char *filterarg2,
+				 size_t expected_normal, size_t expected_multiple)
+{
+	int multiple;
+	struct settings settings;
+
+	igt_fixture
+		init_settings(&settings);
+
+	for (multiple = 0; multiple < 2; multiple++) {
+		igt_subtest_f("job-list-filters-%s-%s", name, multiple ? "multiple" : "normal") {
+			struct job_list list;
+			char *argv[] = { "runner",
+					 /* Ugly but does the trick */
+					 multiple ? "--multiple-mode" : "--sync",
+					 filterarg1, filterarg2,
+					 testdatadir,
+					 "path-to-results",
+			};
+			bool success = false;
+			size_t size;
+
+			init_job_list(&list);
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+			success = create_job_list(&list, &settings);
+			size = list.size;
+
+			if (success)
+				debug_print_executions(&list);
+
+			free_job_list(&list);
+
+			igt_assert_f(success, "Job list creation failed\n");
+			igt_assert_eq(size, multiple ? expected_multiple : expected_normal);
+		}
+	}
+
+	igt_fixture
+		free_settings(&settings);
+}
+
+static void clear_directory_fd(int dirfd)
+{
+	DIR* d;
+	struct dirent *dirent;
+
+	d = fdopendir(dirfd);
+
+	if (dirfd < 0 || d == NULL) {
+		return;
+	}
+
+	while ((dirent = readdir(d)) != NULL) {
+		if (strcmp(dirent->d_name, ".") &&
+		    strcmp(dirent->d_name, "..")) {
+			if (dirent->d_type == DT_REG) {
+				unlinkat(dirfd, dirent->d_name, 0);
+			} else if (dirent->d_type == DT_DIR) {
+				clear_directory_fd(openat(dirfd, dirent->d_name, O_DIRECTORY | O_RDONLY));
+				unlinkat(dirfd, dirent->d_name, AT_REMOVEDIR);
+			}
+		}
+	}
+
+	closedir(d);
+}
+
+static void clear_directory(char *name)
+{
+	int dirfd = open(name, O_DIRECTORY | O_RDONLY);
+	clear_directory_fd(dirfd);
+	rmdir(name);
+}
+
+static void assert_settings_equal(struct settings *one, struct settings *two)
+{
+	/*
+	 * Regex lists are not serialized, and thus won't be compared
+	 * here.
+	 */
+	igt_assert_eq(one->abort_on_error, two->abort_on_error);
+	igt_assert_eqstr(one->test_list, two->test_list);
+	igt_assert_eqstr(one->name, two->name);
+	igt_assert_eq(one->dry_run, two->dry_run);
+	igt_assert_eq(one->sync, two->sync);
+	igt_assert_eq(one->log_level, two->log_level);
+	igt_assert_eq(one->overwrite, two->overwrite);
+	igt_assert_eq(one->multiple_mode, two->multiple_mode);
+	igt_assert_eq(one->inactivity_timeout, two->inactivity_timeout);
+	igt_assert_eq(one->use_watchdog, two->use_watchdog);
+	igt_assert_eqstr(one->test_root, two->test_root);
+	igt_assert_eqstr(one->results_path, two->results_path);
+}
+
+static void assert_job_list_equal(struct job_list *one, struct job_list *two)
+{
+	size_t i, k;
+
+	igt_assert_eq(one->size, two->size);
+
+	for (i = 0; i < one->size; i++) {
+		struct job_list_entry *eone = &one->entries[i];
+		struct job_list_entry *etwo = &two->entries[i];
+
+		igt_assert_eqstr(eone->binary, etwo->binary);
+		igt_assert_eq(eone->subtest_count, etwo->subtest_count);
+
+		for (k = 0; k < eone->subtest_count; k++) {
+			igt_assert_eqstr(eone->subtests[k], etwo->subtests[k]);
+		}
+	}
+}
+
+static void assert_execution_created(int dirfd, char *name)
+{
+	int fd;
+
+	igt_assert_f((fd = openat(dirfd, name, O_RDONLY)) >= 0,
+		     "Execute didn't create %s\n", name);
+	close(fd);
+}
+
+static void assert_execution_results_exist(int dirfd)
+{
+	assert_execution_created(dirfd, "journal.txt");
+	assert_execution_created(dirfd, "out.txt");
+	assert_execution_created(dirfd, "err.txt");
+	assert_execution_created(dirfd, "dmesg.txt");
+}
+
+igt_main
+{
+	struct settings settings;
+
+	igt_fixture
+		init_settings(&settings);
+
+	igt_subtest("default-settings") {
+		char *argv[] = { "runner",
+				 "test-root-dir",
+				 "path-to-results",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+		igt_assert(!settings.abort_on_error);
+		igt_assert(!settings.test_list);
+		igt_assert_eqstr(settings.name, "path-to-results");
+		igt_assert(!settings.dry_run);
+		igt_assert_eq(settings.include_regexes.size, 0);
+		igt_assert_eq(settings.exclude_regexes.size, 0);
+		igt_assert(!settings.sync);
+		igt_assert_eq(settings.log_level, LOG_LEVEL_NORMAL);
+		igt_assert(!settings.overwrite);
+		igt_assert(!settings.multiple_mode);
+		igt_assert_eq(settings.inactivity_timeout, 0);
+		igt_assert(!settings.use_watchdog);
+		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
+		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+	}
+
+	igt_subtest_group {
+		char *cwd;
+		char *path;
+
+		igt_fixture {
+			igt_require((cwd = realpath(".", NULL)) != NULL);
+			path = NULL;
+		}
+
+		igt_subtest("absolute-path-converter") {
+			struct {
+				char *path;
+				bool null;
+			} data[] = { { "simple-name", false },
+				     { "foo/bar", true },
+				     { ".", false },
+			};
+			size_t i;
+
+			for (i = 0; i < ARRAY_SIZE(data); i++) {
+				free(path);
+				path = absolute_path(data[i].path);
+				if (data[i].null) {
+					igt_assert(path == NULL);
+					continue;
+				}
+
+				igt_assert(path[0] == '/');
+				igt_debug("Got path %s for %s\n", path, data[i].path);
+				igt_assert(strstr(path, cwd) == path);
+				if (strcmp(data[i].path, ".")) {
+					igt_assert(strstr(path, data[i].path) != NULL);
+				}
+			}
+		}
+
+		igt_fixture {
+			free(cwd);
+			free(path);
+		}
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		char tmptestlist[] = "tmp.testlist";
+		char pathtotestlist[64];
+		char *path;
+
+		igt_fixture {
+			int dirfd, fd;
+
+			path = NULL;
+
+			igt_require(mkdtemp(dirname) != NULL);
+			igt_require((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0);
+			igt_require((fd = openat(dirfd, tmptestlist, O_CREAT | O_EXCL | O_WRONLY, 0660)) >= 0);
+			close(fd);
+			close(dirfd);
+
+			strcpy(pathtotestlist, dirname);
+			strcat(pathtotestlist, "/");
+			strcat(pathtotestlist, tmptestlist);
+		}
+
+		igt_subtest("absolute-path-usage") {
+			char *argv[] = { "runner",
+					 "--test-list", pathtotestlist,
+					 testdatadir,
+					 dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+			path = realpath(testdatadir, NULL);
+			igt_assert(path != NULL);
+			igt_assert_eqstr(settings.test_root, path);
+			free(path);
+			path = realpath(dirname, NULL);
+			igt_assert(path != NULL);
+			igt_assert_eqstr(settings.results_path, path);
+			free(path);
+			path = realpath(pathtotestlist, NULL);
+			igt_assert(path != NULL);
+			igt_assert_eqstr(settings.test_list, path);
+		}
+
+		igt_fixture {
+			int dirfd;
+
+			igt_require((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0);
+			unlinkat(dirfd, tmptestlist, 0);
+			close(dirfd);
+			rmdir(dirname);
+
+			free(path);
+		}
+	}
+
+	igt_subtest("environment-overrides-test-root-flag") {
+		char *argv[] = { "runner",
+				 "test-root-dir",
+				 "path-to-results",
+		};
+
+		setenv("IGT_TEST_ROOT", testdatadir, 1);
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+		igt_assert(!settings.abort_on_error);
+		igt_assert(!settings.test_list);
+		igt_assert_eqstr(settings.name, "path-to-results");
+		igt_assert(!settings.dry_run);
+		igt_assert_eq(settings.include_regexes.size, 0);
+		igt_assert_eq(settings.exclude_regexes.size, 0);
+		igt_assert(!settings.sync);
+		igt_assert_eq(settings.log_level, LOG_LEVEL_NORMAL);
+		igt_assert(!settings.overwrite);
+		igt_assert(!settings.multiple_mode);
+		igt_assert_eq(settings.inactivity_timeout, 0);
+		igt_assert(!settings.use_watchdog);
+		igt_assert(strstr(settings.test_root, testdatadir) != NULL);
+		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+	}
+
+	igt_fixture {
+		unsetenv("IGT_TEST_ROOT");
+	}
+
+	igt_subtest("parse-all-settings") {
+		char *argv[] = { "runner",
+				 "-n", "foo",
+				 "--abort-on-monitored-error",
+				 "--test-list", "path-to-test-list",
+				 "--ignore-missing",
+				 "--dry-run",
+				 "-t", "pattern1",
+				 "-t", "pattern2",
+				 "-x", "xpattern1",
+				 "-x", "xpattern2",
+				 "-s",
+				 "-l", "verbose",
+				 "--overwrite",
+				 "--multiple-mode",
+				 "--inactivity-timeout", "27",
+				 "--use-watchdog",
+				 "test-root-dir",
+				 "path-to-results",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+		igt_assert(settings.abort_on_error);
+		igt_assert(strstr(settings.test_list, "path-to-test-list") != NULL);
+		igt_assert_eqstr(settings.name, "foo");
+		igt_assert(settings.dry_run);
+		igt_assert_eq(settings.include_regexes.size, 2);
+		igt_assert_eqstr(settings.include_regexes.regex_strings[0], "pattern1");
+		igt_assert_eqstr(settings.include_regexes.regex_strings[1], "pattern2");
+		igt_assert_eq(settings.exclude_regexes.size, 2);
+		igt_assert_eqstr(settings.exclude_regexes.regex_strings[0], "xpattern1");
+		igt_assert_eqstr(settings.exclude_regexes.regex_strings[1], "xpattern2");
+		igt_assert(settings.sync);
+		igt_assert_eq(settings.log_level, LOG_LEVEL_VERBOSE);
+		igt_assert(settings.overwrite);
+		igt_assert(settings.multiple_mode);
+		igt_assert_eq(settings.inactivity_timeout, 27);
+		igt_assert(settings.use_watchdog);
+		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
+		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+	}
+
+	igt_subtest("invalid-option") {
+		char *argv[] = { "runner",
+				 "--no-such-option",
+				 "test-root-dir",
+				 "results-path",
+		};
+
+		igt_assert(!parse_options(ARRAY_SIZE(argv), argv, &settings));
+	}
+
+	igt_subtest("paths-missing") {
+		char *argv[] = { "runner",
+				 "-o",
+		};
+		igt_assert(!parse_options(ARRAY_SIZE(argv), argv, &settings));
+	}
+
+	igt_subtest("log-levels") {
+		char *argv[] = { "runner",
+				 "-l", "normal",
+				 "test-root-dir",
+				 "results-path",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+		igt_assert_eq(settings.log_level, LOG_LEVEL_NORMAL);
+
+		argv[2] = "quiet";
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+		igt_assert_eq(settings.log_level, LOG_LEVEL_QUIET);
+
+		argv[2] = "verbose";
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+		igt_assert_eq(settings.log_level, LOG_LEVEL_VERBOSE);
+	}
+
+	igt_subtest("parse-clears-old-data") {
+		char *argv[] = { "runner",
+				 "-n", "foo",
+				 "--dry-run",
+				 "test-root-dir",
+				 "results-path",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+		igt_assert_eqstr(settings.name, "foo");
+		igt_assert(settings.dry_run);
+		igt_assert(!settings.test_list);
+		igt_assert(!settings.sync);
+
+		argv[1] = "--test-list";
+		argv[3] = "--sync";
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+		igt_assert_eqstr(settings.name, "results-path");
+		igt_assert(!settings.dry_run);
+		igt_assert(strstr(settings.test_list, "foo") != NULL);
+		igt_assert(settings.sync);
+	}
+
+	igt_subtest_group {
+		char filename[] = "tmplistXXXXXX";
+		int fd = -1;
+
+		igt_fixture {
+			igt_require((fd = mkstemp(filename)) >= 0);
+			close(fd);
+		}
+
+		igt_subtest("validate-ok") {
+			char *argv[] = { "runner",
+					 "--test-list", filename,
+					 testdatadir,
+					 "path-to-results",
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+			igt_assert(validate_settings(&settings));
+		}
+
+		igt_fixture {
+			unlink(filename);
+		}
+	}
+
+	igt_subtest("validate-no-test-list") {
+		char *nosuchfile = "no-such-file";
+		char *argv[] = { "runner",
+				 "--test-list", nosuchfile,
+				 testdatadir,
+				 "path-to-results",
+		};
+
+		igt_assert_lt(open(nosuchfile, O_RDONLY), 0);
+		igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+		igt_assert(!validate_settings(&settings));
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		struct job_list list;
+
+		igt_fixture {
+			igt_require(mkdtemp(dirname) != NULL);
+			init_job_list(&list);
+		}
+
+		igt_subtest("job-list-no-test-list-txt") {
+			char *argv[] = { "runner",
+					 dirname,
+					 "path-to-results",
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+			igt_assert(!create_job_list(&list, &settings));
+		}
+
+		igt_fixture {
+			rmdir(dirname);
+			free_job_list(&list);
+		}
+	}
+
+	job_list_filter_test("nofilters", "-n", "placeholderargs", 5, 3);
+	job_list_filter_test("binary-include", "-t", "successtest", 2, 1);
+	job_list_filter_test("binary-exclude", "-x", "successtest", 3, 2);
+	job_list_filter_test("subtest-include", "-t", "first-subtest", 1, 1);
+	job_list_filter_test("subtest-exclude", "-x", "second-subtest", 4, 3);
+
+	igt_subtest_group {
+		char filename[] = "tmplistXXXXXX";
+		char testlisttext[] = "igt@successtest@first-subtest\n"
+			"igt@successtest@second-subtest\n"
+			"igt@nosubtests\n";
+		int fd = -1, multiple;
+		struct job_list list;
+
+		igt_fixture {
+			igt_require((fd = mkstemp(filename)) >= 0);
+			igt_require(write(fd, testlisttext, strlen(testlisttext)) == strlen(testlisttext));
+			close(fd);
+			init_job_list(&list);
+		}
+
+		for (multiple = 0; multiple < 2; multiple++) {
+			igt_subtest_f("job-list-testlist-%s", multiple ? "multiple" : "normal") {
+				char *argv[] = { "runner",
+						 "--test-list", filename,
+						 multiple ? "--multiple-mode" : "--sync",
+						 testdatadir,
+						 "path-to-results",
+				};
+
+				igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+				igt_assert(create_job_list(&list, &settings));
+
+				igt_assert_eq(list.size, multiple ? 2 : 3);
+
+				igt_assert_eqstr(list.entries[0].binary, "successtest");
+				if (!multiple) igt_assert_eqstr(list.entries[1].binary, "successtest");
+				igt_assert_eqstr(list.entries[multiple ? 1 : 2].binary, "nosubtests");
+
+				igt_assert_eq(list.entries[0].subtest_count, multiple ? 2 : 1);
+				igt_assert_eq(list.entries[1].subtest_count, multiple ? 0 : 1);
+				if (!multiple) igt_assert_eq(list.entries[2].subtest_count, 0);
+
+				igt_assert_eqstr(list.entries[0].subtests[0], "first-subtest");
+				igt_assert_eqstr(list.entries[multiple ? 0 : 1].subtests[multiple ? 1 : 0], "second-subtest");
+			}
+		}
+
+		igt_fixture {
+			unlink(filename);
+			free_job_list(&list);
+		}
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		int dirfd = -1, fd = -1;
+		struct settings cmp_settings;
+
+		igt_fixture {
+			igt_require(mkdtemp(dirname) != NULL);
+			rmdir(dirname);
+			init_settings(&cmp_settings);
+		}
+
+		igt_subtest("settings-serialize") {
+			char *argv[] = { "runner",
+					 testdatadir,
+					 dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+
+			igt_assert(serialize_settings(&settings));
+
+			dirfd = open(dirname, O_DIRECTORY, O_RDONLY);
+			igt_assert_f(dirfd >= 0, "Serialization did not create the results directory\n");
+
+			igt_assert_f((fd = openat(dirfd, "metadata.txt", O_RDONLY)),
+				     "Opening %s/metadata.txt failed\n", dirname);
+			close(fd);
+
+			igt_assert_f(read_settings(&cmp_settings, dirfd), "Reading settings failed\n");
+			assert_settings_equal(&settings, &cmp_settings);
+		}
+
+		igt_fixture {
+			close(fd);
+			close(dirfd);
+			clear_directory(dirname);
+			free_settings(&cmp_settings);
+		}
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		int dirfd = -1, fd = -1;
+		struct job_list list, cmp_list;
+		int multiple;
+
+		igt_fixture {
+			init_job_list(&list);
+			init_job_list(&cmp_list);
+			igt_require(mkdtemp(dirname) != NULL);
+			rmdir(dirname);
+		}
+
+		for (multiple = 0; multiple < 2; multiple++) {
+			igt_subtest_f("job-list-serialize-%s", multiple ? "multiple" : "normal") {
+				char *argv[] = { "runner",
+						 /* Ugly */
+						 multiple ? "--multiple-mode" : "--sync",
+						 testdatadir,
+						 dirname,
+				};
+
+				igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+				igt_assert(create_job_list(&list, &settings));
+
+				igt_assert(serialize_settings(&settings));
+				igt_assert(serialize_job_list(&list, &settings));
+
+				dirfd = open(dirname, O_DIRECTORY, O_RDONLY);
+				igt_assert_f(dirfd >= 0, "Serialization did not create the results directory\n");
+
+				igt_assert_f((fd = openat(dirfd, "joblist.txt", O_RDONLY)) >= 0,
+					     "Opening %s/joblist.txt failed\n", dirname);
+				close(fd);
+				fd = -1;
+
+				igt_assert_f(read_job_list(&cmp_list, dirfd), "Reading job list failed\n");
+				assert_job_list_equal(&list, &cmp_list);
+			}
+
+			igt_fixture {
+				close(fd);
+				close(dirfd);
+				clear_directory(dirname);
+				free_job_list(&cmp_list);
+				free_job_list(&list);
+			}
+		}
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		struct job_list list;
+		int dirfd = -1, fd = -1;
+
+		igt_fixture {
+			init_job_list(&list);
+			igt_require(mkdtemp(dirname) != NULL);
+			rmdir(dirname);
+		}
+
+		igt_subtest("execute-initialize-new-run") {
+			struct execute_state state;
+			char *argv[] = { "runner",
+					 testdatadir,
+					 dirname,
+			};
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+			igt_assert(create_job_list(&list, &settings));
+
+			igt_assert(initialize_execute_state(&state, &settings, &list));
+
+			igt_assert_eq(state.next, 0);
+			igt_assert_eq(list.size, 5);
+			igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+				     "Execute state initialization didn't create the results directory.\n");
+			igt_assert_f((fd = openat(dirfd, "metadata.txt", O_RDONLY)) >= 0,
+				     "Execute state initialization didn't serialize settings.\n");
+			close(fd);
+			igt_assert_f((fd = openat(dirfd, "joblist.txt", O_RDONLY)) >= 0,
+				     "Execute state initialization didn't serialize the job list.\n");
+			close(fd);
+			igt_assert_f((fd = openat(dirfd, "journal.txt", O_RDONLY)) < 0,
+				     "Execute state initialization created a journal.\n");
+			igt_assert_f((fd = openat(dirfd, "uname.txt", O_RDONLY)) < 0,
+				     "Execute state initialization created uname.txt.\n");
+		}
+
+		igt_fixture {
+			close(fd);
+			close(dirfd);
+			clear_directory(dirname);
+			free_job_list(&list);
+		}
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		struct job_list list;
+		int dirfd = -1, subdirfd = -1, fd = -1;
+
+		igt_fixture {
+			init_job_list(&list);
+			igt_require(mkdtemp(dirname) != NULL);
+		}
+
+		igt_subtest("execute-initialize-subtest-started") {
+			struct execute_state state;
+			char *argv[] = { "runner",
+					 "--multiple-mode",
+					 "-t", "successtest",
+					 testdatadir,
+					 dirname,
+			};
+			char journaltext[] = "first-subtest\n";
+			char excludestring[] = "!first-subtest";
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+			igt_assert(create_job_list(&list, &settings));
+			igt_assert(list.size == 1);
+			igt_assert(list.entries[0].subtest_count == 0);
+
+			igt_assert(serialize_settings(&settings));
+			igt_assert(serialize_job_list(&list, &settings));
+
+			igt_assert((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0);
+			igt_assert(mkdirat(dirfd, "0", 0770) == 0);
+			igt_assert((subdirfd = openat(dirfd, "0", O_DIRECTORY | O_RDONLY)) >= 0);
+			igt_assert((fd = openat(subdirfd, "journal.txt", O_CREAT | O_WRONLY | O_EXCL, 0660)) >= 0);
+			igt_assert(write(fd, journaltext, strlen(journaltext)) == strlen(journaltext));
+
+			free_job_list(&list);
+			free_settings(&settings);
+			igt_assert(initialize_execute_state_from_resume(dirfd, &state, &settings, &list));
+
+			igt_assert_eq(state.next, 0);
+			igt_assert_eq(list.size, 1);
+			igt_assert_eq(list.entries[0].subtest_count, 2);
+			igt_assert_eqstr(list.entries[0].subtests[0], "*");
+			igt_assert_eqstr(list.entries[0].subtests[1], excludestring);
+		}
+
+		igt_fixture {
+			close(fd);
+			close(subdirfd);
+			close(dirfd);
+			clear_directory(dirname);
+			free_job_list(&list);
+		}
+	}
+
+	igt_subtest_group {
+		char dirname[] = "tmpdirXXXXXX";
+		struct job_list list;
+		int dirfd = -1, subdirfd = -1, fd = -1;
+
+		igt_fixture {
+			init_job_list(&list);
+			igt_require(mkdtemp(dirname) != NULL);
+		}
+
+		igt_subtest("execute-initialize-subtests-complete") {
+			struct execute_state state;
+			char *argv[] = { "runner",
+					 "--multiple-mode",
+					 testdatadir,
+					 dirname,
+			};
+			char journaltext[] = "first-subtest\nsecond-subtest\nexit:0\n";
+
+			igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+			igt_assert(create_job_list(&list, &settings));
+			igt_assert(list.size == 3);
+
+			if (!strcmp(list.entries[0].binary, "no-subtests")) {
+				struct job_list_entry tmp = list.entries[0];
+				list.entries[0] = list.entries[1];
+				list.entries[1] = tmp;
+			}
+
+			igt_assert(list.entries[0].subtest_count == 0);
+
+			igt_assert(serialize_settings(&settings));
+			igt_assert(serialize_job_list(&list, &settings));
+
+			igt_assert_lte(0, dirfd = open(dirname, O_DIRECTORY | O_RDONLY));
+			igt_assert_eq(mkdirat(dirfd, "0", 0770), 0);
+			igt_assert((subdirfd = openat(dirfd, "0", O_DIRECTORY | O_RDONLY)) >= 0);
+			igt_assert_lte(0, fd = openat(subdirfd, "journal.txt", O_CREAT | O_WRONLY | O_EXCL, 0660));
+			igt_assert_eq(write(fd, journaltext, sizeof(journaltext)), sizeof(journaltext));
+
+			free_job_list(&list);
+			free_settings(&settings);
+			igt_assert(initialize_execute_state_from_resume(dirfd, &state, &settings, &list));
+
+			igt_assert_eq(state.next, 1);
+			igt_assert_eq(list.size, 3);
+		}
+
+		igt_fixture {
+			close(fd);
+			close(subdirfd);
+			close(dirfd);
+			clear_directory(dirname);
+			free_job_list(&list);
+		}
+	}
+
+	igt_subtest_group {
+		struct job_list list;
+		int dirfd = -1, subdirfd = -1, fd = -1;
+		int multiple;
+
+		igt_fixture {
+			init_job_list(&list);
+		}
+
+		for (multiple = 0; multiple < 2; multiple++) {
+			char dirname[] = "tmpdirXXXXXX";
+
+			igt_fixture {
+				igt_require(mkdtemp(dirname) != NULL);
+				rmdir(dirname);
+			}
+
+			igt_subtest_f("execute-subtests-%s", multiple ? "multiple" : "normal") {
+				struct execute_state state;
+				char *argv[] = { "runner",
+						 multiple ? "--multiple-mode" : "--sync",
+						 "-t", "-subtest",
+						 testdatadir,
+						 dirname,
+				};
+				char testdirname[16];
+				size_t expected_tests = multiple ? 2 : 3;
+				size_t i;
+
+				igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+				igt_assert(create_job_list(&list, &settings));
+				igt_assert(initialize_execute_state(&state, &settings, &list));
+
+				igt_assert(execute(&state, &settings, &list));
+				igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+					     "Execute didn't create the results directory\n");
+
+				igt_assert_f((fd = openat(dirfd, "uname.txt", O_RDONLY)) >= 0,
+					     "Execute didn't create uname.txt\n");
+				close(fd);
+				fd = -1;
+
+				for (i = 0; i < expected_tests; i++) {
+					snprintf(testdirname, 16, "%zd", i);
+
+					igt_assert_f((subdirfd = openat(dirfd, testdirname, O_DIRECTORY | O_RDONLY)) >= 0,
+						     "Execute didn't create result directory '%s'\n", testdirname);
+					assert_execution_results_exist(subdirfd);
+					close(subdirfd);
+				}
+
+				snprintf(testdirname, 16, "%zd", expected_tests);
+				igt_assert_f((subdirfd = openat(dirfd, testdirname, O_DIRECTORY | O_RDONLY)) < 0,
+					     "Execute created too many directories\n");
+			}
+
+			igt_fixture {
+				close(fd);
+				close(subdirfd);
+				close(dirfd);
+				clear_directory(dirname);
+				free_job_list(&list);
+			}
+		}
+	}
+
+	igt_subtest_group {
+		struct job_list list;
+		int dirfd = -1, subdirfd = -1, fd = -1;
+		int multiple;
+
+		igt_fixture {
+			init_job_list(&list);
+		}
+
+		for (multiple = 0; multiple < 2; multiple++) {
+			char dirname[] = "tmpdirXXXXXX";
+
+			igt_fixture {
+				igt_require(mkdtemp(dirname) != NULL);
+				rmdir(dirname);
+			}
+
+			igt_subtest_f("execute-skipper-journal-%s", multiple ? "multiple" : "normal") {
+				struct execute_state state;
+				char *argv[] = { "runner",
+						 multiple ? "--multiple-mode" : "--sync",
+						 "-t", "skippers",
+						 testdatadir,
+						 dirname,
+				};
+				char *dump;
+				char *expected_0 = multiple ?
+					"skip-one\nskip-two\nexit:77 (" :
+					"skip-one\nexit:77 (";
+				char *expected_1 = "skip-two\nexit:77 (";
+
+				igt_assert(parse_options(ARRAY_SIZE(argv), argv, &settings));
+				igt_assert(create_job_list(&list, &settings));
+				igt_assert(initialize_execute_state(&state, &settings, &list));
+
+				igt_assert(execute(&state, &settings, &list));
+				igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0,
+					     "Execute didn't create the results directory\n");
+
+				igt_assert_f((fd = openat(dirfd, "uname.txt", O_RDONLY)) >= 0,
+					     "Execute didn't create uname.txt\n");
+				close(fd);
+				fd = -1;
+
+				igt_assert_f((subdirfd = openat(dirfd, "0", O_DIRECTORY | O_RDONLY)) >= 0,
+					     "Execute didn't create result directory '0'\n");
+				dump = dump_file(subdirfd, "journal.txt");
+				igt_assert_f(dump != NULL,
+					     "Execute didn't create the journal\n");
+				/* Trim out the runtime */
+				dump[strlen(expected_0)] = '\0';
+				igt_assert_eqstr(dump, expected_0);
+				free(dump);
+				close(subdirfd);
+				subdirfd = -1;
+
+				if (!multiple) {
+					igt_assert_f((subdirfd = openat(dirfd, "1", O_DIRECTORY | O_RDONLY)) >= 0,
+						     "Execute didn't create result directory '1'\n");
+					dump = dump_file(subdirfd, "journal.txt");
+					igt_assert_f(dump != NULL,
+						     "Execute didn't create the journal\n");
+					dump[strlen(expected_1)] = '\0';
+					igt_assert_eqstr(dump, expected_1);
+					free(dump);
+					close(subdirfd);
+					subdirfd = -1;
+				}
+			}
+
+			igt_fixture {
+				close(fd);
+				close(subdirfd);
+				close(dirfd);
+				clear_directory(dirname);
+				free_job_list(&list);
+			}
+		}
+	}
+
+	igt_fixture
+		free_settings(&settings);
+}
diff --git a/runner/testdata/Makefile.am b/runner/testdata/Makefile.am
new file mode 100644
index 00000000..fe225d8d
--- /dev/null
+++ b/runner/testdata/Makefile.am
@@ -0,0 +1,25 @@
+testdata_progs = no-subtests skippers successtest
+
+noinst_PROGRAMS = $(testdata_progs)
+
+test-list.txt: Makefile
+	@echo TESTLIST > $@
+	@echo ${testdata_progs} >> $@
+	@echo END TESTLIST >> $@
+
+noinst_DATA = test-list.txt
+
+all-local: .gitignore
+.gitignore: Makefile.am
+	@echo "$(testdata_progs) test-list.txt /.gitignore" | sed 's/\s\+/\n/g' | sort > $@
+
+CLEANFILES = test-list.txt .gitignore
+
+AM_CFLAGS = $(CWARNFLAGS) -Wno-unused-result $(DEBUG_CFLAGS) \
+	-I$(top_srcdir)/include/drm-uapi \
+	-I$(srcdir)/../.. \
+	-I$(srcdir)/../../lib \
+	$(DRM_CFLAGS) $(CAIRO_CFLAGS) $(LIBUDEV_CFLAGS) \
+	-D_GNU_SOURCE
+
+LDADD = $(top_builddir)/lib/libintel_tools.la
diff --git a/runner/testdata/meson.build b/runner/testdata/meson.build
new file mode 100644
index 00000000..011eff8e
--- /dev/null
+++ b/runner/testdata/meson.build
@@ -0,0 +1,20 @@
+
+testdata_progs = [ 'successtest',
+		   'no-subtests',
+		   'skippers',
+		 ]
+
+testdata_executables = []
+
+foreach prog : testdata_progs
+	testdata_executables += executable(prog, prog + '.c',
+					   dependencies : igt_deps,
+					   install : false)
+endforeach
+
+testdata_list = custom_target('testdata_testlist',
+			      output : 'test-list.txt',
+			      command : [ gen_testlist, '@OUTPUT@', testdata_progs ],
+			      build_by_default : true)
+
+testdata_dir = meson.current_build_dir()
diff --git a/runner/testdata/no-subtests.c b/runner/testdata/no-subtests.c
new file mode 100644
index 00000000..00c92e25
--- /dev/null
+++ b/runner/testdata/no-subtests.c
@@ -0,0 +1,6 @@
+#include "igt.h"
+
+igt_simple_main
+{
+
+}
diff --git a/runner/testdata/skippers.c b/runner/testdata/skippers.c
new file mode 100644
index 00000000..be4a31b4
--- /dev/null
+++ b/runner/testdata/skippers.c
@@ -0,0 +1,14 @@
+#include "igt.h"
+
+igt_main
+{
+	igt_fixture {
+		igt_require_f(false, "Skipping from fixture\n");
+	}
+
+	igt_subtest("skip-one")
+		igt_debug("Should be skipped\n");
+
+	igt_subtest("skip-two")
+		igt_debug("Should be skipped\n");
+}
diff --git a/runner/testdata/successtest.c b/runner/testdata/successtest.c
new file mode 100644
index 00000000..bb411eb4
--- /dev/null
+++ b/runner/testdata/successtest.c
@@ -0,0 +1,10 @@
+#include "igt.h"
+
+igt_main
+{
+	igt_subtest("first-subtest")
+		igt_debug("Running first subtest\n");
+
+	igt_subtest("second-subtest")
+		igt_debug("Running second subtest\n");
+}