diff options
-rw-r--r-- | Makefile.am | 4 | ||||
-rw-r--r-- | configure.ac | 14 | ||||
-rw-r--r-- | meson.build | 6 | ||||
-rw-r--r-- | meson_options.txt | 6 | ||||
-rw-r--r-- | runner/.gitignore | 3 | ||||
-rw-r--r-- | runner/Makefile.am | 31 | ||||
-rw-r--r-- | runner/executor.c | 1022 | ||||
-rw-r--r-- | runner/executor.h | 49 | ||||
-rw-r--r-- | runner/job_list.c | 484 | ||||
-rw-r--r-- | runner/job_list.h | 37 | ||||
-rw-r--r-- | runner/meson.build | 41 | ||||
-rw-r--r-- | runner/output_strings.h | 55 | ||||
-rw-r--r-- | runner/resultgen.c | 962 | ||||
-rw-r--r-- | runner/resultgen.h | 9 | ||||
-rw-r--r-- | runner/results.c | 26 | ||||
-rw-r--r-- | runner/resume.c | 47 | ||||
-rw-r--r-- | runner/runner.c | 40 | ||||
-rw-r--r-- | runner/settings.c | 502 | ||||
-rw-r--r-- | runner/settings.h | 111 |
19 files changed, 3449 insertions, 0 deletions
diff --git a/Makefile.am b/Makefile.am index 5bc486b9..044c0a07 100644 --- a/Makefile.am +++ b/Makefile.am @@ -27,6 +27,10 @@ if BUILD_TESTS SUBDIRS += tests endif +if BUILD_RUNNER +SUBDIRS += runner +endif + if BUILD_X86 if BUILD_ASSEMBLER SUBDIRS += assembler diff --git a/configure.ac b/configure.ac index 416a3240..b73cc9f0 100644 --- a/configure.ac +++ b/configure.ac @@ -374,6 +374,18 @@ fi AM_CONDITIONAL(BUILD_TESTS, [test "x$BUILD_TESTS" = xyes]) AC_DEFINE_UNQUOTED(TARGET_CPU_PLATFORM, ["$host_cpu"], [Target platform]) +AC_ARG_ENABLE(runner, + AS_HELP_STRING([--disable-runner], + [Enable building test runner (default: auto)]), + [BUILD_RUNNER=$enableval], [BUILD_RUNNER="auto"]) +if test "x$BUILD_RUNNER" = xauto; then + PKG_CHECK_EXISTS([json-c], [BUILD_RUNNER=yes], [BUILD_RUNNER=no]) +fi +if test "x$BUILD_RUNNER" = xyes; then + PKG_CHECK_MODULES(JSONC, [json-c]) +fi +AM_CONDITIONAL(BUILD_RUNNER, [test "x$BUILD_RUNNER" = xyes]) + files="broadwell cherryview haswell ivybridge sandybridge valleyview skylake" for file in $files; do REGISTER_FILES="$REGISTER_FILES $file `cat $srcdir/tools/registers/$file`" @@ -408,6 +420,7 @@ AC_CONFIG_FILES([ assembler/test/Makefile assembler/intel-gen4asm.pc overlay/Makefile + runner/Makefile ]) AC_CONFIG_FILES([tools/intel_aubdump], [chmod +x tools/intel_aubdump]) @@ -431,6 +444,7 @@ echo " Assembler : ${enable_assembler}" echo " Debugger : ${enable_debugger}" echo " Overlay : X: ${enable_overlay_xlib}, Xv: ${enable_overlay_xvlib}" echo " x86-specific tools : ${build_x86}" +echo " Test runner : ${BUILD_RUNNER}" echo "" echo " • API-Documentation : ${enable_gtk_doc}" echo " • Fail on warnings : ${enable_werror}" diff --git a/meson.build b/meson.build index 682f44da..d45db9ac 100644 --- a/meson.build +++ b/meson.build @@ -38,6 +38,8 @@ _build_docs = false _docs_required = false _build_tests = false _tests_required = false +_build_runner = false +_runner_required = false build_overlay = get_option('build_overlay') overlay_backends = get_option('overlay_backends') @@ -48,6 +50,7 @@ build_chamelium = get_option('build_chamelium') build_docs = get_option('build_docs') build_tests = get_option('build_tests') with_libdrm = get_option('with_libdrm') +build_runner = get_option('build_runner') _build_overlay = build_overlay != 'false' _overlay_required = build_overlay == 'true' @@ -61,6 +64,8 @@ _build_docs = build_docs != 'false' _docs_required = build_docs == 'true' _build_tests = build_tests != 'false' _tests_required = build_tests == 'true' +_build_runner = build_runner != 'false' +_runner_required = build_runner == 'true' build_info = [] @@ -228,6 +233,7 @@ else endif subdir('benchmarks') subdir('tools') +subdir('runner') if libdrm_intel.found() subdir('assembler') endif diff --git a/meson_options.txt b/meson_options.txt index 05e63463..89a3731c 100644 --- a/meson_options.txt +++ b/meson_options.txt @@ -52,6 +52,12 @@ option('with_libdrm', choices : ['', 'auto', 'intel', 'nouveau', 'amdgpu'], description : 'libdrm libraries to be used') +option('build_runner', + type : 'combo', + value : 'auto', + choices : ['auto', 'true', 'false'], + description : 'Build test runner') + option('use_rpath', type : 'boolean', value : false, diff --git a/runner/.gitignore b/runner/.gitignore new file mode 100644 index 00000000..e3919fa7 --- /dev/null +++ b/runner/.gitignore @@ -0,0 +1,3 @@ +igt_runner +igt_resume +igt_results diff --git a/runner/Makefile.am b/runner/Makefile.am new file mode 100644 index 00000000..9c13a83c --- /dev/null +++ b/runner/Makefile.am @@ -0,0 +1,31 @@ + +if BUILD_RUNNER + +runnerlib = librunner.la +noinst_LTLIBRARIES = $(runnerlib) +librunner_la_SOURCES = \ + settings.c \ + job_list.c \ + executor.c \ + resultgen.c \ + $(NULL) + +bin_PROGRAMS = \ + igt_runner \ + igt_resume \ + igt_results \ + $(NULL) + +LDADD = $(runnerlib) $(JSONC_LIBS) ../lib/libintel_tools.la + +igt_runner_SOURCES = runner.c +igt_resume_SOURCES = resume.c +igt_results_SOURCES = results.c + +AM_CFLAGS = $(JSONC_CFLAGS) \ + $(CWARNFLAGS) -Wno-unused-result $(DEBUG_CFLAGS) \ + -I$(srcdir)/.. \ + -I$(srcdir)/../lib \ + -D_GNU_SOURCE + +endif diff --git a/runner/executor.c b/runner/executor.c new file mode 100644 index 00000000..60cf4da5 --- /dev/null +++ b/runner/executor.c @@ -0,0 +1,1022 @@ +#include <errno.h> +#include <fcntl.h> +#include <linux/watchdog.h> +#include <signal.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/ioctl.h> +#include <sys/select.h> +#include <sys/signalfd.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <sys/utsname.h> +#include <sys/wait.h> +#include <time.h> +#include <unistd.h> + +#include "igt_core.h" +#include "executor.h" +#include "output_strings.h" + +static struct { + int *fds; + size_t num_dogs; +} watchdogs; + +static void close_watchdogs(struct settings *settings) +{ + size_t i; + + if (settings && settings->log_level >= LOG_LEVEL_VERBOSE) + printf("Closing watchdogs\n"); + + for (i = 0; i < watchdogs.num_dogs; i++) { + write(watchdogs.fds[i], "V", 1); + close(watchdogs.fds[i]); + } +} + +static void close_watchdogs_atexit(void) +{ + close_watchdogs(NULL); +} + +static void init_watchdogs(struct settings *settings) +{ + int i; + char name[32]; + int fd; + + memset(&watchdogs, 0, sizeof(watchdogs)); + + if (!settings->use_watchdog || settings->inactivity_timeout <= 0) + return; + + if (settings->log_level >= LOG_LEVEL_VERBOSE) { + printf("Initializing watchdogs\n"); + } + + atexit(close_watchdogs_atexit); + + for (i = 0; ; i++) { + snprintf(name, sizeof(name), "/dev/watchdog%d", i); + if ((fd = open(name, O_RDWR | O_CLOEXEC)) < 0) + break; + + watchdogs.num_dogs++; + watchdogs.fds = realloc(watchdogs.fds, watchdogs.num_dogs * sizeof(int)); + watchdogs.fds[i] = fd; + + if (settings->log_level >= LOG_LEVEL_VERBOSE) + printf(" %s\n", name); + } +} + +static int watchdogs_set_timeout(int timeout) +{ + size_t i; + int orig_timeout = timeout; + + for (i = 0; i < watchdogs.num_dogs; i++) { + if (ioctl(watchdogs.fds[i], WDIOC_SETTIMEOUT, &timeout)) { + write(watchdogs.fds[i], "V", 1); + close(watchdogs.fds[i]); + watchdogs.fds[i] = -1; + continue; + } + + if (timeout < orig_timeout) { + /* + * Timeout of this caliber refused. We want to + * use the same timeout for all devices. + */ + return watchdogs_set_timeout(timeout); + } + } + + return timeout; +} + +static void ping_watchdogs(void) +{ + size_t i; + + for (i = 0; i < watchdogs.num_dogs; i++) { + ioctl(watchdogs.fds[i], WDIOC_KEEPALIVE, 0); + } +} + +static void prune_subtest(struct job_list_entry *entry, char *subtest) +{ + char *excl; + + /* + * Subtest pruning is done by adding exclusion strings to the + * subtest list. The last matching item on the subtest + * selection command line flag decides whether to run a + * subtest, see igt_core.c for details. If the list is empty, + * the expected subtest set is unknown, so we need to add '*' + * first so we can start excluding. + */ + + if (entry->subtest_count == 0) { + entry->subtest_count++; + entry->subtests = realloc(entry->subtests, entry->subtest_count * sizeof(*entry->subtests)); + entry->subtests[0] = strdup("*"); + } + + excl = malloc(strlen(subtest) + 2); + excl[0] = '!'; + strcpy(excl + 1, subtest); + + entry->subtest_count++; + entry->subtests = realloc(entry->subtests, entry->subtest_count * sizeof(*entry->subtests)); + entry->subtests[entry->subtest_count - 1] = excl; +} + +static bool prune_from_journal(struct job_list_entry *entry, int fd) +{ + char *subtest; + FILE *f; + bool any_pruned = false; + + /* + * Each journal line is a subtest that has been started, or + * the line 'exit:$exitcode (time)', or 'timeout:$exitcode (time)'. + */ + + f = fdopen(fd, "r"); + if (!f) + return false; + + while (fscanf(f, "%ms", &subtest) == 1) { + if (!strncmp(subtest, EXECUTOR_EXIT, strlen(EXECUTOR_EXIT))) { + /* Fully done. Mark that by making the binary name invalid. */ + fscanf(f, " (%*fs)"); + entry->binary[0] = '\0'; + free(subtest); + continue; + } + + if (!strncmp(subtest, EXECUTOR_TIMEOUT, strlen(EXECUTOR_TIMEOUT))) { + fscanf(f, " (%*fs)"); + free(subtest); + continue; + } + + prune_subtest(entry, subtest); + + free(subtest); + any_pruned = true; + } + + fclose(f); + return any_pruned; +} + +static const char *filenames[_F_LAST] = { + [_F_JOURNAL] = "journal.txt", + [_F_OUT] = "out.txt", + [_F_ERR] = "err.txt", + [_F_DMESG] = "dmesg.txt", +}; + +static int open_at_end(int dirfd, const char *name) +{ + int fd = openat(dirfd, name, O_RDWR | O_CREAT | O_CLOEXEC, 0666); + char last; + + if (fd >= 0) { + if (lseek(fd, -1, SEEK_END) >= 0 && + read(fd, &last, 1) == 1 && + last != '\n') { + write(fd, "\n", 1); + } + lseek(fd, 0, SEEK_END); + } + + return fd; +} + +static int open_for_reading(int dirfd, const char *name) +{ + return openat(dirfd, name, O_RDONLY); +} + +bool open_output_files(int dirfd, int *fds, bool write) +{ + int i; + int (*openfunc)(int, const char*) = write ? open_at_end : open_for_reading; + + for (i = 0; i < _F_LAST; i++) { + if ((fds[i] = openfunc(dirfd, filenames[i])) < 0) { + while (--i >= 0) + close(fds[i]); + return false; + } + } + + return true; +} + +void close_outputs(int *fds) +{ + int i; + + for (i = 0; i < _F_LAST; i++) { + close(fds[i]); + } +} + +static void dump_dmesg(int kmsgfd, int outfd) +{ + /* + * Write kernel messages to the log file until we reach + * 'now'. Unfortunately, /dev/kmsg doesn't support seeking to + * -1 from SEEK_END so we need to use a second fd to read a + * message to match against, or stop when we reach EAGAIN. + */ + + int comparefd = open("/dev/kmsg", O_RDONLY | O_NONBLOCK); + unsigned flags; + unsigned long long seq, cmpseq, usec; + char cont; + char buf[256]; + ssize_t r; + + if (comparefd < 0) + return; + + if (fcntl(kmsgfd, F_SETFL, O_NONBLOCK)) + return; + + while (1) { + if (comparefd >= 0) { + r = read(comparefd, buf, sizeof(buf) - 1); + if (r < 0) { + if (errno != EAGAIN && errno != EPIPE) + return; + } else { + buf[r] = '\0'; + if (sscanf(buf, "%u,%llu,%llu,%c;", + &flags, &cmpseq, &usec, &cont) == 4) { + /* Reading comparison record done. */ + close(comparefd); + comparefd = -1; + } + } + } + + r = read(kmsgfd, buf, sizeof(buf)); + if (r <= 0) { + if (errno == EPIPE) + continue; + + /* + * If EAGAIN, we're done. If some other error, + * we can't do anything anyway. + */ + return; + } + + write(outfd, buf, r); + + if (comparefd < 0 && sscanf(buf, "%u,%llu,%llu,%c;", + &flags, &seq, &usec, &cont) == 4) { + /* + * Comparison record has been read, compare + * the sequence number to see if we have read + * enough. + */ + if (seq >= cmpseq) + return; + } + } +} + +static bool kill_child(int sig, pid_t child) +{ + /* + * Send the signal to the child directly, and to the child's + * process group. + */ + kill(-child, sig); + if (kill(child, sig) && errno == ESRCH) { + fprintf(stderr, "Child process does not exist. This shouldn't happen.\n"); + return false; + } + + return true; +} + +/* + * Returns: + * =0 - Success + * <0 - Failure executing + * >0 - Timeout happened, need to recreate from journal + */ +static int monitor_output(pid_t child, + int outfd, int errfd, int kmsgfd, int sigfd, + int *outputs, + struct settings *settings) +{ + fd_set set; + char buf[256]; + char *outbuf = NULL; + size_t outbufsize = 0; + char current_subtest[256] = {}; + struct signalfd_siginfo siginfo; + ssize_t s; + int n, status; + int nfds = outfd; + int timeout = settings->inactivity_timeout; + int timeout_intervals = 1, intervals_left; + int wd_extra = 10; + int killed = 0; /* 0 if not killed, signal number otherwise */ + struct timespec time_beg, time_end; + bool aborting = false; + + igt_gettime(&time_beg); + + if (errfd > nfds) + nfds = errfd; + if (kmsgfd > nfds) + nfds = kmsgfd; + if (sigfd > nfds) + nfds = sigfd; + nfds++; + + if (timeout > 0) { + /* + * Use original timeout plus some leeway. If we're still + * alive, we want to kill the test process instead of cutting + * power. + */ + int wd_timeout = watchdogs_set_timeout(timeout + wd_extra); + + if (wd_timeout < timeout + wd_extra) { + /* Watchdog timeout smaller, so ping it more often */ + if (wd_timeout - wd_extra < 0) + wd_extra = wd_timeout / 2; + timeout_intervals = timeout / (wd_timeout - wd_extra); + intervals_left = timeout_intervals; + timeout /= timeout_intervals; + + if (settings->log_level >= LOG_LEVEL_VERBOSE) { + printf("Watchdog doesn't support the timeout we requested (shortened to %d seconds).\n" + "Using %d intervals of %d seconds.\n", + wd_timeout, timeout_intervals, timeout); + } + } + } + + while (outfd >= 0 || errfd >= 0 || sigfd >= 0) { + struct timeval tv = { .tv_sec = timeout }; + + FD_ZERO(&set); + if (outfd >= 0) + FD_SET(outfd, &set); + if (errfd >= 0) + FD_SET(errfd, &set); + if (kmsgfd >= 0) + FD_SET(kmsgfd, &set); + if (sigfd >= 0) + FD_SET(sigfd, &set); + + n = select(nfds, &set, NULL, NULL, timeout == 0 ? NULL : &tv); + if (n < 0) { + /* TODO */ + return -1; + } + + if (n == 0) { + intervals_left--; + if (intervals_left) { + continue; + } + + ping_watchdogs(); + + switch (killed) { + case 0: + if (settings->log_level >= LOG_LEVEL_NORMAL) { + printf("Timeout. Killing the current test with SIGTERM.\n"); + } + + killed = SIGTERM; + if (!kill_child(killed, child)) + return -1; + + /* + * Now continue the loop and let the + * dying child be handled normally. + */ + timeout = 2; /* Timeout for waiting selected by fair dice roll. */ + watchdogs_set_timeout(20); + intervals_left = timeout_intervals = 1; + break; + case SIGTERM: + if (settings->log_level >= LOG_LEVEL_NORMAL) { + printf("Timeout. Killing the current test with SIGKILL.\n"); + } + + killed = SIGKILL; + if (!kill_child(killed, child)) + return -1; + + intervals_left = timeout_intervals = 1; + break; + case SIGKILL: + /* Nothing that can be done, really. Let's tell the caller we want to abort. */ + if (settings->log_level >= LOG_LEVEL_NORMAL) { + fprintf(stderr, "Child refuses to die. Aborting.\n"); + } + close_watchdogs(settings); + free(outbuf); + close(outfd); + close(errfd); + close(kmsgfd); + close(sigfd); + return -1; + } + + continue; + } + + intervals_left = timeout_intervals; + ping_watchdogs(); + + /* TODO: Refactor these handlers to their own functions */ + if (outfd >= 0 && FD_ISSET(outfd, &set)) { + char *newline; + + s = read(outfd, buf, sizeof(buf)); + if (s <= 0) { + if (s < 0) { + fprintf(stderr, "Error reading test's stdout: %s\n", + strerror(errno)); + } + + close(outfd); + outfd = -1; + goto out_end; + } + + write(outputs[_F_OUT], buf, s); + if (settings->sync) { + fdatasync(outputs[_F_OUT]); + } + + outbuf = realloc(outbuf, outbufsize + s); + memcpy(outbuf + outbufsize, buf, s); + outbufsize += s; + + while ((newline = memchr(outbuf, '\n', outbufsize)) != NULL) { + size_t linelen = newline - outbuf + 1; + + if (linelen > strlen(STARTING_SUBTEST) && + !memcmp(outbuf, STARTING_SUBTEST, strlen(STARTING_SUBTEST))) { + write(outputs[_F_JOURNAL], outbuf + strlen(STARTING_SUBTEST), + linelen - strlen(STARTING_SUBTEST)); + memcpy(current_subtest, outbuf + strlen(STARTING_SUBTEST), + linelen - strlen(STARTING_SUBTEST)); + current_subtest[linelen - strlen(STARTING_SUBTEST)] = '\0'; + + if (settings->log_level >= LOG_LEVEL_VERBOSE) { + fwrite(outbuf, 1, linelen, stdout); + } + } + if (linelen > strlen(SUBTEST_RESULT) && + !memcmp(outbuf, SUBTEST_RESULT, strlen(SUBTEST_RESULT))) { + char *delim = memchr(outbuf, ':', linelen); + + if (delim != NULL) { + size_t subtestlen = delim - outbuf - strlen(SUBTEST_RESULT); + if (memcmp(current_subtest, outbuf + strlen(SUBTEST_RESULT), + subtestlen)) { + /* Result for a test that didn't ever start */ + write(outputs[_F_JOURNAL], + outbuf + strlen(SUBTEST_RESULT), + subtestlen); + write(outputs[_F_JOURNAL], "\n", 1); + if (settings->sync) { + fdatasync(outputs[_F_JOURNAL]); + } + current_subtest[0] = '\0'; + } + + if (settings->log_level >= LOG_LEVEL_VERBOSE) { + fwrite(outbuf, 1, linelen, stdout); + } + } + } + + memmove(outbuf, newline + 1, outbufsize - linelen); + outbufsize -= linelen; + } + } + out_end: + + if (errfd >= 0 && FD_ISSET(errfd, &set)) { + s = read(errfd, buf, sizeof(buf)); + if (s <= 0) { + if (s < 0) { + fprintf(stderr, "Error reading test's stderr: %s\n", + strerror(errno)); + } + close(errfd); + errfd = -1; + } else { + write(outputs[_F_ERR], buf, s); + if (settings->sync) { + fdatasync(outputs[_F_ERR]); + } + } + } + + if (kmsgfd >= 0 && FD_ISSET(kmsgfd, &set)) { + s = read(kmsgfd, buf, sizeof(buf)); + if (s < 0) { + if (errno != EPIPE) { + fprintf(stderr, "Error reading from kmsg, stopping monitoring: %s\n", + strerror(errno)); + close(kmsgfd); + kmsgfd = -1; + } + } else { + write(outputs[_F_DMESG], buf, s); + if (settings->sync) { + fdatasync(outputs[_F_DMESG]); + } + } + } + + if (sigfd >= 0 && FD_ISSET(sigfd, &set)) { + double time; + + s = read(sigfd, &siginfo, sizeof(siginfo)); + if (s < 0) { + fprintf(stderr, "Error reading from signalfd: %s\n", + strerror(errno)); + continue; + } else if (siginfo.ssi_signo == SIGCHLD) { + if (child != waitpid(child, &status, WNOHANG)) { + fprintf(stderr, "Failed to reap child\n"); + status = 9999; + } else if (WIFEXITED(status)) { + status = WEXITSTATUS(status); + if (status >= 128) { + status = 128 - status; + } + } else if (WIFSIGNALED(status)) { + status = -WTERMSIG(status); + } else { + status = 9999; + } + } else { + /* We're dying, so we're taking them with us */ + if (settings->log_level >= LOG_LEVEL_NORMAL) + printf("Abort requested, terminating children\n"); + + aborting = true; + timeout = 2; + killed = SIGTERM; + if (!kill_child(killed, child)) + return -1; + + continue; + } + + igt_gettime(&time_end); + + time = igt_time_elapsed(&time_beg, &time_end); + if (time < 0.0) + time = 0.0; + + if (!aborting) { + dprintf(outputs[_F_JOURNAL], "%s%d (%.3fs)\n", + killed ? EXECUTOR_TIMEOUT : EXECUTOR_EXIT, + status, time); + if (settings->sync) { + fdatasync(outputs[_F_JOURNAL]); + } + } + + close(sigfd); + sigfd = -1; + child = 0; + } + } + + dump_dmesg(kmsgfd, outputs[_F_DMESG]); + if (settings->sync) + fdatasync(outputs[_F_DMESG]); + + free(outbuf); + close(outfd); + close(errfd); + close(kmsgfd); + close(sigfd); + + if (aborting) + return -1; + + return killed; +} + +static void execute_test_process(int outfd, int errfd, + struct settings *settings, + struct job_list_entry *entry) +{ + char *argv[4] = {}; + size_t rootlen; + + dup2(outfd, STDOUT_FILENO); + dup2(errfd, STDERR_FILENO); + + setpgid(0, 0); + + rootlen = strlen(settings->test_root); + argv[0] = malloc(rootlen + strlen(entry->binary) + 2); + strcpy(argv[0], settings->test_root); + argv[0][rootlen] = '/'; + strcpy(argv[0] + rootlen + 1, entry->binary); + + if (entry->subtest_count) { + size_t argsize; + size_t i; + + argv[1] = strdup("--run-subtest"); + argsize = strlen(entry->subtests[0]); + argv[2] = malloc(argsize + 1); + strcpy(argv[2], entry->subtests[0]); + + for (i = 1; i < entry->subtest_count; i++) { + char *sub = entry->subtests[i]; + size_t sublen = strlen(sub); + + argv[2] = realloc(argv[2], argsize + sublen + 2); + argv[2][argsize] = ','; + strcpy(argv[2] + argsize + 1, sub); + argsize += sublen + 1; + } + } + + execv(argv[0], argv); + fprintf(stderr, "Cannot execute %s\n", argv[0]); + exit(IGT_EXIT_INVALID); +} + +static int digits(size_t num) +{ + int ret = 0; + while (num) { + num /= 10; + ret++; + } + + if (ret == 0) ret++; + return ret; +} + +/* + * Returns: + * =0 - Success + * <0 - Failure executing + * >0 - Timeout happened, need to recreate from journal + */ +static int execute_entry(size_t idx, + size_t total, + struct settings *settings, + struct job_list_entry *entry, + int testdirfd, int resdirfd) +{ + int dirfd; + int outputs[_F_LAST]; + int kmsgfd; + int sigfd; + sigset_t mask; + int outpipe[2] = { -1, -1 }; + int errpipe[2] = { -1, -1 }; + char name[32]; + pid_t child; + int result; + + snprintf(name, sizeof(name), "%zd", idx); + mkdirat(resdirfd, name, 0777); + if ((dirfd = openat(resdirfd, name, O_DIRECTORY | O_RDONLY | O_CLOEXEC)) < 0) { + fprintf(stderr, "Error accessing individual test result directory\n"); + return -1; + } + + if (!open_output_files(dirfd, outputs, true)) { + close(dirfd); + fprintf(stderr, "Error opening output files\n"); + return -1; + } + + if (settings->sync) { + fsync(dirfd); + fsync(resdirfd); + } + + if (pipe(outpipe) || pipe(errpipe)) { + close_outputs(outputs); + close(dirfd); + close(outpipe[0]); + close(outpipe[1]); + close(errpipe[0]); + close(errpipe[1]); + fprintf(stderr, "Error creating pipes: %s\n", strerror(errno)); + return -1; + } + + if ((kmsgfd = open("/dev/kmsg", O_RDONLY | O_CLOEXEC)) < 0) { + fprintf(stderr, "Warning: Cannot open /dev/kmsg\n"); + } else { + /* TODO: Checking of abort conditions in pre-execute dmesg */ + lseek(kmsgfd, 0, SEEK_END); + } + + sigemptyset(&mask); + sigaddset(&mask, SIGCHLD); + sigaddset(&mask, SIGINT); + sigaddset(&mask, SIGTERM); + sigaddset(&mask, SIGQUIT); + sigprocmask(SIG_BLOCK, &mask, NULL); + sigfd = signalfd(-1, &mask, O_CLOEXEC); + + if (sigfd < 0) { + /* TODO: Handle better */ + fprintf(stderr, "Cannot monitor child process with signalfd\n"); + close(outpipe[0]); + close(errpipe[0]); + close(outpipe[1]); + close(errpipe[1]); + close(kmsgfd); + close_outputs(outputs); + close(dirfd); + return -1; + } + + if (settings->log_level >= LOG_LEVEL_NORMAL) { + int width = digits(total); + printf("[%0*zd/%0*zd] %s", width, idx + 1, width, total, entry->binary); + if (entry->subtest_count > 0) { + size_t i; + const char *delim = ""; + + printf(" ("); + for (i = 0; i < entry->subtest_count; i++) { + printf("%s%s", delim, entry->subtests[i]); + delim = ", "; + } + printf(")"); + } + printf("\n"); + } + + if ((child = fork())) { + int outfd = outpipe[0]; + int errfd = errpipe[0]; + close(outpipe[1]); + close(errpipe[1]); + + result = monitor_output(child, outfd, errfd, kmsgfd, sigfd, + outputs, settings); + } else { + int outfd = outpipe[1]; + int errfd = errpipe[1]; + close(outpipe[0]); + close(errpipe[0]); + + sigprocmask(SIG_UNBLOCK, &mask, NULL); + + setenv("IGT_SENTINEL_ON_STDERR", "1", 1); + + execute_test_process(outfd, errfd, settings, entry); + } + + /* TODO: Refactor this whole function to use onion teardown */ + close(outpipe[1]); + close(errpipe[1]); + close(kmsgfd); + close_outputs(outputs); + close(dirfd); + + return result; +} + +static int remove_file(int dirfd, const char *name) +{ + return unlinkat(dirfd, name, 0) && errno != ENOENT; +} + +static bool clear_test_result_directory(int dirfd) +{ + int i; + + for (i = 0; i < _F_LAST; i++) { + if (remove_file(dirfd, filenames[i])) { + fprintf(stderr, "Error deleting %s from test result directory: %s\n", + filenames[i], + strerror(errno)); + return false; + } + } + + return true; +} + +static bool clear_old_results(char *path) +{ + int dirfd; + size_t i; + + if ((dirfd = open(path, O_DIRECTORY | O_RDONLY)) < 0) { + if (errno == ENOENT) { + /* Successfully cleared if it doesn't even exist */ + return true; + } + + fprintf(stderr, "Error clearing old results: %s\n", strerror(errno)); + return false; + } + + if (unlinkat(dirfd, "uname.txt", 0) && errno != ENOENT) { + close(dirfd); + fprintf(stderr, "Error clearing old results: %s\n", strerror(errno)); + return false; + } + + for (i = 0; true; i++) { + char name[32]; + int resdirfd; + + snprintf(name, sizeof(name), "%zd", i); + if ((resdirfd = openat(dirfd, name, O_DIRECTORY | O_RDONLY)) < 0) + break; + + if (!clear_test_result_directory(resdirfd)) { + close(resdirfd); + close(dirfd); + return false; + } + close(resdirfd); + if (unlinkat(dirfd, name, AT_REMOVEDIR)) { + fprintf(stderr, + "Warning: Result directory %s contains extra files\n", + name); + } + } + + close(dirfd); + + return true; +} + +bool initialize_execute_state_from_resume(int dirfd, + struct execute_state *state, + struct settings *settings, + struct job_list *list) +{ + struct job_list_entry *entry; + int resdirfd, fd, i; + + free_settings(settings); + free_job_list(list); + memset(state, 0, sizeof(*state)); + + if (!read_settings(settings, dirfd) || + !read_job_list(list, dirfd)) { + close(dirfd); + return false; + } + + for (i = list->size; i >= 0; i--) { + char name[32]; + + snprintf(name, sizeof(name), "%d", i); + if ((resdirfd = openat(dirfd, name, O_DIRECTORY | O_RDONLY)) >= 0) + break; + } + + if (i < 0) + /* Nothing has been executed yet, state is fine as is */ + goto success; + + entry = &list->entries[i]; + state->next = i; + if ((fd = openat(resdirfd, filenames[_F_JOURNAL], O_RDONLY)) >= 0) { + if (!prune_from_journal(entry, fd)) { + /* + * The test does not have subtests, or + * incompleted before the first subtest + * began. Either way, not suitable to + * re-run. + */ + state->next = i + 1; + } else if (entry->binary[0] == '\0') { + /* This test is fully completed */ + state->next = i + 1; + } + + close(fd); + } + + success: + close(resdirfd); + close(dirfd); + + return true; +} + +bool initialize_execute_state(struct execute_state *state, + struct settings *settings, + struct job_list *job_list) +{ + memset(state, 0, sizeof(*state)); + + if (!validate_settings(settings)) + return false; + + if (!serialize_settings(settings) || + !serialize_job_list(job_list, settings)) + return false; + + if (settings->overwrite && + !clear_old_results(settings->results_path)) + return false; + + return true; +} + +bool execute(struct execute_state *state, + struct settings *settings, + struct job_list *job_list) +{ + struct utsname unamebuf; + int resdirfd, testdirfd, unamefd; + + if ((resdirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) { + /* Initialize state should have done this */ + fprintf(stderr, "Error: Failure opening results path %s\n", + settings->results_path); + return false; + } + + if ((testdirfd = open(settings->test_root, O_DIRECTORY | O_RDONLY)) < 0) { + fprintf(stderr, "Error: Failure opening test root %s\n", + settings->test_root); + close(resdirfd); + return false; + } + + /* TODO: On resume, don't rewrite, verify that content matches current instead */ + if ((unamefd = openat(resdirfd, "uname.txt", O_CREAT | O_WRONLY | O_TRUNC, 0666)) < 0) { + fprintf(stderr, "Error: Failure creating opening uname.txt: %s\n", + strerror(errno)); + close(testdirfd); + close(resdirfd); + return false; + } + + init_watchdogs(settings); + + if (!uname(&unamebuf)) { + dprintf(unamefd, "%s %s %s %s %s\n", + unamebuf.sysname, + unamebuf.nodename, + unamebuf.release, + unamebuf.version, + unamebuf.machine); + } else { + dprintf(unamefd, "uname() failed\n"); + } + close(unamefd); + + for (; state->next < job_list->size; + state->next++) { + int result = execute_entry(state->next, + job_list->size, + settings, + &job_list->entries[state->next], + testdirfd, resdirfd); + if (result != 0) { + close(testdirfd); + close_watchdogs(settings); + if (result > 0) { + initialize_execute_state_from_resume(resdirfd, state, settings, job_list); + return execute(state, settings, job_list); + } + close(resdirfd); + return false; + } + } + + close(testdirfd); + close(resdirfd); + close_watchdogs(settings); + return true; +} diff --git a/runner/executor.h b/runner/executor.h new file mode 100644 index 00000000..8fe1605b --- /dev/null +++ b/runner/executor.h @@ -0,0 +1,49 @@ +#ifndef RUNNER_EXECUTOR_H +#define RUNNER_EXECUTOR_H + +#include "job_list.h" +#include "settings.h" + +struct execute_state +{ + size_t next; +}; + +enum { + _F_JOURNAL, + _F_OUT, + _F_ERR, + _F_DMESG, + _F_LAST, +}; + +bool open_output_files(int dirfd, int *fds, bool write); +void close_outputs(int *fds); + +/* + * Initialize execute_state object to a state where it's ready to + * execute. Will validate the settings and serialize both settings and + * the job_list into the result directory, overwriting old files if + * settings set to do so. + */ +bool initialize_execute_state(struct execute_state *state, + struct settings *settings, + struct job_list *job_list); + +/* + * Initialize execute_state object to a state where it's ready to + * resume an already existing run. settings and job_list must have + * been initialized with init_settings et al, and will be read from + * the result directory pointed to by dirfd. + */ +bool initialize_execute_state_from_resume(int dirfd, + struct execute_state *state, + struct settings *settings, + struct job_list *job_list); + +bool execute(struct execute_state *state, + struct settings *settings, + struct job_list *job_list); + + +#endif diff --git a/runner/job_list.c b/runner/job_list.c new file mode 100644 index 00000000..e3f820c3 --- /dev/null +++ b/runner/job_list.c @@ -0,0 +1,484 @@ +#include <errno.h> +#include <fcntl.h> +#include <linux/limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include "job_list.h" +#include "igt_core.h" + +static bool matches_any(const char *str, struct regex_list *list) +{ + size_t i; + + for (i = 0; i < list->size; i++) { + if (regexec(list->regexes[i], str, + (size_t)0, NULL, 0) == 0) { + return true; + } + } + + return false; +} + +static void add_job_list_entry(struct job_list *job_list, + char *binary, + char **subtests, + size_t subtest_count) +{ + struct job_list_entry *entry; + + job_list->size++; + job_list->entries = realloc(job_list->entries, job_list->size * sizeof(*job_list->entries)); + entry = &job_list->entries[job_list->size - 1]; + + entry->binary = binary; + entry->subtests = subtests; + entry->subtest_count = subtest_count; +} + +static void add_subtests(struct job_list *job_list, struct settings *settings, + char *binary, + struct regex_list *include, struct regex_list *exclude) +{ + FILE *p; + char cmd[256] = {}; + char *subtestname; + char **subtests = NULL; + size_t num_subtests = 0; + int s; + + s = snprintf(cmd, sizeof(cmd), "%s/%s --list-subtests", + settings->test_root, binary); + if (s < 0) { + fprintf(stderr, "Failure generating command string, this shouldn't happen.\n"); + return; + } + + if (s >= sizeof(cmd)) { + fprintf(stderr, "Path to binary too long, ignoring: %s/%s\n", + settings->test_root, binary); + return; + } + + p = popen(cmd, "r"); + if (!p) { + fprintf(stderr, "popen failed when executing %s: %s\n", + cmd, + strerror(errno)); + return; + } + + while (fscanf(p, "%ms", &subtestname) == 1) { + if (exclude && exclude->size && matches_any(subtestname, exclude)) { + free(subtestname); + continue; + } + + if (include && include->size && !matches_any(subtestname, include)) { + free(subtestname); + continue; + } + + if (settings->multiple_mode) { + num_subtests++; + subtests = realloc(subtests, num_subtests * sizeof(*subtests)); + subtests[num_subtests - 1] = strdup(subtestname); + } else { + subtests = malloc(sizeof(*subtests)); + *subtests = strdup(subtestname); + add_job_list_entry(job_list, strdup(binary), subtests, 1); + subtests = NULL; + } + + free(subtestname); + } + + if (num_subtests) + add_job_list_entry(job_list, strdup(binary), subtests, num_subtests); + + s = pclose(p); + if (s == 0) { + return; + } else if (s == -1) { + fprintf(stderr, "popen error when executing %s: %s\n", binary, strerror(errno)); + } else if (WIFEXITED(s)) { + if (WEXITSTATUS(s) == IGT_EXIT_INVALID) { + /* No subtests on this one */ + if (exclude && exclude->size && matches_any(binary, exclude)) { + return; + } + if (!include || !include->size || matches_any(binary, include)) { + add_job_list_entry(job_list, strdup(binary), NULL, 0); + return; + } + } + } else { + fprintf(stderr, "Test binary %s died unexpectedly\n", binary); + } +} + +static bool filtered_job_list(struct job_list *job_list, + struct settings *settings, + int fd) +{ + FILE *f; + char buf[128]; + + if (job_list->entries != NULL) { + fprintf(stderr, "Caller didn't clear the job list, this shouldn't happen\n"); + exit(1); + } + + f = fdopen(fd, "r"); + + while (fscanf(f, "%127s", buf) == 1) { + if (!strcmp(buf, "TESTLIST") || !(strcmp(buf, "END"))) + continue; + + /* + * If the binary name matches exclude filters, no + * subtests are added. + */ + if (settings->exclude_regexes.size && matches_any(buf, &settings->exclude_regexes)) + continue; + + /* + * If the binary name matches include filters (or include filters not present), + * all subtests except those matching exclude filters are added. + */ + if (!settings->include_regexes.size || matches_any(buf, &settings->include_regexes)) { + if (settings->multiple_mode && !settings->exclude_regexes.size) + /* + * Optimization; we know that all + * subtests will be included, so we + * get to omit executing + * --list-subtests. + */ + add_job_list_entry(job_list, strdup(buf), NULL, 0); + else + add_subtests(job_list, settings, buf, + NULL, &settings->exclude_regexes); + continue; + } + + /* + * Binary name doesn't match exclude or include filters. + */ + add_subtests(job_list, settings, buf, + &settings->include_regexes, + &settings->exclude_regexes); + } + + return job_list->size != 0; +} + +static bool job_list_from_test_list(struct job_list *job_list, + struct settings *settings) +{ + FILE *f; + char *line = NULL; + size_t line_len = 0; + struct job_list_entry entry = {}; + bool any = false; + + if ((f = fopen(settings->test_list, "r")) == NULL) { + fprintf(stderr, "Cannot open test list file %s\n", settings->test_list); + return false; + } + + while (1) { + char *binary; + char *delim; + + if (getline(&line, &line_len, f) == -1) { + if (errno == EINTR) + continue; + else + break; + } + + /* # starts a comment */ + if ((delim = strchr(line, '#')) != NULL) + *delim = '\0'; + + if (sscanf(line, "igt@%ms", &binary) == 1) { + if ((delim = strchr(binary, '@')) != NULL) + *delim++ = '\0'; + + if (!settings->multiple_mode) { + char **subtests = NULL; + if (delim) { + subtests = malloc(sizeof(char*)); + subtests[0] = strdup(delim); + } + add_job_list_entry(job_list, strdup(binary), + subtests, (size_t)(subtests != NULL)); + any = true; + free(binary); + binary = NULL; + continue; + } + + /* + * If the currently built entry has the same + * binary, add a subtest. Otherwise submit + * what's already built and start a new one. + */ + if (entry.binary && !strcmp(entry.binary, binary)) { + if (!delim) { + /* ... except we didn't get a subtest */ + fprintf(stderr, + "Error: Unexpected test without subtests " + "after same test had subtests\n"); + free(binary); + fclose(f); + return false; + } + entry.subtest_count++; + entry.subtests = realloc(entry.subtests, + entry.subtest_count * + sizeof(*entry.subtests)); + entry.subtests[entry.subtest_count - 1] = strdup(delim); + free(binary); + binary = NULL; + continue; + } + + if (entry.binary) { + add_job_list_entry(job_list, entry.binary, entry.subtests, entry.subtest_count); + any = true; + } + + memset(&entry, 0, sizeof(entry)); + entry.binary = strdup(binary); + if (delim) { + entry.subtests = malloc(sizeof(*entry.subtests)); + entry.subtests[0] = strdup(delim); + entry.subtest_count = 1; + } + + free(binary); + binary = NULL; + } + } + + if (entry.binary) { + add_job_list_entry(job_list, entry.binary, entry.subtests, entry.subtest_count); + any = true; + } + + free(line); + fclose(f); + return any; +} + +void init_job_list(struct job_list *job_list) +{ + memset(job_list, 0, sizeof(*job_list)); +} + +void free_job_list(struct job_list *job_list) +{ + int i, k; + + for (i = 0; i < job_list->size; i++) { + struct job_list_entry *entry = &job_list->entries[i]; + + free(entry->binary); + for (k = 0; k < entry->subtest_count; k++) { + free(entry->subtests[k]); + } + free(entry->subtests); + } + free(job_list->entries); + init_job_list(job_list); +} + +bool create_job_list(struct job_list *job_list, + struct settings *settings) +{ + int dirfd, fd; + bool result; + + if (!settings->test_root) { + fprintf(stderr, "No test root set; this shouldn't happen\n"); + return false; + } + + free_job_list(job_list); + + dirfd = open(settings->test_root, O_DIRECTORY | O_RDONLY); + if (dirfd < 0) { + fprintf(stderr, "Test directory %s cannot be opened\n", settings->test_root); + return false; + } + + fd = openat(dirfd, "test-list.txt", O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Cannot open %s/test-list.txt\n", settings->test_root); + close(dirfd); + return false; + } + + /* + * If a test_list is given (not to be confused with + * test-list.txt), we use it directly without making tests + * list their subtests. If include/exclude filters are given + * we filter them directly from the test_list. + */ + if (settings->test_list) + result = job_list_from_test_list(job_list, settings); + else + result = filtered_job_list(job_list, settings, fd); + + close(fd); + close(dirfd); + + return result; +} + +static char joblist_filename[] = "joblist.txt"; +bool serialize_job_list(struct job_list *job_list, struct settings *settings) +{ + int dirfd, fd; + size_t i, k; + FILE *f; + + if (!settings->results_path) { + fprintf(stderr, "No results-path set; this shouldn't happen\n"); + return false; + } + + if ((dirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) { + mkdir(settings->results_path, 0777); + if ((dirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) { + fprintf(stderr, "Creating results-path failed\n"); + return false; + } + } + + if (!settings->overwrite && + faccessat(dirfd, joblist_filename, F_OK, 0) == 0) { + fprintf(stderr, "Job list file already exists and not overwriting\n"); + close(dirfd); + return false; + } + + if (settings->overwrite && + unlinkat(dirfd, joblist_filename, 0) != 0 && + errno != ENOENT) { + fprintf(stderr, "Error removing old job list\n"); + close(dirfd); + return false; + } + + if ((fd = openat(dirfd, joblist_filename, O_CREAT | O_EXCL | O_WRONLY, 0666)) < 0) { + fprintf(stderr, "Creating job list serialization file failed: %s\n", strerror(errno)); + close(dirfd); + return false; + } + + f = fdopen(fd, "w"); + if (!f) { + close(fd); + close(dirfd); + return false; + } + + for (i = 0; i < job_list->size; i++) { + struct job_list_entry *entry = &job_list->entries[i]; + fputs(entry->binary, f); + + if (entry->subtest_count) { + const char *delim = ""; + + fprintf(f, " "); + + for (k = 0; k < entry->subtest_count; k++) { + fprintf(f, "%s%s", delim, entry->subtests[k]); + delim = ","; + } + } + + fprintf(f, "\n"); + } + + if (settings->sync) { + fsync(fd); + fsync(dirfd); + } + + fclose(f); + close(dirfd); + return true; +} + +bool read_job_list(struct job_list *job_list, int dirfd) +{ + int fd; + FILE *f; + ssize_t read; + char *line = NULL; + size_t line_len = 0; + + free_job_list(job_list); + + if ((fd = openat(dirfd, joblist_filename, O_RDONLY)) < 0) + return false; + + f = fdopen(fd, "r"); + if (!f) { + close(fd); + return false; + } + + while ((read = getline(&line, &line_len, f))) { + char *binary, *sublist, *comma; + char **subtests = NULL; + size_t num_subtests = 0, len; + + if (read < 0) { + if (errno == EINTR) + continue; + else + break; + } + + len = strlen(line); + if (len > 0 && line[len - 1] == '\n') + line[len - 1] = '\0'; + + sublist = strchr(line, ' '); + if (!sublist) { + add_job_list_entry(job_list, strdup(line), NULL, 0); + continue; + } + + *sublist++ = '\0'; + binary = strdup(line); + + do { + comma = strchr(sublist, ','); + if (comma) { + *comma++ = '\0'; + } + + ++num_subtests; + subtests = realloc(subtests, num_subtests * sizeof(*subtests)); + subtests[num_subtests - 1] = strdup(sublist); + sublist = comma; + } while (comma != NULL); + + add_job_list_entry(job_list, binary, subtests, num_subtests); + } + + free(line); + fclose(f); + + return true; +} diff --git a/runner/job_list.h b/runner/job_list.h new file mode 100644 index 00000000..c726ab09 --- /dev/null +++ b/runner/job_list.h @@ -0,0 +1,37 @@ +#ifndef RUNNER_JOB_LIST_H +#define RUNNER_JOB_LIST_H + +#include <stdbool.h> + +#include "settings.h" + +struct job_list_entry { + char *binary; + char **subtests; + /* + * 0 = all, or test has no subtests. + * + * If the original job_list was to run all subtests of a + * binary and such a run was incomplete, resuming from the + * execution journal will fill the subtest array with already + * started subtests prepended with '!' so the test binary will + * not run them. subtest_count will still reflect the size of + * the above array. + */ + size_t subtest_count; +}; + +struct job_list +{ + struct job_list_entry *entries; + size_t size; +}; + +void init_job_list(struct job_list *job_list); +void free_job_list(struct job_list *job_list); +bool create_job_list(struct job_list *job_list, struct settings *settings); + +bool serialize_job_list(struct job_list *job_list, struct settings *settings); +bool read_job_list(struct job_list *job_list, int dirfd); + +#endif diff --git a/runner/meson.build b/runner/meson.build new file mode 100644 index 00000000..9dafb312 --- /dev/null +++ b/runner/meson.build @@ -0,0 +1,41 @@ +jsonc = dependency('json-c', required: _runner_required) + +runnerlib_sources = [ 'settings.c', + 'job_list.c', + 'executor.c', + 'resultgen.c', + ] + +runner_sources = [ 'runner.c' ] +resume_sources = [ 'resume.c' ] +results_sources = [ 'results.c' ] + +if _build_runner and jsonc.found() + subdir('testdata') + + runnerlib = static_library('igt_runner', runnerlib_sources, + include_directories : inc, + dependencies : jsonc) + + runner = executable('igt_runner', runner_sources, + link_with : runnerlib, + install : true, + install_dir : bindir, + dependencies : igt_deps) + + resume = executable('igt_resume', resume_sources, + link_with : runnerlib, + install : true, + install_dir : bindir, + dependencies : igt_deps) + + results = executable('igt_results', results_sources, + link_with : runnerlib, + install : true, + install_dir : bindir, + dependencies : igt_deps) + + build_info += 'Build test runner: Yes' +else + build_info += 'Build test runner: No' +endif diff --git a/runner/output_strings.h b/runner/output_strings.h new file mode 100644 index 00000000..1e52c2ce --- /dev/null +++ b/runner/output_strings.h @@ -0,0 +1,55 @@ +#ifndef RUNNER_OUTPUT_STRINGS_H +#define RUNNER_OUTPUT_STRINGS_H + +/* + * Output when a subtest has begun. Is followed by the subtest name. + * + * Example: + * Starting subtest: subtestname + */ +static const char STARTING_SUBTEST[] = "Starting subtest: "; + +/* + * Output when a subtest has ended. Is followed by the subtest name + * and optionally its runtime. + * + * Examples: + * Subtest subtestname: SKIP + * Subtest subtestname: PASS (0.003s) + */ +static const char SUBTEST_RESULT[] = "Subtest "; + +/* + * Output in dmesg when a subtest has begin. Is followed by the subtest name. + * + * Example: + * [IGT] test-binary-name: starting subtest subtestname + */ +static const char STARTING_SUBTEST_DMESG[] = ": starting subtest "; + +/* + * Output when a test process is executed. + * + * Example: + * IGT-Version: 1.22-gde9af343 (x86_64) (Linux: 4.12.0-1-amd64 x86_64) + */ +static const char IGT_VERSIONSTRING[] = "IGT-Version: "; + +/* + * Output by the executor to mark the test's exit code. + * + * Example: + * exit:77 (0.003s) + */ +static const char EXECUTOR_EXIT[] = "exit:"; + +/* + * Output by the executor to mark the test as timeouted, with an exit + * code. + * + * Example: + * timeout:-15 (360.000s) + */ +static const char EXECUTOR_TIMEOUT[] = "timeout:"; + +#endif diff --git a/runner/resultgen.c b/runner/resultgen.c new file mode 100644 index 00000000..347b52a4 --- /dev/null +++ b/runner/resultgen.c @@ -0,0 +1,962 @@ +#include <ctype.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <sys/types.h> +#include <unistd.h> + +#include <json.h> + +#include "igt_core.h" +#include "resultgen.h" +#include "settings.h" +#include "executor.h" +#include "output_strings.h" + +struct subtests +{ + char **names; + size_t size; +}; + +/* + * A lot of string handling here operates on an mmapped buffer, and + * thus we can't assume null-terminated strings. Buffers will be + * passed around as pointer+size, or pointer+pointer-past-the-end, the + * mem*() family of functions is used instead of str*(). + */ + +static char *find_line_starting_with(char *haystack, const char *needle, char *end) +{ + while (haystack < end) { + char *line_end = memchr(haystack, '\n', end - haystack); + + if (end - haystack < strlen(needle)) + return NULL; + if (!memcmp(haystack, needle, strlen(needle))) + return haystack; + if (line_end == NULL) + return NULL; + haystack = line_end + 1; + } + + return NULL; +} + +static char *find_line_starting_with_either(char *haystack, + const char *needle1, + const char *needle2, + char *end) +{ + while (haystack < end) { + char *line_end = memchr(haystack, '\n', end - haystack); + size_t linelen = line_end != NULL ? line_end - haystack : end - haystack; + if ((linelen >= strlen(needle1) && !memcmp(haystack, needle1, strlen(needle1))) || + (linelen >= strlen(needle2) && !memcmp(haystack, needle2, strlen(needle2)))) + return haystack; + + if (line_end == NULL) + return NULL; + + haystack = line_end + 1; + } + + return NULL; +} + +static char *next_line(char *line, char *bufend) +{ + char *ret; + + if (!line) + return NULL; + + ret = memchr(line, '\n', bufend - line); + if (ret) + ret++; + + if (ret < bufend) + return ret; + else + return NULL; +} + +static char *find_line_after_last(char *begin, + const char *needle1, + const char *needle2, + char *end) +{ + char *one, *two; + char *current_pos = begin; + char *needle1_newline = malloc(strlen(needle1) + 2); + char *needle2_newline = malloc(strlen(needle2) + 2); + + needle1_newline[0] = needle2_newline[0] = '\n'; + strcpy(needle1_newline + 1, needle1); + strcpy(needle2_newline + 1, needle2); + + while (true) { + one = memmem(current_pos, end - current_pos, needle1_newline, strlen(needle1_newline)); + two = memmem(current_pos, end - current_pos, needle2_newline, strlen(needle2_newline)); + if (one == NULL && two == NULL) + break; + + if (one != NULL && current_pos < one) + current_pos = one; + if (two != NULL && current_pos < two) + current_pos = two; + + one = next_line(current_pos, end); + if (one != NULL) + current_pos = one; + } + free(needle1_newline); + free(needle2_newline); + + one = memchr(current_pos, '\n', end - current_pos); + if (one != NULL) + return ++one; + + return current_pos; +} + +static size_t count_lines(const char *buf, const char *bufend) +{ + size_t ret = 0; + while (buf < bufend && (buf = memchr(buf, '\n', bufend - buf)) != NULL) { + ret++; + buf++; + } + + return ret; +} + +static char *lowercase(const char *str) +{ + char *ret = malloc(strlen(str) + 1); + char *q = ret; + + while (*str) { + if (isspace(*str)) + break; + + *q++ = tolower(*str++); + } + *q = '\0'; + + return ret; +} + +static void append_line(char **buf, size_t *buflen, char *line) +{ + size_t linelen = strlen(line); + + *buf = realloc(*buf, *buflen + linelen + 1); + strcpy(*buf + *buflen, line); + *buflen += linelen; +} + +static void generate_piglit_name(const char *binary, const char *subtest, + char *namebuf, size_t namebuf_size) +{ + char *lc_binary = lowercase(binary); + char *lc_subtest = NULL; + + if (!subtest) { + snprintf(namebuf, namebuf_size, "igt@%s", lc_binary); + free(lc_binary); + return; + } + + lc_subtest = lowercase(subtest); + + snprintf(namebuf, namebuf_size, "igt@%s@%s", lc_binary, lc_subtest); + + free(lc_binary); + free(lc_subtest); +} + +static const struct { + const char *output_str; + const char *result_str; +} resultmap[] = { + { "SUCCESS", "pass" }, + { "SKIP", "skip" }, + { "FAIL", "fail" }, + { "CRASH", "crash" }, + { "TIMEOUT", "timeout" }, +}; +static void parse_result_string(char *resultstring, size_t len, const char **result, double *time) +{ + size_t i; + size_t wordlen = 0; + + while (wordlen < len && !isspace(resultstring[wordlen])) { + wordlen++; + } + + *result = NULL; + for (i = 0; i < (sizeof(resultmap) / sizeof(resultmap[0])); i++) { + if (!strncmp(resultstring, resultmap[i].output_str, wordlen)) { + *result = resultmap[i].result_str; + break; + } + } + + /* If the result string is unknown, use incomplete */ + if (!*result) + *result = "incomplete"; + + /* + * Check for subtest runtime after the result. The string is + * '(' followed by the runtime in seconds as floating point, + * followed by 's)'. + */ + wordlen++; + if (wordlen < len && resultstring[wordlen] == '(') { + char *dup; + + wordlen++; + dup = malloc(len - wordlen + 1); + memcpy(dup, resultstring + wordlen, len - wordlen); + dup[len - wordlen] = '\0'; + *time = strtod(dup, NULL); + + free(dup); + } +} + +static void parse_subtest_result(char *subtest, const char **result, double *time, char *buf, char *bufend) +{ + char *line; + char *line_end; + char *resultstring; + size_t linelen; + size_t subtestlen = strlen(subtest); + + *result = NULL; + *time = 0.0; + + if (!buf) return; + + /* + * The result line structure is: + * + * - The string "Subtest " (`SUBTEST_RESULT` from output_strings.h) + * - The subtest name + * - The characters ':' and ' ' + * - Subtest result string + * - Optional: + * -- The characters ' ' and '(' + * -- Subtest runtime in seconds as floating point + * -- The characters 's' and ')' + * + * Example: + * Subtest subtestname: PASS (0.003s) + */ + + line = find_line_starting_with(buf, SUBTEST_RESULT, bufend); + if (!line) { + *result = "incomplete"; + return; + } + + line_end = memchr(line, '\n', bufend - line); + linelen = line_end != NULL ? line_end - line : bufend - line; + + if (strlen(SUBTEST_RESULT) + subtestlen + strlen(": ") > linelen || + strncmp(line + strlen(SUBTEST_RESULT), subtest, subtestlen)) + return parse_subtest_result(subtest, result, time, line + linelen, bufend); + + resultstring = line + strlen(SUBTEST_RESULT) + subtestlen + strlen(": "); + parse_result_string(resultstring, linelen - (resultstring - line), result, time); +} + +static struct json_object *get_or_create_json_object(struct json_object *base, + const char *key) +{ + struct json_object *ret; + + if (json_object_object_get_ex(base, key, &ret)) + return ret; + + ret = json_object_new_object(); + json_object_object_add(base, key, ret); + + return ret; +} + +static void set_result(struct json_object *obj, const char *result) +{ + json_object_object_add(obj, "result", + json_object_new_string(result)); +} + +static void add_runtime(struct json_object *obj, double time) +{ + double oldtime; + struct json_object *timeobj = get_or_create_json_object(obj, "time"); + struct json_object *oldend; + + json_object_object_add(timeobj, "__type__", + json_object_new_string("TimeAttribute")); + json_object_object_add(timeobj, "start", + json_object_new_double(0.0)); + + if (!json_object_object_get_ex(timeobj, "end", &oldend)) { + json_object_object_add(timeobj, "end", + json_object_new_double(time)); + return; + } + + /* Add the runtime to the existing runtime. */ + oldtime = json_object_get_double(oldend); + time += oldtime; + json_object_object_add(timeobj, "end", + json_object_new_double(time)); +} + +static void set_runtime(struct json_object *obj, double time) +{ + struct json_object *timeobj = get_or_create_json_object(obj, "time"); + + json_object_object_add(timeobj, "__type__", + json_object_new_string("TimeAttribute")); + json_object_object_add(timeobj, "start", + json_object_new_double(0.0)); + json_object_object_add(timeobj, "end", + json_object_new_double(time)); +} + +static bool fill_from_output(int fd, const char *binary, const char *key, + struct subtests *subtests, + struct json_object *tests) +{ + char *buf, *bufend; + struct stat statbuf; + char piglit_name[256]; + char *igt_version = NULL; + size_t igt_version_len = 0; + struct json_object *current_test = NULL; + size_t i; + + if (fstat(fd, &statbuf)) + return false; + + if (statbuf.st_size != 0) { + buf = mmap(NULL, statbuf.st_size, PROT_READ, MAP_SHARED, fd, 0); + if (buf == MAP_FAILED) + return false; + } else { + buf = NULL; + } + + bufend = buf + statbuf.st_size; + + igt_version = find_line_starting_with(buf, IGT_VERSIONSTRING, bufend); + if (igt_version) { + char *newline = memchr(igt_version, '\n', bufend - igt_version); + igt_version_len = newline - igt_version; + } + + /* TODO: Refactor to helper functions */ + if (subtests->size == 0) { + /* No subtests */ + generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(tests, piglit_name); + + json_object_object_add(current_test, key, + json_object_new_string_len(buf, statbuf.st_size)); + if (igt_version) + json_object_object_add(current_test, "igt-version", + json_object_new_string_len(igt_version, + igt_version_len)); + + return true; + } + + for (i = 0; i < subtests->size; i++) { + char *this_sub_begin, *this_sub_result; + const char *resulttext; + char *beg, *end, *startline; + double time; + int begin_len; + int result_len; + + generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(tests, piglit_name); + + begin_len = asprintf(&this_sub_begin, "%s%s\n", STARTING_SUBTEST, subtests->names[i]); + result_len = asprintf(&this_sub_result, "%s%s: ", SUBTEST_RESULT, subtests->names[i]); + + if (begin_len < 0 || result_len < 0) { + fprintf(stderr, "Failure generating strings\n"); + return false; + } + + beg = find_line_starting_with(buf, this_sub_begin, bufend); + end = find_line_starting_with(buf, this_sub_result, bufend); + startline = beg; + + free(this_sub_begin); + free(this_sub_result); + + if (beg == NULL && end == NULL) { + /* No output at all */ + beg = bufend; + end = bufend; + } + + if (beg == NULL) { + /* + * Subtest didn't start, probably skipped from + * fixture already. Start from the result + * line, it gets adjusted below. + */ + beg = end; + } + + /* Include the output after the previous subtest output */ + beg = find_line_after_last(buf, + STARTING_SUBTEST, + SUBTEST_RESULT, + beg); + + if (end == NULL) { + /* Incomplete result. Find the next starting subtest or result. */ + end = next_line(startline, bufend); + if (end != NULL) { + end = find_line_starting_with_either(end, + STARTING_SUBTEST, + SUBTEST_RESULT, + bufend); + } + if (end == NULL) { + end = bufend; + } + } else { + /* + * Now pointing to the line where this sub's + * result is. We need to include that of + * course. + */ + char *nexttest = next_line(end, bufend); + + /* Stretch onwards until the next subtest begins or ends */ + if (nexttest != NULL) { + nexttest = find_line_starting_with_either(nexttest, + STARTING_SUBTEST, + SUBTEST_RESULT, + bufend); + } + if (nexttest != NULL) { + end = nexttest; + } else { + end = bufend; + } + } + + json_object_object_add(current_test, key, + json_object_new_string_len(beg, end - beg)); + + if (igt_version) { + json_object_object_add(current_test, "igt-version", + json_object_new_string_len(igt_version, + igt_version_len)); + } + + if (!json_object_object_get_ex(current_test, "result", NULL)) { + parse_subtest_result(subtests->names[i], &resulttext, &time, beg, end); + set_result(current_test, resulttext); + set_runtime(current_test, time); + } + } + + return true; +} + +/* + * This regexp controls the kmsg handling. All kernel log records that + * have log level of warning or higher convert the result to + * dmesg-warn/dmesg-fail unless they match this regexp. + * + * TODO: Move this to external files, i915-suppressions.txt, + * general-suppressions.txt et al. + */ + +#define _ "|" +static const char igt_dmesg_whitelist[] = + "ACPI: button: The lid device is not compliant to SW_LID" _ + "ACPI: .*: Unable to dock!" _ + "IRQ [0-9]+: no longer affine to CPU[0-9]+" _ + "IRQ fixup: irq [0-9]+ move in progress, old vector [0-9]+" _ + /* i915 tests set module options, expected message */ + "Setting dangerous option [a-z_]+ - tainting kernel" _ + /* Raw printk() call, uses default log level (warn) */ + "Suspending console\\(s\\) \\(use no_console_suspend to debug\\)" _ + "atkbd serio[0-9]+: Failed to (deactivate|enable) keyboard on isa[0-9]+/serio[0-9]+" _ + "cache: parent cpu[0-9]+ should not be sleeping" _ + "hpet[0-9]+: lost [0-9]+ rtc interrupts" _ + /* i915 selftests terminate normally with ENODEV from the + * module load after the testing finishes, which produces this + * message. + */ + "i915: probe of [0-9:.]+ failed with error -25" _ + /* swiotbl warns even when asked not to */ + "mock: DMA: Out of SW-IOMMU space for [0-9]+ bytes" _ + "usb usb[0-9]+: root hub lost power or was reset" + ; +#undef _ + +static regex_t re; + +static int init_regex_whitelist(void) +{ + static int status = -1; + + if (status == -1) { + if (regcomp(&re, igt_dmesg_whitelist, REG_EXTENDED | REG_NOSUB) != 0) { + fprintf(stderr, "Cannot compile dmesg whitelist regexp\n"); + status = 1; + return false; + } + + status = 0; + } + + return status; +} + +static bool parse_dmesg_line(char* line, + unsigned *flags, unsigned long long *ts_usec, + char *continuation, char **message) +{ + unsigned long long seq; + int s; + + s = sscanf(line, "%u,%llu,%llu,%c;", flags, &seq, ts_usec, continuation); + if (s != 4) { + /* + * Machine readable key/value pairs begin with + * a space. We ignore them. + */ + if (line[0] != ' ') { + fprintf(stderr, "Cannot parse kmsg record: %s\n", line); + } + return false; + } + + *message = strchr(line, ';'); + if (!message) { + fprintf(stderr, "No ; found in kmsg record, this shouldn't happen\n"); + return false; + } + (*message)++; + + return true; +} + +static void add_dmesg(struct json_object *obj, + const char *dmesg, size_t dmesglen, + const char *warnings, size_t warningslen) +{ + json_object_object_add(obj, "dmesg", + json_object_new_string_len(dmesg, dmesglen)); + + if (warnings) { + json_object_object_add(obj, "dmesg-warnings", + json_object_new_string_len(warnings, warningslen)); + } +} + +static void add_empty_dmesgs_where_missing(struct json_object *tests, + char *binary, + struct subtests *subtests) +{ + struct json_object *current_test; + char piglit_name[256]; + size_t i; + + for (i = 0; i < subtests->size; i++) { + generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(tests, piglit_name); + if (!json_object_object_get_ex(current_test, "dmesg", NULL)) { + add_dmesg(current_test, "", 0, NULL, 0); + } + } + +} + +static bool fill_from_dmesg(int fd, char *binary, + struct subtests *subtests, + struct json_object *tests) +{ + char *line = NULL, *warnings = NULL, *dmesg = NULL; + size_t linelen = 0, warningslen = 0, dmesglen = 0; + struct json_object *current_test = NULL; + FILE *f = fdopen(fd, "r"); + char piglit_name[256]; + ssize_t read; + size_t i; + + if (!f) { + return false; + } + + if (init_regex_whitelist()) { + fclose(f); + return false; + } + + while ((read = getline(&line, &linelen, f)) > 0) { + char *formatted; + unsigned flags; + unsigned long long ts_usec; + char continuation; + char *message, *subtest; + + if (!parse_dmesg_line(line, &flags, &ts_usec, &continuation, &message)) + continue; + + asprintf(&formatted, "<%u> [%llu.%06llu] %s", + flags & 0x07, ts_usec / 1000000, ts_usec % 1000000, message); + + if ((subtest = strstr(message, STARTING_SUBTEST_DMESG)) != NULL) { + if (current_test != NULL) { + /* Done with the previous subtest, file up */ + add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen); + + free(dmesg); + free(warnings); + dmesg = warnings = NULL; + dmesglen = warningslen = 0; + } + + subtest += strlen(STARTING_SUBTEST_DMESG); + generate_piglit_name(binary, subtest, piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(tests, piglit_name); + } + + if ((flags & 0x07) <= 4 && continuation != 'c' && + regexec(&re, message, (size_t)0, NULL, 0) == REG_NOMATCH) { + append_line(&warnings, &warningslen, formatted); + } + append_line(&dmesg, &dmesglen, formatted); + free(formatted); + } + + if (current_test != NULL) { + add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen); + } else { + /* + * Didn't get any subtest messages at all. If there + * are subtests, add all of the dmesg gotten to all of + * them. + */ + for (i = 0; i < subtests->size; i++) { + generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(tests, piglit_name); + /* + * Don't bother with warnings, any subtests + * there are would have skip as their result + * anyway. + */ + add_dmesg(current_test, dmesg, dmesglen, NULL, 0); + } + + if (subtests->size == 0) { + generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(tests, piglit_name); + add_dmesg(current_test, dmesg, dmesglen, warnings, warningslen); + } + } + + add_empty_dmesgs_where_missing(tests, binary, subtests); + + free(dmesg); + free(warnings); + fclose(f); + return true; +} + +static const char *result_from_exitcode(int exitcode) +{ + switch (exitcode) { + case IGT_EXIT_TIMEOUT: + return "timeout"; + case IGT_EXIT_SKIP: + return "skip"; + case IGT_EXIT_SUCCESS: + return "pass"; + case IGT_EXIT_INVALID: + return "notrun"; + default: + return "fail"; + } +} + +static void add_subtest(struct subtests *subtests, char *subtest) +{ + size_t len = strlen(subtest); + + if (len == 0) + return; + + if (subtest[len - 1] == '\n') + subtest[len - 1] = '\0'; + + subtests->size++; + subtests->names = realloc(subtests->names, sizeof(*subtests->names) * subtests->size); + subtests->names[subtests->size - 1] = subtest; +} + +static void fill_from_journal(int fd, char *binary, + struct subtests *subtests, + struct json_object *tests) +{ + FILE *f = fdopen(fd, "r"); + char *line = NULL; + size_t linelen = 0; + ssize_t read; + char exitline[] = "exit:"; + char timeoutline[] = "timeout:"; + int exitcode = -1; + bool has_timeout = false; + + while ((read = getline(&line, &linelen, f)) > 0) { + if (read >= strlen(exitline) && !memcmp(line, exitline, strlen(exitline))) { + char *p = strchr(line, '('); + char piglit_name[256]; + double time = 0.0; + struct json_object *obj; + + generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); + obj = get_or_create_json_object(tests, piglit_name); + + exitcode = atoi(line + strlen(exitline)); + + if (p) + time = strtod(p + 1, NULL); + + add_runtime(obj, time); + } else if (read >= strlen(timeoutline) && !memcmp(line, timeoutline, strlen(timeoutline))) { + has_timeout = true; + + if (subtests->size) { + /* Assign the timeout to the previously appeared subtest */ + char *last_subtest = subtests->names[subtests->size - 1]; + char piglit_name[256]; + char *p = strchr(line, '('); + double time = 0.0; + struct json_object *obj; + + generate_piglit_name(binary, last_subtest, piglit_name, sizeof(piglit_name)); + obj = get_or_create_json_object(tests, piglit_name); + + set_result(obj, "timeout"); + + if (p) + time = strtod(p + 1, NULL); + + /* Add runtime for the subtest... */ + add_runtime(obj, time); + + /* ... and also for the binary */ + generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); + obj = get_or_create_json_object(tests, piglit_name); + add_runtime(obj, time); + } + } else { + add_subtest(subtests, strdup(line)); + } + } + + if (subtests->size == 0) { + char piglit_name[256]; + struct json_object *obj; + const char *result = has_timeout ? "timeout" : result_from_exitcode(exitcode); + + generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); + obj = get_or_create_json_object(tests, piglit_name); + set_result(obj, result); + } + + free(line); +} + +static void override_result_single(struct json_object *obj) +{ + const char *errtext = NULL, *result = NULL; + struct json_object *textobj; + bool dmesgwarns = false; + + if (json_object_object_get_ex(obj, "err", &textobj)) + errtext = json_object_get_string(textobj); + if (json_object_object_get_ex(obj, "result", &textobj)) + result = json_object_get_string(textobj); + if (json_object_object_get_ex(obj, "dmesg-warnings", &textobj)) + dmesgwarns = true; + + if (!strcmp(result, "pass") && + count_lines(errtext, errtext + strlen(errtext)) > 2) { + set_result(obj, "warn"); + } + + if (dmesgwarns) { + if (!strcmp(result, "pass") || !strcmp(result, "warn")) { + set_result(obj, "dmesg-warn"); + } else if (!strcmp(result, "fail")) { + set_result(obj, "dmesg-fail"); + } + } +} + +static void override_results(char *binary, + struct subtests *subtests, + struct json_object *tests) +{ + struct json_object *obj; + char piglit_name[256]; + size_t i; + + if (subtests->size == 0) { + generate_piglit_name(binary, NULL, piglit_name, sizeof(piglit_name)); + obj = get_or_create_json_object(tests, piglit_name); + override_result_single(obj); + return; + } + + for (i = 0; i < subtests->size; i++) { + generate_piglit_name(binary, subtests->names[i], piglit_name, sizeof(piglit_name)); + obj = get_or_create_json_object(tests, piglit_name); + override_result_single(obj); + } +} + +static bool parse_test_directory(int dirfd, char *binary, struct json_object *tests) +{ + int fds[_F_LAST]; + struct subtests subtests = {}; + + if (!open_output_files(dirfd, fds, false)) { + fprintf(stderr, "Error opening output files\n"); + return false; + } + + /* + * fill_from_journal fills the subtests struct and adds + * timeout results where applicable. + */ + fill_from_journal(fds[_F_JOURNAL], binary, &subtests, tests); + + if (!fill_from_output(fds[_F_OUT], binary, "out", &subtests, tests) || + !fill_from_output(fds[_F_ERR], binary, "err", &subtests, tests) || + !fill_from_dmesg(fds[_F_DMESG], binary, &subtests, tests)) { + fprintf(stderr, "Error parsing output files\n"); + return false; + } + + override_results(binary, &subtests, tests); + + close_outputs(fds); + + return true; +} + +bool generate_results(int dirfd) +{ + struct settings settings; + struct job_list job_list; + struct json_object *obj, *tests; + int resultsfd, testdirfd, unamefd; + const char *json_string; + size_t i; + + init_settings(&settings); + init_job_list(&job_list); + + if (!read_settings(&settings, dirfd)) { + fprintf(stderr, "resultgen: Cannot parse settings\n"); + return false; + } + + if (!read_job_list(&job_list, dirfd)) { + fprintf(stderr, "resultgen: Cannot parse job list\n"); + return false; + } + + /* TODO: settings.overwrite */ + if ((resultsfd = openat(dirfd, "results.json", O_WRONLY | O_CREAT | O_TRUNC, 0666)) < 0) { + fprintf(stderr, "resultgen: Cannot create results file\n"); + return false; + } + + obj = json_object_new_object(); + json_object_object_add(obj, "__type__", json_object_new_string("TestrunResult")); + json_object_object_add(obj, "results_version", json_object_new_int(9)); + json_object_object_add(obj, "name", + settings.name ? + json_object_new_string(settings.name) : + json_object_new_string("")); + + if ((unamefd = openat(dirfd, "uname.txt", O_RDONLY)) >= 0) { + char buf[128]; + ssize_t r = read(unamefd, buf, 128); + + if (r > 0 && buf[r - 1] == '\n') + r--; + + json_object_object_add(obj, "uname", + json_object_new_string_len(buf, r)); + close(unamefd); + } + + /* + * Result fields that won't be added: + * + * - glxinfo + * - wglinfo + * - clinfo + * + * Result fields that are TODO: + * + * - lspci + * - options + * - time_elapsed + * - totals + */ + + tests = json_object_new_object(); + json_object_object_add(obj, "tests", tests); + + for (i = 0; i < job_list.size; i++) { + char name[16]; + + snprintf(name, 16, "%zd", i); + if ((testdirfd = openat(dirfd, name, O_DIRECTORY | O_RDONLY)) < 0) { + fprintf(stderr, "Warning: Cannot open result directory %s\n", name); + break; + } + + if (!parse_test_directory(testdirfd, job_list.entries[i].binary, tests)) { + close(resultsfd); + return false; + } + } + + json_string = json_object_to_json_string_ext(obj, JSON_C_TO_STRING_PRETTY); + write(resultsfd, json_string, strlen(json_string)); + return true; +} + +bool generate_results_path(char *resultspath) +{ + int dirfd = open(resultspath, O_DIRECTORY | O_RDONLY); + + if (dirfd < 0) + return false; + + return generate_results(dirfd); +} diff --git a/runner/resultgen.h b/runner/resultgen.h new file mode 100644 index 00000000..83a0876b --- /dev/null +++ b/runner/resultgen.h @@ -0,0 +1,9 @@ +#ifndef RUNNER_RESULTGEN_H +#define RUNNER_RESULTGEN_H + +#include <stdbool.h> + +bool generate_results(int dirfd); +bool generate_results_path(char *resultspath); + +#endif diff --git a/runner/results.c b/runner/results.c new file mode 100644 index 00000000..3eb7cb15 --- /dev/null +++ b/runner/results.c @@ -0,0 +1,26 @@ +#include <fcntl.h> +#include <stdio.h> +#include <stdlib.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "resultgen.h" + +int main(int argc, char **argv) +{ + int dirfd; + + if (argc < 2) + exit(1); + + dirfd = open(argv[1], O_DIRECTORY | O_RDONLY); + if (dirfd < 0) + exit(1); + + if (generate_results(dirfd)) { + printf("Results generated\n"); + exit(0); + } + + exit(1); +} diff --git a/runner/resume.c b/runner/resume.c new file mode 100644 index 00000000..b3a2a71e --- /dev/null +++ b/runner/resume.c @@ -0,0 +1,47 @@ +#include <errno.h> +#include <fcntl.h> +#include <stdio.h> +#include <string.h> +#include <sys/stat.h> +#include <sys/types.h> + +#include "settings.h" +#include "job_list.h" +#include "executor.h" +#include "resultgen.h" + +int main(int argc, char **argv) +{ + struct settings settings; + struct job_list job_list; + struct execute_state state; + int dirfd; + + init_settings(&settings); + init_job_list(&job_list); + + if (argc < 2) { + fprintf(stderr, "Usage: %s results-directory\n", argv[0]); + return 1; + } + + if ((dirfd = open(argv[1], O_RDONLY | O_DIRECTORY)) < 0) { + fprintf(stderr, "Failure opening %s: %s\n", argv[1], strerror(errno)); + return 1; + } + + if (!initialize_execute_state_from_resume(dirfd, &state, &settings, &job_list)) { + return 1; + } + + if (!execute(&state, &settings, &job_list)) { + return 1; + } + + if (!generate_results_path(settings.results_path)) { + return 1; + } + + printf("Done.\n"); + return 0; +} diff --git a/runner/runner.c b/runner/runner.c new file mode 100644 index 00000000..b685786a --- /dev/null +++ b/runner/runner.c @@ -0,0 +1,40 @@ +#include <stdio.h> +#include <string.h> + +#include "settings.h" +#include "job_list.h" +#include "executor.h" +#include "resultgen.h" + +int main(int argc, char **argv) +{ + struct settings settings; + struct job_list job_list; + struct execute_state state; + + init_settings(&settings); + init_job_list(&job_list); + + if (!parse_options(argc, argv, &settings)) { + return 1; + } + + if (!create_job_list(&job_list, &settings)) { + return 1; + } + + if (!initialize_execute_state(&state, &settings, &job_list)) { + return 1; + } + + if (!execute(&state, &settings, &job_list)) { + return 1; + } + + if (!generate_results_path(settings.results_path)) { + return 1; + } + + printf("Done.\n"); + return 0; +} diff --git a/runner/settings.c b/runner/settings.c new file mode 100644 index 00000000..31754a12 --- /dev/null +++ b/runner/settings.c @@ -0,0 +1,502 @@ +#include "settings.h" + +#include <errno.h> +#include <fcntl.h> +#include <getopt.h> +#include <libgen.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <sys/types.h> +#include <sys/stat.h> +#include <unistd.h> + +enum { + OPT_ABORT_ON_ERROR, + OPT_TEST_LIST, + OPT_IGNORE_MISSING, + OPT_HELP = 'h', + OPT_NAME = 'n', + OPT_DRY_RUN = 'd', + OPT_INCLUDE = 't', + OPT_EXCLUDE = 'x', + OPT_SYNC = 's', + OPT_LOG_LEVEL = 'l', + OPT_OVERWRITE = 'o', + OPT_MULTIPLE = 'm', + OPT_TIMEOUT = 'c', + OPT_WATCHDOG = 'g', +}; + +static struct { + int level; + const char *name; +} log_levels[] = { + { LOG_LEVEL_NORMAL, "normal" }, + { LOG_LEVEL_QUIET, "quiet" }, + { LOG_LEVEL_VERBOSE, "verbose" }, + { 0, 0 }, +}; + +static bool set_log_level(struct settings* settings, const char *level) +{ + typeof(*log_levels) *it; + + for (it = log_levels; it->name; it++) { + if (!strcmp(level, it->name)) { + settings->log_level = it->level; + return true; + } + } + + return false; +} + +static const char *usage_str = + "usage: runner [options] [test_root] results-path\n\n" + "Options:\n" + " Piglit compatible:\n" + " -h, --help Show this help message and exit\n" + " -n <test name>, --name <test name>\n" + " Name of this test run\n" + " -d, --dry-run Do not execute the tests\n" + " -t <regex>, --include-tests <regex>\n" + " Run only matching tests (can be used more than once)\n" + " -x <regex>, --exclude-tests <regex>\n" + " Exclude matching tests (can be used more than once)\n" + " --abort-on-monitored-error\n" + " Abort execution when a fatal condition is detected.\n" + " <TODO>\n" + " -s, --sync Sync results to disk after every test\n" + " -l {quiet,verbose,dummy}, --log-level {quiet,verbose,dummy}\n" + " Set the logger verbosity level\n" + " --test-list TEST_LIST\n" + " A file containing a list of tests to run\n" + " -o, --overwrite If the results-path already exists, delete it\n" + " --ignore-missing Ignored but accepted, for piglit compatibility\n" + "\n" + " Incompatible options:\n" + " -m, --multiple-mode Run multiple subtests in the same binary execution.\n" + " If a testlist file is given, consecutive subtests are\n" + " run in the same execution if they are from the same\n" + " binary. Note that in that case relative ordering of the\n" + " subtest execution is dictated by the test binary, not\n" + " the testlist\n" + " --inactivity-timeout <seconds>\n" + " Kill the running test after <seconds> of inactivity in\n" + " the test's stdout, stderr, or dmesg\n" + " --use-watchdog Use hardware watchdog for lethal enforcement of the\n" + " above timeout. Killing the test process is still\n" + " attempted at timeout trigger.\n" + " [test_root] Directory that contains the IGT tests. The environment\n" + " variable IGT_TEST_ROOT will be used if set, overriding\n" + " this option if given.\n" + ; + +static void usage(const char *extra_message, FILE *f) +{ + if (extra_message) + fprintf(f, "%s\n\n", extra_message); + + fputs(usage_str, f); +} + +static bool add_regex(struct regex_list *list, char *new) +{ + regex_t *regex; + size_t buflen; + char *buf; + int s; + + regex = malloc(sizeof(*regex)); + + if ((s = regcomp(regex, new, + REG_EXTENDED | REG_NOSUB)) != 0) { + buflen = regerror(s, regex, NULL, 0); + buf = malloc(buflen); + regerror(s, regex, buf, buflen); + usage(buf, stderr); + + free(buf); + regfree(regex); + free(regex); + return false; + } + + list->regexes = realloc(list->regexes, + (list->size + 1) * sizeof(*list->regexes)); + list->regex_strings = realloc(list->regex_strings, + (list->size + 1) * sizeof(*list->regex_strings)); + list->regexes[list->size] = regex; + list->regex_strings[list->size] = new; + list->size++; + + return true; +} + +static void free_regexes(struct regex_list *regexes) +{ + size_t i; + + for (i = 0; i < regexes->size; i++) { + free(regexes->regex_strings[i]); + regfree(regexes->regexes[i]); + free(regexes->regexes[i]); + } + free(regexes->regex_strings); + free(regexes->regexes); +} + +static bool readable_file(char *filename) +{ + return !access(filename, R_OK); +} + +void init_settings(struct settings *settings) +{ + memset(settings, 0, sizeof(*settings)); +} + +void free_settings(struct settings *settings) +{ + free(settings->test_list); + free(settings->name); + free(settings->test_root); + free(settings->results_path); + + free_regexes(&settings->include_regexes); + free_regexes(&settings->exclude_regexes); + + init_settings(settings); +} + +bool parse_options(int argc, char **argv, + struct settings *settings) +{ + int c; + char *env_test_root; + + static struct option long_options[] = { + {"help", no_argument, NULL, OPT_HELP}, + {"name", required_argument, NULL, OPT_NAME}, + {"dry-run", no_argument, NULL, OPT_DRY_RUN}, + {"include-tests", required_argument, NULL, OPT_INCLUDE}, + {"exclude-tests", required_argument, NULL, OPT_EXCLUDE}, + {"abort-on-monitored-error", no_argument, NULL, OPT_ABORT_ON_ERROR}, + {"sync", no_argument, NULL, OPT_SYNC}, + {"log-level", required_argument, NULL, OPT_LOG_LEVEL}, + {"test-list", required_argument, NULL, OPT_TEST_LIST}, + {"overwrite", no_argument, NULL, OPT_OVERWRITE}, + {"ignore-missing", no_argument, NULL, OPT_IGNORE_MISSING}, + {"multiple-mode", no_argument, NULL, OPT_MULTIPLE}, + {"inactivity-timeout", required_argument, NULL, OPT_TIMEOUT}, + {"use-watchdog", no_argument, NULL, OPT_WATCHDOG}, + { 0, 0, 0, 0}, + }; + + free_settings(settings); + + optind = 1; + + while ((c = getopt_long(argc, argv, "hn:dt:x:sl:om", long_options, NULL)) != -1) { + switch (c) { + case OPT_HELP: + usage(NULL, stdout); + goto error; + case OPT_NAME: + settings->name = strdup(optarg); + break; + case OPT_DRY_RUN: + settings->dry_run = true; + break; + case OPT_INCLUDE: + if (!add_regex(&settings->include_regexes, strdup(optarg))) + goto error; + break; + case OPT_EXCLUDE: + if (!add_regex(&settings->exclude_regexes, strdup(optarg))) + goto error; + break; + case OPT_ABORT_ON_ERROR: + settings->abort_on_error = true; + break; + case OPT_SYNC: + settings->sync = true; + break; + case OPT_LOG_LEVEL: + if (!set_log_level(settings, optarg)) { + usage("Cannot parse log level", stderr); + goto error; + } + break; + case OPT_TEST_LIST: + settings->test_list = absolute_path(optarg); + break; + case OPT_OVERWRITE: + settings->overwrite = true; + break; + case OPT_IGNORE_MISSING: + /* Ignored, piglit compatibility */ + break; + case OPT_MULTIPLE: + settings->multiple_mode = true; + break; + case OPT_TIMEOUT: + settings->inactivity_timeout = atoi(optarg); + break; + case OPT_WATCHDOG: + settings->use_watchdog = true; + break; + case '?': + usage(NULL, stderr); + goto error; + default: + usage("Cannot parse options", stderr); + goto error; + } + } + + switch (argc - optind) { + case 2: + settings->test_root = absolute_path(argv[optind]); + ++optind; + /* fallthrough */ + case 1: + settings->results_path = absolute_path(argv[optind]); + break; + case 0: + usage("Results-path missing", stderr); + goto error; + default: + usage("Extra arguments after results-path", stderr); + goto error; + } + + if ((env_test_root = getenv("IGT_TEST_ROOT")) != NULL) { + free(settings->test_root); + settings->test_root = absolute_path(env_test_root); + } + + if (!settings->test_root) { + usage("Test root not set", stderr); + goto error; + } + + if (!settings->name) { + char *name = strdup(settings->results_path); + settings->name = strdup(basename(name)); + free(name); + } + + return true; + + error: + free_settings(settings); + return false; +} + +bool validate_settings(struct settings *settings) +{ + int dirfd, fd; + + if (settings->test_list && !readable_file(settings->test_list)) { + usage("Cannot open test-list file", stderr); + return false; + } + + if (!settings->results_path) { + usage("No results-path set; this shouldn't happen", stderr); + return false; + } + + if (!settings->test_root) { + usage("No test root set; this shouldn't happen", stderr); + return false; + } + + dirfd = open(settings->test_root, O_DIRECTORY | O_RDONLY); + if (dirfd < 0) { + fprintf(stderr, "Test directory %s cannot be opened\n", settings->test_root); + return false; + } + + fd = openat(dirfd, "test-list.txt", O_RDONLY); + if (fd < 0) { + fprintf(stderr, "Cannot open %s/test-list.txt\n", settings->test_root); + close(dirfd); + return false; + } + + close(fd); + close(dirfd); + + return true; +} + +char *absolute_path(char *path) +{ + char *result = NULL; + char *tmppath, *tmpname; + + result = realpath(path, NULL); + if (result != NULL) + return result; + + tmppath = strdup(path); + tmpname = dirname(tmppath); + free(result); + result = realpath(tmpname, NULL); + free(tmppath); + + if (result != NULL) { + char *ret; + + tmppath = strdup(path); + tmpname = basename(tmppath); + + asprintf(&ret, "%s/%s", result, tmpname); + free(result); + free(tmppath); + return ret; + } + + free(result); + return NULL; +} + +static char settings_filename[] = "metadata.txt"; +bool serialize_settings(struct settings *settings) +{ +#define SERIALIZE_LINE(f, s, name, format) fprintf(f, "%s : " format "\n", #name, s->name) + + int dirfd, fd; + FILE *f; + + if (!settings->results_path) { + usage("No results-path set; this shouldn't happen", stderr); + return false; + } + + if ((dirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) { + mkdir(settings->results_path, 0755); + if ((dirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) { + usage("Creating results-path failed", stderr); + return false; + } + } + + if (!settings->overwrite && + faccessat(dirfd, settings_filename, F_OK, 0) == 0) { + usage("Settings metadata already exists and not overwriting", stderr); + return false; + } + + if (settings->overwrite && + unlinkat(dirfd, settings_filename, 0) != 0 && + errno != ENOENT) { + usage("Error removing old settings metadata", stderr); + return false; + } + + if ((fd = openat(dirfd, settings_filename, O_CREAT | O_EXCL | O_WRONLY, 0666)) < 0) { + char *msg; + + asprintf(&msg, "Creating settings serialization file failed: %s", strerror(errno)); + usage(msg, stderr); + + free(msg); + close(dirfd); + return false; + } + + f = fdopen(fd, "w"); + if (!f) { + close(fd); + close(dirfd); + return false; + } + + SERIALIZE_LINE(f, settings, abort_on_error, "%d"); + if (settings->test_list) + SERIALIZE_LINE(f, settings, test_list, "%s"); + if (settings->name) + SERIALIZE_LINE(f, settings, name, "%s"); + SERIALIZE_LINE(f, settings, dry_run, "%d"); + SERIALIZE_LINE(f, settings, sync, "%d"); + SERIALIZE_LINE(f, settings, log_level, "%d"); + SERIALIZE_LINE(f, settings, overwrite, "%d"); + SERIALIZE_LINE(f, settings, multiple_mode, "%d"); + SERIALIZE_LINE(f, settings, inactivity_timeout, "%d"); + SERIALIZE_LINE(f, settings, use_watchdog, "%d"); + SERIALIZE_LINE(f, settings, test_root, "%s"); + SERIALIZE_LINE(f, settings, results_path, "%s"); + + if (settings->sync) { + fsync(fd); + fsync(dirfd); + } + + fclose(f); + close(dirfd); + return true; + +#undef SERIALIZE_LINE +} + +bool read_settings(struct settings *settings, int dirfd) +{ +#define PARSE_LINE(s, name, val, field, write) \ + if (!strcmp(name, #field)) { \ + s->field = write; \ + free(name); \ + free(val); \ + name = val = NULL; \ + continue; \ + } + + int fd; + FILE *f; + char *name = NULL, *val = NULL; + + free_settings(settings); + + if ((fd = openat(dirfd, settings_filename, O_RDONLY)) < 0) + return false; + + f = fdopen(fd, "r"); + if (!f) { + close(fd); + return false; + } + + while (fscanf(f, "%ms : %ms", &name, &val) == 2) { + int numval = atoi(val); + PARSE_LINE(settings, name, val, abort_on_error, numval); + PARSE_LINE(settings, name, val, test_list, val ? strdup(val) : NULL); + PARSE_LINE(settings, name, val, name, val ? strdup(val) : NULL); + PARSE_LINE(settings, name, val, dry_run, numval); + PARSE_LINE(settings, name, val, sync, numval); + PARSE_LINE(settings, name, val, log_level, numval); + PARSE_LINE(settings, name, val, overwrite, numval); + PARSE_LINE(settings, name, val, multiple_mode, numval); + PARSE_LINE(settings, name, val, inactivity_timeout, numval); + PARSE_LINE(settings, name, val, use_watchdog, numval); + PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL); + PARSE_LINE(settings, name, val, results_path, val ? strdup(val) : NULL); + + printf("Warning: Unknown field in settings file: %s = %s\n", + name, val); + free(name); + free(val); + name = val = NULL; + } + + free(name); + free(val); + fclose(f); + + return true; + +#undef PARSE_LINE +} diff --git a/runner/settings.h b/runner/settings.h new file mode 100644 index 00000000..9d1f03fb --- /dev/null +++ b/runner/settings.h @@ -0,0 +1,111 @@ +#ifndef RUNNER_SETTINGS_H +#define RUNNER_SETTINGS_H + +#include <stdbool.h> +#include <stddef.h> +#include <sys/types.h> +#include <regex.h> + +enum { + LOG_LEVEL_NORMAL = 0, + LOG_LEVEL_QUIET = -1, + LOG_LEVEL_VERBOSE = 1, +}; + +struct regex_list { + char **regex_strings; + regex_t** regexes; + size_t size; +}; + +struct settings { + bool abort_on_error; + char *test_list; + char *name; + bool dry_run; + struct regex_list include_regexes; + struct regex_list exclude_regexes; + bool sync; + int log_level; + bool overwrite; + bool multiple_mode; + int inactivity_timeout; + bool use_watchdog; + char *test_root; + char *results_path; +}; + +/** + * init_settings: + * + * Initializes a settings object to an empty state (all values NULL, 0 + * or false). + * + * @settings: Object to initialize. Storage for it must exist. + */ +void init_settings(struct settings *settings); + +/** + * free_settings: + * + * Releases all allocated resources for a settings object and + * initializes it to an empty state (see #init_settings). + * + * @settings: Object to release and initialize. + */ +void free_settings(struct settings *settings); + +/** + * parse_options: + * + * Parses command line options and sets the settings object to + * designated values. + * + * The function can be called again on the same settings object. The + * old values will be properly released and cleared. On a parse + * failure, the settings object will be in an empty state (see + * #init_settings) and usage instructions will be printed with an + * error message. + * + * @argc: Argument count + * @argv: Argument array. First element is the program name. + * @settings: Settings object to fill with values. Must have proper + * storage. + * + * Returns: True on successful parse, false on error. + */ +bool parse_options(int argc, char **argv, + struct settings *settings); + +/** + * validate_settings: + * + * Checks the settings object against the system to see if executing + * on it can be done. Checks pathnames for existence and access + * rights. Note that this function will not check that the designated + * job listing (through a test-list file or the -t/-x flags) yields a + * non-zero amount of testing to be done. On errors, usage + * instructions will be printed with an error message. + * + * @settings: Settings object to check. + * + * Returns: True on valid settings, false on any error. + */ +bool validate_settings(struct settings *settings); + +/* TODO: Better place for this */ +char *absolute_path(char *path); + +/** + * serialize_settings: + * + * Serializes the settings object to a file in the results_path + * directory. + * + * @settings: Settings object to serialize. + */ +bool serialize_settings(struct settings *settings); + +bool read_settings(struct settings *settings, int dirfd); + +#endif |