runner: Add support for code coverage

The gcc compiler has a feature that enables checking the code coverage
in runtime[1].

[1] See https://www.kernel.org/doc/html/latest/dev-tools/gcov.html

The Linux Kernel comes with an option to enable such feature:

	./scripts/config -e DEBUG_FS -e GCOV_KERNEL

The driver's Makefile also needs change to enable it. For instance, in
order to enable GCOV for all DRM drivers, one would need to run:

	for i in $(find drivers/gpu/drm/ -name Makefile); do
                sed '1 a GCOV_PROFILE := y' -i $i
        done

This patch adds support for it by:

a) Implementing a logic to cleanup the code coverage counters via sysfs;
b) Calling a script responsible for collecging code coverage data.

The implementation works with two modes:

1) It zeroes the counters, run all IGT tests and collects the code
   coverage results at the end.

   This implies that no tests would crash the driver, as otherwise the
   results won't be collected;

   This is faster, as collecting code coverage data can take several
   seconds.

2) For each test, it will clean the code coverage counters, run the and
   collect the results.

   This is more reliable, as a Kernel crash/OOPS won't affect the
   results of the previously ran tests.

Reviewed-by: Petri Latvala <petri.latvala@intel.com>
Signed-off-by: Mauro Carvalho Chehab <mchehab@kernel.org>

4 files changed, 265 insertions, 10 deletions

diff --git a/runner/executor.c b/runner/executor.c
index 15bd53dd..ab00900c 100644
--- a/runner/executor.c
+++ b/runner/executor.c
@@ -1,3 +1,4 @@
+#include <ctype.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <glib.h>
@@ -1693,6 +1694,140 @@ static bool should_die_because_signal(int sigfd)
 	return false;
 }
 
+static char *code_coverage_name(struct settings *settings)
+{
+	const char *start, *end, *fname;
+	char *name;
+	int size;
+
+	if (settings->name && *settings->name)
+		return settings->name;
+	else if (!settings->test_list)
+		return NULL;
+
+	/* Use only the base of the test_list, without path and extension */
+	fname = settings->test_list;
+
+	start = strrchr(fname,'/');
+	if (!start)
+		start = fname;
+
+	end = strrchr(start, '.');
+	if (end)
+		size = end - start;
+	else
+		size = strlen(start);
+
+	name = malloc(size + 1);
+	strncpy(name, fname, size);
+	name[size]  = '\0';
+
+	return name;
+}
+
+static void run_as_root(char * const argv[], int sigfd, char **abortreason)
+{
+	struct signalfd_siginfo siginfo;
+	int status = 0, ret;
+	pid_t child;
+
+	child = fork();
+	if (child < 0) {
+		*abortreason = strdup("Failed to fork");
+		return;
+	}
+
+	if (child == 0) {
+		execv(argv[0], argv);
+		perror (argv[0]);
+		exit(IGT_EXIT_INVALID);
+	}
+
+	if (sigfd >= 0) {
+		while (1) {
+			ret = read(sigfd, &siginfo, sizeof(siginfo));
+			if (ret < 0) {
+				errf("Error reading from signalfd: %m\n");
+				continue;
+			} else if (siginfo.ssi_signo == SIGCHLD) {
+				if (child != waitpid(child, &status, WNOHANG)) {
+					errf("Failed to reap child\n");
+					status = 9999;
+					continue;
+				}
+				break;
+			}
+		}
+	} else {
+		waitpid(child, &status, 0);
+	}
+
+	if (WIFSIGNALED(status))
+		asprintf(abortreason, "%s received signal %d while running\n",argv[0], WTERMSIG(status));
+	else if (!WIFEXITED(status))
+		asprintf(abortreason, "%s aborted with unknown status\n", argv[0]);
+	else if (WEXITSTATUS(status))
+		asprintf(abortreason, "%s returned error %d\n", argv[0], WEXITSTATUS(status));
+}
+
+static void code_coverage_start(struct settings *settings, int sigfd, char **abortreason)
+{
+	int fd;
+
+	fd = open(GCOV_RESET, O_WRONLY);
+	if (fd < 0) {
+		asprintf(abortreason, "Failed to open %s", GCOV_RESET);
+		return;
+	}
+	if (write(fd, "0\n", 2) < 0)
+		*abortreason = strdup("Failed to reset gcov counters");
+
+	close(fd);
+}
+
+static void code_coverage_stop(struct settings *settings, const char *job_name,
+			       int sigfd, char **abortreason)
+{
+	int i, j = 0, last_was_escaped = 1;
+	char fname[PATH_MAX];
+	char name[PATH_MAX];
+	char *argv[3] = {};
+
+	/* If name is empty, use a default */
+	if (!job_name || !*job_name)
+		job_name = "code_coverage";
+
+	/*
+	 * Use only letters, numbers and '_'
+	 *
+	 * This way, the tarball name can be used as testname when lcov runs
+	 */
+	for (i = 0; i < strlen(job_name); i++) {
+		if (!isalpha(job_name[i]) && !isalnum(job_name[i])) {
+			if (last_was_escaped)
+				continue;
+			name[j++] = '_';
+			last_was_escaped = 1;
+		} else {
+			name[j++] = job_name[i];
+			last_was_escaped = 0;
+		}
+	}
+	if (j && last_was_escaped)
+		j--;
+	name[j] = '\0';
+
+	strcpy(fname, settings->results_path);
+	strcat(fname, CODE_COV_RESULTS_PATH "/");
+	strcat(fname, name);
+
+	argv[0] = settings->code_coverage_script;
+	argv[1] = fname;
+
+	outf("Storing code coverage results...\n");
+	run_as_root(argv, sigfd, abortreason);
+}
+
 bool execute(struct execute_state *state,
 	     struct settings *settings,
 	     struct job_list *job_list)
@@ -1709,6 +1844,17 @@ bool execute(struct execute_state *state,
 		return true;
 	}
 
+	if (settings->enable_code_coverage && !settings->cov_results_per_test) {
+		char *reason = NULL;
+
+		code_coverage_start(settings, -1, &reason);
+		if (reason != NULL) {
+			errf("%s\n", reason);
+			free(reason);
+			status = false;
+		}
+	}
+
 	if ((resdirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) {
 		/* Initialize state should have done this */
 		errf("Error: Failure opening results path %s\n",
@@ -1795,6 +1941,7 @@ bool execute(struct execute_state *state,
 	for (; state->next < job_list->size;
 	     state->next++) {
 		char *reason = NULL;
+		char *job_name;
 		int result;
 
 		if (should_die_because_signal(sigfd)) {
@@ -1802,14 +1949,26 @@ bool execute(struct execute_state *state,
 			goto end;
 		}
 
-		result = execute_next_entry(state,
-					    job_list->size,
-					    &time_spent,
-					    settings,
-					    &job_list->entries[state->next],
-					    testdirfd, resdirfd,
-					    sigfd, &sigmask,
-					    &reason);
+		if (settings->cov_results_per_test) {
+			code_coverage_start(settings, sigfd, &reason);
+			job_name = entry_display_name(&job_list->entries[state->next]);
+		}
+
+		if (reason == NULL) {
+			result = execute_next_entry(state,
+						job_list->size,
+						&time_spent,
+						settings,
+						&job_list->entries[state->next],
+						testdirfd, resdirfd,
+						sigfd, &sigmask,
+						&reason);
+
+			if (settings->cov_results_per_test) {
+				code_coverage_stop(settings, job_name, sigfd, &reason);
+				free(job_name);
+			}
+		}
 
 		if (reason != NULL || (reason = need_to_abort(settings)) != NULL) {
 			char *prev = entry_display_name(&job_list->entries[state->next]);
@@ -1864,6 +2023,17 @@ bool execute(struct execute_state *state,
 	}
 
  end:
+	if (settings->enable_code_coverage && !settings->cov_results_per_test) {
+		char *reason = NULL;
+
+		code_coverage_stop(settings, code_coverage_name(settings), -1, &reason);
+		if (reason != NULL) {
+			errf("%s\n", reason);
+			free(reason);
+			status = false;
+		}
+	}
+
 	close_watchdogs(settings);
 	sigprocmask(SIG_UNBLOCK, &sigmask, NULL);
 	/* make sure that we do not leave any signals unhandled */
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index e67e08a8..96ffbf1f 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -443,6 +443,9 @@ igt_main
 				       "--use-watchdog",
 				       "--piglit-style-dmesg",
 				       "--dmesg-warn-level=3",
+				       "--collect-code-cov",
+				       "--coverage-per-test",
+				       "--collect-script", "/usr/bin/true",
 				       "test-root-dir",
 				       "path-to-results",
 		};
diff --git a/runner/settings.c b/runner/settings.c
index ac090b08..a7a12f50 100644
--- a/runner/settings.c
+++ b/runner/settings.c
@@ -25,6 +25,9 @@ enum {
 	OPT_OVERALL_TIMEOUT,
 	OPT_PER_TEST_TIMEOUT,
 	OPT_ALLOW_NON_ROOT,
+	OPT_CODE_COV_SCRIPT,
+	OPT_ENABLE_CODE_COVERAGE,
+	OPT_COV_RESULTS_PER_TEST,
 	OPT_VERSION,
 	OPT_HELP = 'h',
 	OPT_NAME = 'n',
@@ -240,6 +243,13 @@ static const char *usage_str =
 	"                        Exclude all test matching to regexes from FILENAME\n"
 	"                        (can be used more than once)\n"
 	"  -L, --list-all        List all matching subtests instead of running\n"
+	"  --collect-code-cov    Enables gcov-based collect of code coverage for tests.\n"
+	"                        Requires --collect-script FILENAME\n"
+	"  --coverage-per-test   Stores code coverage results per each test.\n"
+	"                        Requires --collect-script FILENAME\n"
+	"  --collect-script FILENAME\n"
+	"                        Use FILENAME as script to collect code coverage data.\n"
+	"\n"
 	"  [test_root]           Directory that contains the IGT tests. The environment\n"
 	"                        variable IGT_TEST_ROOT will be used if set, overriding\n"
 	"                        this option if given.\n"
@@ -338,11 +348,21 @@ static void free_regexes(struct regex_list *regexes)
 	free(regexes->regexes);
 }
 
-static bool readable_file(char *filename)
+static bool readable_file(const char *filename)
 {
 	return !access(filename, R_OK);
 }
 
+static bool writeable_file(const char *filename)
+{
+	return !access(filename, W_OK);
+}
+
+static bool executable_file(const char *filename)
+{
+	return !access(filename, X_OK);
+}
+
 static void print_version(void)
 {
 	struct utsname uts;
@@ -393,6 +413,9 @@ bool parse_options(int argc, char **argv,
 		{"test-list", required_argument, NULL, OPT_TEST_LIST},
 		{"overwrite", no_argument, NULL, OPT_OVERWRITE},
 		{"ignore-missing", no_argument, NULL, OPT_IGNORE_MISSING},
+		{"collect-code-cov", no_argument, NULL, OPT_ENABLE_CODE_COVERAGE},
+		{"coverage-per-test", no_argument, NULL, OPT_COV_RESULTS_PER_TEST},
+		{"collect-script", required_argument, NULL, OPT_CODE_COV_SCRIPT},
 		{"multiple-mode", no_argument, NULL, OPT_MULTIPLE},
 		{"inactivity-timeout", required_argument, NULL, OPT_TIMEOUT},
 		{"per-test-timeout", required_argument, NULL, OPT_PER_TEST_TIMEOUT},
@@ -465,6 +488,16 @@ bool parse_options(int argc, char **argv,
 		case OPT_IGNORE_MISSING:
 			/* Ignored, piglit compatibility */
 			break;
+		case OPT_ENABLE_CODE_COVERAGE:
+			settings->enable_code_coverage = true;
+			break;
+		case OPT_COV_RESULTS_PER_TEST:
+			settings->cov_results_per_test = true;
+			break;
+		case OPT_CODE_COV_SCRIPT:
+			settings->code_coverage_script = absolute_path(optarg);
+			break;
+
 		case OPT_MULTIPLE:
 			settings->multiple_mode = true;
 			break;
@@ -597,6 +630,29 @@ bool validate_settings(struct settings *settings)
 	close(fd);
 	close(dirfd);
 
+	/* enables code coverage when --coverage-per-test is used */
+	if (settings->cov_results_per_test)
+		settings->enable_code_coverage = true;
+
+	if (!settings->allow_non_root && (getuid() != 0)) {
+		fprintf(stderr, "Runner needs to run with UID 0 (root).\n");
+		return false;
+	}
+
+	if (settings->enable_code_coverage) {
+		if (!executable_file(settings->code_coverage_script)) {
+			fprintf(stderr, "%s doesn't exist or is not executable\n", settings->code_coverage_script);
+			return false;
+		}
+		if (!writeable_file(GCOV_RESET)) {
+			if (getuid() != 0)
+				fprintf(stderr, "Code coverage requires root.\n");
+			else
+				fprintf(stderr, "Is GCOV enabled? Can't access %s stat.\n", GCOV_RESET);
+			return false;
+		}
+	}
+
 	return true;
 }
 
@@ -645,7 +701,8 @@ bool serialize_settings(struct settings *settings)
 {
 #define SERIALIZE_LINE(f, s, name, format) fprintf(f, "%s : " format "\n", #name, s->name)
 
-	int dirfd, fd;
+	int dirfd, covfd, fd;
+	char path[PATH_MAX];
 	FILE *f;
 
 	if (!settings->results_path) {
@@ -660,6 +717,18 @@ bool serialize_settings(struct settings *settings)
 			return false;
 		}
 	}
+	if (settings->enable_code_coverage) {
+		strcpy(path, settings->results_path);
+		strcat(path, CODE_COV_RESULTS_PATH);
+		if ((covfd = open(path, O_DIRECTORY | O_RDONLY)) < 0) {
+			if (mkdir(path, 0755)) {
+				usage("Creating code coverage path failed", stderr);
+				return false;
+			}
+		} else {
+			close(covfd);
+		}
+	}
 
 	if (!settings->overwrite &&
 	    faccessat(dirfd, settings_filename, F_OK, 0) == 0) {
@@ -712,6 +781,9 @@ bool serialize_settings(struct settings *settings)
 	SERIALIZE_LINE(f, settings, dmesg_warn_level, "%d");
 	SERIALIZE_LINE(f, settings, test_root, "%s");
 	SERIALIZE_LINE(f, settings, results_path, "%s");
+	SERIALIZE_LINE(f, settings, enable_code_coverage, "%d");
+	SERIALIZE_LINE(f, settings, cov_results_per_test, "%d");
+	SERIALIZE_LINE(f, settings, code_coverage_script, "%s");
 
 	if (settings->sync) {
 		fsync(fd);
@@ -760,6 +832,9 @@ bool read_settings_from_file(struct settings *settings, FILE *f)
 		PARSE_LINE(settings, name, val, dmesg_warn_level, numval);
 		PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL);
 		PARSE_LINE(settings, name, val, results_path, val ? strdup(val) : NULL);
+		PARSE_LINE(settings, name, val, enable_code_coverage, numval);
+		PARSE_LINE(settings, name, val, cov_results_per_test, numval);
+		PARSE_LINE(settings, name, val, code_coverage_script, val ? strdup(val) : NULL);
 
 		printf("Warning: Unknown field in settings file: %s = %s\n",
 		       name, val);
diff --git a/runner/settings.h b/runner/settings.h
index bc61faeb..3c2a3ee9 100644
--- a/runner/settings.h
+++ b/runner/settings.h
@@ -20,6 +20,10 @@ enum {
 
 _Static_assert(ABORT_ALL == (ABORT_TAINT | ABORT_LOCKDEP | ABORT_PING), "ABORT_ALL must be all conditions bitwise or'd");
 
+#define GCOV_DIR		"/sys/kernel/debug/gcov"
+#define GCOV_RESET GCOV_DIR	"/reset"
+#define CODE_COV_RESULTS_PATH	"/code_cov"
+
 struct regex_list {
 	char **regex_strings;
 	GRegex **regexes;
@@ -48,6 +52,9 @@ struct settings {
 	bool piglit_style_dmesg;
 	int dmesg_warn_level;
 	bool list_all;
+	char *code_coverage_script;
+	bool enable_code_coverage;
+	bool cov_results_per_test;
 };
 
 /**