runner: Introduce --piglit-style-dmesg

With the flag, dmesg handling is done exactly as piglit does it: Level
5 (info) and higher dmesg lines, if they match a regexp, cause test
result to change to dmesg-*.

The default is false (use new method).

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Cc: Martin Peres <martin.peres@linux.intel.com>
Acked-by: Martin Peres <martin.peres@linux.intel.com>

4 files changed, 62 insertions, 10 deletions

diff --git a/runner/resultgen.c b/runner/resultgen.c
index b2d080e4..ea680914 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -525,15 +525,22 @@ static const char igt_dmesg_whitelist[] =
 	;
 #undef _
 
+static const char igt_piglit_style_dmesg_blacklist[] =
+	"(\\[drm:|drm_|intel_|i915_)";
+
 static regex_t re;
 
-static int init_regex_whitelist(void)
+static int init_regex_whitelist(struct settings *settings)
 {
 	static int status = -1;
 
 	if (status == -1) {
-		if (regcomp(&re, igt_dmesg_whitelist, REG_EXTENDED | REG_NOSUB) != 0) {
-			fprintf(stderr, "Cannot compile dmesg whitelist regexp\n");
+		const char *regex = settings->piglit_style_dmesg ?
+			igt_piglit_style_dmesg_blacklist :
+			igt_dmesg_whitelist;
+
+		if (regcomp(&re, regex, REG_EXTENDED | REG_NOSUB) != 0) {
+			fprintf(stderr, "Cannot compile dmesg regexp\n");
 			status = 1;
 			return false;
 		}
@@ -604,7 +611,9 @@ static void add_empty_dmesgs_where_missing(struct json_object *tests,
 
 }
 
-static bool fill_from_dmesg(int fd, char *binary,
+static bool fill_from_dmesg(int fd,
+			    struct settings *settings,
+			    char *binary,
 			    struct subtests *subtests,
 			    struct json_object *tests)
 {
@@ -620,7 +629,7 @@ static bool fill_from_dmesg(int fd, char *binary,
 		return false;
 	}
 
-	if (init_regex_whitelist()) {
+	if (init_regex_whitelist(settings)) {
 		fclose(f);
 		return false;
 	}
@@ -654,9 +663,16 @@ static bool fill_from_dmesg(int fd, char *binary,
 			current_test = get_or_create_json_object(tests, piglit_name);
 		}
 
-		if ((flags & 0x07) <= 4 && continuation != 'c' &&
-		    regexec(&re, message, (size_t)0, NULL, 0) == REG_NOMATCH) {
-			append_line(&warnings, &warningslen, formatted);
+		if (settings->piglit_style_dmesg) {
+			if ((flags & 0x07) <= 5 && continuation != 'c' &&
+			    regexec(&re, message, (size_t)0, NULL, 0) != REG_NOMATCH) {
+				append_line(&warnings, &warningslen, formatted);
+			}
+		} else {
+			if ((flags & 0x07) <= 4 && continuation != 'c' &&
+			    regexec(&re, message, (size_t)0, NULL, 0) == REG_NOMATCH) {
+				append_line(&warnings, &warningslen, formatted);
+			}
 		}
 		append_line(&dmesg, &dmesglen, formatted);
 		free(formatted);
@@ -952,6 +968,7 @@ static void add_to_totals(char *binary,
 
 static bool parse_test_directory(int dirfd,
 				 struct job_list_entry *entry,
+				 struct settings *settings,
 				 struct results *results)
 {
 	int fds[_F_LAST];
@@ -970,7 +987,7 @@ static bool parse_test_directory(int dirfd,
 
 	if (!fill_from_output(fds[_F_OUT], entry->binary, "out", &subtests, results->tests) ||
 	    !fill_from_output(fds[_F_ERR], entry->binary, "err", &subtests, results->tests) ||
-	    !fill_from_dmesg(fds[_F_DMESG], entry->binary, &subtests, results->tests)) {
+	    !fill_from_dmesg(fds[_F_DMESG], settings, entry->binary, &subtests, results->tests)) {
 		fprintf(stderr, "Error parsing output files\n");
 		return false;
 	}
@@ -1068,7 +1085,7 @@ bool generate_results(int dirfd)
 			break;
 		}
 
-		if (!parse_test_directory(testdirfd, &job_list.entries[i], &results)) {
+		if (!parse_test_directory(testdirfd, &job_list.entries[i], &settings, &results)) {
 			close(resultsfd);
 			return false;
 		}
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index 942ba26b..1a95a75a 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -152,6 +152,7 @@ static void assert_settings_equal(struct settings *one, struct settings *two)
 	igt_assert_eq(one->use_watchdog, two->use_watchdog);
 	igt_assert_eqstr(one->test_root, two->test_root);
 	igt_assert_eqstr(one->results_path, two->results_path);
+	igt_assert_eq(one->piglit_style_dmesg, two->piglit_style_dmesg);
 }
 
 static void assert_job_list_equal(struct job_list *one, struct job_list *two)
@@ -219,6 +220,7 @@ igt_main
 		igt_assert(!settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+		igt_assert(!settings.piglit_style_dmesg);
 	}
 
 	igt_subtest_group {
@@ -332,6 +334,7 @@ igt_main
 		igt_assert(!settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, testdatadir) != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+		igt_assert(!settings.piglit_style_dmesg);
 	}
 
 	igt_fixture {
@@ -355,6 +358,7 @@ igt_main
 				 "--multiple-mode",
 				 "--inactivity-timeout", "27",
 				 "--use-watchdog",
+				 "--piglit-style-dmesg",
 				 "test-root-dir",
 				 "path-to-results",
 		};
@@ -379,6 +383,7 @@ igt_main
 		igt_assert(settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+		igt_assert(settings.piglit_style_dmesg);
 	}
 
 	igt_subtest("invalid-option") {
@@ -578,6 +583,22 @@ igt_main
 
 		igt_subtest("settings-serialize") {
 			char *argv[] = { "runner",
+					 "-n", "foo",
+					 "--abort-on-monitored-error",
+					 "--test-list", "path-to-test-list",
+					 "--ignore-missing",
+					 "--dry-run",
+					 "-t", "pattern1",
+					 "-t", "pattern2",
+					 "-x", "xpattern1",
+					 "-x", "xpattern2",
+					 "-s",
+					 "-l", "verbose",
+					 "--overwrite",
+					 "--multiple-mode",
+					 "--inactivity-timeout", "27",
+					 "--use-watchdog",
+					 "--piglit-style-dmesg",
 					 testdatadir,
 					 dirname,
 			};
diff --git a/runner/settings.c b/runner/settings.c
index 060459b0..70fff3c0 100644
--- a/runner/settings.c
+++ b/runner/settings.c
@@ -16,6 +16,7 @@ enum {
 	OPT_ABORT_ON_ERROR,
 	OPT_TEST_LIST,
 	OPT_IGNORE_MISSING,
+	OPT_PIGLIT_DMESG,
 	OPT_HELP = 'h',
 	OPT_NAME = 'n',
 	OPT_DRY_RUN = 'd',
@@ -89,6 +90,12 @@ static const char *usage_str =
 	"  --use-watchdog        Use hardware watchdog for lethal enforcement of the\n"
 	"                        above timeout. Killing the test process is still\n"
 	"                        attempted at timeout trigger.\n"
+	"  --piglit-style-dmesg  Filter dmesg like piglit does. Piglit considers matches\n"
+	"                        against a short filter list to mean the test result\n"
+	"                        should be changed to dmesg-warn/dmesg-fail. Without\n"
+	"                        this option everything except matches against a\n"
+	"                        (longer) filter list means the test result should\n"
+	"                        change.\n"
 	"  [test_root]           Directory that contains the IGT tests. The environment\n"
 	"                        variable IGT_TEST_ROOT will be used if set, overriding\n"
 	"                        this option if given.\n"
@@ -192,6 +199,7 @@ bool parse_options(int argc, char **argv,
 		{"multiple-mode", no_argument, NULL, OPT_MULTIPLE},
 		{"inactivity-timeout", required_argument, NULL, OPT_TIMEOUT},
 		{"use-watchdog", no_argument, NULL, OPT_WATCHDOG},
+		{"piglit-style-dmesg", no_argument, NULL, OPT_PIGLIT_DMESG},
 		{ 0, 0, 0, 0},
 	};
 
@@ -248,6 +256,9 @@ bool parse_options(int argc, char **argv,
 		case OPT_WATCHDOG:
 			settings->use_watchdog = true;
 			break;
+		case OPT_PIGLIT_DMESG:
+			settings->piglit_style_dmesg = true;
+			break;
 		case '?':
 			usage(NULL, stderr);
 			goto error;
@@ -438,6 +449,7 @@ bool serialize_settings(struct settings *settings)
 	SERIALIZE_LINE(f, settings, multiple_mode, "%d");
 	SERIALIZE_LINE(f, settings, inactivity_timeout, "%d");
 	SERIALIZE_LINE(f, settings, use_watchdog, "%d");
+	SERIALIZE_LINE(f, settings, piglit_style_dmesg, "%d");
 	SERIALIZE_LINE(f, settings, test_root, "%s");
 	SERIALIZE_LINE(f, settings, results_path, "%s");
 
@@ -491,6 +503,7 @@ bool read_settings(struct settings *settings, int dirfd)
 		PARSE_LINE(settings, name, val, multiple_mode, numval);
 		PARSE_LINE(settings, name, val, inactivity_timeout, numval);
 		PARSE_LINE(settings, name, val, use_watchdog, numval);
+		PARSE_LINE(settings, name, val, piglit_style_dmesg, numval);
 		PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL);
 		PARSE_LINE(settings, name, val, results_path, val ? strdup(val) : NULL);
 
diff --git a/runner/settings.h b/runner/settings.h
index 9d1f03fb..e534b845 100644
--- a/runner/settings.h
+++ b/runner/settings.h
@@ -33,6 +33,7 @@ struct settings {
 	bool use_watchdog;
 	char *test_root;
 	char *results_path;
+	bool piglit_style_dmesg;
 };
 
 /**