runner: Add --dmesg-warn-level switch

This switch allows users to select which dmesg log level is treated as
warning resulting in overriding the test results to
dmesg-fail/dmesg-warn.

Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Reviewed-by: Petri Latvala <petri.latvala@intel.com>

38 files changed, 460 insertions, 3 deletions

diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt
new file mode 100644
index 00000000..d75ffbc7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt
@@ -0,0 +1,7 @@
+15,952,3216186095097,-;[IGT] successtest: executing
+15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+6,951,3216186095083,-;drm_ KERN_INFO, disregard
+7,952,3216186095097,-;drm_ KERN_DEBUG disregard
+4,953,3216186101159,-;drm_ KERN_WARNING disregard
+1,953,3216186101159,-;drm_ KERN_ALERT should trip a warning
+15,955,3216186101160,-;[IGT] successtest: exiting, ret=0
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt
new file mode 100644
index 00000000..dad239d7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt
@@ -0,0 +1,5 @@
+When --dmesg-warn-level is set to 1 to make sure that changing the setting works
+also in this direction.
+
+This makes sure that the piglit-style-dmesg does not clash with
+dmesg-warn-level. All the messages are prefixed with drm_ to fit the filters.
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt
new file mode 100644
index 00000000..81f914a7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt
@@ -0,0 +1 @@
+successtest first-subtest
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt
new file mode 100644
index 00000000..172c0708
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt
@@ -0,0 +1,13 @@
+abort_mask : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 1
+dmesg_warn_level : 1
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
new file mode 100644
index 00000000..520edf08
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json
@@ -0,0 +1,73 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":10,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt@successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"dmesg-warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0.0,
+        "end":0.0
+      },
+      "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] drm_ KERN_INFO, disregard\n<7> [3216186.095097] drm_ KERN_DEBUG disregard\n<4> [3216186.101159] drm_ KERN_WARNING disregard\n<1> [3216186.101159] drm_ KERN_ALERT should trip a warning\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n",
+      "dmesg-warnings":"<1> [3216186.101159] drm_ KERN_ALERT should trip a warning\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt@successtest":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt@successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0.0,
+        "end":0.014
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt
new file mode 100644
index 00000000..546a29ff
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt
@@ -0,0 +1,6 @@
+15,952,3216186095097,-;[IGT] successtest: executing
+15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+6,951,3216186095083,-;drm_ KERN_INFO, should be a warning here too
+7,952,3216186095097,-;drm_ KERN_DEBUG, should not be a warning
+4,953,3216186101159,-;drm_ A proper KERN_WARNING
+15,955,3216186101160,-;[IGT] successtest: exiting, ret=0
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt
new file mode 100644
index 00000000..d2981708
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt
@@ -0,0 +1,7 @@
+When --dmesg-warn-level is set to 6 also KERN_INFO level messages should be
+treated as warnings triggering a result change to dmesg-warn/dmesg-fail.
+
+This makes sure that the piglit-style-dmesg does not clash with
+dmesg-warn-level. All the messages are prefixed with drm_ to fit the filters.
+
+IGT messages were artificially bumped to KERN_DEBUG to not pollute the warnings.
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt
new file mode 100644
index 00000000..81f914a7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt
@@ -0,0 +1 @@
+successtest first-subtest
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt
new file mode 100644
index 00000000..b05377a9
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt
@@ -0,0 +1,13 @@
+abort_mask : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 1
+dmesg_warn_level : 6
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
new file mode 100644
index 00000000..8cd2eeee
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json
@@ -0,0 +1,73 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":10,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt@successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"dmesg-warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0.0,
+        "end":0.0
+      },
+      "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] drm_ KERN_INFO, should be a warning here too\n<7> [3216186.095097] drm_ KERN_DEBUG, should not be a warning\n<4> [3216186.101159] drm_ A proper KERN_WARNING\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n",
+      "dmesg-warnings":"<6> [3216186.095083] drm_ KERN_INFO, should be a warning here too\n<4> [3216186.101159] drm_ A proper KERN_WARNING\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt@successtest":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt@successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0.0,
+        "end":0.014
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt
new file mode 100644
index 00000000..16ef1210
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt
@@ -0,0 +1,6 @@
+15,952,3216186095097,-;[IGT] successtest: executing
+15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+6,951,3216186095083,-;KERN_INFO, should be a warning here too
+7,952,3216186095097,-;KERN_DEBUG, should not be a warning
+4,953,3216186101159,-;A proper KERN_WARNING
+15,955,3216186101160,-;[IGT] successtest: exiting, ret=0
diff --git a/runner/json_tests_data/dmesg-warn-level/0/err.txt b/runner/json_tests_data/dmesg-warn-level/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-warn-level/0/journal.txt b/runner/json_tests_data/dmesg-warn-level/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/dmesg-warn-level/0/out.txt b/runner/json_tests_data/dmesg-warn-level/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-warn-level/README.txt b/runner/json_tests_data/dmesg-warn-level/README.txt
new file mode 100644
index 00000000..adb63998
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/README.txt
@@ -0,0 +1,4 @@
+When --dmesg-warn-level is set to 6 also KERN_INFO level messages should be
+treated as warnings triggering a result change to dmesg-warn/dmesg-fail.
+
+IGT messages were artifically bumped to KERN_DEBUG to not pollute the warnings.
diff --git a/runner/json_tests_data/dmesg-warn-level/endtime.txt b/runner/json_tests_data/dmesg-warn-level/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/dmesg-warn-level/joblist.txt b/runner/json_tests_data/dmesg-warn-level/joblist.txt
new file mode 100644
index 00000000..81f914a7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/joblist.txt
@@ -0,0 +1 @@
+successtest first-subtest
diff --git a/runner/json_tests_data/dmesg-warn-level/metadata.txt b/runner/json_tests_data/dmesg-warn-level/metadata.txt
new file mode 100644
index 00000000..7597a4eb
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/metadata.txt
@@ -0,0 +1,13 @@
+abort_mask : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+dmesg_warn_level : 6
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/dmesg-warn-level/reference.json b/runner/json_tests_data/dmesg-warn-level/reference.json
new file mode 100644
index 00000000..93d9c6ec
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/reference.json
@@ -0,0 +1,73 @@
+{
+  "__type__":"TestrunResult",
+  "results_version":10,
+  "name":"normal-run",
+  "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+  "time_elapsed":{
+    "__type__":"TimeAttribute",
+    "start":1539953735.1110389,
+    "end":1539953735.1723731
+  },
+  "tests":{
+    "igt@successtest@first-subtest":{
+      "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+      "result":"dmesg-warn",
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0.0,
+        "end":0.0
+      },
+      "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+      "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] KERN_INFO, should be a warning here too\n<7> [3216186.095097] KERN_DEBUG, should not be a warning\n<4> [3216186.101159] A proper KERN_WARNING\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n",
+      "dmesg-warnings":"<6> [3216186.095083] KERN_INFO, should be a warning here too\n<4> [3216186.101159] A proper KERN_WARNING\n"
+    }
+  },
+  "totals":{
+    "":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "root":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    },
+    "igt@successtest":{
+      "crash":0,
+      "pass":0,
+      "dmesg-fail":0,
+      "dmesg-warn":1,
+      "skip":0,
+      "incomplete":0,
+      "timeout":0,
+      "notrun":0,
+      "fail":0,
+      "warn":0
+    }
+  },
+  "runtimes":{
+    "igt@successtest":{
+      "time":{
+        "__type__":"TimeAttribute",
+        "start":0.0,
+        "end":0.014
+      }
+    }
+  }
+}
\ No newline at end of file
diff --git a/runner/json_tests_data/dmesg-warn-level/starttime.txt b/runner/json_tests_data/dmesg-warn-level/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/dmesg-warn-level/uname.txt b/runner/json_tests_data/dmesg-warn-level/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-warn-level/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/resultgen.c b/runner/resultgen.c
index 42297d8a..d9702a19 100644
--- a/runner/resultgen.c
+++ b/runner/resultgen.c
@@ -670,12 +670,12 @@ static bool fill_from_dmesg(int fd,
 		}
 
 		if (settings->piglit_style_dmesg) {
-			if ((flags & 0x07) <= 5 && continuation != 'c' &&
+			if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' &&
 			    regexec(&re, message, (size_t)0, NULL, 0) != REG_NOMATCH) {
 				append_line(&warnings, &warningslen, formatted);
 			}
 		} else {
-			if ((flags & 0x07) <= 4 && continuation != 'c' &&
+			if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' &&
 			    regexec(&re, message, (size_t)0, NULL, 0) == REG_NOMATCH) {
 				append_line(&warnings, &warningslen, formatted);
 			}
diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
index 09a93189..7c9540d1 100644
--- a/runner/runner_json_tests.c
+++ b/runner/runner_json_tests.c
@@ -159,6 +159,9 @@ static const char *dirnames[] = {
 	"dmesg-escapes",
 	"notrun-results",
 	"notrun-results-multiple-mode",
+	"dmesg-warn-level",
+	"dmesg-warn-level-piglit-style",
+	"dmesg-warn-level-one-piglit-style"
 };
 
 igt_main
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index 06725536..ade78b18 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -155,6 +155,7 @@ static void assert_settings_equal(struct settings *one, struct settings *two)
 	igt_assert_eqstr(one->test_root, two->test_root);
 	igt_assert_eqstr(one->results_path, two->results_path);
 	igt_assert_eq(one->piglit_style_dmesg, two->piglit_style_dmesg);
+	igt_assert_eq(one->dmesg_warn_level, two->dmesg_warn_level);
 }
 
 static void assert_job_list_equal(struct job_list *one, struct job_list *two)
@@ -223,7 +224,9 @@ igt_main
 		igt_assert(!settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+
 		igt_assert(!settings.piglit_style_dmesg);
+		igt_assert_eq(settings.dmesg_warn_level, 4);
 	}
 
 	igt_subtest_group {
@@ -364,6 +367,7 @@ igt_main
 				       "--overall-timeout", "360",
 				       "--use-watchdog",
 				       "--piglit-style-dmesg",
+				       "--dmesg-warn-level=3",
 				       "test-root-dir",
 				       "path-to-results",
 		};
@@ -389,7 +393,48 @@ igt_main
 		igt_assert(settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
+
+		igt_assert(settings.piglit_style_dmesg);
+		igt_assert_eq(settings.dmesg_warn_level, 3);
+	}
+
+	igt_subtest("dmesg-warn-level-inferred") {
+		const char *argv[] = { "runner",
+				       "test-root-dir",
+				       "path-to-results",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings));
+
+		igt_assert(!settings.piglit_style_dmesg);
+		igt_assert_eq(settings.dmesg_warn_level, 4);
+	}
+
+	igt_subtest("dmesg-warn-level-inferred-with-piglit-style") {
+		const char *argv[] = { "runner",
+				       "--piglit-style-dmesg",
+				       "test-root-dir",
+				       "path-to-results",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings));
+
+		igt_assert(settings.piglit_style_dmesg);
+		igt_assert_eq(settings.dmesg_warn_level, 5);
+	}
+
+	igt_subtest("dmesg-warn-level-overridable-with-piglit-style") {
+		const char *argv[] = { "runner",
+				       "--piglit-style-dmesg",
+				       "--dmesg-warn-level=3",
+				       "test-root-dir",
+				       "path-to-results",
+		};
+
+		igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings));
+
 		igt_assert(settings.piglit_style_dmesg);
+		igt_assert_eq(settings.dmesg_warn_level, 3);
 	}
 
 	igt_subtest("invalid-option") {
@@ -1093,6 +1138,60 @@ igt_main
 	}
 
 	igt_subtest_group {
+		igt_subtest("metadata-read-old-style-infer-dmesg-warn-piglit-style") {
+			char metadata[] = "piglit_style_dmesg : 1\n";
+			FILE *f = fmemopen(metadata, strlen(metadata), "r");
+			igt_assert(f);
+
+			igt_assert(read_settings_from_file(&settings, f));
+
+			igt_assert(settings.piglit_style_dmesg);
+			igt_assert_eq(settings.dmesg_warn_level, 5);
+
+			fclose(f);
+		}
+
+		igt_subtest("metadata-read-overrides-dmesg-warn-piglit-style") {
+			char metadata[] = "piglit_style_dmesg : 1\ndmesg_warn_level : 3";
+			FILE *f = fmemopen(metadata, strlen(metadata), "r");
+			igt_assert(f);
+
+			igt_assert(read_settings_from_file(&settings, f));
+
+			igt_assert(settings.piglit_style_dmesg);
+			igt_assert_eq(settings.dmesg_warn_level, 3);
+
+			fclose(f);
+		}
+
+		igt_subtest("metadata-read-old-style-infer-dmesg-warn-default") {
+			char metadata[] = "piglit_style_dmesg : 0\n";
+			FILE *f = fmemopen(metadata, strlen(metadata), "r");
+			igt_assert(f);
+
+			igt_assert(read_settings_from_file(&settings, f));
+
+			igt_assert(!settings.piglit_style_dmesg);
+			igt_assert_eq(settings.dmesg_warn_level, 4);
+
+			fclose(f);
+		}
+
+		igt_subtest("metadata-read-overrides-dmesg-warn-default") {
+			char metadata[] = "piglit_style_dmesg : 0\ndmesg_warn_level : 3";
+			FILE *f = fmemopen(metadata, strlen(metadata), "r");
+			igt_assert(f);
+
+			igt_assert(read_settings_from_file(&settings, f));
+
+			igt_assert(!settings.piglit_style_dmesg);
+			igt_assert_eq(settings.dmesg_warn_level, 3);
+
+			fclose(f);
+		}
+	}
+
+	igt_subtest_group {
 		struct job_list list;
 		int dirfd = -1, subdirfd = -1, fd = -1;
 		int multiple;
diff --git a/runner/settings.c b/runner/settings.c
index 95f74076..25bcf531 100644
--- a/runner/settings.c
+++ b/runner/settings.c
@@ -17,6 +17,7 @@ enum {
 	OPT_TEST_LIST,
 	OPT_IGNORE_MISSING,
 	OPT_PIGLIT_DMESG,
+	OPT_DMESG_WARN_LEVEL,
 	OPT_OVERALL_TIMEOUT,
 	OPT_HELP = 'h',
 	OPT_NAME = 'n',
@@ -159,12 +160,18 @@ static const char *usage_str =
 	"  --use-watchdog        Use hardware watchdog for lethal enforcement of the\n"
 	"                        above timeout. Killing the test process is still\n"
 	"                        attempted at timeout trigger.\n"
+	"  --dmesg-warn-level <level>\n"
+	"                        Messages with log level equal or lower (more serious)\n"
+	"                        to the given one will override the test result to\n"
+	"                        dmesg-warn/dmesg-fail, assuming they go through filtering.\n"
+	"                        Defaults to 4 (KERN_WARNING).\n"
 	"  --piglit-style-dmesg  Filter dmesg like piglit does. Piglit considers matches\n"
 	"                        against a short filter list to mean the test result\n"
 	"                        should be changed to dmesg-warn/dmesg-fail. Without\n"
 	"                        this option everything except matches against a\n"
 	"                        (longer) filter list means the test result should\n"
-	"                        change.\n"
+	"                        change. KERN_NOTICE dmesg level is treated as warn,\n"
+	"                        unless overridden with --dmesg-warn-level.\n"
 	"  [test_root]           Directory that contains the IGT tests. The environment\n"
 	"                        variable IGT_TEST_ROOT will be used if set, overriding\n"
 	"                        this option if given.\n"
@@ -270,6 +277,7 @@ bool parse_options(int argc, char **argv,
 		{"overall-timeout", required_argument, NULL, OPT_OVERALL_TIMEOUT},
 		{"use-watchdog", no_argument, NULL, OPT_WATCHDOG},
 		{"piglit-style-dmesg", no_argument, NULL, OPT_PIGLIT_DMESG},
+		{"dmesg-warn-level", required_argument, NULL, OPT_DMESG_WARN_LEVEL},
 		{ 0, 0, 0, 0},
 	};
 
@@ -277,6 +285,8 @@ bool parse_options(int argc, char **argv,
 
 	optind = 1;
 
+	settings->dmesg_warn_level = -1;
+
 	while ((c = getopt_long(argc, argv, "hn:dt:x:sl:om", long_options, NULL)) != -1) {
 		switch (c) {
 		case OPT_HELP:
@@ -332,6 +342,11 @@ bool parse_options(int argc, char **argv,
 			break;
 		case OPT_PIGLIT_DMESG:
 			settings->piglit_style_dmesg = true;
+			if (settings->dmesg_warn_level < 0)
+				settings->dmesg_warn_level = 5; /* KERN_NOTICE */
+			break;
+		case OPT_DMESG_WARN_LEVEL:
+			settings->dmesg_warn_level = atoi(optarg);
 			break;
 		case '?':
 			usage(NULL, stderr);
@@ -342,6 +357,9 @@ bool parse_options(int argc, char **argv,
 		}
 	}
 
+	if (settings->dmesg_warn_level < 0)
+		settings->dmesg_warn_level = 4; /* KERN_WARN */
+
 	switch (argc - optind) {
 	case 2:
 		settings->test_root = absolute_path(argv[optind]);
@@ -525,6 +543,7 @@ bool serialize_settings(struct settings *settings)
 	SERIALIZE_LINE(f, settings, overall_timeout, "%d");
 	SERIALIZE_LINE(f, settings, use_watchdog, "%d");
 	SERIALIZE_LINE(f, settings, piglit_style_dmesg, "%d");
+	SERIALIZE_LINE(f, settings, dmesg_warn_level, "%d");
 	SERIALIZE_LINE(f, settings, test_root, "%s");
 	SERIALIZE_LINE(f, settings, results_path, "%s");
 
@@ -553,6 +572,7 @@ bool read_settings_from_file(struct settings *settings, FILE *f)
 
 	char *name = NULL, *val = NULL;
 
+	settings->dmesg_warn_level = -1;
 
 	while (fscanf(f, "%ms : %ms", &name, &val) == 2) {
 		int numval = atoi(val);
@@ -568,6 +588,7 @@ bool read_settings_from_file(struct settings *settings, FILE *f)
 		PARSE_LINE(settings, name, val, overall_timeout, numval);
 		PARSE_LINE(settings, name, val, use_watchdog, numval);
 		PARSE_LINE(settings, name, val, piglit_style_dmesg, numval);
+		PARSE_LINE(settings, name, val, dmesg_warn_level, numval);
 		PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL);
 		PARSE_LINE(settings, name, val, results_path, val ? strdup(val) : NULL);
 
@@ -578,6 +599,13 @@ bool read_settings_from_file(struct settings *settings, FILE *f)
 		name = val = NULL;
 	}
 
+	if (settings->dmesg_warn_level < 0) {
+		if (settings->piglit_style_dmesg)
+			settings->dmesg_warn_level = 5;
+		else
+			settings->dmesg_warn_level = 4;
+	}
+
 	free(name);
 	free(val);
 
diff --git a/runner/settings.h b/runner/settings.h
index ae90897c..672a3af8 100644
--- a/runner/settings.h
+++ b/runner/settings.h
@@ -42,6 +42,7 @@ struct settings {
 	char *test_root;
 	char *results_path;
 	bool piglit_style_dmesg;
+	int dmesg_warn_level;
 };
 
 /**