From ec9792ad770d5055d6f42e2a481b8314754c9218 Mon Sep 17 00:00:00 2001 From: Arkadiusz Hiler Date: Mon, 1 Apr 2019 09:40:28 +0300 Subject: runner: Add --dmesg-warn-level switch This switch allows users to select which dmesg log level is treated as warning resulting in overriding the test results to dmesg-fail/dmesg-warn. Signed-off-by: Arkadiusz Hiler Reviewed-by: Petri Latvala --- .../dmesg-warn-level-one-piglit-style/0/dmesg.txt | 7 ++ .../dmesg-warn-level-one-piglit-style/0/err.txt | 2 + .../0/journal.txt | 2 + .../dmesg-warn-level-one-piglit-style/0/out.txt | 3 + .../dmesg-warn-level-one-piglit-style/README.txt | 5 ++ .../dmesg-warn-level-one-piglit-style/endtime.txt | 1 + .../dmesg-warn-level-one-piglit-style/joblist.txt | 1 + .../dmesg-warn-level-one-piglit-style/metadata.txt | 13 +++ .../reference.json | 73 ++++++++++++++++ .../starttime.txt | 1 + .../dmesg-warn-level-one-piglit-style/uname.txt | 1 + .../dmesg-warn-level-piglit-style/0/dmesg.txt | 6 ++ .../dmesg-warn-level-piglit-style/0/err.txt | 2 + .../dmesg-warn-level-piglit-style/0/journal.txt | 2 + .../dmesg-warn-level-piglit-style/0/out.txt | 3 + .../dmesg-warn-level-piglit-style/README.txt | 7 ++ .../dmesg-warn-level-piglit-style/endtime.txt | 1 + .../dmesg-warn-level-piglit-style/joblist.txt | 1 + .../dmesg-warn-level-piglit-style/metadata.txt | 13 +++ .../dmesg-warn-level-piglit-style/reference.json | 73 ++++++++++++++++ .../dmesg-warn-level-piglit-style/starttime.txt | 1 + .../dmesg-warn-level-piglit-style/uname.txt | 1 + .../json_tests_data/dmesg-warn-level/0/dmesg.txt | 6 ++ runner/json_tests_data/dmesg-warn-level/0/err.txt | 2 + .../json_tests_data/dmesg-warn-level/0/journal.txt | 2 + runner/json_tests_data/dmesg-warn-level/0/out.txt | 3 + runner/json_tests_data/dmesg-warn-level/README.txt | 4 + .../json_tests_data/dmesg-warn-level/endtime.txt | 1 + .../json_tests_data/dmesg-warn-level/joblist.txt | 1 + .../json_tests_data/dmesg-warn-level/metadata.txt | 13 +++ .../dmesg-warn-level/reference.json | 73 ++++++++++++++++ .../json_tests_data/dmesg-warn-level/starttime.txt | 1 + runner/json_tests_data/dmesg-warn-level/uname.txt | 1 + runner/resultgen.c | 4 +- runner/runner_json_tests.c | 3 + runner/runner_tests.c | 99 ++++++++++++++++++++++ runner/settings.c | 30 ++++++- runner/settings.h | 1 + 38 files changed, 460 insertions(+), 3 deletions(-) create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt create mode 100644 runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/0/dmesg.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/0/err.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/0/journal.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/0/out.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/README.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/endtime.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/joblist.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/metadata.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/reference.json create mode 100644 runner/json_tests_data/dmesg-warn-level/starttime.txt create mode 100644 runner/json_tests_data/dmesg-warn-level/uname.txt (limited to 'runner') diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt new file mode 100644 index 00000000..d75ffbc7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/dmesg.txt @@ -0,0 +1,7 @@ +15,952,3216186095097,-;[IGT] successtest: executing +15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +6,951,3216186095083,-;drm_ KERN_INFO, disregard +7,952,3216186095097,-;drm_ KERN_DEBUG disregard +4,953,3216186101159,-;drm_ KERN_WARNING disregard +1,953,3216186101159,-;drm_ KERN_ALERT should trip a warning +15,955,3216186101160,-;[IGT] successtest: exiting, ret=0 diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt new file mode 100644 index 00000000..dad239d7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/README.txt @@ -0,0 +1,5 @@ +When --dmesg-warn-level is set to 1 to make sure that changing the setting works +also in this direction. + +This makes sure that the piglit-style-dmesg does not clash with +dmesg-warn-level. All the messages are prefixed with drm_ to fit the filters. diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt new file mode 100644 index 00000000..81f914a7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/joblist.txt @@ -0,0 +1 @@ +successtest first-subtest diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt new file mode 100644 index 00000000..172c0708 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/metadata.txt @@ -0,0 +1,13 @@ +abort_mask : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 1 +dmesg_warn_level : 1 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json new file mode 100644 index 00000000..520edf08 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json @@ -0,0 +1,73 @@ +{ + "__type__":"TestrunResult", + "results_version":10, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-warn", + "time":{ + "__type__":"TimeAttribute", + "start":0.0, + "end":0.0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] drm_ KERN_INFO, disregard\n<7> [3216186.095097] drm_ KERN_DEBUG disregard\n<4> [3216186.101159] drm_ KERN_WARNING disregard\n<1> [3216186.101159] drm_ KERN_ALERT should trip a warning\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n", + "dmesg-warnings":"<1> [3216186.101159] drm_ KERN_ALERT should trip a warning\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0.0, + "end":0.014 + } + } + } +} \ No newline at end of file diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt new file mode 100644 index 00000000..546a29ff --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/dmesg.txt @@ -0,0 +1,6 @@ +15,952,3216186095097,-;[IGT] successtest: executing +15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +6,951,3216186095083,-;drm_ KERN_INFO, should be a warning here too +7,952,3216186095097,-;drm_ KERN_DEBUG, should not be a warning +4,953,3216186101159,-;drm_ A proper KERN_WARNING +15,955,3216186101160,-;[IGT] successtest: exiting, ret=0 diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt new file mode 100644 index 00000000..d2981708 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/README.txt @@ -0,0 +1,7 @@ +When --dmesg-warn-level is set to 6 also KERN_INFO level messages should be +treated as warnings triggering a result change to dmesg-warn/dmesg-fail. + +This makes sure that the piglit-style-dmesg does not clash with +dmesg-warn-level. All the messages are prefixed with drm_ to fit the filters. + +IGT messages were artificially bumped to KERN_DEBUG to not pollute the warnings. diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt new file mode 100644 index 00000000..81f914a7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/joblist.txt @@ -0,0 +1 @@ +successtest first-subtest diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt new file mode 100644 index 00000000..b05377a9 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/metadata.txt @@ -0,0 +1,13 @@ +abort_mask : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 1 +dmesg_warn_level : 6 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json new file mode 100644 index 00000000..8cd2eeee --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json @@ -0,0 +1,73 @@ +{ + "__type__":"TestrunResult", + "results_version":10, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-warn", + "time":{ + "__type__":"TimeAttribute", + "start":0.0, + "end":0.0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] drm_ KERN_INFO, should be a warning here too\n<7> [3216186.095097] drm_ KERN_DEBUG, should not be a warning\n<4> [3216186.101159] drm_ A proper KERN_WARNING\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n", + "dmesg-warnings":"<6> [3216186.095083] drm_ KERN_INFO, should be a warning here too\n<4> [3216186.101159] drm_ A proper KERN_WARNING\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0.0, + "end":0.014 + } + } + } +} \ No newline at end of file diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt b/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt b/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt new file mode 100644 index 00000000..16ef1210 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/0/dmesg.txt @@ -0,0 +1,6 @@ +15,952,3216186095097,-;[IGT] successtest: executing +15,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +6,951,3216186095083,-;KERN_INFO, should be a warning here too +7,952,3216186095097,-;KERN_DEBUG, should not be a warning +4,953,3216186101159,-;A proper KERN_WARNING +15,955,3216186101160,-;[IGT] successtest: exiting, ret=0 diff --git a/runner/json_tests_data/dmesg-warn-level/0/err.txt b/runner/json_tests_data/dmesg-warn-level/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-warn-level/0/journal.txt b/runner/json_tests_data/dmesg-warn-level/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/dmesg-warn-level/0/out.txt b/runner/json_tests_data/dmesg-warn-level/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-warn-level/README.txt b/runner/json_tests_data/dmesg-warn-level/README.txt new file mode 100644 index 00000000..adb63998 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/README.txt @@ -0,0 +1,4 @@ +When --dmesg-warn-level is set to 6 also KERN_INFO level messages should be +treated as warnings triggering a result change to dmesg-warn/dmesg-fail. + +IGT messages were artifically bumped to KERN_DEBUG to not pollute the warnings. diff --git a/runner/json_tests_data/dmesg-warn-level/endtime.txt b/runner/json_tests_data/dmesg-warn-level/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/dmesg-warn-level/joblist.txt b/runner/json_tests_data/dmesg-warn-level/joblist.txt new file mode 100644 index 00000000..81f914a7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/joblist.txt @@ -0,0 +1 @@ +successtest first-subtest diff --git a/runner/json_tests_data/dmesg-warn-level/metadata.txt b/runner/json_tests_data/dmesg-warn-level/metadata.txt new file mode 100644 index 00000000..7597a4eb --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/metadata.txt @@ -0,0 +1,13 @@ +abort_mask : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +dmesg_warn_level : 6 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/dmesg-warn-level/reference.json b/runner/json_tests_data/dmesg-warn-level/reference.json new file mode 100644 index 00000000..93d9c6ec --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/reference.json @@ -0,0 +1,73 @@ +{ + "__type__":"TestrunResult", + "results_version":10, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-warn", + "time":{ + "__type__":"TimeAttribute", + "start":0.0, + "end":0.0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<7> [3216186.095097] [IGT] successtest: executing\n<7> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.095083] KERN_INFO, should be a warning here too\n<7> [3216186.095097] KERN_DEBUG, should not be a warning\n<4> [3216186.101159] A proper KERN_WARNING\n<7> [3216186.101160] [IGT] successtest: exiting, ret=0\n", + "dmesg-warnings":"<6> [3216186.095083] KERN_INFO, should be a warning here too\n<4> [3216186.101159] A proper KERN_WARNING\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0.0, + "end":0.014 + } + } + } +} \ No newline at end of file diff --git a/runner/json_tests_data/dmesg-warn-level/starttime.txt b/runner/json_tests_data/dmesg-warn-level/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/dmesg-warn-level/uname.txt b/runner/json_tests_data/dmesg-warn-level/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/dmesg-warn-level/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/resultgen.c b/runner/resultgen.c index 42297d8a..d9702a19 100644 --- a/runner/resultgen.c +++ b/runner/resultgen.c @@ -670,12 +670,12 @@ static bool fill_from_dmesg(int fd, } if (settings->piglit_style_dmesg) { - if ((flags & 0x07) <= 5 && continuation != 'c' && + if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' && regexec(&re, message, (size_t)0, NULL, 0) != REG_NOMATCH) { append_line(&warnings, &warningslen, formatted); } } else { - if ((flags & 0x07) <= 4 && continuation != 'c' && + if ((flags & 0x07) <= settings->dmesg_warn_level && continuation != 'c' && regexec(&re, message, (size_t)0, NULL, 0) == REG_NOMATCH) { append_line(&warnings, &warningslen, formatted); } diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c index 09a93189..7c9540d1 100644 --- a/runner/runner_json_tests.c +++ b/runner/runner_json_tests.c @@ -159,6 +159,9 @@ static const char *dirnames[] = { "dmesg-escapes", "notrun-results", "notrun-results-multiple-mode", + "dmesg-warn-level", + "dmesg-warn-level-piglit-style", + "dmesg-warn-level-one-piglit-style" }; igt_main diff --git a/runner/runner_tests.c b/runner/runner_tests.c index 06725536..ade78b18 100644 --- a/runner/runner_tests.c +++ b/runner/runner_tests.c @@ -155,6 +155,7 @@ static void assert_settings_equal(struct settings *one, struct settings *two) igt_assert_eqstr(one->test_root, two->test_root); igt_assert_eqstr(one->results_path, two->results_path); igt_assert_eq(one->piglit_style_dmesg, two->piglit_style_dmesg); + igt_assert_eq(one->dmesg_warn_level, two->dmesg_warn_level); } static void assert_job_list_equal(struct job_list *one, struct job_list *two) @@ -223,7 +224,9 @@ igt_main igt_assert(!settings.use_watchdog); igt_assert(strstr(settings.test_root, "test-root-dir") != NULL); igt_assert(strstr(settings.results_path, "path-to-results") != NULL); + igt_assert(!settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 4); } igt_subtest_group { @@ -364,6 +367,7 @@ igt_main "--overall-timeout", "360", "--use-watchdog", "--piglit-style-dmesg", + "--dmesg-warn-level=3", "test-root-dir", "path-to-results", }; @@ -389,7 +393,48 @@ igt_main igt_assert(settings.use_watchdog); igt_assert(strstr(settings.test_root, "test-root-dir") != NULL); igt_assert(strstr(settings.results_path, "path-to-results") != NULL); + + igt_assert(settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 3); + } + + igt_subtest("dmesg-warn-level-inferred") { + const char *argv[] = { "runner", + "test-root-dir", + "path-to-results", + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings)); + + igt_assert(!settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 4); + } + + igt_subtest("dmesg-warn-level-inferred-with-piglit-style") { + const char *argv[] = { "runner", + "--piglit-style-dmesg", + "test-root-dir", + "path-to-results", + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings)); + + igt_assert(settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 5); + } + + igt_subtest("dmesg-warn-level-overridable-with-piglit-style") { + const char *argv[] = { "runner", + "--piglit-style-dmesg", + "--dmesg-warn-level=3", + "test-root-dir", + "path-to-results", + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, &settings)); + igt_assert(settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 3); } igt_subtest("invalid-option") { @@ -1092,6 +1137,60 @@ igt_main } } + igt_subtest_group { + igt_subtest("metadata-read-old-style-infer-dmesg-warn-piglit-style") { + char metadata[] = "piglit_style_dmesg : 1\n"; + FILE *f = fmemopen(metadata, strlen(metadata), "r"); + igt_assert(f); + + igt_assert(read_settings_from_file(&settings, f)); + + igt_assert(settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 5); + + fclose(f); + } + + igt_subtest("metadata-read-overrides-dmesg-warn-piglit-style") { + char metadata[] = "piglit_style_dmesg : 1\ndmesg_warn_level : 3"; + FILE *f = fmemopen(metadata, strlen(metadata), "r"); + igt_assert(f); + + igt_assert(read_settings_from_file(&settings, f)); + + igt_assert(settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 3); + + fclose(f); + } + + igt_subtest("metadata-read-old-style-infer-dmesg-warn-default") { + char metadata[] = "piglit_style_dmesg : 0\n"; + FILE *f = fmemopen(metadata, strlen(metadata), "r"); + igt_assert(f); + + igt_assert(read_settings_from_file(&settings, f)); + + igt_assert(!settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 4); + + fclose(f); + } + + igt_subtest("metadata-read-overrides-dmesg-warn-default") { + char metadata[] = "piglit_style_dmesg : 0\ndmesg_warn_level : 3"; + FILE *f = fmemopen(metadata, strlen(metadata), "r"); + igt_assert(f); + + igt_assert(read_settings_from_file(&settings, f)); + + igt_assert(!settings.piglit_style_dmesg); + igt_assert_eq(settings.dmesg_warn_level, 3); + + fclose(f); + } + } + igt_subtest_group { struct job_list list; int dirfd = -1, subdirfd = -1, fd = -1; diff --git a/runner/settings.c b/runner/settings.c index 95f74076..25bcf531 100644 --- a/runner/settings.c +++ b/runner/settings.c @@ -17,6 +17,7 @@ enum { OPT_TEST_LIST, OPT_IGNORE_MISSING, OPT_PIGLIT_DMESG, + OPT_DMESG_WARN_LEVEL, OPT_OVERALL_TIMEOUT, OPT_HELP = 'h', OPT_NAME = 'n', @@ -159,12 +160,18 @@ static const char *usage_str = " --use-watchdog Use hardware watchdog for lethal enforcement of the\n" " above timeout. Killing the test process is still\n" " attempted at timeout trigger.\n" + " --dmesg-warn-level \n" + " Messages with log level equal or lower (more serious)\n" + " to the given one will override the test result to\n" + " dmesg-warn/dmesg-fail, assuming they go through filtering.\n" + " Defaults to 4 (KERN_WARNING).\n" " --piglit-style-dmesg Filter dmesg like piglit does. Piglit considers matches\n" " against a short filter list to mean the test result\n" " should be changed to dmesg-warn/dmesg-fail. Without\n" " this option everything except matches against a\n" " (longer) filter list means the test result should\n" - " change.\n" + " change. KERN_NOTICE dmesg level is treated as warn,\n" + " unless overridden with --dmesg-warn-level.\n" " [test_root] Directory that contains the IGT tests. The environment\n" " variable IGT_TEST_ROOT will be used if set, overriding\n" " this option if given.\n" @@ -270,6 +277,7 @@ bool parse_options(int argc, char **argv, {"overall-timeout", required_argument, NULL, OPT_OVERALL_TIMEOUT}, {"use-watchdog", no_argument, NULL, OPT_WATCHDOG}, {"piglit-style-dmesg", no_argument, NULL, OPT_PIGLIT_DMESG}, + {"dmesg-warn-level", required_argument, NULL, OPT_DMESG_WARN_LEVEL}, { 0, 0, 0, 0}, }; @@ -277,6 +285,8 @@ bool parse_options(int argc, char **argv, optind = 1; + settings->dmesg_warn_level = -1; + while ((c = getopt_long(argc, argv, "hn:dt:x:sl:om", long_options, NULL)) != -1) { switch (c) { case OPT_HELP: @@ -332,6 +342,11 @@ bool parse_options(int argc, char **argv, break; case OPT_PIGLIT_DMESG: settings->piglit_style_dmesg = true; + if (settings->dmesg_warn_level < 0) + settings->dmesg_warn_level = 5; /* KERN_NOTICE */ + break; + case OPT_DMESG_WARN_LEVEL: + settings->dmesg_warn_level = atoi(optarg); break; case '?': usage(NULL, stderr); @@ -342,6 +357,9 @@ bool parse_options(int argc, char **argv, } } + if (settings->dmesg_warn_level < 0) + settings->dmesg_warn_level = 4; /* KERN_WARN */ + switch (argc - optind) { case 2: settings->test_root = absolute_path(argv[optind]); @@ -525,6 +543,7 @@ bool serialize_settings(struct settings *settings) SERIALIZE_LINE(f, settings, overall_timeout, "%d"); SERIALIZE_LINE(f, settings, use_watchdog, "%d"); SERIALIZE_LINE(f, settings, piglit_style_dmesg, "%d"); + SERIALIZE_LINE(f, settings, dmesg_warn_level, "%d"); SERIALIZE_LINE(f, settings, test_root, "%s"); SERIALIZE_LINE(f, settings, results_path, "%s"); @@ -553,6 +572,7 @@ bool read_settings_from_file(struct settings *settings, FILE *f) char *name = NULL, *val = NULL; + settings->dmesg_warn_level = -1; while (fscanf(f, "%ms : %ms", &name, &val) == 2) { int numval = atoi(val); @@ -568,6 +588,7 @@ bool read_settings_from_file(struct settings *settings, FILE *f) PARSE_LINE(settings, name, val, overall_timeout, numval); PARSE_LINE(settings, name, val, use_watchdog, numval); PARSE_LINE(settings, name, val, piglit_style_dmesg, numval); + PARSE_LINE(settings, name, val, dmesg_warn_level, numval); PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL); PARSE_LINE(settings, name, val, results_path, val ? strdup(val) : NULL); @@ -578,6 +599,13 @@ bool read_settings_from_file(struct settings *settings, FILE *f) name = val = NULL; } + if (settings->dmesg_warn_level < 0) { + if (settings->piglit_style_dmesg) + settings->dmesg_warn_level = 5; + else + settings->dmesg_warn_level = 4; + } + free(name); free(val); diff --git a/runner/settings.h b/runner/settings.h index ae90897c..672a3af8 100644 --- a/runner/settings.h +++ b/runner/settings.h @@ -42,6 +42,7 @@ struct settings { char *test_root; char *results_path; bool piglit_style_dmesg; + int dmesg_warn_level; }; /** -- cgit v1.2.3