From 321fe77d32fef32ef820f53924045fe6ef0cf6ed Mon Sep 17 00:00:00 2001 From: Petri Latvala Date: Mon, 17 Dec 2018 15:07:40 +0200 Subject: runner: Add explicit "notrun" results for tests that were not executed When possible, all tests we know we were going to attempt to execute now appear in the results as "notrun". The only known case where it's not possible to add an explicit "notrun" is when running in multiple-mode, because "no subtests" and "run all subtests, we didn't list them beforehand" are represented the same. v2: Rebase and adjust to already landed json changes Signed-off-by: Petri Latvala Cc: Martin Peres Cc: Arkadiusz Hiler Acked-by: Martin Peres Reviewed-by: Arkadiusz Hiler --- .../aborted-after-a-test/reference.json | 54 +++++++++- .../json_tests_data/aborted-on-boot/reference.json | 70 +++++++++++- .../incomplete-before-any-subtests/reference.json | 56 +++++++++- .../notrun-results-multiple-mode/0/dmesg.txt | 5 + .../notrun-results-multiple-mode/0/err.txt | 2 + .../notrun-results-multiple-mode/0/journal.txt | 2 + .../notrun-results-multiple-mode/0/out.txt | 3 + .../notrun-results-multiple-mode/README.txt | 4 + .../notrun-results-multiple-mode/endtime.txt | 1 + .../notrun-results-multiple-mode/joblist.txt | 5 + .../notrun-results-multiple-mode/metadata.txt | 12 +++ .../notrun-results-multiple-mode/reference.json | 102 ++++++++++++++++++ .../notrun-results-multiple-mode/starttime.txt | 1 + .../notrun-results-multiple-mode/uname.txt | 1 + runner/json_tests_data/notrun-results/0/dmesg.txt | 5 + runner/json_tests_data/notrun-results/0/err.txt | 2 + .../json_tests_data/notrun-results/0/journal.txt | 2 + runner/json_tests_data/notrun-results/0/out.txt | 3 + runner/json_tests_data/notrun-results/README.txt | 4 + runner/json_tests_data/notrun-results/endtime.txt | 1 + runner/json_tests_data/notrun-results/joblist.txt | 5 + runner/json_tests_data/notrun-results/metadata.txt | 12 +++ .../json_tests_data/notrun-results/reference.json | 120 +++++++++++++++++++++ .../json_tests_data/notrun-results/starttime.txt | 1 + runner/json_tests_data/notrun-results/uname.txt | 1 + runner/resultgen.c | 42 +++++++- runner/runner_json_tests.c | 2 + 27 files changed, 507 insertions(+), 11 deletions(-) create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/err.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/0/out.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/README.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/endtime.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/joblist.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/metadata.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/reference.json create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/starttime.txt create mode 100644 runner/json_tests_data/notrun-results-multiple-mode/uname.txt create mode 100644 runner/json_tests_data/notrun-results/0/dmesg.txt create mode 100644 runner/json_tests_data/notrun-results/0/err.txt create mode 100644 runner/json_tests_data/notrun-results/0/journal.txt create mode 100644 runner/json_tests_data/notrun-results/0/out.txt create mode 100644 runner/json_tests_data/notrun-results/README.txt create mode 100644 runner/json_tests_data/notrun-results/endtime.txt create mode 100644 runner/json_tests_data/notrun-results/joblist.txt create mode 100644 runner/json_tests_data/notrun-results/metadata.txt create mode 100644 runner/json_tests_data/notrun-results/reference.json create mode 100644 runner/json_tests_data/notrun-results/starttime.txt create mode 100644 runner/json_tests_data/notrun-results/uname.txt (limited to 'runner') diff --git a/runner/json_tests_data/aborted-after-a-test/reference.json b/runner/json_tests_data/aborted-after-a-test/reference.json index 19ba9285..06a8bff6 100644 --- a/runner/json_tests_data/aborted-after-a-test/reference.json +++ b/runner/json_tests_data/aborted-after-a-test/reference.json @@ -21,6 +21,30 @@ "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" }, + "igt@successtest@second-subtest":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@no-subtests":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@skippers@skip-one":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@skippers@skip-two":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, "igt@runner@aborted":{ "out":"Aborted after: successtest (first-subtest)\n\nKernel tainted (0x200)\n", "result":"fail", @@ -37,7 +61,7 @@ "skip":0, "incomplete":0, "timeout":0, - "notrun":0, + "notrun":4, "fail":1, "warn":0 }, @@ -49,7 +73,7 @@ "skip":0, "incomplete":0, "timeout":0, - "notrun":0, + "notrun":4, "fail":1, "warn":0 }, @@ -61,7 +85,31 @@ "skip":0, "incomplete":0, "timeout":0, - "notrun":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, "fail":0, "warn":0 }, diff --git a/runner/json_tests_data/aborted-on-boot/reference.json b/runner/json_tests_data/aborted-on-boot/reference.json index 0a8723eb..d354fbac 100644 --- a/runner/json_tests_data/aborted-on-boot/reference.json +++ b/runner/json_tests_data/aborted-on-boot/reference.json @@ -9,6 +9,36 @@ "end":1539953735.1723731 }, "tests":{ + "igt@successtest@first-subtest":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@successtest@second-subtest":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@no-subtests":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@skippers@skip-one":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@skippers@skip-two":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, "igt@runner@aborted":{ "out":"Aborted after: startup\n\nKernel tainted (0x200)\n", "result":"fail", @@ -25,7 +55,7 @@ "skip":0, "incomplete":0, "timeout":0, - "notrun":0, + "notrun":5, "fail":1, "warn":0 }, @@ -37,10 +67,46 @@ "skip":0, "incomplete":0, "timeout":0, - "notrun":0, + "notrun":5, "fail":1, "warn":0 }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, + "fail":0, + "warn":0 + }, "igt@runner":{ "crash":0, "pass":0, diff --git a/runner/json_tests_data/incomplete-before-any-subtests/reference.json b/runner/json_tests_data/incomplete-before-any-subtests/reference.json index 8dc684c8..8404bb66 100644 --- a/runner/json_tests_data/incomplete-before-any-subtests/reference.json +++ b/runner/json_tests_data/incomplete-before-any-subtests/reference.json @@ -15,7 +15,31 @@ "result":"incomplete", "err":"", "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n" - } + }, + "igt@successtest@second-subtest":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@no-subtests":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@skippers@skip-one":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, + "igt@skippers@skip-two":{ + "out":"", + "err":"", + "dmesg":"", + "result":"notrun" + }, }, "totals":{ "":{ @@ -26,7 +50,7 @@ "skip":0, "incomplete":1, "timeout":0, - "notrun":0, + "notrun":4, "fail":0, "warn":0 }, @@ -38,7 +62,7 @@ "skip":0, "incomplete":1, "timeout":0, - "notrun":0, + "notrun":4, "fail":0, "warn":0 }, @@ -50,10 +74,34 @@ "skip":0, "incomplete":1, "timeout":0, - "notrun":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, "fail":0, "warn":0 }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, + "fail":0, + "warn":0 + } }, "runtimes":{ } diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt new file mode 100644 index 00000000..a189e704 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt @@ -0,0 +1,5 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +14,954,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/notrun-results-multiple-mode/README.txt b/runner/json_tests_data/notrun-results-multiple-mode/README.txt new file mode 100644 index 00000000..fedb6cf5 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/README.txt @@ -0,0 +1,4 @@ +Tests that are not executed are expected to be present in results.json +when possible. If run with --multiple-mode, we can't distinguish +tests without subtests from tests where we attempt to execute all +subtests. diff --git a/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt b/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt b/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt b/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt new file mode 100644 index 00000000..e387d8ed --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt @@ -0,0 +1,12 @@ +abort_mask : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 1 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/notrun-results-multiple-mode/reference.json b/runner/json_tests_data/notrun-results-multiple-mode/reference.json new file mode 100644 index 00000000..de1c3c31 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/reference.json @@ -0,0 +1,102 @@ +{ + "__type__":"TestrunResult", + "results_version":10, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@successtest@second-subtest":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@skippers@skip-one":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@skippers@skip-two":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":3, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":3, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.014 + } + }, + } +} diff --git a/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt b/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/notrun-results-multiple-mode/uname.txt b/runner/json_tests_data/notrun-results-multiple-mode/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/notrun-results/0/dmesg.txt b/runner/json_tests_data/notrun-results/0/dmesg.txt new file mode 100644 index 00000000..a189e704 --- /dev/null +++ b/runner/json_tests_data/notrun-results/0/dmesg.txt @@ -0,0 +1,5 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +14,954,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/notrun-results/0/err.txt b/runner/json_tests_data/notrun-results/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/notrun-results/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/notrun-results/0/journal.txt b/runner/json_tests_data/notrun-results/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/notrun-results/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/notrun-results/0/out.txt b/runner/json_tests_data/notrun-results/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/notrun-results/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/notrun-results/README.txt b/runner/json_tests_data/notrun-results/README.txt new file mode 100644 index 00000000..4355aebd --- /dev/null +++ b/runner/json_tests_data/notrun-results/README.txt @@ -0,0 +1,4 @@ +Tests that are not executed are expected to be present in results.json +when possible. If run without --multiple-mode, we can distinguish +tests without subtests from tests where we attempt to execute all +subtests. diff --git a/runner/json_tests_data/notrun-results/endtime.txt b/runner/json_tests_data/notrun-results/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/notrun-results/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/notrun-results/joblist.txt b/runner/json_tests_data/notrun-results/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/notrun-results/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/notrun-results/metadata.txt b/runner/json_tests_data/notrun-results/metadata.txt new file mode 100644 index 00000000..c501ae0e --- /dev/null +++ b/runner/json_tests_data/notrun-results/metadata.txt @@ -0,0 +1,12 @@ +abort_mask : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/notrun-results/reference.json b/runner/json_tests_data/notrun-results/reference.json new file mode 100644 index 00000000..6b5ff69b --- /dev/null +++ b/runner/json_tests_data/notrun-results/reference.json @@ -0,0 +1,120 @@ +{ + "__type__":"TestrunResult", + "results_version":10, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@successtest@second-subtest":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@no-subtests":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@skippers@skip-one":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@skippers@skip-two":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":4, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":4, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.014 + } + }, + } +} diff --git a/runner/json_tests_data/notrun-results/starttime.txt b/runner/json_tests_data/notrun-results/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/notrun-results/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/notrun-results/uname.txt b/runner/json_tests_data/notrun-results/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/notrun-results/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/resultgen.c b/runner/resultgen.c index a1a91cd0..be884955 100644 --- a/runner/resultgen.c +++ b/runner/resultgen.c @@ -1034,6 +1034,44 @@ static bool parse_test_directory(int dirfd, return status; } +static void try_add_notrun_results(const struct job_list_entry *entry, + const struct settings *settings, + struct results *results) +{ + struct subtests subtests = {}; + struct json_object *current_test; + size_t i; + + if (entry->subtest_count == 0) { + char piglit_name[256]; + + /* We cannot distinguish no-subtests from run-all-subtests in multiple-mode */ + if (settings->multiple_mode) + return; + generate_piglit_name(entry->binary, NULL, piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(results->tests, piglit_name); + json_object_object_add(current_test, "out", json_object_new_string("")); + json_object_object_add(current_test, "err", json_object_new_string("")); + json_object_object_add(current_test, "dmesg", json_object_new_string("")); + json_object_object_add(current_test, "result", json_object_new_string("notrun")); + } + + for (i = 0; i < entry->subtest_count; i++) { + char piglit_name[256]; + + generate_piglit_name(entry->binary, entry->subtests[i], piglit_name, sizeof(piglit_name)); + current_test = get_or_create_json_object(results->tests, piglit_name); + json_object_object_add(current_test, "out", json_object_new_string("")); + json_object_object_add(current_test, "err", json_object_new_string("")); + json_object_object_add(current_test, "dmesg", json_object_new_string("")); + json_object_object_add(current_test, "result", json_object_new_string("notrun")); + add_subtest(&subtests, strdup(entry->subtests[i])); + } + + add_to_totals(entry->binary, &subtests, results); + free_subtests(&subtests); +} + static void create_result_root_nodes(struct json_object *root, struct results *results) { @@ -1123,8 +1161,8 @@ struct json_object *generate_results_json(int dirfd) snprintf(name, 16, "%zd", i); if ((testdirfd = openat(dirfd, name, O_DIRECTORY | O_RDONLY)) < 0) { - fprintf(stderr, "Warning: Cannot open result directory %s\n", name); - break; + try_add_notrun_results(&job_list.entries[i], &settings, &results); + continue; } if (!parse_test_directory(testdirfd, &job_list.entries[i], &settings, &results)) { diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c index 17b81524..09a93189 100644 --- a/runner/runner_json_tests.c +++ b/runner/runner_json_tests.c @@ -157,6 +157,8 @@ static const char *dirnames[] = { "aborted-on-boot", "aborted-after-a-test", "dmesg-escapes", + "notrun-results", + "notrun-results-multiple-mode", }; igt_main -- cgit v1.2.3