diff options
author | Petri Latvala <petri.latvala@intel.com> | 2018-12-17 15:07:40 +0200 |
---|---|---|
committer | Petri Latvala <petri.latvala@intel.com> | 2018-12-20 14:10:31 +0200 |
commit | 321fe77d32fef32ef820f53924045fe6ef0cf6ed (patch) | |
tree | 2387bd5733f6c7a3932866276e76dc5debe68334 /runner/json_tests_data/notrun-results-multiple-mode | |
parent | 47f5a57a81b66c21d06695e263a22b87f5a33009 (diff) |
runner: Add explicit "notrun" results for tests that were not executed
When possible, all tests we know we were going to attempt to execute
now appear in the results as "notrun". The only known case where it's
not possible to add an explicit "notrun" is when running in
multiple-mode, because "no subtests" and "run all subtests, we didn't
list them beforehand" are represented the same.
v2: Rebase and adjust to already landed json changes
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Martin Peres <martin.peres@linux.intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Acked-by: Martin Peres <martin.peres@linux.intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Diffstat (limited to 'runner/json_tests_data/notrun-results-multiple-mode')
11 files changed, 138 insertions, 0 deletions
diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt new file mode 100644 index 00000000..a189e704 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/dmesg.txt @@ -0,0 +1,5 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +14,954,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt b/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/notrun-results-multiple-mode/README.txt b/runner/json_tests_data/notrun-results-multiple-mode/README.txt new file mode 100644 index 00000000..fedb6cf5 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/README.txt @@ -0,0 +1,4 @@ +Tests that are not executed are expected to be present in results.json +when possible. If run with --multiple-mode, we can't distinguish +tests without subtests from tests where we attempt to execute all +subtests. diff --git a/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt b/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt b/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt b/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt new file mode 100644 index 00000000..e387d8ed --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/metadata.txt @@ -0,0 +1,12 @@ +abort_mask : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 1 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/notrun-results-multiple-mode/reference.json b/runner/json_tests_data/notrun-results-multiple-mode/reference.json new file mode 100644 index 00000000..de1c3c31 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/reference.json @@ -0,0 +1,102 @@ +{ + "__type__":"TestrunResult", + "results_version":10, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@successtest@second-subtest":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@skippers@skip-one":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + }, + "igt@skippers@skip-two":{ + "result":"notrun", + "out":"", + "err":"", + "dmesg":"" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":3, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":3, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":1, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":2, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.014 + } + }, + } +} diff --git a/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt b/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/notrun-results-multiple-mode/uname.txt b/runner/json_tests_data/notrun-results-multiple-mode/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/notrun-results-multiple-mode/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 |