diff options
author | Petri Latvala <petri.latvala@intel.com> | 2018-10-23 15:20:24 +0300 |
---|---|---|
committer | Petri Latvala <petri.latvala@intel.com> | 2018-11-06 11:52:56 +0200 |
commit | e2d7d22ae7e69d30272f366034977ed5165911dd (patch) | |
tree | 811c1e8561c7873f5c45d652cb341ed07e9d83df /runner/json_tests_data/dmesg-results | |
parent | 6d19a3b4bddb4d8716cf6b763ee784ba9bc9f440 (diff) |
runner: Unit tests for json generation
Test the results.json generation with a top-down approach: With a
directory of test run intermediary logs, check that the resulting json
would match a reference json file.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Diffstat (limited to 'runner/json_tests_data/dmesg-results')
27 files changed, 246 insertions, 0 deletions
diff --git a/runner/json_tests_data/dmesg-results/0/dmesg.txt b/runner/json_tests_data/dmesg-results/0/dmesg.txt new file mode 100644 index 00000000..e50f75db --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/dmesg.txt @@ -0,0 +1,6 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +3,954,3216186101159,-;Warning from kernel +14,955,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/0/err.txt b/runner/json_tests_data/dmesg-results/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-results/0/journal.txt b/runner/json_tests_data/dmesg-results/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/dmesg-results/0/out.txt b/runner/json_tests_data/dmesg-results/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-results/1/dmesg.txt b/runner/json_tests_data/dmesg-results/1/dmesg.txt new file mode 100644 index 00000000..6c816cf2 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/dmesg.txt @@ -0,0 +1,6 @@ +6,956,3216186111837,-;Console: switching to colour dummy device 80x25 +14,957,3216186111851,-;[IGT] successtest: executing +14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest +3,954,3216186101159,-;Warning from kernel +14,959,3216186114814,-;[IGT] successtest: exiting, ret=0 +6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/1/err.txt b/runner/json_tests_data/dmesg-results/1/err.txt new file mode 100644 index 00000000..0f64562f --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/err.txt @@ -0,0 +1,2 @@ +Starting subtest: second-subtest +Subtest second-subtest: FAIL (0.000s) diff --git a/runner/json_tests_data/dmesg-results/1/journal.txt b/runner/json_tests_data/dmesg-results/1/journal.txt new file mode 100644 index 00000000..99f57815 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/journal.txt @@ -0,0 +1,2 @@ +second-subtest +exit:0 (0.013s) diff --git a/runner/json_tests_data/dmesg-results/1/out.txt b/runner/json_tests_data/dmesg-results/1/out.txt new file mode 100644 index 00000000..d2778f3c --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: second-subtest +Subtest second-subtest: FAIL (0.000s) diff --git a/runner/json_tests_data/dmesg-results/2/dmesg.txt b/runner/json_tests_data/dmesg-results/2/dmesg.txt new file mode 100644 index 00000000..913979a8 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/dmesg.txt @@ -0,0 +1,5 @@ +6,961,3216186123400,-;Console: switching to colour dummy device 80x25 +14,962,3216186123414,-;[IGT] no-subtests: executing +3,954,3216186101159,-;Warning from kernel +14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0 +6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/2/err.txt b/runner/json_tests_data/dmesg-results/2/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/err.txt diff --git a/runner/json_tests_data/dmesg-results/2/journal.txt b/runner/json_tests_data/dmesg-results/2/journal.txt new file mode 100644 index 00000000..7151877f --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/journal.txt @@ -0,0 +1 @@ +exit:0 (0.010s) diff --git a/runner/json_tests_data/dmesg-results/2/out.txt b/runner/json_tests_data/dmesg-results/2/out.txt new file mode 100644 index 00000000..695b67c2 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/out.txt @@ -0,0 +1,2 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-results/3/dmesg.txt b/runner/json_tests_data/dmesg-results/3/dmesg.txt new file mode 100644 index 00000000..a61e83e8 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/dmesg.txt @@ -0,0 +1,5 @@ +6,965,3216186135188,-;Console: switching to colour dummy device 80x25 +14,966,3216186135212,-;[IGT] skippers: executing +3,954,3216186101159,-;Warning from kernel +14,967,3216186137075,-;[IGT] skippers: exiting, ret=77 +6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/3/err.txt b/runner/json_tests_data/dmesg-results/3/err.txt new file mode 100644 index 00000000..59b73d09 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/err.txt @@ -0,0 +1 @@ +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/dmesg-results/3/journal.txt b/runner/json_tests_data/dmesg-results/3/journal.txt new file mode 100644 index 00000000..afab8ba6 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/journal.txt @@ -0,0 +1,2 @@ +skip-one +exit:77 (0.011s) diff --git a/runner/json_tests_data/dmesg-results/3/out.txt b/runner/json_tests_data/dmesg-results/3/out.txt new file mode 100644 index 00000000..96284e78 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/dmesg-results/4/dmesg.txt b/runner/json_tests_data/dmesg-results/4/dmesg.txt new file mode 100644 index 00000000..fcb43d57 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/dmesg.txt @@ -0,0 +1,5 @@ +6,969,3216186145899,-;Console: switching to colour dummy device 80x25 +14,970,3216186145912,-;[IGT] skippers: executing +3,954,3216186101159,-;Warning from kernel +14,971,3216186147754,-;[IGT] skippers: exiting, ret=77 +6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/4/err.txt b/runner/json_tests_data/dmesg-results/4/err.txt new file mode 100644 index 00000000..2251da1e --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/err.txt @@ -0,0 +1 @@ +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/dmesg-results/4/journal.txt b/runner/json_tests_data/dmesg-results/4/journal.txt new file mode 100644 index 00000000..a9dba132 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/journal.txt @@ -0,0 +1,2 @@ +skip-two +exit:77 (0.010s) diff --git a/runner/json_tests_data/dmesg-results/4/out.txt b/runner/json_tests_data/dmesg-results/4/out.txt new file mode 100644 index 00000000..2024db8f --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/dmesg-results/README.txt b/runner/json_tests_data/dmesg-results/README.txt new file mode 100644 index 00000000..d928a526 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/README.txt @@ -0,0 +1,3 @@ +Certain types of kernel log messages cause the result to change from +'pass' to 'dmesg-warn', and from 'fail' to 'dmesg-fail'. 'skip' should +stay 'skip'. diff --git a/runner/json_tests_data/dmesg-results/endtime.txt b/runner/json_tests_data/dmesg-results/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/dmesg-results/joblist.txt b/runner/json_tests_data/dmesg-results/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/dmesg-results/metadata.txt b/runner/json_tests_data/dmesg-results/metadata.txt new file mode 100644 index 00000000..1316560d --- /dev/null +++ b/runner/json_tests_data/dmesg-results/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json new file mode 100644 index 00000000..811b63eb --- /dev/null +++ b/runner/json_tests_data/dmesg-results/reference.json @@ -0,0 +1,161 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-warn", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@successtest@second-subtest":{ + "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-fail", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n", + "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + }, + "result":"dmesg-warn", + "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "err":"", + "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@skippers@skip-one":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-one: SKIP\n", + "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-two":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-two: SKIP\n", + "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":0, + "dmesg-fail":1, + "dmesg-warn":2, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":0, + "dmesg-fail":1, + "dmesg-warn":2, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":1, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.027 + } + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + } + }, + "igt@skippers":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.020999999999999998 + } + } + } +} diff --git a/runner/json_tests_data/dmesg-results/starttime.txt b/runner/json_tests_data/dmesg-results/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/dmesg-results/uname.txt b/runner/json_tests_data/dmesg-results/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 |