diff options
| author | Petri Latvala <petri.latvala@intel.com> | 2018-10-23 15:20:24 +0300 |
|---|---|---|
| committer | Petri Latvala <petri.latvala@intel.com> | 2018-11-06 11:52:56 +0200 |
| commit | e2d7d22ae7e69d30272f366034977ed5165911dd (patch) | |
| tree | 811c1e8561c7873f5c45d652cb341ed07e9d83df | |
| parent | 6d19a3b4bddb4d8716cf6b763ee784ba9bc9f440 (diff) | |
runner: Unit tests for json generation
Test the results.json generation with a top-down approach: With a
directory of test run intermediary logs, check that the resulting json
would match a reference json file.
Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
151 files changed, 1489 insertions, 2 deletions
diff --git a/runner/.gitignore b/runner/.gitignore index c5030e66..3812da23 100644 --- a/runner/.gitignore +++ b/runner/.gitignore @@ -2,3 +2,4 @@ igt_runner igt_resume igt_results runner_test +runner_json_test diff --git a/runner/Makefile.am b/runner/Makefile.am index ddcbf86a..b55cf797 100644 --- a/runner/Makefile.am +++ b/runner/Makefile.am @@ -30,8 +30,9 @@ AM_CFLAGS = $(JSONC_CFLAGS) \ -I$(srcdir)/../lib \ -D_GNU_SOURCE -TESTS = runner_test -check_PROGRAMS = runner_test +TESTS = runner_test runner_json_test +check_PROGRAMS = runner_test runner_json_test + runner_test_SOURCES = runner_tests.c runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \ -I$(top_srcdir)/include/drm-uapi \ @@ -40,4 +41,13 @@ runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \ -I$(srcdir)/../lib \ -D_GNU_SOURCE +runner_json_test_SOURCES = runner_json_tests.c +runner_json_test_CFLAGS = -DJSON_TESTS_DIRECTORY=\"$(abs_builddir)/json_tests_data\" \ + $(JSONC_CFLAGS) \ + -I$(top_srcdir)/include/drm-uapi \ + $(DRM_CFLAGS) $(CAIRO_CFLAGS) $(LIBUDEV_CFLAGS) \ + -I$(srcdir)/.. \ + -I$(srcdir)/../lib \ + -D_GNU_SOURCE + endif diff --git a/runner/json_tests_data/README.txt b/runner/json_tests_data/README.txt new file mode 100644 index 00000000..206d7630 --- /dev/null +++ b/runner/json_tests_data/README.txt @@ -0,0 +1,10 @@ +This directory is for testing igt_runner's results.json creation. Each +subdirectory is a run-result directory, referred to from +runner_json_tests.c, with test run intermediary files without any +results.json. A semi-automatically crafted reference.json file in each +respective directory is compared with igt_runner's generated json and +needs to match semantically; indentation and such doesn't matter, data +has to be equal. + +Each directory also contains a README file explaining what the +particular directory aims to test. diff --git a/runner/json_tests_data/dmesg-results/0/dmesg.txt b/runner/json_tests_data/dmesg-results/0/dmesg.txt new file mode 100644 index 00000000..e50f75db --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/dmesg.txt @@ -0,0 +1,6 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +3,954,3216186101159,-;Warning from kernel +14,955,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/0/err.txt b/runner/json_tests_data/dmesg-results/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-results/0/journal.txt b/runner/json_tests_data/dmesg-results/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/dmesg-results/0/out.txt b/runner/json_tests_data/dmesg-results/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-results/1/dmesg.txt b/runner/json_tests_data/dmesg-results/1/dmesg.txt new file mode 100644 index 00000000..6c816cf2 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/dmesg.txt @@ -0,0 +1,6 @@ +6,956,3216186111837,-;Console: switching to colour dummy device 80x25 +14,957,3216186111851,-;[IGT] successtest: executing +14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest +3,954,3216186101159,-;Warning from kernel +14,959,3216186114814,-;[IGT] successtest: exiting, ret=0 +6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/1/err.txt b/runner/json_tests_data/dmesg-results/1/err.txt new file mode 100644 index 00000000..0f64562f --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/err.txt @@ -0,0 +1,2 @@ +Starting subtest: second-subtest +Subtest second-subtest: FAIL (0.000s) diff --git a/runner/json_tests_data/dmesg-results/1/journal.txt b/runner/json_tests_data/dmesg-results/1/journal.txt new file mode 100644 index 00000000..99f57815 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/journal.txt @@ -0,0 +1,2 @@ +second-subtest +exit:0 (0.013s) diff --git a/runner/json_tests_data/dmesg-results/1/out.txt b/runner/json_tests_data/dmesg-results/1/out.txt new file mode 100644 index 00000000..d2778f3c --- /dev/null +++ b/runner/json_tests_data/dmesg-results/1/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: second-subtest +Subtest second-subtest: FAIL (0.000s) diff --git a/runner/json_tests_data/dmesg-results/2/dmesg.txt b/runner/json_tests_data/dmesg-results/2/dmesg.txt new file mode 100644 index 00000000..913979a8 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/dmesg.txt @@ -0,0 +1,5 @@ +6,961,3216186123400,-;Console: switching to colour dummy device 80x25 +14,962,3216186123414,-;[IGT] no-subtests: executing +3,954,3216186101159,-;Warning from kernel +14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0 +6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/2/err.txt b/runner/json_tests_data/dmesg-results/2/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/err.txt diff --git a/runner/json_tests_data/dmesg-results/2/journal.txt b/runner/json_tests_data/dmesg-results/2/journal.txt new file mode 100644 index 00000000..7151877f --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/journal.txt @@ -0,0 +1 @@ +exit:0 (0.010s) diff --git a/runner/json_tests_data/dmesg-results/2/out.txt b/runner/json_tests_data/dmesg-results/2/out.txt new file mode 100644 index 00000000..695b67c2 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/2/out.txt @@ -0,0 +1,2 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +SUCCESS (0.000s) diff --git a/runner/json_tests_data/dmesg-results/3/dmesg.txt b/runner/json_tests_data/dmesg-results/3/dmesg.txt new file mode 100644 index 00000000..a61e83e8 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/dmesg.txt @@ -0,0 +1,5 @@ +6,965,3216186135188,-;Console: switching to colour dummy device 80x25 +14,966,3216186135212,-;[IGT] skippers: executing +3,954,3216186101159,-;Warning from kernel +14,967,3216186137075,-;[IGT] skippers: exiting, ret=77 +6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/3/err.txt b/runner/json_tests_data/dmesg-results/3/err.txt new file mode 100644 index 00000000..59b73d09 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/err.txt @@ -0,0 +1 @@ +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/dmesg-results/3/journal.txt b/runner/json_tests_data/dmesg-results/3/journal.txt new file mode 100644 index 00000000..afab8ba6 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/journal.txt @@ -0,0 +1,2 @@ +skip-one +exit:77 (0.011s) diff --git a/runner/json_tests_data/dmesg-results/3/out.txt b/runner/json_tests_data/dmesg-results/3/out.txt new file mode 100644 index 00000000..96284e78 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/3/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/dmesg-results/4/dmesg.txt b/runner/json_tests_data/dmesg-results/4/dmesg.txt new file mode 100644 index 00000000..fcb43d57 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/dmesg.txt @@ -0,0 +1,5 @@ +6,969,3216186145899,-;Console: switching to colour dummy device 80x25 +14,970,3216186145912,-;[IGT] skippers: executing +3,954,3216186101159,-;Warning from kernel +14,971,3216186147754,-;[IGT] skippers: exiting, ret=77 +6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/dmesg-results/4/err.txt b/runner/json_tests_data/dmesg-results/4/err.txt new file mode 100644 index 00000000..2251da1e --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/err.txt @@ -0,0 +1 @@ +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/dmesg-results/4/journal.txt b/runner/json_tests_data/dmesg-results/4/journal.txt new file mode 100644 index 00000000..a9dba132 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/journal.txt @@ -0,0 +1,2 @@ +skip-two +exit:77 (0.010s) diff --git a/runner/json_tests_data/dmesg-results/4/out.txt b/runner/json_tests_data/dmesg-results/4/out.txt new file mode 100644 index 00000000..2024db8f --- /dev/null +++ b/runner/json_tests_data/dmesg-results/4/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/dmesg-results/README.txt b/runner/json_tests_data/dmesg-results/README.txt new file mode 100644 index 00000000..d928a526 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/README.txt @@ -0,0 +1,3 @@ +Certain types of kernel log messages cause the result to change from +'pass' to 'dmesg-warn', and from 'fail' to 'dmesg-fail'. 'skip' should +stay 'skip'. diff --git a/runner/json_tests_data/dmesg-results/endtime.txt b/runner/json_tests_data/dmesg-results/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/dmesg-results/joblist.txt b/runner/json_tests_data/dmesg-results/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/dmesg-results/metadata.txt b/runner/json_tests_data/dmesg-results/metadata.txt new file mode 100644 index 00000000..1316560d --- /dev/null +++ b/runner/json_tests_data/dmesg-results/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json new file mode 100644 index 00000000..811b63eb --- /dev/null +++ b/runner/json_tests_data/dmesg-results/reference.json @@ -0,0 +1,161 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-warn", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@successtest@second-subtest":{ + "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-fail", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n", + "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + }, + "result":"dmesg-warn", + "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "err":"", + "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@skippers@skip-one":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-one: SKIP\n", + "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-two":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-two: SKIP\n", + "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":0, + "dmesg-fail":1, + "dmesg-warn":2, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":0, + "dmesg-fail":1, + "dmesg-warn":2, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":1, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.027 + } + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + } + }, + "igt@skippers":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.020999999999999998 + } + } + } +} diff --git a/runner/json_tests_data/dmesg-results/starttime.txt b/runner/json_tests_data/dmesg-results/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/dmesg-results/uname.txt b/runner/json_tests_data/dmesg-results/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/dmesg-results/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt new file mode 100644 index 00000000..7c6834d5 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt @@ -0,0 +1,2 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt new file mode 100644 index 00000000..ebaa6300 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt @@ -0,0 +1 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) diff --git a/runner/json_tests_data/incomplete-before-any-subtests/README.txt b/runner/json_tests_data/incomplete-before-any-subtests/README.txt new file mode 100644 index 00000000..fb64f581 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/README.txt @@ -0,0 +1,4 @@ +When a test hangs before it even enters a subtest, we don't have a +journal entry for that subtest. This test checks that the result still +gets attributed to a subtest, since we know from the job_list that a +subtest should be executed. diff --git a/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt b/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt b/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt new file mode 100644 index 00000000..1316560d --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/incomplete-before-any-subtests/reference.json b/runner/json_tests_data/incomplete-before-any-subtests/reference.json new file mode 100644 index 00000000..3eb6c9c1 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/reference.json @@ -0,0 +1,60 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"incomplete", + "err":"", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":1, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":1, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":1, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + }, + "runtimes":{ + } +} diff --git a/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt b/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/incomplete-before-any-subtests/uname.txt b/runner/json_tests_data/incomplete-before-any-subtests/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/incomplete-before-any-subtests/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/normal-run/0/dmesg.txt b/runner/json_tests_data/normal-run/0/dmesg.txt new file mode 100644 index 00000000..a189e704 --- /dev/null +++ b/runner/json_tests_data/normal-run/0/dmesg.txt @@ -0,0 +1,5 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +14,954,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/normal-run/0/err.txt b/runner/json_tests_data/normal-run/0/err.txt new file mode 100644 index 00000000..5dc78057 --- /dev/null +++ b/runner/json_tests_data/normal-run/0/err.txt @@ -0,0 +1,2 @@ +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/normal-run/0/journal.txt b/runner/json_tests_data/normal-run/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/normal-run/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/normal-run/0/out.txt b/runner/json_tests_data/normal-run/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/normal-run/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/normal-run/1/dmesg.txt b/runner/json_tests_data/normal-run/1/dmesg.txt new file mode 100644 index 00000000..59b7cee3 --- /dev/null +++ b/runner/json_tests_data/normal-run/1/dmesg.txt @@ -0,0 +1,5 @@ +6,956,3216186111837,-;Console: switching to colour dummy device 80x25 +14,957,3216186111851,-;[IGT] successtest: executing +14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest +14,959,3216186114814,-;[IGT] successtest: exiting, ret=0 +6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/normal-run/1/err.txt b/runner/json_tests_data/normal-run/1/err.txt new file mode 100644 index 00000000..0f64562f --- /dev/null +++ b/runner/json_tests_data/normal-run/1/err.txt @@ -0,0 +1,2 @@ +Starting subtest: second-subtest +Subtest second-subtest: FAIL (0.000s) diff --git a/runner/json_tests_data/normal-run/1/journal.txt b/runner/json_tests_data/normal-run/1/journal.txt new file mode 100644 index 00000000..99f57815 --- /dev/null +++ b/runner/json_tests_data/normal-run/1/journal.txt @@ -0,0 +1,2 @@ +second-subtest +exit:0 (0.013s) diff --git a/runner/json_tests_data/normal-run/1/out.txt b/runner/json_tests_data/normal-run/1/out.txt new file mode 100644 index 00000000..d2778f3c --- /dev/null +++ b/runner/json_tests_data/normal-run/1/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: second-subtest +Subtest second-subtest: FAIL (0.000s) diff --git a/runner/json_tests_data/normal-run/2/dmesg.txt b/runner/json_tests_data/normal-run/2/dmesg.txt new file mode 100644 index 00000000..998b4797 --- /dev/null +++ b/runner/json_tests_data/normal-run/2/dmesg.txt @@ -0,0 +1,4 @@ +6,961,3216186123400,-;Console: switching to colour dummy device 80x25 +14,962,3216186123414,-;[IGT] no-subtests: executing +14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0 +6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/normal-run/2/err.txt b/runner/json_tests_data/normal-run/2/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/normal-run/2/err.txt diff --git a/runner/json_tests_data/normal-run/2/journal.txt b/runner/json_tests_data/normal-run/2/journal.txt new file mode 100644 index 00000000..7151877f --- /dev/null +++ b/runner/json_tests_data/normal-run/2/journal.txt @@ -0,0 +1 @@ +exit:0 (0.010s) diff --git a/runner/json_tests_data/normal-run/2/out.txt b/runner/json_tests_data/normal-run/2/out.txt new file mode 100644 index 00000000..695b67c2 --- /dev/null +++ b/runner/json_tests_data/normal-run/2/out.txt @@ -0,0 +1,2 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +SUCCESS (0.000s) diff --git a/runner/json_tests_data/normal-run/3/dmesg.txt b/runner/json_tests_data/normal-run/3/dmesg.txt new file mode 100644 index 00000000..21e75031 --- /dev/null +++ b/runner/json_tests_data/normal-run/3/dmesg.txt @@ -0,0 +1,4 @@ +6,965,3216186135188,-;Console: switching to colour dummy device 80x25 +14,966,3216186135212,-;[IGT] skippers: executing +14,967,3216186137075,-;[IGT] skippers: exiting, ret=77 +6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/normal-run/3/err.txt b/runner/json_tests_data/normal-run/3/err.txt new file mode 100644 index 00000000..59b73d09 --- /dev/null +++ b/runner/json_tests_data/normal-run/3/err.txt @@ -0,0 +1 @@ +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/normal-run/3/journal.txt b/runner/json_tests_data/normal-run/3/journal.txt new file mode 100644 index 00000000..afab8ba6 --- /dev/null +++ b/runner/json_tests_data/normal-run/3/journal.txt @@ -0,0 +1,2 @@ +skip-one +exit:77 (0.011s) diff --git a/runner/json_tests_data/normal-run/3/out.txt b/runner/json_tests_data/normal-run/3/out.txt new file mode 100644 index 00000000..96284e78 --- /dev/null +++ b/runner/json_tests_data/normal-run/3/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/normal-run/4/dmesg.txt b/runner/json_tests_data/normal-run/4/dmesg.txt new file mode 100644 index 00000000..737bc692 --- /dev/null +++ b/runner/json_tests_data/normal-run/4/dmesg.txt @@ -0,0 +1,4 @@ +6,969,3216186145899,-;Console: switching to colour dummy device 80x25 +14,970,3216186145912,-;[IGT] skippers: executing +14,971,3216186147754,-;[IGT] skippers: exiting, ret=77 +6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/normal-run/4/err.txt b/runner/json_tests_data/normal-run/4/err.txt new file mode 100644 index 00000000..2251da1e --- /dev/null +++ b/runner/json_tests_data/normal-run/4/err.txt @@ -0,0 +1 @@ +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/normal-run/4/journal.txt b/runner/json_tests_data/normal-run/4/journal.txt new file mode 100644 index 00000000..a9dba132 --- /dev/null +++ b/runner/json_tests_data/normal-run/4/journal.txt @@ -0,0 +1,2 @@ +skip-two +exit:77 (0.010s) diff --git a/runner/json_tests_data/normal-run/4/out.txt b/runner/json_tests_data/normal-run/4/out.txt new file mode 100644 index 00000000..2024db8f --- /dev/null +++ b/runner/json_tests_data/normal-run/4/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/normal-run/README.txt b/runner/json_tests_data/normal-run/README.txt new file mode 100644 index 00000000..64270e88 --- /dev/null +++ b/runner/json_tests_data/normal-run/README.txt @@ -0,0 +1 @@ +A normal test run with pass, fail and skip results. diff --git a/runner/json_tests_data/normal-run/endtime.txt b/runner/json_tests_data/normal-run/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/normal-run/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/normal-run/joblist.txt b/runner/json_tests_data/normal-run/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/normal-run/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/normal-run/metadata.txt b/runner/json_tests_data/normal-run/metadata.txt new file mode 100644 index 00000000..1316560d --- /dev/null +++ b/runner/json_tests_data/normal-run/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/normal-run/reference.json b/runner/json_tests_data/normal-run/reference.json new file mode 100644 index 00000000..0d28b2c2 --- /dev/null +++ b/runner/json_tests_data/normal-run/reference.json @@ -0,0 +1,158 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@successtest@second-subtest":{ + "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"fail", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n", + "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + }, + "result":"pass", + "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "err":"", + "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-one":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-one: SKIP\n", + "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-two":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-two: SKIP\n", + "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":1, + "warn":0 + }, + "root":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":1, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":1, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.027 + } + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + } + }, + "igt@skippers":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.020999999999999998 + } + } + } +} diff --git a/runner/json_tests_data/normal-run/starttime.txt b/runner/json_tests_data/normal-run/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/normal-run/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/normal-run/uname.txt b/runner/json_tests_data/normal-run/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/normal-run/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt new file mode 100644 index 00000000..e50f75db --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt @@ -0,0 +1,6 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +3,954,3216186101159,-;Warning from kernel +14,955,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/piglit-style-dmesg/0/err.txt b/runner/json_tests_data/piglit-style-dmesg/0/err.txt new file mode 100644 index 00000000..e18c00e9 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/0/err.txt @@ -0,0 +1,3 @@ +Starting subtest: first-subtest +This is a warning line +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/piglit-style-dmesg/0/journal.txt b/runner/json_tests_data/piglit-style-dmesg/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/piglit-style-dmesg/0/out.txt b/runner/json_tests_data/piglit-style-dmesg/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt new file mode 100644 index 00000000..59b7cee3 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt @@ -0,0 +1,5 @@ +6,956,3216186111837,-;Console: switching to colour dummy device 80x25 +14,957,3216186111851,-;[IGT] successtest: executing +14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest +14,959,3216186114814,-;[IGT] successtest: exiting, ret=0 +6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/piglit-style-dmesg/1/err.txt b/runner/json_tests_data/piglit-style-dmesg/1/err.txt new file mode 100644 index 00000000..57be9f7b --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/1/err.txt @@ -0,0 +1,2 @@ +Starting subtest: second-subtest +Subtest second-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/piglit-style-dmesg/1/journal.txt b/runner/json_tests_data/piglit-style-dmesg/1/journal.txt new file mode 100644 index 00000000..99f57815 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/1/journal.txt @@ -0,0 +1,2 @@ +second-subtest +exit:0 (0.013s) diff --git a/runner/json_tests_data/piglit-style-dmesg/1/out.txt b/runner/json_tests_data/piglit-style-dmesg/1/out.txt new file mode 100644 index 00000000..24b37244 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/1/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: second-subtest +Subtest second-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt new file mode 100644 index 00000000..998b4797 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt @@ -0,0 +1,4 @@ +6,961,3216186123400,-;Console: switching to colour dummy device 80x25 +14,962,3216186123414,-;[IGT] no-subtests: executing +14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0 +6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/piglit-style-dmesg/2/err.txt b/runner/json_tests_data/piglit-style-dmesg/2/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/2/err.txt diff --git a/runner/json_tests_data/piglit-style-dmesg/2/journal.txt b/runner/json_tests_data/piglit-style-dmesg/2/journal.txt new file mode 100644 index 00000000..7151877f --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/2/journal.txt @@ -0,0 +1 @@ +exit:0 (0.010s) diff --git a/runner/json_tests_data/piglit-style-dmesg/2/out.txt b/runner/json_tests_data/piglit-style-dmesg/2/out.txt new file mode 100644 index 00000000..695b67c2 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/2/out.txt @@ -0,0 +1,2 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +SUCCESS (0.000s) diff --git a/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt new file mode 100644 index 00000000..21e75031 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt @@ -0,0 +1,4 @@ +6,965,3216186135188,-;Console: switching to colour dummy device 80x25 +14,966,3216186135212,-;[IGT] skippers: executing +14,967,3216186137075,-;[IGT] skippers: exiting, ret=77 +6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/piglit-style-dmesg/3/err.txt b/runner/json_tests_data/piglit-style-dmesg/3/err.txt new file mode 100644 index 00000000..59b73d09 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/3/err.txt @@ -0,0 +1 @@ +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/piglit-style-dmesg/3/journal.txt b/runner/json_tests_data/piglit-style-dmesg/3/journal.txt new file mode 100644 index 00000000..afab8ba6 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/3/journal.txt @@ -0,0 +1,2 @@ +skip-one +exit:77 (0.011s) diff --git a/runner/json_tests_data/piglit-style-dmesg/3/out.txt b/runner/json_tests_data/piglit-style-dmesg/3/out.txt new file mode 100644 index 00000000..96284e78 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/3/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt new file mode 100644 index 00000000..737bc692 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt @@ -0,0 +1,4 @@ +6,969,3216186145899,-;Console: switching to colour dummy device 80x25 +14,970,3216186145912,-;[IGT] skippers: executing +14,971,3216186147754,-;[IGT] skippers: exiting, ret=77 +6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/piglit-style-dmesg/4/err.txt b/runner/json_tests_data/piglit-style-dmesg/4/err.txt new file mode 100644 index 00000000..2251da1e --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/4/err.txt @@ -0,0 +1 @@ +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/piglit-style-dmesg/4/journal.txt b/runner/json_tests_data/piglit-style-dmesg/4/journal.txt new file mode 100644 index 00000000..a9dba132 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/4/journal.txt @@ -0,0 +1,2 @@ +skip-two +exit:77 (0.010s) diff --git a/runner/json_tests_data/piglit-style-dmesg/4/out.txt b/runner/json_tests_data/piglit-style-dmesg/4/out.txt new file mode 100644 index 00000000..2024db8f --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/4/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/piglit-style-dmesg/README.txt b/runner/json_tests_data/piglit-style-dmesg/README.txt new file mode 100644 index 00000000..3130337b --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/README.txt @@ -0,0 +1,2 @@ +When --piglit-style-dmesg is used, only a particular pattern of kernel +log messages trigger a result change to dmesg-warn/dmesg-fail. diff --git a/runner/json_tests_data/piglit-style-dmesg/endtime.txt b/runner/json_tests_data/piglit-style-dmesg/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/piglit-style-dmesg/joblist.txt b/runner/json_tests_data/piglit-style-dmesg/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/piglit-style-dmesg/metadata.txt b/runner/json_tests_data/piglit-style-dmesg/metadata.txt new file mode 100644 index 00000000..7f1372c1 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 1 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/piglit-style-dmesg/reference.json b/runner/json_tests_data/piglit-style-dmesg/reference.json new file mode 100644 index 00000000..2257b3c7 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/reference.json @@ -0,0 +1,158 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"warn", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@successtest@second-subtest":{ + "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + }, + "result":"pass", + "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "err":"", + "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-one":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-one: SKIP\n", + "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-two":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-two: SKIP\n", + "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":1 + }, + "root":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":1 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":1 + }, + "igt@no-subtests":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.027 + } + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + } + }, + "igt@skippers":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.020999999999999998 + } + } + } +} diff --git a/runner/json_tests_data/piglit-style-dmesg/starttime.txt b/runner/json_tests_data/piglit-style-dmesg/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/piglit-style-dmesg/uname.txt b/runner/json_tests_data/piglit-style-dmesg/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/piglit-style-dmesg/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt new file mode 100644 index 00000000..e50f75db --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt @@ -0,0 +1,6 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +3,954,3216186101159,-;Warning from kernel +14,955,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt new file mode 100644 index 00000000..e18c00e9 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt @@ -0,0 +1,3 @@ +Starting subtest: first-subtest +This is a warning line +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt new file mode 100644 index 00000000..59b7cee3 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt @@ -0,0 +1,5 @@ +6,956,3216186111837,-;Console: switching to colour dummy device 80x25 +14,957,3216186111851,-;[IGT] successtest: executing +14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest +14,959,3216186114814,-;[IGT] successtest: exiting, ret=0 +6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt new file mode 100644 index 00000000..57be9f7b --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt @@ -0,0 +1,2 @@ +Starting subtest: second-subtest +Subtest second-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt new file mode 100644 index 00000000..99f57815 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt @@ -0,0 +1,2 @@ +second-subtest +exit:0 (0.013s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt new file mode 100644 index 00000000..24b37244 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: second-subtest +Subtest second-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt new file mode 100644 index 00000000..998b4797 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt @@ -0,0 +1,4 @@ +6,961,3216186123400,-;Console: switching to colour dummy device 80x25 +14,962,3216186123414,-;[IGT] no-subtests: executing +14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0 +6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt new file mode 100644 index 00000000..7151877f --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt @@ -0,0 +1 @@ +exit:0 (0.010s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt new file mode 100644 index 00000000..695b67c2 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt @@ -0,0 +1,2 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt new file mode 100644 index 00000000..21e75031 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt @@ -0,0 +1,4 @@ +6,965,3216186135188,-;Console: switching to colour dummy device 80x25 +14,966,3216186135212,-;[IGT] skippers: executing +14,967,3216186137075,-;[IGT] skippers: exiting, ret=77 +6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt new file mode 100644 index 00000000..59b73d09 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt @@ -0,0 +1 @@ +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt new file mode 100644 index 00000000..afab8ba6 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt @@ -0,0 +1,2 @@ +skip-one +exit:77 (0.011s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt new file mode 100644 index 00000000..96284e78 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt new file mode 100644 index 00000000..737bc692 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt @@ -0,0 +1,4 @@ +6,969,3216186145899,-;Console: switching to colour dummy device 80x25 +14,970,3216186145912,-;[IGT] skippers: executing +14,971,3216186147754,-;[IGT] skippers: exiting, ret=77 +6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt new file mode 100644 index 00000000..2251da1e --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt @@ -0,0 +1 @@ +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt new file mode 100644 index 00000000..a9dba132 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt @@ -0,0 +1,2 @@ +skip-two +exit:77 (0.010s) diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt new file mode 100644 index 00000000..2024db8f --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/README.txt b/runner/json_tests_data/warnings-with-dmesg-warns/README.txt new file mode 100644 index 00000000..7ef9ee73 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/README.txt @@ -0,0 +1,3 @@ +A test that has output in stderr should result in 'warn' instead of +'pass'. But if there is a message in the kernel logs that would make +the result 'dmesg-warn' instead, that takes priority over 'warn'. diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt b/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt b/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt b/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt new file mode 100644 index 00000000..1316560d --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json new file mode 100644 index 00000000..9e027efb --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json @@ -0,0 +1,159 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"dmesg-warn", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n", + "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n" + }, + "igt@successtest@second-subtest":{ + "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + }, + "result":"pass", + "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "err":"", + "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-one":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-one: SKIP\n", + "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-two":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-two: SKIP\n", + "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "root":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":1, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@no-subtests":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.027 + } + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + } + }, + "igt@skippers":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.020999999999999998 + } + } + } +} diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt b/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt b/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/json_tests_data/warnings/0/dmesg.txt b/runner/json_tests_data/warnings/0/dmesg.txt new file mode 100644 index 00000000..a189e704 --- /dev/null +++ b/runner/json_tests_data/warnings/0/dmesg.txt @@ -0,0 +1,5 @@ +6,951,3216186095083,-;Console: switching to colour dummy device 80x25 +14,952,3216186095097,-;[IGT] successtest: executing +14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest +14,954,3216186101160,-;[IGT] successtest: exiting, ret=0 +6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings/0/err.txt b/runner/json_tests_data/warnings/0/err.txt new file mode 100644 index 00000000..e18c00e9 --- /dev/null +++ b/runner/json_tests_data/warnings/0/err.txt @@ -0,0 +1,3 @@ +Starting subtest: first-subtest +This is a warning line +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings/0/journal.txt b/runner/json_tests_data/warnings/0/journal.txt new file mode 100644 index 00000000..86a30e07 --- /dev/null +++ b/runner/json_tests_data/warnings/0/journal.txt @@ -0,0 +1,2 @@ +first-subtest +exit:0 (0.014s) diff --git a/runner/json_tests_data/warnings/0/out.txt b/runner/json_tests_data/warnings/0/out.txt new file mode 100644 index 00000000..5946bf31 --- /dev/null +++ b/runner/json_tests_data/warnings/0/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: first-subtest +Subtest first-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings/1/dmesg.txt b/runner/json_tests_data/warnings/1/dmesg.txt new file mode 100644 index 00000000..59b7cee3 --- /dev/null +++ b/runner/json_tests_data/warnings/1/dmesg.txt @@ -0,0 +1,5 @@ +6,956,3216186111837,-;Console: switching to colour dummy device 80x25 +14,957,3216186111851,-;[IGT] successtest: executing +14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest +14,959,3216186114814,-;[IGT] successtest: exiting, ret=0 +6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings/1/err.txt b/runner/json_tests_data/warnings/1/err.txt new file mode 100644 index 00000000..57be9f7b --- /dev/null +++ b/runner/json_tests_data/warnings/1/err.txt @@ -0,0 +1,2 @@ +Starting subtest: second-subtest +Subtest second-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings/1/journal.txt b/runner/json_tests_data/warnings/1/journal.txt new file mode 100644 index 00000000..99f57815 --- /dev/null +++ b/runner/json_tests_data/warnings/1/journal.txt @@ -0,0 +1,2 @@ +second-subtest +exit:0 (0.013s) diff --git a/runner/json_tests_data/warnings/1/out.txt b/runner/json_tests_data/warnings/1/out.txt new file mode 100644 index 00000000..24b37244 --- /dev/null +++ b/runner/json_tests_data/warnings/1/out.txt @@ -0,0 +1,3 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Starting subtest: second-subtest +Subtest second-subtest: SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings/2/dmesg.txt b/runner/json_tests_data/warnings/2/dmesg.txt new file mode 100644 index 00000000..998b4797 --- /dev/null +++ b/runner/json_tests_data/warnings/2/dmesg.txt @@ -0,0 +1,4 @@ +6,961,3216186123400,-;Console: switching to colour dummy device 80x25 +14,962,3216186123414,-;[IGT] no-subtests: executing +14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0 +6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings/2/err.txt b/runner/json_tests_data/warnings/2/err.txt new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/runner/json_tests_data/warnings/2/err.txt diff --git a/runner/json_tests_data/warnings/2/journal.txt b/runner/json_tests_data/warnings/2/journal.txt new file mode 100644 index 00000000..7151877f --- /dev/null +++ b/runner/json_tests_data/warnings/2/journal.txt @@ -0,0 +1 @@ +exit:0 (0.010s) diff --git a/runner/json_tests_data/warnings/2/out.txt b/runner/json_tests_data/warnings/2/out.txt new file mode 100644 index 00000000..695b67c2 --- /dev/null +++ b/runner/json_tests_data/warnings/2/out.txt @@ -0,0 +1,2 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +SUCCESS (0.000s) diff --git a/runner/json_tests_data/warnings/3/dmesg.txt b/runner/json_tests_data/warnings/3/dmesg.txt new file mode 100644 index 00000000..21e75031 --- /dev/null +++ b/runner/json_tests_data/warnings/3/dmesg.txt @@ -0,0 +1,4 @@ +6,965,3216186135188,-;Console: switching to colour dummy device 80x25 +14,966,3216186135212,-;[IGT] skippers: executing +14,967,3216186137075,-;[IGT] skippers: exiting, ret=77 +6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings/3/err.txt b/runner/json_tests_data/warnings/3/err.txt new file mode 100644 index 00000000..59b73d09 --- /dev/null +++ b/runner/json_tests_data/warnings/3/err.txt @@ -0,0 +1 @@ +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/warnings/3/journal.txt b/runner/json_tests_data/warnings/3/journal.txt new file mode 100644 index 00000000..afab8ba6 --- /dev/null +++ b/runner/json_tests_data/warnings/3/journal.txt @@ -0,0 +1,2 @@ +skip-one +exit:77 (0.011s) diff --git a/runner/json_tests_data/warnings/3/out.txt b/runner/json_tests_data/warnings/3/out.txt new file mode 100644 index 00000000..96284e78 --- /dev/null +++ b/runner/json_tests_data/warnings/3/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-one: SKIP diff --git a/runner/json_tests_data/warnings/4/dmesg.txt b/runner/json_tests_data/warnings/4/dmesg.txt new file mode 100644 index 00000000..737bc692 --- /dev/null +++ b/runner/json_tests_data/warnings/4/dmesg.txt @@ -0,0 +1,4 @@ +6,969,3216186145899,-;Console: switching to colour dummy device 80x25 +14,970,3216186145912,-;[IGT] skippers: executing +14,971,3216186147754,-;[IGT] skippers: exiting, ret=77 +6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75 diff --git a/runner/json_tests_data/warnings/4/err.txt b/runner/json_tests_data/warnings/4/err.txt new file mode 100644 index 00000000..2251da1e --- /dev/null +++ b/runner/json_tests_data/warnings/4/err.txt @@ -0,0 +1 @@ +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/warnings/4/journal.txt b/runner/json_tests_data/warnings/4/journal.txt new file mode 100644 index 00000000..a9dba132 --- /dev/null +++ b/runner/json_tests_data/warnings/4/journal.txt @@ -0,0 +1,2 @@ +skip-two +exit:77 (0.010s) diff --git a/runner/json_tests_data/warnings/4/out.txt b/runner/json_tests_data/warnings/4/out.txt new file mode 100644 index 00000000..2024db8f --- /dev/null +++ b/runner/json_tests_data/warnings/4/out.txt @@ -0,0 +1,6 @@ +IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64) +Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6: +Test requirement: false +Skipping from fixture +Last errno: 2, No such file or directory +Subtest skip-two: SKIP diff --git a/runner/json_tests_data/warnings/README.txt b/runner/json_tests_data/warnings/README.txt new file mode 100644 index 00000000..c776add0 --- /dev/null +++ b/runner/json_tests_data/warnings/README.txt @@ -0,0 +1,2 @@ +Any output in stderr should result in the result being 'warn' instead +of 'pass'. diff --git a/runner/json_tests_data/warnings/endtime.txt b/runner/json_tests_data/warnings/endtime.txt new file mode 100644 index 00000000..635f6ae9 --- /dev/null +++ b/runner/json_tests_data/warnings/endtime.txt @@ -0,0 +1 @@ +1539953735.172373 diff --git a/runner/json_tests_data/warnings/joblist.txt b/runner/json_tests_data/warnings/joblist.txt new file mode 100644 index 00000000..31ef8413 --- /dev/null +++ b/runner/json_tests_data/warnings/joblist.txt @@ -0,0 +1,5 @@ +successtest first-subtest +successtest second-subtest +no-subtests +skippers skip-one +skippers skip-two diff --git a/runner/json_tests_data/warnings/metadata.txt b/runner/json_tests_data/warnings/metadata.txt new file mode 100644 index 00000000..1316560d --- /dev/null +++ b/runner/json_tests_data/warnings/metadata.txt @@ -0,0 +1,12 @@ +abort_on_error : 0 +name : normal-run +dry_run : 0 +sync : 0 +log_level : 0 +overwrite : 0 +multiple_mode : 0 +inactivity_timeout : 0 +use_watchdog : 0 +piglit_style_dmesg : 0 +test_root : /path/does/not/exist +results_path : /path/does/not/exist diff --git a/runner/json_tests_data/warnings/reference.json b/runner/json_tests_data/warnings/reference.json new file mode 100644 index 00000000..8d5edbe1 --- /dev/null +++ b/runner/json_tests_data/warnings/reference.json @@ -0,0 +1,158 @@ +{ + "__type__":"TestrunResult", + "results_version":9, + "name":"normal-run", + "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64", + "time_elapsed":{ + "__type__":"TimeAttribute", + "start":1539953735.1110389, + "end":1539953735.1723731 + }, + "tests":{ + "igt@successtest@first-subtest":{ + "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"warn", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@successtest@second-subtest":{ + "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"pass", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n", + "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + }, + "result":"pass", + "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "err":"", + "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-one":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-one: SKIP\n", + "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n" + }, + "igt@skippers@skip-two":{ + "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n", + "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)", + "result":"skip", + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0 + }, + "err":"Subtest skip-two: SKIP\n", + "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n" + } + }, + "totals":{ + "":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":1 + }, + "root":{ + "crash":0, + "pass":2, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":1 + }, + "igt@successtest":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":1 + }, + "igt@no-subtests":{ + "crash":0, + "pass":1, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":0, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + }, + "igt@skippers":{ + "crash":0, + "pass":0, + "dmesg-fail":0, + "dmesg-warn":0, + "skip":2, + "incomplete":0, + "timeout":0, + "notrun":0, + "fail":0, + "warn":0 + } + }, + "runtimes":{ + "igt@successtest":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.027 + } + }, + "igt@no-subtests":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.01 + } + }, + "igt@skippers":{ + "time":{ + "__type__":"TimeAttribute", + "start":0, + "end":0.020999999999999998 + } + } + } +} diff --git a/runner/json_tests_data/warnings/starttime.txt b/runner/json_tests_data/warnings/starttime.txt new file mode 100644 index 00000000..ae038f18 --- /dev/null +++ b/runner/json_tests_data/warnings/starttime.txt @@ -0,0 +1 @@ +1539953735.111039 diff --git a/runner/json_tests_data/warnings/uname.txt b/runner/json_tests_data/warnings/uname.txt new file mode 100644 index 00000000..a7aef6f7 --- /dev/null +++ b/runner/json_tests_data/warnings/uname.txt @@ -0,0 +1 @@ +Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64 diff --git a/runner/meson.build b/runner/meson.build index 6c4b4862..de6e6f1c 100644 --- a/runner/meson.build +++ b/runner/meson.build @@ -10,6 +10,7 @@ runner_sources = [ 'runner.c' ] resume_sources = [ 'resume.c' ] results_sources = [ 'results.c' ] runner_test_sources = [ 'runner_tests.c' ] +runner_json_test_sources = [ 'runner_json_tests.c' ] if _build_runner and jsonc.found() subdir('testdata') @@ -46,6 +47,13 @@ if _build_runner and jsonc.found() dependencies : igt_deps) test('runner', runner_test) + runner_json_test = executable('runner_json_test', runner_json_test_sources, + c_args : '-DJSON_TESTS_DIRECTORY="@0@"'.format(join_paths(meson.current_source_dir(), 'json_tests_data')), + link_with : runnerlib, + install : false, + dependencies : [igt_deps, jsonc]) + test('runner_json', runner_json_test) + build_info += 'Build test runner: Yes' else build_info += 'Build test runner: No' diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c new file mode 100644 index 00000000..758700d4 --- /dev/null +++ b/runner/runner_json_tests.c @@ -0,0 +1,171 @@ +#include <sys/types.h> +#include <sys/stat.h> +#include <fcntl.h> + +#include <json.h> + +#include "igt.h" +#include "resultgen.h" + +static char testdatadir[] = JSON_TESTS_DIRECTORY; + +static struct json_object *read_json(int fd) +{ + struct json_object *obj; + struct json_tokener *tok = json_tokener_new(); + enum json_tokener_error err; + char buf[512]; + ssize_t s; + + do { + s = read(fd, buf, sizeof(buf)); + obj = json_tokener_parse_ex(tok, buf, s); + } while ((err = json_tokener_get_error(tok)) == json_tokener_continue); + + igt_assert_eq(err, json_tokener_success); + + json_tokener_free(tok); + return obj; +} + +static void compare(struct json_object *one, + struct json_object *two); + +static void compare_objects(struct json_object *one, + struct json_object *two) +{ + json_object_iter iter; + struct json_object *subobj; + + json_object_object_foreachC(one, iter) { + igt_debug("Key %s\n", iter.key); + + igt_assert(json_object_object_get_ex(two, iter.key, &subobj)); + + compare(iter.val, subobj); + } +} + +static void compare_arrays(struct json_object *one, + struct json_object *two) +{ + size_t i; + + for (i = 0; i < json_object_array_length(one); i++) { + igt_debug("Array index %zd\n", i); + compare(json_object_array_get_idx(one, i), + json_object_array_get_idx(two, i)); + } +} + +static bool compatible_types(struct json_object *one, + struct json_object *two) +{ + /* + * A double of value 0.0 gets written as "0", which gets read + * as an int. + */ + json_type onetype = json_object_get_type(one); + json_type twotype = json_object_get_type(two); + + switch (onetype) { + case json_type_boolean: + case json_type_string: + case json_type_object: + case json_type_array: + case json_type_null: + return onetype == twotype; + break; + case json_type_double: + case json_type_int: + return twotype == json_type_double || twotype == json_type_int; + break; + } + + igt_assert(!"Cannot be reached"); + return false; +} + +static void compare(struct json_object *one, + struct json_object *two) +{ + igt_assert(compatible_types(one, two)); + + switch (json_object_get_type(one)) { + case json_type_boolean: + igt_assert_eq(json_object_get_boolean(one), json_object_get_boolean(two)); + break; + case json_type_double: + case json_type_int: + /* + * A double of value 0.0 gets written as "0", which + * gets read as an int. Both yield 0.0 with + * json_object_get_double(). Comparing doubles with == + * considered crazy but it's good enough. + */ + igt_assert(json_object_get_double(one) == json_object_get_double(two)); + break; + case json_type_string: + igt_assert(!strcmp(json_object_get_string(one), json_object_get_string(two))); + break; + case json_type_object: + igt_assert_eq(json_object_object_length(one), json_object_object_length(two)); + compare_objects(one, two); + break; + case json_type_array: + igt_assert_eq(json_object_array_length(one), json_object_array_length(two)); + compare_arrays(one, two); + break; + case json_type_null: + break; + default: + igt_assert(!"Cannot be reached"); + } +} + +static void run_results_and_compare(int dirfd, const char *dirname) +{ + int testdirfd = openat(dirfd, dirname, O_RDONLY | O_DIRECTORY); + int reference; + struct json_object *resultsobj, *referenceobj; + + igt_assert_fd(testdirfd); + + igt_assert((resultsobj = generate_results_json(testdirfd)) != NULL); + + reference = openat(testdirfd, "reference.json", O_RDONLY); + close(testdirfd); + + igt_assert_fd(reference); + referenceobj = read_json(reference); + close(reference); + igt_assert(referenceobj != NULL); + + igt_debug("Root object\n"); + compare(resultsobj, referenceobj); + igt_assert_eq(json_object_put(resultsobj), 1); + igt_assert_eq(json_object_put(referenceobj), 1); +} + +static const char *dirnames[] = { + "normal-run", + "warnings", + "warnings-with-dmesg-warns", + "piglit-style-dmesg", + "incomplete-before-any-subtests", + "dmesg-results", +}; + +igt_main +{ + int dirfd = open(testdatadir, O_RDONLY | O_DIRECTORY); + size_t i; + + igt_assert_fd(dirfd); + + for (i = 0; i < ARRAY_SIZE(dirnames); i++) { + igt_subtest(dirnames[i]) { + run_results_and_compare(dirfd, dirnames[i]); + } + } +} |