summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorPetri Latvala <petri.latvala@intel.com>2018-10-23 15:20:24 +0300
committerPetri Latvala <petri.latvala@intel.com>2018-11-06 11:52:56 +0200
commite2d7d22ae7e69d30272f366034977ed5165911dd (patch)
tree811c1e8561c7873f5c45d652cb341ed07e9d83df
parent6d19a3b4bddb4d8716cf6b763ee784ba9bc9f440 (diff)
runner: Unit tests for json generation
Test the results.json generation with a top-down approach: With a directory of test run intermediary logs, check that the resulting json would match a reference json file. Signed-off-by: Petri Latvala <petri.latvala@intel.com> Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com> Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
-rw-r--r--runner/.gitignore1
-rw-r--r--runner/Makefile.am14
-rw-r--r--runner/json_tests_data/README.txt10
-rw-r--r--runner/json_tests_data/dmesg-results/0/dmesg.txt6
-rw-r--r--runner/json_tests_data/dmesg-results/0/err.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/0/journal.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/0/out.txt3
-rw-r--r--runner/json_tests_data/dmesg-results/1/dmesg.txt6
-rw-r--r--runner/json_tests_data/dmesg-results/1/err.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/1/journal.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/1/out.txt3
-rw-r--r--runner/json_tests_data/dmesg-results/2/dmesg.txt5
-rw-r--r--runner/json_tests_data/dmesg-results/2/err.txt0
-rw-r--r--runner/json_tests_data/dmesg-results/2/journal.txt1
-rw-r--r--runner/json_tests_data/dmesg-results/2/out.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/3/dmesg.txt5
-rw-r--r--runner/json_tests_data/dmesg-results/3/err.txt1
-rw-r--r--runner/json_tests_data/dmesg-results/3/journal.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/3/out.txt6
-rw-r--r--runner/json_tests_data/dmesg-results/4/dmesg.txt5
-rw-r--r--runner/json_tests_data/dmesg-results/4/err.txt1
-rw-r--r--runner/json_tests_data/dmesg-results/4/journal.txt2
-rw-r--r--runner/json_tests_data/dmesg-results/4/out.txt6
-rw-r--r--runner/json_tests_data/dmesg-results/README.txt3
-rw-r--r--runner/json_tests_data/dmesg-results/endtime.txt1
-rw-r--r--runner/json_tests_data/dmesg-results/joblist.txt5
-rw-r--r--runner/json_tests_data/dmesg-results/metadata.txt12
-rw-r--r--runner/json_tests_data/dmesg-results/reference.json161
-rw-r--r--runner/json_tests_data/dmesg-results/starttime.txt1
-rw-r--r--runner/json_tests_data/dmesg-results/uname.txt1
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt2
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/0/err.txt0
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt0
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/0/out.txt1
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/README.txt4
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/endtime.txt1
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/joblist.txt5
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/metadata.txt12
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/reference.json60
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/starttime.txt1
-rw-r--r--runner/json_tests_data/incomplete-before-any-subtests/uname.txt1
-rw-r--r--runner/json_tests_data/normal-run/0/dmesg.txt5
-rw-r--r--runner/json_tests_data/normal-run/0/err.txt2
-rw-r--r--runner/json_tests_data/normal-run/0/journal.txt2
-rw-r--r--runner/json_tests_data/normal-run/0/out.txt3
-rw-r--r--runner/json_tests_data/normal-run/1/dmesg.txt5
-rw-r--r--runner/json_tests_data/normal-run/1/err.txt2
-rw-r--r--runner/json_tests_data/normal-run/1/journal.txt2
-rw-r--r--runner/json_tests_data/normal-run/1/out.txt3
-rw-r--r--runner/json_tests_data/normal-run/2/dmesg.txt4
-rw-r--r--runner/json_tests_data/normal-run/2/err.txt0
-rw-r--r--runner/json_tests_data/normal-run/2/journal.txt1
-rw-r--r--runner/json_tests_data/normal-run/2/out.txt2
-rw-r--r--runner/json_tests_data/normal-run/3/dmesg.txt4
-rw-r--r--runner/json_tests_data/normal-run/3/err.txt1
-rw-r--r--runner/json_tests_data/normal-run/3/journal.txt2
-rw-r--r--runner/json_tests_data/normal-run/3/out.txt6
-rw-r--r--runner/json_tests_data/normal-run/4/dmesg.txt4
-rw-r--r--runner/json_tests_data/normal-run/4/err.txt1
-rw-r--r--runner/json_tests_data/normal-run/4/journal.txt2
-rw-r--r--runner/json_tests_data/normal-run/4/out.txt6
-rw-r--r--runner/json_tests_data/normal-run/README.txt1
-rw-r--r--runner/json_tests_data/normal-run/endtime.txt1
-rw-r--r--runner/json_tests_data/normal-run/joblist.txt5
-rw-r--r--runner/json_tests_data/normal-run/metadata.txt12
-rw-r--r--runner/json_tests_data/normal-run/reference.json158
-rw-r--r--runner/json_tests_data/normal-run/starttime.txt1
-rw-r--r--runner/json_tests_data/normal-run/uname.txt1
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt6
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/0/err.txt3
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/0/journal.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/0/out.txt3
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt5
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/1/err.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/1/journal.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/1/out.txt3
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt4
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/2/err.txt0
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/2/journal.txt1
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/2/out.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt4
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/3/err.txt1
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/3/journal.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/3/out.txt6
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt4
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/4/err.txt1
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/4/journal.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/4/out.txt6
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/README.txt2
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/endtime.txt1
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/joblist.txt5
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/metadata.txt12
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/reference.json158
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/starttime.txt1
-rw-r--r--runner/json_tests_data/piglit-style-dmesg/uname.txt1
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt6
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt3
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt2
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt3
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt5
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt2
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt2
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt3
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt4
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt0
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt1
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt2
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt4
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt1
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt2
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt6
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt4
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt1
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt2
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt6
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/README.txt3
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt1
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt5
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt12
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/reference.json159
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt1
-rw-r--r--runner/json_tests_data/warnings-with-dmesg-warns/uname.txt1
-rw-r--r--runner/json_tests_data/warnings/0/dmesg.txt5
-rw-r--r--runner/json_tests_data/warnings/0/err.txt3
-rw-r--r--runner/json_tests_data/warnings/0/journal.txt2
-rw-r--r--runner/json_tests_data/warnings/0/out.txt3
-rw-r--r--runner/json_tests_data/warnings/1/dmesg.txt5
-rw-r--r--runner/json_tests_data/warnings/1/err.txt2
-rw-r--r--runner/json_tests_data/warnings/1/journal.txt2
-rw-r--r--runner/json_tests_data/warnings/1/out.txt3
-rw-r--r--runner/json_tests_data/warnings/2/dmesg.txt4
-rw-r--r--runner/json_tests_data/warnings/2/err.txt0
-rw-r--r--runner/json_tests_data/warnings/2/journal.txt1
-rw-r--r--runner/json_tests_data/warnings/2/out.txt2
-rw-r--r--runner/json_tests_data/warnings/3/dmesg.txt4
-rw-r--r--runner/json_tests_data/warnings/3/err.txt1
-rw-r--r--runner/json_tests_data/warnings/3/journal.txt2
-rw-r--r--runner/json_tests_data/warnings/3/out.txt6
-rw-r--r--runner/json_tests_data/warnings/4/dmesg.txt4
-rw-r--r--runner/json_tests_data/warnings/4/err.txt1
-rw-r--r--runner/json_tests_data/warnings/4/journal.txt2
-rw-r--r--runner/json_tests_data/warnings/4/out.txt6
-rw-r--r--runner/json_tests_data/warnings/README.txt2
-rw-r--r--runner/json_tests_data/warnings/endtime.txt1
-rw-r--r--runner/json_tests_data/warnings/joblist.txt5
-rw-r--r--runner/json_tests_data/warnings/metadata.txt12
-rw-r--r--runner/json_tests_data/warnings/reference.json158
-rw-r--r--runner/json_tests_data/warnings/starttime.txt1
-rw-r--r--runner/json_tests_data/warnings/uname.txt1
-rw-r--r--runner/meson.build8
-rw-r--r--runner/runner_json_tests.c171
151 files changed, 1489 insertions, 2 deletions
diff --git a/runner/.gitignore b/runner/.gitignore
index c5030e66..3812da23 100644
--- a/runner/.gitignore
+++ b/runner/.gitignore
@@ -2,3 +2,4 @@ igt_runner
igt_resume
igt_results
runner_test
+runner_json_test
diff --git a/runner/Makefile.am b/runner/Makefile.am
index ddcbf86a..b55cf797 100644
--- a/runner/Makefile.am
+++ b/runner/Makefile.am
@@ -30,8 +30,9 @@ AM_CFLAGS = $(JSONC_CFLAGS) \
-I$(srcdir)/../lib \
-D_GNU_SOURCE
-TESTS = runner_test
-check_PROGRAMS = runner_test
+TESTS = runner_test runner_json_test
+check_PROGRAMS = runner_test runner_json_test
+
runner_test_SOURCES = runner_tests.c
runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \
-I$(top_srcdir)/include/drm-uapi \
@@ -40,4 +41,13 @@ runner_test_CFLAGS = -DTESTDATA_DIRECTORY=\"$(abs_builddir)/testdata\" \
-I$(srcdir)/../lib \
-D_GNU_SOURCE
+runner_json_test_SOURCES = runner_json_tests.c
+runner_json_test_CFLAGS = -DJSON_TESTS_DIRECTORY=\"$(abs_builddir)/json_tests_data\" \
+ $(JSONC_CFLAGS) \
+ -I$(top_srcdir)/include/drm-uapi \
+ $(DRM_CFLAGS) $(CAIRO_CFLAGS) $(LIBUDEV_CFLAGS) \
+ -I$(srcdir)/.. \
+ -I$(srcdir)/../lib \
+ -D_GNU_SOURCE
+
endif
diff --git a/runner/json_tests_data/README.txt b/runner/json_tests_data/README.txt
new file mode 100644
index 00000000..206d7630
--- /dev/null
+++ b/runner/json_tests_data/README.txt
@@ -0,0 +1,10 @@
+This directory is for testing igt_runner's results.json creation. Each
+subdirectory is a run-result directory, referred to from
+runner_json_tests.c, with test run intermediary files without any
+results.json. A semi-automatically crafted reference.json file in each
+respective directory is compared with igt_runner's generated json and
+needs to match semantically; indentation and such doesn't matter, data
+has to be equal.
+
+Each directory also contains a README file explaining what the
+particular directory aims to test.
diff --git a/runner/json_tests_data/dmesg-results/0/dmesg.txt b/runner/json_tests_data/dmesg-results/0/dmesg.txt
new file mode 100644
index 00000000..e50f75db
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/dmesg.txt
@@ -0,0 +1,6 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+3,954,3216186101159,-;Warning from kernel
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/0/err.txt b/runner/json_tests_data/dmesg-results/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/0/journal.txt b/runner/json_tests_data/dmesg-results/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/dmesg-results/0/out.txt b/runner/json_tests_data/dmesg-results/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/1/dmesg.txt b/runner/json_tests_data/dmesg-results/1/dmesg.txt
new file mode 100644
index 00000000..6c816cf2
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/dmesg.txt
@@ -0,0 +1,6 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+3,954,3216186101159,-;Warning from kernel
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/1/err.txt b/runner/json_tests_data/dmesg-results/1/err.txt
new file mode 100644
index 00000000..0f64562f
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/1/journal.txt b/runner/json_tests_data/dmesg-results/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/dmesg-results/1/out.txt b/runner/json_tests_data/dmesg-results/1/out.txt
new file mode 100644
index 00000000..d2778f3c
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/2/dmesg.txt b/runner/json_tests_data/dmesg-results/2/dmesg.txt
new file mode 100644
index 00000000..913979a8
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/dmesg.txt
@@ -0,0 +1,5 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+3,954,3216186101159,-;Warning from kernel
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/2/err.txt b/runner/json_tests_data/dmesg-results/2/err.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/err.txt
diff --git a/runner/json_tests_data/dmesg-results/2/journal.txt b/runner/json_tests_data/dmesg-results/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/dmesg-results/2/out.txt b/runner/json_tests_data/dmesg-results/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/dmesg-results/3/dmesg.txt b/runner/json_tests_data/dmesg-results/3/dmesg.txt
new file mode 100644
index 00000000..a61e83e8
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/dmesg.txt
@@ -0,0 +1,5 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+3,954,3216186101159,-;Warning from kernel
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/3/err.txt b/runner/json_tests_data/dmesg-results/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/dmesg-results/3/journal.txt b/runner/json_tests_data/dmesg-results/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/dmesg-results/3/out.txt b/runner/json_tests_data/dmesg-results/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/dmesg-results/4/dmesg.txt b/runner/json_tests_data/dmesg-results/4/dmesg.txt
new file mode 100644
index 00000000..fcb43d57
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/dmesg.txt
@@ -0,0 +1,5 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+3,954,3216186101159,-;Warning from kernel
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/dmesg-results/4/err.txt b/runner/json_tests_data/dmesg-results/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/dmesg-results/4/journal.txt b/runner/json_tests_data/dmesg-results/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/dmesg-results/4/out.txt b/runner/json_tests_data/dmesg-results/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/dmesg-results/README.txt b/runner/json_tests_data/dmesg-results/README.txt
new file mode 100644
index 00000000..d928a526
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/README.txt
@@ -0,0 +1,3 @@
+Certain types of kernel log messages cause the result to change from
+'pass' to 'dmesg-warn', and from 'fail' to 'dmesg-fail'. 'skip' should
+stay 'skip'.
diff --git a/runner/json_tests_data/dmesg-results/endtime.txt b/runner/json_tests_data/dmesg-results/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/dmesg-results/joblist.txt b/runner/json_tests_data/dmesg-results/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/dmesg-results/metadata.txt b/runner/json_tests_data/dmesg-results/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json
new file mode 100644
index 00000000..811b63eb
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/reference.json
@@ -0,0 +1,161 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt@successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"dmesg-warn",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n",
+ "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+ },
+ "igt@successtest@second-subtest":{
+ "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"dmesg-fail",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+ "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n",
+ "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ },
+ "result":"dmesg-warn",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "err":"",
+ "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n",
+ "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+ },
+ "igt@skippers@skip-one":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-one: SKIP\n",
+ "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-two":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-two: SKIP\n",
+ "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":1,
+ "dmesg-warn":2,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":1,
+ "dmesg-warn":2,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@successtest":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":1,
+ "dmesg-warn":1,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@no-subtests":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":1,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt@successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.027
+ }
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ }
+ },
+ "igt@skippers":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.020999999999999998
+ }
+ }
+ }
+}
diff --git a/runner/json_tests_data/dmesg-results/starttime.txt b/runner/json_tests_data/dmesg-results/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/dmesg-results/uname.txt b/runner/json_tests_data/dmesg-results/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/dmesg-results/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt
new file mode 100644
index 00000000..7c6834d5
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/0/dmesg.txt
@@ -0,0 +1,2 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/0/err.txt
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/0/journal.txt
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt b/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt
new file mode 100644
index 00000000..ebaa6300
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/0/out.txt
@@ -0,0 +1 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/README.txt b/runner/json_tests_data/incomplete-before-any-subtests/README.txt
new file mode 100644
index 00000000..fb64f581
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/README.txt
@@ -0,0 +1,4 @@
+When a test hangs before it even enters a subtest, we don't have a
+journal entry for that subtest. This test checks that the result still
+gets attributed to a subtest, since we know from the job_list that a
+subtest should be executed.
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt b/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt b/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/reference.json b/runner/json_tests_data/incomplete-before-any-subtests/reference.json
new file mode 100644
index 00000000..3eb6c9c1
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/reference.json
@@ -0,0 +1,60 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt@successtest@first-subtest":{
+ "out":"",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"incomplete",
+ "err":"",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":1,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":1,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@successtest":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":1,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ },
+ "runtimes":{
+ }
+}
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt b/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/incomplete-before-any-subtests/uname.txt b/runner/json_tests_data/incomplete-before-any-subtests/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/incomplete-before-any-subtests/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/normal-run/0/dmesg.txt b/runner/json_tests_data/normal-run/0/dmesg.txt
new file mode 100644
index 00000000..a189e704
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,954,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/0/err.txt b/runner/json_tests_data/normal-run/0/err.txt
new file mode 100644
index 00000000..5dc78057
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/normal-run/0/journal.txt b/runner/json_tests_data/normal-run/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/normal-run/0/out.txt b/runner/json_tests_data/normal-run/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/normal-run/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/normal-run/1/dmesg.txt b/runner/json_tests_data/normal-run/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/1/err.txt b/runner/json_tests_data/normal-run/1/err.txt
new file mode 100644
index 00000000..0f64562f
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/normal-run/1/journal.txt b/runner/json_tests_data/normal-run/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/normal-run/1/out.txt b/runner/json_tests_data/normal-run/1/out.txt
new file mode 100644
index 00000000..d2778f3c
--- /dev/null
+++ b/runner/json_tests_data/normal-run/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: FAIL (0.000s)
diff --git a/runner/json_tests_data/normal-run/2/dmesg.txt b/runner/json_tests_data/normal-run/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/2/err.txt b/runner/json_tests_data/normal-run/2/err.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/err.txt
diff --git a/runner/json_tests_data/normal-run/2/journal.txt b/runner/json_tests_data/normal-run/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/normal-run/2/out.txt b/runner/json_tests_data/normal-run/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/normal-run/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/normal-run/3/dmesg.txt b/runner/json_tests_data/normal-run/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/3/err.txt b/runner/json_tests_data/normal-run/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/normal-run/3/journal.txt b/runner/json_tests_data/normal-run/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/normal-run/3/out.txt b/runner/json_tests_data/normal-run/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/normal-run/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/normal-run/4/dmesg.txt b/runner/json_tests_data/normal-run/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/normal-run/4/err.txt b/runner/json_tests_data/normal-run/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/normal-run/4/journal.txt b/runner/json_tests_data/normal-run/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/normal-run/4/out.txt b/runner/json_tests_data/normal-run/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/normal-run/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/normal-run/README.txt b/runner/json_tests_data/normal-run/README.txt
new file mode 100644
index 00000000..64270e88
--- /dev/null
+++ b/runner/json_tests_data/normal-run/README.txt
@@ -0,0 +1 @@
+A normal test run with pass, fail and skip results.
diff --git a/runner/json_tests_data/normal-run/endtime.txt b/runner/json_tests_data/normal-run/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/normal-run/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/normal-run/joblist.txt b/runner/json_tests_data/normal-run/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/normal-run/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/normal-run/metadata.txt b/runner/json_tests_data/normal-run/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/normal-run/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/normal-run/reference.json b/runner/json_tests_data/normal-run/reference.json
new file mode 100644
index 00000000..0d28b2c2
--- /dev/null
+++ b/runner/json_tests_data/normal-run/reference.json
@@ -0,0 +1,158 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt@successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@successtest@second-subtest":{
+ "out":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"fail",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: second-subtest\nSubtest second-subtest: FAIL (0.000s)\n",
+ "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ },
+ "result":"pass",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "err":"",
+ "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-one":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-one: SKIP\n",
+ "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-two":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-two: SKIP\n",
+ "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":1,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":1,
+ "warn":0
+ },
+ "igt@successtest":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":1,
+ "warn":0
+ },
+ "igt@no-subtests":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt@successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.027
+ }
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ }
+ },
+ "igt@skippers":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.020999999999999998
+ }
+ }
+ }
+}
diff --git a/runner/json_tests_data/normal-run/starttime.txt b/runner/json_tests_data/normal-run/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/normal-run/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/normal-run/uname.txt b/runner/json_tests_data/normal-run/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/normal-run/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt
new file mode 100644
index 00000000..e50f75db
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/dmesg.txt
@@ -0,0 +1,6 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+3,954,3216186101159,-;Warning from kernel
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/err.txt b/runner/json_tests_data/piglit-style-dmesg/0/err.txt
new file mode 100644
index 00000000..e18c00e9
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/err.txt
@@ -0,0 +1,3 @@
+Starting subtest: first-subtest
+This is a warning line
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/journal.txt b/runner/json_tests_data/piglit-style-dmesg/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/0/out.txt b/runner/json_tests_data/piglit-style-dmesg/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/err.txt b/runner/json_tests_data/piglit-style-dmesg/1/err.txt
new file mode 100644
index 00000000..57be9f7b
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/journal.txt b/runner/json_tests_data/piglit-style-dmesg/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/1/out.txt b/runner/json_tests_data/piglit-style-dmesg/1/out.txt
new file mode 100644
index 00000000..24b37244
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/err.txt b/runner/json_tests_data/piglit-style-dmesg/2/err.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/err.txt
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/journal.txt b/runner/json_tests_data/piglit-style-dmesg/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/2/out.txt b/runner/json_tests_data/piglit-style-dmesg/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/err.txt b/runner/json_tests_data/piglit-style-dmesg/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/journal.txt b/runner/json_tests_data/piglit-style-dmesg/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/3/out.txt b/runner/json_tests_data/piglit-style-dmesg/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt b/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/err.txt b/runner/json_tests_data/piglit-style-dmesg/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/journal.txt b/runner/json_tests_data/piglit-style-dmesg/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/piglit-style-dmesg/4/out.txt b/runner/json_tests_data/piglit-style-dmesg/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/piglit-style-dmesg/README.txt b/runner/json_tests_data/piglit-style-dmesg/README.txt
new file mode 100644
index 00000000..3130337b
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/README.txt
@@ -0,0 +1,2 @@
+When --piglit-style-dmesg is used, only a particular pattern of kernel
+log messages trigger a result change to dmesg-warn/dmesg-fail.
diff --git a/runner/json_tests_data/piglit-style-dmesg/endtime.txt b/runner/json_tests_data/piglit-style-dmesg/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/piglit-style-dmesg/joblist.txt b/runner/json_tests_data/piglit-style-dmesg/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/piglit-style-dmesg/metadata.txt b/runner/json_tests_data/piglit-style-dmesg/metadata.txt
new file mode 100644
index 00000000..7f1372c1
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 1
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/piglit-style-dmesg/reference.json b/runner/json_tests_data/piglit-style-dmesg/reference.json
new file mode 100644
index 00000000..2257b3c7
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/reference.json
@@ -0,0 +1,158 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt@successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"warn",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@successtest@second-subtest":{
+ "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ },
+ "result":"pass",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "err":"",
+ "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-one":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-one: SKIP\n",
+ "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-two":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-two: SKIP\n",
+ "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":1
+ },
+ "root":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":1
+ },
+ "igt@successtest":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":1
+ },
+ "igt@no-subtests":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt@successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.027
+ }
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ }
+ },
+ "igt@skippers":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.020999999999999998
+ }
+ }
+ }
+}
diff --git a/runner/json_tests_data/piglit-style-dmesg/starttime.txt b/runner/json_tests_data/piglit-style-dmesg/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/piglit-style-dmesg/uname.txt b/runner/json_tests_data/piglit-style-dmesg/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/piglit-style-dmesg/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt
new file mode 100644
index 00000000..e50f75db
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/dmesg.txt
@@ -0,0 +1,6 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+3,954,3216186101159,-;Warning from kernel
+14,955,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,956,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt
new file mode 100644
index 00000000..e18c00e9
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/err.txt
@@ -0,0 +1,3 @@
+Starting subtest: first-subtest
+This is a warning line
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt
new file mode 100644
index 00000000..57be9f7b
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt
new file mode 100644
index 00000000..24b37244
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/err.txt
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt b/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/README.txt b/runner/json_tests_data/warnings-with-dmesg-warns/README.txt
new file mode 100644
index 00000000..7ef9ee73
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/README.txt
@@ -0,0 +1,3 @@
+A test that has output in stderr should result in 'warn' instead of
+'pass'. But if there is a message in the kernel logs that would make
+the result 'dmesg-warn' instead, that takes priority over 'warn'.
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt b/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt b/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt b/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
new file mode 100644
index 00000000..9e027efb
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json
@@ -0,0 +1,159 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt@successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"dmesg-warn",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<3> [3216186.101159] Warning from kernel\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n",
+ "dmesg-warnings":"<3> [3216186.101159] Warning from kernel\n"
+ },
+ "igt@successtest@second-subtest":{
+ "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ },
+ "result":"pass",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "err":"",
+ "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-one":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-one: SKIP\n",
+ "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-two":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-two: SKIP\n",
+ "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":1,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "root":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":1,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@successtest":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":1,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@no-subtests":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt@successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.027
+ }
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ }
+ },
+ "igt@skippers":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.020999999999999998
+ }
+ }
+ }
+}
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt b/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt b/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/warnings-with-dmesg-warns/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/json_tests_data/warnings/0/dmesg.txt b/runner/json_tests_data/warnings/0/dmesg.txt
new file mode 100644
index 00000000..a189e704
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/dmesg.txt
@@ -0,0 +1,5 @@
+6,951,3216186095083,-;Console: switching to colour dummy device 80x25
+14,952,3216186095097,-;[IGT] successtest: executing
+14,953,3216186101115,-;[IGT] successtest: starting subtest first-subtest
+14,954,3216186101160,-;[IGT] successtest: exiting, ret=0
+6,955,3216186101299,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/0/err.txt b/runner/json_tests_data/warnings/0/err.txt
new file mode 100644
index 00000000..e18c00e9
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/err.txt
@@ -0,0 +1,3 @@
+Starting subtest: first-subtest
+This is a warning line
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/0/journal.txt b/runner/json_tests_data/warnings/0/journal.txt
new file mode 100644
index 00000000..86a30e07
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/journal.txt
@@ -0,0 +1,2 @@
+first-subtest
+exit:0 (0.014s)
diff --git a/runner/json_tests_data/warnings/0/out.txt b/runner/json_tests_data/warnings/0/out.txt
new file mode 100644
index 00000000..5946bf31
--- /dev/null
+++ b/runner/json_tests_data/warnings/0/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: first-subtest
+Subtest first-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/1/dmesg.txt b/runner/json_tests_data/warnings/1/dmesg.txt
new file mode 100644
index 00000000..59b7cee3
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/dmesg.txt
@@ -0,0 +1,5 @@
+6,956,3216186111837,-;Console: switching to colour dummy device 80x25
+14,957,3216186111851,-;[IGT] successtest: executing
+14,958,3216186114762,-;[IGT] successtest: starting subtest second-subtest
+14,959,3216186114814,-;[IGT] successtest: exiting, ret=0
+6,960,3216186114933,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/1/err.txt b/runner/json_tests_data/warnings/1/err.txt
new file mode 100644
index 00000000..57be9f7b
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/err.txt
@@ -0,0 +1,2 @@
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/1/journal.txt b/runner/json_tests_data/warnings/1/journal.txt
new file mode 100644
index 00000000..99f57815
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/journal.txt
@@ -0,0 +1,2 @@
+second-subtest
+exit:0 (0.013s)
diff --git a/runner/json_tests_data/warnings/1/out.txt b/runner/json_tests_data/warnings/1/out.txt
new file mode 100644
index 00000000..24b37244
--- /dev/null
+++ b/runner/json_tests_data/warnings/1/out.txt
@@ -0,0 +1,3 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Starting subtest: second-subtest
+Subtest second-subtest: SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/2/dmesg.txt b/runner/json_tests_data/warnings/2/dmesg.txt
new file mode 100644
index 00000000..998b4797
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/dmesg.txt
@@ -0,0 +1,4 @@
+6,961,3216186123400,-;Console: switching to colour dummy device 80x25
+14,962,3216186123414,-;[IGT] no-subtests: executing
+14,963,3216186125204,-;[IGT] no-subtests: exiting, ret=0
+6,964,3216186125374,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/2/err.txt b/runner/json_tests_data/warnings/2/err.txt
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/err.txt
diff --git a/runner/json_tests_data/warnings/2/journal.txt b/runner/json_tests_data/warnings/2/journal.txt
new file mode 100644
index 00000000..7151877f
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/journal.txt
@@ -0,0 +1 @@
+exit:0 (0.010s)
diff --git a/runner/json_tests_data/warnings/2/out.txt b/runner/json_tests_data/warnings/2/out.txt
new file mode 100644
index 00000000..695b67c2
--- /dev/null
+++ b/runner/json_tests_data/warnings/2/out.txt
@@ -0,0 +1,2 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+SUCCESS (0.000s)
diff --git a/runner/json_tests_data/warnings/3/dmesg.txt b/runner/json_tests_data/warnings/3/dmesg.txt
new file mode 100644
index 00000000..21e75031
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/dmesg.txt
@@ -0,0 +1,4 @@
+6,965,3216186135188,-;Console: switching to colour dummy device 80x25
+14,966,3216186135212,-;[IGT] skippers: executing
+14,967,3216186137075,-;[IGT] skippers: exiting, ret=77
+6,968,3216186137206,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/3/err.txt b/runner/json_tests_data/warnings/3/err.txt
new file mode 100644
index 00000000..59b73d09
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/err.txt
@@ -0,0 +1 @@
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings/3/journal.txt b/runner/json_tests_data/warnings/3/journal.txt
new file mode 100644
index 00000000..afab8ba6
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/journal.txt
@@ -0,0 +1,2 @@
+skip-one
+exit:77 (0.011s)
diff --git a/runner/json_tests_data/warnings/3/out.txt b/runner/json_tests_data/warnings/3/out.txt
new file mode 100644
index 00000000..96284e78
--- /dev/null
+++ b/runner/json_tests_data/warnings/3/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-one: SKIP
diff --git a/runner/json_tests_data/warnings/4/dmesg.txt b/runner/json_tests_data/warnings/4/dmesg.txt
new file mode 100644
index 00000000..737bc692
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/dmesg.txt
@@ -0,0 +1,4 @@
+6,969,3216186145899,-;Console: switching to colour dummy device 80x25
+14,970,3216186145912,-;[IGT] skippers: executing
+14,971,3216186147754,-;[IGT] skippers: exiting, ret=77
+6,972,3216186147894,-;Console: switching to colour frame buffer device 240x75
diff --git a/runner/json_tests_data/warnings/4/err.txt b/runner/json_tests_data/warnings/4/err.txt
new file mode 100644
index 00000000..2251da1e
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/err.txt
@@ -0,0 +1 @@
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings/4/journal.txt b/runner/json_tests_data/warnings/4/journal.txt
new file mode 100644
index 00000000..a9dba132
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/journal.txt
@@ -0,0 +1,2 @@
+skip-two
+exit:77 (0.010s)
diff --git a/runner/json_tests_data/warnings/4/out.txt b/runner/json_tests_data/warnings/4/out.txt
new file mode 100644
index 00000000..2024db8f
--- /dev/null
+++ b/runner/json_tests_data/warnings/4/out.txt
@@ -0,0 +1,6 @@
+IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)
+Test requirement not met in function __real_main3, file ../runner/testdata/skippers.c:6:
+Test requirement: false
+Skipping from fixture
+Last errno: 2, No such file or directory
+Subtest skip-two: SKIP
diff --git a/runner/json_tests_data/warnings/README.txt b/runner/json_tests_data/warnings/README.txt
new file mode 100644
index 00000000..c776add0
--- /dev/null
+++ b/runner/json_tests_data/warnings/README.txt
@@ -0,0 +1,2 @@
+Any output in stderr should result in the result being 'warn' instead
+of 'pass'.
diff --git a/runner/json_tests_data/warnings/endtime.txt b/runner/json_tests_data/warnings/endtime.txt
new file mode 100644
index 00000000..635f6ae9
--- /dev/null
+++ b/runner/json_tests_data/warnings/endtime.txt
@@ -0,0 +1 @@
+1539953735.172373
diff --git a/runner/json_tests_data/warnings/joblist.txt b/runner/json_tests_data/warnings/joblist.txt
new file mode 100644
index 00000000..31ef8413
--- /dev/null
+++ b/runner/json_tests_data/warnings/joblist.txt
@@ -0,0 +1,5 @@
+successtest first-subtest
+successtest second-subtest
+no-subtests
+skippers skip-one
+skippers skip-two
diff --git a/runner/json_tests_data/warnings/metadata.txt b/runner/json_tests_data/warnings/metadata.txt
new file mode 100644
index 00000000..1316560d
--- /dev/null
+++ b/runner/json_tests_data/warnings/metadata.txt
@@ -0,0 +1,12 @@
+abort_on_error : 0
+name : normal-run
+dry_run : 0
+sync : 0
+log_level : 0
+overwrite : 0
+multiple_mode : 0
+inactivity_timeout : 0
+use_watchdog : 0
+piglit_style_dmesg : 0
+test_root : /path/does/not/exist
+results_path : /path/does/not/exist
diff --git a/runner/json_tests_data/warnings/reference.json b/runner/json_tests_data/warnings/reference.json
new file mode 100644
index 00000000..8d5edbe1
--- /dev/null
+++ b/runner/json_tests_data/warnings/reference.json
@@ -0,0 +1,158 @@
+{
+ "__type__":"TestrunResult",
+ "results_version":9,
+ "name":"normal-run",
+ "uname":"Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64",
+ "time_elapsed":{
+ "__type__":"TimeAttribute",
+ "start":1539953735.1110389,
+ "end":1539953735.1723731
+ },
+ "tests":{
+ "igt@successtest@first-subtest":{
+ "out":"Starting subtest: first-subtest\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"warn",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: first-subtest\nThis is a warning line\nSubtest first-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.095083] Console: switching to colour dummy device 80x25\n<6> [3216186.095097] [IGT] successtest: executing\n<6> [3216186.101115] [IGT] successtest: starting subtest first-subtest\n<6> [3216186.101160] [IGT] successtest: exiting, ret=0\n<6> [3216186.101299] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@successtest@second-subtest":{
+ "out":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"pass",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Starting subtest: second-subtest\nSubtest second-subtest: SUCCESS (0.000s)\n",
+ "dmesg":"<6> [3216186.111837] Console: switching to colour dummy device 80x25\n<6> [3216186.111851] [IGT] successtest: executing\n<6> [3216186.114762] [IGT] successtest: starting subtest second-subtest\n<6> [3216186.114814] [IGT] successtest: exiting, ret=0\n<6> [3216186.114933] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ },
+ "result":"pass",
+ "out":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)\nSUCCESS (0.000s)\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "err":"",
+ "dmesg":"<6> [3216186.123400] Console: switching to colour dummy device 80x25\n<6> [3216186.123414] [IGT] no-subtests: executing\n<6> [3216186.125204] [IGT] no-subtests: exiting, ret=0\n<6> [3216186.125374] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-one":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-one: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-one: SKIP\n",
+ "dmesg":"<6> [3216186.135188] Console: switching to colour dummy device 80x25\n<6> [3216186.135212] [IGT] skippers: executing\n<6> [3216186.137075] [IGT] skippers: exiting, ret=77\n<6> [3216186.137206] Console: switching to colour frame buffer device 240x75\n"
+ },
+ "igt@skippers@skip-two":{
+ "out":"Test requirement not met in function __real_main3, file ..\/runner\/testdata\/skippers.c:6:\nTest requirement: false\nSkipping from fixture\nLast errno: 2, No such file or directory\nSubtest skip-two: SKIP\n",
+ "igt-version":"IGT-Version: 1.23-g0c763bfd (x86_64) (Linux: 4.18.0-1-amd64 x86_64)",
+ "result":"skip",
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0
+ },
+ "err":"Subtest skip-two: SKIP\n",
+ "dmesg":"<6> [3216186.145899] Console: switching to colour dummy device 80x25\n<6> [3216186.145912] [IGT] skippers: executing\n<6> [3216186.147754] [IGT] skippers: exiting, ret=77\n<6> [3216186.147894] Console: switching to colour frame buffer device 240x75\n"
+ }
+ },
+ "totals":{
+ "":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":1
+ },
+ "root":{
+ "crash":0,
+ "pass":2,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":1
+ },
+ "igt@successtest":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":1
+ },
+ "igt@no-subtests":{
+ "crash":0,
+ "pass":1,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":0,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ },
+ "igt@skippers":{
+ "crash":0,
+ "pass":0,
+ "dmesg-fail":0,
+ "dmesg-warn":0,
+ "skip":2,
+ "incomplete":0,
+ "timeout":0,
+ "notrun":0,
+ "fail":0,
+ "warn":0
+ }
+ },
+ "runtimes":{
+ "igt@successtest":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.027
+ }
+ },
+ "igt@no-subtests":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.01
+ }
+ },
+ "igt@skippers":{
+ "time":{
+ "__type__":"TimeAttribute",
+ "start":0,
+ "end":0.020999999999999998
+ }
+ }
+ }
+}
diff --git a/runner/json_tests_data/warnings/starttime.txt b/runner/json_tests_data/warnings/starttime.txt
new file mode 100644
index 00000000..ae038f18
--- /dev/null
+++ b/runner/json_tests_data/warnings/starttime.txt
@@ -0,0 +1 @@
+1539953735.111039
diff --git a/runner/json_tests_data/warnings/uname.txt b/runner/json_tests_data/warnings/uname.txt
new file mode 100644
index 00000000..a7aef6f7
--- /dev/null
+++ b/runner/json_tests_data/warnings/uname.txt
@@ -0,0 +1 @@
+Linux hostname 4.18.0-1-amd64 #1 SMP Debian 4.18.6-1 (2018-09-06) x86_64
diff --git a/runner/meson.build b/runner/meson.build
index 6c4b4862..de6e6f1c 100644
--- a/runner/meson.build
+++ b/runner/meson.build
@@ -10,6 +10,7 @@ runner_sources = [ 'runner.c' ]
resume_sources = [ 'resume.c' ]
results_sources = [ 'results.c' ]
runner_test_sources = [ 'runner_tests.c' ]
+runner_json_test_sources = [ 'runner_json_tests.c' ]
if _build_runner and jsonc.found()
subdir('testdata')
@@ -46,6 +47,13 @@ if _build_runner and jsonc.found()
dependencies : igt_deps)
test('runner', runner_test)
+ runner_json_test = executable('runner_json_test', runner_json_test_sources,
+ c_args : '-DJSON_TESTS_DIRECTORY="@0@"'.format(join_paths(meson.current_source_dir(), 'json_tests_data')),
+ link_with : runnerlib,
+ install : false,
+ dependencies : [igt_deps, jsonc])
+ test('runner_json', runner_json_test)
+
build_info += 'Build test runner: Yes'
else
build_info += 'Build test runner: No'
diff --git a/runner/runner_json_tests.c b/runner/runner_json_tests.c
new file mode 100644
index 00000000..758700d4
--- /dev/null
+++ b/runner/runner_json_tests.c
@@ -0,0 +1,171 @@
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+
+#include <json.h>
+
+#include "igt.h"
+#include "resultgen.h"
+
+static char testdatadir[] = JSON_TESTS_DIRECTORY;
+
+static struct json_object *read_json(int fd)
+{
+ struct json_object *obj;
+ struct json_tokener *tok = json_tokener_new();
+ enum json_tokener_error err;
+ char buf[512];
+ ssize_t s;
+
+ do {
+ s = read(fd, buf, sizeof(buf));
+ obj = json_tokener_parse_ex(tok, buf, s);
+ } while ((err = json_tokener_get_error(tok)) == json_tokener_continue);
+
+ igt_assert_eq(err, json_tokener_success);
+
+ json_tokener_free(tok);
+ return obj;
+}
+
+static void compare(struct json_object *one,
+ struct json_object *two);
+
+static void compare_objects(struct json_object *one,
+ struct json_object *two)
+{
+ json_object_iter iter;
+ struct json_object *subobj;
+
+ json_object_object_foreachC(one, iter) {
+ igt_debug("Key %s\n", iter.key);
+
+ igt_assert(json_object_object_get_ex(two, iter.key, &subobj));
+
+ compare(iter.val, subobj);
+ }
+}
+
+static void compare_arrays(struct json_object *one,
+ struct json_object *two)
+{
+ size_t i;
+
+ for (i = 0; i < json_object_array_length(one); i++) {
+ igt_debug("Array index %zd\n", i);
+ compare(json_object_array_get_idx(one, i),
+ json_object_array_get_idx(two, i));
+ }
+}
+
+static bool compatible_types(struct json_object *one,
+ struct json_object *two)
+{
+ /*
+ * A double of value 0.0 gets written as "0", which gets read
+ * as an int.
+ */
+ json_type onetype = json_object_get_type(one);
+ json_type twotype = json_object_get_type(two);
+
+ switch (onetype) {
+ case json_type_boolean:
+ case json_type_string:
+ case json_type_object:
+ case json_type_array:
+ case json_type_null:
+ return onetype == twotype;
+ break;
+ case json_type_double:
+ case json_type_int:
+ return twotype == json_type_double || twotype == json_type_int;
+ break;
+ }
+
+ igt_assert(!"Cannot be reached");
+ return false;
+}
+
+static void compare(struct json_object *one,
+ struct json_object *two)
+{
+ igt_assert(compatible_types(one, two));
+
+ switch (json_object_get_type(one)) {
+ case json_type_boolean:
+ igt_assert_eq(json_object_get_boolean(one), json_object_get_boolean(two));
+ break;
+ case json_type_double:
+ case json_type_int:
+ /*
+ * A double of value 0.0 gets written as "0", which
+ * gets read as an int. Both yield 0.0 with
+ * json_object_get_double(). Comparing doubles with ==
+ * considered crazy but it's good enough.
+ */
+ igt_assert(json_object_get_double(one) == json_object_get_double(two));
+ break;
+ case json_type_string:
+ igt_assert(!strcmp(json_object_get_string(one), json_object_get_string(two)));
+ break;
+ case json_type_object:
+ igt_assert_eq(json_object_object_length(one), json_object_object_length(two));
+ compare_objects(one, two);
+ break;
+ case json_type_array:
+ igt_assert_eq(json_object_array_length(one), json_object_array_length(two));
+ compare_arrays(one, two);
+ break;
+ case json_type_null:
+ break;
+ default:
+ igt_assert(!"Cannot be reached");
+ }
+}
+
+static void run_results_and_compare(int dirfd, const char *dirname)
+{
+ int testdirfd = openat(dirfd, dirname, O_RDONLY | O_DIRECTORY);
+ int reference;
+ struct json_object *resultsobj, *referenceobj;
+
+ igt_assert_fd(testdirfd);
+
+ igt_assert((resultsobj = generate_results_json(testdirfd)) != NULL);
+
+ reference = openat(testdirfd, "reference.json", O_RDONLY);
+ close(testdirfd);
+
+ igt_assert_fd(reference);
+ referenceobj = read_json(reference);
+ close(reference);
+ igt_assert(referenceobj != NULL);
+
+ igt_debug("Root object\n");
+ compare(resultsobj, referenceobj);
+ igt_assert_eq(json_object_put(resultsobj), 1);
+ igt_assert_eq(json_object_put(referenceobj), 1);
+}
+
+static const char *dirnames[] = {
+ "normal-run",
+ "warnings",
+ "warnings-with-dmesg-warns",
+ "piglit-style-dmesg",
+ "incomplete-before-any-subtests",
+ "dmesg-results",
+};
+
+igt_main
+{
+ int dirfd = open(testdatadir, O_RDONLY | O_DIRECTORY);
+ size_t i;
+
+ igt_assert_fd(dirfd);
+
+ for (i = 0; i < ARRAY_SIZE(dirnames); i++) {
+ igt_subtest(dirnames[i]) {
+ run_results_and_compare(dirfd, dirnames[i]);
+ }
+ }
+}