diff options
26 files changed, 547 insertions, 6 deletions
diff --git a/runner/executor.c b/runner/executor.c index c4b99115..a56cb5d6 100644 --- a/runner/executor.c +++ b/runner/executor.c @@ -1031,6 +1031,9 @@ static int monitor_output(pid_t child, fdatasync(outputs[_F_JOURNAL]); } + if (status == IGT_EXIT_ABORT) + aborting = true; + if (time_spent) *time_spent = time; } diff --git a/runner/json_tests_data/aborted-after-a-test/reference.json b/runner/json_tests_data/aborted-after-a-test/reference.json index 583242c7..0776f758 100644 --- a/runner/json_tests_data/aborted-after-a-test/reference.json +++ b/runner/json_tests_data/aborted-after-a-test/reference.json @@ -60,6 +60,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":4, "fail":1, @@ -72,6 +73,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":4, "fail":1, @@ -84,6 +86,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -96,6 +99,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -108,6 +112,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":2, "fail":0, @@ -120,6 +125,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":1, diff --git a/runner/json_tests_data/aborted-on-boot/reference.json b/runner/json_tests_data/aborted-on-boot/reference.json index d354fbac..75f19466 100644 --- a/runner/json_tests_data/aborted-on-boot/reference.json +++ b/runner/json_tests_data/aborted-on-boot/reference.json @@ -54,6 +54,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":5, "fail":1, @@ -66,6 +67,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":5, "fail":1, @@ -78,6 +80,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":2, "fail":0, @@ -90,6 +93,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -102,6 +106,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":2, "fail":0, @@ -114,6 +119,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":1, diff --git a/runner/json_tests_data/dmesg-escapes/reference.json b/runner/json_tests_data/dmesg-escapes/reference.json index 70d6b366..91c57310 100644 --- a/runner/json_tests_data/dmesg-escapes/reference.json +++ b/runner/json_tests_data/dmesg-escapes/reference.json @@ -30,6 +30,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -42,6 +43,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -54,6 +56,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/dmesg-results/reference.json b/runner/json_tests_data/dmesg-results/reference.json index 0edbae93..e9e01185 100644 --- a/runner/json_tests_data/dmesg-results/reference.json +++ b/runner/json_tests_data/dmesg-results/reference.json @@ -81,6 +81,7 @@ "dmesg-warn":2, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -93,6 +94,7 @@ "dmesg-warn":2, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -105,6 +107,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -117,6 +120,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -129,6 +133,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json index 4ccb18ae..8d266cdf 100644 --- a/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json +++ b/runner/json_tests_data/dmesg-warn-level-one-piglit-style/reference.json @@ -31,6 +31,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -43,6 +44,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -55,6 +57,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json index d706ee4c..4a1e8b31 100644 --- a/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json +++ b/runner/json_tests_data/dmesg-warn-level-piglit-style/reference.json @@ -31,6 +31,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -43,6 +44,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -55,6 +57,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/dmesg-warn-level/reference.json b/runner/json_tests_data/dmesg-warn-level/reference.json index 11cc39d9..400e9cfb 100644 --- a/runner/json_tests_data/dmesg-warn-level/reference.json +++ b/runner/json_tests_data/dmesg-warn-level/reference.json @@ -31,6 +31,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -43,6 +44,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -55,6 +57,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json b/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json index 370fce4d..514de06a 100644 --- a/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json +++ b/runner/json_tests_data/dynamic-subtest-name-in-multiple-subtests/reference.json @@ -42,6 +42,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -54,6 +55,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -66,6 +68,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/dynamic-subtests/reference.json b/runner/json_tests_data/dynamic-subtests/reference.json index ca8f6cd7..c013d282 100644 --- a/runner/json_tests_data/dynamic-subtests/reference.json +++ b/runner/json_tests_data/dynamic-subtests/reference.json @@ -90,6 +90,7 @@ "dmesg-warn":0, "skip":1, "incomplete":1, + "abort":0, "timeout":0, "notrun":0, "fail":2, @@ -102,6 +103,7 @@ "dmesg-warn":0, "skip":1, "incomplete":1, + "abort":0, "timeout":0, "notrun":0, "fail":2, @@ -114,6 +116,7 @@ "dmesg-warn":0, "skip":1, "incomplete":1, + "abort":0, "timeout":0, "notrun":0, "fail":2, diff --git a/runner/json_tests_data/empty-result-files/reference.json b/runner/json_tests_data/empty-result-files/reference.json index ef225601..f81ffb81 100644 --- a/runner/json_tests_data/empty-result-files/reference.json +++ b/runner/json_tests_data/empty-result-files/reference.json @@ -24,6 +24,7 @@ "dmesg-warn":0, "skip":0, "incomplete":1, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -36,6 +37,7 @@ "dmesg-warn":0, "skip":0, "incomplete":1, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -48,6 +50,7 @@ "dmesg-warn":0, "skip":0, "incomplete":1, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/incomplete-before-any-subtests/reference.json b/runner/json_tests_data/incomplete-before-any-subtests/reference.json index b8a76dd7..2a4bd456 100644 --- a/runner/json_tests_data/incomplete-before-any-subtests/reference.json +++ b/runner/json_tests_data/incomplete-before-any-subtests/reference.json @@ -49,6 +49,7 @@ "dmesg-warn":0, "skip":0, "incomplete":1, + "abort":0, "timeout":0, "notrun":4, "fail":0, @@ -61,6 +62,7 @@ "dmesg-warn":0, "skip":0, "incomplete":1, + "abort":0, "timeout":0, "notrun":4, "fail":0, @@ -73,6 +75,7 @@ "dmesg-warn":0, "skip":0, "incomplete":1, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -85,6 +88,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -97,6 +101,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":2, "fail":0, diff --git a/runner/json_tests_data/normal-run/reference.json b/runner/json_tests_data/normal-run/reference.json index 982038f9..0a00b1ca 100644 --- a/runner/json_tests_data/normal-run/reference.json +++ b/runner/json_tests_data/normal-run/reference.json @@ -78,6 +78,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":1, @@ -90,6 +91,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":1, @@ -102,6 +104,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":1, @@ -114,6 +117,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -126,6 +130,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/notrun-results-multiple-mode/reference.json b/runner/json_tests_data/notrun-results-multiple-mode/reference.json index 492c0a9e..3f8b7fb0 100644 --- a/runner/json_tests_data/notrun-results-multiple-mode/reference.json +++ b/runner/json_tests_data/notrun-results-multiple-mode/reference.json @@ -48,6 +48,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":3, "fail":0, @@ -60,6 +61,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":3, "fail":0, @@ -72,6 +74,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -84,6 +87,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":2, "fail":0, diff --git a/runner/json_tests_data/notrun-results/reference.json b/runner/json_tests_data/notrun-results/reference.json index 49a2f693..800de38c 100644 --- a/runner/json_tests_data/notrun-results/reference.json +++ b/runner/json_tests_data/notrun-results/reference.json @@ -54,6 +54,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":4, "fail":0, @@ -66,6 +67,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":4, "fail":0, @@ -78,6 +80,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -90,6 +93,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":1, "fail":0, @@ -102,6 +106,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":2, "fail":0, diff --git a/runner/json_tests_data/piglit-style-dmesg/reference.json b/runner/json_tests_data/piglit-style-dmesg/reference.json index 45d6108e..bf5d86ee 100644 --- a/runner/json_tests_data/piglit-style-dmesg/reference.json +++ b/runner/json_tests_data/piglit-style-dmesg/reference.json @@ -78,6 +78,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -90,6 +91,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -102,6 +104,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -114,6 +117,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -126,6 +130,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/unprintable-characters/reference.json b/runner/json_tests_data/unprintable-characters/reference.json index d3add1eb..88c62c34 100644 --- a/runner/json_tests_data/unprintable-characters/reference.json +++ b/runner/json_tests_data/unprintable-characters/reference.json @@ -30,6 +30,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -42,6 +43,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -54,6 +56,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -69,4 +72,4 @@ } } } -}
\ No newline at end of file +} diff --git a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json index fa571703..bd0bb3a3 100644 --- a/runner/json_tests_data/warnings-with-dmesg-warns/reference.json +++ b/runner/json_tests_data/warnings-with-dmesg-warns/reference.json @@ -79,6 +79,7 @@ "dmesg-warn":1, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -91,6 +92,7 @@ "dmesg-warn":1, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -103,6 +105,7 @@ "dmesg-warn":1, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -115,6 +118,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -127,6 +131,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/json_tests_data/warnings/reference.json b/runner/json_tests_data/warnings/reference.json index 53e0c3c7..a2b79da9 100644 --- a/runner/json_tests_data/warnings/reference.json +++ b/runner/json_tests_data/warnings/reference.json @@ -78,6 +78,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -90,6 +91,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -102,6 +104,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -114,6 +117,7 @@ "dmesg-warn":0, "skip":0, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, @@ -126,6 +130,7 @@ "dmesg-warn":0, "skip":2, "incomplete":0, + "abort":0, "timeout":0, "notrun":0, "fail":0, diff --git a/runner/resultgen.c b/runner/resultgen.c index 611d36cb..fe59aafe 100644 --- a/runner/resultgen.c +++ b/runner/resultgen.c @@ -645,6 +645,23 @@ static void process_dynamic_subtest_output(const char *piglit_name, &dynresulttext, &dyntime, dyn_result_idx < 0 ? NULL : matches.items[dyn_result_idx].where, dynend); + + /* + * If a dynamic subsubtest is considered incomplete we + * need to check parent's status first, to be sure that + * the binary hasn't aborted (exit code). If it has + * aborted then we have to attribute this status to our + * subsubtest. + */ + if (!strcmp(dynresulttext, "incomplete")) { + struct json_object *parent_subtest; + + if (json_object_object_get_ex(tests, piglit_name, &parent_subtest) && + json_object_object_get_ex(parent_subtest, "result", &parent_subtest) && + !strcmp(json_object_get_string(parent_subtest), "abort")) + dynresulttext = "abort"; + } + set_result(current_dynamic_test, dynresulttext); set_runtime(current_dynamic_test, dyntime); } @@ -1078,6 +1095,8 @@ static const char *result_from_exitcode(int exitcode) return "pass"; case IGT_EXIT_INVALID: return "skip"; + case IGT_EXIT_ABORT: + return "abort"; case INCOMPLETE_EXITCODE: return "incomplete"; default: @@ -1154,6 +1173,17 @@ static void fill_from_journal(int fd, } } + if (subtests->size && exitcode == IGT_EXIT_ABORT) { + char *last_subtest = subtests->subs[subtests->size - 1].name; + char subtest_piglit_name[256]; + struct json_object *subtest_obj; + + generate_piglit_name(entry->binary, last_subtest, subtest_piglit_name, sizeof(subtest_piglit_name)); + subtest_obj = get_or_create_json_object(tests, subtest_piglit_name); + + set_result(subtest_obj, "abort"); + } + if (subtests->size == 0) { char *subtestname = NULL; char piglit_name[256]; @@ -1322,6 +1352,7 @@ static struct json_object *get_totals_object(struct json_object *totals, json_object_object_add(obj, "dmesg-warn", json_object_new_int(0)); json_object_object_add(obj, "skip", json_object_new_int(0)); json_object_object_add(obj, "incomplete", json_object_new_int(0)); + json_object_object_add(obj, "abort", json_object_new_int(0)); json_object_object_add(obj, "timeout", json_object_new_int(0)); json_object_object_add(obj, "notrun", json_object_new_int(0)); json_object_object_add(obj, "fail", json_object_new_int(0)); diff --git a/runner/runner_tests.c b/runner/runner_tests.c index 139ee9fd..60e00960 100644 --- a/runner/runner_tests.c +++ b/runner/runner_tests.c @@ -28,9 +28,10 @@ static const char testdatadir[] = TESTDATA_DIRECTORY; * that test binaries without subtests should still be counted as one * for this macro. */ -#define NUM_TESTDATA_SUBTESTS 6 +#define NUM_TESTDATA_SUBTESTS 15 +#define NUM_TESTDATA_ABORT_SUBTESTS 9 /* The total number of test binaries in runner/testdata/ */ -#define NUM_TESTDATA_BINARIES 4 +#define NUM_TESTDATA_BINARIES 8 static const char *igt_get_result(struct json_object *tests, const char* testname) { @@ -954,6 +955,7 @@ igt_main struct execute_state state; const char *argv[] = { "runner", "--dry-run", + "-x", "^abort", testdatadir, dirname, }; @@ -964,7 +966,7 @@ igt_main igt_assert(initialize_execute_state(&state, settings, list)); igt_assert_eq(state.next, 0); igt_assert(state.dry); - igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS); + igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS - NUM_TESTDATA_ABORT_SUBTESTS); igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, "Dry run initialization didn't create the results directory.\n"); @@ -985,7 +987,7 @@ igt_main igt_assert(initialize_execute_state_from_resume(dirfd, &state, settings, list)); igt_assert_eq(state.next, 0); igt_assert(!state.dry); - igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS); + igt_assert_eq(list->size, NUM_TESTDATA_SUBTESTS - NUM_TESTDATA_ABORT_SUBTESTS); /* initialize_execute_state_from_resume() closes the dirfd */ igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, "Dry run resume somehow deleted the results directory.\n"); @@ -1512,7 +1514,7 @@ igt_main struct execute_state state; struct json_object *results, *tests; const char *argv[] = { "runner", - "-t", "dynamic", + "-t", "^dynamic$", testdatadir, dirname, }; @@ -1541,6 +1543,284 @@ igt_main } } + igt_subtest_group { + struct job_list *list = malloc(sizeof(*list)); + volatile int dirfd = -1; + char dirname[] = "tmpdirXXXXXX"; + + igt_fixture { + igt_require(mkdtemp(dirname) != NULL); + rmdir(dirname); + + init_job_list(list); + } + + igt_subtest("execute-abort-simple") { + struct execute_state state; + struct json_object *results, *tests; + const char *argv[] = { "runner", + "-t", "^abort-simple$", + testdatadir, + dirname, + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings)); + igt_assert(create_job_list(list, settings)); + igt_assert(initialize_execute_state(&state, settings, list)); + igt_assert(!execute(&state, settings, list)); /* false = signal abort */ + + igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, + "Execute didn't create the results directory\n"); + igt_assert_f((results = generate_results_json(dirfd)) != NULL, + "Results parsing failed\n"); + + igt_assert(json_object_object_get_ex(results, "tests", &tests)); + + igt_assert_eqstr(igt_get_result(tests, "igt@abort-simple"), "abort"); + + igt_assert_eq(json_object_put(results), 1); + } + + igt_fixture { + close(dirfd); + clear_directory(dirname); + free_job_list(list); + } + } + + igt_subtest_group { + struct job_list *list = malloc(sizeof(*list)); + volatile int dirfd = -1; + + for (int multiple = 0; multiple <= 1; ++multiple) { + char dirname[] = "tmpdirXXXXXX"; + + igt_fixture { + igt_require(mkdtemp(dirname) != NULL); + rmdir(dirname); + + init_job_list(list); + } + + igt_subtest_f("execute-abort%s", multiple ? "-multiple" : "") { + struct execute_state state; + struct json_object *results, *tests; + const char *argv[] = { "runner", + "-t", "^abort$", + multiple ? "--multiple-mode" : "--sync", + testdatadir, + dirname, + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings)); + igt_assert(create_job_list(list, settings)); + igt_assert(initialize_execute_state(&state, settings, list)); + igt_assert(!execute(&state, settings, list)); /* false = signal abort */ + + igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, + "Execute didn't create the results directory\n"); + igt_assert_f((results = generate_results_json(dirfd)) != NULL, + "Results parsing failed\n"); + + igt_assert(json_object_object_get_ex(results, "tests", &tests)); + + igt_assert_eqstr(igt_get_result(tests, "igt@abort@a-subtest"), "pass"); + igt_assert_eqstr(igt_get_result(tests, "igt@abort@b-subtest"), "abort"); + + if (multiple) /* no notrun injection for multiple mode */ + igt_assert_no_result_for(tests, "igt@abort@c-subtest"); + else + igt_assert_eqstr(igt_get_result(tests, "igt@abort@c-subtest"), "notrun"); + + igt_assert_eq(json_object_put(results), 1); + } + + igt_fixture { + close(dirfd); + clear_directory(dirname); + free_job_list(list); + } + } + } + + igt_subtest_group { + struct job_list *list = malloc(sizeof(*list)); + volatile int dirfd = -1; + + for (int multiple = 0; multiple <= 1; ++multiple) { + char dirname[] = "tmpdirXXXXXX"; + + igt_fixture { + igt_require(mkdtemp(dirname) != NULL); + rmdir(dirname); + + init_job_list(list); + } + + igt_subtest_f("execute-abort-fixture%s", multiple ? "-multiple" : "") { + struct execute_state state; + struct json_object *results, *tests; + const char *argv[] = { "runner", multiple ? "--multiple-mode" : "--sync", + "-t", "^abort-fixture$", + testdatadir, + dirname, + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings)); + igt_assert(create_job_list(list, settings)); + igt_assert(initialize_execute_state(&state, settings, list)); + igt_assert(!execute(&state, settings, list)); /* false = signal abort */ + + igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, + "Execute didn't create the results directory\n"); + igt_assert_f((results = generate_results_json(dirfd)) != NULL, + "Results parsing failed\n"); + + igt_assert(json_object_object_get_ex(results, "tests", &tests)); + + if (multiple) { + /* + * running the whole binary via -t, no + * way of blaming the particular subtest + */ + igt_assert_eqstr(igt_get_result(tests, "igt@abort-fixture"), "abort"); + igt_assert_no_result_for(tests, "igt@abort-fixture@a-subtest"); + igt_assert_no_result_for(tests, "igt@abort-fixture@b-subtest"); + } else { + igt_assert_eqstr(igt_get_result(tests, "igt@abort-fixture@a-subtest"), "abort"); + igt_assert_eqstr(igt_get_result(tests, "igt@abort-fixture@b-subtest"), "notrun"); + } + + igt_assert_eq(json_object_put(results), 1); + } + + igt_fixture { + close(dirfd); + clear_directory(dirname); + free_job_list(list); + } + } + } + + igt_subtest_group { + struct job_list *list = malloc(sizeof(*list)); + volatile int dirfd = -1; + + for (int multiple = 0; multiple <= 1; ++multiple) { + char dirname[] = "tmpdirXXXXXX"; + char filename[] = "tmplistXXXXXX"; + const char testlisttext[] = "igt@abort-fixture@b-subtest\n" + "igt@abort-fixture@a-subtest\n"; + + igt_fixture { + int fd; + igt_require((fd = mkstemp(filename)) >= 0); + igt_require(write(fd, testlisttext, strlen(testlisttext)) == strlen(testlisttext)); + close(fd); + igt_require(mkdtemp(dirname) != NULL); + rmdir(dirname); + + init_job_list(list); + } + + igt_subtest_f("execute-abort-fixture-testlist%s", multiple ? "-multiple" : "") { + struct execute_state state; + struct json_object *results, *tests; + const char *argv[] = { "runner", multiple ? "--multiple-mode" : "--sync", + "--test-list", filename, + testdatadir, + dirname, + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings)); + igt_assert(create_job_list(list, settings)); + igt_assert(initialize_execute_state(&state, settings, list)); + igt_assert(!execute(&state, settings, list)); /* false = signal abort */ + + igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, + "Execute didn't create the results directory\n"); + igt_assert_f((results = generate_results_json(dirfd)) != NULL, + "Results parsing failed\n"); + + igt_assert(json_object_object_get_ex(results, "tests", &tests)); + + if (multiple) /* multiple mode = no notruns */ + igt_assert_no_result_for(tests, "igt@abort-fixture@a-subtest"); + else + igt_assert_eqstr(igt_get_result(tests, "igt@abort-fixture@a-subtest"), "notrun"); + + + igt_assert_eqstr(igt_get_result(tests, "igt@abort-fixture@b-subtest"), "abort"); + + igt_assert_eq(json_object_put(results), 1); + } + + igt_fixture { + unlink(filename); + close(dirfd); + clear_directory(dirname); + free_job_list(list); + } + } + } + + igt_subtest_group { + struct job_list *list = malloc(sizeof(*list)); + volatile int dirfd = -1; + + for (int multiple = 0; multiple <= 1; ++multiple) { + char dirname[] = "tmpdirXXXXXX"; + + igt_fixture { + igt_require(mkdtemp(dirname) != NULL); + rmdir(dirname); + + init_job_list(list); + } + + igt_subtest_f("execute-abort-dynamic%s", multiple ? "-multiple" : "") { + struct execute_state state; + struct json_object *results, *tests; + const char *argv[] = { "runner", multiple ? "--multiple-mode" : "--sync", + "-t", "^abort-dynamic$", + testdatadir, + dirname, + }; + + igt_assert(parse_options(ARRAY_SIZE(argv), (char**)argv, settings)); + igt_assert(create_job_list(list, settings)); + igt_assert(initialize_execute_state(&state, settings, list)); + igt_assert(!execute(&state, settings, list)); /* false = signal abort */ + + igt_assert_f((dirfd = open(dirname, O_DIRECTORY | O_RDONLY)) >= 0, + "Execute didn't create the results directory\n"); + igt_assert_f((results = generate_results_json(dirfd)) != NULL, + "Results parsing failed\n"); + + igt_assert(json_object_object_get_ex(results, "tests", &tests)); + + igt_assert_eqstr(igt_get_result(tests, "igt@abort-dynamic@a-subtest@dynamic-1"), "pass"); + igt_assert_eqstr(igt_get_result(tests, "igt@abort-dynamic@b-subtest@dynamic-1"), "pass"); + igt_assert_eqstr(igt_get_result(tests, "igt@abort-dynamic@b-subtest@dynamic-2"), "abort"); + + igt_assert_no_result_for(tests, "igt@abort-dynamic@b-subtest@dynamic-3"); + + if (multiple) /* multiple mode = no notruns */ + igt_assert_no_result_for(tests, "igt@abort-dynamic@c-subtest"); + else + igt_assert_eqstr(igt_get_result(tests, "igt@abort-dynamic@c-subtest"), "notrun"); + + igt_assert_eq(json_object_put(results), 1); + } + + igt_fixture { + close(dirfd); + clear_directory(dirname); + free_job_list(list); + } + } + } + igt_subtest("file-descriptor-leakage") { int i; diff --git a/runner/testdata/abort-dynamic.c b/runner/testdata/abort-dynamic.c new file mode 100644 index 00000000..66cf13a7 --- /dev/null +++ b/runner/testdata/abort-dynamic.c @@ -0,0 +1,46 @@ +/* + * Copyright © 2020 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "igt.h" + +igt_main +{ + igt_subtest_with_dynamic("a-subtest") { + igt_dynamic("dynamic-1") + ; + } + + igt_subtest_with_dynamic("b-subtest") { + igt_dynamic("dynamic-1") + ; + + igt_dynamic("dynamic-2") + igt_abort_on_f(true, "I'm out!\n"); + + igt_dynamic("dynamic-3") + ; + } + + igt_subtest("c-subtest") + ; +} diff --git a/runner/testdata/abort-fixture.c b/runner/testdata/abort-fixture.c new file mode 100644 index 00000000..6a1cb986 --- /dev/null +++ b/runner/testdata/abort-fixture.c @@ -0,0 +1,37 @@ +/* + * Copyright © 2020 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "igt.h" + +igt_main +{ + igt_fixture { + igt_abort_on_f(true, "I'm out!\n"); + } + + igt_subtest("a-subtest") + ; + + igt_subtest("b-subtest") + ; +} diff --git a/runner/testdata/abort-simple.c b/runner/testdata/abort-simple.c new file mode 100644 index 00000000..94eef72c --- /dev/null +++ b/runner/testdata/abort-simple.c @@ -0,0 +1,29 @@ +/* + * Copyright © 2020 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "igt.h" + +igt_simple_main +{ + igt_abort_on_f(true, "I'm out!\n"); +} diff --git a/runner/testdata/abort.c b/runner/testdata/abort.c new file mode 100644 index 00000000..074946a4 --- /dev/null +++ b/runner/testdata/abort.c @@ -0,0 +1,36 @@ +/* + * Copyright © 2020 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include "igt.h" + +igt_main +{ + igt_subtest("a-subtest") + ; + + igt_subtest("b-subtest") + igt_abort_on_f(true, "I'm out!\n"); + + igt_subtest("c-subtest") + ; +} diff --git a/runner/testdata/meson.build b/runner/testdata/meson.build index 631ba5b9..1f822591 100644 --- a/runner/testdata/meson.build +++ b/runner/testdata/meson.build @@ -3,6 +3,10 @@ testdata_progs = [ 'successtest', 'no-subtests', 'skippers', 'dynamic', + 'abort', + 'abort-dynamic', + 'abort-fixture', + 'abort-simple', ] testdata_executables = [] |