diff options
| author | Arkadiusz Hiler <arkadiusz.hiler@intel.com> | 2019-07-04 14:55:01 +0300 |
|---|---|---|
| committer | Arkadiusz Hiler <arkadiusz.hiler@intel.com> | 2019-07-08 09:29:43 +0300 |
| commit | 78428bf16ef96ba30892f6a256d67830a3d84395 (patch) | |
| tree | 07c9a044e8d573b57ed6f8ae0c85843b455970d7 /lib | |
| parent | 3356087442806675438319578f1c964e51ee4965 (diff) | |
lib/igt_core: Add support for subtest descriptions
This patch adds igt_description() which attaches a description to the
following igt_subtest or igt_subtest_group block.
Descriptions are accessible via './test --describe[=pattern]'
Subtest description is its own igt_describe as well as igt_describes
of all the parenting igt_subtest_groups, starting from the outermost
scope.
Examples of code and produced outputs are included in
lib/test/igt_describe.c and as a documentation comment on igt_describe()
macro.
v2: address Simon's review
Cc: Simon Ser <simon.ser@intel.com>
Cc: Petri Latvala <petri.latvala@intel.com>
Signed-off-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Acked-by: Petri Latvala <petri.latvala@intel.com>
Reviewed-by: Simon Ser <simon.ser@intel.com>
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/igt_core.c | 179 | ||||
| -rw-r--r-- | lib/igt_core.h | 137 | ||||
| -rw-r--r-- | lib/tests/igt_describe.c | 282 | ||||
| -rw-r--r-- | lib/tests/meson.build | 1 |
4 files changed, 578 insertions, 21 deletions
diff --git a/lib/igt_core.c b/lib/igt_core.c index 4c5f4dd5..1cbb09f9 100644 --- a/lib/igt_core.c +++ b/lib/igt_core.c @@ -70,6 +70,7 @@ #include "igt_sysfs.h" #include "igt_sysrq.h" #include "igt_rc.h" +#include "igt_list.h" #define UNW_LOCAL_ONLY #include <libunwind.h> @@ -260,6 +261,7 @@ bool igt_skip_crc_compare; /* subtests helpers */ static bool list_subtests = false; +static bool describe_subtests = false; static char *run_single_subtest = NULL; static bool run_single_subtest_found = false; static const char *in_subtest = NULL; @@ -272,6 +274,16 @@ static enum { CONT = 0, SKIP, FAIL } skip_subtests_henceforth = CONT; +static char __current_description[512]; + +struct description_node { + char desc[sizeof(__current_description)]; + struct igt_list link; +}; + +static struct igt_list subgroup_descriptions; + + bool __igt_plain_output = false; /* fork support state */ @@ -286,6 +298,7 @@ enum { * conflict with core ones */ OPT_LIST_SUBTESTS = 500, + OPT_DESCRIBE_SUBTESTS, OPT_RUN_SUBTEST, OPT_DESCRIPTION, OPT_DEBUG, @@ -530,10 +543,59 @@ static void common_exit_handler(int sig) assert(sig != 0 || igt_exit_called); } +static void print_line_wrapping(const char *indent, const char *text) +{ + char *copy, *curr, *next_space; + int current_line_length = 0; + bool done = false; + + const int total_line_length = 80; + const int line_length = total_line_length - strlen(indent); + + copy = malloc(strlen(text) + 1); + memcpy(copy, text, strlen(text) + 1); + + curr = copy; + + printf("%s", indent); + + while (!done) { + next_space = strchr(curr, ' '); + + if (!next_space) { /* no more spaces, print everything that is left */ + done = true; + next_space = strchr(curr, '\0'); + } + + *next_space = '\0'; + + if ((next_space - curr) + current_line_length > line_length && curr != copy) { + printf("\n%s", indent); + current_line_length = 0; + } + + if (current_line_length == 0) + printf("%s", curr); /* first word in a line, don't space out */ + else + printf(" %s", curr); + + current_line_length += next_space - curr; + curr = next_space + 1; + } + + printf("\n"); + + free(copy); +} + + static void print_test_description(void) { - if (&__igt_test_description) - printf("%s\n", __igt_test_description); + if (&__igt_test_description) { + print_line_wrapping("", __igt_test_description); + if (describe_subtests) + printf("\n"); + } } static void print_version(void) @@ -561,6 +623,7 @@ static void print_usage(const char *help_str, bool output_on_stderr) " --interactive-debug[=domain]\n" " --skip-crc-compare\n" " --help-description\n" + " --describe\n" " --help|-h\n"); if (help_str) fprintf(f, "%s\n", help_str); @@ -674,6 +737,7 @@ static int common_init(int *argc, char **argv, int c, option_index = 0, i, x; static struct option long_options[] = { {"list-subtests", no_argument, NULL, OPT_LIST_SUBTESTS}, + {"describe", optional_argument, NULL, OPT_DESCRIBE_SUBTESTS}, {"run-subtest", required_argument, NULL, OPT_RUN_SUBTEST}, {"help-description", no_argument, NULL, OPT_DESCRIPTION}, {"debug", optional_argument, NULL, OPT_DEBUG}, @@ -691,6 +755,7 @@ static int common_init(int *argc, char **argv, int ret = 0; common_init_env(); + igt_list_init(&subgroup_descriptions); command_str = argv[0]; if (strrchr(command_str, '/')) @@ -781,6 +846,13 @@ static int common_init(int *argc, char **argv, if (!run_single_subtest) list_subtests = true; break; + case OPT_DESCRIBE_SUBTESTS: + if (optarg) + run_single_subtest = strdup(optarg); + list_subtests = true; + describe_subtests = true; + print_test_description(); + break; case OPT_RUN_SUBTEST: assert(optarg); if (!list_subtests) @@ -941,12 +1013,41 @@ void igt_simple_init_parse_opts(int *argc, char **argv, extra_opt_handler, handler_data); } +static void _clear_current_description(void) { + __current_description[0] = '\0'; +} + +static void __igt_print_description(const char *subtest_name, const char *file, int line) +{ + struct description_node *desc; + const char indent[] = " "; + bool has_doc = false; + + + printf("SUB %s %s:%d:\n", subtest_name, file, line); + + igt_list_for_each(desc, &subgroup_descriptions, link) { + print_line_wrapping(indent, desc->desc); + printf("\n"); + has_doc = true; + } + + if (__current_description[0] != '\0') { + print_line_wrapping(indent, __current_description); + printf("\n"); + has_doc = true; + } + + if (!has_doc) + printf("%sNO DOCUMENTATION!\n\n", indent); +} + /* * Note: Testcases which use these helpers MUST NOT output anything to stdout * outside of places protected by igt_run_subtest checks - the piglit * runner adds every line to the subtest list. */ -bool __igt_run_subtest(const char *subtest_name) +bool __igt_run_subtest(const char *subtest_name, const char *file, const int line) { int i; @@ -961,18 +1062,25 @@ bool __igt_run_subtest(const char *subtest_name) igt_exit(); } - if (list_subtests) { - printf("%s\n", subtest_name); - return false; - } - if (run_single_subtest) { - if (uwildmat(subtest_name, run_single_subtest) == 0) + if (uwildmat(subtest_name, run_single_subtest) == 0) { + _clear_current_description(); return false; - else + } else { run_single_subtest_found = true; + } } + if (describe_subtests) { + __igt_print_description(subtest_name, file, line); + _clear_current_description(); + return false; + } else if (list_subtests) { + printf("%s\n", subtest_name); + return false; + } + + if (skip_subtests_henceforth) { printf("%sSubtest %s: %s%s\n", (!__igt_plain_output) ? "\x1b[1m" : "", subtest_name, @@ -1021,15 +1129,32 @@ bool igt_only_list_subtests(void) return list_subtests; } -void __igt_subtest_group_save(int *save) + + +void __igt_subtest_group_save(int *save, int *desc) { assert(test_with_subtests); + if (__current_description[0] != '\0') { + struct description_node *new = calloc(1, sizeof(*new)); + memcpy(new->desc, __current_description, sizeof(__current_description)); + igt_list_add_tail(&new->link, &subgroup_descriptions); + _clear_current_description(); + *desc = true; + } + *save = skip_subtests_henceforth; } -void __igt_subtest_group_restore(int save) +void __igt_subtest_group_restore(int save, int desc) { + if (desc) { + struct description_node *last = + igt_list_last_entry(&subgroup_descriptions, last, link); + igt_list_del(&last->link); + free(last); + } + skip_subtests_henceforth = save; } @@ -1236,6 +1361,34 @@ bool igt_can_fail(void) return !test_with_subtests || in_fixture || in_subtest; } +/** + * igt_describe_f: + * @fmt: format string containing description + * @...: argument used by the format string + * + * Attach a description to the following #igt_subtest or #igt_subtest_group + * block. + * + * Check #igt_describe for more details. + * + */ +void igt_describe_f(const char *fmt, ...) +{ + int ret; + va_list args; + + if (!describe_subtests) + return; + + va_start(args, fmt); + + ret = vsnprintf(__current_description, sizeof(__current_description), fmt, args); + + va_end(args); + + assert(ret < sizeof(__current_description)); +} + static bool running_under_gdb(void) { char pathname[30], buf[1024]; @@ -1547,7 +1700,7 @@ void igt_exit(void) g_key_file_free(igt_key_file); if (run_single_subtest && !run_single_subtest_found) { - igt_warn("Unknown subtest: %s\n", run_single_subtest); + igt_critical("Unknown subtest: %s\n", run_single_subtest); exit(IGT_EXIT_INVALID); } diff --git a/lib/igt_core.h b/lib/igt_core.h index 101518f2..177d2431 100644 --- a/lib/igt_core.h +++ b/lib/igt_core.h @@ -174,7 +174,7 @@ int igt_subtest_init_parse_opts(int *argc, char **argv, #define igt_subtest_init(argc, argv) \ igt_subtest_init_parse_opts(&argc, argv, NULL, NULL, NULL, NULL, NULL); -bool __igt_run_subtest(const char *subtest_name); +bool __igt_run_subtest(const char *subtest_name, const char *file, const int line); #define __igt_tokencat2(x, y) x ## y /** @@ -198,14 +198,14 @@ bool __igt_run_subtest(const char *subtest_name); * * This is a simpler version of igt_subtest_f() */ -#define igt_subtest(name) for (; __igt_run_subtest((name)) && \ +#define igt_subtest(name) for (; __igt_run_subtest((name), __FILE__, __LINE__) && \ (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \ igt_success()) #define __igt_subtest_f(tmp, format...) \ for (char tmp [256]; \ snprintf( tmp , sizeof( tmp ), \ format), \ - __igt_run_subtest( tmp ) && \ + __igt_run_subtest(tmp, __FILE__, __LINE__) && \ (sigsetjmp(igt_subtest_jmpbuf, 1) == 0); \ igt_success()) @@ -227,8 +227,8 @@ bool __igt_run_subtest(const char *subtest_name); const char *igt_subtest_name(void); bool igt_only_list_subtests(void); -void __igt_subtest_group_save(int *); -void __igt_subtest_group_restore(int); +void __igt_subtest_group_save(int *, int *); +void __igt_subtest_group_restore(int, int); /** * igt_subtest_group: * @@ -245,11 +245,14 @@ void __igt_subtest_group_restore(int); * group will fail or skip. Subtest groups can be arbitrarily nested. */ #define igt_subtest_group for (int igt_tokencat(__tmpint,__LINE__) = 0, \ - igt_tokencat(__save,__LINE__) = 0; \ + igt_tokencat(__save,__LINE__) = 0, \ + igt_tokencat(__desc,__LINE__) = 0; \ igt_tokencat(__tmpint,__LINE__) < 1 && \ - (__igt_subtest_group_save(& igt_tokencat(__save,__LINE__) ), true); \ + (__igt_subtest_group_save(& igt_tokencat(__save,__LINE__), \ + & igt_tokencat(__desc,__LINE__) ), true); \ igt_tokencat(__tmpint,__LINE__) ++, \ - __igt_subtest_group_restore(igt_tokencat(__save,__LINE__) )) + __igt_subtest_group_restore(igt_tokencat(__save,__LINE__), \ + igt_tokencat(__desc,__LINE__))) /** * igt_main_args: @@ -385,6 +388,124 @@ static inline void igt_ignore_warn(bool value) { } +__attribute__((format(printf, 1, 2))) +void igt_describe_f(const char *fmt, ...); + +/** + * igt_describe: + * @dsc: string containing description + * + * Attach a description to the following #igt_subtest or #igt_subtest_group + * block. + * + * The description should complement the test/subtest name and provide more + * context on what is being tested. It should explain the idea of the test and + * do not mention implementation details, so that it never goes out of date. + * + * DO: + * * focus on the userspace's perspective + * * try to capture the reason for the test's existence + * * be brief + * + * DON'T: + * * try to translate the code into English + * * explain all the checks the test does + * * delve on the implementation + * + * Good examples: + * * "make sure that legacy cursor updates do not stall atomic commits" + * * "check that atomic updates of many planes are indeed atomic and take + * effect immediately after the commit" + * * "make sure that the meta-data exposed by the kernel to the userspace + * is correct and matches the used EDID" + * + * Bad examples: + * * "spawn 10 threads, each pinning cpu core with a busy loop..." + * * "randomly generate holes in a primary plane then try to cover each hole + * with a plane and make sure that CRC matches, do 25 gazillion rounds of + * that..." + * + * + * Resulting #igt_subtest documentation is a concatenation of its own + * description and all the parenting #igt_subtest_group descriptions, starting + * from the outermost one. Example: + * + * |[<!-- language="C" --> + * #include "igt.h" + * + * IGT_TEST_DESCRIPTION("Global description of the whole binary"); + * igt_main + * { + * igt_describe("Desc of the subgroup with A and B"); + * igt_subtest_group { + * igt_describe("Desc of the subtest A"); + * igt_subtest("subtest-a") { + * ... + * } + * + * igt_describe("Desc of the subtest B"); + * igt_subtest("subtest-b") { + * ... + * } + * } + * + * igt_describe("Desc of the subtest C"); + * igt_subtest("subtest-c") { + * ... + * } + * } + * ]| + * + * It's will accessible via --describe command line switch: + * + * |[ + * $ test --describe + * Global description of the whole binary + * + * SUB subtest-a test.c:5: + * Desc of the subgroup with A and B + * + * Desc of the subtest A + * + * SUB subtest-b test.c:10: + * Desc of the subgroup with A and B + * + * Desc of the subtest B + * + * SUB subtest-c test.c:15: + * Desc of the subtest C + * ]| + * + * Every single #igt_subtest does not have to be preceded with a #igt_describe + * as long as it has good-enough explanation provided on the #igt_subtest_group + * level. + * + * Example: + * + * |[<!-- language="C" --> + * #include "igt.h" + * + * igt_main + * { + * igt_describe("check xyz with different tilings"); + * igt_subtest_group { + * // no need for extra description, group is enough and tiling is + * // obvious from the test name + * igt_subtest("foo-tiling-x") { + * ... + * } + * + * igt_subtest("foo-tiling-y") { + * ... + * } + * } + * } + * ]| + * + */ +#define igt_describe(dsc) \ + igt_describe_f("%s", dsc) + /** * igt_assert: * @expr: condition to test diff --git a/lib/tests/igt_describe.c b/lib/tests/igt_describe.c new file mode 100644 index 00000000..2ea47e9d --- /dev/null +++ b/lib/tests/igt_describe.c @@ -0,0 +1,282 @@ +/* + * Copyright © 2019 Intel Corporation + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice (including the next + * paragraph) shall be included in all copies or substantial portions of the + * Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +#include <sys/wait.h> +#include <string.h> +#include <stdlib.h> + +#include "drmtest.h" +#include "igt_tests_common.h" + +IGT_TEST_DESCRIPTION("the top level description"); +static void fake_main(int argc, char **argv) { + igt_subtest_init(argc, argv); + + igt_describe("Basic A"); + igt_subtest("A") + ; + + igt_fixture + printf("should not be executed!\n"); + + igt_describe("Group with B, C & D"); + igt_subtest_group { + igt_describe("Basic B"); + igt_subtest("B") + ; + + if (!igt_only_list_subtests()) + printf("should not be executed!\n"); + + igt_describe("Group with C & D"); + igt_subtest_group { + igt_describe("Basic C"); + igt_subtest("C") + printf("should not be executed!\n"); + + // NO DOC + igt_subtest("D") + ; + } + } + + // NO DOC + igt_subtest_group { + // NO DOC + igt_subtest("E") + ; + } + + // NO DOC + igt_subtest("F") + ; + + igt_describe("this description should be so long that it wraps itself nicely in the terminal " + "this description should be so long that it wraps itself nicely in the terminal " + "this description should be so long that it wraps itself nicely in the terminal " + "this description should be so long that it wraps itself nicely in the terminal " + "this description should be so long that it wraps itself nicely in the terminal " + "this description should be so long that it wraps itself nicely in the terminal"); + igt_subtest("G") + ; + + igt_describe("verylongwordthatshoudlbeprintedeventhoughitspastthewrppinglimit" + "verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit " + "verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit" + "verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit"); + igt_subtest("F") + ; + + igt_exit(); +} + +static const char DESCRIBE_ALL_OUTPUT[] = \ + "the top level description\n" + "\n" + "SUB A ../lib/tests/igt_describe.c:36:\n" + " Basic A\n" + "\n" + "SUB B ../lib/tests/igt_describe.c:45:\n" + " Group with B, C & D\n" + "\n" + " Basic B\n" + "\n" + "SUB C ../lib/tests/igt_describe.c:54:\n" + " Group with B, C & D\n" + "\n" + " Group with C & D\n" + "\n" + " Basic C\n" + "\n" + "SUB D ../lib/tests/igt_describe.c:58:\n" + " Group with B, C & D\n" + "\n" + " Group with C & D\n" + "\n" + "SUB E ../lib/tests/igt_describe.c:66:\n" + " NO DOCUMENTATION!\n" + "\n" + "SUB F ../lib/tests/igt_describe.c:71:\n" + " NO DOCUMENTATION!\n" + "\n" + "SUB G ../lib/tests/igt_describe.c:80:\n" + " this description should be so long that it wraps itself nicely in the terminal this\n" + " description should be so long that it wraps itself nicely in the terminal this description\n" + " should be so long that it wraps itself nicely in the terminal this description should be so\n" + " long that it wraps itself nicely in the terminal this description should be so long that it\n" + " wraps itself nicely in the terminal this description should be so long that it wraps itself\n" + " nicely in the terminal\n" + "\n" + "SUB F ../lib/tests/igt_describe.c:87:\n" + " verylongwordthatshoudlbeprintedeventhoughitspastthewrppinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n" + " verylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimitverylongwordthatshoudlbeprintedeventhoughitspastthewrappinglimit\n\n"; + +static const char JUST_C_OUTPUT[] = \ + "the top level description\n" + "\n" + "SUB C ../lib/tests/igt_describe.c:54:\n" + " Group with B, C & D\n" + "\n" + " Group with C & D\n" + "\n" + " Basic C\n" + "\n"; + +static void assert_pipe_empty(int fd) +{ + char buf[5]; + internal_assert(0 == read(fd, buf, sizeof(buf))); +} + +static void read_whole_pipe(int fd, char *buf, size_t buflen) +{ + ssize_t readlen; + off_t offset; + + offset = 0; + while ((readlen = read(fd, buf+offset, buflen-offset))) { + if (readlen == -1) { + if (errno == EINTR) { + continue; + } else { + printf("read failed with %s\n", strerror(errno)); + exit(1); + } + } + internal_assert(readlen != -1); + offset += readlen; + } +} + +static pid_t do_fork(int argc, char **argv, int *out, int *err) +{ + int outfd[2], errfd[2]; + pid_t pid; + + internal_assert(pipe(outfd) != -1); + internal_assert(pipe(errfd) != -1); + + pid = fork(); + internal_assert(pid != -1); + + if (pid == 0) { + while (dup2(outfd[1], STDOUT_FILENO) == -1 && errno == EINTR) {} + while (dup2(errfd[1], STDERR_FILENO) == -1 && errno == EINTR) {} + + close(outfd[0]); + close(outfd[1]); + close(errfd[0]); + close(errfd[1]); + + fake_main(argc, argv); + + exit(-1); + } else { + /* close the writing ends */ + close(outfd[1]); + close(errfd[1]); + + *out = outfd[0]; + *err = errfd[0]; + + return pid; + } +} + +static int _wait(pid_t pid, int *status) { + int ret; + + do { + ret = waitpid(pid, status, 0); + } while (ret == -1 && errno == EINTR); + + return ret; +} + +int main(int argc, char **argv) +{ + char prog[] = "igt_describe"; + int status; + int outfd, errfd; + + /* describe all subtest */ { + static char out[4096]; + char arg[] = "--describe"; + char *fake_argv[] = {prog, arg}; + int fake_argc = ARRAY_SIZE(fake_argv); + + pid_t pid = do_fork(fake_argc, fake_argv, &outfd, &errfd); + + read_whole_pipe(outfd, out, sizeof(out)); + assert_pipe_empty(errfd); + + internal_assert(_wait(pid, &status) != -1); + internal_assert(WIFEXITED(status)); + internal_assert(WEXITSTATUS(status) == IGT_EXIT_SUCCESS); + internal_assert(0 == strcmp(DESCRIBE_ALL_OUTPUT, out)); + + close(outfd); + close(errfd); + } + + /* describe C using a pattern */ { + static char out[4096]; + char arg[] = "--describe=C"; + char *fake_argv[] = {prog, arg}; + int fake_argc = ARRAY_SIZE(fake_argv); + + pid_t pid = do_fork(fake_argc, fake_argv, &outfd, &errfd); + + read_whole_pipe(outfd, out, sizeof(out)); + assert_pipe_empty(errfd); + + internal_assert(_wait(pid, &status) != -1); + internal_assert(WIFEXITED(status)); + internal_assert(WEXITSTATUS(status) == IGT_EXIT_SUCCESS); + internal_assert(0 == strcmp(JUST_C_OUTPUT, out)); + + close(outfd); + close(errfd); + } + + /* fail describing with a bad pattern */ { + static char err[4096]; + char arg[] = "--describe=Z"; + char *fake_argv[] = {prog, arg}; + int fake_argc = ARRAY_SIZE(fake_argv); + + pid_t pid = do_fork(fake_argc, fake_argv, &outfd, &errfd); + + read_whole_pipe(errfd, err, sizeof(err)); + + internal_assert(_wait(pid, &status) != -1); + internal_assert(WIFEXITED(status)); + internal_assert(WEXITSTATUS(status) == IGT_EXIT_INVALID); + internal_assert(strstr(err, "Unknown subtest: Z")); + + close(outfd); + close(errfd); + } + + return 0; +} diff --git a/lib/tests/meson.build b/lib/tests/meson.build index b930ee6e..4d907e34 100644 --- a/lib/tests/meson.build +++ b/lib/tests/meson.build @@ -3,6 +3,7 @@ lib_tests = [ 'igt_can_fail', 'igt_can_fail_simple', 'igt_conflicting_args', + 'igt_describe', 'igt_edid', 'igt_exit_handler', 'igt_fork', |
