runner: Add --overall-timeout

With --overall-timeout $foo, the runner will stop executing new tests
when $foo seconds have already been used.

A resumed run will start over with no time used, using the same
timeout. This allows for executing a long list of tests piecemeal, in
about $foo length executions.

Signed-off-by: Petri Latvala <petri.latvala@intel.com>
Bugzilla: https://bugs.freedesktop.org/show_bug.cgi?id=106127
Cc: Arkadiusz Hiler <arkadiusz.hiler@intel.com>
Cc: Tomi Sarvela <tomi.p.sarvela@intel.com>
Cc: Martin Peres <martin.peres@linux.intel.com>
Reviewed-by: Arkadiusz Hiler <arkadiusz.hiler@intel.com>

5 files changed, 114 insertions, 20 deletions

diff --git a/runner/executor.c b/runner/executor.c
index 8b87a421..fc79f772 100644
--- a/runner/executor.c
+++ b/runner/executor.c
@@ -335,6 +335,7 @@ static bool kill_child(int sig, pid_t child)
 static int monitor_output(pid_t child,
 			   int outfd, int errfd, int kmsgfd, int sigfd,
 			   int *outputs,
+			   double *time_spent,
 			   struct settings *settings)
 {
 	fd_set set;
@@ -619,6 +620,9 @@ static int monitor_output(pid_t child,
 				if (settings->sync) {
 					fdatasync(outputs[_F_JOURNAL]);
 				}
+
+				if (time_spent)
+					*time_spent = time;
 			}
 
 			close(sigfd);
@@ -698,17 +702,30 @@ static int digits(size_t num)
 	return ret;
 }
 
+static void print_time_left(struct execute_state *state,
+			    struct settings *settings)
+{
+	int width;
+
+	if (settings->overall_timeout <= 0)
+		return;
+
+	width = digits(settings->overall_timeout);
+	printf("(%*.0fs left) ", width, state->time_left);
+}
+
 /*
  * Returns:
  *  =0 - Success
  *  <0 - Failure executing
  *  >0 - Timeout happened, need to recreate from journal
  */
-static int execute_entry(size_t idx,
-			  size_t total,
-			  struct settings *settings,
-			  struct job_list_entry *entry,
-			  int testdirfd, int resdirfd)
+static int execute_next_entry(struct execute_state *state,
+			      size_t total,
+			      double *time_spent,
+			      struct settings *settings,
+			      struct job_list_entry *entry,
+			      int testdirfd, int resdirfd)
 {
 	int dirfd;
 	int outputs[_F_LAST];
@@ -720,6 +737,7 @@ static int execute_entry(size_t idx,
 	char name[32];
 	pid_t child;
 	int result;
+	size_t idx = state->next;
 
 	snprintf(name, sizeof(name), "%zd", idx);
 	mkdirat(resdirfd, name, 0777);
@@ -780,7 +798,12 @@ static int execute_entry(size_t idx,
 
 	if (settings->log_level >= LOG_LEVEL_NORMAL) {
 		int width = digits(total);
-		printf("[%0*zd/%0*zd] %s", width, idx + 1, width, total, entry->binary);
+		printf("[%0*zd/%0*zd] ", width, idx + 1, width, total);
+
+		print_time_left(state, settings);
+
+		printf("%s", entry->binary);
+
 		if (entry->subtest_count > 0) {
 			size_t i;
 			const char *delim = "";
@@ -792,6 +815,7 @@ static int execute_entry(size_t idx,
 			}
 			printf(")");
 		}
+
 		printf("\n");
 	}
 
@@ -810,7 +834,7 @@ static int execute_entry(size_t idx,
 		close(errpipe[1]);
 
 		result = monitor_output(child, outfd, errfd, kmsgfd, sigfd,
-					outputs, settings);
+					outputs, time_spent, settings);
 	} else {
 		int outfd = outpipe[1];
 		int errfd = errpipe[1];
@@ -913,6 +937,15 @@ static double timeofday_double()
 	return 0.0;
 }
 
+static void init_time_left(struct execute_state *state,
+			   struct settings *settings)
+{
+	if (settings->overall_timeout <= 0)
+		state->time_left = -1;
+	else
+		state->time_left = settings->overall_timeout;
+}
+
 bool initialize_execute_state_from_resume(int dirfd,
 					  struct execute_state *state,
 					  struct settings *settings,
@@ -931,6 +964,8 @@ bool initialize_execute_state_from_resume(int dirfd,
 		return false;
 	}
 
+	init_time_left(state, settings);
+
 	for (i = list->size; i >= 0; i--) {
 		char name[32];
 
@@ -986,15 +1021,37 @@ bool initialize_execute_state(struct execute_state *state,
 	    !clear_old_results(settings->results_path))
 		return false;
 
+	init_time_left(state, settings);
+
 	return true;
 }
 
+static void reduce_time_left(struct settings *settings,
+			     struct execute_state *state,
+			     double time_spent)
+{
+	if (state->time_left < 0)
+		return;
+
+	if (time_spent > state->time_left)
+		state->time_left = 0.0;
+	else
+		state->time_left -= time_spent;
+}
+
+static bool overall_timeout_exceeded(struct execute_state *state)
+{
+	return state->time_left == 0.0;
+}
+
 bool execute(struct execute_state *state,
 	     struct settings *settings,
 	     struct job_list *job_list)
 {
 	struct utsname unamebuf;
 	int resdirfd, testdirfd, unamefd, timefd;
+	double time_spent = 0.0;
+	bool status = true;
 
 	if ((resdirfd = open(settings->results_path, O_DIRECTORY | O_RDONLY)) < 0) {
 		/* Initialize state should have done this */
@@ -1045,20 +1102,36 @@ bool execute(struct execute_state *state,
 
 	for (; state->next < job_list->size;
 	     state->next++) {
-		int result = execute_entry(state->next,
-					   job_list->size,
-					   settings,
-					   &job_list->entries[state->next],
-					   testdirfd, resdirfd);
-		if (result != 0) {
+		int result = execute_next_entry(state,
+						job_list->size,
+						&time_spent,
+						settings,
+						&job_list->entries[state->next],
+						testdirfd, resdirfd);
+
+		if (result < 0) {
+			status = false;
+			break;
+		}
+
+		reduce_time_left(settings, state, time_spent);
+
+		if (overall_timeout_exceeded(state)) {
+			if (settings->log_level >= LOG_LEVEL_NORMAL) {
+				printf("Overall timeout time exceeded, stopping.\n");
+			}
+
+			break;
+		}
+
+		if (result > 0) {
+			double time_left = state->time_left;
+
 			close(testdirfd);
 			close_watchdogs(settings);
-			if (result > 0) {
-				initialize_execute_state_from_resume(resdirfd, state, settings, job_list);
-				return execute(state, settings, job_list);
-			}
-			close(resdirfd);
-			return false;
+			initialize_execute_state_from_resume(resdirfd, state, settings, job_list);
+			state->time_left = time_left;
+			return execute(state, settings, job_list);
 		}
 	}
 
@@ -1070,5 +1143,5 @@ bool execute(struct execute_state *state,
 	close(testdirfd);
 	close(resdirfd);
 	close_watchdogs(settings);
-	return true;
+	return status;
 }
diff --git a/runner/executor.h b/runner/executor.h
index 8fe1605b..252339ab 100644
--- a/runner/executor.h
+++ b/runner/executor.h
@@ -7,6 +7,12 @@
 struct execute_state
 {
 	size_t next;
+	/*
+	 * < 0 : No overall timeout used.
+	 * = 0 : Timeouted, don't execute any more.
+	 * > 0 : Timeout in use, time left.
+	 */
+	double time_left;
 };
 
 enum {
diff --git a/runner/runner_tests.c b/runner/runner_tests.c
index b18af3a0..9c0f9eb0 100644
--- a/runner/runner_tests.c
+++ b/runner/runner_tests.c
@@ -219,6 +219,7 @@ igt_main
 		igt_assert(!settings.overwrite);
 		igt_assert(!settings.multiple_mode);
 		igt_assert_eq(settings.inactivity_timeout, 0);
+		igt_assert_eq(settings.overall_timeout, 0);
 		igt_assert(!settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
@@ -333,6 +334,7 @@ igt_main
 		igt_assert(!settings.overwrite);
 		igt_assert(!settings.multiple_mode);
 		igt_assert_eq(settings.inactivity_timeout, 0);
+		igt_assert_eq(settings.overall_timeout, 0);
 		igt_assert(!settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, testdatadir) != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
@@ -359,6 +361,7 @@ igt_main
 				 "--overwrite",
 				 "--multiple-mode",
 				 "--inactivity-timeout", "27",
+				 "--overall-timeout", "360",
 				 "--use-watchdog",
 				 "--piglit-style-dmesg",
 				 "test-root-dir",
@@ -382,6 +385,7 @@ igt_main
 		igt_assert(settings.overwrite);
 		igt_assert(settings.multiple_mode);
 		igt_assert_eq(settings.inactivity_timeout, 27);
+		igt_assert_eq(settings.overall_timeout, 360);
 		igt_assert(settings.use_watchdog);
 		igt_assert(strstr(settings.test_root, "test-root-dir") != NULL);
 		igt_assert(strstr(settings.results_path, "path-to-results") != NULL);
@@ -619,6 +623,7 @@ igt_main
 					 "--overwrite",
 					 "--multiple-mode",
 					 "--inactivity-timeout", "27",
+					 "--overall-timeout", "360",
 					 "--use-watchdog",
 					 "--piglit-style-dmesg",
 					 testdatadir,
diff --git a/runner/settings.c b/runner/settings.c
index 70fff3c0..e2401455 100644
--- a/runner/settings.c
+++ b/runner/settings.c
@@ -17,6 +17,7 @@ enum {
 	OPT_TEST_LIST,
 	OPT_IGNORE_MISSING,
 	OPT_PIGLIT_DMESG,
+	OPT_OVERALL_TIMEOUT,
 	OPT_HELP = 'h',
 	OPT_NAME = 'n',
 	OPT_DRY_RUN = 'd',
@@ -87,6 +88,8 @@ static const char *usage_str =
 	"  --inactivity-timeout <seconds>\n"
 	"                        Kill the running test after <seconds> of inactivity in\n"
 	"                        the test's stdout, stderr, or dmesg\n"
+	"  --overall-timeout <seconds>\n"
+	"                        Don't execute more tests after <seconds> has elapsed\n"
 	"  --use-watchdog        Use hardware watchdog for lethal enforcement of the\n"
 	"                        above timeout. Killing the test process is still\n"
 	"                        attempted at timeout trigger.\n"
@@ -198,6 +201,7 @@ bool parse_options(int argc, char **argv,
 		{"ignore-missing", no_argument, NULL, OPT_IGNORE_MISSING},
 		{"multiple-mode", no_argument, NULL, OPT_MULTIPLE},
 		{"inactivity-timeout", required_argument, NULL, OPT_TIMEOUT},
+		{"overall-timeout", required_argument, NULL, OPT_OVERALL_TIMEOUT},
 		{"use-watchdog", no_argument, NULL, OPT_WATCHDOG},
 		{"piglit-style-dmesg", no_argument, NULL, OPT_PIGLIT_DMESG},
 		{ 0, 0, 0, 0},
@@ -253,6 +257,9 @@ bool parse_options(int argc, char **argv,
 		case OPT_TIMEOUT:
 			settings->inactivity_timeout = atoi(optarg);
 			break;
+		case OPT_OVERALL_TIMEOUT:
+			settings->overall_timeout = atoi(optarg);
+			break;
 		case OPT_WATCHDOG:
 			settings->use_watchdog = true;
 			break;
@@ -448,6 +455,7 @@ bool serialize_settings(struct settings *settings)
 	SERIALIZE_LINE(f, settings, overwrite, "%d");
 	SERIALIZE_LINE(f, settings, multiple_mode, "%d");
 	SERIALIZE_LINE(f, settings, inactivity_timeout, "%d");
+	SERIALIZE_LINE(f, settings, overall_timeout, "%d");
 	SERIALIZE_LINE(f, settings, use_watchdog, "%d");
 	SERIALIZE_LINE(f, settings, piglit_style_dmesg, "%d");
 	SERIALIZE_LINE(f, settings, test_root, "%s");
@@ -502,6 +510,7 @@ bool read_settings(struct settings *settings, int dirfd)
 		PARSE_LINE(settings, name, val, overwrite, numval);
 		PARSE_LINE(settings, name, val, multiple_mode, numval);
 		PARSE_LINE(settings, name, val, inactivity_timeout, numval);
+		PARSE_LINE(settings, name, val, overall_timeout, numval);
 		PARSE_LINE(settings, name, val, use_watchdog, numval);
 		PARSE_LINE(settings, name, val, piglit_style_dmesg, numval);
 		PARSE_LINE(settings, name, val, test_root, val ? strdup(val) : NULL);
diff --git a/runner/settings.h b/runner/settings.h
index e534b845..b489abc5 100644
--- a/runner/settings.h
+++ b/runner/settings.h
@@ -30,6 +30,7 @@ struct settings {
 	bool overwrite;
 	bool multiple_mode;
 	int inactivity_timeout;
+	int overall_timeout;
 	bool use_watchdog;
 	char *test_root;
 	char *results_path;