tests/perf_pmu: Handle CPU hotplug failures better

CPU hotplug, especially CPU0, can be flaky on commodity hardware.

To improve test reliability and reponse times when testing larger runs we
need to handle those cases better.

Handle failures to off-line a CPU by immediately skipping the test, and
failures to on-line a CPU by immediately rebooting the machine.

This patch includes igt_sysrq_reboot implementation from Chris Wilson.

v2: Halt by default, reboot if env variable IGT_REBOOT_ON_FATAL_ERROR is
    set. (Petri Latvala)

v3: Add missign docs and update stale comment. (Petri Latvala)

v4: Use pause instead of sleep. (Chris Wilson)
v5: Newlines! (Chris Wilson)

Signed-off-by: Tvrtko Ursulin <tvrtko.ursulin@intel.com>
Cc: Chris Wilson <chris@chris-wilson.co.uk>
Cc: Petri Latvala <petri.latvala@intel.com>
Cc: Tomi Sarvela <tomi.p.sarvela@intel.com>
Reviewed-by: Petri Latvala <petri.latvala@intel.com>

1 files changed, 31 insertions, 7 deletions

diff --git a/tests/perf_pmu.c b/tests/perf_pmu.c
index 4713c98c..9ebffc64 100644
--- a/tests/perf_pmu.c
+++ b/tests/perf_pmu.c
@@ -990,6 +990,7 @@ static void cpu_hotplug(int gem_fd)
 	int link[2];
 	int fd, ret;
 	int cur = 0;
+	char buf;
 
 	igt_skip_on(IS_BROXTON(intel_get_drm_devid(gem_fd)));
 	igt_require(cpu0_hotplug_support());
@@ -1036,9 +1037,32 @@ static void cpu_hotplug(int gem_fd)
 			}
 
 			/* Offline followed by online a CPU. */
-			igt_assert_eq(write(cpufd, "0", 2), 2);
+
+			ret = write(cpufd, "0", 2);
+			if (ret < 0) {
+				/*
+				 * If we failed to offline a CPU we don't want
+				 * to proceed.
+				 */
+				igt_warn("Failed to offline cpu%u! (%d)\n",
+					 cpu, errno);
+				igt_assert_eq(write(link[1], "s", 1), 1);
+				break;
+			}
+
 			usleep(1e6);
-			igt_assert_eq(write(cpufd, "1", 2), 2);
+
+			ret = write(cpufd, "1", 2);
+			if (ret < 0) {
+				/*
+				 * Failed to bring a CPU back online is fatal
+				 * for the sanity of a test run so stop further
+				 * testing.
+				 */
+				igt_warn("Failed to online cpu%u! (%d)\n",
+					 cpu, errno);
+				igt_fatal_error();
+			}
 
 			close(cpufd);
 			cpu++;
@@ -1052,15 +1076,12 @@ static void cpu_hotplug(int gem_fd)
 	 * until the CPU core shuffler finishes one loop.
 	 */
 	for (;;) {
-		char buf;
-		int ret2;
-
 		usleep(500e3);
 		end_spin(gem_fd, spin[cur], 0);
 
 		/* Check if the child is signaling completion. */
-		ret2 = read(link[0], &buf, 1);
-		if ( ret2 == 1 || (ret2 < 0 && errno != EAGAIN))
+		ret = read(link[0], &buf, 1);
+		if ( ret == 1 || (ret < 0 && errno != EAGAIN))
 			break;
 
 		igt_spin_batch_free(gem_fd, spin[cur]);
@@ -1079,6 +1100,9 @@ static void cpu_hotplug(int gem_fd)
 	close(fd);
 	close(link[0]);
 
+	/* Skip if child signals a problem with offlining a CPU. */
+	igt_skip_on(buf == 's');
+
 	assert_within_epsilon(val, ts[1] - ts[0], tolerance);
 }