/* * Copyright © 2017 Intel Corporation * * Permission is hereby granted, free of charge, to any person obtaining a * copy of this software and associated documentation files (the "Software"), * to deal in the Software without restriction, including without limitation * the rights to use, copy, modify, merge, publish, distribute, sublicense, * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: * * The above copyright notice and this permission notice (including the next * paragraph) shall be included in all copies or substantial portions of the * Software. * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. * */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "intel_chipset.h" #include "drm.h" #include "ioctl_wrappers.h" #include "drmtest.h" #include "intel_io.h" #include "igt_aux.h" #include "igt_rand.h" #include "ewma.h" enum intel_engine_id { RCS, BCS, VCS, VCS1, VCS2, VECS, NUM_ENGINES }; struct duration { unsigned int min, max; }; enum w_type { BATCH, SYNC, DELAY, PERIOD, THROTTLE, QD_THROTTLE }; struct w_step { /* Workload step metadata */ enum w_type type; unsigned int context; unsigned int engine; struct duration duration; int nr_deps; int *dep; int wait; /* Implementation details */ unsigned int idx; struct igt_list rq_link; struct drm_i915_gem_execbuffer2 eb; struct drm_i915_gem_exec_object2 *obj; struct drm_i915_gem_relocation_entry reloc[4]; unsigned long bb_sz; uint32_t bb_handle; uint32_t *mapped_batch; uint32_t *seqno_value; uint32_t *seqno_address; uint32_t *rt0_value; uint32_t *rt0_address; uint32_t *rt1_address; uint32_t *latch_value; uint32_t *latch_address; unsigned int mapped_len; }; DECLARE_EWMA(uint64_t, rt, 4, 2) struct workload { unsigned int nr_steps; struct w_step *steps; uint32_t prng; struct timespec repeat_start; int pipe[2]; unsigned int nr_ctxs; uint32_t *ctx_id; uint32_t seqno[NUM_ENGINES]; uint32_t status_page_handle; uint32_t *status_page; unsigned int vcs_rr; unsigned long qd_sum[NUM_ENGINES]; unsigned long nr_bb[NUM_ENGINES]; struct igt_list requests[NUM_ENGINES]; unsigned int nrequest[NUM_ENGINES]; union { struct rtavg { struct ewma_rt avg[NUM_ENGINES]; uint32_t last[NUM_ENGINES]; } rt; }; }; static const unsigned int nop_calibration_us = 1000; static unsigned long nop_calibration; static int verbose = 1; static int fd; #define SWAPVCS (1<<0) #define SEQNO (1<<1) #define BALANCE (1<<2) #define RT (1<<3) #define VCS2REMAP (1<<4) #define INITVCSRR (1<<5) #define SYNCEDCLIENTS (1<<6) #define VCS_SEQNO_IDX(engine) (((engine) - VCS1) * 16) #define VCS_SEQNO_OFFSET(engine) (VCS_SEQNO_IDX(engine) * sizeof(uint32_t)) #define RCS_TIMESTAMP (0x2000 + 0x358) #define REG(x) (volatile uint32_t *)((volatile char *)igt_global_mmio + x) static const char *ring_str_map[NUM_ENGINES] = { [RCS] = "RCS", [BCS] = "BCS", [VCS] = "VCS", [VCS1] = "VCS1", [VCS2] = "VCS2", [VECS] = "VECS", }; static int parse_dependencies(unsigned int nr_steps, struct w_step *w, char *_desc) { char *desc = strdup(_desc); char *token, *tctx = NULL, *tstart = desc; int dep; igt_assert(desc); w->nr_deps = 0; w->dep = NULL; while ((token = strtok_r(tstart, "/", &tctx)) != NULL) { tstart = NULL; dep = atoi(token); if (dep > 0 || ((int)nr_steps + dep) < 0) { if (w->dep) free(w-dep); return -1; } if (dep < 0) { w->nr_deps++; w->dep = realloc(w->dep, sizeof(*w->dep) * w->nr_deps); igt_assert(w->dep); w->dep[w->nr_deps - 1] = dep; } } free(desc); return 0; } static struct workload *parse_workload(char *_desc, unsigned int flags) { struct workload *wrk; unsigned int nr_steps = 0; char *desc = strdup(_desc); char *_token, *token, *tctx = NULL, *tstart = desc; char *field, *fctx = NULL, *fstart; struct w_step step, *steps = NULL; bool bcs_used = false; unsigned int valid; int tmp; igt_assert(desc); while ((_token = strtok_r(tstart, ",", &tctx)) != NULL) { tstart = NULL; token = strdup(_token); igt_assert(token); fstart = token; valid = 0; memset(&step, 0, sizeof(step)); if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { fstart = NULL; if (!strcasecmp(field, "d")) { if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { tmp = atoi(field); if (tmp <= 0) { if (verbose) fprintf(stderr, "Invalid delay at step %u!\n", nr_steps); return NULL; } step.type = DELAY; step.wait = tmp; goto add_step; } } else if (!strcasecmp(field, "p")) { if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { tmp = atoi(field); if (tmp <= 0) { if (verbose) fprintf(stderr, "Invalid period at step %u!\n", nr_steps); return NULL; } step.type = PERIOD; step.wait = tmp; goto add_step; } } else if (!strcasecmp(field, "s")) { if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { tmp = atoi(field); if (tmp >= 0 || ((int)nr_steps + tmp) < 0) { if (verbose) fprintf(stderr, "Invalid sync target at step %u!\n", nr_steps); return NULL; } step.type = SYNC; step.wait = tmp; goto add_step; } } else if (!strcasecmp(field, "t")) { if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { tmp = atoi(field); if (tmp < 0) { if (verbose) fprintf(stderr, "Invalid throttle at step %u!\n", nr_steps); return NULL; } step.type = THROTTLE; step.wait = tmp; goto add_step; } } else if (!strcasecmp(field, "q")) { if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { tmp = atoi(field); if (tmp < 0) { if (verbose) fprintf(stderr, "Invalid qd throttle at step %u!\n", nr_steps); return NULL; } step.type = QD_THROTTLE; step.wait = tmp; goto add_step; } } tmp = atoi(field); if (tmp < 0) { if (verbose) fprintf(stderr, "Invalid ctx id at step %u!\n", nr_steps); return NULL; } step.context = tmp; valid++; } if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { unsigned int i, old_valid = valid; fstart = NULL; for (i = 0; i < ARRAY_SIZE(ring_str_map); i++) { if (!strcasecmp(field, ring_str_map[i])) { step.engine = i; if (step.engine == BCS) bcs_used = true; valid++; break; } } if (old_valid == valid) { if (verbose) fprintf(stderr, "Invalid engine id at step %u!\n", nr_steps); return NULL; } } if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { char *sep = NULL; long int tmpl; fstart = NULL; tmpl = strtol(field, &sep, 10); if (tmpl == 0 || tmpl == LONG_MIN || tmpl == LONG_MAX) { if (verbose) fprintf(stderr, "Invalid duration at step %u!\n", nr_steps); return NULL; } step.duration.min = tmpl; if (sep && *sep == '-') { tmpl = strtol(sep + 1, NULL, 10); if (tmpl == 0 || tmpl == LONG_MIN || tmpl == LONG_MAX) { if (verbose) fprintf(stderr, "Invalid duration range at step %u!\n", nr_steps); return NULL; } step.duration.max = tmpl; } else { step.duration.max = step.duration.min; } valid++; } if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { fstart = NULL; tmp = parse_dependencies(nr_steps, &step, field); if (tmp < 0) { if (verbose) fprintf(stderr, "Invalid dependency at step %u!\n", nr_steps); return NULL; } valid++; } if ((field = strtok_r(fstart, ".", &fctx)) != NULL) { fstart = NULL; if (strlen(field) != 1 || (field[0] != '0' && field[0] != '1')) { if (verbose) fprintf(stderr, "Invalid wait boolean at step %u!\n", nr_steps); return NULL; } step.wait = field[0] - '0'; valid++; } if (valid != 5) { if (verbose) fprintf(stderr, "Invalid record at step %u!\n", nr_steps); return NULL; } step.type = BATCH; add_step: step.idx = nr_steps++; steps = realloc(steps, sizeof(step) * nr_steps); igt_assert(steps); memcpy(&steps[nr_steps - 1], &step, sizeof(step)); free(token); } wrk = malloc(sizeof(*wrk)); igt_assert(wrk); wrk->nr_steps = nr_steps; wrk->steps = steps; free(desc); if (bcs_used && verbose) printf("BCS usage in workload with VCS2 remapping enabled!\n"); return wrk; } static struct workload * clone_workload(struct workload *_wrk) { struct workload *wrk; int i; wrk = malloc(sizeof(*wrk)); igt_assert(wrk); memset(wrk, 0, sizeof(*wrk)); wrk->nr_steps = _wrk->nr_steps; wrk->steps = calloc(wrk->nr_steps, sizeof(struct w_step)); igt_assert(wrk->steps); memcpy(wrk->steps, _wrk->steps, sizeof(struct w_step) * wrk->nr_steps); for (i = 0; i < NUM_ENGINES; i++) igt_list_init(&wrk->requests[i]); return wrk; } #define rounddown(x, y) (x - (x%y)) #ifndef PAGE_SIZE #define PAGE_SIZE (4096) #endif static unsigned int get_duration(struct duration *dur) { if (dur->min == dur->max) return dur->min; else return dur->min + hars_petruska_f54_1_random_unsafe() % (dur->max + 1 - dur->min); } static unsigned long get_bb_sz(unsigned int duration) { return ALIGN(duration * nop_calibration * sizeof(uint32_t) / nop_calibration_us, sizeof(uint32_t)); } static void terminate_bb(struct w_step *w, unsigned int flags) { const uint32_t bbe = 0xa << 23; unsigned long mmap_start, mmap_len; unsigned long batch_start = w->bb_sz; uint32_t *ptr, *cs; igt_assert(((flags & RT) && (flags & SEQNO)) || !(flags & RT)); batch_start -= sizeof(uint32_t); /* bbend */ if (flags & SEQNO) batch_start -= 4 * sizeof(uint32_t); if (flags & RT) batch_start -= 12 * sizeof(uint32_t); mmap_start = rounddown(batch_start, PAGE_SIZE); mmap_len = w->bb_sz - mmap_start; gem_set_domain(fd, w->bb_handle, I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); ptr = gem_mmap__wc(fd, w->bb_handle, mmap_start, mmap_len, PROT_WRITE); cs = (uint32_t *)((char *)ptr + batch_start - mmap_start); if (flags & SEQNO) { w->reloc[0].offset = batch_start + sizeof(uint32_t); batch_start += 4 * sizeof(uint32_t); *cs++ = MI_STORE_DWORD_IMM; w->seqno_address = cs; *cs++ = 0; *cs++ = 0; w->seqno_value = cs; *cs++ = 0; } if (flags & RT) { w->reloc[1].offset = batch_start + sizeof(uint32_t); batch_start += 4 * sizeof(uint32_t); *cs++ = MI_STORE_DWORD_IMM; w->rt0_address = cs; *cs++ = 0; *cs++ = 0; w->rt0_value = cs; *cs++ = 0; w->reloc[2].offset = batch_start + 2 * sizeof(uint32_t); batch_start += 4 * sizeof(uint32_t); *cs++ = 0x24 << 23 | 2; /* MI_STORE_REG_MEM */ *cs++ = RCS_TIMESTAMP; w->rt1_address = cs; *cs++ = 0; *cs++ = 0; w->reloc[3].offset = batch_start + sizeof(uint32_t); batch_start += 4 * sizeof(uint32_t); *cs++ = MI_STORE_DWORD_IMM; w->latch_address = cs; *cs++ = 0; *cs++ = 0; w->latch_value = cs; *cs++ = 0; } *cs = bbe; w->mapped_batch = ptr; w->mapped_len = mmap_len; } static const unsigned int eb_engine_map[NUM_ENGINES] = { [RCS] = I915_EXEC_RENDER, [BCS] = I915_EXEC_BLT, [VCS] = I915_EXEC_BSD, [VCS1] = I915_EXEC_BSD | I915_EXEC_BSD_RING1, [VCS2] = I915_EXEC_BSD | I915_EXEC_BSD_RING2, [VECS] = I915_EXEC_VEBOX }; static void eb_update_flags(struct w_step *w, enum intel_engine_id engine, unsigned int flags) { if (engine == VCS2 && (flags & VCS2REMAP)) engine = BCS; w->eb.flags = eb_engine_map[engine]; w->eb.flags |= I915_EXEC_HANDLE_LUT; w->eb.flags |= I915_EXEC_NO_RELOC; } static void alloc_step_batch(struct workload *wrk, struct w_step *w, unsigned int flags) { enum intel_engine_id engine = w->engine; unsigned int j = 0; unsigned int nr_obj = 3 + w->nr_deps; unsigned int i; w->obj = calloc(nr_obj, sizeof(*w->obj)); igt_assert(w->obj); w->obj[j].handle = gem_create(fd, 4096); w->obj[j].flags = EXEC_OBJECT_WRITE; j++; igt_assert(j < nr_obj); if (flags & SEQNO) { w->obj[j].handle = wrk->status_page_handle; j++; igt_assert(j < nr_obj); } for (i = 0; i < w->nr_deps; i++) { igt_assert(w->dep[i] <= 0); if (w->dep[i]) { int dep_idx = w->idx + w->dep[i]; igt_assert(dep_idx >= 0 && dep_idx < wrk->nr_steps); igt_assert(wrk->steps[dep_idx].type == BATCH); w->obj[j].handle = wrk->steps[dep_idx].obj[0].handle; j++; igt_assert(j < nr_obj); } } w->bb_sz = get_bb_sz(w->duration.max); w->bb_handle = w->obj[j].handle = gem_create(fd, w->bb_sz); terminate_bb(w, flags); if (flags & SEQNO) { w->obj[j].relocs_ptr = to_user_pointer(&w->reloc); if (flags & RT) w->obj[j].relocation_count = 4; else w->obj[j].relocation_count = 1; for (i = 0; i < w->obj[j].relocation_count; i++) w->reloc[i].target_handle = 1; } w->eb.buffers_ptr = to_user_pointer(w->obj); w->eb.buffer_count = j + 1; w->eb.rsvd1 = wrk->ctx_id[w->context]; if (flags & SWAPVCS && engine == VCS1) engine = VCS2; else if (flags & SWAPVCS && engine == VCS2) engine = VCS1; eb_update_flags(w, engine, flags); #ifdef DEBUG printf("%u: %u:|", w->idx, w->eb.buffer_count); for (i = 0; i <= j; i++) printf("%x|", w->obj[i].handle); printf(" %10lu flags=%llx bb=%x[%u] ctx[%u]=%u\n", w->bb_sz, w->eb.flags, w->bb_handle, j, w->context, wrk->ctx_id[w->context]); #endif } static void prepare_workload(unsigned int id, struct workload *wrk, unsigned int flags) { int max_ctx = -1; struct w_step *w; int i; wrk->prng = rand(); if (flags & INITVCSRR) wrk->vcs_rr = id & 1; if (flags & SEQNO) { const unsigned int status_sz = sizeof(uint32_t); uint32_t handle = gem_create(fd, status_sz); gem_set_caching(fd, handle, I915_CACHING_CACHED); wrk->status_page_handle = handle; wrk->status_page = gem_mmap__cpu(fd, handle, 0, status_sz, PROT_READ); } for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) { if ((int)w->context > max_ctx) { int delta = w->context + 1 - wrk->nr_ctxs; wrk->nr_ctxs += delta; wrk->ctx_id = realloc(wrk->ctx_id, wrk->nr_ctxs * sizeof(uint32_t)); memset(&wrk->ctx_id[wrk->nr_ctxs - delta], 0, delta * sizeof(uint32_t)); max_ctx = w->context; } if (!wrk->ctx_id[w->context]) { struct drm_i915_gem_context_create arg = {}; drmIoctl(fd, DRM_IOCTL_I915_GEM_CONTEXT_CREATE, &arg); igt_assert(arg.ctx_id); wrk->ctx_id[w->context] = arg.ctx_id; } } for (i = 0, w = wrk->steps; i < wrk->nr_steps; i++, w++) { unsigned int _flags = flags; enum intel_engine_id engine = w->engine; if (w->type != BATCH) continue; if (engine != VCS && engine != VCS1 && engine != VCS2) _flags &= ~(SEQNO | RT); if (engine == VCS) _flags &= ~SWAPVCS; alloc_step_batch(wrk, w, _flags); } } static double elapsed(const struct timespec *start, const struct timespec *end) { return (end->tv_sec - start->tv_sec) + (end->tv_nsec - start->tv_nsec) / 1e9; } static int elapsed_us(const struct timespec *start, const struct timespec *end) { return elapsed(start, end) * 1e6; } static enum intel_engine_id get_vcs_engine(unsigned int n) { const enum intel_engine_id vcs_engines[2] = { VCS1, VCS2 }; igt_assert(n < ARRAY_SIZE(vcs_engines)); return vcs_engines[n]; } struct workload_balancer { unsigned int (*get_qd)(const struct workload_balancer *balancer, struct workload *wrk, enum intel_engine_id engine); enum intel_engine_id (*balance)(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w); }; static enum intel_engine_id rr_balance(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w) { unsigned int engine; engine = get_vcs_engine(wrk->vcs_rr); wrk->vcs_rr ^= 1; return engine; } static const struct workload_balancer rr_balancer = { .balance = rr_balance, }; static unsigned int get_qd_depth(const struct workload_balancer *balancer, struct workload *wrk, enum intel_engine_id engine) { return wrk->seqno[engine] - wrk->status_page[VCS_SEQNO_IDX(engine)]; } static enum intel_engine_id qd_balance(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w) { enum intel_engine_id engine; long qd[NUM_ENGINES]; unsigned int n; igt_assert(w->engine == VCS); qd[VCS1] = balancer->get_qd(balancer, wrk, VCS1); wrk->qd_sum[VCS1] += qd[VCS1]; qd[VCS2] = balancer->get_qd(balancer, wrk, VCS2); wrk->qd_sum[VCS2] += qd[VCS2]; if (qd[VCS1] < qd[VCS2]) n = 0; else if (qd[VCS2] < qd[VCS1]) n = 1; else n = wrk->vcs_rr; engine = get_vcs_engine(n); wrk->vcs_rr = n ^ 1; #ifdef DEBUG printf("qd_balance: 1:%ld 2:%ld rr:%u = %u\t(%lu - %u) (%lu - %u)\n", qd[VCS1], qd[VCS2], wrk->vcs_rr, engine, wrk->seqno[VCS1], wrk->status_page[VCS_SEQNO_IDX(VCS1)], wrk->seqno[VCS2], wrk->status_page[VCS_SEQNO_IDX(VCS2)]); #endif return engine; } static const struct workload_balancer qd_balancer = { .get_qd = get_qd_depth, .balance = qd_balance, }; static enum intel_engine_id __rt_select_engine(struct workload *wrk, unsigned long *qd, bool random) { unsigned int n; qd[VCS1] >>= 10; qd[VCS2] >>= 10; if (qd[VCS1] < qd[VCS2]) n = 0; else if (qd[VCS2] < qd[VCS1]) n = 1; else if (random) n = hars_petruska_f54_1_random(&wrk->prng) & 1; else n = wrk->vcs_rr; wrk->vcs_rr = n ^ 1; return get_vcs_engine(n); } struct rt_depth { uint32_t seqno; uint32_t submitted; uint32_t completed; }; #define READ_ONCE(x) (*(volatile typeof(x) *)(&(x))) static void get_rt_depth(struct workload *wrk, unsigned int engine, struct rt_depth *rt) { const unsigned int idx = VCS_SEQNO_IDX(engine); uint32_t latch; do { latch = READ_ONCE(wrk->status_page[idx + 3]); rt->submitted = wrk->status_page[idx + 1]; rt->completed = wrk->status_page[idx + 2]; rt->seqno = READ_ONCE(wrk->status_page[idx]); } while (latch != rt->seqno); } static enum intel_engine_id __rt_balance(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w, bool random) { unsigned long qd[NUM_ENGINES]; struct rt_depth results; igt_assert(w->engine == VCS); /* Estimate the "speed" of the most recent batch * (finish time - submit time) * and use that as an approximate for the total remaining time for * all batches on that engine, plus the time we expect this batch to * take. We try to keep the total balanced between the engines. */ get_rt_depth(wrk, VCS1, &results); qd[VCS1] = wrk->seqno[VCS1] - results.seqno; wrk->qd_sum[VCS1] += qd[VCS1]; qd[VCS1] = (qd[VCS1] + 1) * (results.completed - results.submitted); #ifdef DEBUG printf("qd[0] = %d (%d - %d) x %d (%d - %d) = %ld\n", wrk->seqno[VCS1] - wrk->status_page[0], wrk->seqno[VCS1], wrk->status_page[0], wrk->status_page[2] - wrk->status_page[1], wrk->status_page[2], wrk->status_page[1], qd[VCS1]); #endif get_rt_depth(wrk, VCS2, &results); qd[VCS2] = wrk->seqno[VCS2] - results.seqno; wrk->qd_sum[VCS2] += qd[VCS2]; qd[VCS2] = (qd[VCS2] + 1) * (results.completed - results.submitted); #ifdef DEBUG printf("qd[1] = %d (%d - %d) x %d (%d - %d) = %ld\n", wrk->seqno[VCS2] - wrk->status_page[16], wrk->seqno[VCS2], wrk->status_page[16], wrk->status_page[18] - wrk->status_page[17], wrk->status_page[18], wrk->status_page[17], qd[VCS2]); #endif return __rt_select_engine(wrk, qd, random); } static enum intel_engine_id rt_balance(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w) { return __rt_balance(balancer, wrk, w, false); } static const struct workload_balancer rt_balancer = { .get_qd = get_qd_depth, .balance = rt_balance, }; static enum intel_engine_id rtr_balance(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w) { return __rt_balance(balancer, wrk, w, true); } static const struct workload_balancer rtr_balancer = { .get_qd = get_qd_depth, .balance = rtr_balance, }; static enum intel_engine_id rtavg_balance(const struct workload_balancer *balancer, struct workload *wrk, struct w_step *w) { unsigned long qd[NUM_ENGINES]; struct rt_depth results; igt_assert(w->engine == VCS); /* Estimate the average "speed" of the most recent batches * (finish time - submit time) * and use that as an approximate for the total remaining time for * all batches on that engine plus the time we expect to execute in. * We try to keep the total remaining balanced between the engines. */ get_rt_depth(wrk, VCS1, &results); if (results.seqno != wrk->rt.last[VCS1]) { igt_assert((long)(results.completed - results.submitted) > 0); ewma_rt_add(&wrk->rt.avg[VCS1], results.completed - results.submitted); wrk->rt.last[VCS1] = results.seqno; } qd[VCS1] = wrk->seqno[VCS1] - results.seqno; wrk->qd_sum[VCS1] += qd[VCS1]; qd[VCS1] = (qd[VCS1] + 1) * ewma_rt_read(&wrk->rt.avg[VCS1]); #ifdef DEBUG printf("qd[0] = %d (%d - %d) x %ld (%d) = %ld\n", wrk->seqno[VCS1] - wrk->status_page[0], wrk->seqno[VCS1], wrk->status_page[0], ewma_rt_read(&wrk->rt.avg[VCS1]), wrk->status_page[2] - wrk->status_page[1], qd[VCS1]); #endif get_rt_depth(wrk, VCS2, &results); if (results.seqno != wrk->rt.last[VCS2]) { igt_assert((long)(results.completed - results.submitted) > 0); ewma_rt_add(&wrk->rt.avg[VCS2], results.completed - results.submitted); wrk->rt.last[VCS2] = results.seqno; } qd[VCS2] = wrk->seqno[VCS2] - results.seqno; wrk->qd_sum[VCS2] += qd[VCS2]; qd[VCS2] = (qd[VCS2] + 1) * ewma_rt_read(&wrk->rt.avg[VCS2]); #ifdef DEBUG printf("qd[1] = %d (%d - %d) x %ld (%d) = %ld\n", wrk->seqno[VCS2] - wrk->status_page[16], wrk->seqno[VCS2], wrk->status_page[16], ewma_rt_read(&wrk->rt.avg[VCS2]), wrk->status_page[18] - wrk->status_page[17], qd[VCS2]); #endif return __rt_select_engine(wrk, qd, false); } static const struct workload_balancer rtavg_balancer = { .get_qd = get_qd_depth, .balance = rtavg_balance, }; static void update_bb_seqno(struct w_step *w, enum intel_engine_id engine, uint32_t seqno) { igt_assert(engine == VCS1 || engine == VCS2); gem_set_domain(fd, w->bb_handle, I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); w->reloc[0].delta = VCS_SEQNO_OFFSET(engine); *w->seqno_value = seqno; *w->seqno_address = w->reloc[0].presumed_offset + w->reloc[0].delta; /* If not using NO_RELOC, force the relocations */ if (!(w->eb.flags & I915_EXEC_NO_RELOC)) w->reloc[0].presumed_offset = -1; } static void update_bb_rt(struct w_step *w, enum intel_engine_id engine, uint32_t seqno) { igt_assert(engine == VCS1 || engine == VCS2); gem_set_domain(fd, w->bb_handle, I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); w->reloc[1].delta = VCS_SEQNO_OFFSET(engine) + sizeof(uint32_t); w->reloc[2].delta = VCS_SEQNO_OFFSET(engine) + 2 * sizeof(uint32_t); w->reloc[3].delta = VCS_SEQNO_OFFSET(engine) + 3 * sizeof(uint32_t); *w->latch_value = seqno; *w->latch_address = w->reloc[3].presumed_offset + w->reloc[3].delta; *w->rt0_value = *REG(RCS_TIMESTAMP); *w->rt0_address = w->reloc[1].presumed_offset + w->reloc[1].delta; *w->rt1_address = w->reloc[2].presumed_offset + w->reloc[2].delta; /* If not using NO_RELOC, force the relocations */ if (!(w->eb.flags & I915_EXEC_NO_RELOC)) { w->reloc[1].presumed_offset = -1; w->reloc[2].presumed_offset = -1; } } static void w_sync_to(struct workload *wrk, struct w_step *w, int target) { if (target < 0) target = wrk->nr_steps + target; igt_assert(target < wrk->nr_steps); while (wrk->steps[target].type != BATCH) { if (--target < 0) target = wrk->nr_steps + target; } igt_assert(target < wrk->nr_steps); igt_assert(wrk->steps[target].type == BATCH); gem_sync(fd, wrk->steps[target].obj[0].handle); } static void init_status_page(struct workload *wrk) { struct drm_i915_gem_exec_object2 obj[2] = {}; struct drm_i915_gem_relocation_entry reloc[4] = {}; struct drm_i915_gem_execbuffer2 eb = { .buffer_count = 2, .buffers_ptr = to_user_pointer(obj) }; uint32_t *base; /* Want to make sure that the balancer has a reasonable view of * the background busyness of each engine. To do that we occasionally * send a dummy batch down the pipeline. */ if (!wrk->status_page_handle) return; obj[0].handle = wrk->status_page_handle; /* As the expected offset is untracked, do a quick nop to query it */ obj[1].handle = gem_create(fd, 4096); base = gem_mmap__wc(fd, obj[1].handle, 0, 4096, PROT_WRITE); gem_set_domain(fd, obj[1].handle, I915_GEM_DOMAIN_WC, I915_GEM_DOMAIN_WC); *base = MI_BATCH_BUFFER_END; gem_execbuf(fd, &eb); obj[1].relocs_ptr = to_user_pointer(reloc); obj[1].relocation_count = ARRAY_SIZE(reloc); reloc[0].offset = sizeof(uint32_t); reloc[1].offset = 5 * sizeof(uint32_t); reloc[2].offset = 10 * sizeof(uint32_t); reloc[3].offset = 13 * sizeof(uint32_t); for (int engine = VCS1; engine <= VCS2; engine++) { uint32_t *cs = base + engine * 128 / sizeof(*cs); uint64_t addr; reloc[0].delta = VCS_SEQNO_OFFSET(engine); reloc[0].presumed_offset = obj[0].offset; addr = reloc[0].presumed_offset + reloc[0].delta; *cs++ = MI_STORE_DWORD_IMM; *cs++ = addr; *cs++ = addr >> 32; *cs++ = ++wrk->seqno[engine]; reloc[1].delta = VCS_SEQNO_OFFSET(engine) + sizeof(uint32_t); reloc[1].presumed_offset = obj[0].offset; addr = reloc[1].presumed_offset + reloc[1].delta; *cs++ = MI_STORE_DWORD_IMM; *cs++ = addr; *cs++ = addr >> 32; *cs++ = *REG(RCS_TIMESTAMP); reloc[2].delta = VCS_SEQNO_OFFSET(engine) + 2*sizeof(uint32_t); reloc[2].presumed_offset = obj[0].offset; addr = reloc[2].presumed_offset + reloc[2].delta; *cs++ = 0x24 << 23 | 2; /* MI_STORE_REG_MEM */ *cs++ = RCS_TIMESTAMP; *cs++ = addr; *cs++ = addr >> 32; reloc[3].delta = VCS_SEQNO_OFFSET(engine) + 3*sizeof(uint32_t); reloc[3].presumed_offset = obj[0].offset; addr = reloc[3].presumed_offset + reloc[3].delta; *cs++ = MI_STORE_DWORD_IMM; *cs++ = addr; *cs++ = addr >> 32; *cs++ = wrk->seqno[engine]; *cs++ = MI_BATCH_BUFFER_END; eb.flags = eb_engine_map[engine]; eb.flags |= I915_EXEC_HANDLE_LUT; eb.flags |= I915_EXEC_NO_RELOC; eb.batch_start_offset = 128 * engine; gem_execbuf(fd, &eb); } munmap(base, 4096); gem_close(fd, obj[1].handle); } static void run_workload(unsigned int id, struct workload *wrk, bool background, int pipe_fd, const struct workload_balancer *balancer, unsigned int repeat, unsigned int flags) { struct timespec t_start, t_end; struct w_step *w; bool run = true; int throttle = -1; int qd_throttle = -1; int i, j; clock_gettime(CLOCK_MONOTONIC, &t_start); hars_petruska_f54_1_random_seed((flags & SYNCEDCLIENTS) ? 0 : id); for (j = 0; run && (background || j < repeat); j++) { clock_gettime(CLOCK_MONOTONIC, &wrk->repeat_start); init_status_page(wrk); for (i = 0, w = wrk->steps; run && (i < wrk->nr_steps); i++, w++) { enum intel_engine_id engine = w->engine; int do_sleep = 0; if (w->type == DELAY) { do_sleep = w->wait; } else if (w->type == PERIOD) { struct timespec now; clock_gettime(CLOCK_MONOTONIC, &now); do_sleep = w->wait - elapsed_us(&wrk->repeat_start, &now); if (do_sleep < 0) { if (verbose > 1) { printf("%u: Dropped period @ %u/%u (%dus late)!\n", id, j, i, do_sleep); continue; } } } else if (w->type == SYNC) { unsigned int s_idx = i + w->wait; igt_assert(i > 0 && i < wrk->nr_steps); igt_assert(wrk->steps[s_idx].type == BATCH); gem_sync(fd, wrk->steps[s_idx].obj[0].handle); continue; } else if (w->type == THROTTLE) { throttle = w->wait; continue; } else if (w->type == QD_THROTTLE) { qd_throttle = w->wait; continue; } if (do_sleep) { usleep(do_sleep); continue; } wrk->nr_bb[engine]++; if (engine == VCS && balancer) { engine = balancer->balance(balancer, wrk, w); wrk->nr_bb[engine]++; eb_update_flags(w, engine, flags); if (flags & SEQNO) update_bb_seqno(w, engine, ++wrk->seqno[engine]); if (flags & RT) update_bb_rt(w, engine, wrk->seqno[engine]); } if (w->duration.min != w->duration.max) { unsigned int d = get_duration(&w->duration); unsigned long offset; offset = ALIGN(w->bb_sz - get_bb_sz(d), 2 * sizeof(uint32_t)); w->eb.batch_start_offset = offset; } if (throttle > 0) w_sync_to(wrk, w, i - throttle); gem_execbuf(fd, &w->eb); igt_list_add_tail(&w->rq_link, &wrk->requests[engine]); wrk->nrequest[engine]++; if (pipe_fd >= 0) { struct pollfd fds; fds.fd = pipe_fd; fds.events = POLLHUP; if (poll(&fds, 1, 0)) { run = false; break; } } if (w->wait) gem_sync(fd, w->obj[0].handle); if (qd_throttle > 0) { while (wrk->nrequest[engine] > qd_throttle) { struct w_step *s; s = igt_list_first_entry(&wrk->requests[engine], s, rq_link); gem_sync(fd, s->obj[0].handle); igt_list_del(&s->rq_link); wrk->nrequest[engine]--; } } } } for (i = 0; i < NUM_ENGINES; i++) { if (!wrk->nrequest[i]) continue; w = igt_list_last_entry(&wrk->requests[i], w, rq_link); gem_sync(fd, w->obj[0].handle); } clock_gettime(CLOCK_MONOTONIC, &t_end); if (verbose > 1) { double t = elapsed(&t_start, &t_end); if (!balancer) printf("%c%u: %.3fs elapsed (%.3f workloads/s)\n", background ? ' ' : '*', id, t, repeat / t); else if (!balancer->get_qd) printf("%c%u: %.3fs elapsed (%.3f workloads/s). %lu (%lu + %lu) total VCS batches.\n", background ? ' ' : '*', id, t, repeat / t, wrk->nr_bb[VCS], wrk->nr_bb[VCS1], wrk->nr_bb[VCS2]); else printf("%c%u: %.3fs elapsed (%.3f workloads/s). %lu (%lu + %lu) total VCS batches. Average queue depths %.3f, %.3f.\n", background ? ' ' : '*', id, t, repeat / t, wrk->nr_bb[VCS], wrk->nr_bb[VCS1], wrk->nr_bb[VCS2], (double)wrk->qd_sum[VCS1] / wrk->nr_bb[VCS], (double)wrk->qd_sum[VCS2] / wrk->nr_bb[VCS]); } } static void fini_workload(struct workload *wrk) { free(wrk->steps); free(wrk); } static unsigned long calibrate_nop(unsigned int tolerance_pct) { const uint32_t bbe = 0xa << 23; unsigned int loops = 17; unsigned int usecs = nop_calibration_us; struct drm_i915_gem_exec_object2 obj = {}; struct drm_i915_gem_execbuffer2 eb = { .buffer_count = 1, .buffers_ptr = (uintptr_t)&obj}; long size, last_size; struct timespec t_0, t_end; clock_gettime(CLOCK_MONOTONIC, &t_0); size = 256 * 1024; do { struct timespec t_start; obj.handle = gem_create(fd, size); gem_write(fd, obj.handle, size - sizeof(bbe), &bbe, sizeof(bbe)); gem_execbuf(fd, &eb); gem_sync(fd, obj.handle); clock_gettime(CLOCK_MONOTONIC, &t_start); for (int loop = 0; loop < loops; loop++) gem_execbuf(fd, &eb); gem_sync(fd, obj.handle); clock_gettime(CLOCK_MONOTONIC, &t_end); gem_close(fd, obj.handle); last_size = size; size = loops * size / elapsed(&t_start, &t_end) / 1e6 * usecs; size = ALIGN(size, sizeof(uint32_t)); } while (elapsed(&t_0, &t_end) < 5 || abs(size - last_size) > (size * tolerance_pct / 100)); return size / sizeof(uint32_t); } static void print_help(void) { puts( "Usage: gem_wsim [OPTIONS]\n" "\n" "Runs a simulated workload on the GPU.\n" "When ran without arguments performs a GPU calibration result of which needs\n" "to be provided when running the simulation in subsequent invocations.\n" "\n" "Options:\n" " -h This text.\n" " -q Be quiet - do not output anything to stdout.\n" " -n Nop calibration value.\n" " -t Nop calibration tolerance percentage.\n" " Use when there is a difficulty obtaining calibration\n" " with the default settings.\n" " -w Filename or a workload descriptor.\n" " Can be given multiple times.\n" " -W Filename or a master workload descriptor.\n" " Only one master workload can be optinally specified\n" " in which case all other workloads become background\n" " ones and run as long as the master.\n" " -r How many times to emit the workload.\n" " -c Fork N clients emitting the workload simultaneously.\n" " -x Swap VCS1 and VCS2 engines in every other client.\n" " -b Load balancing to use. (0: rr, 1: qd, 2: rt, 3: rtr)\n" " Balancers can be specified either as names or as their\n" " id numbers as listed above.\n" " -2 Remap VCS2 to BCS.\n" " -R Round-robin initial VCS assignment per client.\n" " -S Synchronize the sequence of random batch durations\n" " between clients.\n" ); } static char *load_workload_descriptor(char *filename) { struct stat sbuf; char *buf; int infd, ret, i; ssize_t len; ret = stat(filename, &sbuf); if (ret || !S_ISREG(sbuf.st_mode)) return filename; igt_assert(sbuf.st_size < 1024 * 1024); /* Just so. */ buf = malloc(sbuf.st_size); igt_assert(buf); infd = open(filename, O_RDONLY); igt_assert(infd >= 0); len = read(infd, buf, sbuf.st_size); igt_assert(len == sbuf.st_size); close(infd); for (i = 0; i < len; i++) { if (buf[i] == '\n') buf[i] = ','; } len--; while (buf[len] == ',') buf[len--] = 0; return buf; } static char ** add_workload_arg(char **w_args, unsigned int nr_args, char *w_arg) { w_args = realloc(w_args, sizeof(char *) * nr_args); igt_assert(w_args); w_args[nr_args - 1] = w_arg; return w_args; } static int parse_balancing_mode(char *str) { const char *modes[] = { "rr", "qd", "rt", "rtr" , "rtavg" }; int mode = -1; unsigned int i; for (i = 0; i < ARRAY_SIZE(modes); i++) { if (!strcasecmp(str, modes[i])) { mode = i; break; } } return mode; } static void init_clocks(void) { struct timespec t_start, t_end; uint32_t rcs_start, rcs_end; double overhead, t; intel_register_access_init(intel_get_pci_device(), false, fd); if (verbose <= 1) return; clock_gettime(CLOCK_MONOTONIC, &t_start); for (int i = 0; i < 100; i++) rcs_start = *REG(RCS_TIMESTAMP); clock_gettime(CLOCK_MONOTONIC, &t_end); overhead = 2 * elapsed(&t_start, &t_end) / 100; clock_gettime(CLOCK_MONOTONIC, &t_start); for (int i = 0; i < 100; i++) clock_gettime(CLOCK_MONOTONIC, &t_end); clock_gettime(CLOCK_MONOTONIC, &t_end); overhead += elapsed(&t_start, &t_end) / 100; clock_gettime(CLOCK_MONOTONIC, &t_start); rcs_start = *REG(RCS_TIMESTAMP); usleep(100); rcs_end = *REG(RCS_TIMESTAMP); clock_gettime(CLOCK_MONOTONIC, &t_end); t = elapsed(&t_start, &t_end) - overhead; printf("%d cycles in %.1fus, i.e. 1024 cycles takes %1.fus\n", rcs_end - rcs_start, 1e6*t, 1024e6 * t / (rcs_end - rcs_start)); } int main(int argc, char **argv) { unsigned int repeat = 1; unsigned int clients = 1; unsigned int flags = 0; struct timespec t_start, t_end; struct workload **w, **wrk = NULL; unsigned int nr_w_args = 0; int master_workload = -1; char **w_args = NULL; unsigned int tolerance_pct = 1; const struct workload_balancer *balancer = NULL; char *endptr = NULL; double t; int i, c; fd = drm_open_driver(DRIVER_INTEL); init_clocks(); while ((c = getopt(argc, argv, "q2RSc:n:r:xw:W:t:b:vh")) != -1) { switch (c) { case 'W': if (master_workload >= 0) { if (verbose) fprintf(stderr, "Only one master workload can be given!\n"); return 1; } master_workload = nr_w_args; /* Fall through */ case 'w': w_args = add_workload_arg(w_args, ++nr_w_args, optarg); break; case 'c': clients = strtol(optarg, NULL, 0); break; case 't': tolerance_pct = strtol(optarg, NULL, 0); break; case 'n': nop_calibration = strtol(optarg, NULL, 0); break; case 'r': repeat = strtol(optarg, NULL, 0); break; case 'q': verbose = 0; break; case 'v': verbose++; break; case 'x': flags |= SWAPVCS; break; case '2': flags |= VCS2REMAP; break; case 'R': flags |= INITVCSRR; break; case 'S': flags |= SYNCEDCLIENTS; break; case 'b': i = parse_balancing_mode(optarg); if (i < 0) { i = strtol(optarg, &endptr, 0); if (endptr && *endptr) i = -1; } switch (i) { case 0: if (verbose > 1) printf("Using rr balancer\n"); balancer = &rr_balancer; flags |= BALANCE; break; case 1: if (verbose > 1) printf("Using qd balancer\n"); igt_assert(intel_gen(intel_get_drm_devid(fd)) >= 8); balancer = &qd_balancer; flags |= SEQNO | BALANCE; break; case 2: if (verbose > 1) printf("Using rt balancer\n"); igt_assert(intel_gen(intel_get_drm_devid(fd)) >= 8); balancer = &rt_balancer; flags |= SEQNO | BALANCE | RT; break; case 3: if (verbose > 1) printf("Using rtr balancer\n"); igt_assert(intel_gen(intel_get_drm_devid(fd)) >= 8); balancer = &rtr_balancer; flags |= SEQNO | BALANCE | RT; break; case 4: if (verbose > 1) printf("Using rtavg balancer\n"); igt_assert(intel_gen(intel_get_drm_devid(fd)) >= 8); balancer = &rtavg_balancer; flags |= SEQNO | BALANCE | RT; break; default: if (verbose) fprintf(stderr, "Unknown balancing mode '%s'!\n", optarg); return 1; } break; case 'h': print_help(); return 0; default: return 1; } } if (!nop_calibration) { if (verbose > 1) printf("Calibrating nop delay with %u%% tolerance...\n", tolerance_pct); nop_calibration = calibrate_nop(tolerance_pct); if (verbose) printf("Nop calibration for %uus delay is %lu.\n", nop_calibration_us, nop_calibration); return 0; } if (!nr_w_args) { if (verbose) fprintf(stderr, "No workload descriptor(s)!\n"); return 1; } if (nr_w_args > 1 && clients > 1) { if (verbose) fprintf(stderr, "Cloned clients cannot be combined with multiple workloads!\n"); return 1; } wrk = calloc(nr_w_args, sizeof(*wrk)); igt_assert(wrk); for (i = 0; i < nr_w_args; i++) { w_args[i] = load_workload_descriptor(w_args[i]); if (!w_args[i]) { if (verbose) fprintf(stderr, "Failed to load workload descriptor %u!\n", i); return 1; } wrk[i] = parse_workload(w_args[i], flags); if (!wrk[i]) { if (verbose) fprintf(stderr, "Failed to parse workload %u!\n", i); return 1; } } if (verbose > 1) { printf("Using %lu nop calibration for %uus delay.\n", nop_calibration, nop_calibration_us); if (nr_w_args > 1) clients = nr_w_args; printf("%u client%s.\n", clients, clients > 1 ? "s" : ""); if (flags & SWAPVCS) printf("Swapping VCS rings between clients.\n"); } if (master_workload >= 0 && clients == 1) master_workload = -1; w = calloc(clients, sizeof(struct workload *)); igt_assert(w); for (i = 0; i < clients; i++) { unsigned int flags_ = flags; w[i] = clone_workload(wrk[nr_w_args > 1 ? i : 0]); if (master_workload >= 0) { int ret = pipe(w[i]->pipe); igt_assert(ret == 0); } if (flags & SWAPVCS && i & 1) flags_ &= ~SWAPVCS; prepare_workload(i, w[i], flags_); } clock_gettime(CLOCK_MONOTONIC, &t_start); igt_fork(child, clients) { int pipe_fd = -1; bool background = false; if (master_workload >= 0) { close(w[child]->pipe[0]); if (child != master_workload) { pipe_fd = w[child]->pipe[1]; background = true; } else { close(w[child]->pipe[1]); } } run_workload(child, w[child], background, pipe_fd, balancer, repeat, flags); } if (master_workload >= 0) { int status = -1; pid_t pid; for (i = 0; i < clients; i++) close(w[i]->pipe[1]); pid = wait(&status); if (pid >= 0) igt_child_done(pid); for (i = 0; i < clients; i++) close(w[i]->pipe[0]); } igt_waitchildren(); clock_gettime(CLOCK_MONOTONIC, &t_end); t = elapsed(&t_start, &t_end); if (verbose) printf("%.3fs elapsed (%.3f workloads/s)\n", t, clients * repeat / t); for (i = 0; i < clients; i++) fini_workload(w[i]); free(w); for (i = 0; i < nr_w_args; i++) fini_workload(wrk[i]); free(w_args); return 0; }