diff options
Diffstat (limited to 'drivers/gpu/drm')
42 files changed, 322 insertions, 200 deletions
diff --git a/drivers/gpu/drm/i915/display/intel_fbc.c b/drivers/gpu/drm/i915/display/intel_fbc.c index 465dc4e97ea8..bcdffe62f3cb 100644 --- a/drivers/gpu/drm/i915/display/intel_fbc.c +++ b/drivers/gpu/drm/i915/display/intel_fbc.c @@ -605,7 +605,7 @@ static void ivb_fbc_activate(struct intel_fbc *fbc) else if (DISPLAY_VER(i915) == 9) skl_fbc_program_cfb_stride(fbc); - if (i915->ggtt.num_fences) + if (to_gt(i915)->ggtt->num_fences) snb_fbc_program_fence(fbc); intel_de_write(i915, ILK_DPFC_CONTROL(fbc->id), diff --git a/drivers/gpu/drm/i915/display/intel_fbdev.c b/drivers/gpu/drm/i915/display/intel_fbdev.c index adc3a81be9f7..41d279db2be6 100644 --- a/drivers/gpu/drm/i915/display/intel_fbdev.c +++ b/drivers/gpu/drm/i915/display/intel_fbdev.c @@ -180,7 +180,7 @@ static int intelfb_create(struct drm_fb_helper *helper, struct drm_device *dev = helper->dev; struct drm_i915_private *dev_priv = to_i915(dev); struct pci_dev *pdev = to_pci_dev(dev_priv->drm.dev); - struct i915_ggtt *ggtt = &dev_priv->ggtt; + struct i915_ggtt *ggtt = to_gt(dev_priv)->ggtt; const struct i915_ggtt_view view = { .type = I915_GGTT_VIEW_NORMAL, }; diff --git a/drivers/gpu/drm/i915/display/intel_plane_initial.c b/drivers/gpu/drm/i915/display/intel_plane_initial.c index 01ce1d72297f..e4186a0b8edb 100644 --- a/drivers/gpu/drm/i915/display/intel_plane_initial.c +++ b/drivers/gpu/drm/i915/display/intel_plane_initial.c @@ -94,7 +94,7 @@ initial_plane_vma(struct drm_i915_private *i915, goto err_obj; } - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &to_gt(i915)->ggtt->vm, NULL); if (IS_ERR(vma)) goto err_obj; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.c b/drivers/gpu/drm/i915/gem/i915_gem_context.c index 00327b750fbb..ebbac2ea0833 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.c @@ -571,10 +571,6 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, struct intel_engine_cs **siblings = NULL; intel_engine_mask_t prev_mask; - /* FIXME: This is NIY for execlists */ - if (!(intel_uc_uses_guc_submission(&to_gt(i915)->uc))) - return -ENODEV; - if (get_user(slot, &ext->engine_index)) return -EFAULT; @@ -584,6 +580,13 @@ set_proto_ctx_engines_parallel_submit(struct i915_user_extension __user *base, if (get_user(num_siblings, &ext->num_siblings)) return -EFAULT; + if (!intel_uc_uses_guc_submission(&to_gt(i915)->uc) && + num_siblings != 1) { + drm_dbg(&i915->drm, "Only 1 sibling (%d) supported in non-GuC mode\n", + num_siblings); + return -EINVAL; + } + if (slot >= set->num_engines) { drm_dbg(&i915->drm, "Invalid placement value, %d >= %d\n", slot, set->num_engines); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_context.h b/drivers/gpu/drm/i915/gem/i915_gem_context.h index babfecb17ad1..e5b0f66ea1fe 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_context.h +++ b/drivers/gpu/drm/i915/gem/i915_gem_context.h @@ -174,7 +174,7 @@ i915_gem_context_get_eb_vm(struct i915_gem_context *ctx) vm = ctx->vm; if (!vm) - vm = &ctx->i915->ggtt.vm; + vm = &to_gt(ctx->i915)->ggtt->vm; vm = i915_vm_get(vm); return vm; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c index 3a5b247be738..c0e6712651f3 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_execbuffer.c @@ -1095,7 +1095,7 @@ static inline struct i915_ggtt *cache_to_ggtt(struct reloc_cache *cache) { struct drm_i915_private *i915 = container_of(cache, struct i915_execbuffer, reloc_cache)->i915; - return &i915->ggtt; + return to_gt(i915)->ggtt; } static void reloc_cache_unmap(struct reloc_cache *cache) diff --git a/drivers/gpu/drm/i915/gem/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/i915_gem_mman.c index aaf970c37aa2..ee5ec0fd4807 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_mman.c @@ -295,7 +295,7 @@ static vm_fault_t vm_fault_gtt(struct vm_fault *vmf) struct drm_device *dev = obj->base.dev; struct drm_i915_private *i915 = to_i915(dev); struct intel_runtime_pm *rpm = &i915->runtime_pm; - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; bool write = area->vm_flags & VM_WRITE; struct i915_gem_ww_ctx ww; intel_wakeref_t wakeref; @@ -388,16 +388,16 @@ retry: assert_rpm_wakelock_held(rpm); /* Mark as being mmapped into userspace for later revocation */ - mutex_lock(&i915->ggtt.vm.mutex); + mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!i915_vma_set_userfault(vma) && !obj->userfault_count++) - list_add(&obj->userfault_link, &i915->ggtt.userfault_list); - mutex_unlock(&i915->ggtt.vm.mutex); + list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list); + mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); /* Track the mmo associated with the fenced vma */ vma->mmo = mmo; if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND) - intel_wakeref_auto(&i915->ggtt.userfault_wakeref, + intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)); if (write) { @@ -512,7 +512,7 @@ void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) * wakeref. */ wakeref = intel_runtime_pm_get(&i915->runtime_pm); - mutex_lock(&i915->ggtt.vm.mutex); + mutex_lock(&to_gt(i915)->ggtt->vm.mutex); if (!obj->userfault_count) goto out; @@ -530,7 +530,7 @@ void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj) wmb(); out: - mutex_unlock(&i915->ggtt.vm.mutex); + mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); intel_runtime_pm_put(&i915->runtime_pm, wakeref); } @@ -733,13 +733,14 @@ i915_gem_dumb_mmap_offset(struct drm_file *file, u32 handle, u64 *offset) { + struct drm_i915_private *i915 = to_i915(dev); enum i915_mmap_type mmap_type; if (HAS_LMEM(to_i915(dev))) mmap_type = I915_MMAP_TYPE_FIXED; else if (pat_enabled()) mmap_type = I915_MMAP_TYPE_WC; - else if (!i915_ggtt_has_aperture(&to_i915(dev)->ggtt)) + else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV; else mmap_type = I915_MMAP_TYPE_GTT; @@ -787,7 +788,7 @@ i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data, switch (args->flags) { case I915_MMAP_OFFSET_GTT: - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return -ENODEV; type = I915_MMAP_TYPE_GTT; break; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_pm.c b/drivers/gpu/drm/i915/gem/i915_gem_pm.c index ac56124760e1..6da68b38f00f 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_pm.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_pm.c @@ -23,7 +23,7 @@ void i915_gem_suspend(struct drm_i915_private *i915) { GEM_TRACE("%s\n", dev_name(i915->drm.dev)); - intel_wakeref_auto(&i915->ggtt.userfault_wakeref, 0); + intel_wakeref_auto(&to_gt(i915)->ggtt->userfault_wakeref, 0); flush_workqueue(i915->wq); /* diff --git a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c index cc927e49d21f..f9f68830c084 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_shrinker.c @@ -401,9 +401,9 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr I915_SHRINK_VMAPS); /* We also want to clear any cached iomaps as they wrap vmap */ - mutex_lock(&i915->ggtt.vm.mutex); + mutex_lock(&to_gt(i915)->ggtt->vm.mutex); list_for_each_entry_safe(vma, next, - &i915->ggtt.vm.bound_list, vm_link) { + &to_gt(i915)->ggtt->vm.bound_list, vm_link) { unsigned long count = vma->node.size >> PAGE_SHIFT; struct drm_i915_gem_object *obj = vma->obj; @@ -418,7 +418,7 @@ i915_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr i915_gem_object_unlock(obj); } - mutex_unlock(&i915->ggtt.vm.mutex); + mutex_unlock(&to_gt(i915)->ggtt->vm.mutex); *(unsigned long *)ptr += freed_pages; return NOTIFY_DONE; diff --git a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c index 7df50fd6cc7b..26975d857776 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_stolen.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_stolen.c @@ -71,7 +71,7 @@ void i915_gem_stolen_remove_node(struct drm_i915_private *i915, static int i915_adjust_stolen(struct drm_i915_private *i915, struct resource *dsm) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct intel_uncore *uncore = ggtt->vm.gt->uncore; struct resource *r; @@ -582,6 +582,7 @@ i915_pages_create_for_stolen(struct drm_device *dev, static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); struct sg_table *pages = i915_pages_create_for_stolen(obj->base.dev, obj->stolen->start, @@ -589,7 +590,7 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) if (IS_ERR(pages)) return PTR_ERR(pages); - dbg_poison(&to_i915(obj->base.dev)->ggtt, + dbg_poison(to_gt(i915)->ggtt, sg_dma_address(pages->sgl), sg_dma_len(pages->sgl), POISON_INUSE); @@ -602,9 +603,10 @@ static int i915_gem_object_get_pages_stolen(struct drm_i915_gem_object *obj) static void i915_gem_object_put_pages_stolen(struct drm_i915_gem_object *obj, struct sg_table *pages) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); /* Should only be called from i915_gem_object_release_stolen() */ - dbg_poison(&to_i915(obj->base.dev)->ggtt, + dbg_poison(to_gt(i915)->ggtt, sg_dma_address(pages->sgl), sg_dma_len(pages->sgl), POISON_FREE); diff --git a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c index ef4d0f7dc118..c3d432e314c9 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_tiling.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_tiling.c @@ -181,7 +181,8 @@ static int i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj, int tiling_mode, unsigned int stride) { - struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt; + struct drm_i915_private *i915 = to_i915(obj->base.dev); + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct i915_vma *vma, *vn; LIST_HEAD(unbind); int ret = 0; @@ -336,7 +337,7 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int err; - if (!dev_priv->ggtt.num_fences) + if (!to_gt(dev_priv)->ggtt->num_fences) return -EOPNOTSUPP; obj = i915_gem_object_lookup(file, args->handle); @@ -362,9 +363,9 @@ i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data, args->stride = 0; } else { if (args->tiling_mode == I915_TILING_X) - args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_x; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x; else - args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_y; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y; /* Hide bit 17 swizzling from the user. This prevents old Mesa * from aborting the application on sw fallbacks to bit 17, @@ -419,7 +420,7 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, struct drm_i915_gem_object *obj; int err = -ENOENT; - if (!dev_priv->ggtt.num_fences) + if (!to_gt(dev_priv)->ggtt->num_fences) return -EOPNOTSUPP; rcu_read_lock(); @@ -435,10 +436,10 @@ i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data, switch (args->tiling_mode) { case I915_TILING_X: - args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_x; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_x; break; case I915_TILING_Y: - args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_y; + args->swizzle_mode = to_gt(dev_priv)->ggtt->bit_6_swizzle_y; break; default: case I915_TILING_NONE: diff --git a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c index ee9612a3ee5e..8653855d808b 100644 --- a/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c +++ b/drivers/gpu/drm/i915/gem/i915_gem_ttm_move.c @@ -525,7 +525,7 @@ int i915_ttm_move(struct ttm_buffer_object *bo, bool evict, return ret; } - migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, bo->ttm, + migration_fence = __i915_ttm_move(bo, ctx, clear, dst_mem, ttm, dst_rsgt, true, &deps); i915_deps_fini(&deps); } diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c index 75947e9dada2..c08f766e6e15 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_client_blt.c @@ -543,7 +543,7 @@ static bool has_bit17_swizzle(int sw) static bool bad_swizzling(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; if (i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) return true; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c index 3f41fe5ec9d4..80d99b9c694f 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_context.c @@ -1374,7 +1374,7 @@ static int igt_ctx_readonly(void *arg) goto out_file; } - vm = ctx->vm ?: &i915->ggtt.alias->vm; + vm = ctx->vm ?: &to_gt(i915)->ggtt->alias->vm; if (!vm || !vm->has_read_only) { err = 0; goto out_file; diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c index 743e6ab2c40b..743a098facf2 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_mman.c @@ -307,7 +307,7 @@ static int igt_partial_tiling(void *arg) int tiling; int err; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; /* We want to check the page mapping and fencing of a large object @@ -320,7 +320,7 @@ static int igt_partial_tiling(void *arg) obj = huge_gem_object(i915, nreal << PAGE_SHIFT, - (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); + (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -366,10 +366,10 @@ static int igt_partial_tiling(void *arg) tile.tiling = tiling; switch (tiling) { case I915_TILING_X: - tile.swizzle = i915->ggtt.bit_6_swizzle_x; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; break; case I915_TILING_Y: - tile.swizzle = i915->ggtt.bit_6_swizzle_y; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; break; } @@ -440,7 +440,7 @@ static int igt_smoke_tiling(void *arg) IGT_TIMEOUT(end); int err; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; /* @@ -457,7 +457,7 @@ static int igt_smoke_tiling(void *arg) obj = huge_gem_object(i915, nreal << PAGE_SHIFT, - (1 + next_prime_number(i915->ggtt.vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); + (1 + next_prime_number(to_gt(i915)->ggtt->vm.total >> PAGE_SHIFT)) << PAGE_SHIFT); if (IS_ERR(obj)) return PTR_ERR(obj); @@ -486,10 +486,10 @@ static int igt_smoke_tiling(void *arg) break; case I915_TILING_X: - tile.swizzle = i915->ggtt.bit_6_swizzle_x; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_x; break; case I915_TILING_Y: - tile.swizzle = i915->ggtt.bit_6_swizzle_y; + tile.swizzle = to_gt(i915)->ggtt->bit_6_swizzle_y; break; } @@ -856,6 +856,7 @@ static int wc_check(struct drm_i915_gem_object *obj) static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) { + struct drm_i915_private *i915 = to_i915(obj->base.dev); bool no_map; if (obj->ops->mmap_offset) @@ -864,7 +865,7 @@ static bool can_mmap(struct drm_i915_gem_object *obj, enum i915_mmap_type type) return false; if (type == I915_MMAP_TYPE_GTT && - !i915_ggtt_has_aperture(&to_i915(obj->base.dev)->ggtt)) + !i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return false; i915_gem_object_lock(obj, NULL); diff --git a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c index 740ee8086a27..fe0a890775e2 100644 --- a/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c +++ b/drivers/gpu/drm/i915/gem/selftests/i915_gem_object.c @@ -43,7 +43,7 @@ static int igt_gem_huge(void *arg) obj = huge_gem_object(i915, nreal * PAGE_SIZE, - i915->ggtt.vm.total + PAGE_SIZE); + to_gt(i915)->ggtt->vm.total + PAGE_SIZE); if (IS_ERR(obj)) return PTR_ERR(obj); diff --git a/drivers/gpu/drm/i915/gt/intel_context.c b/drivers/gpu/drm/i915/gt/intel_context.c index ba083d800a08..5d0ec7c49b6a 100644 --- a/drivers/gpu/drm/i915/gt/intel_context.c +++ b/drivers/gpu/drm/i915/gt/intel_context.c @@ -79,7 +79,8 @@ static int intel_context_active_acquire(struct intel_context *ce) __i915_active_acquire(&ce->active); - if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine)) + if (intel_context_is_barrier(ce) || intel_engine_uses_guc(ce->engine) || + intel_context_is_parallel(ce)) return 0; /* Preallocate tracking nodes */ @@ -563,7 +564,6 @@ void intel_context_bind_parent_child(struct intel_context *parent, * Callers responsibility to validate that this function is used * correctly but we use GEM_BUG_ON here ensure that they do. */ - GEM_BUG_ON(!intel_engine_uses_guc(parent->engine)); GEM_BUG_ON(intel_context_is_pinned(parent)); GEM_BUG_ON(intel_context_is_child(parent)); GEM_BUG_ON(intel_context_is_pinned(child)); diff --git a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c index a69df5e9e77a..be56d0b41892 100644 --- a/drivers/gpu/drm/i915/gt/intel_execlists_submission.c +++ b/drivers/gpu/drm/i915/gt/intel_execlists_submission.c @@ -2599,6 +2599,43 @@ static void execlists_context_cancel_request(struct intel_context *ce, current->comm); } +static struct intel_context * +execlists_create_parallel(struct intel_engine_cs **engines, + unsigned int num_siblings, + unsigned int width) +{ + struct intel_context *parent = NULL, *ce, *err; + int i; + + GEM_BUG_ON(num_siblings != 1); + + for (i = 0; i < width; ++i) { + ce = intel_context_create(engines[i]); + if (IS_ERR(ce)) { + err = ce; + goto unwind; + } + + if (i == 0) + parent = ce; + else + intel_context_bind_parent_child(parent, ce); + } + + parent->parallel.fence_context = dma_fence_context_alloc(1); + + intel_context_set_nopreempt(parent); + for_each_child(parent, ce) + intel_context_set_nopreempt(ce); + + return parent; + +unwind: + if (parent) + intel_context_put(parent); + return err; +} + static const struct intel_context_ops execlists_context_ops = { .flags = COPS_HAS_INFLIGHT, @@ -2617,6 +2654,7 @@ static const struct intel_context_ops execlists_context_ops = { .reset = lrc_reset, .destroy = lrc_destroy, + .create_parallel = execlists_create_parallel, .create_virtual = execlists_create_virtual, }; diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt.c b/drivers/gpu/drm/i915/gt/intel_ggtt.c index 5263dda7f8d5..ab6c4322dc08 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt.c @@ -86,7 +86,7 @@ int i915_ggtt_init_hw(struct drm_i915_private *i915) * beyond the end of the batch buffer, across the page boundary, * and beyond the end of the GTT if we do not provide a guard. */ - ret = ggtt_init_hw(&i915->ggtt); + ret = ggtt_init_hw(to_gt(i915)->ggtt); if (ret) return ret; @@ -722,14 +722,14 @@ int i915_init_ggtt(struct drm_i915_private *i915) { int ret; - ret = init_ggtt(&i915->ggtt); + ret = init_ggtt(to_gt(i915)->ggtt); if (ret) return ret; if (INTEL_PPGTT(i915) == INTEL_PPGTT_ALIASING) { - ret = init_aliasing_ppgtt(&i915->ggtt); + ret = init_aliasing_ppgtt(to_gt(i915)->ggtt); if (ret) - cleanup_init_ggtt(&i915->ggtt); + cleanup_init_ggtt(to_gt(i915)->ggtt); } return 0; @@ -772,7 +772,7 @@ static void ggtt_cleanup_hw(struct i915_ggtt *ggtt) */ void i915_ggtt_driver_release(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; fini_aliasing_ppgtt(ggtt); @@ -787,7 +787,7 @@ void i915_ggtt_driver_release(struct drm_i915_private *i915) */ void i915_ggtt_driver_late_release(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; GEM_WARN_ON(kref_read(&ggtt->vm.resv_ref) != 1); dma_resv_fini(&ggtt->vm._resv); @@ -1208,7 +1208,7 @@ int i915_ggtt_probe_hw(struct drm_i915_private *i915) { int ret; - ret = ggtt_probe_hw(&i915->ggtt, to_gt(i915)); + ret = ggtt_probe_hw(to_gt(i915)->ggtt, to_gt(i915)); if (ret) return ret; diff --git a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c index f8948de72036..beabf3bc9b75 100644 --- a/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c +++ b/drivers/gpu/drm/i915/gt/intel_ggtt_fencing.c @@ -728,8 +728,8 @@ static void detect_bit_6_swizzle(struct i915_ggtt *ggtt) swizzle_y = I915_BIT_6_SWIZZLE_NONE; } - i915->ggtt.bit_6_swizzle_x = swizzle_x; - i915->ggtt.bit_6_swizzle_y = swizzle_y; + to_gt(i915)->ggtt->bit_6_swizzle_x = swizzle_x; + to_gt(i915)->ggtt->bit_6_swizzle_y = swizzle_y; } /* @@ -896,7 +896,7 @@ void intel_gt_init_swizzling(struct intel_gt *gt) struct intel_uncore *uncore = gt->uncore; if (GRAPHICS_VER(i915) < 5 || - i915->ggtt.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) + to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) return; intel_uncore_rmw(uncore, DISP_ARB_CTL, 0, DISP_TILE_SURFACE_SWIZZLING); diff --git a/drivers/gpu/drm/i915/gt/intel_gt.c b/drivers/gpu/drm/i915/gt/intel_gt.c index f98f0fb21efb..298ff32c8d0c 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.c +++ b/drivers/gpu/drm/i915/gt/intel_gt.c @@ -3,6 +3,7 @@ * Copyright © 2019 Intel Corporation */ +#include <drm/drm_managed.h> #include <drm/intel-gtt.h> #include "intel_gt_debugfs.h" @@ -85,9 +86,11 @@ int intel_gt_probe_lmem(struct intel_gt *gt) return 0; } -void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt) +int intel_gt_assign_ggtt(struct intel_gt *gt) { - gt->ggtt = ggtt; + gt->ggtt = drmm_kzalloc(>->i915->drm, sizeof(*gt->ggtt), GFP_KERNEL); + + return gt->ggtt ? 0 : -ENOMEM; } static const struct intel_mmio_range icl_l3bank_steering_table[] = { diff --git a/drivers/gpu/drm/i915/gt/intel_gt.h b/drivers/gpu/drm/i915/gt/intel_gt.h index 3ace129eb2af..94e1bac8c0cc 100644 --- a/drivers/gpu/drm/i915/gt/intel_gt.h +++ b/drivers/gpu/drm/i915/gt/intel_gt.h @@ -36,7 +36,7 @@ static inline struct intel_gt *huc_to_gt(struct intel_huc *huc) void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915); void __intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915); -void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt); +int intel_gt_assign_ggtt(struct intel_gt *gt); int intel_gt_probe_lmem(struct intel_gt *gt); int intel_gt_init_mmio(struct intel_gt *gt); int __must_check intel_gt_init_hw(struct intel_gt *gt); diff --git a/drivers/gpu/drm/i915/gt/intel_lrc.c b/drivers/gpu/drm/i915/gt/intel_lrc.c index b3489599e4de..84456ffeb4cd 100644 --- a/drivers/gpu/drm/i915/gt/intel_lrc.c +++ b/drivers/gpu/drm/i915/gt/intel_lrc.c @@ -1065,6 +1065,10 @@ lrc_pin(struct intel_context *ce, void lrc_unpin(struct intel_context *ce) { + if (unlikely(ce->parallel.last_rq)) { + i915_request_put(ce->parallel.last_rq); + ce->parallel.last_rq = NULL; + } check_redzone((void *)ce->lrc_reg_state - LRC_STATE_OFFSET, ce->engine); } diff --git a/drivers/gpu/drm/i915/gt/intel_region_lmem.c b/drivers/gpu/drm/i915/gt/intel_region_lmem.c index fde2dcb59809..21215a080088 100644 --- a/drivers/gpu/drm/i915/gt/intel_region_lmem.c +++ b/drivers/gpu/drm/i915/gt/intel_region_lmem.c @@ -15,7 +15,7 @@ static int init_fake_lmem_bar(struct intel_memory_region *mem) { struct drm_i915_private *i915 = mem->i915; - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; unsigned long n; int ret; @@ -131,7 +131,7 @@ intel_gt_setup_fake_lmem(struct intel_gt *gt) if (!i915->params.fake_lmem_start) return ERR_PTR(-ENODEV); - GEM_BUG_ON(i915_ggtt_has_aperture(&i915->ggtt)); + GEM_BUG_ON(i915_ggtt_has_aperture(to_gt(i915)->ggtt)); /* Your mappable aperture belongs to me now! */ mappable_end = pci_resource_len(pdev, 2); diff --git a/drivers/gpu/drm/i915/gt/selftest_reset.c b/drivers/gpu/drm/i915/gt/selftest_reset.c index 8a873f6bda7f..37c38bdd5f47 100644 --- a/drivers/gpu/drm/i915/gt/selftest_reset.c +++ b/drivers/gpu/drm/i915/gt/selftest_reset.c @@ -19,7 +19,7 @@ __igt_reset_stolen(struct intel_gt *gt, intel_engine_mask_t mask, const char *msg) { - struct i915_ggtt *ggtt = >->i915->ggtt; + struct i915_ggtt *ggtt = gt->ggtt; const struct resource *dsm = >->i915->dsm; resource_size_t num_pages, page; struct intel_engine_cs *engine; diff --git a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c index e7517206af82..9989d121127d 100644 --- a/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c +++ b/drivers/gpu/drm/i915/gt/uc/intel_guc_submission.c @@ -1349,7 +1349,8 @@ submission_disabled(struct intel_guc *guc) struct i915_sched_engine * const sched_engine = guc->sched_engine; return unlikely(!sched_engine || - !__tasklet_is_enabled(&sched_engine->tasklet)); + !__tasklet_is_enabled(&sched_engine->tasklet) || + intel_gt_is_wedged(guc_to_gt(guc))); } static void disable_submission(struct intel_guc *guc) @@ -1725,7 +1726,7 @@ void intel_guc_submission_reset_finish(struct intel_guc *guc) { /* Reset called during driver load or during wedge? */ if (unlikely(!guc_submission_initialized(guc) || - test_bit(I915_WEDGED, &guc_to_gt(guc)->reset.flags))) { + intel_gt_is_wedged(guc_to_gt(guc)))) { return; } @@ -3248,8 +3249,6 @@ static void guc_parent_context_unpin(struct intel_context *ce) GEM_BUG_ON(!intel_context_is_parent(ce)); GEM_BUG_ON(!intel_engine_is_virtual(ce->engine)); - if (ce->parallel.last_rq) - i915_request_put(ce->parallel.last_rq); unpin_guc_id(guc, ce); lrc_unpin(ce); } @@ -3979,6 +3978,11 @@ static void guc_handle_context_reset(struct intel_guc *guc, !context_blocked(ce))) { capture_error_state(guc, ce); guc_context_replay(ce); + } else { + drm_err(&guc_to_gt(guc)->i915->drm, + "Invalid GuC engine reset notificaion for 0x%04X on %s: banned = %d, blocked = %d", + ce->guc_id.id, ce->engine->name, intel_context_is_banned(ce), + context_blocked(ce)); } } diff --git a/drivers/gpu/drm/i915/gvt/dmabuf.c b/drivers/gpu/drm/i915/gvt/dmabuf.c index 8e65cd8258b9..94c3eb1586b0 100644 --- a/drivers/gpu/drm/i915/gvt/dmabuf.c +++ b/drivers/gpu/drm/i915/gvt/dmabuf.c @@ -84,7 +84,7 @@ static int vgpu_gem_get_pages( kfree(st); return ret; } - gtt_entries = (gen8_pte_t __iomem *)dev_priv->ggtt.gsm + + gtt_entries = (gen8_pte_t __iomem *)to_gt(dev_priv)->ggtt->gsm + (fb_info->start >> PAGE_SHIFT); for_each_sg(st->sgl, sg, page_num, i) { dma_addr_t dma_addr = diff --git a/drivers/gpu/drm/i915/i915_debugfs.c b/drivers/gpu/drm/i915/i915_debugfs.c index e0e052cdf8b8..6966fe08df92 100644 --- a/drivers/gpu/drm/i915/i915_debugfs.c +++ b/drivers/gpu/drm/i915/i915_debugfs.c @@ -390,9 +390,9 @@ static int i915_swizzle_info(struct seq_file *m, void *data) intel_wakeref_t wakeref; seq_printf(m, "bit6 swizzle for X-tiling = %s\n", - swizzle_string(dev_priv->ggtt.bit_6_swizzle_x)); + swizzle_string(to_gt(dev_priv)->ggtt->bit_6_swizzle_x)); seq_printf(m, "bit6 swizzle for Y-tiling = %s\n", - swizzle_string(dev_priv->ggtt.bit_6_swizzle_y)); + swizzle_string(to_gt(dev_priv)->ggtt->bit_6_swizzle_y)); if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) seq_puts(m, "L-shaped memory detected\n"); diff --git a/drivers/gpu/drm/i915/i915_driver.c b/drivers/gpu/drm/i915/i915_driver.c index 95174938b160..5f2343389b5e 100644 --- a/drivers/gpu/drm/i915/i915_driver.c +++ b/drivers/gpu/drm/i915/i915_driver.c @@ -571,6 +571,10 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) i915_perf_init(dev_priv); + ret = intel_gt_assign_ggtt(to_gt(dev_priv)); + if (ret) + goto err_perf; + ret = i915_ggtt_probe_hw(dev_priv); if (ret) goto err_perf; @@ -587,8 +591,6 @@ static int i915_driver_hw_probe(struct drm_i915_private *dev_priv) if (ret) goto err_ggtt; - intel_gt_init_hw_early(to_gt(dev_priv), &dev_priv->ggtt); - ret = intel_gt_probe_lmem(to_gt(dev_priv)); if (ret) goto err_mem_regions; @@ -1146,7 +1148,7 @@ static int i915_drm_suspend(struct drm_device *dev) /* Must be called before GGTT is suspended. */ intel_dpt_suspend(dev_priv); - i915_ggtt_suspend(&dev_priv->ggtt); + i915_ggtt_suspend(to_gt(dev_priv)->ggtt); i915_save_display(dev_priv); @@ -1270,7 +1272,7 @@ static int i915_drm_resume(struct drm_device *dev) if (ret) drm_err(&dev_priv->drm, "failed to re-enable GGTT\n"); - i915_ggtt_resume(&dev_priv->ggtt); + i915_ggtt_resume(to_gt(dev_priv)->ggtt); /* Must be called after GGTT is resumed. */ intel_dpt_resume(dev_priv); diff --git a/drivers/gpu/drm/i915/i915_drv.h b/drivers/gpu/drm/i915/i915_drv.h index 3967748ba347..2f9336302e6c 100644 --- a/drivers/gpu/drm/i915/i915_drv.h +++ b/drivers/gpu/drm/i915/i915_drv.h @@ -824,8 +824,6 @@ struct drm_i915_private { struct drm_atomic_state *modeset_restore_state; struct drm_modeset_acquire_ctx reset_ctx; - struct i915_ggtt ggtt; /* VM representing the global address space */ - struct i915_gem_mm mm; /* Kernel Modesetting */ @@ -1747,7 +1745,7 @@ static inline bool i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_objec { struct drm_i915_private *i915 = to_i915(obj->base.dev); - return i915->ggtt.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && + return to_gt(i915)->ggtt->bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && i915_gem_object_is_tiled(obj); } diff --git a/drivers/gpu/drm/i915/i915_gem.c b/drivers/gpu/drm/i915/i915_gem.c index 915bf431f320..e3730096abd9 100644 --- a/drivers/gpu/drm/i915/i915_gem.c +++ b/drivers/gpu/drm/i915/i915_gem.c @@ -88,7 +88,8 @@ int i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, struct drm_file *file) { - struct i915_ggtt *ggtt = &to_i915(dev)->ggtt; + struct drm_i915_private *i915 = to_i915(dev); + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct drm_i915_gem_get_aperture *args = data; struct i915_vma *vma; u64 pinned; @@ -289,7 +290,7 @@ static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj, bool write) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct i915_vma *vma; struct i915_gem_ww_ctx ww; int ret; @@ -350,7 +351,7 @@ static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj, struct i915_vma *vma) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; i915_gem_object_unpin_pages(obj); if (drm_mm_node_allocated(node)) { @@ -366,7 +367,7 @@ i915_gem_gtt_pread(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pread *args) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; intel_wakeref_t wakeref; struct drm_mm_node node; void __user *user_data; @@ -522,7 +523,7 @@ i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj, const struct drm_i915_gem_pwrite *args) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct intel_runtime_pm *rpm = &i915->runtime_pm; intel_wakeref_t wakeref; struct drm_mm_node node; @@ -823,7 +824,7 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) */ list_for_each_entry_safe(obj, on, - &i915->ggtt.userfault_list, userfault_link) + &to_gt(i915)->ggtt->userfault_list, userfault_link) __i915_gem_object_release_mmap_gtt(obj); /* @@ -831,8 +832,8 @@ void i915_gem_runtime_suspend(struct drm_i915_private *i915) * in use by hardware (i.e. they are pinned), we should not be powering * down! All other fences will be reacquired by the user upon waking. */ - for (i = 0; i < i915->ggtt.num_fences; i++) { - struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i]; + for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) { + struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i]; /* * Ideally we want to assert that the fence register is not @@ -873,7 +874,7 @@ i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj, u64 size, u64 alignment, u64 flags) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct i915_vma *vma; int ret; @@ -1123,7 +1124,7 @@ err_unlock: /* Minimal basic recovery for KMS */ ret = i915_ggtt_enable_hw(dev_priv); - i915_ggtt_resume(&dev_priv->ggtt); + i915_ggtt_resume(to_gt(dev_priv)->ggtt); intel_init_clock_gating(dev_priv); } @@ -1146,7 +1147,7 @@ void i915_gem_driver_unregister(struct drm_i915_private *i915) void i915_gem_driver_remove(struct drm_i915_private *dev_priv) { - intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref); + intel_wakeref_auto_fini(&to_gt(dev_priv)->ggtt->userfault_wakeref); i915_gem_suspend_late(dev_priv); intel_gt_driver_remove(to_gt(dev_priv)); diff --git a/drivers/gpu/drm/i915/i915_gem_gtt.c b/drivers/gpu/drm/i915/i915_gem_gtt.c index cd5f2348a187..2f2ba7a2955d 100644 --- a/drivers/gpu/drm/i915/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/i915_gem_gtt.c @@ -56,7 +56,7 @@ void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj, struct sg_table *pages) { struct drm_i915_private *i915 = to_i915(obj->base.dev); - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; /* XXX This does not prevent more requests being submitted! */ if (unlikely(ggtt->do_idle_maps)) @@ -103,7 +103,7 @@ int i915_gem_gtt_reserve(struct i915_address_space *vm, GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT)); GEM_BUG_ON(range_overflows(offset, size, vm->total)); - GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm); + GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm); GEM_BUG_ON(drm_mm_node_allocated(node)); node->size = size; @@ -201,7 +201,7 @@ int i915_gem_gtt_insert(struct i915_address_space *vm, GEM_BUG_ON(start >= end); GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE)); GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE)); - GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm); + GEM_BUG_ON(vm == &to_gt(vm->i915)->ggtt->alias->vm); GEM_BUG_ON(drm_mm_node_allocated(node)); if (unlikely(range_overflows(start, size, end))) diff --git a/drivers/gpu/drm/i915/i915_getparam.c b/drivers/gpu/drm/i915/i915_getparam.c index 7f80ad247bc8..5b8a2157d797 100644 --- a/drivers/gpu/drm/i915/i915_getparam.c +++ b/drivers/gpu/drm/i915/i915_getparam.c @@ -31,7 +31,7 @@ int i915_getparam_ioctl(struct drm_device *dev, void *data, value = pdev->revision; break; case I915_PARAM_NUM_FENCES_AVAIL: - value = i915->ggtt.num_fences; + value = to_gt(i915)->ggtt->num_fences; break; case I915_PARAM_HAS_OVERLAY: value = !!i915->overlay; diff --git a/drivers/gpu/drm/i915/i915_perf.c b/drivers/gpu/drm/i915/i915_perf.c index 170bba913c30..298857b69180 100644 --- a/drivers/gpu/drm/i915/i915_perf.c +++ b/drivers/gpu/drm/i915/i915_perf.c @@ -1630,8 +1630,8 @@ static int alloc_noa_wait(struct i915_perf_stream *stream) struct drm_i915_gem_object *bo; struct i915_vma *vma; const u64 delay_ticks = 0xffffffffffffffff - - intel_gt_ns_to_clock_interval(stream->perf->i915->ggtt.vm.gt, - atomic64_read(&stream->perf->noa_programming_delay)); + intel_gt_ns_to_clock_interval(to_gt(stream->perf->i915), + atomic64_read(&stream->perf->noa_programming_delay)); const u32 base = stream->engine->mmio_base; #define CS_GPR(x) GEN8_RING_CS_GPR(base, x) u32 *batch, *ts0, *cs, *jump; @@ -3542,7 +3542,7 @@ err: static u64 oa_exponent_to_ns(struct i915_perf *perf, int exponent) { - return intel_gt_clock_interval_to_ns(perf->i915->ggtt.vm.gt, + return intel_gt_clock_interval_to_ns(to_gt(perf->i915), 2ULL << exponent); } diff --git a/drivers/gpu/drm/i915/i915_vma.c b/drivers/gpu/drm/i915/i915_vma.c index 29a858c53bdd..be208a8f1ed0 100644 --- a/drivers/gpu/drm/i915/i915_vma.c +++ b/drivers/gpu/drm/i915/i915_vma.c @@ -909,30 +909,39 @@ err_st_alloc: } static struct scatterlist * -remap_pages(struct drm_i915_gem_object *obj, - unsigned int offset, unsigned int alignment_pad, - unsigned int width, unsigned int height, - unsigned int src_stride, unsigned int dst_stride, - struct sg_table *st, struct scatterlist *sg) +add_padding_pages(unsigned int count, + struct sg_table *st, struct scatterlist *sg) +{ + st->nents++; + + /* + * The DE ignores the PTEs for the padding tiles, the sg entry + * here is just a convenience to indicate how many padding PTEs + * to insert at this spot. + */ + sg_set_page(sg, NULL, count * I915_GTT_PAGE_SIZE, 0); + sg_dma_address(sg) = 0; + sg_dma_len(sg) = count * I915_GTT_PAGE_SIZE; + sg = sg_next(sg); + + return sg; +} + +static struct scatterlist * +remap_tiled_color_plane_pages(struct drm_i915_gem_object *obj, + unsigned int offset, unsigned int alignment_pad, + unsigned int width, unsigned int height, + unsigned int src_stride, unsigned int dst_stride, + struct sg_table *st, struct scatterlist *sg, + unsigned int *gtt_offset) { unsigned int row; if (!width || !height) return sg; - if (alignment_pad) { - st->nents++; - - /* - * The DE ignores the PTEs for the padding tiles, the sg entry - * here is just a convenience to indicate how many padding PTEs - * to insert at this spot. - */ - sg_set_page(sg, NULL, alignment_pad * 4096, 0); - sg_dma_address(sg) = 0; - sg_dma_len(sg) = alignment_pad * 4096; - sg = sg_next(sg); - } + if (alignment_pad) + sg = add_padding_pages(alignment_pad, st, sg); for (row = 0; row < height; row++) { unsigned int left = width * I915_GTT_PAGE_SIZE; @@ -969,18 +978,98 @@ remap_pages(struct drm_i915_gem_object *obj, if (!left) continue; + sg = add_padding_pages(left >> PAGE_SHIFT, st, sg); + } + + *gtt_offset += alignment_pad + dst_stride * height; + + return sg; +} + +static struct scatterlist * +remap_contiguous_pages(struct drm_i915_gem_object *obj, + unsigned int obj_offset, + unsigned int count, + struct sg_table *st, struct scatterlist *sg) +{ + struct scatterlist *iter; + unsigned int offset; + + iter = i915_gem_object_get_sg_dma(obj, obj_offset, &offset); + GEM_BUG_ON(!iter); + + do { + unsigned int len; + + len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), + count << PAGE_SHIFT); + sg_set_page(sg, NULL, len, 0); + sg_dma_address(sg) = + sg_dma_address(iter) + (offset << PAGE_SHIFT); + sg_dma_len(sg) = len; + st->nents++; + count -= len >> PAGE_SHIFT; + if (count == 0) + return sg; - /* - * The DE ignores the PTEs for the padding tiles, the sg entry - * here is just a conenience to indicate how many padding PTEs - * to insert at this spot. - */ - sg_set_page(sg, NULL, left, 0); - sg_dma_address(sg) = 0; - sg_dma_len(sg) = left; - sg = sg_next(sg); - } + sg = __sg_next(sg); + iter = __sg_next(iter); + offset = 0; + } while (1); +} + +static struct scatterlist * +remap_linear_color_plane_pages(struct drm_i915_gem_object *obj, + unsigned int obj_offset, unsigned int alignment_pad, + unsigned int size, + struct sg_table *st, struct scatterlist *sg, + unsigned int *gtt_offset) +{ + if (!size) + return sg; + + if (alignment_pad) + sg = add_padding_pages(alignment_pad, st, sg); + + sg = remap_contiguous_pages(obj, obj_offset, size, st, sg); + sg = sg_next(sg); + + *gtt_offset += alignment_pad + size; + + return sg; +} + +static struct scatterlist * +remap_color_plane_pages(const struct intel_remapped_info *rem_info, + struct drm_i915_gem_object *obj, + int color_plane, + struct sg_table *st, struct scatterlist *sg, + unsigned int *gtt_offset) +{ + unsigned int alignment_pad = 0; + + if (rem_info->plane_alignment) + alignment_pad = ALIGN(*gtt_offset, rem_info->plane_alignment) - *gtt_offset; + + if (rem_info->plane[color_plane].linear) + sg = remap_linear_color_plane_pages(obj, + rem_info->plane[color_plane].offset, + alignment_pad, + rem_info->plane[color_plane].size, + st, sg, + gtt_offset); + + else + sg = remap_tiled_color_plane_pages(obj, + rem_info->plane[color_plane].offset, + alignment_pad, + rem_info->plane[color_plane].width, + rem_info->plane[color_plane].height, + rem_info->plane[color_plane].src_stride, + rem_info->plane[color_plane].dst_stride, + st, sg, + gtt_offset); return sg; } @@ -1009,21 +1098,8 @@ intel_remap_pages(struct intel_remapped_info *rem_info, st->nents = 0; sg = st->sgl; - for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) { - unsigned int alignment_pad = 0; - - if (rem_info->plane_alignment) - alignment_pad = ALIGN(gtt_offset, rem_info->plane_alignment) - gtt_offset; - - sg = remap_pages(obj, - rem_info->plane[i].offset, alignment_pad, - rem_info->plane[i].width, rem_info->plane[i].height, - rem_info->plane[i].src_stride, rem_info->plane[i].dst_stride, - st, sg); - - gtt_offset += alignment_pad + - rem_info->plane[i].dst_stride * rem_info->plane[i].height; - } + for (i = 0 ; i < ARRAY_SIZE(rem_info->plane); i++) + sg = remap_color_plane_pages(rem_info, obj, i, st, sg, >t_offset); i915_sg_trim(st); @@ -1045,9 +1121,8 @@ intel_partial_pages(const struct i915_ggtt_view *view, struct drm_i915_gem_object *obj) { struct sg_table *st; - struct scatterlist *sg, *iter; + struct scatterlist *sg; unsigned int count = view->partial.size; - unsigned int offset; int ret = -ENOMEM; st = kmalloc(sizeof(*st), GFP_KERNEL); @@ -1058,34 +1133,14 @@ intel_partial_pages(const struct i915_ggtt_view *view, if (ret) goto err_sg_alloc; - iter = i915_gem_object_get_sg_dma(obj, view->partial.offset, &offset); - GEM_BUG_ON(!iter); - - sg = st->sgl; st->nents = 0; - do { - unsigned int len; - - len = min(sg_dma_len(iter) - (offset << PAGE_SHIFT), - count << PAGE_SHIFT); - sg_set_page(sg, NULL, len, 0); - sg_dma_address(sg) = - sg_dma_address(iter) + (offset << PAGE_SHIFT); - sg_dma_len(sg) = len; - st->nents++; - count -= len >> PAGE_SHIFT; - if (count == 0) { - sg_mark_end(sg); - i915_sg_trim(st); /* Drop any unused tail entries. */ + sg = remap_contiguous_pages(obj, view->partial.offset, count, st, st->sgl); - return st; - } + sg_mark_end(sg); + i915_sg_trim(st); /* Drop any unused tail entries. */ - sg = __sg_next(sg); - iter = __sg_next(iter); - offset = 0; - } while (1); + return st; err_sg_alloc: kfree(st); diff --git a/drivers/gpu/drm/i915/selftests/i915_gem.c b/drivers/gpu/drm/i915/selftests/i915_gem.c index b5576888cd78..1628b81d0a35 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem.c @@ -41,7 +41,7 @@ static int switch_to_context(struct i915_gem_context *ctx) static void trash_stolen(struct drm_i915_private *i915) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; const u64 slot = ggtt->error_capture.start; const resource_size_t size = resource_size(&i915->dsm); unsigned long page; @@ -99,7 +99,7 @@ static void igt_pm_suspend(struct drm_i915_private *i915) intel_wakeref_t wakeref; with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_suspend(&i915->ggtt); + i915_ggtt_suspend(to_gt(i915)->ggtt); i915_gem_suspend_late(i915); } } @@ -109,7 +109,7 @@ static void igt_pm_hibernate(struct drm_i915_private *i915) intel_wakeref_t wakeref; with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_suspend(&i915->ggtt); + i915_ggtt_suspend(to_gt(i915)->ggtt); i915_gem_freeze(i915); i915_gem_freeze_late(i915); @@ -125,7 +125,7 @@ static void igt_pm_resume(struct drm_i915_private *i915) * that runtime-pm just works. */ with_intel_runtime_pm(&i915->runtime_pm, wakeref) { - i915_ggtt_resume(&i915->ggtt); + i915_ggtt_resume(to_gt(i915)->ggtt); i915_gem_resume(i915); } } diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c index 575705c3bce9..357ced0b88e7 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_gtt.c @@ -1122,7 +1122,7 @@ static int exercise_ggtt(struct drm_i915_private *i915, u64 hole_start, u64 hole_end, unsigned long end_time)) { - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; u64 hole_start, hole_end, last = 0; struct drm_mm_node *node; IGT_TIMEOUT(end_time); @@ -1182,7 +1182,7 @@ static int igt_ggtt_page(void *arg) const unsigned int count = PAGE_SIZE/sizeof(u32); I915_RND_STATE(prng); struct drm_i915_private *i915 = arg; - struct i915_ggtt *ggtt = &i915->ggtt; + struct i915_ggtt *ggtt = to_gt(i915)->ggtt; struct drm_i915_gem_object *obj; intel_wakeref_t wakeref; struct drm_mm_node tmp; @@ -1737,26 +1737,28 @@ int i915_gem_gtt_mock_selftests(void) SUBTEST(igt_gtt_insert), }; struct drm_i915_private *i915; - struct i915_ggtt *ggtt; + struct intel_gt *gt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; - ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); - if (!ggtt) { - err = -ENOMEM; + /* allocate the ggtt */ + err = intel_gt_assign_ggtt(to_gt(i915)); + if (err) goto out_put; - } - mock_init_ggtt(i915, ggtt); - err = i915_subtests(tests, ggtt); + gt = to_gt(i915); + + mock_init_ggtt(gt); + + err = i915_subtests(tests, gt->ggtt); mock_device_flush(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(ggtt); - kfree(ggtt); + mock_fini_ggtt(gt->ggtt); + out_put: mock_destroy_device(i915); return err; @@ -2114,7 +2116,7 @@ int i915_gem_gtt_live_selftests(struct drm_i915_private *i915) SUBTEST(igt_cs_tlb), }; - GEM_BUG_ON(offset_in_page(i915->ggtt.vm.total)); + GEM_BUG_ON(offset_in_page(to_gt(i915)->ggtt->vm.total)); return i915_subtests(tests, i915); } diff --git a/drivers/gpu/drm/i915/selftests/i915_request.c b/drivers/gpu/drm/i915/selftests/i915_request.c index 92a859b34190..7f66f6d299b2 100644 --- a/drivers/gpu/drm/i915/selftests/i915_request.c +++ b/drivers/gpu/drm/i915/selftests/i915_request.c @@ -843,7 +843,7 @@ static struct i915_vma *empty_batch(struct drm_i915_private *i915) intel_gt_chipset_flush(to_gt(i915)); - vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL); + vma = i915_vma_instance(obj, &to_gt(i915)->ggtt->vm, NULL); if (IS_ERR(vma)) { err = PTR_ERR(vma); goto err; diff --git a/drivers/gpu/drm/i915/selftests/i915_vma.c b/drivers/gpu/drm/i915/selftests/i915_vma.c index 5c5809dfe9b2..de37cfa4c65f 100644 --- a/drivers/gpu/drm/i915/selftests/i915_vma.c +++ b/drivers/gpu/drm/i915/selftests/i915_vma.c @@ -922,26 +922,28 @@ int i915_vma_mock_selftests(void) SUBTEST(igt_vma_partial), }; struct drm_i915_private *i915; - struct i915_ggtt *ggtt; + struct intel_gt *gt; int err; i915 = mock_gem_device(); if (!i915) return -ENOMEM; - ggtt = kmalloc(sizeof(*ggtt), GFP_KERNEL); - if (!ggtt) { - err = -ENOMEM; + /* allocate the ggtt */ + err = intel_gt_assign_ggtt(to_gt(i915)); + if (err) goto out_put; - } - mock_init_ggtt(i915, ggtt); - err = i915_subtests(tests, ggtt); + gt = to_gt(i915); + + mock_init_ggtt(gt); + + err = i915_subtests(tests, gt->ggtt); mock_device_flush(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(ggtt); - kfree(ggtt); + mock_fini_ggtt(gt->ggtt); + out_put: mock_destroy_device(i915); return err; @@ -982,7 +984,7 @@ static int igt_vma_remapped_gtt(void *arg) intel_wakeref_t wakeref; int err = 0; - if (!i915_ggtt_has_aperture(&i915->ggtt)) + if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt)) return 0; obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE); diff --git a/drivers/gpu/drm/i915/selftests/mock_gem_device.c b/drivers/gpu/drm/i915/selftests/mock_gem_device.c index 8aa7b1d33865..28a0f054009a 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gem_device.c +++ b/drivers/gpu/drm/i915/selftests/mock_gem_device.c @@ -69,7 +69,7 @@ static void mock_device_release(struct drm_device *dev) i915_gem_drain_workqueue(i915); i915_gem_drain_freed_objects(i915); - mock_fini_ggtt(&i915->ggtt); + mock_fini_ggtt(to_gt(i915)->ggtt); destroy_workqueue(i915->wq); intel_region_ttm_device_fini(i915); @@ -194,8 +194,13 @@ struct drm_i915_private *mock_gem_device(void) mock_init_contexts(i915); - mock_init_ggtt(i915, &i915->ggtt); - to_gt(i915)->vm = i915_vm_get(&i915->ggtt.vm); + /* allocate the ggtt */ + ret = intel_gt_assign_ggtt(to_gt(i915)); + if (ret) + goto err_unlock; + + mock_init_ggtt(to_gt(i915)); + to_gt(i915)->vm = i915_vm_get(&to_gt(i915)->ggtt->vm); mkwrite_device_info(i915)->platform_engine_mask = BIT(0); to_gt(i915)->info.engine_mask = BIT(0); diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.c b/drivers/gpu/drm/i915/selftests/mock_gtt.c index 1802baf80a17..7863d0f4bbd7 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.c +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.c @@ -104,12 +104,12 @@ static void mock_unbind_ggtt(struct i915_address_space *vm, { } -void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) +void mock_init_ggtt(struct intel_gt *gt) { - memset(ggtt, 0, sizeof(*ggtt)); + struct i915_ggtt *ggtt = gt->ggtt; - ggtt->vm.gt = to_gt(i915); - ggtt->vm.i915 = i915; + ggtt->vm.gt = gt; + ggtt->vm.i915 = gt->i915; ggtt->vm.is_ggtt = true; ggtt->gmadr = (struct resource) DEFINE_RES_MEM(0, 2048 * PAGE_SIZE); @@ -128,7 +128,6 @@ void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt) ggtt->vm.vma_ops.unbind_vma = mock_unbind_ggtt; i915_address_space_init(&ggtt->vm, VM_CLASS_GGTT); - to_gt(i915)->ggtt = ggtt; } void mock_fini_ggtt(struct i915_ggtt *ggtt) diff --git a/drivers/gpu/drm/i915/selftests/mock_gtt.h b/drivers/gpu/drm/i915/selftests/mock_gtt.h index e3f224f43beb..d6eb90bd7f3f 100644 --- a/drivers/gpu/drm/i915/selftests/mock_gtt.h +++ b/drivers/gpu/drm/i915/selftests/mock_gtt.h @@ -27,8 +27,9 @@ struct drm_i915_private; struct i915_ggtt; +struct intel_gt; -void mock_init_ggtt(struct drm_i915_private *i915, struct i915_ggtt *ggtt); +void mock_init_ggtt(struct intel_gt *gt); void mock_fini_ggtt(struct i915_ggtt *ggtt); struct i915_ppgtt *mock_ppgtt(struct drm_i915_private *i915, const char *name); |