path: root/drivers/gpu/drm/i915/gem/i915_gem_vm_bind_object.c

// SPDX-License-Identifier: MIT
/*
 * Copyright © 2022 Intel Corporation
 */

#include <linux/interval_tree_generic.h>

#include <drm/drm_syncobj.h>

#include "gem/i915_gem_vm_bind.h"
#include "gt/gen8_engine_cs.h"

#include "i915_drv.h"
#include "i915_gem_gtt.h"

#define START(node) ((node)->start)
#define LAST(node) ((node)->last)

INTERVAL_TREE_DEFINE(struct i915_vma, rb, u64, __subtree_last,
		     START, LAST, static inline, i915_vm_bind_it)

#undef START
#undef LAST

/**
 * DOC: VM_BIND/UNBIND ioctls
 *
 * DRM_I915_GEM_VM_BIND/UNBIND ioctls allows UMD to bind/unbind GEM buffer
 * objects (BOs) or sections of a BOs at specified GPU virtual addresses on a
 * specified address space (VM). Multiple mappings can map to the same physical
 * pages of an object (aliasing). These mappings (also referred to as persistent
 * mappings) will be persistent across multiple GPU submissions (execbuf calls)
 * issued by the UMD, without user having to provide a list of all required
 * mappings during each submission (as required by older execbuf mode).
 *
 * The VM_BIND/UNBIND calls allow UMDs to request a timeline out fence for
 * signaling the completion of bind/unbind operation.
 *
 * VM_BIND feature is advertised to user via I915_PARAM_VM_BIND_VERSION.
 * User has to opt-in for VM_BIND mode of binding for an address space (VM)
 * during VM creation time via I915_VM_CREATE_FLAGS_USE_VM_BIND extension.
 *
 * VM_BIND/UNBIND ioctl calls executed on different CPU threads concurrently
 * are not ordered. Furthermore, parts of the VM_BIND/UNBIND operations can be
 * done asynchronously, when valid out fence is specified.
 *
 * VM_BIND locking order is as below.
 *
 * 1) Lock-A: A vm_bind mutex will protect vm_bind lists. This lock is taken in
 *    vm_bind/vm_unbind ioctl calls, in the execbuf path and while releasing the
 *    mapping.
 *
 *    In future, when GPU page faults are supported, we can potentially use a
 *    rwsem instead, so that multiple page fault handlers can take the read
 *    side lock to lookup the mapping and hence can run in parallel.
 *    The older execbuf mode of binding do not need this lock.
 *
 * 2) Lock-B: The object's dma-resv lock will protect i915_vma state and needs
 *    to be held while binding/unbinding a vma in the async worker and while
 *    updating dma-resv fence list of an object. Note that private BOs of a VM
 *    will all share a dma-resv object.
 *
 *    The future system allocator support will use the HMM prescribed locking
 *    instead.
 *
 * 3) Lock-C: Spinlock/s to protect some of the VM's lists like the list of
 *    invalidated vmas (due to eviction and userptr invalidation) etc.
 */

struct i915_vma *
i915_gem_vm_bind_lookup_vma(struct i915_address_space *vm, u64 va)
{
	struct i915_vma *vma, *temp;

	assert_vm_bind_held(vm);

	vma = i915_vm_bind_it_iter_first(&vm->va, va, va);
	/* Working around compiler error, remove later */
	if (vma)
		temp = i915_vm_bind_it_iter_next(vma, va + vma->size, -1);
	return vma;
}

void i915_gem_vm_bind_remove(struct i915_vma *vma, bool release_obj)
{
	assert_vm_bind_held(vma->vm);

	spin_lock(&vma->vm->vm_rebind_lock);
	if (!list_empty(&vma->vm_rebind_link))
		list_del_init(&vma->vm_rebind_link);
	i915_vma_set_purged(vma);
	i915_vma_set_freed(vma);
	spin_unlock(&vma->vm->vm_rebind_lock);

	if (!list_empty(&vma->vm_bind_link)) {
		list_del_init(&vma->vm_bind_link);
		list_del_init(&vma->non_priv_vm_bind_link);
		i915_vm_bind_it_remove(vma, &vma->vm->va);

		/* Release object */
		if (release_obj)
			i915_vma_put(vma);
	}
}

static int i915_vm_bind_add_fence(struct drm_file *file, struct i915_vma *vma,
				  u32 handle, u64 point)
{
	struct drm_syncobj *syncobj;

	syncobj = drm_syncobj_find(file, handle);
	if (!syncobj) {
		DRM_DEBUG("Invalid syncobj handle provided\n");
		return -ENOENT;
	}

	/*
	 * For timeline syncobjs we need to preallocate chains for
	 * later signaling.
	 */
	if (point) {
		vma->vm_bind_fence.chain_fence = dma_fence_chain_alloc();
		if (!vma->vm_bind_fence.chain_fence) {
			drm_syncobj_put(syncobj);
			return -ENOMEM;
		}
	} else {
		vma->vm_bind_fence.chain_fence = NULL;
	}

	vma->vm_bind_fence.syncobj = syncobj;
	vma->vm_bind_fence.value = point;

	return 0;
}

static void i915_vm_bind_put_fence(struct i915_vma *vma)
{
	if (!vma->vm_bind_fence.syncobj)
		return;

	drm_syncobj_put(vma->vm_bind_fence.syncobj);
	dma_fence_chain_free(vma->vm_bind_fence.chain_fence);
}

void i915_vm_bind_signal_fence(struct i915_vma *vma,
			       struct dma_fence * const fence)
{
	struct drm_syncobj *syncobj = vma->vm_bind_fence.syncobj;

	if (!syncobj)
		return;

	if (vma->vm_bind_fence.chain_fence) {
		drm_syncobj_add_point(syncobj,
				      vma->vm_bind_fence.chain_fence,
				      fence, vma->vm_bind_fence.value);
		/*
		 * The chain's ownership is transferred to the
		 * timeline.
		 */
		vma->vm_bind_fence.chain_fence = NULL;
	} else {
		drm_syncobj_replace_fence(syncobj, fence);
	}
}

int i915_gem_vm_unbind_obj(struct i915_address_space *vm,
			   struct drm_i915_gem_vm_unbind *va)
{
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma;
	int ret;

	va->start = gen8_noncanonical_addr(va->start);
	ret = i915_gem_vm_bind_lock_interruptible(vm);
	if (ret)
		return ret;

	vma = i915_gem_vm_bind_lookup_vma(vm, va->start);
	if (!vma) {
		ret = -ENOENT;
		goto out_unlock;
	}

	if (vma->size != va->length)
		ret = -EINVAL;
	else
		i915_gem_vm_bind_remove(vma, false);

out_unlock:
	i915_gem_vm_bind_unlock(vm);
	if (ret || !vma)
		return ret;

	/* Destroy vma and then release object */
	obj = vma->obj;
	ret = i915_gem_object_lock(obj, NULL);
	if (ret)
		return ret;

	i915_vma_destroy(vma);
	i915_gem_object_unlock(obj);
	i915_gem_object_put(obj);

	return 0;
}

static struct i915_vma *vm_bind_get_vma(struct i915_address_space *vm,
					struct drm_i915_gem_object *obj,
					struct drm_i915_gem_vm_bind *va)
{
	struct i915_ggtt_view view;
	struct i915_vma *vma;

	va->start = gen8_noncanonical_addr(va->start);
	vma = i915_gem_vm_bind_lookup_vma(vm, va->start);
	if (vma)
		return ERR_PTR(-EEXIST);

	view.type = I915_GGTT_VIEW_PARTIAL;
	view.partial.offset = va->offset >> PAGE_SHIFT;
	view.partial.size = va->length >> PAGE_SHIFT;
	vma = i915_vma_instance(obj, vm, &view);
	if (IS_ERR(vma))
		return vma;

	vma->start = va->start;
	vma->last = va->start + va->length - 1;
	i915_vma_set_persistent(vma);

	return vma;
}

int i915_gem_vm_bind_obj(struct i915_address_space *vm,
			 struct drm_i915_gem_vm_bind *va,
			 struct drm_file *file)
{
	struct drm_i915_gem_object *obj;
	struct i915_vma *vma = NULL;
	struct i915_gem_ww_ctx ww;
	u64 pin_flags;
	int ret = 0;

	if (!vm->vm_bind_mode)
		return -EOPNOTSUPP;

	obj = i915_gem_object_lookup(file, va->handle);
	if (!obj)
		return -ENOENT;

	if (!va->length ||
	    !IS_ALIGNED(va->offset | va->length,
			i915_gem_object_max_page_size(obj->mm.placements,
						      obj->mm.n_placements)) ||
	    range_overflows_t(u64, va->offset, va->length, obj->base.size)) {
		ret = -EINVAL;
		goto put_obj;
	}

	if (obj->priv_root && obj->priv_root != vm->root_obj) {
		ret = -EINVAL;
		goto put_obj;
	}

	if (i915_gem_object_is_userptr(obj)) {
		ret = i915_gem_object_userptr_submit_init(obj);
		if (ret)
			goto put_obj;
	}

	ret = i915_gem_vm_bind_lock_interruptible(vm);
	if (ret)
		goto put_obj;

	vma = vm_bind_get_vma(vm, obj, va);
	if (IS_ERR(vma)) {
		ret = PTR_ERR(vma);
		goto unlock_vm;
	}

	i915_gem_ww_ctx_init(&ww, true);

	if (va->fence.flags & I915_TIMELINE_FENCE_SIGNAL) {
		ret = i915_vm_bind_add_fence(file, vma, va->fence.handle,
					     va->fence.value);
		if (ret)
			goto put_vma;
	}

	pin_flags = va->start | PIN_OFFSET_FIXED | PIN_USER;
retry:
	ret = i915_gem_object_lock(vma->obj, &ww);
	if (ret)
		goto out_ww;

	ret = i915_vma_pin_ww(vma, &ww, 0, 0, pin_flags);
	if (ret)
		goto out_ww;

	/* Make it evictable */
	__i915_vma_unpin(vma);

#ifdef CONFIG_MMU_NOTIFIER
	if (i915_gem_object_is_userptr(obj)) {
		write_lock(&vm->i915->mm.notifier_lock);
		ret = i915_gem_object_userptr_submit_done(obj);
		write_unlock(&vm->i915->mm.notifier_lock);
		if (ret)
			goto out_ww;
	}
#endif

	list_add_tail(&vma->vm_bind_link, &vm->vm_bound_list);
	i915_vm_bind_it_insert(vma, &vm->va);
	if (!obj->priv_root)
		list_add_tail(&vma->non_priv_vm_bind_link,
			      &vm->non_priv_vm_bind_list);

	/* Hold object reference until vm_unbind */
	i915_gem_object_get(vma->obj);
out_ww:
	if (ret == -EDEADLK) {
		ret = i915_gem_ww_ctx_backoff(&ww);
		if (!ret)
			goto retry;
	}

	i915_vm_bind_put_fence(vma);
put_vma:
	if (ret) {
		i915_vma_set_freed(vma);
		i915_vma_destroy(vma);
	}

	i915_gem_ww_ctx_fini(&ww);
unlock_vm:
	i915_gem_vm_bind_unlock(vm);
put_obj:
	i915_gem_object_put(obj);
	return ret;
}