drivers/gpu/drm/xe/xe_dma_buf.c - third_party/linux - Git at Google

 // SPDX-License-Identifier: MIT
 /*
  * Copyright © 2022 Intel Corporation
  */

 #include "xe_dma_buf.h"

 #include <kunit/test.h>
 #include <linux/dma-buf.h>
 #include <linux/pci-p2pdma.h>

 #include <drm/drm_device.h>
 #include <drm/drm_prime.h>
 #include <drm/ttm/ttm_tt.h>

 #include "tests/xe_test.h"
 #include "xe_bo.h"
 #include "xe_device.h"
 #include "xe_ttm_vram_mgr.h"
 #include "xe_vm.h"

 MODULE_IMPORT_NS(DMA_BUF);

 static int xe_dma_buf_attach(struct dma_buf *dmabuf,
 			     struct dma_buf_attachment *attach)
 {
 	struct drm_gem_object *obj = attach->dmabuf->priv;

 	if (attach->peer2peer &&
 	    pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
 		attach->peer2peer = false;

 	if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
 		return -EOPNOTSUPP;

 	xe_device_mem_access_get(to_xe_device(obj->dev));
 	return 0;
 }

 static void xe_dma_buf_detach(struct dma_buf *dmabuf,
 			      struct dma_buf_attachment *attach)
 {
 	struct drm_gem_object *obj = attach->dmabuf->priv;

 	xe_device_mem_access_put(to_xe_device(obj->dev));
 }

 static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
 {
 	struct drm_gem_object *obj = attach->dmabuf->priv;
 	struct xe_bo *bo = gem_to_xe_bo(obj);
 	struct xe_device *xe = xe_bo_device(bo);
 	int ret;

 	/*
 	 * For now only support pinning in TT memory, for two reasons:
 	 * 1) Avoid pinning in a placement not accessible to some importers.
 	 * 2) Pinning in VRAM requires PIN accounting which is a to-do.
 	 */
 	if (xe_bo_is_pinned(bo) && bo->ttm.resource->placement != XE_PL_TT) {
 		drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n");
 		return -EINVAL;
 	}

 	ret = xe_bo_migrate(bo, XE_PL_TT);
 	if (ret) {
 		if (ret != -EINTR && ret != -ERESTARTSYS)
 			drm_dbg(&xe->drm,
 				"Failed migrating dma-buf to TT memory: %pe\n",
 				ERR_PTR(ret));
 		return ret;
 	}

 	ret = xe_bo_pin_external(bo);
 	xe_assert(xe, !ret);

 	return 0;
 }

 static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
 {
 	struct drm_gem_object *obj = attach->dmabuf->priv;
 	struct xe_bo *bo = gem_to_xe_bo(obj);

 	xe_bo_unpin_external(bo);
 }

 static struct sg_table *xe_dma_buf_map(struct dma_buf_attachment *attach,
 				       enum dma_data_direction dir)
 {
 	struct dma_buf *dma_buf = attach->dmabuf;
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct xe_bo *bo = gem_to_xe_bo(obj);
 	struct sg_table *sgt;
 	int r = 0;

 	if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
 		return ERR_PTR(-EOPNOTSUPP);

 	if (!xe_bo_is_pinned(bo)) {
 		if (!attach->peer2peer)
 			r = xe_bo_migrate(bo, XE_PL_TT);
 		else
 			r = xe_bo_validate(bo, NULL, false);
 		if (r)
 			return ERR_PTR(r);
 	}

 	switch (bo->ttm.resource->mem_type) {
 	case XE_PL_TT:
 		sgt = drm_prime_pages_to_sg(obj->dev,
 					    bo->ttm.ttm->pages,
 					    bo->ttm.ttm->num_pages);
 		if (IS_ERR(sgt))
 			return sgt;

 		if (dma_map_sgtable(attach->dev, sgt, dir,
 				    DMA_ATTR_SKIP_CPU_SYNC))
 			goto error_free;
 		break;

 	case XE_PL_VRAM0:
 	case XE_PL_VRAM1:
 		r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo),
 					      bo->ttm.resource, 0,
 					      bo->ttm.base.size, attach->dev,
 					      dir, &sgt);
 		if (r)
 			return ERR_PTR(r);
 		break;
 	default:
 		return ERR_PTR(-EINVAL);
 	}

 	return sgt;

 error_free:
 	sg_free_table(sgt);
 	kfree(sgt);
 	return ERR_PTR(-EBUSY);
 }

 static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
 			     struct sg_table *sgt,
 			     enum dma_data_direction dir)
 {
 	struct dma_buf *dma_buf = attach->dmabuf;
 	struct xe_bo *bo = gem_to_xe_bo(dma_buf->priv);

 	if (!xe_bo_is_vram(bo)) {
 		dma_unmap_sgtable(attach->dev, sgt, dir, 0);
 		sg_free_table(sgt);
 		kfree(sgt);
 	} else {
 		xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt);
 	}
 }

 static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
 				       enum dma_data_direction direction)
 {
 	struct drm_gem_object *obj = dma_buf->priv;
 	struct xe_bo *bo = gem_to_xe_bo(obj);
 	bool reads =  (direction == DMA_BIDIRECTIONAL ||
 		       direction == DMA_FROM_DEVICE);

 	if (!reads)
 		return 0;

 	/* Can we do interruptible lock here? */
 	xe_bo_lock(bo, false);
 	(void)xe_bo_migrate(bo, XE_PL_TT);
 	xe_bo_unlock(bo);

 	return 0;
 }

 static const struct dma_buf_ops xe_dmabuf_ops = {
 	.attach = xe_dma_buf_attach,
 	.detach = xe_dma_buf_detach,
 	.pin = xe_dma_buf_pin,
 	.unpin = xe_dma_buf_unpin,
 	.map_dma_buf = xe_dma_buf_map,
 	.unmap_dma_buf = xe_dma_buf_unmap,
 	.release = drm_gem_dmabuf_release,
 	.begin_cpu_access = xe_dma_buf_begin_cpu_access,
 	.mmap = drm_gem_dmabuf_mmap,
 	.vmap = drm_gem_dmabuf_vmap,
 	.vunmap = drm_gem_dmabuf_vunmap,
 };

 struct dma_buf *xe_gem_prime_export(struct drm_gem_object *obj, int flags)
 {
 	struct xe_bo *bo = gem_to_xe_bo(obj);
 	struct dma_buf *buf;

 	if (bo->vm)
 		return ERR_PTR(-EPERM);

 	buf = drm_gem_prime_export(obj, flags);
 	if (!IS_ERR(buf))
 		buf->ops = &xe_dmabuf_ops;

 	return buf;
 }

 static struct drm_gem_object *
 xe_dma_buf_init_obj(struct drm_device *dev, struct xe_bo *storage,
 		    struct dma_buf *dma_buf)
 {
 	struct dma_resv *resv = dma_buf->resv;
 	struct xe_device *xe = to_xe_device(dev);
 	struct xe_bo *bo;
 	int ret;

 	dma_resv_lock(resv, NULL);
 	bo = ___xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
 				    0, /* Will require 1way or 2way for vm_bind */
 				    ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT);
 	if (IS_ERR(bo)) {
 		ret = PTR_ERR(bo);
 		goto error;
 	}
 	dma_resv_unlock(resv);

 	return &bo->ttm.base;

 error:
 	dma_resv_unlock(resv);
 	return ERR_PTR(ret);
 }

 static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
 {
 	struct drm_gem_object *obj = attach->importer_priv;
 	struct xe_bo *bo = gem_to_xe_bo(obj);

 	XE_WARN_ON(xe_bo_evict(bo, false));
 }

 static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
 	.allow_peer2peer = true,
 	.move_notify = xe_dma_buf_move_notify
 };

 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)

 struct dma_buf_test_params {
 	struct xe_test_priv base;
 	const struct dma_buf_attach_ops *attach_ops;
 	bool force_different_devices;
 	u32 mem_mask;
 };

 #define to_dma_buf_test_params(_priv) \
 	container_of(_priv, struct dma_buf_test_params, base)
 #endif

 struct drm_gem_object *xe_gem_prime_import(struct drm_device *dev,
 					   struct dma_buf *dma_buf)
 {
 	XE_TEST_DECLARE(struct dma_buf_test_params *test =
 			to_dma_buf_test_params
 			(xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
 	const struct dma_buf_attach_ops *attach_ops;
 	struct dma_buf_attachment *attach;
 	struct drm_gem_object *obj;
 	struct xe_bo *bo;

 	if (dma_buf->ops == &xe_dmabuf_ops) {
 		obj = dma_buf->priv;
 		if (obj->dev == dev &&
 		    !XE_TEST_ONLY(test && test->force_different_devices)) {
 			/*
 			 * Importing dmabuf exported from out own gem increases
 			 * refcount on gem itself instead of f_count of dmabuf.
 			 */
 			drm_gem_object_get(obj);
 			return obj;
 		}
 	}

 	/*
 	 * Don't publish the bo until we have a valid attachment, and a
 	 * valid attachment needs the bo address. So pre-create a bo before
 	 * creating the attachment and publish.
 	 */
 	bo = xe_bo_alloc();
 	if (IS_ERR(bo))
 		return ERR_CAST(bo);

 	attach_ops = &xe_dma_buf_attach_ops;
 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 	if (test)
 		attach_ops = test->attach_ops;
 #endif

 	attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
 	if (IS_ERR(attach)) {
 		obj = ERR_CAST(attach);
 		goto out_err;
 	}

 	/* Errors here will take care of freeing the bo. */
 	obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
 	if (IS_ERR(obj))
 		return obj;


 	get_dma_buf(dma_buf);
 	obj->import_attach = attach;
 	return obj;

 out_err:
 	xe_bo_free(bo);

 	return obj;
 }

 #if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
 #include "tests/xe_dma_buf.c"
 #endif
	// SPDX-License-Identifier: MIT
	/*
	* Copyright © 2022 Intel Corporation
	*/

	#include "xe_dma_buf.h"

	#include <kunit/test.h>
	#include <linux/dma-buf.h>
	#include <linux/pci-p2pdma.h>

	#include <drm/drm_device.h>
	#include <drm/drm_prime.h>
	#include <drm/ttm/ttm_tt.h>

	#include "tests/xe_test.h"
	#include "xe_bo.h"
	#include "xe_device.h"
	#include "xe_ttm_vram_mgr.h"
	#include "xe_vm.h"

	MODULE_IMPORT_NS(DMA_BUF);

	static int xe_dma_buf_attach(struct dma_buf *dmabuf,
	struct dma_buf_attachment *attach)
	{
	struct drm_gem_object *obj = attach->dmabuf->priv;

	if (attach->peer2peer &&
	pci_p2pdma_distance(to_pci_dev(obj->dev->dev), attach->dev, false) < 0)
	attach->peer2peer = false;

	if (!attach->peer2peer && !xe_bo_can_migrate(gem_to_xe_bo(obj), XE_PL_TT))
	return -EOPNOTSUPP;

	xe_device_mem_access_get(to_xe_device(obj->dev));
	return 0;
	}

	static void xe_dma_buf_detach(struct dma_buf *dmabuf,
	struct dma_buf_attachment *attach)
	{
	struct drm_gem_object *obj = attach->dmabuf->priv;

	xe_device_mem_access_put(to_xe_device(obj->dev));
	}

	static int xe_dma_buf_pin(struct dma_buf_attachment *attach)
	{
	struct drm_gem_object *obj = attach->dmabuf->priv;
	struct xe_bo *bo = gem_to_xe_bo(obj);
	struct xe_device *xe = xe_bo_device(bo);
	int ret;

	/*
	* For now only support pinning in TT memory, for two reasons:
	* 1) Avoid pinning in a placement not accessible to some importers.
	* 2) Pinning in VRAM requires PIN accounting which is a to-do.
	*/
	if (xe_bo_is_pinned(bo) && bo->ttm.resource->placement != XE_PL_TT) {
	drm_dbg(&xe->drm, "Can't migrate pinned bo for dma-buf pin.\n");
	return -EINVAL;
	}

	ret = xe_bo_migrate(bo, XE_PL_TT);
	if (ret) {
	if (ret != -EINTR && ret != -ERESTARTSYS)
	drm_dbg(&xe->drm,
	"Failed migrating dma-buf to TT memory: %pe\n",
	ERR_PTR(ret));
	return ret;
	}

	ret = xe_bo_pin_external(bo);
	xe_assert(xe, !ret);

	return 0;
	}

	static void xe_dma_buf_unpin(struct dma_buf_attachment *attach)
	{
	struct drm_gem_object *obj = attach->dmabuf->priv;
	struct xe_bo *bo = gem_to_xe_bo(obj);

	xe_bo_unpin_external(bo);
	}

	static struct sg_table xe_dma_buf_map(struct dma_buf_attachment attach,
	enum dma_data_direction dir)
	{
	struct dma_buf *dma_buf = attach->dmabuf;
	struct drm_gem_object *obj = dma_buf->priv;
	struct xe_bo *bo = gem_to_xe_bo(obj);
	struct sg_table *sgt;
	int r = 0;

	if (!attach->peer2peer && !xe_bo_can_migrate(bo, XE_PL_TT))
	return ERR_PTR(-EOPNOTSUPP);

	if (!xe_bo_is_pinned(bo)) {
	if (!attach->peer2peer)
	r = xe_bo_migrate(bo, XE_PL_TT);
	else
	r = xe_bo_validate(bo, NULL, false);
	if (r)
	return ERR_PTR(r);
	}

	switch (bo->ttm.resource->mem_type) {
	case XE_PL_TT:
	sgt = drm_prime_pages_to_sg(obj->dev,
	bo->ttm.ttm->pages,
	bo->ttm.ttm->num_pages);
	if (IS_ERR(sgt))
	return sgt;

	if (dma_map_sgtable(attach->dev, sgt, dir,
	DMA_ATTR_SKIP_CPU_SYNC))
	goto error_free;
	break;

	case XE_PL_VRAM0:
	case XE_PL_VRAM1:
	r = xe_ttm_vram_mgr_alloc_sgt(xe_bo_device(bo),
	bo->ttm.resource, 0,
	bo->ttm.base.size, attach->dev,
	dir, &sgt);
	if (r)
	return ERR_PTR(r);
	break;
	default:
	return ERR_PTR(-EINVAL);
	}

	return sgt;

	error_free:
	sg_free_table(sgt);
	kfree(sgt);
	return ERR_PTR(-EBUSY);
	}

	static void xe_dma_buf_unmap(struct dma_buf_attachment *attach,
	struct sg_table *sgt,
	enum dma_data_direction dir)
	{
	struct dma_buf *dma_buf = attach->dmabuf;
	struct xe_bo *bo = gem_to_xe_bo(dma_buf->priv);

	if (!xe_bo_is_vram(bo)) {
	dma_unmap_sgtable(attach->dev, sgt, dir, 0);
	sg_free_table(sgt);
	kfree(sgt);
	} else {
	xe_ttm_vram_mgr_free_sgt(attach->dev, dir, sgt);
	}
	}

	static int xe_dma_buf_begin_cpu_access(struct dma_buf *dma_buf,
	enum dma_data_direction direction)
	{
	struct drm_gem_object *obj = dma_buf->priv;
	struct xe_bo *bo = gem_to_xe_bo(obj);
	bool reads = (direction == DMA_BIDIRECTIONAL \|\|
	direction == DMA_FROM_DEVICE);

	if (!reads)
	return 0;

	/* Can we do interruptible lock here? */
	xe_bo_lock(bo, false);
	(void)xe_bo_migrate(bo, XE_PL_TT);
	xe_bo_unlock(bo);

	return 0;
	}

	static const struct dma_buf_ops xe_dmabuf_ops = {
	.attach = xe_dma_buf_attach,
	.detach = xe_dma_buf_detach,
	.pin = xe_dma_buf_pin,
	.unpin = xe_dma_buf_unpin,
	.map_dma_buf = xe_dma_buf_map,
	.unmap_dma_buf = xe_dma_buf_unmap,
	.release = drm_gem_dmabuf_release,
	.begin_cpu_access = xe_dma_buf_begin_cpu_access,
	.mmap = drm_gem_dmabuf_mmap,
	.vmap = drm_gem_dmabuf_vmap,
	.vunmap = drm_gem_dmabuf_vunmap,
	};

	struct dma_buf xe_gem_prime_export(struct drm_gem_object obj, int flags)
	{
	struct xe_bo *bo = gem_to_xe_bo(obj);
	struct dma_buf *buf;

	if (bo->vm)
	return ERR_PTR(-EPERM);

	buf = drm_gem_prime_export(obj, flags);
	if (!IS_ERR(buf))
	buf->ops = &xe_dmabuf_ops;

	return buf;
	}

	static struct drm_gem_object *
	xe_dma_buf_init_obj(struct drm_device dev, struct xe_bo storage,
	struct dma_buf *dma_buf)
	{
	struct dma_resv *resv = dma_buf->resv;
	struct xe_device *xe = to_xe_device(dev);
	struct xe_bo *bo;
	int ret;

	dma_resv_lock(resv, NULL);
	bo = ___xe_bo_create_locked(xe, storage, NULL, resv, NULL, dma_buf->size,
	0, /* Will require 1way or 2way for vm_bind */
	ttm_bo_type_sg, XE_BO_CREATE_SYSTEM_BIT);
	if (IS_ERR(bo)) {
	ret = PTR_ERR(bo);
	goto error;
	}
	dma_resv_unlock(resv);

	return &bo->ttm.base;

	error:
	dma_resv_unlock(resv);
	return ERR_PTR(ret);
	}

	static void xe_dma_buf_move_notify(struct dma_buf_attachment *attach)
	{
	struct drm_gem_object *obj = attach->importer_priv;
	struct xe_bo *bo = gem_to_xe_bo(obj);

	XE_WARN_ON(xe_bo_evict(bo, false));
	}

	static const struct dma_buf_attach_ops xe_dma_buf_attach_ops = {
	.allow_peer2peer = true,
	.move_notify = xe_dma_buf_move_notify
	};

	#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)

	struct dma_buf_test_params {
	struct xe_test_priv base;
	const struct dma_buf_attach_ops *attach_ops;
	bool force_different_devices;
	u32 mem_mask;
	};

	#define to_dma_buf_test_params(_priv) \
	container_of(_priv, struct dma_buf_test_params, base)
	#endif

	struct drm_gem_object xe_gem_prime_import(struct drm_device dev,
	struct dma_buf *dma_buf)
	{
	XE_TEST_DECLARE(struct dma_buf_test_params *test =
	to_dma_buf_test_params
	(xe_cur_kunit_priv(XE_TEST_LIVE_DMA_BUF));)
	const struct dma_buf_attach_ops *attach_ops;
	struct dma_buf_attachment *attach;
	struct drm_gem_object *obj;
	struct xe_bo *bo;

	if (dma_buf->ops == &xe_dmabuf_ops) {
	obj = dma_buf->priv;
	if (obj->dev == dev &&
	!XE_TEST_ONLY(test && test->force_different_devices)) {
	/*
	* Importing dmabuf exported from out own gem increases
	* refcount on gem itself instead of f_count of dmabuf.
	*/
	drm_gem_object_get(obj);
	return obj;
	}
	}

	/*
	* Don't publish the bo until we have a valid attachment, and a
	* valid attachment needs the bo address. So pre-create a bo before
	* creating the attachment and publish.
	*/
	bo = xe_bo_alloc();
	if (IS_ERR(bo))
	return ERR_CAST(bo);

	attach_ops = &xe_dma_buf_attach_ops;
	#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
	if (test)
	attach_ops = test->attach_ops;
	#endif

	attach = dma_buf_dynamic_attach(dma_buf, dev->dev, attach_ops, &bo->ttm.base);
	if (IS_ERR(attach)) {
	obj = ERR_CAST(attach);
	goto out_err;
	}

	/* Errors here will take care of freeing the bo. */
	obj = xe_dma_buf_init_obj(dev, bo, dma_buf);
	if (IS_ERR(obj))
	return obj;


	get_dma_buf(dma_buf);
	obj->import_attach = attach;
	return obj;

	out_err:
	xe_bo_free(bo);

	return obj;
	}

	#if IS_ENABLED(CONFIG_DRM_XE_KUNIT_TEST)
	#include "tests/xe_dma_buf.c"
	#endif