drivers/vfio/pci/ism/main.c - third_party/linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * vfio-ISM driver for s390
  *
  * Copyright IBM Corp.
  */

 #include <linux/slab.h>
 #include "../vfio_pci_priv.h"

 #define ISM_VFIO_PCI_OFFSET_SHIFT   48
 #define ISM_VFIO_PCI_OFFSET_TO_INDEX(off) ((off) >> ISM_VFIO_PCI_OFFSET_SHIFT)
 #define ISM_VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << ISM_VFIO_PCI_OFFSET_SHIFT)
 #define ISM_VFIO_PCI_OFFSET_MASK (((u64)(1) << ISM_VFIO_PCI_OFFSET_SHIFT) - 1)

 /*
  * Use __zpci_load() to bypass automatic use of
  * PCI MIO instructions which are not supported on ISM devices
  */
 #define ISM_READ(size)                                                        \
 	static int ism_read##size(struct zpci_dev *zdev, int bar,             \
 				  size_t *filled, char __user *buf,           \
 				  loff_t off)                                 \
 	{                                                                     \
 		u64 req, tmp;                                                 \
 		u##size val;                                                  \
 		int ret;                                                      \
 									      \
 		req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, sizeof(val)); \
 		ret = __zpci_load(&tmp, req, off);                            \
 		if (ret)                                                      \
 			return ret;                                           \
 		val = (u##size)tmp;                                           \
 		if (copy_to_user(buf, &val, sizeof(val)))                     \
 			return -EFAULT;                                       \
 		*filled = sizeof(val);                                        \
 		return 0;						      \
 	}

 ISM_READ(64);
 ISM_READ(32);
 ISM_READ(16);
 ISM_READ(8);

 struct ism_vfio_pci_core_device {
 	struct vfio_pci_core_device core_device;
 	struct kmem_cache *store_block_cache;
 };

 static int ism_vfio_pci_open_device(struct vfio_device *core_vdev)
 {
 	struct ism_vfio_pci_core_device *ivpcd;
 	struct vfio_pci_core_device *vdev;
 	int ret;

 	ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device,
 			     core_device.vdev);
 	vdev = &ivpcd->core_device;

 	ret = vfio_pci_core_enable(vdev);
 	if (ret)
 		return ret;

 	vfio_pci_core_finish_enable(vdev);
 	return 0;
 }

 /*
  * ism_vfio_pci_do_io_r()
  *
  * On s390, kernel primitives such as ioread() and iowrite() are switched over
  * from function-handle-based PCI load/stores instructions to PCI memory-I/O (MIO)
  * loads/stores when these are available and not explicitly disabled. Since these
  * instructions cannot be used with ISM devices, ensure that classic
  * function-handle-based PCI instructions are used instead.
  */
 static ssize_t ism_vfio_pci_do_io_r(struct vfio_pci_core_device *vdev,
 				    char __user *buf, loff_t off, size_t count,
 				    int bar)
 {
 	struct zpci_dev *zdev = to_zpci(vdev->pdev);
 	ssize_t done = 0;
 	int ret;

 	while (count) {
 		size_t filled;

 		if (count >= 8 && IS_ALIGNED(off, 8)) {
 			ret = ism_read64(zdev, bar, &filled, buf, off);
 			if (ret)
 				return ret;
 		} else if (count >= 4 && IS_ALIGNED(off, 4)) {
 			ret = ism_read32(zdev, bar, &filled, buf, off);
 			if (ret)
 				return ret;
 		} else if (count >= 2 && IS_ALIGNED(off, 2)) {
 			ret = ism_read16(zdev, bar, &filled, buf, off);
 			if (ret)
 				return ret;
 		} else {
 			ret = ism_read8(zdev, bar, &filled, buf, off);
 			if (ret)
 				return ret;
 		}

 		count -= filled;
 		done += filled;
 		off += filled;
 		buf += filled;
 	}

 	return done;
 }

 /*
  * ism_vfio_pci_do_io_w()
  *
  * Ensure that the PCI store block (PCISTB) instruction is used as required by the
  * ISM device. The ISM device also uses a 256 TiB BAR 0 for write operations,
  * which requires a 48bit region address space (ISM_VFIO_PCI_OFFSET_SHIFT).
  */
 static ssize_t ism_vfio_pci_do_io_w(struct vfio_pci_core_device *vdev,
 				    char __user *buf, loff_t off, size_t count,
 				    int bar)
 {
 	struct zpci_dev *zdev = to_zpci(vdev->pdev);
 	struct ism_vfio_pci_core_device *ivpcd;
 	ssize_t ret;
 	void *data;
 	u64 req;

 	if (count > zdev->maxstbl)
 		return -EINVAL;
 	if (((off % PAGE_SIZE) + count) > PAGE_SIZE)
 		return -EINVAL;

 	ivpcd = container_of(vdev, struct ism_vfio_pci_core_device,
 			     core_device);
 	data = kmem_cache_alloc(ivpcd->store_block_cache, GFP_KERNEL);
 	if (!data)
 		return -ENOMEM;

 	if (copy_from_user(data, buf, count)) {
 		ret = -EFAULT;
 		goto out_free;
 	}

 	req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, count);
 	ret = __zpci_store_block(data, req, off);
 	if (ret)
 		goto out_free;

 	ret = count;

 out_free:
 	kmem_cache_free(ivpcd->store_block_cache, data);
 	return ret;
 }

 static ssize_t ism_vfio_pci_bar_rw(struct vfio_pci_core_device *vdev,
 				   char __user *buf, size_t count, loff_t *ppos,
 				   bool iswrite)
 {
 	int bar = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 	loff_t pos = *ppos & ISM_VFIO_PCI_OFFSET_MASK;
 	resource_size_t end;
 	ssize_t done = 0;

 	if (pci_resource_start(vdev->pdev, bar))
 		end = pci_resource_len(vdev->pdev, bar);
 	else
 		return -EINVAL;

 	if (pos >= end)
 		return -EINVAL;

 	count = min(count, (size_t)(end - pos));

 	if (iswrite)
 		done = ism_vfio_pci_do_io_w(vdev, buf, pos, count, bar);
 	else
 		done = ism_vfio_pci_do_io_r(vdev, buf, pos, count, bar);

 	if (done >= 0)
 		*ppos += done;

 	return done;
 }

 static ssize_t ism_vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
 				      char __user *buf, size_t count,
 				      loff_t *ppos, bool iswrite)
 {
 	loff_t pos = *ppos;
 	size_t done = 0;
 	int ret = 0;

 	pos &= ISM_VFIO_PCI_OFFSET_MASK;

 	while (count) {
 		/*
 		 * zPCI must not use MIO instructions for config space access,
 		 * so we can use common code path here.
 		 */
 		ret = vfio_pci_config_rw_single(vdev, buf, count, &pos, iswrite);
 		if (ret < 0)
 			return ret;

 		count -= ret;
 		done += ret;
 		buf += ret;
 		pos += ret;
 	}

 	*ppos += done;

 	return done;
 }

 static ssize_t ism_vfio_pci_rw(struct vfio_device *core_vdev, char __user *buf,
 			       size_t count, loff_t *ppos, bool iswrite)
 {
 	unsigned int index = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos);
 	struct vfio_pci_core_device *vdev;
 	int ret;

 	vdev = container_of(core_vdev, struct vfio_pci_core_device, vdev);

 	if (!count)
 		return 0;

 	switch (index) {
 	case VFIO_PCI_CONFIG_REGION_INDEX:
 		ret = ism_vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
 		break;

 	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
 		ret = ism_vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
 		break;

 	default:
 		return -EINVAL;
 	}

 	return ret;
 }

 static ssize_t ism_vfio_pci_read(struct vfio_device *core_vdev,
 				 char __user *buf, size_t count, loff_t *ppos)
 {
 	return ism_vfio_pci_rw(core_vdev, buf, count, ppos, false);
 }

 static ssize_t ism_vfio_pci_write(struct vfio_device *core_vdev,
 				  const char __user *buf, size_t count,
 				  loff_t *ppos)
 {
 	return ism_vfio_pci_rw(core_vdev, (char __user *)buf, count, ppos,
 			       true);
 }

 static int ism_vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
 					      struct vfio_region_info *info,
 					      struct vfio_info_cap *caps)
 {
 	struct vfio_pci_core_device *vdev =
 		container_of(core_vdev, struct vfio_pci_core_device, vdev);
 	struct pci_dev *pdev = vdev->pdev;

 	switch (info->index) {
 	case VFIO_PCI_CONFIG_REGION_INDEX:
 		info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index);
 		info->size = pdev->cfg_size;
 		info->flags = VFIO_REGION_INFO_FLAG_READ |
 			      VFIO_REGION_INFO_FLAG_WRITE;
 		break;
 	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
 		info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index);
 		info->size = pci_resource_len(pdev, info->index);
 		if (!info->size) {
 			info->flags = 0;
 			break;
 		}
 		info->flags = VFIO_REGION_INFO_FLAG_READ |
 			      VFIO_REGION_INFO_FLAG_WRITE;
 		break;
 	default:
 		info->offset = 0;
 		info->size = 0;
 		info->flags = 0;
 		return -EINVAL;
 	}
 	return 0;
 }

 static int ism_vfio_pci_init_dev(struct vfio_device *core_vdev)
 {
 	struct zpci_dev *zdev = to_zpci(to_pci_dev(core_vdev->dev));
 	struct ism_vfio_pci_core_device *ivpcd;
 	char cache_name[20];
 	int ret;

 	ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device,
 			     core_device.vdev);

 	snprintf(cache_name, sizeof(cache_name), "ism_sb_fid_%08x", zdev->fid);

 	ivpcd->store_block_cache =
 		kmem_cache_create(cache_name, zdev->maxstbl,
 				  (&(struct kmem_cache_args){
 					  .align = PAGE_SIZE,
 					  .useroffset = 0,
 					  .usersize = zdev->maxstbl,
 				  }),
 				  (SLAB_RECLAIM_ACCOUNT | SLAB_ACCOUNT));
 	if (!ivpcd->store_block_cache)
 		return -ENOMEM;

 	ret = vfio_pci_core_init_dev(core_vdev);
 	if (ret)
 		kmem_cache_destroy(ivpcd->store_block_cache);

 	return ret;
 }

 static void ism_vfio_pci_release_dev(struct vfio_device *core_vdev)
 {
 	struct ism_vfio_pci_core_device *ivpcd = container_of(
 		core_vdev, struct ism_vfio_pci_core_device, core_device.vdev);

 	kmem_cache_destroy(ivpcd->store_block_cache);
 	vfio_pci_core_release_dev(core_vdev);
 }

 static const struct vfio_device_ops ism_pci_ops = {
 	.name = "ism-vfio-pci",
 	.init = ism_vfio_pci_init_dev,
 	.release = ism_vfio_pci_release_dev,
 	.open_device = ism_vfio_pci_open_device,
 	.close_device = vfio_pci_core_close_device,
 	.ioctl = vfio_pci_core_ioctl,
 	.get_region_info_caps = ism_vfio_pci_ioctl_get_region_info,
 	.device_feature = vfio_pci_core_ioctl_feature,
 	.read = ism_vfio_pci_read,
 	.write = ism_vfio_pci_write,
 	.request = vfio_pci_core_request,
 	.match = vfio_pci_core_match,
 	.match_token_uuid = vfio_pci_core_match_token_uuid,
 	.bind_iommufd = vfio_iommufd_physical_bind,
 	.unbind_iommufd = vfio_iommufd_physical_unbind,
 	.attach_ioas = vfio_iommufd_physical_attach_ioas,
 	.detach_ioas = vfio_iommufd_physical_detach_ioas,
 };

 static int ism_vfio_pci_probe(struct pci_dev *pdev,
 			      const struct pci_device_id *id)
 {
 	struct ism_vfio_pci_core_device *ivpcd;
 	int ret;

 	ivpcd = vfio_alloc_device(ism_vfio_pci_core_device, core_device.vdev,
 				  &pdev->dev, &ism_pci_ops);
 	if (IS_ERR(ivpcd))
 		return PTR_ERR(ivpcd);

 	dev_set_drvdata(&pdev->dev, &ivpcd->core_device);

 	ret = vfio_pci_core_register_device(&ivpcd->core_device);
 	if (ret)
 		vfio_put_device(&ivpcd->core_device.vdev);

 	return ret;
 }

 static void ism_vfio_pci_remove(struct pci_dev *pdev)
 {
 	struct vfio_pci_core_device *core_device;
 	struct ism_vfio_pci_core_device *ivpcd;

 	core_device = dev_get_drvdata(&pdev->dev);
 	ivpcd = container_of(core_device, struct ism_vfio_pci_core_device,
 			     core_device);

 	vfio_pci_core_unregister_device(&ivpcd->core_device);
 	vfio_put_device(&ivpcd->core_device.vdev);
 }

 static const struct pci_device_id ism_device_table[] = {
 	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_IBM,
 					  PCI_DEVICE_ID_IBM_ISM) },
 	{}
 };
 MODULE_DEVICE_TABLE(pci, ism_device_table);

 static struct pci_driver ism_vfio_pci_driver = {
 	.name = KBUILD_MODNAME,
 	.id_table = ism_device_table,
 	.probe = ism_vfio_pci_probe,
 	.remove = ism_vfio_pci_remove,
 	.err_handler = &vfio_pci_core_err_handlers,
 	.driver_managed_dma = true,
 };

 module_pci_driver(ism_vfio_pci_driver);

 MODULE_LICENSE("GPL");
 MODULE_DESCRIPTION("vfio-pci variant driver for the IBM Internal Shared Memory (ISM) device");
 MODULE_AUTHOR("IBM Corporation");
	// SPDX-License-Identifier: GPL-2.0
	/*
	* vfio-ISM driver for s390
	*
	* Copyright IBM Corp.
	*/

	#include <linux/slab.h>
	#include "../vfio_pci_priv.h"

	#define ISM_VFIO_PCI_OFFSET_SHIFT 48
	#define ISM_VFIO_PCI_OFFSET_TO_INDEX(off) ((off) >> ISM_VFIO_PCI_OFFSET_SHIFT)
	#define ISM_VFIO_PCI_INDEX_TO_OFFSET(index) ((u64)(index) << ISM_VFIO_PCI_OFFSET_SHIFT)
	#define ISM_VFIO_PCI_OFFSET_MASK (((u64)(1) << ISM_VFIO_PCI_OFFSET_SHIFT) - 1)

	/*
	* Use __zpci_load() to bypass automatic use of
	* PCI MIO instructions which are not supported on ISM devices
	*/
	#define ISM_READ(size) \
	static int ism_read##size(struct zpci_dev *zdev, int bar, \
	size_t filled, char __user buf, \
	loff_t off) \
	{ \
	u64 req, tmp; \
	u##size val; \
	int ret; \
	\
	req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, sizeof(val)); \
	ret = __zpci_load(&tmp, req, off); \
	if (ret) \
	return ret; \
	val = (u##size)tmp; \
	if (copy_to_user(buf, &val, sizeof(val))) \
	return -EFAULT; \
	*filled = sizeof(val); \
	return 0; \
	}

	ISM_READ(64);
	ISM_READ(32);
	ISM_READ(16);
	ISM_READ(8);

	struct ism_vfio_pci_core_device {
	struct vfio_pci_core_device core_device;
	struct kmem_cache *store_block_cache;
	};

	static int ism_vfio_pci_open_device(struct vfio_device *core_vdev)
	{
	struct ism_vfio_pci_core_device *ivpcd;
	struct vfio_pci_core_device *vdev;
	int ret;

	ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device,
	core_device.vdev);
	vdev = &ivpcd->core_device;

	ret = vfio_pci_core_enable(vdev);
	if (ret)
	return ret;

	vfio_pci_core_finish_enable(vdev);
	return 0;
	}

	/*
	* ism_vfio_pci_do_io_r()
	*
	* On s390, kernel primitives such as ioread() and iowrite() are switched over
	* from function-handle-based PCI load/stores instructions to PCI memory-I/O (MIO)
	* loads/stores when these are available and not explicitly disabled. Since these
	* instructions cannot be used with ISM devices, ensure that classic
	* function-handle-based PCI instructions are used instead.
	*/
	static ssize_t ism_vfio_pci_do_io_r(struct vfio_pci_core_device *vdev,
	char __user *buf, loff_t off, size_t count,
	int bar)
	{
	struct zpci_dev *zdev = to_zpci(vdev->pdev);
	ssize_t done = 0;
	int ret;

	while (count) {
	size_t filled;

	if (count >= 8 && IS_ALIGNED(off, 8)) {
	ret = ism_read64(zdev, bar, &filled, buf, off);
	if (ret)
	return ret;
	} else if (count >= 4 && IS_ALIGNED(off, 4)) {
	ret = ism_read32(zdev, bar, &filled, buf, off);
	if (ret)
	return ret;
	} else if (count >= 2 && IS_ALIGNED(off, 2)) {
	ret = ism_read16(zdev, bar, &filled, buf, off);
	if (ret)
	return ret;
	} else {
	ret = ism_read8(zdev, bar, &filled, buf, off);
	if (ret)
	return ret;
	}

	count -= filled;
	done += filled;
	off += filled;
	buf += filled;
	}

	return done;
	}

	/*
	* ism_vfio_pci_do_io_w()
	*
	* Ensure that the PCI store block (PCISTB) instruction is used as required by the
	* ISM device. The ISM device also uses a 256 TiB BAR 0 for write operations,
	* which requires a 48bit region address space (ISM_VFIO_PCI_OFFSET_SHIFT).
	*/
	static ssize_t ism_vfio_pci_do_io_w(struct vfio_pci_core_device *vdev,
	char __user *buf, loff_t off, size_t count,
	int bar)
	{
	struct zpci_dev *zdev = to_zpci(vdev->pdev);
	struct ism_vfio_pci_core_device *ivpcd;
	ssize_t ret;
	void *data;
	u64 req;

	if (count > zdev->maxstbl)
	return -EINVAL;
	if (((off % PAGE_SIZE) + count) > PAGE_SIZE)
	return -EINVAL;

	ivpcd = container_of(vdev, struct ism_vfio_pci_core_device,
	core_device);
	data = kmem_cache_alloc(ivpcd->store_block_cache, GFP_KERNEL);
	if (!data)
	return -ENOMEM;

	if (copy_from_user(data, buf, count)) {
	ret = -EFAULT;
	goto out_free;
	}

	req = ZPCI_CREATE_REQ(READ_ONCE(zdev->fh), bar, count);
	ret = __zpci_store_block(data, req, off);
	if (ret)
	goto out_free;

	ret = count;

	out_free:
	kmem_cache_free(ivpcd->store_block_cache, data);
	return ret;
	}

	static ssize_t ism_vfio_pci_bar_rw(struct vfio_pci_core_device *vdev,
	char __user buf, size_t count, loff_t ppos,
	bool iswrite)
	{
	int bar = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos);
	loff_t pos = *ppos & ISM_VFIO_PCI_OFFSET_MASK;
	resource_size_t end;
	ssize_t done = 0;

	if (pci_resource_start(vdev->pdev, bar))
	end = pci_resource_len(vdev->pdev, bar);
	else
	return -EINVAL;

	if (pos >= end)
	return -EINVAL;

	count = min(count, (size_t)(end - pos));

	if (iswrite)
	done = ism_vfio_pci_do_io_w(vdev, buf, pos, count, bar);
	else
	done = ism_vfio_pci_do_io_r(vdev, buf, pos, count, bar);

	if (done >= 0)
	*ppos += done;

	return done;
	}

	static ssize_t ism_vfio_pci_config_rw(struct vfio_pci_core_device *vdev,
	char __user *buf, size_t count,
	loff_t *ppos, bool iswrite)
	{
	loff_t pos = *ppos;
	size_t done = 0;
	int ret = 0;

	pos &= ISM_VFIO_PCI_OFFSET_MASK;

	while (count) {
	/*
	* zPCI must not use MIO instructions for config space access,
	* so we can use common code path here.
	*/
	ret = vfio_pci_config_rw_single(vdev, buf, count, &pos, iswrite);
	if (ret < 0)
	return ret;

	count -= ret;
	done += ret;
	buf += ret;
	pos += ret;
	}

	*ppos += done;

	return done;
	}

	static ssize_t ism_vfio_pci_rw(struct vfio_device core_vdev, char __user buf,
	size_t count, loff_t *ppos, bool iswrite)
	{
	unsigned int index = ISM_VFIO_PCI_OFFSET_TO_INDEX(*ppos);
	struct vfio_pci_core_device *vdev;
	int ret;

	vdev = container_of(core_vdev, struct vfio_pci_core_device, vdev);

	if (!count)
	return 0;

	switch (index) {
	case VFIO_PCI_CONFIG_REGION_INDEX:
	ret = ism_vfio_pci_config_rw(vdev, buf, count, ppos, iswrite);
	break;

	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
	ret = ism_vfio_pci_bar_rw(vdev, buf, count, ppos, iswrite);
	break;

	default:
	return -EINVAL;
	}

	return ret;
	}

	static ssize_t ism_vfio_pci_read(struct vfio_device *core_vdev,
	char __user buf, size_t count, loff_t ppos)
	{
	return ism_vfio_pci_rw(core_vdev, buf, count, ppos, false);
	}

	static ssize_t ism_vfio_pci_write(struct vfio_device *core_vdev,
	const char __user *buf, size_t count,
	loff_t *ppos)
	{
	return ism_vfio_pci_rw(core_vdev, (char __user *)buf, count, ppos,
	true);
	}

	static int ism_vfio_pci_ioctl_get_region_info(struct vfio_device *core_vdev,
	struct vfio_region_info *info,
	struct vfio_info_cap *caps)
	{
	struct vfio_pci_core_device *vdev =
	container_of(core_vdev, struct vfio_pci_core_device, vdev);
	struct pci_dev *pdev = vdev->pdev;

	switch (info->index) {
	case VFIO_PCI_CONFIG_REGION_INDEX:
	info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index);
	info->size = pdev->cfg_size;
	info->flags = VFIO_REGION_INFO_FLAG_READ \|
	VFIO_REGION_INFO_FLAG_WRITE;
	break;
	case VFIO_PCI_BAR0_REGION_INDEX ... VFIO_PCI_BAR5_REGION_INDEX:
	info->offset = ISM_VFIO_PCI_INDEX_TO_OFFSET(info->index);
	info->size = pci_resource_len(pdev, info->index);
	if (!info->size) {
	info->flags = 0;
	break;
	}
	info->flags = VFIO_REGION_INFO_FLAG_READ \|
	VFIO_REGION_INFO_FLAG_WRITE;
	break;
	default:
	info->offset = 0;
	info->size = 0;
	info->flags = 0;
	return -EINVAL;
	}
	return 0;
	}

	static int ism_vfio_pci_init_dev(struct vfio_device *core_vdev)
	{
	struct zpci_dev *zdev = to_zpci(to_pci_dev(core_vdev->dev));
	struct ism_vfio_pci_core_device *ivpcd;
	char cache_name[20];
	int ret;

	ivpcd = container_of(core_vdev, struct ism_vfio_pci_core_device,
	core_device.vdev);

	snprintf(cache_name, sizeof(cache_name), "ism_sb_fid_%08x", zdev->fid);

	ivpcd->store_block_cache =
	kmem_cache_create(cache_name, zdev->maxstbl,
	(&(struct kmem_cache_args){
	.align = PAGE_SIZE,
	.useroffset = 0,
	.usersize = zdev->maxstbl,
	}),
	(SLAB_RECLAIM_ACCOUNT \| SLAB_ACCOUNT));
	if (!ivpcd->store_block_cache)
	return -ENOMEM;

	ret = vfio_pci_core_init_dev(core_vdev);
	if (ret)
	kmem_cache_destroy(ivpcd->store_block_cache);

	return ret;
	}

	static void ism_vfio_pci_release_dev(struct vfio_device *core_vdev)
	{
	struct ism_vfio_pci_core_device *ivpcd = container_of(
	core_vdev, struct ism_vfio_pci_core_device, core_device.vdev);

	kmem_cache_destroy(ivpcd->store_block_cache);
	vfio_pci_core_release_dev(core_vdev);
	}

	static const struct vfio_device_ops ism_pci_ops = {
	.name = "ism-vfio-pci",
	.init = ism_vfio_pci_init_dev,
	.release = ism_vfio_pci_release_dev,
	.open_device = ism_vfio_pci_open_device,
	.close_device = vfio_pci_core_close_device,
	.ioctl = vfio_pci_core_ioctl,
	.get_region_info_caps = ism_vfio_pci_ioctl_get_region_info,
	.device_feature = vfio_pci_core_ioctl_feature,
	.read = ism_vfio_pci_read,
	.write = ism_vfio_pci_write,
	.request = vfio_pci_core_request,
	.match = vfio_pci_core_match,
	.match_token_uuid = vfio_pci_core_match_token_uuid,
	.bind_iommufd = vfio_iommufd_physical_bind,
	.unbind_iommufd = vfio_iommufd_physical_unbind,
	.attach_ioas = vfio_iommufd_physical_attach_ioas,
	.detach_ioas = vfio_iommufd_physical_detach_ioas,
	};

	static int ism_vfio_pci_probe(struct pci_dev *pdev,
	const struct pci_device_id *id)
	{
	struct ism_vfio_pci_core_device *ivpcd;
	int ret;

	ivpcd = vfio_alloc_device(ism_vfio_pci_core_device, core_device.vdev,
	&pdev->dev, &ism_pci_ops);
	if (IS_ERR(ivpcd))
	return PTR_ERR(ivpcd);

	dev_set_drvdata(&pdev->dev, &ivpcd->core_device);

	ret = vfio_pci_core_register_device(&ivpcd->core_device);
	if (ret)
	vfio_put_device(&ivpcd->core_device.vdev);

	return ret;
	}

	static void ism_vfio_pci_remove(struct pci_dev *pdev)
	{
	struct vfio_pci_core_device *core_device;
	struct ism_vfio_pci_core_device *ivpcd;

	core_device = dev_get_drvdata(&pdev->dev);
	ivpcd = container_of(core_device, struct ism_vfio_pci_core_device,
	core_device);

	vfio_pci_core_unregister_device(&ivpcd->core_device);
	vfio_put_device(&ivpcd->core_device.vdev);
	}

	static const struct pci_device_id ism_device_table[] = {
	{ PCI_DRIVER_OVERRIDE_DEVICE_VFIO(PCI_VENDOR_ID_IBM,
	PCI_DEVICE_ID_IBM_ISM) },
	{}
	};
	MODULE_DEVICE_TABLE(pci, ism_device_table);

	static struct pci_driver ism_vfio_pci_driver = {
	.name = KBUILD_MODNAME,
	.id_table = ism_device_table,
	.probe = ism_vfio_pci_probe,
	.remove = ism_vfio_pci_remove,
	.err_handler = &vfio_pci_core_err_handlers,
	.driver_managed_dma = true,
	};

	module_pci_driver(ism_vfio_pci_driver);

	MODULE_LICENSE("GPL");
	MODULE_DESCRIPTION("vfio-pci variant driver for the IBM Internal Shared Memory (ISM) device");
	MODULE_AUTHOR("IBM Corporation");