drivers/dma/pci-coiommu.c - third_party/linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 #include <linux/kernel.h>
 #include <linux/module.h>
 #include <linux/pci.h>
 #include <linux/init.h>
 #include <linux/sizes.h>
 #include <linux/mutex.h>
 #include <linux/coiommu_dev.h>

 #define COIOMMU_MMIO_BAR	0
 #define COIOMMU_NOTIFY_BAR	2
 #define COIOMMU_TOPOLOGY_BAR	4

 #define COIOMMU_CMD_DEACTIVATE		0
 #define COIOMMU_CMD_ACTIVATE		1
 #define COIOMMU_CMD_PARK_UNPIN		2
 #define COIOMMU_CMD_UNPARK_UNPIN	3
 #define PIN_PAGES_IN_BATCH	(1UL << 63)

 struct coiommu_mmio_info {
 	u64 dtt_addr;
 	u64 command;
 	u64 dtt_level;
 	u8 triggers[];
 };

 struct coiommu_notify_info {
 	u64 data;
 };

 struct coiommu_dev {
 	struct coiommu_mmio_info *mmio_info;
 	struct coiommu_notify_info *notify_map;
 };

 static struct pci_device_id pci_coiommu_ids[] = {
 	{ PCI_DEVICE(PCI_VENDOR_ID_COIOMMU, PCI_DEVICE_ID_COIOMMU), },
 	{ 0, }
 };
 MODULE_DEVICE_TABLE(pci, pci_coiommu_ids);

 static void coiommu_execute_cmd(u64 cmdval, void *__iomem cmd)
 {
 	/* MMIO write is not a sync write. The backend can handle this
 	 * MMIO write async so the frontend can be returned back before
 	 * this request is really completed.
 	 *
 	 * To make sure the cmd is really completed by the backend, we
 	 * can simply read the cmd MMIO again. Once the read is completed
 	 * then the previous write has been also completed.
 	 */
 	writeq(cmdval, cmd);
 	readq(cmd);
 }

 static void sync_notify(struct coiommu_dev *cidev, u64 data)
 {
 	unsigned long flags;
 	struct coiommu_notify_info *info;
 	unsigned long timeout;
 	u64 id, busy;

 	local_irq_save(flags);

 	id = smp_processor_id();
 	info = cidev->notify_map + id;

 	busy = readq((void *__iomem)info);
 	if (busy) {
 		pr_err("%s: coiommu unexpected busy on CPU%lld: notify info 0x%llx\n",
 			__func__, id, busy);
 		local_irq_restore(flags);
 		return;
 	}

 	writeq(data, (void *__iomem)info);

 	writeb(1, (void *__iomem)(cidev->mmio_info->triggers + id));

 	/*
 	 * Set a timeout in case the backend coiommu cannot complete
 	 * the request.
 	 */
 	timeout = READ_ONCE(jiffies) + msecs_to_jiffies(5000);
 	while (readq((void *__iomem)info)) {
 		if (time_after(READ_ONCE(jiffies), timeout)) {
 			pr_err("%s: coIOMMU timeout waiting for complete\n",
 				__func__);
 			break;
 		}
 	}

 	local_irq_restore(flags);
 }

 static int coiommu_execute_req(struct coiommu_dev *dev, unsigned long pfn, unsigned short bdf)
 {
 	if (!dev)
 		return -EINVAL;

 	sync_notify(dev, (pfn << 16) | bdf);

 	return 0;
 }

 static int coiommu_execute_reqs(struct coiommu_dev *dev, struct pin_pages_info *pin_info)
 {
 	if (!pin_info)
 		return -EINVAL;

 	if (!pin_info->nr_pages)
 		return 0;

 	sync_notify(dev, (__pa(pin_info)) | PIN_PAGES_IN_BATCH);
 	return 0;
 }

 static void coiommu_park_unpin(struct coiommu_dev *dev, bool park)
 {
 	u64 cmdval = park ? COIOMMU_CMD_PARK_UNPIN : COIOMMU_CMD_UNPARK_UNPIN;

 	coiommu_execute_cmd(cmdval, (void *__iomem)&dev->mmio_info->command);
 }

 static const struct coiommu_dev_ops dev_ops = {
 	.execute_request = coiommu_execute_req,
 	.execute_requests = coiommu_execute_reqs,
 	.park_unpin = coiommu_park_unpin,
 };

 static int pci_coiommu_probe(struct pci_dev *dev, const struct pci_device_id *id)
 {
 	struct coiommu_dev *cidev;
 	u64 dtt_addr, dtt_level;
 	int ret;

 	ret = pci_enable_device(dev);
 	if (ret)
 		return ret;

 	cidev = kzalloc(sizeof(struct coiommu_dev), GFP_KERNEL);
 	if (!cidev)
 		return -ENOMEM;

 	/*
 	 * MMIO bar which contains mmio infos and trigger register.
 	 * Each CPU has one byte trigger register to notify backend about
 	 * the action. So Bar length should be longer than the mmio info size
 	 * plus the number of possible cpus.
 	 */
 	if (pci_resource_len(dev, COIOMMU_MMIO_BAR) <
 		(sizeof(struct coiommu_mmio_info) + num_possible_cpus())) {
 		dev_err(&dev->dev, "Invalid bar%d length 0x%llx\n",
 				COIOMMU_MMIO_BAR,
 				pci_resource_len(dev, COIOMMU_MMIO_BAR));
 		ret = -ENXIO;
 		goto free_dev;
 	}

 	cidev->mmio_info = pci_iomap(dev, COIOMMU_MMIO_BAR,
 				     pci_resource_len(dev, COIOMMU_MMIO_BAR));
 	if (!cidev->mmio_info) {
 		ret = -ENXIO;
 		goto free_dev;
 	}

 	/*
 	 * Notification bar. Each CPU has a 8byte register to hold the
 	 * GFP pin info. So Bar length should be longer than multiply the
 	 * number of possible cpus with 8.
 	 */
 	if (pci_resource_len(dev, COIOMMU_NOTIFY_BAR) <
 			num_possible_cpus() * 8) {
 		dev_err(&dev->dev, "Invalid bar%d length 0x%llx\n",
 				COIOMMU_NOTIFY_BAR,
 				pci_resource_len(dev, COIOMMU_NOTIFY_BAR));
 		ret = -ENXIO;
 		goto free_mmio;
 	}

 	cidev->notify_map = pci_iomap(dev, COIOMMU_NOTIFY_BAR,
 				pci_resource_len(dev, COIOMMU_NOTIFY_BAR));
 	if (!cidev->notify_map) {
 		ret = -ENXIO;
 		goto free_mmio;
 	}

 	ret = coiommu_enable_dtt(&dtt_addr, &dtt_level);
 	if (ret) {
 		dev_err(&dev->dev, "Failed to setup DTT\n");
 		goto free_notify;
 	}

 	/*
 	 * Set the dtt_addr and dtt_level to the backend. The subsequent
 	 * activate command takes care synchronizing these async writes.
 	 */
 	writeq(dtt_addr, (void *__iomem)&cidev->mmio_info->dtt_addr);
 	writeq(dtt_level, (void *__iomem)&cidev->mmio_info->dtt_level);

 	coiommu_execute_cmd(COIOMMU_CMD_ACTIVATE,
 			(void *__iomem)&cidev->mmio_info->command);

 	/* The last step to enable coiommu */
 	ret = coiommu_setup_dev_ops(&dev_ops, cidev);
 	if (ret)
 		goto deactivate;

 	dev_set_drvdata(&dev->dev, cidev);

 	return 0;

 deactivate:
 	coiommu_execute_cmd(COIOMMU_CMD_DEACTIVATE,
 			(void *__iomem)&cidev->mmio_info->command);
 	coiommu_disable_dtt();
 free_notify:
 	pci_iounmap(dev, cidev->notify_map);
 free_mmio:
 	pci_iounmap(dev, cidev->mmio_info);
 free_dev:
 	kfree(cidev);
 	return ret;
 }

 static void pci_coiommu_remove(struct pci_dev *dev)
 {
 	struct coiommu_dev *cidev = dev_get_drvdata(&dev->dev);

 	if (!cidev)
 		return;

 	/* deactivate the device before disable dtt as
 	 * device may be using dtt.
 	 */
 	coiommu_execute_cmd(COIOMMU_CMD_DEACTIVATE,
 			(void *__iomem)&cidev->mmio_info->command);
 	coiommu_disable_dtt();
 	pci_iounmap(dev, cidev->mmio_info);
 	pci_iounmap(dev, cidev->notify_map);
 	kfree(cidev);
 }

 static struct pci_driver pci_coiommu_driver = {
 	.name = "pci-coiommu",
 	.id_table = pci_coiommu_ids,
 	.probe = pci_coiommu_probe,
 	.remove = pci_coiommu_remove,
 };

 static int __init pci_coiommu_init(void)
 {
 	return pci_register_driver(&pci_coiommu_driver);
 }

 static void __exit pci_coiommu_exit(void)
 {
 	pci_unregister_driver(&pci_coiommu_driver);
 }

 MODULE_LICENSE("GPL v2");

 module_init(pci_coiommu_init);
 module_exit(pci_coiommu_exit);

 static void setup_coiommu(struct pci_dev *dev)
 {
 	void *__iomem bar;
 	unsigned short ep_count, *endpoints;
 	unsigned short total_count, bdf;
 	unsigned long bar_len;
 	int i;

 	if (pci_enable_device(dev))
 		return;
 	/*
 	 * Map device topology bar which contains BDF to indicate which
 	 * device is attached to coiommu.
 	 */
 	bar_len = pci_resource_len(dev, COIOMMU_TOPOLOGY_BAR);
 	bar = pci_iomap(dev, COIOMMU_TOPOLOGY_BAR, bar_len);
 	if (!bar) {
 		dev_err(&dev->dev, "%s: failed to get topology bar\n",
 				__func__);
 		return;
 	}

 	total_count = readw(bar);
 	dev_info(&dev->dev, "%s: endpoint count %d\n", __func__, total_count);

 	if (!total_count)
 		goto out;

 	if (((total_count + 1) * sizeof(unsigned short)) > bar_len) {
 		dev_warn(&dev->dev, "%s: topology bar is too small\n",
 				__func__);
 		total_count = bar_len / sizeof(unsigned short) - 1;
 	}

 	endpoints = kcalloc(total_count, sizeof(unsigned short), GFP_KERNEL);
 	if (!endpoints)
 		goto out;

 	/* start from index 1 as to skip the total_count register */
 	for (i = 1, ep_count = 0; i < total_count + 1; i++) {
 		bdf = readw(bar + i * sizeof(unsigned short));
 		if (!bdf) {
 			pr_err("%s: Get invalid bdf\n", __func__);
 			continue;
 		}

 		endpoints[ep_count++] = bdf;
 		dev_info(&dev->dev, "%s: endpoint %02x:%02x.%01x\n", __func__,
 			 PCI_BUS_NUM(bdf), PCI_SLOT(bdf), PCI_FUNC(bdf));
 	}

 	coiommu_init(ep_count, endpoints);
 	kfree(endpoints);
 out:
 	pci_iounmap(dev, bar);
 }
 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_COIOMMU, PCI_DEVICE_ID_COIOMMU,
 			setup_coiommu);
	// SPDX-License-Identifier: GPL-2.0
	#include <linux/kernel.h>
	#include <linux/module.h>
	#include <linux/pci.h>
	#include <linux/init.h>
	#include <linux/sizes.h>
	#include <linux/mutex.h>
	#include <linux/coiommu_dev.h>

	#define COIOMMU_MMIO_BAR 0
	#define COIOMMU_NOTIFY_BAR 2
	#define COIOMMU_TOPOLOGY_BAR 4

	#define COIOMMU_CMD_DEACTIVATE 0
	#define COIOMMU_CMD_ACTIVATE 1
	#define COIOMMU_CMD_PARK_UNPIN 2
	#define COIOMMU_CMD_UNPARK_UNPIN 3
	#define PIN_PAGES_IN_BATCH (1UL << 63)

	struct coiommu_mmio_info {
	u64 dtt_addr;
	u64 command;
	u64 dtt_level;
	u8 triggers[];
	};

	struct coiommu_notify_info {
	u64 data;
	};

	struct coiommu_dev {
	struct coiommu_mmio_info *mmio_info;
	struct coiommu_notify_info *notify_map;
	};

	static struct pci_device_id pci_coiommu_ids[] = {
	{ PCI_DEVICE(PCI_VENDOR_ID_COIOMMU, PCI_DEVICE_ID_COIOMMU), },
	{ 0, }
	};
	MODULE_DEVICE_TABLE(pci, pci_coiommu_ids);

	static void coiommu_execute_cmd(u64 cmdval, void *__iomem cmd)
	{
	/* MMIO write is not a sync write. The backend can handle this
	* MMIO write async so the frontend can be returned back before
	* this request is really completed.
	*
	* To make sure the cmd is really completed by the backend, we
	* can simply read the cmd MMIO again. Once the read is completed
	* then the previous write has been also completed.
	*/
	writeq(cmdval, cmd);
	readq(cmd);
	}

	static void sync_notify(struct coiommu_dev *cidev, u64 data)
	{
	unsigned long flags;
	struct coiommu_notify_info *info;
	unsigned long timeout;
	u64 id, busy;

	local_irq_save(flags);

	id = smp_processor_id();
	info = cidev->notify_map + id;

	busy = readq((void *__iomem)info);
	if (busy) {
	pr_err("%s: coiommu unexpected busy on CPU%lld: notify info 0x%llx\n",
	__func__, id, busy);
	local_irq_restore(flags);
	return;
	}

	writeq(data, (void *__iomem)info);

	writeb(1, (void *__iomem)(cidev->mmio_info->triggers + id));

	/*
	* Set a timeout in case the backend coiommu cannot complete
	* the request.
	*/
	timeout = READ_ONCE(jiffies) + msecs_to_jiffies(5000);
	while (readq((void *__iomem)info)) {
	if (time_after(READ_ONCE(jiffies), timeout)) {
	pr_err("%s: coIOMMU timeout waiting for complete\n",
	__func__);
	break;
	}
	}

	local_irq_restore(flags);
	}

	static int coiommu_execute_req(struct coiommu_dev *dev, unsigned long pfn, unsigned short bdf)
	{
	if (!dev)
	return -EINVAL;

	sync_notify(dev, (pfn << 16) \| bdf);

	return 0;
	}

	static int coiommu_execute_reqs(struct coiommu_dev dev, struct pin_pages_info pin_info)
	{
	if (!pin_info)
	return -EINVAL;

	if (!pin_info->nr_pages)
	return 0;

	sync_notify(dev, (__pa(pin_info)) \| PIN_PAGES_IN_BATCH);
	return 0;
	}

	static void coiommu_park_unpin(struct coiommu_dev *dev, bool park)
	{
	u64 cmdval = park ? COIOMMU_CMD_PARK_UNPIN : COIOMMU_CMD_UNPARK_UNPIN;

	coiommu_execute_cmd(cmdval, (void *__iomem)&dev->mmio_info->command);
	}

	static const struct coiommu_dev_ops dev_ops = {
	.execute_request = coiommu_execute_req,
	.execute_requests = coiommu_execute_reqs,
	.park_unpin = coiommu_park_unpin,
	};

	static int pci_coiommu_probe(struct pci_dev dev, const struct pci_device_id id)
	{
	struct coiommu_dev *cidev;
	u64 dtt_addr, dtt_level;
	int ret;

	ret = pci_enable_device(dev);
	if (ret)
	return ret;

	cidev = kzalloc(sizeof(struct coiommu_dev), GFP_KERNEL);
	if (!cidev)
	return -ENOMEM;

	/*
	* MMIO bar which contains mmio infos and trigger register.
	* Each CPU has one byte trigger register to notify backend about
	* the action. So Bar length should be longer than the mmio info size
	* plus the number of possible cpus.
	*/
	if (pci_resource_len(dev, COIOMMU_MMIO_BAR) <
	(sizeof(struct coiommu_mmio_info) + num_possible_cpus())) {
	dev_err(&dev->dev, "Invalid bar%d length 0x%llx\n",
	COIOMMU_MMIO_BAR,
	pci_resource_len(dev, COIOMMU_MMIO_BAR));
	ret = -ENXIO;
	goto free_dev;
	}

	cidev->mmio_info = pci_iomap(dev, COIOMMU_MMIO_BAR,
	pci_resource_len(dev, COIOMMU_MMIO_BAR));
	if (!cidev->mmio_info) {
	ret = -ENXIO;
	goto free_dev;
	}

	/*
	* Notification bar. Each CPU has a 8byte register to hold the
	* GFP pin info. So Bar length should be longer than multiply the
	* number of possible cpus with 8.
	*/
	if (pci_resource_len(dev, COIOMMU_NOTIFY_BAR) <
	num_possible_cpus() * 8) {
	dev_err(&dev->dev, "Invalid bar%d length 0x%llx\n",
	COIOMMU_NOTIFY_BAR,
	pci_resource_len(dev, COIOMMU_NOTIFY_BAR));
	ret = -ENXIO;
	goto free_mmio;
	}

	cidev->notify_map = pci_iomap(dev, COIOMMU_NOTIFY_BAR,
	pci_resource_len(dev, COIOMMU_NOTIFY_BAR));
	if (!cidev->notify_map) {
	ret = -ENXIO;
	goto free_mmio;
	}

	ret = coiommu_enable_dtt(&dtt_addr, &dtt_level);
	if (ret) {
	dev_err(&dev->dev, "Failed to setup DTT\n");
	goto free_notify;
	}

	/*
	* Set the dtt_addr and dtt_level to the backend. The subsequent
	* activate command takes care synchronizing these async writes.
	*/
	writeq(dtt_addr, (void *__iomem)&cidev->mmio_info->dtt_addr);
	writeq(dtt_level, (void *__iomem)&cidev->mmio_info->dtt_level);

	coiommu_execute_cmd(COIOMMU_CMD_ACTIVATE,
	(void *__iomem)&cidev->mmio_info->command);

	/* The last step to enable coiommu */
	ret = coiommu_setup_dev_ops(&dev_ops, cidev);
	if (ret)
	goto deactivate;

	dev_set_drvdata(&dev->dev, cidev);

	return 0;

	deactivate:
	coiommu_execute_cmd(COIOMMU_CMD_DEACTIVATE,
	(void *__iomem)&cidev->mmio_info->command);
	coiommu_disable_dtt();
	free_notify:
	pci_iounmap(dev, cidev->notify_map);
	free_mmio:
	pci_iounmap(dev, cidev->mmio_info);
	free_dev:
	kfree(cidev);
	return ret;
	}

	static void pci_coiommu_remove(struct pci_dev *dev)
	{
	struct coiommu_dev *cidev = dev_get_drvdata(&dev->dev);

	if (!cidev)
	return;

	/* deactivate the device before disable dtt as
	* device may be using dtt.
	*/
	coiommu_execute_cmd(COIOMMU_CMD_DEACTIVATE,
	(void *__iomem)&cidev->mmio_info->command);
	coiommu_disable_dtt();
	pci_iounmap(dev, cidev->mmio_info);
	pci_iounmap(dev, cidev->notify_map);
	kfree(cidev);
	}

	static struct pci_driver pci_coiommu_driver = {
	.name = "pci-coiommu",
	.id_table = pci_coiommu_ids,
	.probe = pci_coiommu_probe,
	.remove = pci_coiommu_remove,
	};

	static int __init pci_coiommu_init(void)
	{
	return pci_register_driver(&pci_coiommu_driver);
	}

	static void __exit pci_coiommu_exit(void)
	{
	pci_unregister_driver(&pci_coiommu_driver);
	}

	MODULE_LICENSE("GPL v2");

	module_init(pci_coiommu_init);
	module_exit(pci_coiommu_exit);

	static void setup_coiommu(struct pci_dev *dev)
	{
	void *__iomem bar;
	unsigned short ep_count, *endpoints;
	unsigned short total_count, bdf;
	unsigned long bar_len;
	int i;

	if (pci_enable_device(dev))
	return;
	/*
	* Map device topology bar which contains BDF to indicate which
	* device is attached to coiommu.
	*/
	bar_len = pci_resource_len(dev, COIOMMU_TOPOLOGY_BAR);
	bar = pci_iomap(dev, COIOMMU_TOPOLOGY_BAR, bar_len);
	if (!bar) {
	dev_err(&dev->dev, "%s: failed to get topology bar\n",
	__func__);
	return;
	}

	total_count = readw(bar);
	dev_info(&dev->dev, "%s: endpoint count %d\n", __func__, total_count);

	if (!total_count)
	goto out;

	if (((total_count + 1) * sizeof(unsigned short)) > bar_len) {
	dev_warn(&dev->dev, "%s: topology bar is too small\n",
	__func__);
	total_count = bar_len / sizeof(unsigned short) - 1;
	}

	endpoints = kcalloc(total_count, sizeof(unsigned short), GFP_KERNEL);
	if (!endpoints)
	goto out;

	/* start from index 1 as to skip the total_count register */
	for (i = 1, ep_count = 0; i < total_count + 1; i++) {
	bdf = readw(bar + i * sizeof(unsigned short));
	if (!bdf) {
	pr_err("%s: Get invalid bdf\n", __func__);
	continue;
	}

	endpoints[ep_count++] = bdf;
	dev_info(&dev->dev, "%s: endpoint %02x:%02x.%01x\n", __func__,
	PCI_BUS_NUM(bdf), PCI_SLOT(bdf), PCI_FUNC(bdf));
	}

	coiommu_init(ep_count, endpoints);
	kfree(endpoints);
	out:
	pci_iounmap(dev, bar);
	}
	DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_COIOMMU, PCI_DEVICE_ID_COIOMMU,
	setup_coiommu);