drivers/char/tpm/tpm_virtio.c - third_party/linux - Git at Google

 // SPDX-License-Identifier: GPL-2.0
 /*
  * Copyright 2019 Google Inc.
  *
  * Author: David Tolnay <dtolnay@gmail.com>
  *
  * ---
  *
  * Device driver for TPM over virtio.
  *
  * This driver employs a single virtio queue to handle both send and recv. TPM
  * commands are sent over virtio to the hypervisor during a TPM send operation
  * and responses are received over the same queue during a recv operation.
  *
  * The driver contains a single buffer that is the only buffer we ever place on
  * the virtio queue. Commands are copied from the caller's command buffer into
  * the driver's buffer before handing off to virtio, and responses are received
  * into the driver's buffer then copied into the caller's response buffer. This
  * allows us to be resilient to timeouts. When a send or recv operation times
  * out, the caller is free to destroy their buffer; we don't want the hypervisor
  * continuing to perform reads or writes against that destroyed buffer.
  *
  * This driver does not support concurrent send and recv operations. Mutual
  * exclusion is upheld by the tpm_mutex lock held in tpm-interface.c around the
  * calls to chip->ops->send and chip->ops->recv.
  *
  * The intended hypervisor-side implementation is as follows.
  *
  *     while true:
  *         await next virtio buffer.
  *         expect first descriptor in chain to be guest-to-host.
  *         read tpm command from that buffer.
  *         synchronously perform TPM work determined by the command.
  *         expect second descriptor in chain to be host-to-guest.
  *         write TPM response into that buffer.
  *         place buffer on virtio used queue indicating how many bytes written.
  */

 #include <linux/virtio_config.h>

 #include <uapi/linux/virtio_ids.h>

 #include "tpm.h"

 /*
  * Timeout duration when waiting on the hypervisor to complete its end of the
  * TPM operation. This timeout is relatively high because certain TPM operations
  * can take dozens of seconds.
  */
 #define TPM_VIRTIO_TIMEOUT (120 * HZ)

 struct vtpm_device {
 	/*
 	 * Data structure for integration with the common code of the TPM driver
 	 * in tpm-chip.c.
 	 */
 	struct tpm_chip *chip;

 	/*
 	 * Virtio queue for sending TPM commands out of the virtual machine and
 	 * receiving TPM responses back from the hypervisor.
 	 */
 	struct virtqueue *vq;

 	/*
 	 * Completion that is notified when a virtio operation has been
 	 * fulfilled by the hypervisor.
 	 */
 	struct completion complete;

 	/*
 	 * Whether driver currently holds ownership of the virtqueue buffer.
 	 * When false, the hypervisor is in the process of reading or writing
 	 * the buffer and the driver must not touch it.
 	 */
 	bool driver_has_buffer;

 	/*
 	 * Whether during the most recent TPM operation, a virtqueue_kick failed
 	 * or a wait timed out.
 	 *
 	 * The next send or recv operation will attempt a kick upon seeing this
 	 * status. That should clear up the queue in the case that the
 	 * hypervisor became temporarily nonresponsive, such as by resource
 	 * exhaustion on the host. The extra kick enables recovery from kicks
 	 * going unnoticed by the hypervisor as well as recovery from virtio
 	 * callbacks going unnoticed by the guest kernel.
 	 */
 	bool needs_kick;

 	/* Number of bytes available to read from the virtqueue buffer. */
 	unsigned int readable;

 	/*
 	 * Buffer in which all virtio transfers take place. Buffer size is the
 	 * maximum legal TPM command or response message size.
 	 */
 	u8 virtqueue_buffer[TPM_BUFSIZE];
 };

 /*
  * Wait for ownership of the virtqueue buffer.
  *
  * The why-string should begin with "waiting to..." or "waiting for..." with no
  * trailing newline. It will appear in log output.
  *
  * Returns zero for success, otherwise negative error.
  */
 static int vtpm_wait_for_buffer(struct vtpm_device *dev, const char *why)
 {
 	int ret;
 	struct tpm_chip *chip = dev->chip;
 	unsigned long deadline = jiffies + TPM_VIRTIO_TIMEOUT;

 	/* Kick queue if needed. */
 	if (dev->needs_kick) {
 		bool did_kick = virtqueue_kick(dev->vq);
 		if (!did_kick) {
 			dev_notice(&chip->dev, "kick failed; will retry\n");
 			return -EBUSY;
 		}
 		dev->needs_kick = false;
 	}

 	while (!dev->driver_has_buffer) {
 		unsigned long now = jiffies;

 		/* Check timeout, otherwise `deadline - now` may underflow. */
 		if time_after_eq(now, deadline) {
 			dev_warn(&chip->dev, "timed out %s\n", why);
 			dev->needs_kick = true;
 			return -ETIMEDOUT;
 		}

 		/*
 		 * Wait to be signaled by virtio callback.
 		 *
 		 * Positive ret is jiffies remaining until timeout when the
 		 * completion occurred, which means successful completion. Zero
 		 * ret is timeout. Negative ret is error.
 		 */
 		ret = wait_for_completion_killable_timeout(
 				&dev->complete, deadline - now);

 		/* Log if completion did not occur. */
 		if (ret == -ERESTARTSYS) {
 			/* Not a warning if it was simply interrupted. */
 			dev_dbg(&chip->dev, "interrupted %s\n", why);
 		} else if (ret == 0) {
 			dev_warn(&chip->dev, "timed out %s\n", why);
 			ret = -ETIMEDOUT;
 		} else if (ret < 0) {
 			dev_warn(&chip->dev, "failed while %s: error %d\n",
 					why, -ret);
 		}

 		/*
 		 * Return error if completion did not occur. Schedule kick to be
 		 * retried at the start of the next send/recv to help unblock
 		 * the queue.
 		 */
 		if (ret < 0) {
 			dev->needs_kick = true;
 			return ret;
 		}

 		/* Completion occurred. Expect response buffer back. */
 		if (virtqueue_get_buf(dev->vq, &dev->readable)) {
 			dev->driver_has_buffer = true;

 			if (dev->readable > TPM_BUFSIZE) {
 				dev_crit(&chip->dev,
 						"hypervisor bug: response exceeds max size,"
 						" %u > %u\n",
 						dev->readable,
 						(unsigned int) TPM_BUFSIZE);
 				dev->readable = TPM_BUFSIZE;
 				return -EPROTO;
 			}
 		}
 	}

 	return 0;
 }

 static int vtpm_op_send(struct tpm_chip *chip, u8 *caller_buf, size_t len)
 {
 	int ret;
 	bool did_kick;
 	struct scatterlist sg_outbuf, sg_inbuf;
 	struct scatterlist *sgs[2] = { &sg_outbuf, &sg_inbuf };
 	struct vtpm_device *dev = dev_get_drvdata(&chip->dev);
 	u8 *virtqueue_buf = dev->virtqueue_buffer;

 	dev_dbg(&chip->dev, "vtpm_op_send %zu bytes\n", len);

 	if (len > TPM_BUFSIZE) {
 		dev_err(&chip->dev,
 				"command is too long, %zu > %zu\n",
 				len, (size_t) TPM_BUFSIZE);
 		return -EINVAL;
 	}

 	/*
 	 * Wait until hypervisor relinquishes ownership of the virtqueue buffer.
 	 *
 	 * This may block if the previous recv operation timed out in the guest
 	 * kernel but is still being processed by the hypervisor. Also may block
 	 * if send operations are performed back-to-back, such as if something
 	 * in the caller failed in between a send and recv.
 	 *
 	 * During normal operation absent of any errors or timeouts, this does
 	 * not block.
 	 */
 	ret = vtpm_wait_for_buffer(dev, "waiting to begin send");
 	if (ret) {
 		return ret;
 	}

 	/* Driver owns virtqueue buffer and may now write into it. */
 	memcpy(virtqueue_buf, caller_buf, len);

 	/*
 	 * Enqueue the virtqueue buffer once as outgoing virtio data (written by
 	 * the virtual machine and read by the hypervisor) and again as incoming
 	 * data (written by the hypervisor and read by the virtual machine).
 	 * This step moves ownership of the virtqueue buffer from driver to
 	 * hypervisor.
 	 *
 	 * Note that we don't know here how big of a buffer the caller will use
 	 * with their later call to recv. We allow the hypervisor to write up to
 	 * the TPM max message size. If the caller ends up using a smaller
 	 * buffer with recv that is too small to hold the entire response, the
 	 * recv will return an error. This conceivably breaks TPM
 	 * implementations that want to produce a different verbosity of
 	 * response depending on the receiver's buffer size.
 	 */
 	sg_init_one(&sg_outbuf, virtqueue_buf, len);
 	sg_init_one(&sg_inbuf, virtqueue_buf, TPM_BUFSIZE);
 	ret = virtqueue_add_sgs(dev->vq, sgs, 1, 1, virtqueue_buf, GFP_KERNEL);
 	if (ret) {
 		dev_err(&chip->dev, "failed virtqueue_add_sgs\n");
 		return ret;
 	}

 	/* Kick the other end of the virtqueue after having added a buffer. */
 	did_kick = virtqueue_kick(dev->vq);
 	if (!did_kick) {
 		dev->needs_kick = true;
 		dev_notice(&chip->dev, "kick failed; will retry\n");

 		/*
 		 * We return 0 anyway because what the caller doesn't know can't
 		 * hurt them. They can call recv and it will retry the kick. If
 		 * that works, everything is fine.
 		 *
 		 * If the retry in recv fails too, they will get -EBUSY from
 		 * recv.
 		 */
 	}

 	/*
 	 * Hypervisor is now processing the TPM command asynchronously. It will
 	 * read the command from the output buffer and write the response into
 	 * the input buffer (which are the same buffer). When done, it will send
 	 * back the buffers over virtio and the driver's virtio callback will
 	 * complete dev->complete so that we know the response is ready to be
 	 * read.
 	 *
 	 * It is important to have copied data out of the caller's buffer into
 	 * the driver's virtqueue buffer because the caller is free to destroy
 	 * their buffer when this call returns. We can't avoid copying by
 	 * waiting here for the hypervisor to finish reading, because if that
 	 * wait times out, we return and the caller may destroy their buffer
 	 * while the hypervisor is continuing to read from it.
 	 */
 	dev->driver_has_buffer = false;
 	return 0;
 }

 static int vtpm_op_recv(struct tpm_chip *chip, u8 *caller_buf, size_t len)
 {
 	int ret;
 	struct vtpm_device *dev = dev_get_drvdata(&chip->dev);
 	u8 *virtqueue_buf = dev->virtqueue_buffer;

 	dev_dbg(&chip->dev, "vtpm_op_recv\n");

 	/*
 	 * Wait until the virtqueue buffer is owned by the driver.
 	 *
 	 * This will usually block while the hypervisor finishes processing the
 	 * most recent TPM command.
 	 */
 	ret = vtpm_wait_for_buffer(dev, "waiting for TPM response");
 	if (ret) {
 		return ret;
 	}

 	dev_dbg(&chip->dev, "received %u bytes\n", dev->readable);

 	if (dev->readable > len) {
 		dev_notice(&chip->dev,
 				"TPM response is bigger than receiver's buffer:"
 				" %u > %zu\n",
 				dev->readable, len);
 		return -EINVAL;
 	}

 	/* Copy response over to the caller. */
 	memcpy(caller_buf, virtqueue_buf, dev->readable);

 	return dev->readable;
 }

 static void vtpm_op_cancel(struct tpm_chip *chip)
 {
 	/*
 	 * Cancel is not called in this driver's use of tpm-interface.c. It may
 	 * be triggered through tpm-sysfs but that can be implemented as needed.
 	 * Be aware that tpm-sysfs performs cancellation without holding the
 	 * tpm_mutex that protects our send and recv operations, so a future
 	 * implementation will need to consider thread safety of concurrent
 	 * send/recv and cancel.
 	 */
 	dev_notice(&chip->dev, "cancellation is not implemented\n");
 }

 static u8 vtpm_op_status(struct tpm_chip *chip)
 {
 	/*
 	 * Status is for TPM drivers that want tpm-interface.c to poll for
 	 * completion before calling recv. Usually this is when the hardware
 	 * needs to be polled i.e. there is no other way for recv to block on
 	 * the TPM command completion.
 	 *
 	 * Polling goes until `(status & complete_mask) == complete_val`. This
 	 * driver defines both complete_mask and complete_val as 0 and blocks on
 	 * our own completion object in recv instead.
 	 */
 	return 0;
 }

 static const struct tpm_class_ops vtpm_ops = {
 	.flags = TPM_OPS_AUTO_STARTUP,
 	.send = vtpm_op_send,
 	.recv = vtpm_op_recv,
 	.cancel = vtpm_op_cancel,
 	.status = vtpm_op_status,
 	.req_complete_mask = 0,
 	.req_complete_val = 0,
 };

 static void vtpm_virtio_complete(struct virtqueue *vq)
 {
 	struct virtio_device *vdev = vq->vdev;
 	struct vtpm_device *dev = vdev->priv;

 	complete(&dev->complete);
 }

 static int vtpm_probe(struct virtio_device *vdev)
 {
 	int err;
 	struct vtpm_device *dev;
 	struct virtqueue *vq;
 	struct tpm_chip *chip;

 	dev_dbg(&vdev->dev, "vtpm_probe\n");

 	dev = kzalloc(sizeof(struct vtpm_device), GFP_KERNEL);
 	if (!dev) {
 		err = -ENOMEM;
 		dev_err(&vdev->dev, "failed kzalloc\n");
 		goto err_dev_alloc;
 	}
 	vdev->priv = dev;

 	vq = virtio_find_single_vq(vdev, vtpm_virtio_complete, "vtpm");
 	if (IS_ERR(vq)) {
 		err = PTR_ERR(vq);
 		dev_err(&vdev->dev, "failed virtio_find_single_vq\n");
 		goto err_virtio_find;
 	}
 	dev->vq = vq;

 	chip = tpm_chip_alloc(&vdev->dev, &vtpm_ops);
 	if (IS_ERR(chip)) {
 		err = PTR_ERR(chip);
 		dev_err(&vdev->dev, "failed tpm_chip_alloc\n");
 		goto err_chip_alloc;
 	}
 	dev_set_drvdata(&chip->dev, dev);
 	chip->flags |= TPM_CHIP_FLAG_TPM2;
 	dev->chip = chip;

 	init_completion(&dev->complete);
 	dev->driver_has_buffer = true;
 	dev->needs_kick = false;
 	dev->readable = 0;

 	/*
 	 * Required in order to enable vq use in probe function for auto
 	 * startup.
 	 */
 	virtio_device_ready(vdev);

 	err = tpm_chip_register(dev->chip);
 	if (err) {
 		dev_err(&vdev->dev, "failed tpm_chip_register\n");
 		goto err_chip_register;
 	}

 	return 0;

 err_chip_register:
 	put_device(&dev->chip->dev);
 err_chip_alloc:
 	vdev->config->del_vqs(vdev);
 err_virtio_find:
 	kfree(dev);
 err_dev_alloc:
 	return err;
 }

 static void vtpm_remove(struct virtio_device *vdev)
 {
 	struct vtpm_device *dev = vdev->priv;

 	/* Undo tpm_chip_register. */
 	tpm_chip_unregister(dev->chip);

 	/* Undo tpm_chip_alloc. */
 	put_device(&dev->chip->dev);

 	vdev->config->reset(vdev);
 	vdev->config->del_vqs(vdev);

 	kfree(dev);
 }

 static struct virtio_device_id id_table[] = {
 	{
 		.device = VIRTIO_ID_TPM,
 		.vendor = VIRTIO_DEV_ANY_ID,
 	},
 	{},
 };

 static struct virtio_driver vtpm_driver = {
 	.driver.name = KBUILD_MODNAME,
 	.driver.owner = THIS_MODULE,
 	.id_table = id_table,
 	.probe = vtpm_probe,
 	.remove = vtpm_remove,
 };

 module_virtio_driver(vtpm_driver);

 MODULE_AUTHOR("David Tolnay (dtolnay@gmail.com)");
 MODULE_DESCRIPTION("Virtio vTPM Driver");
 MODULE_VERSION("1.0");
 MODULE_LICENSE("GPL");
	// SPDX-License-Identifier: GPL-2.0
	/*
	* Copyright 2019 Google Inc.
	*
	* Author: David Tolnay <dtolnay@gmail.com>
	*
	* ---
	*
	* Device driver for TPM over virtio.
	*
	* This driver employs a single virtio queue to handle both send and recv. TPM
	* commands are sent over virtio to the hypervisor during a TPM send operation
	* and responses are received over the same queue during a recv operation.
	*
	* The driver contains a single buffer that is the only buffer we ever place on
	* the virtio queue. Commands are copied from the caller's command buffer into
	* the driver's buffer before handing off to virtio, and responses are received
	* into the driver's buffer then copied into the caller's response buffer. This
	* allows us to be resilient to timeouts. When a send or recv operation times
	* out, the caller is free to destroy their buffer; we don't want the hypervisor
	* continuing to perform reads or writes against that destroyed buffer.
	*
	* This driver does not support concurrent send and recv operations. Mutual
	* exclusion is upheld by the tpm_mutex lock held in tpm-interface.c around the
	* calls to chip->ops->send and chip->ops->recv.
	*
	* The intended hypervisor-side implementation is as follows.
	*
	* while true:
	* await next virtio buffer.
	* expect first descriptor in chain to be guest-to-host.
	* read tpm command from that buffer.
	* synchronously perform TPM work determined by the command.
	* expect second descriptor in chain to be host-to-guest.
	* write TPM response into that buffer.
	* place buffer on virtio used queue indicating how many bytes written.
	*/

	#include <linux/virtio_config.h>

	#include <uapi/linux/virtio_ids.h>

	#include "tpm.h"

	/*
	* Timeout duration when waiting on the hypervisor to complete its end of the
	* TPM operation. This timeout is relatively high because certain TPM operations
	* can take dozens of seconds.
	*/
	#define TPM_VIRTIO_TIMEOUT (120 * HZ)

	struct vtpm_device {
	/*
	* Data structure for integration with the common code of the TPM driver
	* in tpm-chip.c.
	*/
	struct tpm_chip *chip;

	/*
	* Virtio queue for sending TPM commands out of the virtual machine and
	* receiving TPM responses back from the hypervisor.
	*/
	struct virtqueue *vq;

	/*
	* Completion that is notified when a virtio operation has been
	* fulfilled by the hypervisor.
	*/
	struct completion complete;

	/*
	* Whether driver currently holds ownership of the virtqueue buffer.
	* When false, the hypervisor is in the process of reading or writing
	* the buffer and the driver must not touch it.
	*/
	bool driver_has_buffer;

	/*
	* Whether during the most recent TPM operation, a virtqueue_kick failed
	* or a wait timed out.
	*
	* The next send or recv operation will attempt a kick upon seeing this
	* status. That should clear up the queue in the case that the
	* hypervisor became temporarily nonresponsive, such as by resource
	* exhaustion on the host. The extra kick enables recovery from kicks
	* going unnoticed by the hypervisor as well as recovery from virtio
	* callbacks going unnoticed by the guest kernel.
	*/
	bool needs_kick;

	/* Number of bytes available to read from the virtqueue buffer. */
	unsigned int readable;

	/*
	* Buffer in which all virtio transfers take place. Buffer size is the
	* maximum legal TPM command or response message size.
	*/
	u8 virtqueue_buffer[TPM_BUFSIZE];
	};

	/*
	* Wait for ownership of the virtqueue buffer.
	*
	* The why-string should begin with "waiting to..." or "waiting for..." with no
	* trailing newline. It will appear in log output.
	*
	* Returns zero for success, otherwise negative error.
	*/
	static int vtpm_wait_for_buffer(struct vtpm_device dev, const char why)
	{
	int ret;
	struct tpm_chip *chip = dev->chip;
	unsigned long deadline = jiffies + TPM_VIRTIO_TIMEOUT;

	/* Kick queue if needed. */
	if (dev->needs_kick) {
	bool did_kick = virtqueue_kick(dev->vq);
	if (!did_kick) {
	dev_notice(&chip->dev, "kick failed; will retry\n");
	return -EBUSY;
	}
	dev->needs_kick = false;
	}

	while (!dev->driver_has_buffer) {
	unsigned long now = jiffies;

	/* Check timeout, otherwise `deadline - now` may underflow. */
	if time_after_eq(now, deadline) {
	dev_warn(&chip->dev, "timed out %s\n", why);
	dev->needs_kick = true;
	return -ETIMEDOUT;
	}

	/*
	* Wait to be signaled by virtio callback.
	*
	* Positive ret is jiffies remaining until timeout when the
	* completion occurred, which means successful completion. Zero
	* ret is timeout. Negative ret is error.
	*/
	ret = wait_for_completion_killable_timeout(
	&dev->complete, deadline - now);

	/* Log if completion did not occur. */
	if (ret == -ERESTARTSYS) {
	/* Not a warning if it was simply interrupted. */
	dev_dbg(&chip->dev, "interrupted %s\n", why);
	} else if (ret == 0) {
	dev_warn(&chip->dev, "timed out %s\n", why);
	ret = -ETIMEDOUT;
	} else if (ret < 0) {
	dev_warn(&chip->dev, "failed while %s: error %d\n",
	why, -ret);
	}

	/*
	* Return error if completion did not occur. Schedule kick to be
	* retried at the start of the next send/recv to help unblock
	* the queue.
	*/
	if (ret < 0) {
	dev->needs_kick = true;
	return ret;
	}

	/* Completion occurred. Expect response buffer back. */
	if (virtqueue_get_buf(dev->vq, &dev->readable)) {
	dev->driver_has_buffer = true;

	if (dev->readable > TPM_BUFSIZE) {
	dev_crit(&chip->dev,
	"hypervisor bug: response exceeds max size,"
	" %u > %u\n",
	dev->readable,
	(unsigned int) TPM_BUFSIZE);
	dev->readable = TPM_BUFSIZE;
	return -EPROTO;
	}
	}
	}

	return 0;
	}

	static int vtpm_op_send(struct tpm_chip chip, u8 caller_buf, size_t len)
	{
	int ret;
	bool did_kick;
	struct scatterlist sg_outbuf, sg_inbuf;
	struct scatterlist *sgs[2] = { &sg_outbuf, &sg_inbuf };
	struct vtpm_device *dev = dev_get_drvdata(&chip->dev);
	u8 *virtqueue_buf = dev->virtqueue_buffer;

	dev_dbg(&chip->dev, "vtpm_op_send %zu bytes\n", len);

	if (len > TPM_BUFSIZE) {
	dev_err(&chip->dev,
	"command is too long, %zu > %zu\n",
	len, (size_t) TPM_BUFSIZE);
	return -EINVAL;
	}

	/*
	* Wait until hypervisor relinquishes ownership of the virtqueue buffer.
	*
	* This may block if the previous recv operation timed out in the guest
	* kernel but is still being processed by the hypervisor. Also may block
	* if send operations are performed back-to-back, such as if something
	* in the caller failed in between a send and recv.
	*
	* During normal operation absent of any errors or timeouts, this does
	* not block.
	*/
	ret = vtpm_wait_for_buffer(dev, "waiting to begin send");
	if (ret) {
	return ret;
	}

	/* Driver owns virtqueue buffer and may now write into it. */
	memcpy(virtqueue_buf, caller_buf, len);

	/*
	* Enqueue the virtqueue buffer once as outgoing virtio data (written by
	* the virtual machine and read by the hypervisor) and again as incoming
	* data (written by the hypervisor and read by the virtual machine).
	* This step moves ownership of the virtqueue buffer from driver to
	* hypervisor.
	*
	* Note that we don't know here how big of a buffer the caller will use
	* with their later call to recv. We allow the hypervisor to write up to
	* the TPM max message size. If the caller ends up using a smaller
	* buffer with recv that is too small to hold the entire response, the
	* recv will return an error. This conceivably breaks TPM
	* implementations that want to produce a different verbosity of
	* response depending on the receiver's buffer size.
	*/
	sg_init_one(&sg_outbuf, virtqueue_buf, len);
	sg_init_one(&sg_inbuf, virtqueue_buf, TPM_BUFSIZE);
	ret = virtqueue_add_sgs(dev->vq, sgs, 1, 1, virtqueue_buf, GFP_KERNEL);
	if (ret) {
	dev_err(&chip->dev, "failed virtqueue_add_sgs\n");
	return ret;
	}

	/* Kick the other end of the virtqueue after having added a buffer. */
	did_kick = virtqueue_kick(dev->vq);
	if (!did_kick) {
	dev->needs_kick = true;
	dev_notice(&chip->dev, "kick failed; will retry\n");

	/*
	* We return 0 anyway because what the caller doesn't know can't
	* hurt them. They can call recv and it will retry the kick. If
	* that works, everything is fine.
	*
	* If the retry in recv fails too, they will get -EBUSY from
	* recv.
	*/
	}

	/*
	* Hypervisor is now processing the TPM command asynchronously. It will
	* read the command from the output buffer and write the response into
	* the input buffer (which are the same buffer). When done, it will send
	* back the buffers over virtio and the driver's virtio callback will
	* complete dev->complete so that we know the response is ready to be
	* read.
	*
	* It is important to have copied data out of the caller's buffer into
	* the driver's virtqueue buffer because the caller is free to destroy
	* their buffer when this call returns. We can't avoid copying by
	* waiting here for the hypervisor to finish reading, because if that
	* wait times out, we return and the caller may destroy their buffer
	* while the hypervisor is continuing to read from it.
	*/
	dev->driver_has_buffer = false;
	return 0;
	}

	static int vtpm_op_recv(struct tpm_chip chip, u8 caller_buf, size_t len)
	{
	int ret;
	struct vtpm_device *dev = dev_get_drvdata(&chip->dev);
	u8 *virtqueue_buf = dev->virtqueue_buffer;

	dev_dbg(&chip->dev, "vtpm_op_recv\n");

	/*
	* Wait until the virtqueue buffer is owned by the driver.
	*
	* This will usually block while the hypervisor finishes processing the
	* most recent TPM command.
	*/
	ret = vtpm_wait_for_buffer(dev, "waiting for TPM response");
	if (ret) {
	return ret;
	}

	dev_dbg(&chip->dev, "received %u bytes\n", dev->readable);

	if (dev->readable > len) {
	dev_notice(&chip->dev,
	"TPM response is bigger than receiver's buffer:"
	" %u > %zu\n",
	dev->readable, len);
	return -EINVAL;
	}

	/* Copy response over to the caller. */
	memcpy(caller_buf, virtqueue_buf, dev->readable);

	return dev->readable;
	}

	static void vtpm_op_cancel(struct tpm_chip *chip)
	{
	/*
	* Cancel is not called in this driver's use of tpm-interface.c. It may
	* be triggered through tpm-sysfs but that can be implemented as needed.
	* Be aware that tpm-sysfs performs cancellation without holding the
	* tpm_mutex that protects our send and recv operations, so a future
	* implementation will need to consider thread safety of concurrent
	* send/recv and cancel.
	*/
	dev_notice(&chip->dev, "cancellation is not implemented\n");
	}

	static u8 vtpm_op_status(struct tpm_chip *chip)
	{
	/*
	* Status is for TPM drivers that want tpm-interface.c to poll for
	* completion before calling recv. Usually this is when the hardware
	* needs to be polled i.e. there is no other way for recv to block on
	* the TPM command completion.
	*
	* Polling goes until `(status & complete_mask) == complete_val`. This
	* driver defines both complete_mask and complete_val as 0 and blocks on
	* our own completion object in recv instead.
	*/
	return 0;
	}

	static const struct tpm_class_ops vtpm_ops = {
	.flags = TPM_OPS_AUTO_STARTUP,
	.send = vtpm_op_send,
	.recv = vtpm_op_recv,
	.cancel = vtpm_op_cancel,
	.status = vtpm_op_status,
	.req_complete_mask = 0,
	.req_complete_val = 0,
	};

	static void vtpm_virtio_complete(struct virtqueue *vq)
	{
	struct virtio_device *vdev = vq->vdev;
	struct vtpm_device *dev = vdev->priv;

	complete(&dev->complete);
	}

	static int vtpm_probe(struct virtio_device *vdev)
	{
	int err;
	struct vtpm_device *dev;
	struct virtqueue *vq;
	struct tpm_chip *chip;

	dev_dbg(&vdev->dev, "vtpm_probe\n");

	dev = kzalloc(sizeof(struct vtpm_device), GFP_KERNEL);
	if (!dev) {
	err = -ENOMEM;
	dev_err(&vdev->dev, "failed kzalloc\n");
	goto err_dev_alloc;
	}
	vdev->priv = dev;

	vq = virtio_find_single_vq(vdev, vtpm_virtio_complete, "vtpm");
	if (IS_ERR(vq)) {
	err = PTR_ERR(vq);
	dev_err(&vdev->dev, "failed virtio_find_single_vq\n");
	goto err_virtio_find;
	}
	dev->vq = vq;

	chip = tpm_chip_alloc(&vdev->dev, &vtpm_ops);
	if (IS_ERR(chip)) {
	err = PTR_ERR(chip);
	dev_err(&vdev->dev, "failed tpm_chip_alloc\n");
	goto err_chip_alloc;
	}
	dev_set_drvdata(&chip->dev, dev);
	chip->flags \|= TPM_CHIP_FLAG_TPM2;
	dev->chip = chip;

	init_completion(&dev->complete);
	dev->driver_has_buffer = true;
	dev->needs_kick = false;
	dev->readable = 0;

	/*
	* Required in order to enable vq use in probe function for auto
	* startup.
	*/
	virtio_device_ready(vdev);

	err = tpm_chip_register(dev->chip);
	if (err) {
	dev_err(&vdev->dev, "failed tpm_chip_register\n");
	goto err_chip_register;
	}

	return 0;

	err_chip_register:
	put_device(&dev->chip->dev);
	err_chip_alloc:
	vdev->config->del_vqs(vdev);
	err_virtio_find:
	kfree(dev);
	err_dev_alloc:
	return err;
	}

	static void vtpm_remove(struct virtio_device *vdev)
	{
	struct vtpm_device *dev = vdev->priv;

	/* Undo tpm_chip_register. */
	tpm_chip_unregister(dev->chip);

	/* Undo tpm_chip_alloc. */
	put_device(&dev->chip->dev);

	vdev->config->reset(vdev);
	vdev->config->del_vqs(vdev);

	kfree(dev);
	}

	static struct virtio_device_id id_table[] = {
	{
	.device = VIRTIO_ID_TPM,
	.vendor = VIRTIO_DEV_ANY_ID,
	},
	{},
	};

	static struct virtio_driver vtpm_driver = {
	.driver.name = KBUILD_MODNAME,
	.driver.owner = THIS_MODULE,
	.id_table = id_table,
	.probe = vtpm_probe,
	.remove = vtpm_remove,
	};

	module_virtio_driver(vtpm_driver);

	MODULE_AUTHOR("David Tolnay (dtolnay@gmail.com)");
	MODULE_DESCRIPTION("Virtio vTPM Driver");
	MODULE_VERSION("1.0");
	MODULE_LICENSE("GPL");