| // SPDX-License-Identifier: GPL-2.0 |
| /* |
| * Copyright 2019 Google Inc. |
| * |
| * Author: David Tolnay <dtolnay@gmail.com> |
| * |
| * --- |
| * |
| * Device driver for TPM over virtio. |
| * |
| * This driver employs a single virtio queue to handle both send and recv. TPM |
| * commands are sent over virtio to the hypervisor during a TPM send operation |
| * and responses are received over the same queue during a recv operation. |
| * |
| * The driver contains a single buffer that is the only buffer we ever place on |
| * the virtio queue. Commands are copied from the caller's command buffer into |
| * the driver's buffer before handing off to virtio, and responses are received |
| * into the driver's buffer then copied into the caller's response buffer. This |
| * allows us to be resilient to timeouts. When a send or recv operation times |
| * out, the caller is free to destroy their buffer; we don't want the hypervisor |
| * continuing to perform reads or writes against that destroyed buffer. |
| * |
| * This driver does not support concurrent send and recv operations. Mutual |
| * exclusion is upheld by the tpm_mutex lock held in tpm-interface.c around the |
| * calls to chip->ops->send and chip->ops->recv. |
| * |
| * The intended hypervisor-side implementation is as follows. |
| * |
| * while true: |
| * await next virtio buffer. |
| * expect first descriptor in chain to be guest-to-host. |
| * read tpm command from that buffer. |
| * synchronously perform TPM work determined by the command. |
| * expect second descriptor in chain to be host-to-guest. |
| * write TPM response into that buffer. |
| * place buffer on virtio used queue indicating how many bytes written. |
| */ |
| |
| #include <linux/virtio_config.h> |
| |
| #include <uapi/linux/virtio_ids.h> |
| |
| #include "tpm.h" |
| |
| /* |
| * Timeout duration when waiting on the hypervisor to complete its end of the |
| * TPM operation. This timeout is relatively high because certain TPM operations |
| * can take dozens of seconds. |
| */ |
| #define TPM_VIRTIO_TIMEOUT (120 * HZ) |
| |
| struct vtpm_device { |
| /* |
| * Data structure for integration with the common code of the TPM driver |
| * in tpm-chip.c. |
| */ |
| struct tpm_chip *chip; |
| |
| /* |
| * Virtio queue for sending TPM commands out of the virtual machine and |
| * receiving TPM responses back from the hypervisor. |
| */ |
| struct virtqueue *vq; |
| |
| /* |
| * Completion that is notified when a virtio operation has been |
| * fulfilled by the hypervisor. |
| */ |
| struct completion complete; |
| |
| /* |
| * Whether driver currently holds ownership of the virtqueue buffer. |
| * When false, the hypervisor is in the process of reading or writing |
| * the buffer and the driver must not touch it. |
| */ |
| bool driver_has_buffer; |
| |
| /* |
| * Whether during the most recent TPM operation, a virtqueue_kick failed |
| * or a wait timed out. |
| * |
| * The next send or recv operation will attempt a kick upon seeing this |
| * status. That should clear up the queue in the case that the |
| * hypervisor became temporarily nonresponsive, such as by resource |
| * exhaustion on the host. The extra kick enables recovery from kicks |
| * going unnoticed by the hypervisor as well as recovery from virtio |
| * callbacks going unnoticed by the guest kernel. |
| */ |
| bool needs_kick; |
| |
| /* Number of bytes available to read from the virtqueue buffer. */ |
| unsigned int readable; |
| |
| /* |
| * Buffer in which all virtio transfers take place. Buffer size is the |
| * maximum legal TPM command or response message size. |
| */ |
| u8 virtqueue_buffer[TPM_BUFSIZE]; |
| }; |
| |
| /* |
| * Wait for ownership of the virtqueue buffer. |
| * |
| * The why-string should begin with "waiting to..." or "waiting for..." with no |
| * trailing newline. It will appear in log output. |
| * |
| * Returns zero for success, otherwise negative error. |
| */ |
| static int vtpm_wait_for_buffer(struct vtpm_device *dev, const char *why) |
| { |
| int ret; |
| struct tpm_chip *chip = dev->chip; |
| unsigned long deadline = jiffies + TPM_VIRTIO_TIMEOUT; |
| |
| /* Kick queue if needed. */ |
| if (dev->needs_kick) { |
| bool did_kick = virtqueue_kick(dev->vq); |
| if (!did_kick) { |
| dev_notice(&chip->dev, "kick failed; will retry\n"); |
| return -EBUSY; |
| } |
| dev->needs_kick = false; |
| } |
| |
| while (!dev->driver_has_buffer) { |
| unsigned long now = jiffies; |
| |
| /* Check timeout, otherwise `deadline - now` may underflow. */ |
| if time_after_eq(now, deadline) { |
| dev_warn(&chip->dev, "timed out %s\n", why); |
| dev->needs_kick = true; |
| return -ETIMEDOUT; |
| } |
| |
| /* |
| * Wait to be signaled by virtio callback. |
| * |
| * Positive ret is jiffies remaining until timeout when the |
| * completion occurred, which means successful completion. Zero |
| * ret is timeout. Negative ret is error. |
| */ |
| ret = wait_for_completion_killable_timeout( |
| &dev->complete, deadline - now); |
| |
| /* Log if completion did not occur. */ |
| if (ret == -ERESTARTSYS) { |
| /* Not a warning if it was simply interrupted. */ |
| dev_dbg(&chip->dev, "interrupted %s\n", why); |
| } else if (ret == 0) { |
| dev_warn(&chip->dev, "timed out %s\n", why); |
| ret = -ETIMEDOUT; |
| } else if (ret < 0) { |
| dev_warn(&chip->dev, "failed while %s: error %d\n", |
| why, -ret); |
| } |
| |
| /* |
| * Return error if completion did not occur. Schedule kick to be |
| * retried at the start of the next send/recv to help unblock |
| * the queue. |
| */ |
| if (ret < 0) { |
| dev->needs_kick = true; |
| return ret; |
| } |
| |
| /* Completion occurred. Expect response buffer back. */ |
| if (virtqueue_get_buf(dev->vq, &dev->readable)) { |
| dev->driver_has_buffer = true; |
| |
| if (dev->readable > TPM_BUFSIZE) { |
| dev_crit(&chip->dev, |
| "hypervisor bug: response exceeds max size," |
| " %u > %u\n", |
| dev->readable, |
| (unsigned int) TPM_BUFSIZE); |
| dev->readable = TPM_BUFSIZE; |
| return -EPROTO; |
| } |
| } |
| } |
| |
| return 0; |
| } |
| |
| static int vtpm_op_send(struct tpm_chip *chip, u8 *caller_buf, size_t len) |
| { |
| int ret; |
| bool did_kick; |
| struct scatterlist sg_outbuf, sg_inbuf; |
| struct scatterlist *sgs[2] = { &sg_outbuf, &sg_inbuf }; |
| struct vtpm_device *dev = dev_get_drvdata(&chip->dev); |
| u8 *virtqueue_buf = dev->virtqueue_buffer; |
| |
| dev_dbg(&chip->dev, "vtpm_op_send %zu bytes\n", len); |
| |
| if (len > TPM_BUFSIZE) { |
| dev_err(&chip->dev, |
| "command is too long, %zu > %zu\n", |
| len, (size_t) TPM_BUFSIZE); |
| return -EINVAL; |
| } |
| |
| /* |
| * Wait until hypervisor relinquishes ownership of the virtqueue buffer. |
| * |
| * This may block if the previous recv operation timed out in the guest |
| * kernel but is still being processed by the hypervisor. Also may block |
| * if send operations are performed back-to-back, such as if something |
| * in the caller failed in between a send and recv. |
| * |
| * During normal operation absent of any errors or timeouts, this does |
| * not block. |
| */ |
| ret = vtpm_wait_for_buffer(dev, "waiting to begin send"); |
| if (ret) { |
| return ret; |
| } |
| |
| /* Driver owns virtqueue buffer and may now write into it. */ |
| memcpy(virtqueue_buf, caller_buf, len); |
| |
| /* |
| * Enqueue the virtqueue buffer once as outgoing virtio data (written by |
| * the virtual machine and read by the hypervisor) and again as incoming |
| * data (written by the hypervisor and read by the virtual machine). |
| * This step moves ownership of the virtqueue buffer from driver to |
| * hypervisor. |
| * |
| * Note that we don't know here how big of a buffer the caller will use |
| * with their later call to recv. We allow the hypervisor to write up to |
| * the TPM max message size. If the caller ends up using a smaller |
| * buffer with recv that is too small to hold the entire response, the |
| * recv will return an error. This conceivably breaks TPM |
| * implementations that want to produce a different verbosity of |
| * response depending on the receiver's buffer size. |
| */ |
| sg_init_one(&sg_outbuf, virtqueue_buf, len); |
| sg_init_one(&sg_inbuf, virtqueue_buf, TPM_BUFSIZE); |
| ret = virtqueue_add_sgs(dev->vq, sgs, 1, 1, virtqueue_buf, GFP_KERNEL); |
| if (ret) { |
| dev_err(&chip->dev, "failed virtqueue_add_sgs\n"); |
| return ret; |
| } |
| |
| /* Kick the other end of the virtqueue after having added a buffer. */ |
| did_kick = virtqueue_kick(dev->vq); |
| if (!did_kick) { |
| dev->needs_kick = true; |
| dev_notice(&chip->dev, "kick failed; will retry\n"); |
| |
| /* |
| * We return 0 anyway because what the caller doesn't know can't |
| * hurt them. They can call recv and it will retry the kick. If |
| * that works, everything is fine. |
| * |
| * If the retry in recv fails too, they will get -EBUSY from |
| * recv. |
| */ |
| } |
| |
| /* |
| * Hypervisor is now processing the TPM command asynchronously. It will |
| * read the command from the output buffer and write the response into |
| * the input buffer (which are the same buffer). When done, it will send |
| * back the buffers over virtio and the driver's virtio callback will |
| * complete dev->complete so that we know the response is ready to be |
| * read. |
| * |
| * It is important to have copied data out of the caller's buffer into |
| * the driver's virtqueue buffer because the caller is free to destroy |
| * their buffer when this call returns. We can't avoid copying by |
| * waiting here for the hypervisor to finish reading, because if that |
| * wait times out, we return and the caller may destroy their buffer |
| * while the hypervisor is continuing to read from it. |
| */ |
| dev->driver_has_buffer = false; |
| return 0; |
| } |
| |
| static int vtpm_op_recv(struct tpm_chip *chip, u8 *caller_buf, size_t len) |
| { |
| int ret; |
| struct vtpm_device *dev = dev_get_drvdata(&chip->dev); |
| u8 *virtqueue_buf = dev->virtqueue_buffer; |
| |
| dev_dbg(&chip->dev, "vtpm_op_recv\n"); |
| |
| /* |
| * Wait until the virtqueue buffer is owned by the driver. |
| * |
| * This will usually block while the hypervisor finishes processing the |
| * most recent TPM command. |
| */ |
| ret = vtpm_wait_for_buffer(dev, "waiting for TPM response"); |
| if (ret) { |
| return ret; |
| } |
| |
| dev_dbg(&chip->dev, "received %u bytes\n", dev->readable); |
| |
| if (dev->readable > len) { |
| dev_notice(&chip->dev, |
| "TPM response is bigger than receiver's buffer:" |
| " %u > %zu\n", |
| dev->readable, len); |
| return -EINVAL; |
| } |
| |
| /* Copy response over to the caller. */ |
| memcpy(caller_buf, virtqueue_buf, dev->readable); |
| |
| return dev->readable; |
| } |
| |
| static void vtpm_op_cancel(struct tpm_chip *chip) |
| { |
| /* |
| * Cancel is not called in this driver's use of tpm-interface.c. It may |
| * be triggered through tpm-sysfs but that can be implemented as needed. |
| * Be aware that tpm-sysfs performs cancellation without holding the |
| * tpm_mutex that protects our send and recv operations, so a future |
| * implementation will need to consider thread safety of concurrent |
| * send/recv and cancel. |
| */ |
| dev_notice(&chip->dev, "cancellation is not implemented\n"); |
| } |
| |
| static u8 vtpm_op_status(struct tpm_chip *chip) |
| { |
| /* |
| * Status is for TPM drivers that want tpm-interface.c to poll for |
| * completion before calling recv. Usually this is when the hardware |
| * needs to be polled i.e. there is no other way for recv to block on |
| * the TPM command completion. |
| * |
| * Polling goes until `(status & complete_mask) == complete_val`. This |
| * driver defines both complete_mask and complete_val as 0 and blocks on |
| * our own completion object in recv instead. |
| */ |
| return 0; |
| } |
| |
| static const struct tpm_class_ops vtpm_ops = { |
| .flags = TPM_OPS_AUTO_STARTUP, |
| .send = vtpm_op_send, |
| .recv = vtpm_op_recv, |
| .cancel = vtpm_op_cancel, |
| .status = vtpm_op_status, |
| .req_complete_mask = 0, |
| .req_complete_val = 0, |
| }; |
| |
| static void vtpm_virtio_complete(struct virtqueue *vq) |
| { |
| struct virtio_device *vdev = vq->vdev; |
| struct vtpm_device *dev = vdev->priv; |
| |
| complete(&dev->complete); |
| } |
| |
| static int vtpm_probe(struct virtio_device *vdev) |
| { |
| int err; |
| struct vtpm_device *dev; |
| struct virtqueue *vq; |
| struct tpm_chip *chip; |
| |
| dev_dbg(&vdev->dev, "vtpm_probe\n"); |
| |
| dev = kzalloc(sizeof(struct vtpm_device), GFP_KERNEL); |
| if (!dev) { |
| err = -ENOMEM; |
| dev_err(&vdev->dev, "failed kzalloc\n"); |
| goto err_dev_alloc; |
| } |
| vdev->priv = dev; |
| |
| vq = virtio_find_single_vq(vdev, vtpm_virtio_complete, "vtpm"); |
| if (IS_ERR(vq)) { |
| err = PTR_ERR(vq); |
| dev_err(&vdev->dev, "failed virtio_find_single_vq\n"); |
| goto err_virtio_find; |
| } |
| dev->vq = vq; |
| |
| chip = tpm_chip_alloc(&vdev->dev, &vtpm_ops); |
| if (IS_ERR(chip)) { |
| err = PTR_ERR(chip); |
| dev_err(&vdev->dev, "failed tpm_chip_alloc\n"); |
| goto err_chip_alloc; |
| } |
| dev_set_drvdata(&chip->dev, dev); |
| chip->flags |= TPM_CHIP_FLAG_TPM2; |
| dev->chip = chip; |
| |
| init_completion(&dev->complete); |
| dev->driver_has_buffer = true; |
| dev->needs_kick = false; |
| dev->readable = 0; |
| |
| /* |
| * Required in order to enable vq use in probe function for auto |
| * startup. |
| */ |
| virtio_device_ready(vdev); |
| |
| err = tpm_chip_register(dev->chip); |
| if (err) { |
| dev_err(&vdev->dev, "failed tpm_chip_register\n"); |
| goto err_chip_register; |
| } |
| |
| return 0; |
| |
| err_chip_register: |
| put_device(&dev->chip->dev); |
| err_chip_alloc: |
| vdev->config->del_vqs(vdev); |
| err_virtio_find: |
| kfree(dev); |
| err_dev_alloc: |
| return err; |
| } |
| |
| static void vtpm_remove(struct virtio_device *vdev) |
| { |
| struct vtpm_device *dev = vdev->priv; |
| |
| /* Undo tpm_chip_register. */ |
| tpm_chip_unregister(dev->chip); |
| |
| /* Undo tpm_chip_alloc. */ |
| put_device(&dev->chip->dev); |
| |
| vdev->config->reset(vdev); |
| vdev->config->del_vqs(vdev); |
| |
| kfree(dev); |
| } |
| |
| static struct virtio_device_id id_table[] = { |
| { |
| .device = VIRTIO_ID_TPM, |
| .vendor = VIRTIO_DEV_ANY_ID, |
| }, |
| {}, |
| }; |
| |
| static struct virtio_driver vtpm_driver = { |
| .driver.name = KBUILD_MODNAME, |
| .driver.owner = THIS_MODULE, |
| .id_table = id_table, |
| .probe = vtpm_probe, |
| .remove = vtpm_remove, |
| }; |
| |
| module_virtio_driver(vtpm_driver); |
| |
| MODULE_AUTHOR("David Tolnay (dtolnay@gmail.com)"); |
| MODULE_DESCRIPTION("Virtio vTPM Driver"); |
| MODULE_VERSION("1.0"); |
| MODULE_LICENSE("GPL"); |