drivers/virtio/virtio_magma.c

// Copyright 2019 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

#include <linux/anon_inodes.h>
#include <linux/cdev.h>
#include <linux/compat.h>
#include <linux/hashtable.h>
#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/virtio.h>
#include <linux/virtio_magma.h>
#include <linux/vmalloc.h>

#define VQ_DESCRIPTOR_SIZE PAGE_SIZE
#define CONNECTIONS_HASHTABLE_BITS 4
#define CONNECTION_OBJECTS_HASHTABLE_BITS 12
#define NOTIFICATION_MAX_BYTES 65536
#define COMMAND_OK(command, request_ptr, response_ptr)                         \
	((command)->request_size >= sizeof(*(request_ptr)) &&                  \
	 (command)->response_size >= sizeof(*(response_ptr)))
#define WAYLAND_DEVICE_PATH "/dev/wl0"
#define MESSAGE_CACHE_OBJECT_SIZE 64

struct virtmagma_info {
	dev_t dev_num;
	struct device *dev;
	struct class *class;
	struct cdev cdev;

	struct mutex vq_out_lock;
	struct virtqueue *vq_out;
	struct work_struct out_vq_work;
	wait_queue_head_t out_waitq;

	struct mutex instances_lock;
	struct idr instances;
};

enum virtmagma_connection_object_type {
	MAGMA_BUFFER,
	MAGMA_SEMAPHORE,
	MAGMA_CONTEXT
};

static const char *virtmagma_connection_object_type_string(
	enum virtmagma_connection_object_type type)
{
	switch (type) {
	case MAGMA_BUFFER:
		return "MAGMA_BUFFER";
	case MAGMA_SEMAPHORE:
		return "MAGMA_SEMAPHORE";
	case MAGMA_CONTEXT:
		return "MAGMA_CONTEXT";
	default:
		return "[UNKNOWN]";
	}
}

struct virtmagma_buffer {
	size_t size_requested;
	size_t size_allocated;
};

struct virtmagma_semaphore {
	uint8_t dummy;
};

struct virtmagma_context {
	uint8_t dummy;
};

struct virtmagma_connection;
struct virtmagma_connection_object {
	struct virtmagma_connection *parent_connection;
	enum virtmagma_connection_object_type type;
	uint64_t host_value;
	union {
		struct virtmagma_buffer buffer;
		struct virtmagma_semaphore semaphore;
		struct virtmagma_context context;
	};
	struct hlist_node node;
};

struct virtmagma_connection {
	struct virtmagma_instance *parent_instance;
	uint64_t host_value;
	DECLARE_HASHTABLE(objects, CONNECTION_OBJECTS_HASHTABLE_BITS);
	struct hlist_node node;
};

struct virtmagma_instance {
	struct virtmagma_info *vi;
	int id;
	DECLARE_HASHTABLE(connections, CONNECTIONS_HASHTABLE_BITS);
	int mmfd;
	struct {
		struct virtmagma_buffer *buffer;
		u64 phys_addr;
		bool pending;
	} mmap_params;
	struct {
		pid_t pid;
		pid_t tgid;
		char comm[TASK_COMM_LEN];
	} creator;
	struct kmem_cache *msg_cache;
	void *wayland_device_private_data;
};

struct virtmagma_virtio_command {
	void *request_ptr;
	size_t request_size;
	void *response_ptr;
	size_t response_size;
};

static void virtmagma_cache_ctor(void *p)
{
	memset(p, 0, MESSAGE_CACHE_OBJECT_SIZE);
}

static int vq_out_send_sync(struct virtmagma_info *vi,
			    struct virtmagma_virtio_command *command)
{
	int ret;
	DECLARE_COMPLETION_ONSTACK(finish_completion);
	struct scatterlist sg_out;
	struct scatterlist sg_in;
	struct scatterlist *sgs[] = { &sg_out, &sg_in };
	init_completion(&finish_completion);
	sg_init_one(&sg_out, command->request_ptr, command->request_size);
	sg_init_one(&sg_in, command->response_ptr, command->response_size);

	mutex_lock(&vi->vq_out_lock);
	while ((ret = virtqueue_add_sgs(vi->vq_out, sgs, 1, 1,
					&finish_completion, GFP_KERNEL)) ==
	       -ENOSPC) {
		mutex_unlock(&vi->vq_out_lock);
		if (!wait_event_timeout(vi->out_waitq, vi->vq_out->num_free > 0,
					HZ))
			return -EBUSY;
		mutex_lock(&vi->vq_out_lock);
	}
	if (!ret)
		virtqueue_kick(vi->vq_out);
	mutex_unlock(&vi->vq_out_lock);

	wait_for_completion(&finish_completion);

	return ret;
}

/* Verify that a virtio command's response matches its expected response.
   Note that a match indicates only that the proxying of the magma command
   has succeeded, not necessarily that the magma command itself did. */
static int virtmagma_check_expected_response_type(void *request, void *response)
{
	struct virtio_magma_ctrl_hdr *request_hdr = request;
	struct virtio_magma_ctrl_hdr *response_hdr = response;
	if (virtio_magma_expected_response_type(request_hdr->type) !=
	    response_hdr->type) {
		pr_warn("virtmagma: unexpected virtio response %s (%d) to request %s (%d)",
			virtio_magma_ctrl_type_string(response_hdr->type),
			response_hdr->type,
			virtio_magma_ctrl_type_string(request_hdr->type),
			request_hdr->type);
		return -EIO;
	}
	return 0;
}

static struct virtmagma_connection *
	get_connection(struct virtmagma_instance *instance, uint64_t id)
{
	struct virtmagma_connection *connection = NULL;
	hash_for_each_possible (instance->connections, connection, node, id) {
		if (connection->host_value == id)
			break;
	}
	if (!connection) {
		pr_warn("virtmagma: invalid connection id %lld", id);
	}
	return connection;
}

static struct virtmagma_connection_object *
	get_connection_object(struct virtmagma_connection *connection,
			      uint64_t id,
			      enum virtmagma_connection_object_type type)
{
	struct virtmagma_connection_object *object = NULL;
	hash_for_each_possible (connection->objects, object, node, id) {
		if (object->type == type && object->host_value == id)
			break;
	}
	if (!object) {
		pr_warn("virtmagma: invalid %s object id %lld",
			virtmagma_connection_object_type_string(type), id);
	}
	return object;
}

static int control_type(void *p)
{
	return ((struct virtio_magma_ctrl_hdr *)p)->type;
}

static int release_buffer(struct virtmagma_buffer *buffer)
{
	int ret;
	struct virtio_magma_release_buffer_ctrl *request;
	struct virtio_magma_release_buffer_resp *response;
	struct virtmagma_virtio_command command;
	struct virtmagma_connection_object *object = container_of(
		buffer, struct virtmagma_connection_object, buffer);
	BUILD_BUG_ON(sizeof(*request) > MESSAGE_CACHE_OBJECT_SIZE);
	BUILD_BUG_ON(sizeof(*response) > MESSAGE_CACHE_OBJECT_SIZE);

	request = kmem_cache_alloc(
		object->parent_connection->parent_instance->msg_cache,
		GFP_KERNEL);
	if (!request)
		return -ENOMEM;

	response = kmem_cache_alloc(
		object->parent_connection->parent_instance->msg_cache,
		GFP_KERNEL);
	if (!response) {
		ret = -ENOMEM;
		goto free_request;
	}

	request->hdr.type = VIRTIO_MAGMA_CMD_RELEASE_BUFFER;
	request->connection = object->parent_connection->host_value;
	request->buffer = object->host_value;

	command.request_ptr = request;
	command.request_size = sizeof(*request);
	command.response_ptr = response;
	command.response_size = sizeof(*response);

	ret = vq_out_send_sync(object->parent_connection->parent_instance->vi,
			       &command);
	if (ret)
		goto free_response;

	if (response->hdr.type != VIRTIO_MAGMA_RESP_RELEASE_BUFFER)
		ret = -EIO;

free_response:
	kmem_cache_free(object->parent_connection->parent_instance->msg_cache,
			response);

free_request:
	kmem_cache_free(object->parent_connection->parent_instance->msg_cache,
			request);

	return ret;
}

static int release_semaphore(struct virtmagma_semaphore *semaphore)
{
	int ret;
	struct virtio_magma_release_semaphore_ctrl *request;
	struct virtio_magma_release_semaphore_resp *response;
	struct virtmagma_virtio_command command;
	struct virtmagma_connection_object *object = container_of(
		semaphore, struct virtmagma_connection_object, semaphore);
	BUILD_BUG_ON(sizeof(*request) > MESSAGE_CACHE_OBJECT_SIZE);
	BUILD_BUG_ON(sizeof(*response) > MESSAGE_CACHE_OBJECT_SIZE);

	request = kmem_cache_alloc(
		object->parent_connection->parent_instance->msg_cache,
		GFP_KERNEL);
	if (!request)
		return -ENOMEM;

	response = kmem_cache_alloc(
		object->parent_connection->parent_instance->msg_cache,
		GFP_KERNEL);
	if (!response) {
		ret = -ENOMEM;
		goto free_request;
	}

	request->hdr.type = VIRTIO_MAGMA_CMD_RELEASE_SEMAPHORE;
	request->connection = object->parent_connection->host_value;
	request->semaphore = object->host_value;

	command.request_ptr = request;
	command.request_size = sizeof(*request);
	command.response_ptr = response;
	command.response_size = sizeof(*response);

	ret = vq_out_send_sync(object->parent_connection->parent_instance->vi,
			       &command);
	if (ret)
		goto free_response;

	if (response->hdr.type != VIRTIO_MAGMA_RESP_RELEASE_SEMAPHORE)
		ret = -EIO;

free_response:
	kmem_cache_free(object->parent_connection->parent_instance->msg_cache,
			response);

free_request:
	kmem_cache_free(object->parent_connection->parent_instance->msg_cache,
			request);

	return ret;
}

static int release_context(struct virtmagma_context *context)
{
	int ret;
	struct virtio_magma_release_context_ctrl *request;
	struct virtio_magma_release_context_resp *response;
	struct virtmagma_virtio_command command;
	struct virtmagma_connection_object *object = container_of(
		context, struct virtmagma_connection_object, context);
	BUILD_BUG_ON(sizeof(*request) > MESSAGE_CACHE_OBJECT_SIZE);
	BUILD_BUG_ON(sizeof(*response) > MESSAGE_CACHE_OBJECT_SIZE);

	request = kmem_cache_alloc(
		object->parent_connection->parent_instance->msg_cache,
		GFP_KERNEL);
	if (!request)
		return -ENOMEM;

	response = kmem_cache_alloc(
		object->parent_connection->parent_instance->msg_cache,
		GFP_KERNEL);
	if (!response) {
		ret = -ENOMEM;
		goto free_request;
	}

	request->hdr.type = VIRTIO_MAGMA_CMD_RELEASE_CONTEXT;
	request->connection = object->parent_connection->host_value;
	request->context_id = object->host_value;

	command.request_ptr = request;
	command.request_size = sizeof(*request);
	command.response_ptr = response;
	command.response_size = sizeof(*response);

	ret = vq_out_send_sync(object->parent_connection->parent_instance->vi,
			       &command);
	if (ret)
		goto free_response;

	if (response->hdr.type != VIRTIO_MAGMA_RESP_RELEASE_CONTEXT)
		ret = -EIO;

free_response:
	kmem_cache_free(object->parent_connection->parent_instance->msg_cache,
			response);

free_request:
	kmem_cache_free(object->parent_connection->parent_instance->msg_cache,
			request);

	return ret;
}

static int release_connection(struct virtmagma_connection *connection)
{
	int ret;
	int bkt;
	struct virtmagma_connection_object *object;
	struct virtio_magma_release_connection_ctrl *request;
	struct virtio_magma_release_connection_resp *response;
	struct virtmagma_virtio_command command;
	uint64_t leaked_buffers = 0;
	uint64_t leaked_semaphores = 0;
	uint64_t leaked_contexts = 0;

	/* first, release any child objects */

	hash_for_each (connection->objects, bkt, object, node) {
		switch (object->type) {
		case MAGMA_BUFFER:
			release_buffer(&object->buffer);
			++leaked_buffers;
			break;
		case MAGMA_CONTEXT:
			release_context(&object->context);
			++leaked_contexts;
			break;
		case MAGMA_SEMAPHORE:
			release_semaphore(&object->semaphore);
			++leaked_semaphores;
			break;
		default:
			pr_err("virtmagma: unknown connection object (%d)",
			       object->type);
			break;
		}
	}
	if (leaked_buffers || leaked_semaphores || leaked_contexts) {
		pr_info("virtmagma: connection %lld from command %s closed with leaked objects:\n",
			connection->host_value,
			connection->parent_instance->creator.comm);
		pr_cont("virtmagma: buffers: %lld\n", leaked_buffers);
		pr_cont("virtmagma: semaphores: %lld\n", leaked_semaphores);
		pr_cont("virtmagma: contexts: %lld\n", leaked_contexts);
	}

	/* now release the connection */

	BUILD_BUG_ON(sizeof(*request) > MESSAGE_CACHE_OBJECT_SIZE);
	BUILD_BUG_ON(sizeof(*response) > MESSAGE_CACHE_OBJECT_SIZE);

	request = kmem_cache_alloc(connection->parent_instance->msg_cache,
				   GFP_KERNEL);
	if (!request)
		return -ENOMEM;

	response = kmem_cache_alloc(connection->parent_instance->msg_cache,
				    GFP_KERNEL);
	if (!response) {
		ret = -ENOMEM;
		goto free_request;
	}

	request->hdr.type = VIRTIO_MAGMA_CMD_RELEASE_CONNECTION;
	request->connection = connection->host_value;

	command.request_ptr = request;
	command.request_size = sizeof(*request);
	command.response_ptr = response;
	command.response_size = sizeof(*response);

	ret = vq_out_send_sync(connection->parent_instance->vi, &command);
	if (ret)
		goto free_response;

	if (response->hdr.type != VIRTIO_MAGMA_RESP_RELEASE_CONNECTION)
		ret = -EIO;

free_response:
	kmem_cache_free(connection->parent_instance->msg_cache, response);

free_request:
	kmem_cache_free(connection->parent_instance->msg_cache, request);

	return ret;
}

static int destroy_connection(struct virtmagma_connection *connection)
{
	int bkt;
	struct virtmagma_connection_object *object;
	hash_for_each (connection->objects, bkt, object, node) {
		hash_del(&object->node);
		kfree(object);
	}
	hash_del(&connection->node);
	kfree(connection);
	return 0;
}

static int destroy_instance(int id, void *p, void *data)
{
	struct virtmagma_instance *instance = p;
	struct virtmagma_connection *connection;
	int bkt;
	uint64_t leaked_connections = 0;

	instance = p;

	hash_for_each (instance->connections, bkt, connection, node) {
		++leaked_connections;
	}
	if (leaked_connections) {
		pr_info("virtmagma: command %s exited with %lld leaked connections",
			instance->creator.comm, leaked_connections);
	}
	hash_for_each (instance->connections, bkt, connection, node) {
		release_connection(connection);
		destroy_connection(connection);
	}

	kmem_cache_destroy(instance->msg_cache);

	kfree(instance);
	return 0;
}

static int virtmagma_mmfd_mmap(struct file *filp, struct vm_area_struct *vma)
{
	struct virtmagma_instance *instance = filp->private_data;
	unsigned long vm_size = vma->vm_end - vma->vm_start;
	size_t max_map_size;
	int ret;

	if (!instance)
		return -ENODEV;

	if (!instance->mmap_params.pending) {
		pr_warn("virtmagma: user called mmap on the mmfd without first submitting a magma_map ioctl");
		return -EINVAL;
	}

	instance->mmap_params.pending = false;

	max_map_size = instance->mmap_params.buffer->size_allocated;
	max_map_size = PAGE_ALIGN(max_map_size);

	if (vm_size > max_map_size) {
		pr_warn("virtmagma: user tried to mmap with a size (%ld) larger than the buffer's size (%ld)",
			vm_size, max_map_size);
		return -EINVAL;
	}

	ret = io_remap_pfn_range(vma, vma->vm_start,
				 instance->mmap_params.phys_addr / PAGE_SIZE,
				 vm_size, vma->vm_page_prot);

	if (ret)
		return ret;

	vma->vm_flags |= VM_PFNMAP | VM_IO | VM_DONTEXPAND | VM_DONTDUMP;

	return ret;
}

static const struct file_operations virtmagma_mmfd_fops = {
	.mmap = virtmagma_mmfd_mmap,
};

static int create_instance(struct virtmagma_info *vi,
			   struct virtmagma_instance **instance_out)
{
	int ret;
	struct file *filp;
	struct virtmagma_instance *instance;

	*instance_out = NULL;

	instance = kzalloc(sizeof(*instance), GFP_KERNEL);
	if (!instance)
		return -ENOMEM;
	instance->vi = vi;

	mutex_lock(&vi->instances_lock);
	ret = idr_alloc(&vi->instances, instance, 1, -1, GFP_KERNEL);
	mutex_unlock(&vi->instances_lock);
	if (ret <= 0) {
		ret = -ENOMEM;
		goto free_instance;
	}
	instance->id = ret;

	ret = anon_inode_getfd("[virtmagma_mmfd]", &virtmagma_mmfd_fops,
			       instance, O_RDWR);
	if (ret < 0) {
		goto free_instance;
	}
	instance->mmfd = ret;

	hash_init(instance->connections);
	instance->creator.pid = current->pid;
	instance->creator.tgid = current->tgid;
	memcpy(instance->creator.comm, current->comm,
	       sizeof(instance->creator.comm));
	instance->creator.comm[sizeof(instance->creator.comm) - 1] = 0;

	filp = filp_open(WAYLAND_DEVICE_PATH, O_RDWR, 0);
	if (IS_ERR_OR_NULL(filp)) {
		pr_warn("virtmagma: failed to open wayland device at %s\n",
			WAYLAND_DEVICE_PATH);
		pr_cont("virtmagma: magma_export will not be available\n");
	} else {
		instance->wayland_device_private_data = filp->private_data;
		filp_close(filp, 0);
	}

	instance->msg_cache =
		kmem_cache_create("virtmagma_cache", MESSAGE_CACHE_OBJECT_SIZE,
				  MESSAGE_CACHE_OBJECT_SIZE, 0,
				  virtmagma_cache_ctor);
	if (!instance->msg_cache) {
		pr_err("virtmagma: failed to create message cache");
		return -ENOMEM;
	}

	*instance_out = instance;

	return 0;

free_instance:
	kfree(instance);

	return ret;
}

static int virtmagma_command_magma_create_connection2(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection;
	struct virtio_magma_create_connection2_ctrl *request =
		command->request_ptr;
	struct virtio_magma_create_connection2_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* pass on magma errors without creating a connection object */
	if (response->result_return) {
		pr_warn("virtmagma: magma_create_connection returned %d",
			(int32_t)response->result_return);
		return 0; /* the ioctl is still successful */
	}

	connection = kzalloc(sizeof(*connection), GFP_KERNEL);
	if (!connection)
		return -ENOMEM;

	connection->parent_instance = instance;
	connection->host_value = response->connection_out;
	hash_init(connection->objects);

	hash_add(instance->connections, &connection->node,
		 connection->host_value);

	return 0;
}

static int virtmagma_command_magma_release_connection(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection = NULL;
	struct virtio_magma_release_connection_ctrl *request =
		command->request_ptr;
	struct virtio_magma_release_connection_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	ret = release_connection(connection);
	if (ret)
		return ret;

	return destroy_connection(connection);
}

static int virtmagma_command_magma_create_context(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection;
	struct virtmagma_connection_object *object;
	struct virtio_magma_create_context_ctrl *request = command->request_ptr;
	struct virtio_magma_create_context_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* magma_create_context does not return errors */

	object = kzalloc(sizeof(*object), GFP_KERNEL);
	if (!object)
		return -ENOMEM;

	object->parent_connection = connection;
	object->host_value = response->context_id_out;
	object->type = MAGMA_CONTEXT;

	hash_add(connection->objects, &object->node, object->host_value);

	return 0;
}

static int virtmagma_command_magma_release_context(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection = NULL;
	struct virtmagma_connection_object *object = NULL;
	struct virtio_magma_release_context_ctrl *request =
		command->request_ptr;
	struct virtio_magma_release_context_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	object = get_connection_object(connection, request->context_id,
				       MAGMA_CONTEXT);

	// Send the call even if bad context_id; tests check that magma error
	// is set.
	ret = vq_out_send_sync(instance->vi, command);

	if (object) {
		hash_del(&object->node);
		kfree(object);
	}

	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	return 0;
}

static int virtmagma_command_magma_create_buffer(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection;
	struct virtmagma_connection_object *object;
	struct virtio_magma_create_buffer_ctrl *request = command->request_ptr;
	struct virtio_magma_create_buffer_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* pass on magma errors without creating a buffer object */
	if (response->result_return) {
		pr_warn("virtmagma: magma_create_buffer returned %d",
			(int32_t)response->result_return);
		return 0; /* the ioctl is still successful */
	}

	object = kzalloc(sizeof(*object), GFP_KERNEL);
	if (!object)
		return -ENOMEM;

	object->parent_connection = connection;
	object->host_value = response->buffer_out;
	object->type = MAGMA_BUFFER;
	object->buffer.size_requested = request->size;
	object->buffer.size_allocated = response->size_out;

	hash_add(connection->objects, &object->node, object->host_value);

	return 0;
}

static int virtmagma_command_magma_release_buffer(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection = NULL;
	struct virtmagma_connection_object *object = NULL;
	struct virtio_magma_release_buffer_ctrl *request = command->request_ptr;
	struct virtio_magma_release_buffer_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	object = get_connection_object(connection, request->buffer,
				       MAGMA_BUFFER);
	if (!object)
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	hash_del(&object->node);
	kfree(object);

	return 0;
}

static int virtmagma_command_magma_execute_command_buffer_with_resources(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	struct virtio_magma_execute_command_buffer_with_resources_ctrl *request =
		command->request_ptr;
	struct virtio_magma_execute_command_buffer_with_resources_resp
		*response = command->response_ptr;
	struct virtmagma_connection *connection;
	struct virtmagma_command_buffer virt_command_buffer;
	char *dst_ptr;
	int ret;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	if (!get_connection_object(connection, request->context_id,
				   MAGMA_CONTEXT))
		return -EINVAL;

	/* The virtmagma_comand_buffer includes the resources and semaphore data
	   and their lengths, so we ignore the corresponding members of the request. */
	ret = copy_from_user(&virt_command_buffer,
			     (void *)request->command_buffer,
			     sizeof(virt_command_buffer));
	if (ret)
		return ret;

	/* reallocate request buffer with enough space for the structures */
	command->request_size = sizeof(*request) +
				virt_command_buffer.command_buffer_size +
				virt_command_buffer.resource_size +
				virt_command_buffer.semaphore_size;
	/* memory will be freed by the caller */
	dst_ptr = kzalloc(command->request_size, GFP_KERNEL);
	if (!dst_ptr)
		return -ENOMEM;

	memcpy(dst_ptr, request, sizeof(*request));
	command->request_ptr = dst_ptr;

	dst_ptr += sizeof(*request);
	ret = copy_from_user(dst_ptr,
			     (void *)virt_command_buffer.command_buffer,
			     virt_command_buffer.command_buffer_size);
	if (ret)
		return ret;

	dst_ptr += virt_command_buffer.command_buffer_size;
	ret = copy_from_user(dst_ptr, (void *)virt_command_buffer.resources,
			     virt_command_buffer.resource_size);
	if (ret)
		return ret;

	dst_ptr += virt_command_buffer.resource_size;
	ret = copy_from_user(dst_ptr, (void *)virt_command_buffer.semaphores,
			     virt_command_buffer.semaphore_size);
	if (ret)
		return ret;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	return 0;
}

static int virtmagma_command_magma_create_semaphore(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection;
	struct virtmagma_connection_object *object;
	struct virtio_magma_create_semaphore_ctrl *request =
		command->request_ptr;
	struct virtio_magma_create_semaphore_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* pass on magma errors without creating a semaphore object */
	if (response->result_return) {
		pr_warn("virtmagma: magma_create_semaphore returned %d",
			(int32_t)response->result_return);
		return 0; /* the ioctl is still successful */
	}

	object = kzalloc(sizeof(*object), GFP_KERNEL);
	if (!object)
		return -ENOMEM;

	object->parent_connection = connection;
	object->host_value = response->semaphore_out;
	object->type = MAGMA_SEMAPHORE;

	hash_add(connection->objects, &object->node, object->host_value);

	return 0;
}

static int virtmagma_command_magma_release_semaphore(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection = NULL;
	struct virtmagma_connection_object *object = NULL;
	struct virtio_magma_release_semaphore_ctrl *request =
		command->request_ptr;
	struct virtio_magma_release_semaphore_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	object = get_connection_object(connection, request->semaphore,
				       MAGMA_SEMAPHORE);
	if (!object)
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	hash_del(&object->node);
	kfree(object);

	return 0;
}

static int virtmagma_command_magma_map(struct virtmagma_instance *instance,
				       struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtmagma_connection *connection;
	struct virtmagma_connection_object *object;
	struct virtio_magma_map_ctrl *request = command->request_ptr;
	/* this ioctl has a size_t output parameter appended */
	struct {
		struct virtio_magma_map_resp virtio_response;
		size_t size_to_mmap_out;
	} *response = command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	connection = get_connection(instance, request->connection);
	if (!connection)
		return -EINVAL;

	object = get_connection_object(connection, request->buffer,
				       MAGMA_BUFFER);
	if (!object)
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* store parameters for subsequent mmap call */

	instance->mmap_params.buffer = &object->buffer;
	instance->mmap_params.phys_addr = response->virtio_response.addr_out;
	instance->mmap_params.pending = true;

	/* user must use the returned size in its subsequent mmap call */

	response->size_to_mmap_out = object->buffer.size_requested;

	return 0;
}

static int
	virtmagma_command_magma_poll(struct virtmagma_instance *instance,
				     struct virtmagma_virtio_command *command)
{
	int ret;
	size_t items_size;
	struct virtio_magma_poll_ctrl *request = command->request_ptr;
	struct virtio_magma_poll_resp *response = command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	/* reallocate request buffer with enough space for the items */
	/* note: request->count is used as the byte count */
	items_size = request->count;
	command->request_size = sizeof(*request) + items_size;
	command->request_ptr = kzalloc(command->request_size, GFP_KERNEL);
	if (!command->request_ptr)
		return -ENOMEM;

	memcpy(command->request_ptr, request, sizeof(*request));
	ret = copy_from_user((char *)command->request_ptr + sizeof(*request),
			     (void *)request->items, items_size);
	if (ret)
		return ret;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	return copy_to_user((void *)request->items,
			    (char *)command->request_ptr + sizeof(*request),
			    items_size);
}

static int virtmagma_command_magma_read_notification_channel(
	struct virtmagma_instance *instance,
	struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtio_magma_read_notification_channel_ctrl *request =
		command->request_ptr;
	struct virtio_magma_read_notification_channel_resp *response =
		command->response_ptr;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	/* reallocate response buffer with additional space for notification data.
	   note that the size is not modified, as we only want the response struct
	   itself to be copied back to the user by our caller */

	command->response_ptr = response =
		kzalloc(sizeof(*response) + NOTIFICATION_MAX_BYTES, GFP_KERNEL);
	if (!command->response_ptr)
		return -ENOMEM;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* pass on magma errors without writing to the buffer */
	if (response->result_return) {
		pr_warn("virtmagma: magma_read_notification_channel returned %d",
			(int32_t)response->result_return);
		return 0; /* the ioctl is still successful */
	}

	if (response->buffer_size_out > request->buffer_size) {
		pr_err("virtmagma: magma_read_notification_channel returned buffer_size_out (%lld) larger than buffer_size (%lld)",
		       response->buffer_size_out, request->buffer_size);
		return -EIO;
	}

	return copy_to_user((void *)request->buffer,
			    (char *)command->response_ptr + sizeof(*response),
			    response->buffer_size_out);
}

#if IS_ENABLED(CONFIG_VIRTIO_WL)
/* use the implementation in the virtio_wl module */
extern int virtwl_create_fd_for_vfd(void *filp_private_data, uint32_t vfd_id);
#else
#define virtwl_create_fd_for_vfd(a, b) (-ENODEV)
#endif

static int
	virtmagma_command_magma_export(struct virtmagma_instance *instance,
				       struct virtmagma_virtio_command *command)
{
	int ret;
	struct virtio_magma_export_ctrl *request = command->request_ptr;
	struct virtio_magma_export_resp *response = command->response_ptr;

	if (!instance->wayland_device_private_data)
		return -ENODEV;

	if (!COMMAND_OK(command, request, response))
		return -EINVAL;

	ret = vq_out_send_sync(instance->vi, command);
	if (ret)
		return ret;

	ret = virtmagma_check_expected_response_type(request, response);
	if (ret)
		return ret;

	/* pass on magma errors without creating a vfd */
	if (response->result_return) {
		pr_warn("virtmagma: magma_export returned %d",
			(int32_t)response->result_return);
		return 0; /* the ioctl is still successful */
	}

	ret = virtwl_create_fd_for_vfd(instance->wayland_device_private_data,
				       response->buffer_handle_out);
	if (ret < 0) {
		pr_err("virtmagma: failed to get vfd creation info for vfd id %lld",
		       response->buffer_handle_out);
		return ret;
	}

	response->buffer_handle_out = ret;

	return 0;
}

static int virtmagma_ioctl_handshake(struct file *filp, void __user *ptr)
{
	struct virtmagma_ioctl_args_handshake ioctl_args;
	int ret = copy_from_user(&ioctl_args, ptr, sizeof(ioctl_args));
	if (ret)
		return ret;
	if (ioctl_args.handshake_inout != VIRTMAGMA_HANDSHAKE_SEND)
		return -EINVAL;
	ioctl_args.handshake_inout = VIRTMAGMA_HANDSHAKE_RECV;
	ioctl_args.version_out = VIRTMAGMA_VERSION;
	return copy_to_user(ptr, &ioctl_args, sizeof(ioctl_args));
}

static int virtmagma_ioctl_get_mmfd(struct file *filp, void __user *ptr)
{
	struct virtmagma_ioctl_args_get_mmfd ioctl_args;
	struct virtmagma_instance *instance = filp->private_data;
	if (!instance)
		return -ENODEV;
	ioctl_args.fd_out = instance->mmfd;
	return copy_to_user(ptr, &ioctl_args, sizeof(ioctl_args));
}

int virtmagma_ioctl_magma_command(struct file *filp, void __user *ptr)
{
	struct virtmagma_ioctl_args_magma_command ioctl_args;
	struct virtmagma_virtio_command command;
	void *request;
	void *response;
	int request_type;
	int ret;
	struct virtmagma_instance *instance = filp->private_data;
	command.request_ptr = NULL;
	command.response_ptr = NULL;

	if (!instance)
		return -ENODEV;

	if (instance->mmap_params.pending) {
		pr_warn("virtmagma: user failed to mmap on the mmfd after submitting a magma_map ioctl");
		return -EINVAL;
	}

	/* copy in command arguments */

	ret = copy_from_user(&ioctl_args, ptr, sizeof(ioctl_args));
	if (ret)
		return ret;

	/* verify userspace-provided pointers are accessible */

	ret = !access_ok(VERIFY_READ, (void *)ioctl_args.request_address,
			 ioctl_args.request_size);
	if (ret)
		return -EFAULT;
	ret = !access_ok(VERIFY_WRITE, (void *)ioctl_args.response_address,
			 ioctl_args.response_size);
	if (ret)
		return -EFAULT;

	/* allocate buffers and copy in userspace data */

	if (ioctl_args.request_size <= MESSAGE_CACHE_OBJECT_SIZE)
		request = kmem_cache_alloc(instance->msg_cache, GFP_KERNEL);
	else
		request = kzalloc(ioctl_args.request_size, GFP_KERNEL);
	if (!request)
		return -ENOMEM;
	if (ioctl_args.response_size <= MESSAGE_CACHE_OBJECT_SIZE)
		response = kmem_cache_alloc(instance->msg_cache, GFP_KERNEL);
	else
		response = kzalloc(ioctl_args.response_size, GFP_KERNEL);
	if (!response) {
		ret = -ENOMEM;
		goto free_request;
	}
	command.request_ptr = request;
	command.response_ptr = response;
	command.request_size = ioctl_args.request_size;
	command.response_size = ioctl_args.response_size;

	ret = copy_from_user(command.request_ptr,
			     (void *)ioctl_args.request_address,
			     ioctl_args.request_size);
	if (ret)
		goto free_response;

	request_type = control_type(command.request_ptr);
	switch (request_type) {
	case VIRTIO_MAGMA_CMD_CREATE_CONNECTION2:
		ret = virtmagma_command_magma_create_connection2(instance,
								 &command);
		break;
	case VIRTIO_MAGMA_CMD_RELEASE_CONNECTION:
		ret = virtmagma_command_magma_release_connection(instance,
								 &command);
		break;
	case VIRTIO_MAGMA_CMD_CREATE_CONTEXT:
		ret = virtmagma_command_magma_create_context(instance,
							     &command);
		break;
	case VIRTIO_MAGMA_CMD_RELEASE_CONTEXT:
		ret = virtmagma_command_magma_release_context(instance,
							      &command);
		break;
	case VIRTIO_MAGMA_CMD_CREATE_BUFFER:
		ret = virtmagma_command_magma_create_buffer(instance, &command);
		break;
	case VIRTIO_MAGMA_CMD_RELEASE_BUFFER:
		ret = virtmagma_command_magma_release_buffer(instance,
							     &command);
		break;
	case VIRTIO_MAGMA_CMD_EXECUTE_COMMAND_BUFFER_WITH_RESOURCES:
		ret = virtmagma_command_magma_execute_command_buffer_with_resources(
			instance, &command);
		break;
	case VIRTIO_MAGMA_CMD_CREATE_SEMAPHORE:
		ret = virtmagma_command_magma_create_semaphore(instance,
							       &command);
		break;
	case VIRTIO_MAGMA_CMD_RELEASE_SEMAPHORE:
		ret = virtmagma_command_magma_release_semaphore(instance,
								&command);
		break;
	case VIRTIO_MAGMA_CMD_MAP:
		ret = virtmagma_command_magma_map(instance, &command);
		break;
	case VIRTIO_MAGMA_CMD_POLL:
		ret = virtmagma_command_magma_poll(instance, &command);
		break;
	case VIRTIO_MAGMA_CMD_READ_NOTIFICATION_CHANNEL:
		ret = virtmagma_command_magma_read_notification_channel(
			instance, &command);
		break;
	case VIRTIO_MAGMA_CMD_EXPORT:
		ret = virtmagma_command_magma_export(instance, &command);
		break;
	/* pass-through handlers */
	case VIRTIO_MAGMA_CMD_QUERY2:
	case VIRTIO_MAGMA_CMD_GET_ERROR:
	case VIRTIO_MAGMA_CMD_GET_BUFFER_ID:
	case VIRTIO_MAGMA_CMD_GET_BUFFER_SIZE:
	case VIRTIO_MAGMA_CMD_CLEAN_CACHE:
	case VIRTIO_MAGMA_CMD_SET_CACHE_POLICY:
	case VIRTIO_MAGMA_CMD_GET_BUFFER_CACHE_POLICY:
	case VIRTIO_MAGMA_CMD_GET_BUFFER_IS_MAPPABLE:
	case VIRTIO_MAGMA_CMD_UNMAP:
	case VIRTIO_MAGMA_CMD_MAP_BUFFER_GPU:
	case VIRTIO_MAGMA_CMD_UNMAP_BUFFER_GPU:
	case VIRTIO_MAGMA_CMD_COMMIT_BUFFER:
	case VIRTIO_MAGMA_CMD_GET_SEMAPHORE_ID:
	case VIRTIO_MAGMA_CMD_SIGNAL_SEMAPHORE:
	case VIRTIO_MAGMA_CMD_RESET_SEMAPHORE:
	case VIRTIO_MAGMA_CMD_GET_NOTIFICATION_CHANNEL_HANDLE:
	case VIRTIO_MAGMA_CMD_DEVICE_IMPORT:
	case VIRTIO_MAGMA_CMD_DEVICE_RELEASE:
	case VIRTIO_MAGMA_CMD_RELEASE_BUFFER_HANDLE:
		ret = vq_out_send_sync(instance->vi, &command);
		if (!ret)
			ret = virtmagma_check_expected_response_type(
				command.request_ptr, command.response_ptr);
		break;
	default:
		pr_warn("virtmagma: command %s (%d) not implemented",
			virtio_magma_ctrl_type_string(request_type),
			request_type);
		ret = -EINVAL;
		break;
	}
	if (ret) {
		pr_err("virtmagma: error handling command %s (%d)",
		       virtio_magma_ctrl_type_string(request_type),
		       request_type);
		dump_stack();
		goto free_response;
	}

	/* copy responses back to userspace */

	ret = copy_to_user((void *)ioctl_args.response_address,
			   command.response_ptr, ioctl_args.response_size);

free_response:
	if (ioctl_args.request_size <= MESSAGE_CACHE_OBJECT_SIZE)
		kmem_cache_free(instance->msg_cache, request);
	else
		kfree(request);

free_request:
	if (ioctl_args.response_size <= MESSAGE_CACHE_OBJECT_SIZE)
		kmem_cache_free(instance->msg_cache, response);
	else
		kfree(response);

	/* Some of the handlers above may override the command members and hand
	   off allocation ownership back to us. Free them now. */

	if (command.request_ptr && command.request_ptr != request)
		kfree(command.request_ptr);

	if (command.response_ptr && command.response_ptr != response)
		kfree(command.response_ptr);

	return ret;
}

static int virtmagma_open(struct inode *inodep, struct file *filp)
{
	int ret;
	struct virtmagma_instance *instance;
	struct virtmagma_info *vi =
		container_of(inodep->i_cdev, struct virtmagma_info, cdev);

	ret = create_instance(vi, &instance);
	if (ret)
		return ret;

	filp->private_data = instance;

	return 0;
}

static int virtmagma_release(struct inode *inodep, struct file *filp)
{
	struct virtmagma_instance *instance = filp->private_data;
	return destroy_instance(instance->id, instance, instance->vi);
}

static void vq_out_work_handler(struct work_struct *work)
{
	struct virtmagma_info *vi =
		container_of(work, struct virtmagma_info, out_vq_work);
	unsigned int len;
	struct completion *finish_completion;
	bool wake_waitq = false;

	mutex_lock(&vi->vq_out_lock);
	while ((finish_completion = virtqueue_get_buf(vi->vq_out, &len)) !=
	       NULL) {
		wake_waitq = true;
		complete(finish_completion);
	}
	mutex_unlock(&vi->vq_out_lock);

	if (wake_waitq)
		wake_up_interruptible_all(&vi->out_waitq);
}

static void vq_out_cb(struct virtqueue *vq)
{
	struct virtmagma_info *vi = vq->vdev->priv;
	schedule_work(&vi->out_vq_work);
}

static long virtmagma_ioctl_common(struct file *filp, unsigned int cmd,
				   void __user *ptr)
{
	switch (cmd) {
	case VIRTMAGMA_IOCTL_HANDSHAKE:
		return virtmagma_ioctl_handshake(filp, ptr);
	case VIRTMAGMA_IOCTL_GET_MMFD:
		return virtmagma_ioctl_get_mmfd(filp, ptr);
	case VIRTMAGMA_IOCTL_MAGMA_COMMAND:
		return virtmagma_ioctl_magma_command(filp, ptr);
	default:
		return -ENOTTY;
	}
}

static long virtmagma_ioctl(struct file *filp, unsigned int cmd,
			    unsigned long arg)
{
	return virtmagma_ioctl_common(filp, cmd, (void __user *)arg);
}

#ifdef CONFIG_COMPAT
static long virtmagma_ioctl_compat(struct file *filp, unsigned int cmd,
				   unsigned long arg)
{
	return virtmagma_ioctl_common(filp, cmd, compat_ptr(arg));
}
#else
#define virtmagma_ioctl_compat NULL
#endif

static const struct file_operations virtmagma_fops = {
	.open = virtmagma_open,
	.unlocked_ioctl = virtmagma_ioctl,
	.compat_ioctl = virtmagma_ioctl_compat,
	.release = virtmagma_release,
};

static int virtmagma_probe(struct virtio_device *vdev)
{
	int ret;
	struct virtmagma_info *vi = NULL;
	static const char *vq_out_name = "out";
	vq_callback_t *callback = &vq_out_cb;

	vi = kzalloc(sizeof(struct virtmagma_info), GFP_KERNEL);
	if (!vi)
		return -ENOMEM;

	vdev->priv = vi;

	ret = alloc_chrdev_region(&vi->dev_num, 0, 1, "magma");
	if (ret) {
		ret = -ENOMEM;
		pr_warn("virtmagma: failed to allocate wl chrdev region: %d\n",
			ret);
		goto free_vi;
	}

	vi->class = class_create(THIS_MODULE, "magma");
	if (IS_ERR(vi->class)) {
		ret = PTR_ERR(vi->class);
		pr_warn("virtmagma: failed to create magma class: %d\n", ret);
		goto unregister_region;
	}

	vi->dev = device_create(vi->class, NULL, vi->dev_num, vi, "magma%d", 0);
	if (IS_ERR(vi->dev)) {
		ret = PTR_ERR(vi->dev);
		pr_warn("virtmagma: failed to create magma0 device: %d\n", ret);
		goto destroy_class;
	}

	cdev_init(&vi->cdev, &virtmagma_fops);
	ret = cdev_add(&vi->cdev, vi->dev_num, 1);
	if (ret) {
		pr_warn("virtmagma: failed to add virtio magma character device to system: %d\n",
			ret);
		goto destroy_device;
	}

	mutex_init(&vi->vq_out_lock);
	mutex_init(&vi->instances_lock);
	idr_init(&vi->instances);

	ret = virtio_find_vqs(vdev, 1, &vi->vq_out, &callback, &vq_out_name,
			      NULL);
	if (ret) {
		pr_warn("virtmagma: failed to find virtio magma out queue: %d\n",
			ret);
		goto del_cdev;
	}

	INIT_WORK(&vi->out_vq_work, vq_out_work_handler);
	init_waitqueue_head(&vi->out_waitq);

	virtio_device_ready(vdev);

	return 0;

del_cdev:
	cdev_del(&vi->cdev);
destroy_device:
	put_device(vi->dev);
destroy_class:
	class_destroy(vi->class);
unregister_region:
	unregister_chrdev_region(vi->dev_num, 0);
free_vi:
	kfree(vi);
	return ret;
}

static void virtmagma_remove(struct virtio_device *vdev)
{
	struct virtmagma_info *vi = vdev->priv;

	idr_for_each(&vi->instances, destroy_instance, vi);
	mutex_destroy(&vi->instances_lock);
	idr_destroy(&vi->instances);
	cdev_del(&vi->cdev);
	put_device(vi->dev);
	class_destroy(vi->class);
	unregister_chrdev_region(vi->dev_num, 0);
	kfree(vi);
}

static void virtmagma_scan(struct virtio_device *vdev)
{
}

static struct virtio_device_id id_table[] = {
	{ VIRTIO_ID_MAGMA, VIRTIO_DEV_ANY_ID },
	{ 0 },
};

static struct virtio_driver virtio_magma_driver = {
	.driver.name = KBUILD_MODNAME,
	.driver.owner = THIS_MODULE,
	.id_table = id_table,
	.probe = virtmagma_probe,
	.remove = virtmagma_remove,
	.scan = virtmagma_scan,
};

module_virtio_driver(virtio_magma_driver);
MODULE_DEVICE_TABLE(virtio, id_table);
MODULE_DESCRIPTION("Virtio Magma driver");
MODULE_LICENSE("GPL");