drivers/vfio/pci/mlx5/cmd.h - third_party/linux - Git at Google


            
            /* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */

            
            /*

            
             * Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.

            
             */


            #ifndef MLX5_VFIO_CMD_H

            
            #define MLX5_VFIO_CMD_H


            #include <linux/kernel.h>

            
            #include <linux/vfio_pci_core.h>

            
            #include <linux/mlx5/driver.h>

            
            #include <linux/mlx5/vport.h>

            
            #include <linux/mlx5/cq.h>

            
            #include <linux/mlx5/qp.h>


            enum mlx5_vf_migf_state {

            
            	MLX5_MIGF_STATE_ERROR = 1,

            
            	MLX5_MIGF_STATE_PRE_COPY_ERROR,

            
            	MLX5_MIGF_STATE_PRE_COPY,

            
            	MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK,

            
            	MLX5_MIGF_STATE_COMPLETE,

            
            };


            enum mlx5_vf_load_state {

            
            	MLX5_VF_LOAD_STATE_READ_HEADER,

            
            	MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,

            
            	MLX5_VF_LOAD_STATE_READ_HEADER_DATA,

            
            	MLX5_VF_LOAD_STATE_PREP_IMAGE,

            
            	MLX5_VF_LOAD_STATE_READ_IMAGE,

            
            	MLX5_VF_LOAD_STATE_LOAD_IMAGE,

            
            };


            struct mlx5_vf_migration_tag_stop_copy_data {

            
            	__le64 stop_copy_size;

            
            };


            enum mlx5_vf_migf_header_flags {

            
            	MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,

            
            	MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,

            
            };


            enum mlx5_vf_migf_header_tag {

            
            	MLX5_MIGF_HEADER_TAG_FW_DATA = 0,

            
            	MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,

            
            };


            struct mlx5_vf_migration_header {

            
            	__le64 record_size;

            
            	/* For future use in case we may need to change the kernel protocol */

            
            	__le32 flags; /* Use mlx5_vf_migf_header_flags */

            
            	__le32 tag; /* Use mlx5_vf_migf_header_tag */

            
            	__u8 data[]; /* Its size is given in the record_size */

            
            };


            struct mlx5_vhca_data_buffer {

            
            	struct page **page_list;

            
            	struct dma_iova_state state;

            
            	loff_t start_pos;

            
            	u64 length;

            
            	u32 npages;

            
            	u32 mkey;

            
            	u32 *mkey_in;

            
            	enum dma_data_direction dma_dir;

            
            	u8 stop_copy_chunk_num;

            
            	bool pre_copy_init_bytes_chunk;

            
            	struct list_head buf_elm;

            
            	struct mlx5_vf_migration_file *migf;

            
            };


            struct mlx5vf_async_data {

            
            	struct mlx5_async_work cb_work;

            
            	struct work_struct work;

            
            	struct mlx5_vhca_data_buffer *buf;

            
            	struct mlx5_vhca_data_buffer *header_buf;

            
            	int status;

            
            	u8 stop_copy_chunk:1;

            
            	void *out;

            
            };


            struct mlx5vf_save_work_data {

            
            	struct mlx5_vf_migration_file *migf;

            
            	size_t next_required_umem_size;

            
            	struct work_struct work;

            
            	u8 chunk_num;

            
            };


            #define MAX_NUM_CHUNKS 2


            struct mlx5_vf_migration_file {

            
            	struct file *filp;

            
            	struct mutex lock;

            
            	enum mlx5_vf_migf_state state;


            	enum mlx5_vf_load_state load_state;

            
            	u32 pdn;

            
            	loff_t max_pos;

            
            	u64 record_size;

            
            	u32 record_tag;

            
            	u64 stop_copy_prep_size;

            
            	u64 pre_copy_initial_bytes;

            
            	u64 pre_copy_initial_bytes_start;

            
            	size_t next_required_umem_size;

            
            	u8 num_ready_chunks;

            
            	/* Upon chunk mode preserve another set of buffers for stop_copy phase */

            
            	struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];

            
            	struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];

            
            	struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];

            
            	spinlock_t list_lock;

            
            	struct list_head buf_list;

            
            	struct list_head avail_list;

            
            	struct mlx5vf_pci_core_device *mvdev;

            
            	wait_queue_head_t poll_wait;

            
            	struct completion save_comp;

            
            	struct mlx5_async_ctx async_ctx;

            
            	struct mlx5vf_async_data async_data;

            
            	u8 inflight_save:1;

            
            };


            struct mlx5_vhca_cq_buf {

            
            	struct mlx5_frag_buf_ctrl fbc;

            
            	struct mlx5_frag_buf frag_buf;

            
            	int cqe_size;

            
            	int nent;

            
            };


            struct mlx5_vhca_cq {

            
            	struct mlx5_vhca_cq_buf buf;

            
            	struct mlx5_db db;

            
            	struct mlx5_core_cq mcq;

            
            	size_t ncqe;

            
            };


            struct mlx5_vhca_recv_buf {

            
            	u32 npages;

            
            	struct page **page_list;

            
            	struct dma_iova_state state;

            
            	u32 next_rq_offset;

            
            	u32 *mkey_in;

            
            	u32 mkey;

            
            };


            struct mlx5_vhca_qp {

            
            	struct mlx5_frag_buf buf;

            
            	struct mlx5_db db;

            
            	struct mlx5_vhca_recv_buf recv_buf;

            
            	u32 tracked_page_size;

            
            	u32 max_msg_size;

            
            	u32 qpn;

            
            	struct {

            
            		unsigned int pc;

            
            		unsigned int cc;

            
            		unsigned int wqe_cnt;

            
            		__be32 *db;

            
            		struct mlx5_frag_buf_ctrl fbc;

            
            	} rq;

            
            };


            struct mlx5_vhca_page_tracker {

            
            	u32 id;

            
            	u32 pdn;

            
            	/* Flags modified at runtime - dedicated storage unit */

            
            	u8 is_err;

            
            	u8 object_changed;

            
            	int status;

            
            	struct mlx5_uars_page *uar;

            
            	struct mlx5_vhca_cq cq;

            
            	struct mlx5_vhca_qp *host_qp;

            
            	struct mlx5_vhca_qp *fw_qp;

            
            	struct mlx5_nb nb;

            
            };


            struct mlx5vf_pci_core_device {

            
            	struct vfio_pci_core_device core_device;

            
            	int vf_id;

            
            	u16 vhca_id;

            
            	/* Flags only modified on setup/release - bitfield ok */

            
            	u8 migrate_cap:1;

            
            	u8 chunk_mode:1;

            
            	u8 mig_state_cap:1;

            
            	/* Flags modified at runtime - dedicated storage unit */

            
            	u8 mdev_detach;

            
            	u8 log_active;

            
            	u8 deferred_reset;

            
            	struct completion tracker_comp;

            
            	/* protect migration state */

            
            	struct mutex state_mutex;

            
            	enum vfio_device_mig_state mig_state;

            
            	/* protect the reset_done flow */

            
            	spinlock_t reset_lock;

            
            	struct mlx5_vf_migration_file *resuming_migf;

            
            	struct mlx5_vf_migration_file *saving_migf;

            
            	struct mlx5_vhca_page_tracker tracker;

            
            	struct workqueue_struct *cb_wq;

            
            	struct notifier_block nb;

            
            	struct mlx5_core_dev *mdev;

            
            };


            enum {

            
            	MLX5VF_QUERY_INC = (1UL << 0),

            
            	MLX5VF_QUERY_FINAL = (1UL << 1),

            
            	MLX5VF_QUERY_CLEANUP = (1UL << 2),

            
            };


            int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);

            
            int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);

            
            int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,

            
            					  size_t *state_size, u64 *total_size,

            
            					  u8 *migration_state, u8 query_flags);

            
            void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,

            
            			       const struct vfio_migration_ops *mig_ops,

            
            			       const struct vfio_log_ops *log_ops);

            
            void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);

            
            void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);

            
            int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,

            
            			       struct mlx5_vf_migration_file *migf,

            
            			       struct mlx5_vhca_data_buffer *buf, bool inc,

            
            			       bool track);

            
            int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,

            
            			       struct mlx5_vf_migration_file *migf,

            
            			       struct mlx5_vhca_data_buffer *buf);

            
            int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);

            
            void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);

            
            void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);

            
            struct mlx5_vhca_data_buffer *

            
            mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,

            
            			 enum dma_data_direction dma_dir);

            
            void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);

            
            struct mlx5_vhca_data_buffer *

            
            mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,

            
            		       enum dma_data_direction dma_dir);

            
            void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);

            
            static inline struct page *

            
            mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,

            
            			  unsigned long offset)

            
            {

            
            	int page_entry = offset / PAGE_SIZE;


            	if (page_entry >= buf->npages)

            
            		return NULL;


            	return buf->page_list[page_entry];

            
            }

            
            void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);

            
            void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,

            
            			enum mlx5_vf_migf_state *last_save_state);

            
            void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);

            
            void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,

            
            				   u8 chunk_num, size_t next_required_umem_size);

            
            int mlx5vf_start_page_tracker(struct vfio_device *vdev,

            
            		struct rb_root_cached *ranges, u32 nnodes, u64 *page_size);

            
            int mlx5vf_stop_page_tracker(struct vfio_device *vdev);

            
            int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,

            
            			unsigned long length, struct iova_bitmap *dirty);

            
            #endif /* MLX5_VFIO_CMD_H */
	/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */
	/*
	* Copyright (c) 2021-2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
	*/

	#ifndef MLX5_VFIO_CMD_H
	#define MLX5_VFIO_CMD_H

	#include <linux/kernel.h>
	#include <linux/vfio_pci_core.h>
	#include <linux/mlx5/driver.h>
	#include <linux/mlx5/vport.h>
	#include <linux/mlx5/cq.h>
	#include <linux/mlx5/qp.h>

	enum mlx5_vf_migf_state {
	MLX5_MIGF_STATE_ERROR = 1,
	MLX5_MIGF_STATE_PRE_COPY_ERROR,
	MLX5_MIGF_STATE_PRE_COPY,
	MLX5_MIGF_STATE_SAVE_STOP_COPY_CHUNK,
	MLX5_MIGF_STATE_COMPLETE,
	};

	enum mlx5_vf_load_state {
	MLX5_VF_LOAD_STATE_READ_HEADER,
	MLX5_VF_LOAD_STATE_PREP_HEADER_DATA,
	MLX5_VF_LOAD_STATE_READ_HEADER_DATA,
	MLX5_VF_LOAD_STATE_PREP_IMAGE,
	MLX5_VF_LOAD_STATE_READ_IMAGE,
	MLX5_VF_LOAD_STATE_LOAD_IMAGE,
	};

	struct mlx5_vf_migration_tag_stop_copy_data {
	__le64 stop_copy_size;
	};

	enum mlx5_vf_migf_header_flags {
	MLX5_MIGF_HEADER_FLAGS_TAG_MANDATORY = 0,
	MLX5_MIGF_HEADER_FLAGS_TAG_OPTIONAL = 1 << 0,
	};

	enum mlx5_vf_migf_header_tag {
	MLX5_MIGF_HEADER_TAG_FW_DATA = 0,
	MLX5_MIGF_HEADER_TAG_STOP_COPY_SIZE = 1 << 0,
	};

	struct mlx5_vf_migration_header {
	__le64 record_size;
	/* For future use in case we may need to change the kernel protocol */
	__le32 flags; /* Use mlx5_vf_migf_header_flags */
	__le32 tag; /* Use mlx5_vf_migf_header_tag */
	__u8 data[]; /* Its size is given in the record_size */
	};

	struct mlx5_vhca_data_buffer {
	struct page **page_list;
	struct dma_iova_state state;
	loff_t start_pos;
	u64 length;
	u32 npages;
	u32 mkey;
	u32 *mkey_in;
	enum dma_data_direction dma_dir;
	u8 stop_copy_chunk_num;
	bool pre_copy_init_bytes_chunk;
	struct list_head buf_elm;
	struct mlx5_vf_migration_file *migf;
	};

	struct mlx5vf_async_data {
	struct mlx5_async_work cb_work;
	struct work_struct work;
	struct mlx5_vhca_data_buffer *buf;
	struct mlx5_vhca_data_buffer *header_buf;
	int status;
	u8 stop_copy_chunk:1;
	void *out;
	};

	struct mlx5vf_save_work_data {
	struct mlx5_vf_migration_file *migf;
	size_t next_required_umem_size;
	struct work_struct work;
	u8 chunk_num;
	};

	#define MAX_NUM_CHUNKS 2

	struct mlx5_vf_migration_file {
	struct file *filp;
	struct mutex lock;
	enum mlx5_vf_migf_state state;

	enum mlx5_vf_load_state load_state;
	u32 pdn;
	loff_t max_pos;
	u64 record_size;
	u32 record_tag;
	u64 stop_copy_prep_size;
	u64 pre_copy_initial_bytes;
	u64 pre_copy_initial_bytes_start;
	size_t next_required_umem_size;
	u8 num_ready_chunks;
	/* Upon chunk mode preserve another set of buffers for stop_copy phase */
	struct mlx5_vhca_data_buffer *buf[MAX_NUM_CHUNKS];
	struct mlx5_vhca_data_buffer *buf_header[MAX_NUM_CHUNKS];
	struct mlx5vf_save_work_data save_data[MAX_NUM_CHUNKS];
	spinlock_t list_lock;
	struct list_head buf_list;
	struct list_head avail_list;
	struct mlx5vf_pci_core_device *mvdev;
	wait_queue_head_t poll_wait;
	struct completion save_comp;
	struct mlx5_async_ctx async_ctx;
	struct mlx5vf_async_data async_data;
	u8 inflight_save:1;
	};

	struct mlx5_vhca_cq_buf {
	struct mlx5_frag_buf_ctrl fbc;
	struct mlx5_frag_buf frag_buf;
	int cqe_size;
	int nent;
	};

	struct mlx5_vhca_cq {
	struct mlx5_vhca_cq_buf buf;
	struct mlx5_db db;
	struct mlx5_core_cq mcq;
	size_t ncqe;
	};

	struct mlx5_vhca_recv_buf {
	u32 npages;
	struct page **page_list;
	struct dma_iova_state state;
	u32 next_rq_offset;
	u32 *mkey_in;
	u32 mkey;
	};

	struct mlx5_vhca_qp {
	struct mlx5_frag_buf buf;
	struct mlx5_db db;
	struct mlx5_vhca_recv_buf recv_buf;
	u32 tracked_page_size;
	u32 max_msg_size;
	u32 qpn;
	struct {
	unsigned int pc;
	unsigned int cc;
	unsigned int wqe_cnt;
	__be32 *db;
	struct mlx5_frag_buf_ctrl fbc;
	} rq;
	};

	struct mlx5_vhca_page_tracker {
	u32 id;
	u32 pdn;
	/* Flags modified at runtime - dedicated storage unit */
	u8 is_err;
	u8 object_changed;
	int status;
	struct mlx5_uars_page *uar;
	struct mlx5_vhca_cq cq;
	struct mlx5_vhca_qp *host_qp;
	struct mlx5_vhca_qp *fw_qp;
	struct mlx5_nb nb;
	};

	struct mlx5vf_pci_core_device {
	struct vfio_pci_core_device core_device;
	int vf_id;
	u16 vhca_id;
	/* Flags only modified on setup/release - bitfield ok */
	u8 migrate_cap:1;
	u8 chunk_mode:1;
	u8 mig_state_cap:1;
	/* Flags modified at runtime - dedicated storage unit */
	u8 mdev_detach;
	u8 log_active;
	u8 deferred_reset;
	struct completion tracker_comp;
	/* protect migration state */
	struct mutex state_mutex;
	enum vfio_device_mig_state mig_state;
	/* protect the reset_done flow */
	spinlock_t reset_lock;
	struct mlx5_vf_migration_file *resuming_migf;
	struct mlx5_vf_migration_file *saving_migf;
	struct mlx5_vhca_page_tracker tracker;
	struct workqueue_struct *cb_wq;
	struct notifier_block nb;
	struct mlx5_core_dev *mdev;
	};

	enum {
	MLX5VF_QUERY_INC = (1UL << 0),
	MLX5VF_QUERY_FINAL = (1UL << 1),
	MLX5VF_QUERY_CLEANUP = (1UL << 2),
	};

	int mlx5vf_cmd_suspend_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
	int mlx5vf_cmd_resume_vhca(struct mlx5vf_pci_core_device *mvdev, u16 op_mod);
	int mlx5vf_cmd_query_vhca_migration_state(struct mlx5vf_pci_core_device *mvdev,
	size_t state_size, u64 total_size,
	u8 *migration_state, u8 query_flags);
	void mlx5vf_cmd_set_migratable(struct mlx5vf_pci_core_device *mvdev,
	const struct vfio_migration_ops *mig_ops,
	const struct vfio_log_ops *log_ops);
	void mlx5vf_cmd_remove_migratable(struct mlx5vf_pci_core_device *mvdev);
	void mlx5vf_cmd_close_migratable(struct mlx5vf_pci_core_device *mvdev);
	int mlx5vf_cmd_save_vhca_state(struct mlx5vf_pci_core_device *mvdev,
	struct mlx5_vf_migration_file *migf,
	struct mlx5_vhca_data_buffer *buf, bool inc,
	bool track);
	int mlx5vf_cmd_load_vhca_state(struct mlx5vf_pci_core_device *mvdev,
	struct mlx5_vf_migration_file *migf,
	struct mlx5_vhca_data_buffer *buf);
	int mlx5vf_cmd_alloc_pd(struct mlx5_vf_migration_file *migf);
	void mlx5vf_cmd_dealloc_pd(struct mlx5_vf_migration_file *migf);
	void mlx5fv_cmd_clean_migf_resources(struct mlx5_vf_migration_file *migf);
	struct mlx5_vhca_data_buffer *
	mlx5vf_alloc_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,
	enum dma_data_direction dma_dir);
	void mlx5vf_free_data_buffer(struct mlx5_vhca_data_buffer *buf);
	struct mlx5_vhca_data_buffer *
	mlx5vf_get_data_buffer(struct mlx5_vf_migration_file *migf, u32 npages,
	enum dma_data_direction dma_dir);
	void mlx5vf_put_data_buffer(struct mlx5_vhca_data_buffer *buf);
	static inline struct page *
	mlx5vf_get_migration_page(struct mlx5_vhca_data_buffer *buf,
	unsigned long offset)
	{
	int page_entry = offset / PAGE_SIZE;

	if (page_entry >= buf->npages)
	return NULL;

	return buf->page_list[page_entry];
	}
	void mlx5vf_state_mutex_unlock(struct mlx5vf_pci_core_device *mvdev);
	void mlx5vf_disable_fds(struct mlx5vf_pci_core_device *mvdev,
	enum mlx5_vf_migf_state *last_save_state);
	void mlx5vf_mig_file_cleanup_cb(struct work_struct *_work);
	void mlx5vf_mig_file_set_save_work(struct mlx5_vf_migration_file *migf,
	u8 chunk_num, size_t next_required_umem_size);
	int mlx5vf_start_page_tracker(struct vfio_device *vdev,
	struct rb_root_cached ranges, u32 nnodes, u64 page_size);
	int mlx5vf_stop_page_tracker(struct vfio_device *vdev);
	int mlx5vf_tracker_read_and_clear(struct vfio_device *vdev, unsigned long iova,
	unsigned long length, struct iova_bitmap *dirty);
	#endif /* MLX5_VFIO_CMD_H */