drivers/net/ethernet/intel/idpf/idpf.h

/* SPDX-License-Identifier: GPL-2.0-only */
/* Copyright (C) 2023 Intel Corporation */

#ifndef _IDPF_H_
#define _IDPF_H_

/* Forward declaration */
struct idpf_adapter;
struct idpf_vport;
struct idpf_vport_max_q;
struct idpf_q_vec_rsrc;
struct idpf_rss_data;

#include <net/pkt_sched.h>
#include <linux/aer.h>
#include <linux/etherdevice.h>
#include <linux/ioport.h>
#include <linux/pci.h>
#include <linux/bitfield.h>
#include <linux/sctp.h>
#include <linux/ethtool_netlink.h>
#include <net/gro.h>

#include <linux/net/intel/iidc_rdma.h>
#include <linux/net/intel/iidc_rdma_idpf.h>
#include <linux/net/intel/libie/controlq.h>
#include <linux/net/intel/virtchnl2.h>

#include "idpf_txrx.h"

#define GETMAXVAL(num_bits)		GENMASK((num_bits) - 1, 0)

#define IDPF_NO_FREE_SLOT		0xffff

/* Default Mailbox settings */
#define IDPF_NUM_FILTERS_PER_MSG	20
#define IDPF_NUM_DFLT_MBX_Q		2	/* includes both TX and RX */
#define IDPF_DFLT_MBX_Q_LEN		64
/* maximum number of times to try before resetting mailbox */
#define IDPF_MB_MAX_ERR			20
#define IDPF_NUM_CHUNKS_PER_MSG(struct_sz, chunk_sz)	\
	((LIBIE_CTLQ_MAX_BUF_LEN - (struct_sz)) / (chunk_sz))

#define IDPF_WAIT_FOR_MARKER_TIMEO	500
#define IDPF_MAX_WAIT			500

/* available message levels */
#define IDPF_AVAIL_NETIF_M (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)

#define IDPF_DIM_PROFILE_SLOTS  5

#define IDPF_VIRTCHNL_VERSION_MAJOR VIRTCHNL2_VERSION_MAJOR_2
#define IDPF_VIRTCHNL_VERSION_MINOR VIRTCHNL2_VERSION_MINOR_0

/**
 * struct idpf_mac_filter
 * @list: list member field
 * @macaddr: MAC address
 * @remove: filter should be removed (virtchnl)
 * @add: filter should be added (virtchnl)
 */
struct idpf_mac_filter {
	struct list_head list;
	u8 macaddr[ETH_ALEN];
	bool remove;
	bool add;
};

/**
 * enum idpf_state - State machine to handle bring up
 * @__IDPF_VER_CHECK: Negotiate virtchnl version
 * @__IDPF_GET_CAPS: Negotiate capabilities
 * @__IDPF_INIT_SW: Init based on given capabilities
 * @__IDPF_STATE_LAST: Must be last, used to determine size
 */
enum idpf_state {
	__IDPF_VER_CHECK,
	__IDPF_GET_CAPS,
	__IDPF_INIT_SW,
	__IDPF_STATE_LAST,
};

/**
 * enum idpf_flags - Hard reset causes.
 * @IDPF_HR_FUNC_RESET: Hard reset when TxRx timeout
 * @IDPF_HR_DRV_LOAD: Set on driver load for a clean HW
 * @IDPF_HR_RESET_IN_PROG: Reset in progress
 * @IDPF_REMOVE_IN_PROG: Driver remove in progress
 * @IDPF_MB_INTR_MODE: Mailbox in interrupt mode
 * @IDPF_VC_CORE_INIT: virtchnl core has been init
 * @IDPF_FLAGS_NBITS: Must be last
 */
enum idpf_flags {
	IDPF_HR_FUNC_RESET,
	IDPF_HR_DRV_LOAD,
	IDPF_HR_RESET_IN_PROG,
	IDPF_REMOVE_IN_PROG,
	IDPF_MB_INTR_MODE,
	IDPF_VC_CORE_INIT,
	IDPF_FLAGS_NBITS,
};

/**
 * enum idpf_cap_field - Offsets into capabilities struct for specific caps
 * @IDPF_BASE_CAPS: generic base capabilities
 * @IDPF_CSUM_CAPS: checksum offload capabilities
 * @IDPF_SEG_CAPS: segmentation offload capabilities
 * @IDPF_RSS_CAPS: RSS offload capabilities
 * @IDPF_HSPLIT_CAPS: Header split capabilities
 * @IDPF_RSC_CAPS: RSC offload capabilities
 * @IDPF_OTHER_CAPS: miscellaneous offloads
 *
 * Used when checking for a specific capability flag since different capability
 * sets are not mutually exclusive numerically, the caller must specify which
 * type of capability they are checking for.
 */
enum idpf_cap_field {
	IDPF_BASE_CAPS		= -1,
	IDPF_CSUM_CAPS		= offsetof(struct virtchnl2_get_capabilities,
					   csum_caps),
	IDPF_SEG_CAPS		= offsetof(struct virtchnl2_get_capabilities,
					   seg_caps),
	IDPF_RSS_CAPS		= offsetof(struct virtchnl2_get_capabilities,
					   rss_caps),
	IDPF_HSPLIT_CAPS	= offsetof(struct virtchnl2_get_capabilities,
					   hsplit_caps),
	IDPF_RSC_CAPS		= offsetof(struct virtchnl2_get_capabilities,
					   rsc_caps),
	IDPF_OTHER_CAPS		= offsetof(struct virtchnl2_get_capabilities,
					   other_caps),
};

/**
 * enum idpf_vport_state - Current vport state
 * @IDPF_VPORT_UP: Vport is up
 * @IDPF_VPORT_STATE_NBITS: Must be last, number of states
 */
enum idpf_vport_state {
	IDPF_VPORT_UP,
	IDPF_VPORT_STATE_NBITS
};

/**
 * struct idpf_netdev_priv - Struct to store vport back pointer
 * @adapter: Adapter back pointer
 * @vport: Vport back pointer
 * @vport_id: Vport identifier
 * @link_speed_mbps: Link speed in mbps
 * @vport_idx: Relative vport index
 * @max_tx_hdr_size: Max header length hardware can support
 * @tx_max_bufs: Max buffers that can be transmitted with scatter-gather
 * @state: See enum idpf_vport_state
 * @netstats: Packet and byte stats
 * @stats_lock: Lock to protect stats update
 */
struct idpf_netdev_priv {
	struct idpf_adapter *adapter;
	struct idpf_vport *vport;
	u32 vport_id;
	u32 link_speed_mbps;
	u16 vport_idx;
	u16 max_tx_hdr_size;
	u16 tx_max_bufs;
	DECLARE_BITMAP(state, IDPF_VPORT_STATE_NBITS);
	struct rtnl_link_stats64 netstats;
	spinlock_t stats_lock;
};

/**
 * struct idpf_reset_reg - Reset register offsets/masks
 * @rstat: Reset status register
 * @rstat_m: Reset status mask
 */
struct idpf_reset_reg {
	void __iomem *rstat;
	u32 rstat_m;
};

/**
 * struct idpf_vport_max_q - Queue limits
 * @max_rxq: Maximum number of RX queues supported
 * @max_txq: Maixmum number of TX queues supported
 * @max_bufq: In splitq, maximum number of buffer queues supported
 * @max_complq: In splitq, maximum number of completion queues supported
 */
struct idpf_vport_max_q {
	u16 max_rxq;
	u16 max_txq;
	u16 max_bufq;
	u16 max_complq;
};

/**
 * struct idpf_reg_ops - Device specific register operation function pointers
 * @ctlq_reg_init: Mailbox control queue register initialization
 * @intr_reg_init: Traffic interrupt register initialization
 * @mb_intr_reg_init: Mailbox interrupt register initialization
 * @reset_reg_init: Reset register initialization
 * @trigger_reset: Trigger a reset to occur
 * @ptp_reg_init: PTP register initialization
 */
struct idpf_reg_ops {
	void (*ctlq_reg_init)(struct libie_mmio_info *mmio,
			      struct libie_ctlq_create_info *cctlq_info);
	int (*intr_reg_init)(struct idpf_vport *vport,
			     struct idpf_q_vec_rsrc *rsrc);
	void (*mb_intr_reg_init)(struct idpf_adapter *adapter);
	void (*reset_reg_init)(struct idpf_adapter *adapter);
	void (*trigger_reset)(struct idpf_adapter *adapter,
			      enum idpf_flags trig_cause);
	void (*ptp_reg_init)(const struct idpf_adapter *adapter);
};

#define IDPF_MMIO_REG_NUM_STATIC	2
#define IDPF_PF_MBX_REGION_SZ		4096
#define IDPF_PF_RSTAT_REGION_SZ		2048
#define IDPF_VF_MBX_REGION_SZ		10240
#define IDPF_VF_RSTAT_REGION_SZ		2048

/**
 * struct idpf_dev_ops - Device specific operations
 * @reg_ops: Register operations
 * @idc_init: IDC initialization
 * @static_reg_info: array of mailbox and rstat register info
 */
struct idpf_dev_ops {
	struct idpf_reg_ops reg_ops;

	int (*idc_init)(struct idpf_adapter *adapter);

	/* static_reg_info[0] is mailbox region, static_reg_info[1] is rstat */
	struct resource static_reg_info[IDPF_MMIO_REG_NUM_STATIC];
};

/**
 * enum idpf_vport_reset_cause - Vport soft reset causes
 * @IDPF_SR_Q_CHANGE: Soft reset queue change
 * @IDPF_SR_Q_DESC_CHANGE: Soft reset descriptor change
 * @IDPF_SR_MTU_CHANGE: Soft reset MTU change
 * @IDPF_SR_RSC_CHANGE: Soft reset RSC change
 */
enum idpf_vport_reset_cause {
	IDPF_SR_Q_CHANGE,
	IDPF_SR_Q_DESC_CHANGE,
	IDPF_SR_MTU_CHANGE,
	IDPF_SR_RSC_CHANGE,
};

/**
 * enum idpf_vport_flags - Vport flags
 * @IDPF_VPORT_DEL_QUEUES: To send delete queues message
 * @IDPF_VPORT_FLAGS_NBITS: Must be last
 */
enum idpf_vport_flags {
	IDPF_VPORT_DEL_QUEUES,
	IDPF_VPORT_FLAGS_NBITS,
};

/**
 * struct idpf_tstamp_stats - Tx timestamp statistics
 * @stats_sync: See struct u64_stats_sync
 * @packets: Number of packets successfully timestamped by the hardware
 * @discarded: Number of Tx skbs discarded due to cached PHC
 *	       being too old to correctly extend timestamp
 * @flushed: Number of Tx skbs flushed due to interface closed
 */
struct idpf_tstamp_stats {
	struct u64_stats_sync stats_sync;
	u64_stats_t packets;
	u64_stats_t discarded;
	u64_stats_t flushed;
};

struct idpf_port_stats {
	struct u64_stats_sync stats_sync;
	u64_stats_t rx_hw_csum_err;
	u64_stats_t rx_hsplit;
	u64_stats_t rx_hsplit_hbo;
	u64_stats_t rx_bad_descs;
	u64_stats_t tx_linearize;
	u64_stats_t tx_busy;
	u64_stats_t tx_drops;
	u64_stats_t tx_dma_map_errs;
	struct virtchnl2_vport_stats vport_stats;
};

struct idpf_fsteer_fltr {
	struct list_head list;
	struct ethtool_rx_flow_spec fs;
};

/**
 * struct idpf_q_vec_rsrc - handle for queue and vector resources
 * @dev: device pointer for DMA mapping
 * @q_vectors: array of queue vectors
 * @q_vector_idxs: starting index of queue vectors
 * @num_q_vectors: number of IRQ vectors allocated
 * @noirq_v_idx: ID of the NOIRQ vector
 * @noirq_dyn_ctl_ena: value to write to the above to enable it
 * @noirq_dyn_ctl: register to enable/disable the vector for NOIRQ queues
 * @txq_grps: array of TX queue groups
 * @txq_desc_count: TX queue descriptor count
 * @complq_desc_count: completion queue descriptor count
 * @txq_model: split queue or single queue queuing model
 * @num_txq: number of allocated TX queues
 * @num_complq: number of allocated completion queues
 * @num_txq_grp: number of TX queue groups
 * @xdp_txq_offset: index of the first XDPSQ (== number of regular SQs)
 * @num_rxq_grp: number of RX queues in a group
 * @rxq_model: splitq queue or single queue queuing model
 * @rxq_grps: total number of RX groups. Number of groups * number of RX per
 *	      group will yield total number of RX queues.
 * @num_rxq: number of allocated RX queues
 * @num_bufq: number of allocated buffer queues
 * @rxq_desc_count: RX queue descriptor count. *MUST* have enough descriptors
 *		    to complete all buffer descriptors for all buffer queues in
 *		    the worst case.
 * @bufq_desc_count: buffer queue descriptor count
 * @num_bufqs_per_qgrp: buffer queues per RX queue in a given grouping
 * @base_rxd: true if the driver should use base descriptors instead of flex
 */
struct idpf_q_vec_rsrc {
	struct device		*dev;
	struct idpf_q_vector	*q_vectors;
	u16			*q_vector_idxs;
	u16			num_q_vectors;
	u16			noirq_v_idx;
	u32			noirq_dyn_ctl_ena;
	void __iomem		*noirq_dyn_ctl;

	struct idpf_txq_group	*txq_grps;
	u32			txq_desc_count;
	u32			complq_desc_count;
	u32			txq_model;
	u16			num_txq;
	u16			num_complq;
	u16			num_txq_grp;
	u16			xdp_txq_offset;

	u16			num_rxq_grp;
	u32			rxq_model;
	struct idpf_rxq_group	*rxq_grps;
	u16			num_rxq;
	u16			num_bufq;
	u32			rxq_desc_count;
	u32			bufq_desc_count[IDPF_MAX_BUFQS_PER_RXQ_GRP];
	u8			num_bufqs_per_qgrp;
	bool			base_rxd;
};

/**
 * struct idpf_vport - Handle for netdevices and queue resources
 * @dflt_qv_rsrc: contains default queue and vector resources
 * @txqs: Used only in hotpath to get to the right queue very fast
 * @num_txq: Number of allocated TX queues
 * @num_xdp_txq: number of XDPSQs
 * @xdpsq_share: whether XDPSQ sharing is enabled
 * @xdp_prog: installed XDP program
 * @vdev_info: IDC vport device info pointer
 * @adapter: back pointer to associated adapter
 * @netdev: Associated net_device. Each vport should have one and only one
 *	    associated netdev.
 * @flags: See enum idpf_vport_flags
 * @compln_clean_budget: Work budget for completion clean
 * @vport_id: Device given vport identifier
 * @vport_type: Default SRIOV, SIOV, etc.
 * @idx: Software index in adapter vports struct
 * @max_mtu: device given max possible MTU
 * @default_mac_addr: device will give a default MAC to use
 * @rx_itr_profile: RX profiles for Dynamic Interrupt Moderation
 * @tx_itr_profile: TX profiles for Dynamic Interrupt Moderation
 * @port_stats: per port csum, header split, and other offload stats
 * @default_vport: Use this vport if one isn't specified
 * @crc_enable: Enable CRC insertion offload
 * @link_up: True if link is up
 * @tx_tstamp_caps: Capabilities negotiated for Tx timestamping
 * @tstamp_config: The Tx tstamp config
 * @tstamp_task: Tx timestamping task
 * @tstamp_stats: Tx timestamping statistics
 */
struct idpf_vport {
	struct idpf_q_vec_rsrc dflt_qv_rsrc;
	struct idpf_tx_queue **txqs;
	u16 num_txq;
	u16 num_xdp_txq;
	bool xdpsq_share;
	struct bpf_prog *xdp_prog;

	struct iidc_rdma_vport_dev_info *vdev_info;

	struct idpf_adapter *adapter;
	struct net_device *netdev;
	DECLARE_BITMAP(flags, IDPF_VPORT_FLAGS_NBITS);
	u32 compln_clean_budget;
	u32 vport_id;
	u16 vport_type;
	u16 idx;

	u16 max_mtu;
	u8 default_mac_addr[ETH_ALEN];
	u16 rx_itr_profile[IDPF_DIM_PROFILE_SLOTS];
	u16 tx_itr_profile[IDPF_DIM_PROFILE_SLOTS];

	struct idpf_port_stats port_stats;
	bool default_vport;
	bool crc_enable;
	bool link_up;

	struct idpf_ptp_vport_tx_tstamp_caps *tx_tstamp_caps;
	struct kernel_hwtstamp_config tstamp_config;
	struct work_struct tstamp_task;
	struct idpf_tstamp_stats tstamp_stats;
};

/**
 * enum idpf_user_flags
 * @__IDPF_USER_FLAG_HSPLIT: header split state
 * @__IDPF_PROMISC_UC: Unicast promiscuous mode
 * @__IDPF_PROMISC_MC: Multicast promiscuous mode
 * @__IDPF_USER_FLAGS_NBITS: Must be last
 */
enum idpf_user_flags {
	__IDPF_USER_FLAG_HSPLIT = 0U,
	__IDPF_PROMISC_UC = 32,
	__IDPF_PROMISC_MC,

	__IDPF_USER_FLAGS_NBITS,
};

/**
 * struct idpf_rss_data - Associated RSS data
 * @rss_key_size: Size of RSS hash key
 * @rss_key: RSS hash key
 * @rss_lut_size: Size of RSS lookup table
 * @rss_lut: RSS lookup table
 */
struct idpf_rss_data {
	u16 rss_key_size;
	u8 *rss_key;
	u16 rss_lut_size;
	u32 *rss_lut;
};

/**
 * struct idpf_q_coalesce - User defined coalescing configuration values for
 *			   a single queue.
 * @tx_intr_mode: Dynamic TX ITR or not
 * @rx_intr_mode: Dynamic RX ITR or not
 * @tx_coalesce_usecs: TX interrupt throttling rate
 * @rx_coalesce_usecs: RX interrupt throttling rate
 *
 * Used to restore user coalescing configuration after a reset.
 */
struct idpf_q_coalesce {
	u32 tx_intr_mode;
	u32 rx_intr_mode;
	u32 tx_coalesce_usecs;
	u32 rx_coalesce_usecs;
};

/**
 * struct idpf_vport_user_config_data - User defined configuration values for
 *					each vport.
 * @rss_data: See struct idpf_rss_data
 * @q_coalesce: Array of per queue coalescing data
 * @num_req_tx_qs: Number of user requested TX queues through ethtool
 * @num_req_rx_qs: Number of user requested RX queues through ethtool
 * @num_req_txq_desc: Number of user requested TX queue descriptors through
 *		      ethtool
 * @num_req_rxq_desc: Number of user requested RX queue descriptors through
 *		      ethtool
 * @xdp_prog: requested XDP program to install
 * @user_flags: User toggled config flags
 * @mac_filter_list: List of MAC filters
 * @num_fsteer_fltrs: number of flow steering filters
 * @flow_steer_list: list of flow steering filters
 *
 * Used to restore configuration after a reset as the vport will get wiped.
 */
struct idpf_vport_user_config_data {
	struct idpf_rss_data rss_data;
	struct idpf_q_coalesce *q_coalesce;
	u16 num_req_tx_qs;
	u16 num_req_rx_qs;
	u32 num_req_txq_desc;
	u32 num_req_rxq_desc;
	struct bpf_prog *xdp_prog;
	DECLARE_BITMAP(user_flags, __IDPF_USER_FLAGS_NBITS);
	struct list_head mac_filter_list;
	u32 num_fsteer_fltrs;
	struct list_head flow_steer_list;
};

/**
 * enum idpf_vport_config_flags - Vport config flags
 * @IDPF_VPORT_REG_NETDEV: Register netdev
 * @IDPF_VPORT_UP_REQUESTED: Set if interface up is requested on core reset
 * @IDPF_VPORT_CONFIG_FLAGS_NBITS: Must be last
 */
enum idpf_vport_config_flags {
	IDPF_VPORT_REG_NETDEV,
	IDPF_VPORT_UP_REQUESTED,
	IDPF_VPORT_CONFIG_FLAGS_NBITS,
};

/**
 * struct idpf_avail_queue_info
 * @avail_rxq: Available RX queues
 * @avail_txq: Available TX queues
 * @avail_bufq: Available buffer queues
 * @avail_complq: Available completion queues
 *
 * Maintain total queues available after allocating max queues to each vport.
 */
struct idpf_avail_queue_info {
	u16 avail_rxq;
	u16 avail_txq;
	u16 avail_bufq;
	u16 avail_complq;
};

/**
 * struct idpf_vector_info - Utility structure to pass function arguments as a
 *			     structure
 * @num_req_vecs: Vectors required based on the number of queues updated by the
 *		  user via ethtool
 * @num_curr_vecs: Current number of vectors, must be >= @num_req_vecs
 * @index: Relative starting index for vectors
 * @default_vport: Vectors are for default vport
 */
struct idpf_vector_info {
	u16 num_req_vecs;
	u16 num_curr_vecs;
	u16 index;
	bool default_vport;
};

/**
 * struct idpf_vector_lifo - Stack to maintain vector indexes used for vector
 *			     distribution algorithm
 * @top: Points to stack top i.e. next available vector index
 * @base: Always points to start of the free pool
 * @size: Total size of the vector stack
 * @vec_idx: Array to store all the vector indexes
 *
 * Vector stack maintains all the relative vector indexes at the *adapter*
 * level. This stack is divided into 2 parts, first one is called as 'default
 * pool' and other one is called 'free pool'.  Vector distribution algorithm
 * gives priority to default vports in a way that at least IDPF_MIN_Q_VEC
 * vectors are allocated per default vport and the relative vector indexes for
 * those are maintained in default pool. Free pool contains all the unallocated
 * vector indexes which can be allocated on-demand basis. Mailbox vector index
 * is maintained in the default pool of the stack.
 */
struct idpf_vector_lifo {
	u16 top;
	u16 base;
	u16 size;
	u16 *vec_idx;
};

/**
 * struct idpf_queue_id_reg_chunk - individual queue ID and register chunk
 * @qtail_reg_start: queue tail register offset
 * @qtail_reg_spacing: queue tail register spacing
 * @type: queue type of the queues in the chunk
 * @start_queue_id: starting queue ID in the chunk
 * @num_queues: number of queues in the chunk
 */
struct idpf_queue_id_reg_chunk {
	u64 qtail_reg_start;
	u32 qtail_reg_spacing;
	u32 type;
	u32 start_queue_id;
	u32 num_queues;
};

/**
 * struct idpf_queue_id_reg_info - queue ID and register chunk info received
 *				   over the mailbox
 * @num_chunks: number of chunks
 * @queue_chunks: array of chunks
 */
struct idpf_queue_id_reg_info {
	u16 num_chunks;
	struct idpf_queue_id_reg_chunk *queue_chunks;
};

/**
 * struct idpf_vport_config - Vport configuration data
 * @user_config: see struct idpf_vport_user_config_data
 * @max_q: Maximum possible queues
 * @qid_reg_info: Struct to store the queue ID and register info
 * @mac_filter_list_lock: Lock to protect mac filters
 * @flow_steer_list_lock: Lock to protect fsteer filters
 * @flags: See enum idpf_vport_config_flags
 */
struct idpf_vport_config {
	struct idpf_vport_user_config_data user_config;
	struct idpf_vport_max_q max_q;
	struct idpf_queue_id_reg_info qid_reg_info;
	spinlock_t mac_filter_list_lock;
	spinlock_t flow_steer_list_lock;
	DECLARE_BITMAP(flags, IDPF_VPORT_CONFIG_FLAGS_NBITS);
};

#define idpf_for_each_vport(adapter, iter) \
	for (struct idpf_vport **__##iter = &(adapter)->vports[0], \
	     *iter = (adapter)->max_vports ? *__##iter : NULL; \
	     iter; \
	     iter = (++__##iter) < &(adapter)->vports[(adapter)->max_vports] ? \
	     *__##iter : NULL)

/**
 * struct idpf_adapter - Device data struct generated on probe
 * @pdev: PCI device struct given on probe
 * @virt_ver_maj: Virtchnl version major
 * @virt_ver_min: Virtchnl version minor
 * @msg_enable: Debug message level enabled
 * @mb_wait_count: Number of times mailbox was attempted initialization
 * @state: Init state machine
 * @flags: See enum idpf_flags
 * @reset_reg: See struct idpf_reset_reg
 * @ctlq_ctx: controlq context
 * @asq: Send control queue info
 * @arq: Receive control queue info
 * @xnm: Xn transaction manager
 * @num_avail_msix: Available number of MSIX vectors
 * @num_msix_entries: Number of entries in MSIX table
 * @msix_entries: MSIX table
 * @num_rdma_msix_entries: Available number of MSIX vectors for RDMA
 * @rdma_msix_entries: RDMA MSIX table
 * @req_vec_chunks: Requested vector chunk data
 * @mb_vector: Mailbox vector data
 * @vector_stack: Stack to store the msix vector indexes
 * @irq_mb_handler: Handler for hard interrupt for mailbox
 * @tx_timeout_count: Number of TX timeouts that have occurred
 * @avail_queues: Device given queue limits
 * @vports: Array to store vports created by the driver
 * @netdevs: Associated Vport netdevs
 * @vport_params_recvd: Vport params received
 * @vport_ids: Array of device given vport identifiers
 * @singleq_pt_lkup: Lookup table for singleq RX ptypes
 * @splitq_pt_lkup: Lookup table for splitq RX ptypes
 * @vport_config: Vport config parameters
 * @max_vports: Maximum vports that can be allocated
 * @num_alloc_vports: Current number of vports allocated
 * @next_vport: Next free slot in pf->vport[] - 0-based!
 * @init_task: Initialization task
 * @init_wq: Workqueue for initialization task
 * @serv_task: Periodically recurring maintenance task
 * @serv_wq: Workqueue for service task
 * @mbx_task: Task to handle mailbox interrupts
 * @mbx_wq: Workqueue for mailbox responses
 * @vc_event_task: Task to handle out of band virtchnl event notifications
 * @vc_event_wq: Workqueue for virtchnl events
 * @stats_task: Periodic statistics retrieval task
 * @stats_wq: Workqueue for statistics task
 * @caps: Negotiated capabilities with device
 * @dev_ops: See idpf_dev_ops
 * @cdev_info: IDC core device info pointer
 * @num_vfs: Number of allocated VFs through sysfs. PF does not directly talk
 *	     to VFs but is used to initialize them
 * @crc_enable: Enable CRC insertion offload
 * @req_tx_splitq: TX split or single queue model to request
 * @req_rx_splitq: RX split or single queue model to request
 * @vport_ctrl_lock: Lock to protect the vport control flow
 * @vector_lock: Lock to protect vector distribution
 * @queue_lock: Lock to protect queue distribution
 * @vc_buf_lock: Lock to protect virtchnl buffer
 * @ptp: Storage for PTP-related data
 */
struct idpf_adapter {
	struct pci_dev *pdev;
	u32 virt_ver_maj;
	u32 virt_ver_min;

	u32 msg_enable;
	u32 mb_wait_count;
	enum idpf_state state;
	DECLARE_BITMAP(flags, IDPF_FLAGS_NBITS);
	struct idpf_reset_reg reset_reg;
	struct libie_ctlq_ctx ctlq_ctx;
	struct libie_ctlq_info *asq;
	struct libie_ctlq_info *arq;
	struct libie_ctlq_xn_manager *xnm;
	u16 num_avail_msix;
	u16 num_msix_entries;
	struct msix_entry *msix_entries;
	u16 num_rdma_msix_entries;
	struct msix_entry *rdma_msix_entries;
	struct virtchnl2_alloc_vectors *req_vec_chunks;
	struct idpf_q_vector mb_vector;
	struct idpf_vector_lifo vector_stack;
	irqreturn_t (*irq_mb_handler)(int irq, void *data);

	u32 tx_timeout_count;
	struct idpf_avail_queue_info avail_queues;
	struct idpf_vport **vports;
	struct net_device **netdevs;
	struct virtchnl2_create_vport **vport_params_recvd;
	u32 *vport_ids;

	struct libeth_rx_pt *singleq_pt_lkup;
	struct libeth_rx_pt *splitq_pt_lkup;

	struct idpf_vport_config **vport_config;
	u16 max_vports;
	u16 num_alloc_vports;
	u16 next_vport;

	struct delayed_work init_task;
	struct workqueue_struct *init_wq;
	struct delayed_work serv_task;
	struct workqueue_struct *serv_wq;
	struct delayed_work mbx_task;
	struct workqueue_struct *mbx_wq;
	struct delayed_work vc_event_task;
	struct workqueue_struct *vc_event_wq;
	struct delayed_work stats_task;
	struct workqueue_struct *stats_wq;
	struct virtchnl2_get_capabilities caps;

	struct idpf_dev_ops dev_ops;
	struct iidc_rdma_core_dev_info *cdev_info;
	int num_vfs;
	bool crc_enable;
	bool req_tx_splitq;
	bool req_rx_splitq;

	struct mutex vport_ctrl_lock;
	struct mutex vector_lock;
	struct mutex queue_lock;
	struct mutex vc_buf_lock;

	struct idpf_ptp *ptp;
};

/**
 * idpf_is_queue_model_split - check if queue model is split
 * @q_model: queue model single or split
 *
 * Returns true if queue model is split else false
 */
static inline int idpf_is_queue_model_split(u16 q_model)
{
	return !IS_ENABLED(CONFIG_IDPF_SINGLEQ) ||
	       q_model == VIRTCHNL2_QUEUE_MODEL_SPLIT;
}

static inline bool idpf_xdp_enabled(const struct idpf_vport *vport)
{
	return vport->adapter && vport->xdp_prog;
}

#define idpf_is_cap_ena(adapter, field, flag) \
	idpf_is_capability_ena(adapter, false, field, flag)
#define idpf_is_cap_ena_all(adapter, field, flag) \
	idpf_is_capability_ena(adapter, true, field, flag)

bool idpf_is_capability_ena(struct idpf_adapter *adapter, bool all,
			    enum idpf_cap_field field, u64 flag);

/**
 * idpf_is_rdma_cap_ena - Determine if RDMA is supported
 * @adapter: private data struct
 *
 * Return: true if RDMA capability is enabled, false otherwise
 */
static inline bool idpf_is_rdma_cap_ena(struct idpf_adapter *adapter)
{
	return idpf_is_cap_ena(adapter, IDPF_OTHER_CAPS, VIRTCHNL2_CAP_RDMA);
}

#define IDPF_CAP_RSS (\
	VIRTCHNL2_FLOW_IPV4_TCP		|\
	VIRTCHNL2_FLOW_IPV4_UDP		|\
	VIRTCHNL2_FLOW_IPV4_SCTP	|\
	VIRTCHNL2_FLOW_IPV4_OTHER	|\
	VIRTCHNL2_FLOW_IPV6_TCP		|\
	VIRTCHNL2_FLOW_IPV6_UDP		|\
	VIRTCHNL2_FLOW_IPV6_SCTP	|\
	VIRTCHNL2_FLOW_IPV6_OTHER)

#define IDPF_CAP_RSC (\
	VIRTCHNL2_CAP_RSC_IPV4_TCP	|\
	VIRTCHNL2_CAP_RSC_IPV6_TCP)

#define IDPF_CAP_HSPLIT	(\
	VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V4	|\
	VIRTCHNL2_CAP_RX_HSPLIT_AT_L4V6)

#define IDPF_CAP_TX_CSUM_L4V4 (\
	VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_TCP	|\
	VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_UDP)

#define IDPF_CAP_TX_CSUM_L4V6 (\
	VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_TCP	|\
	VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_UDP)

#define IDPF_CAP_RX_CSUM (\
	VIRTCHNL2_CAP_RX_CSUM_L3_IPV4		|\
	VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_TCP	|\
	VIRTCHNL2_CAP_RX_CSUM_L4_IPV4_UDP	|\
	VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_TCP	|\
	VIRTCHNL2_CAP_RX_CSUM_L4_IPV6_UDP)

#define IDPF_CAP_TX_SCTP_CSUM (\
	VIRTCHNL2_CAP_TX_CSUM_L4_IPV4_SCTP	|\
	VIRTCHNL2_CAP_TX_CSUM_L4_IPV6_SCTP)

#define IDPF_CAP_TUNNEL_TX_CSUM (\
	VIRTCHNL2_CAP_TX_CSUM_L3_SINGLE_TUNNEL	|\
	VIRTCHNL2_CAP_TX_CSUM_L4_SINGLE_TUNNEL)

/**
 * idpf_get_reserved_vecs - Get reserved vectors
 * @adapter: private data struct
 */
static inline u16 idpf_get_reserved_vecs(struct idpf_adapter *adapter)
{
	return le16_to_cpu(adapter->caps.num_allocated_vectors);
}

/**
 * idpf_get_reserved_rdma_vecs - Get reserved RDMA vectors
 * @adapter: private data struct
 *
 * Return: number of vectors reserved for RDMA
 */
static inline u16 idpf_get_reserved_rdma_vecs(struct idpf_adapter *adapter)
{
	return le16_to_cpu(adapter->caps.num_rdma_allocated_vectors);
}

/**
 * idpf_get_default_vports - Get default number of vports
 * @adapter: private data struct
 */
static inline u16 idpf_get_default_vports(struct idpf_adapter *adapter)
{
	return le16_to_cpu(adapter->caps.default_num_vports);
}

/**
 * idpf_get_max_vports - Get max number of vports
 * @adapter: private data struct
 */
static inline u16 idpf_get_max_vports(struct idpf_adapter *adapter)
{
	return le16_to_cpu(adapter->caps.max_vports);
}

/**
 * idpf_get_max_tx_bufs - Get max scatter-gather buffers supported by the device
 * @adapter: private data struct
 */
static inline unsigned int idpf_get_max_tx_bufs(struct idpf_adapter *adapter)
{
	return adapter->caps.max_sg_bufs_per_tx_pkt;
}

/**
 * idpf_get_min_tx_pkt_len - Get min packet length supported by the device
 * @adapter: private data struct
 */
static inline u8 idpf_get_min_tx_pkt_len(struct idpf_adapter *adapter)
{
	u8 pkt_len = adapter->caps.min_sso_packet_len;

	return pkt_len ? pkt_len : IDPF_TX_MIN_PKT_LEN;
}

/**
 * idpf_is_reset_detected - check if we were reset at some point
 * @adapter: driver specific private structure
 *
 * Returns true if we are either in reset currently or were previously reset.
 */
static inline bool idpf_is_reset_detected(struct idpf_adapter *adapter)
{
	struct libie_ctlq_info *arq = adapter->arq;

	if (!arq)
		return true;

	return !(readl(arq->reg.len) & arq->reg.len_mask);
}

/**
 * idpf_is_reset_in_prog - check if reset is in progress
 * @adapter: driver specific private structure
 *
 * Returns true if hard reset is in progress, false otherwise
 */
static inline bool idpf_is_reset_in_prog(struct idpf_adapter *adapter)
{
	return (test_bit(IDPF_HR_RESET_IN_PROG, adapter->flags) ||
		test_bit(IDPF_HR_FUNC_RESET, adapter->flags) ||
		test_bit(IDPF_HR_DRV_LOAD, adapter->flags));
}

/**
 * idpf_netdev_to_vport - get a vport handle from a netdev
 * @netdev: network interface device structure
 */
static inline struct idpf_vport *idpf_netdev_to_vport(struct net_device *netdev)
{
	struct idpf_netdev_priv *np = netdev_priv(netdev);

	return np->vport;
}

/**
 * idpf_netdev_to_adapter - Get adapter handle from a netdev
 * @netdev: Network interface device structure
 */
static inline struct idpf_adapter *idpf_netdev_to_adapter(struct net_device *netdev)
{
	struct idpf_netdev_priv *np = netdev_priv(netdev);

	return np->adapter;
}

/**
 * idpf_is_feature_ena - Determine if a particular feature is enabled
 * @vport: Vport to check
 * @feature: Netdev flag to check
 *
 * Returns true or false if a particular feature is enabled.
 */
static inline bool idpf_is_feature_ena(const struct idpf_vport *vport,
				       netdev_features_t feature)
{
	return vport->netdev->features & feature;
}

/**
 * idpf_get_max_tx_hdr_size -- get the size of tx header
 * @adapter: Driver specific private structure
 */
static inline u16 idpf_get_max_tx_hdr_size(struct idpf_adapter *adapter)
{
	return le16_to_cpu(adapter->caps.max_tx_hdr_size);
}

/**
 * idpf_vport_ctrl_lock - Acquire the vport control lock
 * @netdev: Network interface device structure
 *
 * This lock should be used by non-datapath code to protect against vport
 * destruction.
 */
static inline void idpf_vport_ctrl_lock(struct net_device *netdev)
{
	struct idpf_netdev_priv *np = netdev_priv(netdev);

	mutex_lock(&np->adapter->vport_ctrl_lock);
}

/**
 * idpf_vport_ctrl_unlock - Release the vport control lock
 * @netdev: Network interface device structure
 */
static inline void idpf_vport_ctrl_unlock(struct net_device *netdev)
{
	struct idpf_netdev_priv *np = netdev_priv(netdev);

	mutex_unlock(&np->adapter->vport_ctrl_lock);
}

static inline bool idpf_vport_ctrl_is_locked(struct net_device *netdev)
{
	struct idpf_netdev_priv *np = netdev_priv(netdev);

	return mutex_is_locked(&np->adapter->vport_ctrl_lock);
}

void idpf_statistics_task(struct work_struct *work);
void idpf_init_task(struct work_struct *work);
void idpf_service_task(struct work_struct *work);
void idpf_mbx_task(struct work_struct *work);
void idpf_vc_event_task(struct work_struct *work);
void idpf_dev_ops_init(struct idpf_adapter *adapter);
void idpf_vf_dev_ops_init(struct idpf_adapter *adapter);
int idpf_intr_req(struct idpf_adapter *adapter);
void idpf_mb_intr_rel_irq(struct idpf_adapter *adapter);
void idpf_intr_rel(struct idpf_adapter *adapter);
u16 idpf_get_max_tx_hdr_size(struct idpf_adapter *adapter);
int idpf_initiate_soft_reset(struct idpf_vport *vport,
			     enum idpf_vport_reset_cause reset_cause);
void idpf_deinit_task(struct idpf_adapter *adapter);
int idpf_req_rel_vector_indexes(struct idpf_adapter *adapter,
				u16 *q_vector_idxs,
				struct idpf_vector_info *vec_info);
void idpf_set_ethtool_ops(struct net_device *netdev);
void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector,
			       u16 itr, bool tx);
int idpf_sriov_configure(struct pci_dev *pdev, int num_vfs);

u8 idpf_vport_get_hsplit(const struct idpf_vport *vport);
bool idpf_vport_set_hsplit(const struct idpf_vport *vport, u8 val);
int idpf_idc_init(struct idpf_adapter *adapter);
int idpf_idc_init_aux_core_dev(struct idpf_adapter *adapter,
			       enum iidc_function_type ftype);
void idpf_idc_deinit_core_aux_device(struct idpf_adapter *adapter);
void idpf_idc_deinit_vport_aux_device(struct iidc_rdma_vport_dev_info *vdev_info);
void idpf_idc_issue_reset_event(struct iidc_rdma_core_dev_info *cdev_info);
void idpf_idc_vdev_mtu_event(struct iidc_rdma_vport_dev_info *vdev_info,
			     enum iidc_rdma_event_type event_type);

int idpf_add_del_fsteer_filters(struct idpf_adapter *adapter,
				struct virtchnl2_flow_rule_add_del *rule,
				enum virtchnl2_op opcode);
#endif /* !_IDPF_H_ */