drivers/infiniband/hw/ionic/ionic_ibdev.h

/* SPDX-License-Identifier: GPL-2.0 */
/* Copyright (C) 2018-2025, Advanced Micro Devices, Inc. */

#ifndef _IONIC_IBDEV_H_
#define _IONIC_IBDEV_H_

#include <rdma/ib_verbs.h>
#include <rdma/ib_pack.h>
#include <rdma/iter.h>
#include <rdma/uverbs_ioctl.h>

#include <rdma/ionic-abi.h>
#include <ionic_api.h>
#include <ionic_regs.h>

#include "ionic_fw.h"
#include "ionic_queue.h"
#include "ionic_res.h"

#include "ionic_lif_cfg.h"

/* Config knobs */
#define IONIC_EQ_DEPTH 511
#define IONIC_EQ_COUNT 32
#define IONIC_AQ_DEPTH 63
#define IONIC_AQ_COUNT 4
#define IONIC_EQ_ISR_BUDGET 10
#define IONIC_EQ_WORK_BUDGET 1000
#define IONIC_MAX_RD_ATOM 16
#define IONIC_PKEY_TBL_LEN 1
#define IONIC_GID_TBL_LEN 256

#define IONIC_MAX_QPID 0xffffff
#define IONIC_SPEC_HIGH 8
#define IONIC_MAX_PD 1024
#define IONIC_SQCMB_ORDER 5
#define IONIC_RQCMB_ORDER 0

#define IONIC_MAX_SRQ_SGES	2
#define IONIC_MAX_SRQ_LIMIT	0xffff
#define IONIC_MAX_SRQ_DEPTH	0xffff

#define IONIC_META_LAST		((void *)1ul)
#define IONIC_META_POSTED	((void *)2ul)

#define IONIC_CQ_GRACE 100

#define IONIC_ROCE_UDP_SPORT	28272
#define IONIC_DMA_LKEY		0
#define IONIC_DMA_RKEY		IONIC_DMA_LKEY

#define IONIC_CMB_SUPPORTED \
	(IONIC_CMB_ENABLE | IONIC_CMB_REQUIRE | IONIC_CMB_EXPDB | \
	 IONIC_CMB_WC | IONIC_CMB_UC)

/* resource is not reserved on the device, indicated in tbl_order */
#define IONIC_RES_INVALID	-1

struct ionic_aq;
struct ionic_cq;
struct ionic_eq;
struct ionic_vcq;

enum ionic_admin_state {
	IONIC_ADMIN_ACTIVE, /* submitting admin commands to queue */
	IONIC_ADMIN_PAUSED, /* not submitting, but may complete normally */
	IONIC_ADMIN_KILLED, /* not submitting, locally completed */
};

enum ionic_admin_flags {
	IONIC_ADMIN_F_BUSYWAIT  = BIT(0),	/* Don't sleep */
	IONIC_ADMIN_F_TEARDOWN  = BIT(1),	/* In destroy path */
	IONIC_ADMIN_F_INTERRUPT = BIT(2),	/* Interruptible w/timeout */
};

enum ionic_mmap_flag {
	IONIC_MMAP_WC = BIT(0),
	IONIC_MMAP_PHC = BIT(1),
};

struct ionic_mmap_entry {
	struct rdma_user_mmap_entry rdma_entry;
	unsigned long size;
	unsigned long pfn;
	u8 mmap_flags;
};

struct ionic_ibdev {
	struct ib_device	ibdev;

	struct ionic_lif_cfg	lif_cfg;

	struct xarray		qp_tbl;
	struct xarray		cq_tbl;
	struct xarray		srq_tbl;

	struct ionic_resid_bits	inuse_dbid;
	struct ionic_resid_bits	inuse_pdid;
	struct ionic_resid_bits	inuse_ahid;
	struct ionic_resid_bits	inuse_mrid;
	struct ionic_resid_bits	inuse_qpid;
	struct ionic_resid_bits	inuse_cqid;

	u8			half_cqid_udma_shift;
	u8			half_qpid_udma_shift;
	u8			next_qpid_udma_idx;
	u8			next_mrkey;

	struct work_struct	reset_work;
	bool			reset_posted;
	u32			reset_cnt;

	struct delayed_work	admin_dwork;
	struct ionic_aq		**aq_vec;
	atomic_t		admin_state;

	struct ionic_eq		**eq_vec;

	struct ionic_v1_stat	*hw_stats;
	void			*hw_stats_buf;
	struct rdma_stat_desc	*hw_stats_hdrs;
	struct ionic_counter_stats *counter_stats;
	int			hw_stats_count;
};

struct ionic_eq {
	struct ionic_ibdev	*dev;

	u32			eqid;
	u32			intr;

	struct ionic_queue	q;

	int			armed;
	bool			enable;

	struct work_struct	work;

	int			irq;
	char			name[32];
};

struct ionic_admin_wr {
	struct completion		work;
	struct list_head		aq_ent;
	struct ionic_v1_admin_wqe	wqe;
	struct ionic_v1_cqe		cqe;
	struct ionic_aq			*aq;
	int				status;
};

struct ionic_admin_wr_q {
	struct ionic_admin_wr	*wr;
	int			wqe_strides;
};

struct ionic_aq {
	struct ionic_ibdev	*dev;
	struct ionic_vcq	*vcq;

	struct work_struct	work;

	atomic_t		admin_state;
	unsigned long		stamp;
	bool			armed;

	u32			aqid;
	u32			cqid;

	spinlock_t		lock; /* for posting */
	struct ionic_queue	q;
	struct ionic_admin_wr_q	*q_wr;
	struct list_head	wr_prod;
	struct list_head	wr_post;
};

struct ionic_ctx {
	struct ib_ucontext	ibctx;
	u32			dbid;
	struct rdma_user_mmap_entry	*mmap_dbell;
	struct rdma_user_mmap_entry	*mmap_phc;
};

struct ionic_tbl_buf {
	u32		tbl_limit;
	u32		tbl_pages;
	size_t		tbl_size;
	__le64		*tbl_buf;
	dma_addr_t	tbl_dma;
	u8		page_size_log2;
};

struct ionic_pd {
	struct ib_pd		ibpd;

	u32			pdid;
	u32			flags;
};

struct ionic_cq {
	struct ionic_vcq	*vcq;

	u32			cqid;
	u32			eqid;

	spinlock_t		lock; /* for polling */
	struct list_head	poll_sq;
	bool			flush;
	struct list_head	flush_sq;
	struct list_head	flush_rq;
	struct list_head	ibkill_flush_ent;

	struct ionic_queue	q;
	bool			color;
	int			credit;
	u16			arm_any_prod;
	u16			arm_sol_prod;

	struct kref		cq_kref;
	struct completion	cq_rel_comp;

	/* infrequently accessed, keep at end */
	struct ib_umem		*umem;
};

struct ionic_vcq {
	struct ib_cq		ibcq;
	struct ionic_cq		cq[2];
	u8			udma_mask;
	u8			poll_idx;
};

struct ionic_sq_meta {
	u64			wrid;
	u32			len;
	u16			seq;
	u8			ibop;
	u8			ibsts;
	u8			remote:1;
	u8			signal:1;
	u8			local_comp:1;
};

struct ionic_rq_meta {
	struct ionic_rq_meta	*next;
	u64			wrid;
};

struct ionic_rq {
	struct ionic_queue		q;
	u32				qid;
	u32				cqid;
	struct list_head		cq_flush;
	spinlock_t			lock; /* for posting and polling */

	phys_addr_t			cmb_addr;
	struct rdma_user_mmap_entry	*mmap_cmb;

	struct ionic_rq_meta		*meta;
	struct ionic_rq_meta		*meta_head;
	struct ib_umem			*umem;

	int				spec;
	int				cmb_order;
	u32				cmb_pgid;
	u16				old_prod;
	u8				cmb;
	bool				flush;
};

struct ionic_srq {
	struct ib_srq		ibsrq;
	struct ionic_rq		rq;
	struct kref		kref;
	struct completion	rel_comp;
	u16			srq_limit;
	u8			udma_idx;
};

struct ionic_qp {
	struct ib_qp		ibqp;
	enum ib_qp_state	state;

	u32			qpid;
	u32			ahid;
	u32			sq_cqid;
	u8			udma_idx;
	u8			has_ah:1;
	u8			has_sq:1;
	u8			sig_all:1;

	struct list_head	qp_list_counter;

	struct list_head	cq_poll_sq;
	struct list_head	cq_flush_sq;
	struct list_head	ibkill_flush_ent;

	spinlock_t		sq_lock; /* for posting and polling */
	struct ionic_queue	sq;
	struct ionic_sq_meta	*sq_meta;
	u16			*sq_msn_idx;
	int			sq_spec;
	u16			sq_old_prod;
	u16			sq_msn_prod;
	u16			sq_msn_cons;
	u8			sq_cmb;
	bool			sq_flush;
	bool			sq_flush_rcvd;

	struct ionic_rq		rq;

	struct kref		qp_kref;
	struct completion	qp_rel_comp;

	/* infrequently accessed, keep at end */
	int			sgid_index;
	int			sq_cmb_order;
	u32			sq_cmb_pgid;
	phys_addr_t		sq_cmb_addr;
	struct rdma_user_mmap_entry *mmap_sq_cmb;

	struct ib_umem		*sq_umem;

	int			dcqcn_profile;

	struct ib_ud_header	*hdr;
	struct ionic_srq	*srq;
};

struct ionic_ah {
	struct ib_ah		ibah;
	u32			ahid;
	int			sgid_index;
	struct ib_ud_header	hdr;
};

struct ionic_mr {
	union {
		struct ib_mr	ibmr;
		struct ib_mw	ibmw;
	};

	u32			mrid;
	int			flags;

	struct ib_umem		*umem;
	struct ionic_tbl_buf	buf;
	bool			created;
};

struct ionic_counter_stats {
	int queue_stats_count;
	struct ionic_v1_stat *hdr;
	struct rdma_stat_desc *stats_hdrs;
	struct ida counter_ida;
};

struct ionic_rdma_counter {
	struct rdma_counter rdma_counter;
	void *vals;
	struct list_head qp_list;
};

static inline struct ionic_rdma_counter *
to_ionic_rdma_counter(struct rdma_counter *counter)
{
	return container_of(counter, struct ionic_rdma_counter, rdma_counter);
}

static inline struct ionic_ibdev *to_ionic_ibdev(struct ib_device *ibdev)
{
	return container_of(ibdev, struct ionic_ibdev, ibdev);
}

static inline struct ionic_ctx *to_ionic_ctx(struct ib_ucontext *ibctx)
{
	return container_of(ibctx, struct ionic_ctx, ibctx);
}

static inline struct ionic_ctx *to_ionic_ctx_uobj(struct ib_uobject *uobj)
{
	if (!uobj)
		return NULL;

	if (!uobj->context)
		return NULL;

	return to_ionic_ctx(uobj->context);
}

static inline struct ionic_pd *to_ionic_pd(struct ib_pd *ibpd)
{
	return container_of(ibpd, struct ionic_pd, ibpd);
}

static inline struct ionic_mr *to_ionic_mr(struct ib_mr *ibmr)
{
	return container_of(ibmr, struct ionic_mr, ibmr);
}

static inline struct ionic_mr *to_ionic_mw(struct ib_mw *ibmw)
{
	return container_of(ibmw, struct ionic_mr, ibmw);
}

static inline struct ionic_vcq *to_ionic_vcq(struct ib_cq *ibcq)
{
	return container_of(ibcq, struct ionic_vcq, ibcq);
}

static inline struct ionic_cq *to_ionic_vcq_cq(struct ib_cq *ibcq,
					       uint8_t udma_idx)
{
	return &to_ionic_vcq(ibcq)->cq[udma_idx];
}

static inline struct ionic_qp *to_ionic_qp(struct ib_qp *ibqp)
{
	return container_of(ibqp, struct ionic_qp, ibqp);
}

static inline struct ionic_srq *to_ionic_srq(struct ib_srq *ibsrq)
{
	return container_of(ibsrq, struct ionic_srq, ibsrq);
}

static inline struct ionic_ah *to_ionic_ah(struct ib_ah *ibah)
{
	return container_of(ibah, struct ionic_ah, ibah);
}

static inline u32 ionic_ctx_dbid(struct ionic_ibdev *dev,
				 struct ionic_ctx *ctx)
{
	if (!ctx)
		return dev->lif_cfg.dbid;

	return ctx->dbid;
}

static inline u32 ionic_obj_dbid(struct ionic_ibdev *dev,
				 struct ib_uobject *uobj)
{
	return ionic_ctx_dbid(dev, to_ionic_ctx_uobj(uobj));
}

static inline bool ionic_ibop_is_local(enum ib_wr_opcode op)
{
	return op == IB_WR_LOCAL_INV || op == IB_WR_REG_MR;
}

static inline void ionic_qp_complete(struct kref *kref)
{
	struct ionic_qp *qp = container_of(kref, struct ionic_qp, qp_kref);

	complete(&qp->qp_rel_comp);
}

static inline void ionic_cq_complete(struct kref *kref)
{
	struct ionic_cq *cq = container_of(kref, struct ionic_cq, cq_kref);

	complete(&cq->cq_rel_comp);
}

static inline bool ionic_fw_has_qid_alloc(struct ionic_ibdev *dev,
					  enum ionic_lif_rdma_alloc_qid qtype)
{
	return dev->lif_cfg.alloc_qid_cap & qtype;
}

static inline void ionic_srq_complete(struct kref *kref)
{
	struct ionic_srq *srq = container_of(kref, struct ionic_srq, kref);

	complete(&srq->rel_comp);
}

/* ionic_admin.c */
extern struct workqueue_struct *ionic_evt_workq;
void ionic_admin_post(struct ionic_ibdev *dev, struct ionic_admin_wr *wr);
int ionic_admin_wait(struct ionic_ibdev *dev, struct ionic_admin_wr *wr,
		     enum ionic_admin_flags);

int ionic_rdma_reset_devcmd(struct ionic_ibdev *dev);

int ionic_create_rdma_admin(struct ionic_ibdev *dev);
void ionic_destroy_rdma_admin(struct ionic_ibdev *dev);
void ionic_kill_rdma_admin(struct ionic_ibdev *dev, bool fatal_path);

/* ionic_controlpath.c */
int ionic_create_cq_common(struct ionic_vcq *vcq,
			   struct ionic_tbl_buf *buf,
			   const struct ib_cq_init_attr *attr,
			   struct ionic_ctx *ctx,
			   struct ib_udata *udata,
			   struct ionic_qdesc *req_cq,
			   int udma_idx);
int ionic_post_create_cq_cmd(struct ionic_cq *cq,
			     struct ib_udata *udata,
			     __u32 *resp_cqid);
void ionic_pre_destroy_cq_cmd(struct ionic_ibdev *dev, struct ionic_cq *cq);
void ionic_destroy_cq_common(struct ionic_ibdev *dev, struct ionic_cq *cq);
void ionic_flush_qp(struct ionic_ibdev *dev, struct ionic_qp *qp);
void ionic_notify_flush_cq(struct ionic_cq *cq);

int ionic_alloc_ucontext(struct ib_ucontext *ibctx, struct ib_udata *udata);
void ionic_dealloc_ucontext(struct ib_ucontext *ibctx);
int ionic_mmap(struct ib_ucontext *ibctx, struct vm_area_struct *vma);
void ionic_mmap_free(struct rdma_user_mmap_entry *rdma_entry);
int ionic_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
int ionic_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata);
int ionic_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
		    struct ib_udata *udata);
int ionic_query_ah(struct ib_ah *ibah, struct rdma_ah_attr *ah_attr);
int ionic_destroy_ah(struct ib_ah *ibah, u32 flags);
struct ib_mr *ionic_get_dma_mr(struct ib_pd *ibpd, int access);
struct ib_mr *ionic_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 length,
				u64 addr, int access, struct ib_dmah *dmah,
				struct ib_udata *udata);
struct ib_mr *ionic_reg_user_mr_dmabuf(struct ib_pd *ibpd, u64 offset,
				       u64 length, u64 addr, int fd, int access,
					   struct ib_dmah *dmah,
				       struct uverbs_attr_bundle *attrs);
int ionic_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata);
struct ib_mr *ionic_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type type,
			     u32 max_sg);
int ionic_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents,
		    unsigned int *sg_offset);
int ionic_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata);
int ionic_dealloc_mw(struct ib_mw *ibmw);
int ionic_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
		    struct uverbs_attr_bundle *attrs);
int ionic_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata);
int ionic_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
		    struct ib_udata *udata);
int ionic_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
		    struct ib_udata *udata);
int ionic_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int mask,
		   struct ib_qp_init_attr *init_attr);
int ionic_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
int ionic_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *attr,
		     struct ib_udata *udata);
int ionic_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata);
int ionic_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr);
int ionic_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
		     enum ib_srq_attr_mask attr_mask, struct ib_udata *udata);

/* ionic_datapath.c */
int ionic_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
		    const struct ib_send_wr **bad);
int ionic_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
		    const struct ib_recv_wr **bad);
int ionic_poll_cq(struct ib_cq *ibcq, int nwc, struct ib_wc *wc);
int ionic_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags);

/* ionic_hw_stats.c */
void ionic_stats_init(struct ionic_ibdev *dev);
void ionic_stats_cleanup(struct ionic_ibdev *dev);

/* ionic_pgtbl.c */
__le64 ionic_pgtbl_dma(struct ionic_tbl_buf *buf, u64 va);
__be64 ionic_pgtbl_off(struct ionic_tbl_buf *buf, u64 va);
int ionic_pgtbl_page(struct ionic_tbl_buf *buf, u64 dma);
int ionic_pgtbl_init(struct ionic_ibdev *dev,
		     struct ionic_tbl_buf *buf,
		     struct ib_umem *umem,
		     dma_addr_t dma,
		     int limit,
		     u64 page_size);
void ionic_pgtbl_unbuf(struct ionic_ibdev *dev, struct ionic_tbl_buf *buf);
#endif /* _IONIC_IBDEV_H_ */