drivers/net/ethernet/intel/idpf/xdp.c

// SPDX-License-Identifier: GPL-2.0-only
/* Copyright (C) 2025 Intel Corporation */

#include "idpf.h"
#include "idpf_ptp.h"
#include "idpf_virtchnl.h"
#include "xdp.h"
#include "xsk.h"

static int idpf_rxq_for_each(const struct idpf_q_vec_rsrc *rsrc,
			     int (*fn)(struct idpf_rx_queue *rxq, void *arg),
			     void *arg)
{
	bool splitq = idpf_is_queue_model_split(rsrc->rxq_model);

	if (!rsrc->rxq_grps)
		return -ENETDOWN;

	for (u32 i = 0; i < rsrc->num_rxq_grp; i++) {
		const struct idpf_rxq_group *rx_qgrp = &rsrc->rxq_grps[i];
		u32 num_rxq;

		if (splitq)
			num_rxq = rx_qgrp->splitq.num_rxq_sets;
		else
			num_rxq = rx_qgrp->singleq.num_rxq;

		for (u32 j = 0; j < num_rxq; j++) {
			struct idpf_rx_queue *q;
			int err;

			if (splitq)
				q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
			else
				q = rx_qgrp->singleq.rxqs[j];

			err = fn(q, arg);
			if (err)
				return err;
		}
	}

	return 0;
}

static int __idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq, void *arg)
{
	const struct idpf_vport *vport = rxq->q_vector->vport;
	const struct idpf_q_vec_rsrc *rsrc;
	u32 frag_size = 0;
	bool split;
	int err;

	if (idpf_queue_has(XSK, rxq))
		frag_size = rxq->bufq_sets[0].bufq.truesize;

	err = __xdp_rxq_info_reg(&rxq->xdp_rxq, vport->netdev, rxq->idx,
				 rxq->q_vector->napi.napi_id,
				 frag_size);
	if (err)
		return err;

	rsrc = &vport->dflt_qv_rsrc;
	split = idpf_is_queue_model_split(rsrc->rxq_model);

	if (idpf_queue_has(XSK, rxq)) {
		err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq,
						 MEM_TYPE_XSK_BUFF_POOL,
						 rxq->pool);
		if (err)
			goto unreg;
	} else {
		const struct page_pool *pp;

		pp = split ? rxq->bufq_sets[0].bufq.pp : rxq->pp;
		xdp_rxq_info_attach_page_pool(&rxq->xdp_rxq, pp);
	}

	if (!split)
		return 0;

	rxq->xdpsqs = &vport->txqs[rsrc->xdp_txq_offset];
	rxq->num_xdp_txq = vport->num_xdp_txq;

	return 0;

unreg:
	xdp_rxq_info_unreg(&rxq->xdp_rxq);

	return err;
}

int idpf_xdp_rxq_info_init(struct idpf_rx_queue *rxq)
{
	return __idpf_xdp_rxq_info_init(rxq, NULL);
}

int idpf_xdp_rxq_info_init_all(const struct idpf_q_vec_rsrc *rsrc)
{
	return idpf_rxq_for_each(rsrc, __idpf_xdp_rxq_info_init, NULL);
}

static int __idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, void *arg)
{
	if (idpf_is_queue_model_split((size_t)arg)) {
		rxq->xdpsqs = NULL;
		rxq->num_xdp_txq = 0;
	}

	if (!idpf_queue_has(XSK, rxq))
		xdp_rxq_info_detach_mem_model(&rxq->xdp_rxq);

	xdp_rxq_info_unreg(&rxq->xdp_rxq);

	return 0;
}

void idpf_xdp_rxq_info_deinit(struct idpf_rx_queue *rxq, u32 model)
{
	__idpf_xdp_rxq_info_deinit(rxq, (void *)(size_t)model);
}

void idpf_xdp_rxq_info_deinit_all(const struct idpf_q_vec_rsrc *rsrc)
{
	idpf_rxq_for_each(rsrc, __idpf_xdp_rxq_info_deinit,
			  (void *)(size_t)rsrc->rxq_model);
}

static int idpf_xdp_rxq_assign_prog(struct idpf_rx_queue *rxq, void *arg)
{
	struct bpf_prog *prog = arg;
	struct bpf_prog *old;

	if (prog)
		bpf_prog_inc(prog);

	old = rcu_replace_pointer(rxq->xdp_prog, prog, lockdep_rtnl_is_held());
	if (old)
		bpf_prog_put(old);

	return 0;
}

void idpf_xdp_copy_prog_to_rqs(const struct idpf_q_vec_rsrc *rsrc,
			       struct bpf_prog *xdp_prog)
{
	idpf_rxq_for_each(rsrc, idpf_xdp_rxq_assign_prog, xdp_prog);
}

static void idpf_xdp_tx_timer(struct work_struct *work);

int idpf_xdpsqs_get(const struct idpf_vport *vport)
{
	struct libeth_xdpsq_timer **timers __free(kvfree) = NULL;
	struct net_device *dev;
	u32 sqs;

	if (!idpf_xdp_enabled(vport))
		return 0;

	timers = kvzalloc_objs(*timers, vport->num_xdp_txq);
	if (!timers)
		return -ENOMEM;

	for (u32 i = 0; i < vport->num_xdp_txq; i++) {
		timers[i] = kzalloc_node(sizeof(*timers[i]), GFP_KERNEL,
					 cpu_to_mem(i));
		if (!timers[i]) {
			for (int j = i - 1; j >= 0; j--)
				kfree(timers[j]);

			return -ENOMEM;
		}
	}

	dev = vport->netdev;
	sqs = vport->dflt_qv_rsrc.xdp_txq_offset;

	for (u32 i = sqs; i < vport->num_txq; i++) {
		struct idpf_tx_queue *xdpsq = vport->txqs[i];

		xdpsq->complq = xdpsq->txq_grp->complq;
		kfree(xdpsq->refillq);
		xdpsq->refillq = NULL;

		idpf_queue_clear(FLOW_SCH_EN, xdpsq);
		idpf_queue_clear(FLOW_SCH_EN, xdpsq->complq);
		idpf_queue_set(NOIRQ, xdpsq);
		idpf_queue_set(XDP, xdpsq);
		idpf_queue_set(XDP, xdpsq->complq);

		xdpsq->timer = timers[i - sqs];
		libeth_xdpsq_get(&xdpsq->xdp_lock, dev, vport->xdpsq_share);
		libeth_xdpsq_init_timer(xdpsq->timer, xdpsq, &xdpsq->xdp_lock,
					idpf_xdp_tx_timer);

		xdpsq->pending = 0;
		xdpsq->xdp_tx = 0;
		xdpsq->thresh = libeth_xdp_queue_threshold(xdpsq->desc_count);
	}

	return 0;
}

void idpf_xdpsqs_put(const struct idpf_vport *vport)
{
	struct net_device *dev;
	u32 sqs;

	if (!idpf_xdp_enabled(vport))
		return;

	dev = vport->netdev;
	sqs = vport->dflt_qv_rsrc.xdp_txq_offset;

	for (u32 i = sqs; i < vport->num_txq; i++) {
		struct idpf_tx_queue *xdpsq = vport->txqs[i];

		if (!idpf_queue_has_clear(XDP, xdpsq))
			continue;

		libeth_xdpsq_deinit_timer(xdpsq->timer);
		libeth_xdpsq_put(&xdpsq->xdp_lock, dev);

		kfree(xdpsq->timer);
		xdpsq->refillq = NULL;
		idpf_queue_clear(NOIRQ, xdpsq);
	}
}

static int idpf_xdp_parse_cqe(const struct idpf_splitq_4b_tx_compl_desc *desc,
			      bool gen)
{
	u32 val;

#ifdef __LIBETH_WORD_ACCESS
	val = *(const u32 *)desc;
#else
	val = ((u32)le16_to_cpu(desc->q_head_compl_tag.q_head) << 16) |
	      le16_to_cpu(desc->qid_comptype_gen);
#endif
	if (!!(val & IDPF_TXD_COMPLQ_GEN_M) != gen)
		return -ENODATA;

	if (unlikely((val & GENMASK(IDPF_TXD_COMPLQ_GEN_S - 1, 0)) !=
		     FIELD_PREP(IDPF_TXD_COMPLQ_COMPL_TYPE_M,
				IDPF_TXD_COMPLT_RS)))
		return -EINVAL;

	return upper_16_bits(val);
}

u32 idpf_xdpsq_poll(struct idpf_tx_queue *xdpsq, u32 budget)
{
	struct idpf_compl_queue *cq = xdpsq->complq;
	u32 tx_ntc = xdpsq->next_to_clean;
	u32 tx_cnt = xdpsq->desc_count;
	u32 ntc = cq->next_to_clean;
	u32 cnt = cq->desc_count;
	u32 done_frames;
	bool gen;

	gen = idpf_queue_has(GEN_CHK, cq);

	for (done_frames = 0; done_frames < budget; ) {
		int ret;

		ret = idpf_xdp_parse_cqe(&cq->comp_4b[ntc], gen);
		if (ret >= 0) {
			done_frames = ret > tx_ntc ? ret - tx_ntc :
						     ret + tx_cnt - tx_ntc;
			goto next;
		}

		switch (ret) {
		case -ENODATA:
			goto out;
		case -EINVAL:
			break;
		}

next:
		if (unlikely(++ntc == cnt)) {
			ntc = 0;
			gen = !gen;
			idpf_queue_change(GEN_CHK, cq);
		}
	}

out:
	cq->next_to_clean = ntc;

	return done_frames;
}

static u32 idpf_xdpsq_complete(void *_xdpsq, u32 budget)
{
	struct libeth_xdpsq_napi_stats ss = { };
	struct idpf_tx_queue *xdpsq = _xdpsq;
	u32 tx_ntc = xdpsq->next_to_clean;
	u32 tx_cnt = xdpsq->desc_count;
	struct xdp_frame_bulk bq;
	struct libeth_cq_pp cp = {
		.dev	= xdpsq->dev,
		.bq	= &bq,
		.xss	= &ss,
		.napi	= true,
	};
	u32 done_frames;

	done_frames = idpf_xdpsq_poll(xdpsq, budget);
	if (unlikely(!done_frames))
		return 0;

	xdp_frame_bulk_init(&bq);

	for (u32 i = 0; likely(i < done_frames); i++) {
		libeth_xdp_complete_tx(&xdpsq->tx_buf[tx_ntc], &cp);

		if (unlikely(++tx_ntc == tx_cnt))
			tx_ntc = 0;
	}

	xdp_flush_frame_bulk(&bq);

	xdpsq->next_to_clean = tx_ntc;
	xdpsq->pending -= done_frames;
	xdpsq->xdp_tx -= cp.xdp_tx;

	return done_frames;
}

static u32 idpf_xdp_tx_prep(void *_xdpsq, struct libeth_xdpsq *sq)
{
	struct idpf_tx_queue *xdpsq = _xdpsq;
	u32 free;

	libeth_xdpsq_lock(&xdpsq->xdp_lock);

	free = xdpsq->desc_count - xdpsq->pending;
	if (free < xdpsq->thresh)
		free += idpf_xdpsq_complete(xdpsq, xdpsq->thresh);

	*sq = (struct libeth_xdpsq){
		.sqes		= xdpsq->tx_buf,
		.descs		= xdpsq->desc_ring,
		.count		= xdpsq->desc_count,
		.lock		= &xdpsq->xdp_lock,
		.ntu		= &xdpsq->next_to_use,
		.pending	= &xdpsq->pending,
		.xdp_tx		= &xdpsq->xdp_tx,
	};

	return free;
}

LIBETH_XDP_DEFINE_START();
LIBETH_XDP_DEFINE_TIMER(static idpf_xdp_tx_timer, idpf_xdpsq_complete);
LIBETH_XDP_DEFINE_FLUSH_TX(idpf_xdp_tx_flush_bulk, idpf_xdp_tx_prep,
			   idpf_xdp_tx_xmit);
LIBETH_XDP_DEFINE_FLUSH_XMIT(static idpf_xdp_xmit_flush_bulk, idpf_xdp_tx_prep,
			     idpf_xdp_tx_xmit);
LIBETH_XDP_DEFINE_END();

int idpf_xdp_xmit(struct net_device *dev, int n, struct xdp_frame **frames,
		  u32 flags)
{
	const struct idpf_netdev_priv *np = netdev_priv(dev);
	const struct idpf_vport *vport = np->vport;
	u32 xdp_txq_offset;

	if (unlikely(!netif_carrier_ok(dev) || !vport->link_up))
		return -ENETDOWN;

	xdp_txq_offset = vport->dflt_qv_rsrc.xdp_txq_offset;

	return libeth_xdp_xmit_do_bulk(dev, n, frames, flags,
				       &vport->txqs[xdp_txq_offset],
				       vport->num_xdp_txq,
				       idpf_xdp_xmit_flush_bulk,
				       idpf_xdp_tx_finalize);
}

static int idpf_xdpmo_rx_hash(const struct xdp_md *ctx, u32 *hash,
			      enum xdp_rss_hash_type *rss_type)
{
	const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx;
	struct idpf_xdp_rx_desc desc __uninitialized;
	const struct idpf_rx_queue *rxq;
	struct libeth_rx_pt pt;

	rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq);

	idpf_xdp_get_qw0(&desc, xdp->desc);

	pt = rxq->rx_ptype_lkup[idpf_xdp_rx_pt(&desc)];
	if (!libeth_rx_pt_has_hash(rxq->xdp_rxq.dev, pt))
		return -ENODATA;

	idpf_xdp_get_qw2(&desc, xdp->desc);

	return libeth_xdpmo_rx_hash(hash, rss_type, idpf_xdp_rx_hash(&desc),
				    pt);
}

static int idpf_xdpmo_rx_timestamp(const struct xdp_md *ctx, u64 *timestamp)
{
	const struct libeth_xdp_buff *xdp = (typeof(xdp))ctx;
	struct idpf_xdp_rx_desc desc __uninitialized;
	const struct idpf_rx_queue *rxq;
	u64 cached_time, ts_ns;
	u32 ts_high;

	rxq = libeth_xdp_buff_to_rq(xdp, typeof(*rxq), xdp_rxq);

	if (!idpf_queue_has(PTP, rxq))
		return -ENODATA;

	idpf_xdp_get_qw1(&desc, xdp->desc);

	if (!(idpf_xdp_rx_ts_low(&desc) & VIRTCHNL2_RX_FLEX_TSTAMP_VALID))
		return -ENODATA;

	cached_time = READ_ONCE(rxq->cached_phc_time);

	idpf_xdp_get_qw3(&desc, xdp->desc);

	ts_high = idpf_xdp_rx_ts_high(&desc);
	ts_ns = idpf_ptp_tstamp_extend_32b_to_64b(cached_time, ts_high);

	*timestamp = ts_ns;
	return 0;
}

static const struct xdp_metadata_ops idpf_xdpmo = {
	.xmo_rx_hash		= idpf_xdpmo_rx_hash,
	.xmo_rx_timestamp	= idpf_xdpmo_rx_timestamp,
};

void idpf_xdp_set_features(const struct idpf_vport *vport)
{
	if (!idpf_is_queue_model_split(vport->dflt_qv_rsrc.rxq_model))
		return;

	libeth_xdp_set_features_noredir(vport->netdev, &idpf_xdpmo,
					idpf_get_max_tx_bufs(vport->adapter),
					libeth_xsktmo);
}

static int idpf_xdp_setup_prog(struct idpf_vport *vport,
			       const struct netdev_bpf *xdp)
{
	const struct idpf_netdev_priv *np = netdev_priv(vport->netdev);
	const struct idpf_q_vec_rsrc *rsrc = &vport->dflt_qv_rsrc;
	struct bpf_prog *old, *prog = xdp->prog;
	struct idpf_vport_config *cfg;
	int ret;

	cfg = vport->adapter->vport_config[vport->idx];

	if (test_bit(IDPF_REMOVE_IN_PROG, vport->adapter->flags) ||
	    !test_bit(IDPF_VPORT_REG_NETDEV, cfg->flags) ||
	    !!vport->xdp_prog == !!prog) {
		if (test_bit(IDPF_VPORT_UP, np->state))
			idpf_xdp_copy_prog_to_rqs(rsrc, prog);

		old = xchg(&vport->xdp_prog, prog);
		if (old)
			bpf_prog_put(old);

		cfg->user_config.xdp_prog = prog;

		return 0;
	}

	if (!vport->num_xdp_txq && vport->num_txq == cfg->max_q.max_txq) {
		NL_SET_ERR_MSG_MOD(xdp->extack,
				   "No Tx queues available for XDP, please decrease the number of regular SQs");
		return -ENOSPC;
	}

	old = cfg->user_config.xdp_prog;
	cfg->user_config.xdp_prog = prog;

	ret = idpf_initiate_soft_reset(vport, IDPF_SR_Q_CHANGE);
	if (ret) {
		NL_SET_ERR_MSG_MOD(xdp->extack,
				   "Could not reopen the vport after XDP setup");

		cfg->user_config.xdp_prog = old;
		old = prog;
	}

	if (old)
		bpf_prog_put(old);

	libeth_xdp_set_redirect(vport->netdev, vport->xdp_prog);

	return ret;
}

int idpf_xdp(struct net_device *dev, struct netdev_bpf *xdp)
{
	struct idpf_vport *vport;
	int ret;

	idpf_vport_ctrl_lock(dev);
	vport = idpf_netdev_to_vport(dev);

	if (!idpf_is_queue_model_split(vport->dflt_qv_rsrc.txq_model))
		goto notsupp;

	switch (xdp->command) {
	case XDP_SETUP_PROG:
		ret = idpf_xdp_setup_prog(vport, xdp);
		break;
	case XDP_SETUP_XSK_POOL:
		ret = idpf_xsk_pool_setup(vport, xdp);
		break;
	default:
notsupp:
		ret = -EOPNOTSUPP;
		break;
	}

	idpf_vport_ctrl_unlock(dev);

	return ret;
}