tools/testing/selftests/bpf/xskxceiver.c

// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2020 Intel Corporation. */

/*
 * Some functions in this program are taken from
 * Linux kernel samples/bpf/xdpsock* and modified
 * for use.
 *
 * See test_xsk.sh for detailed information on test topology
 * and prerequisite network setup.
 *
 * This test program contains two threads, each thread is single socket with
 * a unique UMEM. It validates in-order packet delivery and packet content
 * by sending packets to each other.
 *
 * Tests Information:
 * ------------------
 * These selftests test AF_XDP SKB and Native/DRV modes using veth
 * Virtual Ethernet interfaces.
 *
 * For each mode, the following tests are run:
 *    a. nopoll - soft-irq processing in run-to-completion mode
 *    b. poll - using poll() syscall
 *    c. Socket Teardown
 *       Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
 *       both sockets, then repeat multiple times. Only nopoll mode is used
 *    d. Bi-directional sockets
 *       Configure sockets as bi-directional tx/rx sockets, sets up fill and
 *       completion rings on each socket, tx/rx in both directions. Only nopoll
 *       mode is used
 *    e. Statistics
 *       Trigger some error conditions and ensure that the appropriate statistics
 *       are incremented. Within this test, the following statistics are tested:
 *       i.   rx dropped
 *            Increase the UMEM frame headroom to a value which results in
 *            insufficient space in the rx buffer for both the packet and the headroom.
 *       ii.  tx invalid
 *            Set the 'len' field of tx descriptors to an invalid value (umem frame
 *            size + 1).
 *       iii. rx ring full
 *            Reduce the size of the RX ring to a fraction of the fill ring size.
 *       iv.  fill queue empty
 *            Do not populate the fill queue and then try to receive pkts.
 *    f. bpf_link resource persistence
 *       Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
 *       then remove xsk sockets from queue 0 on both veth interfaces and
 *       finally run a traffic on queues ids 1
 *    g. unaligned mode
 *    h. tests for invalid and corner case Tx descriptors so that the correct ones
 *       are discarded and let through, respectively.
 *    i. 2K frame size tests
 *    j. If multi-buffer is supported, send 9k packets divided into 3 frames
 *    k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
 *       using unaligned mode
 *    l. If multi-buffer is supported, try various nasty combinations of descriptors to
 *       check if they pass the validation or not
 *
 * Flow:
 * -----
 * - Single process spawns two threads: Tx and Rx
 * - Each of these two threads attach to a veth interface
 * - Each thread creates one AF_XDP socket connected to a unique umem for each
 *   veth interface
 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
 * - Rx thread verifies if all packets were received and delivered in-order,
 *   and have the right content
 *
 * Enable/disable packet dump mode:
 * --------------------------
 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
 */

#define _GNU_SOURCE
#include <assert.h>
#include <fcntl.h>
#include <getopt.h>
#include <linux/if_link.h>
#include <linux/if_ether.h>
#include <linux/mman.h>
#include <linux/netdev.h>
#include <linux/ethtool.h>
#include <linux/align.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <locale.h>
#include <stdio.h>
#include <stdlib.h>
#include <libgen.h>
#include <stddef.h>
#include <sys/mman.h>
#include <sys/types.h>

#include "prog_tests/test_xsk.h"
#include "xsk_xdp_progs.skel.h"
#include "xsk.h"
#include "xskxceiver.h"
#include <bpf/bpf.h>
#include <linux/filter.h>
#include "kselftest.h"
#include "xsk_xdp_common.h"

#include <network_helpers.h>

static bool opt_print_tests;
static enum test_mode opt_mode = TEST_MODE_ALL;
static u32 opt_run_test = RUN_ALL_TESTS;

void test__fail(void) { /* for network_helpers.c */ }

static void __exit_with_error(int error, const char *file, const char *func, int line)
{
	ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line,
			      error, strerror(error));
	ksft_exit_xfail();
}

#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)

static bool ifobj_zc_avail(struct ifobject *ifobject)
{
	size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
	int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
	struct xsk_socket_info *xsk;
	struct xsk_umem_info *umem;
	bool zc_avail = false;
	void *bufs;
	int ret;

	bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
	if (bufs == MAP_FAILED)
		exit_with_error(errno);

	umem = calloc(1, sizeof(struct xsk_umem_info));
	if (!umem) {
		munmap(bufs, umem_sz);
		exit_with_error(ENOMEM);
	}
	umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
	ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
	if (ret)
		exit_with_error(-ret);

	xsk = calloc(1, sizeof(struct xsk_socket_info));
	if (!xsk)
		goto out;
	ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
	ifobject->rx_on = true;
	xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
	ret = xsk_configure_socket(xsk, umem, ifobject, false);
	if (!ret)
		zc_avail = true;

	xsk_socket__delete(xsk->xsk);
	free(xsk);
out:
	munmap(umem->buffer, umem_sz);
	xsk_umem__delete(umem->umem);
	free(umem);
	return zc_avail;
}

static struct option long_options[] = {
	{"interface", required_argument, 0, 'i'},
	{"busy-poll", no_argument, 0, 'b'},
	{"verbose", no_argument, 0, 'v'},
	{"mode", required_argument, 0, 'm'},
	{"list", no_argument, 0, 'l'},
	{"test", required_argument, 0, 't'},
	{"help", no_argument, 0, 'h'},
	{0, 0, 0, 0}
};

static void print_usage(char **argv)
{
	const char *str =
		"  Usage: xskxceiver [OPTIONS]\n"
		"  Options:\n"
		"  -i, --interface      Use interface\n"
		"  -v, --verbose        Verbose output\n"
		"  -b, --busy-poll      Enable busy poll\n"
		"  -m, --mode           Run only mode skb, drv, or zc\n"
		"  -l, --list           List all available tests\n"
		"  -t, --test           Run a specific test. Enter number from -l option.\n"
		"  -h, --help           Display this help and exit\n";

	ksft_print_msg(str, basename(argv[0]));
	ksft_exit_xfail();
}

static bool validate_interface(struct ifobject *ifobj)
{
	if (!strcmp(ifobj->ifname, ""))
		return false;
	return true;
}

static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
			       char **argv)
{
	struct ifobject *ifobj;
	u32 interface_nb = 0;
	int option_index, c;

	opterr = 0;

	for (;;) {
		c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
		if (c == -1)
			break;

		switch (c) {
		case 'i':
			if (interface_nb == 0)
				ifobj = ifobj_tx;
			else if (interface_nb == 1)
				ifobj = ifobj_rx;
			else
				break;

			memcpy(ifobj->ifname, optarg,
			       min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));

			ifobj->ifindex = if_nametoindex(ifobj->ifname);
			if (!ifobj->ifindex)
				exit_with_error(errno);

			interface_nb++;
			break;
		case 'v':
			opt_verbose = true;
			break;
		case 'b':
			ifobj_tx->busy_poll = true;
			ifobj_rx->busy_poll = true;
			break;
		case 'm':
			if (!strncmp("skb", optarg, strlen(optarg)))
				opt_mode = TEST_MODE_SKB;
			else if (!strncmp("drv", optarg, strlen(optarg)))
				opt_mode = TEST_MODE_DRV;
			else if (!strncmp("zc", optarg, strlen(optarg)))
				opt_mode = TEST_MODE_ZC;
			else
				print_usage(argv);
			break;
		case 'l':
			opt_print_tests = true;
			break;
		case 't':
			errno = 0;
			opt_run_test = strtol(optarg, NULL, 0);
			if (errno)
				print_usage(argv);
			break;
		case 'h':
		default:
			print_usage(argv);
		}
	}
}

static void xsk_unload_xdp_programs(struct ifobject *ifobj)
{
	xsk_xdp_progs__destroy(ifobj->xdp_progs);
}

static void run_pkt_test(struct test_spec *test)
{
	int ret;

	ret = test->test_func(test);

	switch (ret) {
	case TEST_PASS:
		ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
				      test->name);
		break;
	case TEST_SKIP:
		ksft_test_result_skip("SKIP: %s %s%s\n", mode_string(test), busy_poll_string(test),
				      test->name);
		break;
	case TEST_FAILURE:
		ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
				      test->name);
		break;
	default:
		ksft_test_result_fail("FAIL: %s %s%s -- Unexpected returned value (%d)\n",
				      mode_string(test), busy_poll_string(test), test->name, ret);
	}

	pkt_stream_restore_default(test);
}

static bool is_xdp_supported(int ifindex)
{
	int flags = XDP_FLAGS_DRV_MODE;

	LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
	struct bpf_insn insns[2] = {
		BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
		BPF_EXIT_INSN()
	};
	int prog_fd, insn_cnt = ARRAY_SIZE(insns);
	int err;

	prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
	if (prog_fd < 0)
		return false;

	err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
	if (err) {
		close(prog_fd);
		return false;
	}

	bpf_xdp_detach(ifindex, flags, NULL);
	close(prog_fd);

	return true;
}

static void print_tests(void)
{
	u32 i;

	printf("Tests:\n");
	for (i = 0; i < ARRAY_SIZE(tests); i++)
		printf("%u: %s\n", i, tests[i].name);
	for (i = ARRAY_SIZE(tests); i < ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests); i++)
		printf("%u: %s\n", i, ci_skip_tests[i - ARRAY_SIZE(tests)].name);
}

int main(int argc, char **argv)
{
	const size_t total_tests = ARRAY_SIZE(tests) + ARRAY_SIZE(ci_skip_tests);
	u32 cache_line_size, max_frags, umem_tailroom;
	struct pkt_stream *rx_pkt_stream_default;
	struct pkt_stream *tx_pkt_stream_default;
	struct ifobject *ifobj_tx, *ifobj_rx;
	u32 i, j, failed_tests = 0, nb_tests;
	int modes = TEST_MODE_SKB + 1;
	struct test_spec test;
	bool shared_netdev;
	int ret;

	/* Use libbpf 1.0 API mode */
	libbpf_set_strict_mode(LIBBPF_STRICT_ALL);

	ifobj_tx = ifobject_create();
	if (!ifobj_tx)
		exit_with_error(ENOMEM);
	ifobj_rx = ifobject_create();
	if (!ifobj_rx)
		exit_with_error(ENOMEM);

	setlocale(LC_ALL, "");

	cache_line_size = read_procfs_val(SMP_CACHE_BYTES_PATH);
	if (!cache_line_size) {
		ksft_print_msg("Can't get SMP_CACHE_BYTES from system, using default (64)\n");
		cache_line_size = 64;
	}

	max_frags = read_procfs_val(MAX_SKB_FRAGS_PATH);
	if (!max_frags) {
		ksft_print_msg("Can't get MAX_SKB_FRAGS from system, using default (17)\n");
		max_frags = 17;
	}
	ifobj_tx->max_skb_frags = max_frags;
	ifobj_rx->max_skb_frags = max_frags;

	/* 48 bytes is a part of skb_shared_info w/o frags array;
	 * 16 bytes is sizeof(skb_frag_t)
	 */
	umem_tailroom = ALIGN(48 + (max_frags * 16), cache_line_size);
	ifobj_tx->umem_tailroom = umem_tailroom;
	ifobj_rx->umem_tailroom = umem_tailroom;

	parse_command_line(ifobj_tx, ifobj_rx, argc, argv);

	if (opt_print_tests) {
		print_tests();
		ksft_exit_xpass();
	}
	if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= total_tests) {
		ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
		ksft_exit_xfail();
	}

	shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
	ifobj_tx->shared_umem = shared_netdev;
	ifobj_rx->shared_umem = shared_netdev;

	if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
		print_usage(argv);

	if (is_xdp_supported(ifobj_tx->ifindex)) {
		modes++;
		if (ifobj_zc_avail(ifobj_tx))
			modes++;
	}

	ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
	if (!ret) {
		ifobj_tx->hw_ring_size_supp = true;
		ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;
		ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;
	}

	if (init_iface(ifobj_rx, worker_testapp_validate_rx) ||
	    init_iface(ifobj_tx, worker_testapp_validate_tx)) {
		ksft_print_msg("Error : can't initialize interfaces\n");
		ksft_exit_xfail();
	}

	test_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
	tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
	rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
	if (!tx_pkt_stream_default || !rx_pkt_stream_default)
		exit_with_error(ENOMEM);
	test.tx_pkt_stream_default = tx_pkt_stream_default;
	test.rx_pkt_stream_default = rx_pkt_stream_default;

	if (opt_run_test == RUN_ALL_TESTS)
		nb_tests = total_tests;
	else
		nb_tests = 1;
	if (opt_mode == TEST_MODE_ALL) {
		ksft_set_plan(modes * nb_tests);
	} else {
		if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
			ksft_print_msg("Error: XDP_DRV mode not supported.\n");
			ksft_exit_xfail();
		}
		if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
			ksft_print_msg("Error: zero-copy mode not supported.\n");
			ksft_exit_xfail();
		}

		ksft_set_plan(nb_tests);
	}

	for (i = 0; i < modes; i++) {
		if (opt_mode != TEST_MODE_ALL && i != opt_mode)
			continue;

		for (j = 0; j < total_tests; j++) {
			if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
				continue;

			if (j < ARRAY_SIZE(tests))
				test_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
			else
				test_init(&test, ifobj_tx, ifobj_rx, i,
					  &ci_skip_tests[j - ARRAY_SIZE(tests)]);
			run_pkt_test(&test);
			usleep(USLEEP_MAX);

			if (test.fail)
				failed_tests++;
		}
	}

	if (ifobj_tx->hw_ring_size_supp)
		hw_ring_size_reset(ifobj_tx);

	pkt_stream_delete(tx_pkt_stream_default);
	pkt_stream_delete(rx_pkt_stream_default);
	xsk_unload_xdp_programs(ifobj_tx);
	xsk_unload_xdp_programs(ifobj_rx);
	ifobject_delete(ifobj_tx);
	ifobject_delete(ifobj_rx);

	if (failed_tests)
		ksft_exit_fail();
	else
		ksft_exit_pass();
}