tools/sched_ext/scx_qmap.h

/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Shared definitions between scx_qmap.bpf.c and scx_qmap.c.
 *
 * The scheduler keeps all state in a single BPF arena map. struct
 * qmap_arena is the one object that lives at the base of the arena and is
 * mmap'd into userspace so the loader can read counters directly.
 *
 * Copyright (c) 2026 Meta Platforms, Inc. and affiliates.
 * Copyright (c) 2026 Tejun Heo <tj@kernel.org>
 */
#ifndef __SCX_QMAP_H
#define __SCX_QMAP_H

#ifdef __BPF__
#include <scx/bpf_arena_common.bpf.h>
#else
#include <linux/types.h>
#include <scx/bpf_arena_common.h>
#endif

#define MAX_SUB_SCHEDS		8
#define MAX_PARTS		(MAX_SUB_SCHEDS + 1)	/* participants: children + self */

/*
 * cpu_ctxs[] is sized to a fixed cap so the layout is shared between BPF and
 * userspace. Keep this in sync with NR_CPUS used by the BPF side.
 */
#define SCX_QMAP_MAX_CPUS	1024

/*
 * An owner id identifies who holds a cid: a child slot in [0, MAX_SUB_SCHEDS),
 * CID_SELF for this node, CID_NONE for a cid not currently held, or CID_SHARED
 * for a cid in the round-robin pool (its live holder is rr_slots[rr_pos]). Used
 * by the partition's cid_owner[].
 */
#define CID_SELF	(-1)
#define CID_NONE	(-2)
#define CID_SHARED	(-3)

/* -C cid-override test modes. Selects cid_override_mode in scx_qmap.bpf.c. */
enum qmap_cid_override {
	QMAP_CID_OVR_OFF	= 0,	/* disabled */
	QMAP_CID_OVR_SHUFFLE	= 1,	/* valid reversed cpu->cid mapping */
	QMAP_CID_OVR_BAD_DUP	= 2,	/* invalid: duplicate cid assignment */
	QMAP_CID_OVR_BAD_RANGE	= 3,	/* invalid: out-of-range cid */
	QMAP_CID_OVR_BAD_MONO	= 4,	/* invalid: non-monotonic shard_start */
};

struct cpu_ctx {
	u64 dsp_idx;		/* dispatch index */
	u64 dsp_cnt;		/* remaining count */
	u32 avg_weight;
	u32 cpuperf_target;
};

struct qmap_fifo {
	struct task_ctx __arena *head;
	struct task_ctx __arena *tail;
	s32 idx;
};

/* -J fault-injection modes. Selects inject_mode in struct qmap_arena. */
enum qmap_inject {
	QMAP_INJ_OFF		= 0,
	QMAP_INJ_WRONG_CID	= 1,	/* dispatch to a cid we don't hold */
	QMAP_INJ_INIT_FAIL	= 2,	/* fail init_task for "qmfail*" comms */
	QMAP_INJ_CGRP_INIT_FAIL	= 3,	/* fail cpuctl_init for "qmfail*" cgroups */
};

/*
 * scx_cmask's are embedded in struct qmap_arena with inline backing storage.
 * The bpf side uses &field.mask with the normal cmask_* helpers. Userspace
 * doesn't have access to the type definition and sees same-sized opaque words.
 * _Static_assert()'s in .bpf.c ensure that they are in sync.
 */
#define QMAP_CMASK_WORDS	(((SCX_QMAP_MAX_CPUS) + 63) / 64 + 1)
struct qmap_cmask {
#ifdef __BPF__
	union {
		struct scx_cmask mask;
		u64 words[QMAP_CMASK_WORDS + 2];
	};
#else
	u64 words[QMAP_CMASK_WORDS + 2];
#endif
};

/* Opaque to userspace; defined in scx_qmap.bpf.c. */
struct task_ctx;

/* per-direct-child state for the sub-scheduler */
struct sub_sched_ctx {
	u64 cgroup_id;
	u32 weight;			/* cpu.weight, seeded at attach, then set_weight */
	u64 nr_dsps;
	struct qmap_cmask granted_cids;	/* cids granted excl to this child */
	struct qmap_cmask prev_granted;	/* last grant, for delta calculation */
};

/*
 * compute_partition() builds the following from this node's held caps, and
 * apply_partition()/rr_advance() execute it. Userspace only reads for the
 * hierarchy display.
 */
struct qmap_partition {
	u32 nr_excl;			/* number of excl-held (delegatable) cids */
	s32 cid_owner[SCX_QMAP_MAX_CPUS]; /* per cid: owner id, or CID_NONE */
	s32 shared_cids[MAX_PARTS];	/* the round-robin cid pool */
	u32 nr_shared;			/* number of shared_cids entries */
	u64 rr_slots[MAX_PARTS];	/* rotation order: holder cgroup_id, 0 = self */
	u32 nr_rr;			/* number of rr_slots entries */
	u32 rr_pos;			/* current rotation index */
};

struct qmap_arena {
	/* userspace-visible stats */
	u64 nr_enqueued, nr_dispatched, nr_reenqueued, nr_reenqueued_cid0;
	u64 nr_dequeued, nr_ddsp_from_enq;
	u64 nr_core_sched_execed;
	u64 nr_expedited_local, nr_expedited_remote;
	u64 nr_expedited_lost, nr_expedited_from_timer;
	u64 nr_highpri_queued;
	u32 test_error_cnt;
	u32 cpuperf_min, cpuperf_avg, cpuperf_max;
	u32 cpuperf_target_min, cpuperf_target_avg, cpuperf_target_max;

	/* kernel-side runtime state */
	u64 core_sched_head_seqs[5];
	u64 core_sched_tail_seqs[5];

	struct cpu_ctx cpu_ctxs[SCX_QMAP_MAX_CPUS];

	/* cid-override test input, populated by the loader before attach */
	__s32 cid_override_cpu_to_cid[SCX_QMAP_MAX_CPUS];
	__s32 cid_override_shard_start[SCX_QMAP_MAX_CPUS];

	/* task_ctx slab; allocated and threaded by qmap_init() */
	struct task_ctx __arena *task_ctxs;
	struct task_ctx __arena *task_free_head;

	/* five priority FIFOs, each a doubly-linked list through task_ctx */
	struct qmap_fifo fifos[5];

	/*
	 * Hierarchical sub-scheduling state. See the design comment at the top
	 * of scx_qmap.bpf.c.
	 */
	u32 nr_cids;			/* cid count, cached at init */

	/* bpf-owned partition: read by userspace for display */
	struct qmap_partition part;

	struct sub_sched_ctx sub_sched_ctxs[MAX_SUB_SCHEDS]; /* per-child context */
	u64 nr_sub_scheds;		/* number of attached children */

	/* bpf-internal per-cid state */
	u8 cid_shared[SCX_QMAP_MAX_CPUS]; /* per cid: 1 if held shared (ENQ_IMMED-only) */

	/* allocated cid-time, charged per owner by account_alloc() */
	u64 alloc_ns[MAX_SUB_SCHEDS];	/* per child slot */
	u64 self_alloc_ns;
	u64 alloc_ts;			/* last accounting timestamp */
	u64 alloc_window_ns;		/* total accounted time, the alloc denominator */

	/* bpf-internal cmasks (embedded, see struct qmap_cmask) */
	struct qmap_cmask self_cids;	/* cids this node runs its own tasks on */
	struct qmap_cmask avail_cids;	/* cids with caps in effect on the cpu */
	struct qmap_cmask usable_cids;	/* self_cids & avail_cids, placeable right now */
	struct qmap_cmask idle_cids;	/* idle state of all cids regardless of delegation */
	struct qmap_cmask rr_cids;	/* the shared pool, as a mask for grant/revoke */

	/* scratch cmasks */
	struct qmap_cmask to_revoke_cids; /* delta cids to revoke */
	struct qmap_cmask to_grant_cids; /* delta cids to grant */
	struct qmap_cmask usable_scratch; /* refresh_usable() build area */
	struct qmap_cmask prev_rr_cids; /* previous shared pool, to clear stale grants */
	struct qmap_cmask held_excl;	/* cids held excl (ENQ): delegatable */
	struct qmap_cmask held_shared;	/* cids held shared (ENQ_IMMED only): self-local */

	/* bpf -> userspace: stats */
	u64 nr_reenq_cap;		/* SCX_TASK_REENQ_CAP bounces */
	u64 nr_reenq_immed;		/* SCX_TASK_REENQ_IMMED bounces */
	u64 nr_inject_attempts;		/* fault-injection: dispatches to an unheld cid */
	u64 nr_rescue_dsp;		/* SCX_ENQ_RESCUE dispatch attempts */
	u32 inject_mode;		/* fault-injection mode (QMAP_INJ_*) */
};

#endif /* __SCX_QMAP_H */