include/drm/ttm/ttm_resource.h

/*
 * Copyright 2020 Advanced Micro Devices, Inc.
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 * OTHER DEALINGS IN THE SOFTWARE.
 *
 * Authors: Christian König
 */

#ifndef _TTM_RESOURCE_H_
#define _TTM_RESOURCE_H_

#include <linux/types.h>
#include <linux/list.h>
#include <linux/mutex.h>
#include <linux/iosys-map.h>
#include <linux/dma-fence.h>

#include <drm/ttm/ttm_caching.h>
#include <drm/ttm/ttm_kmap_iter.h>

#define TTM_MAX_BO_PRIORITY	4U
#define TTM_NUM_MEM_TYPES 9

struct dentry;
struct dmem_cgroup_device;
struct dmem_cgroup_region;
struct drm_printer;
struct ttm_device;
struct ttm_resource_manager;
struct ttm_resource;
struct ttm_place;
struct ttm_buffer_object;
struct ttm_placement;
struct iosys_map;
struct io_mapping;
struct sg_table;
struct scatterlist;

/**
 * define TTM_NUM_MOVE_FENCES - How many entities can be used for evictions
 *
 * Pipelined evictions can be spread on multiple entities. This
 * is the max number of entities that can be used by the driver
 * for that purpose.
 */
#define TTM_NUM_MOVE_FENCES 8

/**
 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses
 */
enum ttm_lru_item_type {
	/** @TTM_LRU_RESOURCE: The resource subclass */
	TTM_LRU_RESOURCE,
	/** @TTM_LRU_HITCH: The iterator hitch subclass */
	TTM_LRU_HITCH
};

/**
 * struct ttm_lru_item - The TTM lru list node base class
 * @link: The list link
 * @type: The subclass type
 */
struct ttm_lru_item {
	struct list_head link;
	enum ttm_lru_item_type type;
};

/**
 * ttm_lru_item_init() - initialize a struct ttm_lru_item
 * @item: The item to initialize
 * @type: The subclass type
 */
static inline void ttm_lru_item_init(struct ttm_lru_item *item,
				     enum ttm_lru_item_type type)
{
	item->type = type;
	INIT_LIST_HEAD(&item->link);
}

static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item)
{
	return item->type == TTM_LRU_RESOURCE;
}

struct ttm_resource_manager_func {
	/**
	 * struct ttm_resource_manager_func member alloc
	 *
	 * @man: Pointer to a memory type manager.
	 * @bo: Pointer to the buffer object we're allocating space for.
	 * @place: Placement details.
	 * @res: Resulting pointer to the ttm_resource.
	 *
	 * This function should allocate space in the memory type managed
	 * by @man. Placement details if applicable are given by @place. If
	 * successful, a filled in ttm_resource object should be returned in
	 * @res. @res::start should be set to a value identifying the beginning
	 * of the range allocated, and the function should return zero.
	 * If the manager can't fulfill the request -ENOSPC should be returned.
	 * If a system error occurred, preventing the request to be fulfilled,
	 * the function should return a negative error code.
	 *
	 * This function may not be called from within atomic context and needs
	 * to take care of its own locking to protect any data structures
	 * managing the space.
	 */
	int  (*alloc)(struct ttm_resource_manager *man,
		      struct ttm_buffer_object *bo,
		      const struct ttm_place *place,
		      struct ttm_resource **res);

	/**
	 * struct ttm_resource_manager_func member free
	 *
	 * @man: Pointer to a memory type manager.
	 * @res: Pointer to a struct ttm_resource to be freed.
	 *
	 * This function frees memory type resources previously allocated.
	 * May not be called from within atomic context.
	 */
	void (*free)(struct ttm_resource_manager *man,
		     struct ttm_resource *res);

	/**
	 * struct ttm_resource_manager_func member intersects
	 *
	 * @man: Pointer to a memory type manager.
	 * @res: Pointer to a struct ttm_resource to be checked.
	 * @place: Placement to check against.
	 * @size: Size of the check.
	 *
	 * Test if @res intersects with @place + @size. Used to judge if
	 * evictions are valueable or not.
	 */
	bool (*intersects)(struct ttm_resource_manager *man,
			   struct ttm_resource *res,
			   const struct ttm_place *place,
			   size_t size);

	/**
	 * struct ttm_resource_manager_func member compatible
	 *
	 * @man: Pointer to a memory type manager.
	 * @res: Pointer to a struct ttm_resource to be checked.
	 * @place: Placement to check against.
	 * @size: Size of the check.
	 *
	 * Test if @res compatible with @place + @size. Used to check of
	 * the need to move the backing store or not.
	 */
	bool (*compatible)(struct ttm_resource_manager *man,
			   struct ttm_resource *res,
			   const struct ttm_place *place,
			   size_t size);

	/**
	 * struct ttm_resource_manager_func member debug
	 *
	 * @man: Pointer to a memory type manager.
	 * @printer: Prefix to be used in printout to identify the caller.
	 *
	 * This function is called to print out the state of the memory
	 * type manager to aid debugging of out-of-memory conditions.
	 * It may not be called from within atomic context.
	 */
	void (*debug)(struct ttm_resource_manager *man,
		      struct drm_printer *printer);
};

/**
 * struct ttm_resource_manager
 *
 * @use_type: The memory type is enabled.
 * @use_tt: If a TT object should be used for the backing store.
 * @size: Size of the managed region.
 * @bdev: ttm device this manager belongs to
 * @func: structure pointer implementing the range manager. See above
 * @eviction_lock: lock for eviction fences
 * @eviction_fences: The fences of the last pipelined move operation.
 * @lru: The lru list for this memory type.
 *
 * This structure is used to identify and manage memory types for a device.
 */
struct ttm_resource_manager {
	/*
	 * No protection. Constant from start.
	 */
	bool use_type;
	bool use_tt;
	struct ttm_device *bdev;
	uint64_t size;
	const struct ttm_resource_manager_func *func;

	/* This is very similar to a dma_resv object, but locking rules make
	 * it difficult to use one in this context.
	 */
	spinlock_t eviction_lock;
	struct dma_fence *eviction_fences[TTM_NUM_MOVE_FENCES];

	/*
	 * Protected by the bdev->lru_lock.
	 */
	struct list_head lru[TTM_MAX_BO_PRIORITY];

	/**
	 * @usage: How much of the resources are used, protected by the
	 * bdev->lru_lock.
	 */
	uint64_t usage;

	/**
	 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL.
	 */
	struct dmem_cgroup_region *cg;
};

/**
 * struct ttm_bus_placement
 *
 * @addr:		mapped virtual address
 * @offset:		physical addr
 * @is_iomem:		is this io memory ?
 * @caching:		See enum ttm_caching
 *
 * Structure indicating the bus placement of an object.
 */
struct ttm_bus_placement {
	void			*addr;
	phys_addr_t		offset;
	bool			is_iomem;
	enum ttm_caching	caching;
};

/**
 * struct ttm_resource
 *
 * @start: Start of the allocation.
 * @size: Actual size of resource in bytes.
 * @mem_type: Resource type of the allocation.
 * @placement: Placement flags.
 * @bus: Placement on io bus accessible to the CPU
 * @bo: weak reference to the BO, protected by ttm_device::lru_lock
 * @css: cgroup state this resource is charged to
 *
 * Structure indicating the placement and space resources used by a
 * buffer object.
 */
struct ttm_resource {
	unsigned long start;
	size_t size;
	uint32_t mem_type;
	uint32_t placement;
	struct ttm_bus_placement bus;
	struct ttm_buffer_object *bo;

	struct dmem_cgroup_pool_state *css;

	/**
	 * @lru: Least recently used list, see &ttm_resource_manager.lru
	 */
	struct ttm_lru_item lru;
};

/**
 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource
 * @item: The struct ttm_lru_item to downcast
 *
 * Return: Pointer to the embedding struct ttm_resource
 */
static inline struct ttm_resource *
ttm_lru_item_to_res(struct ttm_lru_item *item)
{
	return container_of(item, struct ttm_resource, lru);
}

/**
 * struct ttm_lru_bulk_move_pos
 *
 * @first: first res in the bulk move range
 * @last: last res in the bulk move range
 *
 * Range of resources for a lru bulk move.
 */
struct ttm_lru_bulk_move_pos {
	struct ttm_resource *first;
	struct ttm_resource *last;
};

/**
 * struct ttm_lru_bulk_move
 * @pos: first/last lru entry for resources in the each domain/priority
 * @cursor_list: The list of cursors currently traversing any of
 * the sublists of @pos. Protected by the ttm device's lru_lock.
 *
 * Container for the current bulk move state. Should be used with
 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move().
 * All BOs in a bulk_move structure need to share the same reservation object to
 * ensure that the bulk as a whole is locked for eviction even if only one BO of
 * the bulk is evicted.
 */
struct ttm_lru_bulk_move {
	struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY];
	struct list_head cursor_list;
};

/**
 * struct ttm_resource_cursor
 * @man: The resource manager currently being iterated over
 * @hitch: A hitch list node inserted before the next resource
 * to iterate over.
 * @bulk_link: A list link for the list of cursors traversing the
 * bulk sublist of @bulk. Protected by the ttm device's lru_lock.
 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is
 * inserted to. NULL if none. Never dereference this pointer since
 * the struct ttm_lru_bulk_move object pointed to might have been
 * freed. The pointer is only for comparison.
 * @mem_type: The memory type of the LRU list being traversed.
 * This field is valid iff @bulk != NULL.
 * @priority: the current priority
 *
 * Cursor to iterate over the resources in a manager.
 */
struct ttm_resource_cursor {
	struct ttm_resource_manager *man;
	struct ttm_lru_item hitch;
	struct list_head bulk_link;
	struct ttm_lru_bulk_move *bulk;
	unsigned int mem_type;
	unsigned int priority;
};

void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor,
			      struct ttm_resource_manager *man);

void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor);

/**
 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping +
 * struct sg_table backed struct ttm_resource.
 * @base: Embedded struct ttm_kmap_iter providing the usage interface.
 * @iomap: struct io_mapping representing the underlying linear io_memory.
 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource.
 * @start: Offset that needs to be subtracted from @st to make
 * sg_dma_address(st->sgl) - @start == 0 for @iomap start.
 * @cache: Scatterlist traversal cache for fast lookups.
 * @cache.sg: Pointer to the currently cached scatterlist segment.
 * @cache.i: First index of @sg. PAGE_SIZE granularity.
 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity.
 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity.
 */
struct ttm_kmap_iter_iomap {
	struct ttm_kmap_iter base;
	struct io_mapping *iomap;
	struct sg_table *st;
	resource_size_t start;
	struct {
		struct scatterlist *sg;
		pgoff_t i;
		pgoff_t end;
		pgoff_t offs;
	} cache;
};

/**
 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io
 * @base: The base iterator
 * @dmap: Points to the starting address of the region
 * @needs_unmap: Whether we need to unmap on fini
 */
struct ttm_kmap_iter_linear_io {
	struct ttm_kmap_iter base;
	struct iosys_map dmap;
	bool needs_unmap;
};

/**
 * ttm_resource_manager_set_used
 *
 * @man: A memory manager object.
 * @used: usage state to set.
 *
 * Set the manager in use flag. If disabled the manager is no longer
 * used for object placement.
 */
static inline void
ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used)
{
	int i;

	for (i = 0; i < TTM_MAX_BO_PRIORITY; i++)
		WARN_ON(!list_empty(&man->lru[i]));
	man->use_type = used;
}

/**
 * ttm_resource_manager_used
 *
 * @man: Manager to get used state for
 *
 * Get the in use flag for a manager.
 * Returns:
 * true is used, false if not.
 */
static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man)
{
	return man->use_type;
}

/**
 * ttm_resource_manager_cleanup
 *
 * @man: A memory manager object.
 *
 * Cleanup the move fences from the memory manager object.
 */
static inline void
ttm_resource_manager_cleanup(struct ttm_resource_manager *man)
{
	int i;

	for (i = 0; i < TTM_NUM_MOVE_FENCES; i++) {
		dma_fence_put(man->eviction_fences[i]);
		man->eviction_fences[i] = NULL;
	}
}

void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk);
void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk);
void ttm_lru_bulk_move_fini(struct ttm_device *bdev,
			    struct ttm_lru_bulk_move *bulk);

void ttm_resource_add_bulk_move(struct ttm_resource *res,
				struct ttm_buffer_object *bo);
void ttm_resource_del_bulk_move(struct ttm_resource *res,
				struct ttm_buffer_object *bo);
void ttm_resource_del_bulk_move_unevictable(struct ttm_resource *res,
					    struct ttm_buffer_object *bo);
void ttm_resource_move_to_lru_tail(struct ttm_resource *res);

void ttm_resource_init(struct ttm_buffer_object *bo,
                       const struct ttm_place *place,
                       struct ttm_resource *res);
void ttm_resource_fini(struct ttm_resource_manager *man,
		       struct ttm_resource *res);

int ttm_resource_try_charge(struct ttm_buffer_object *bo,
			    const struct ttm_place *place,
			    struct dmem_cgroup_pool_state **ret_pool,
			    struct dmem_cgroup_pool_state **ret_limit_pool);
int ttm_resource_alloc(struct ttm_buffer_object *bo,
		       const struct ttm_place *place,
		       struct ttm_resource **res,
		       struct dmem_cgroup_pool_state *charge_pool);
void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res);
bool ttm_resource_intersects(struct ttm_device *bdev,
			     struct ttm_resource *res,
			     const struct ttm_place *place,
			     size_t size);
bool ttm_resource_compatible(struct ttm_resource *res,
			     struct ttm_placement *placement,
			     bool evicting);
void ttm_resource_set_bo(struct ttm_resource *res,
			 struct ttm_buffer_object *bo);

void ttm_resource_manager_init(struct ttm_resource_manager *man,
			       struct ttm_device *bdev,
			       uint64_t size);

void ttm_resource_manager_set_dmem_region(struct ttm_resource_manager *man,
					  struct dmem_cgroup_region *region);

int ttm_resource_manager_dmem_reclaim(struct dmem_cgroup_pool_state *pool,
				      u64 target_bytes, void *priv);

int ttm_resource_manager_evict_all(struct ttm_device *bdev,
				   struct ttm_resource_manager *man);

uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man);
void ttm_resource_manager_debug(struct ttm_resource_manager *man,
				struct drm_printer *p);

struct ttm_resource *
ttm_resource_manager_first(struct ttm_resource_cursor *cursor);
struct ttm_resource *
ttm_resource_manager_next(struct ttm_resource_cursor *cursor);

struct ttm_resource *
ttm_lru_first_res_or_null(struct list_head *head);

/**
 * ttm_resource_manager_for_each_res - iterate over all resources
 * @cursor: struct ttm_resource_cursor for the current position
 * @res: the current resource
 *
 * Iterate over all the evictable resources in a resource manager.
 */
#define ttm_resource_manager_for_each_res(cursor, res)	\
	for (res = ttm_resource_manager_first(cursor); res;	\
	     res = ttm_resource_manager_next(cursor))

struct ttm_kmap_iter *
ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io,
			 struct io_mapping *iomap,
			 struct sg_table *st,
			 resource_size_t start);

struct ttm_kmap_iter_linear_io;

struct ttm_kmap_iter *
ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io,
			     struct ttm_device *bdev,
			     struct ttm_resource *mem);

void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io,
				  struct ttm_device *bdev,
				  struct ttm_resource *mem);

void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man,
					 struct dentry * parent,
					 const char *name);
#endif