drivers/md/dm-pcache/cache_segment.c

// SPDX-License-Identifier: GPL-2.0-or-later

#include "cache_dev.h"
#include "cache.h"
#include "backing_dev.h"
#include "dm_pcache.h"

static inline struct pcache_segment_info *get_seg_info_addr(struct pcache_cache_segment *cache_seg)
{
	struct pcache_segment_info *seg_info_addr;
	u32 seg_id = cache_seg->segment.seg_id;
	void *seg_addr;

	seg_addr = CACHE_DEV_SEGMENT(cache_seg->cache->cache_dev, seg_id);
	seg_info_addr = seg_addr + PCACHE_SEG_INFO_SIZE * cache_seg->info_index;

	return seg_info_addr;
}

static void cache_seg_info_write(struct pcache_cache_segment *cache_seg)
{
	struct pcache_segment_info *seg_info_addr;
	struct pcache_segment_info *seg_info = &cache_seg->cache_seg_info;

	mutex_lock(&cache_seg->info_lock);
	seg_info->header.seq++;
	seg_info->header.crc = pcache_meta_crc(&seg_info->header, sizeof(struct pcache_segment_info));

	cache_seg->info_index = (cache_seg->info_index + 1) % PCACHE_META_INDEX_MAX;

	seg_info_addr = get_seg_info_addr(cache_seg);
	memcpy_flushcache(seg_info_addr, seg_info, sizeof(struct pcache_segment_info));
	pmem_wmb();
	mutex_unlock(&cache_seg->info_lock);
}

static int cache_seg_info_load(struct pcache_cache_segment *cache_seg)
{
	struct pcache_segment_info *cache_seg_info_addr_base, *cache_seg_info_addr;
	struct pcache_cache_dev *cache_dev = cache_seg->cache->cache_dev;
	struct dm_pcache *pcache = CACHE_DEV_TO_PCACHE(cache_dev);
	u32 seg_id = cache_seg->segment.seg_id;
	int ret = 0;

	cache_seg_info_addr_base = CACHE_DEV_SEGMENT(cache_dev, seg_id);

	mutex_lock(&cache_seg->info_lock);
	cache_seg_info_addr = pcache_meta_find_latest(&cache_seg_info_addr_base->header,
						sizeof(struct pcache_segment_info),
						PCACHE_SEG_INFO_SIZE,
						&cache_seg->cache_seg_info);
	if (IS_ERR(cache_seg_info_addr)) {
		ret = PTR_ERR(cache_seg_info_addr);
		goto out;
	} else if (!cache_seg_info_addr) {
		ret = -EIO;
		goto out;
	}

	cache_seg->info_index =
		((char *)cache_seg_info_addr - (char *)cache_seg_info_addr_base) /
		PCACHE_SEG_INFO_SIZE;
out:
	mutex_unlock(&cache_seg->info_lock);

	if (ret)
		pcache_dev_err(pcache, "can't read segment info of segment: %u, ret: %d\n",
			      cache_seg->segment.seg_id, ret);
	return ret;
}

static int cache_seg_ctrl_load(struct pcache_cache_segment *cache_seg)
{
	struct pcache_cache_seg_ctrl *cache_seg_ctrl = cache_seg->cache_seg_ctrl;
	struct pcache_cache_seg_gen cache_seg_gen, *cache_seg_gen_addr;
	int ret = 0;

	cache_seg_gen_addr = pcache_meta_find_latest(&cache_seg_ctrl->gen->header,
					     sizeof(struct pcache_cache_seg_gen),
					     sizeof(struct pcache_cache_seg_gen),
					     &cache_seg_gen);
	if (IS_ERR(cache_seg_gen_addr)) {
		ret = PTR_ERR(cache_seg_gen_addr);
		goto out;
	}

	if (!cache_seg_gen_addr) {
		cache_seg->gen = 0;
		cache_seg->gen_seq = 0;
		cache_seg->gen_index = 0;
		goto out;
	}

	cache_seg->gen = cache_seg_gen.gen;
	cache_seg->gen_seq = cache_seg_gen.header.seq;
	cache_seg->gen_index = (cache_seg_gen_addr - cache_seg_ctrl->gen);
out:

	return ret;
}

static inline struct pcache_cache_seg_gen *get_cache_seg_gen_addr(struct pcache_cache_segment *cache_seg)
{
	struct pcache_cache_seg_ctrl *cache_seg_ctrl = cache_seg->cache_seg_ctrl;

	return (cache_seg_ctrl->gen + cache_seg->gen_index);
}

/*
 * cache_seg_ctrl_write - write cache segment control information
 * @seg: the cache segment to update
 *
 * This function writes the control information of a cache segment to media.
 *
 * Although this updates shared control data, we intentionally do not use
 * any locking here.  All accesses to control information are single-threaded:
 *
 *   - All reads occur during the init phase, where no concurrent writes
 *     can happen.
 *   - Writes happen once during init and once when the last reference
 *     to the segment is dropped in cache_seg_put().
 *
 * Both cases are guaranteed to be single-threaded, so there is no risk
 * of concurrent read/write races.
 */
static void cache_seg_ctrl_write(struct pcache_cache_segment *cache_seg)
{
	struct pcache_cache_seg_gen cache_seg_gen;

	cache_seg_gen.gen = cache_seg->gen;
	cache_seg_gen.header.seq = ++cache_seg->gen_seq;
	cache_seg_gen.header.crc = pcache_meta_crc(&cache_seg_gen.header,
						 sizeof(struct pcache_cache_seg_gen));

	cache_seg->gen_index = (cache_seg->gen_index + 1) % PCACHE_META_INDEX_MAX;

	memcpy_flushcache(get_cache_seg_gen_addr(cache_seg), &cache_seg_gen, sizeof(struct pcache_cache_seg_gen));
	pmem_wmb();
}

static void cache_seg_ctrl_init(struct pcache_cache_segment *cache_seg)
{
	cache_seg->gen = 0;
	cache_seg->gen_seq = 0;
	cache_seg->gen_index = 0;
	cache_seg_ctrl_write(cache_seg);
}

static int cache_seg_meta_load(struct pcache_cache_segment *cache_seg)
{
	int ret;

	ret = cache_seg_info_load(cache_seg);
	if (ret)
		goto err;

	ret = cache_seg_ctrl_load(cache_seg);
	if (ret)
		goto err;

	return 0;
err:
	return ret;
}

/**
 * cache_seg_set_next_seg - Sets the ID of the next segment
 * @cache_seg: Pointer to the cache segment structure.
 * @seg_id: The segment ID to set as the next segment.
 *
 * A pcache_cache allocates multiple cache segments, which are linked together
 * through next_seg. When loading a pcache_cache, the first cache segment can
 * be found using cache->seg_id, which allows access to all the cache segments.
 */
void cache_seg_set_next_seg(struct pcache_cache_segment *cache_seg, u32 seg_id)
{
	cache_seg->cache_seg_info.flags |= PCACHE_SEG_INFO_FLAGS_HAS_NEXT;
	cache_seg->cache_seg_info.next_seg = seg_id;
	cache_seg_info_write(cache_seg);
}

int cache_seg_init(struct pcache_cache *cache, u32 seg_id, u32 cache_seg_id,
		   bool new_cache)
{
	struct pcache_cache_dev *cache_dev = cache->cache_dev;
	struct pcache_cache_segment *cache_seg = &cache->segments[cache_seg_id];
	struct pcache_segment_init_options seg_options = { 0 };
	struct pcache_segment *segment = &cache_seg->segment;
	int ret;

	cache_seg->cache = cache;
	cache_seg->cache_seg_id = cache_seg_id;
	spin_lock_init(&cache_seg->gen_lock);
	atomic_set(&cache_seg->refs, 0);
	mutex_init(&cache_seg->info_lock);

	/* init pcache_segment */
	seg_options.type = PCACHE_SEGMENT_TYPE_CACHE_DATA;
	seg_options.data_off = PCACHE_CACHE_SEG_CTRL_OFF + PCACHE_CACHE_SEG_CTRL_SIZE;
	seg_options.seg_id = seg_id;
	seg_options.seg_info = &cache_seg->cache_seg_info;
	pcache_segment_init(cache_dev, segment, &seg_options);

	cache_seg->cache_seg_ctrl = CACHE_DEV_SEGMENT(cache_dev, seg_id) + PCACHE_CACHE_SEG_CTRL_OFF;

	if (new_cache) {
		cache_dev_zero_range(cache_dev, CACHE_DEV_SEGMENT(cache_dev, seg_id),
				     PCACHE_SEG_INFO_SIZE * PCACHE_META_INDEX_MAX +
				     PCACHE_CACHE_SEG_CTRL_SIZE);

		cache_seg_ctrl_init(cache_seg);

		cache_seg->info_index = 0;
		cache_seg_info_write(cache_seg);

		/* clear outdated kset in segment */
		memcpy_flushcache(segment->data, &pcache_empty_kset, sizeof(struct pcache_cache_kset_onmedia));
		pmem_wmb();
	} else {
		ret = cache_seg_meta_load(cache_seg);
		if (ret)
			goto err;
	}

	return 0;
err:
	return ret;
}

/**
 * get_cache_segment - Retrieves a free cache segment from the cache.
 * @cache: Pointer to the cache structure.
 *
 * This function attempts to find a free cache segment that can be used.
 * It locks the segment map and checks for the next available segment ID.
 * If a free segment is found, it initializes it and returns a pointer to the
 * cache segment structure. Returns NULL if no segments are available.
 */
struct pcache_cache_segment *get_cache_segment(struct pcache_cache *cache)
{
	struct pcache_cache_segment *cache_seg;
	u32 seg_id;

	spin_lock(&cache->seg_map_lock);
again:
	/*
	 * Only allocate initialized segments. cache_segs_init() initializes
	 * cache_info.n_segs of the cache->n_segs device segments; a forged
	 * smaller cache_info.n_segs leaves the rest as zeroed structs whose data
	 * pointer is NULL. Bounding the search to cache_info.n_segs keeps such a
	 * segment from reaching cache_kset_close(), which writes through it.
	 */
	seg_id = find_next_zero_bit(cache->seg_map, cache->cache_info.n_segs,
				    cache->last_cache_seg);
	if (seg_id == cache->cache_info.n_segs) {
		/* reset the hint of ->last_cache_seg and retry */
		if (cache->last_cache_seg) {
			cache->last_cache_seg = 0;
			goto again;
		}
		cache->cache_full = true;
		spin_unlock(&cache->seg_map_lock);
		return NULL;
	}

	/*
	 * found an available cache_seg, mark it used in seg_map
	 * and update the search hint ->last_cache_seg
	 */
	__set_bit(seg_id, cache->seg_map);
	cache->last_cache_seg = seg_id;
	spin_unlock(&cache->seg_map_lock);

	cache_seg = &cache->segments[seg_id];
	cache_seg->cache_seg_id = seg_id;

	return cache_seg;
}

static void cache_seg_gen_increase(struct pcache_cache_segment *cache_seg)
{
	spin_lock(&cache_seg->gen_lock);
	cache_seg->gen++;
	spin_unlock(&cache_seg->gen_lock);

	cache_seg_ctrl_write(cache_seg);
}

void cache_seg_get(struct pcache_cache_segment *cache_seg)
{
	atomic_inc(&cache_seg->refs);
}

static void cache_seg_invalidate(struct pcache_cache_segment *cache_seg)
{
	struct pcache_cache *cache;

	cache = cache_seg->cache;
	cache_seg_gen_increase(cache_seg);

	spin_lock(&cache->seg_map_lock);
	if (cache->cache_full)
		cache->cache_full = false;
	__clear_bit(cache_seg->cache_seg_id, cache->seg_map);
	spin_unlock(&cache->seg_map_lock);

	pcache_defer_reqs_kick(CACHE_TO_PCACHE(cache));
	/* clean_work will clean the bad key in key_tree*/
	queue_work(cache_get_wq(cache), &cache->clean_work);
}

void cache_seg_put(struct pcache_cache_segment *cache_seg)
{
	if (atomic_dec_and_test(&cache_seg->refs))
		cache_seg_invalidate(cache_seg);
}