drivers/md/dm-pcache/cache_writeback.c

// SPDX-License-Identifier: GPL-2.0-or-later

#include <linux/bio.h>

#include "cache.h"
#include "backing_dev.h"
#include "cache_dev.h"
#include "dm_pcache.h"

static void writeback_ctx_end(struct pcache_cache *cache, int ret)
{
	if (ret && !cache->writeback_ctx.ret) {
		pcache_dev_err(CACHE_TO_PCACHE(cache), "writeback error: %d", ret);
		cache->writeback_ctx.ret = ret;
	}

	if (!atomic_dec_and_test(&cache->writeback_ctx.pending))
		return;

	if (!cache->writeback_ctx.ret) {
		backing_dev_flush(cache->backing_dev);

		mutex_lock(&cache->dirty_tail_lock);
		cache_pos_advance(&cache->dirty_tail, cache->writeback_ctx.advance);
		cache_encode_dirty_tail(cache);
		mutex_unlock(&cache->dirty_tail_lock);
	}
	queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0);
}

static void writeback_end_req(struct pcache_backing_dev_req *backing_req, int ret)
{
	struct pcache_cache *cache = backing_req->priv_data;

	mutex_lock(&cache->writeback_lock);
	writeback_ctx_end(cache, ret);
	mutex_unlock(&cache->writeback_lock);
}

static inline bool is_cache_clean(struct pcache_cache *cache, struct pcache_cache_pos *dirty_tail)
{
	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
	struct pcache_cache_kset_onmedia *kset_onmedia;
	u32 to_copy;
	void *addr;
	int ret;

	addr = cache_pos_addr(dirty_tail);
	kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;

	to_copy = min(PCACHE_KSET_ONMEDIA_SIZE_MAX, cache_seg_remain(dirty_tail));
	ret = copy_mc_to_kernel(kset_onmedia, addr, to_copy);
	if (ret) {
		pcache_dev_err(pcache, "error to read kset: %d", ret);
		return true;
	}

	/* Reject a corrupted or out-of-bounds kset before reading its keys */
	if (!kset_onmedia_valid(kset_onmedia)) {
		pcache_dev_debug(pcache, "dirty_tail: %u:%u invalid kset magic: %llx, key_num: %u\n",
				dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off,
				kset_onmedia->magic, kset_onmedia->key_num);
		return true;
	}

	/* Verify the CRC checksum for data integrity */
	if (kset_onmedia->crc != cache_kset_crc(kset_onmedia)) {
		pcache_dev_debug(pcache, "dirty_tail: %u:%u crc: %x, not expected: %x\n",
				dirty_tail->cache_seg->cache_seg_id, dirty_tail->seg_off,
				cache_kset_crc(kset_onmedia), kset_onmedia->crc);
		return true;
	}

	return false;
}

void cache_writeback_exit(struct pcache_cache *cache)
{
	cancel_delayed_work_sync(&cache->writeback_work);
	backing_dev_flush(cache->backing_dev);
	cache_tree_exit(&cache->writeback_key_tree);
}

int cache_writeback_init(struct pcache_cache *cache)
{
	int ret;

	ret = cache_tree_init(cache, &cache->writeback_key_tree, 1);
	if (ret)
		goto err;

	atomic_set(&cache->writeback_ctx.pending, 0);

	/* Queue delayed work to start writeback handling */
	queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, 0);

	return 0;
err:
	return ret;
}

static void cache_key_writeback(struct pcache_cache *cache, struct pcache_cache_key *key)
{
	struct pcache_backing_dev_req *writeback_req;
	struct pcache_backing_dev_req_opts writeback_req_opts = { 0 };
	struct pcache_cache_pos *pos;
	void *addr;
	u32 seg_remain, req_len, done = 0;

	if (cache_key_clean(key))
		return;

	pos = &key->cache_pos;

	seg_remain = cache_seg_remain(pos);
	BUG_ON(seg_remain < key->len);
next_req:
	addr = cache_pos_addr(pos) + done;
	req_len = backing_dev_req_coalesced_max_len(addr, key->len - done);

	writeback_req_opts.type = BACKING_DEV_REQ_TYPE_KMEM;
	writeback_req_opts.gfp_mask = GFP_NOIO;
	writeback_req_opts.end_fn = writeback_end_req;
	writeback_req_opts.priv_data = cache;

	writeback_req_opts.kmem.data = addr;
	writeback_req_opts.kmem.opf = REQ_OP_WRITE;
	writeback_req_opts.kmem.len = req_len;
	writeback_req_opts.kmem.backing_off = key->off + done;

	writeback_req = backing_dev_req_create(cache->backing_dev, &writeback_req_opts);

	atomic_inc(&cache->writeback_ctx.pending);
	backing_dev_req_submit(writeback_req, true);

	done += req_len;
	if (done < key->len)
		goto next_req;
}

static void cache_wb_tree_writeback(struct pcache_cache *cache, u32 advance)
{
	struct pcache_cache_tree *cache_tree = &cache->writeback_key_tree;
	struct pcache_cache_subtree *cache_subtree;
	struct rb_node *node;
	struct pcache_cache_key *key;
	u32 i;

	cache->writeback_ctx.ret = 0;
	cache->writeback_ctx.advance = advance;
	atomic_set(&cache->writeback_ctx.pending, 1);

	for (i = 0; i < cache_tree->n_subtrees; i++) {
		cache_subtree = &cache_tree->subtrees[i];

		node = rb_first(&cache_subtree->root);
		while (node) {
			key = CACHE_KEY(node);
			node = rb_next(node);

			cache_key_writeback(cache, key);
			cache_key_delete(key);
		}
	}
	writeback_ctx_end(cache, 0);
}

static int cache_kset_insert_tree(struct pcache_cache *cache, struct pcache_cache_kset_onmedia *kset_onmedia)
{
	struct pcache_cache_key_onmedia *key_onmedia;
	struct pcache_cache_subtree *cache_subtree;
	struct pcache_cache_key *key;
	int ret;
	u32 i;

	/* Iterate through all keys in the kset and write each back to storage */
	for (i = 0; i < kset_onmedia->key_num; i++) {
		key_onmedia = &kset_onmedia->data[i];

		key = cache_key_alloc(&cache->writeback_key_tree, GFP_NOIO);
		ret = cache_key_decode(cache, key_onmedia, key);
		if (ret) {
			cache_key_put(key);
			goto clear_tree;
		}

		cache_subtree = get_subtree(&cache->writeback_key_tree, key->off);
		spin_lock(&cache_subtree->tree_lock);
		cache_key_insert(&cache->writeback_key_tree, key, true);
		spin_unlock(&cache_subtree->tree_lock);
	}

	return 0;
clear_tree:
	cache_tree_clear(&cache->writeback_key_tree);
	return ret;
}

static int last_kset_writeback(struct pcache_cache *cache,
		struct pcache_cache_kset_onmedia *last_kset_onmedia)
{
	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
	struct pcache_cache_segment *next_seg;

	if (!cache_seg_id_valid(cache, last_kset_onmedia->next_cache_seg_id)) {
		pcache_dev_err(pcache, "invalid next_cache_seg_id %u in writeback (n_segs %u)\n",
				last_kset_onmedia->next_cache_seg_id, cache->n_segs);
		return -EIO;
	}

	pcache_dev_debug(pcache, "last kset, next: %u\n", last_kset_onmedia->next_cache_seg_id);

	next_seg = &cache->segments[last_kset_onmedia->next_cache_seg_id];

	mutex_lock(&cache->dirty_tail_lock);
	cache->dirty_tail.cache_seg = next_seg;
	cache->dirty_tail.seg_off = 0;
	cache_encode_dirty_tail(cache);
	mutex_unlock(&cache->dirty_tail_lock);

	return 0;
}

void cache_writeback_fn(struct work_struct *work)
{
	struct pcache_cache *cache = container_of(work, struct pcache_cache, writeback_work.work);
	struct dm_pcache *pcache = CACHE_TO_PCACHE(cache);
	struct pcache_cache_pos dirty_tail;
	struct pcache_cache_kset_onmedia *kset_onmedia;
	u32 delay;
	int ret;

	mutex_lock(&cache->writeback_lock);
	if (atomic_read(&cache->writeback_ctx.pending))
		goto unlock;

	if (pcache_is_stopping(pcache))
		goto unlock;

	if (atomic_read(&cache->writeback_errors))
		goto unlock;

	kset_onmedia = (struct pcache_cache_kset_onmedia *)cache->wb_kset_onmedia_buf;

	mutex_lock(&cache->dirty_tail_lock);
	cache_pos_copy(&dirty_tail, &cache->dirty_tail);
	mutex_unlock(&cache->dirty_tail_lock);

	if (is_cache_clean(cache, &dirty_tail)) {
		delay = PCACHE_CACHE_WRITEBACK_INTERVAL;
		goto queue_work;
	}

	if (kset_onmedia->flags & PCACHE_KSET_FLAGS_LAST) {
		ret = last_kset_writeback(cache, kset_onmedia);
		if (ret) {
			atomic_inc(&cache->writeback_errors);
			goto unlock;
		}
		delay = 0;
		goto queue_work;
	}

	if (get_kset_onmedia_size(kset_onmedia) > cache_seg_remain(&dirty_tail)) {
		atomic_inc(&cache->writeback_errors);
		goto unlock;
	}

	ret = cache_kset_insert_tree(cache, kset_onmedia);
	if (ret) {
		atomic_inc(&cache->writeback_errors);
		goto unlock;
	}

	cache_wb_tree_writeback(cache, get_kset_onmedia_size(kset_onmedia));
	delay = 0;
queue_work:
	queue_delayed_work(cache_get_wq(cache), &cache->writeback_work, delay);
unlock:
	mutex_unlock(&cache->writeback_lock);
}