drivers/md/dm-pcache/backing_dev.c

// SPDX-License-Identifier: GPL-2.0-or-later
#include <linux/blkdev.h>

#include "../dm-core.h"
#include "pcache_internal.h"
#include "cache_dev.h"
#include "backing_dev.h"
#include "cache.h"
#include "dm_pcache.h"

static struct kmem_cache *backing_req_cache;
static struct kmem_cache *backing_bvec_cache;

static void backing_dev_exit(struct pcache_backing_dev *backing_dev)
{
	mempool_exit(&backing_dev->req_pool);
	mempool_exit(&backing_dev->bvec_pool);
}

static void req_submit_fn(struct work_struct *work);
static void req_complete_fn(struct work_struct *work);
static int backing_dev_init(struct dm_pcache *pcache)
{
	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
	int ret;

	ret = mempool_init_slab_pool(&backing_dev->req_pool, 128, backing_req_cache);
	if (ret)
		goto err;

	ret = mempool_init_slab_pool(&backing_dev->bvec_pool, 128, backing_bvec_cache);
	if (ret)
		goto req_pool_exit;

	INIT_LIST_HEAD(&backing_dev->submit_list);
	INIT_LIST_HEAD(&backing_dev->complete_list);
	spin_lock_init(&backing_dev->submit_lock);
	spin_lock_init(&backing_dev->complete_lock);
	INIT_WORK(&backing_dev->req_submit_work, req_submit_fn);
	INIT_WORK(&backing_dev->req_complete_work, req_complete_fn);
	atomic_set(&backing_dev->inflight_reqs, 0);
	init_waitqueue_head(&backing_dev->inflight_wq);

	return 0;

req_pool_exit:
	mempool_exit(&backing_dev->req_pool);
err:
	return ret;
}

int backing_dev_start(struct dm_pcache *pcache)
{
	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;
	int ret;

	ret = backing_dev_init(pcache);
	if (ret)
		return ret;

	backing_dev->dev_size = bdev_nr_sectors(backing_dev->dm_dev->bdev);

	return 0;
}

void backing_dev_stop(struct dm_pcache *pcache)
{
	struct pcache_backing_dev *backing_dev = &pcache->backing_dev;

	/*
	 * There should not be any new request comming, just wait
	 * inflight requests done.
	 */
	wait_event(backing_dev->inflight_wq,
			atomic_read(&backing_dev->inflight_reqs) == 0);

	flush_work(&backing_dev->req_submit_work);
	flush_work(&backing_dev->req_complete_work);

	backing_dev_exit(backing_dev);
}

/* pcache_backing_dev_req functions */
void backing_dev_req_end(struct pcache_backing_dev_req *backing_req)
{
	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;

	if (backing_req->end_req)
		backing_req->end_req(backing_req, backing_req->ret);

	switch (backing_req->type) {
	case BACKING_DEV_REQ_TYPE_REQ:
		if (backing_req->req.upper_req)
			pcache_req_put(backing_req->req.upper_req, backing_req->ret);
		break;
	case BACKING_DEV_REQ_TYPE_KMEM:
		if (backing_req->kmem.bvecs != backing_req->kmem.inline_bvecs)
			mempool_free(backing_req->kmem.bvecs, &backing_dev->bvec_pool);
		break;
	default:
		BUG();
	}

	mempool_free(backing_req, &backing_dev->req_pool);

	if (atomic_dec_and_test(&backing_dev->inflight_reqs))
		wake_up(&backing_dev->inflight_wq);
}

static void req_complete_fn(struct work_struct *work)
{
	struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_complete_work);
	struct pcache_backing_dev_req *backing_req;
	LIST_HEAD(tmp_list);

	spin_lock_irq(&backing_dev->complete_lock);
	list_splice_init(&backing_dev->complete_list, &tmp_list);
	spin_unlock_irq(&backing_dev->complete_lock);

	while (!list_empty(&tmp_list)) {
		backing_req = list_first_entry(&tmp_list,
					    struct pcache_backing_dev_req, node);
		list_del_init(&backing_req->node);
		backing_dev_req_end(backing_req);
	}
}

static void backing_dev_bio_end(struct bio *bio)
{
	struct pcache_backing_dev_req *backing_req = bio->bi_private;
	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
	unsigned long flags;

	backing_req->ret = blk_status_to_errno(bio->bi_status);

	spin_lock_irqsave(&backing_dev->complete_lock, flags);
	list_move_tail(&backing_req->node, &backing_dev->complete_list);
	queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_complete_work);
	spin_unlock_irqrestore(&backing_dev->complete_lock, flags);
}

static void req_submit_fn(struct work_struct *work)
{
	struct pcache_backing_dev *backing_dev = container_of(work, struct pcache_backing_dev, req_submit_work);
	struct pcache_backing_dev_req *backing_req;
	LIST_HEAD(tmp_list);

	spin_lock(&backing_dev->submit_lock);
	list_splice_init(&backing_dev->submit_list, &tmp_list);
	spin_unlock(&backing_dev->submit_lock);

	while (!list_empty(&tmp_list)) {
		backing_req = list_first_entry(&tmp_list,
					    struct pcache_backing_dev_req, node);
		list_del_init(&backing_req->node);
		submit_bio_noacct(&backing_req->bio);
	}
}

void backing_dev_req_submit(struct pcache_backing_dev_req *backing_req, bool direct)
{
	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;

	if (direct) {
		submit_bio_noacct(&backing_req->bio);
		return;
	}

	spin_lock(&backing_dev->submit_lock);
	list_add_tail(&backing_req->node, &backing_dev->submit_list);
	queue_work(BACKING_DEV_TO_PCACHE(backing_dev)->task_wq, &backing_dev->req_submit_work);
	spin_unlock(&backing_dev->submit_lock);
}

static void bio_map(struct bio *bio, void *base, size_t size)
{
	struct page *page;
	unsigned int offset;
	unsigned int len;

	if (!is_vmalloc_addr(base)) {
		page = virt_to_page(base);
		offset = offset_in_page(base);

		BUG_ON(!bio_add_page(bio, page, size, offset));
		return;
	}

	flush_kernel_vmap_range(base, size);
	while (size) {
		page = vmalloc_to_page(base);
		offset = offset_in_page(base);
		len = min_t(size_t, PAGE_SIZE - offset, size);

		BUG_ON(!bio_add_page(bio, page, len, offset));
		size -= len;
		base += len;
	}
}

static struct pcache_backing_dev_req *req_type_req_alloc(struct pcache_backing_dev *backing_dev,
							struct pcache_backing_dev_req_opts *opts)
{
	struct pcache_request *pcache_req = opts->req.upper_req;
	struct pcache_backing_dev_req *backing_req;
	struct bio *orig = pcache_req->bio;

	backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask);
	if (!backing_req)
		return NULL;

	memset(backing_req, 0, sizeof(struct pcache_backing_dev_req));

	bio_init_clone(backing_dev->dm_dev->bdev, &backing_req->bio, orig, opts->gfp_mask);

	backing_req->type = BACKING_DEV_REQ_TYPE_REQ;
	backing_req->backing_dev = backing_dev;
	atomic_inc(&backing_dev->inflight_reqs);

	return backing_req;
}

static struct pcache_backing_dev_req *kmem_type_req_alloc(struct pcache_backing_dev *backing_dev,
						struct pcache_backing_dev_req_opts *opts)
{
	struct pcache_backing_dev_req *backing_req;
	u32 n_vecs = bio_add_max_vecs(opts->kmem.data, opts->kmem.len);

	backing_req = mempool_alloc(&backing_dev->req_pool, opts->gfp_mask);
	if (!backing_req)
		return NULL;

	memset(backing_req, 0, sizeof(struct pcache_backing_dev_req));

	if (n_vecs > BACKING_DEV_REQ_INLINE_BVECS) {
		backing_req->kmem.bvecs = mempool_alloc(&backing_dev->bvec_pool, opts->gfp_mask);
		if (!backing_req->kmem.bvecs)
			goto free_backing_req;
	} else {
		backing_req->kmem.bvecs = backing_req->kmem.inline_bvecs;
	}

	backing_req->kmem.n_vecs = n_vecs;
	backing_req->type = BACKING_DEV_REQ_TYPE_KMEM;
	backing_req->backing_dev = backing_dev;
	atomic_inc(&backing_dev->inflight_reqs);

	return backing_req;

free_backing_req:
	mempool_free(backing_req, &backing_dev->req_pool);
	return NULL;
}

struct pcache_backing_dev_req *backing_dev_req_alloc(struct pcache_backing_dev *backing_dev,
						struct pcache_backing_dev_req_opts *opts)
{
	if (opts->type == BACKING_DEV_REQ_TYPE_REQ)
		return req_type_req_alloc(backing_dev, opts);

	if (opts->type == BACKING_DEV_REQ_TYPE_KMEM)
		return kmem_type_req_alloc(backing_dev, opts);

	BUG();
}

static void req_type_req_init(struct pcache_backing_dev_req *backing_req,
			struct pcache_backing_dev_req_opts *opts)
{
	struct pcache_request *pcache_req = opts->req.upper_req;
	struct bio *clone;
	u32 off = opts->req.req_off;
	u32 len = opts->req.len;

	clone = &backing_req->bio;
	BUG_ON(off & SECTOR_MASK);
	BUG_ON(len & SECTOR_MASK);
	bio_trim(clone, off >> SECTOR_SHIFT, len >> SECTOR_SHIFT);

	clone->bi_iter.bi_sector = (pcache_req->off + off) >> SECTOR_SHIFT;
	clone->bi_private = backing_req;
	clone->bi_end_io = backing_dev_bio_end;

	INIT_LIST_HEAD(&backing_req->node);
	backing_req->end_req     = opts->end_fn;

	pcache_req_get(pcache_req);
	backing_req->req.upper_req	= pcache_req;
	backing_req->req.bio_off	= off;
}

static void kmem_type_req_init(struct pcache_backing_dev_req *backing_req,
			struct pcache_backing_dev_req_opts *opts)
{
	struct pcache_backing_dev *backing_dev = backing_req->backing_dev;
	struct bio *backing_bio;

	bio_init(&backing_req->bio, backing_dev->dm_dev->bdev, backing_req->kmem.bvecs,
			backing_req->kmem.n_vecs, opts->kmem.opf);

	backing_bio = &backing_req->bio;
	bio_map(backing_bio, opts->kmem.data, opts->kmem.len);

	backing_bio->bi_iter.bi_sector = (opts->kmem.backing_off) >> SECTOR_SHIFT;
	backing_bio->bi_private = backing_req;
	backing_bio->bi_end_io = backing_dev_bio_end;

	INIT_LIST_HEAD(&backing_req->node);
	backing_req->end_req	= opts->end_fn;
	backing_req->priv_data	= opts->priv_data;
}

void backing_dev_req_init(struct pcache_backing_dev_req *backing_req,
			struct pcache_backing_dev_req_opts *opts)
{
	if (opts->type == BACKING_DEV_REQ_TYPE_REQ)
		return req_type_req_init(backing_req, opts);

	if (opts->type == BACKING_DEV_REQ_TYPE_KMEM)
		return kmem_type_req_init(backing_req, opts);

	BUG();
}

struct pcache_backing_dev_req *backing_dev_req_create(struct pcache_backing_dev *backing_dev,
						struct pcache_backing_dev_req_opts *opts)
{
	struct pcache_backing_dev_req *backing_req;

	backing_req = backing_dev_req_alloc(backing_dev, opts);
	if (!backing_req)
		return NULL;

	backing_dev_req_init(backing_req, opts);

	return backing_req;
}

void backing_dev_flush(struct pcache_backing_dev *backing_dev)
{
	blkdev_issue_flush(backing_dev->dm_dev->bdev);
}

int pcache_backing_init(void)
{
	u32 max_bvecs = (PCACHE_CACHE_SUBTREE_SIZE >> PAGE_SHIFT) + 1;
	int ret;

	backing_req_cache = KMEM_CACHE(pcache_backing_dev_req, 0);
	if (!backing_req_cache) {
		ret = -ENOMEM;
		goto err;
	}

	backing_bvec_cache = kmem_cache_create("pcache-bvec-slab",
					max_bvecs * sizeof(struct bio_vec),
					0, 0, NULL);
	if (!backing_bvec_cache) {
		ret = -ENOMEM;
		goto destroy_req_cache;
	}

	return 0;
destroy_req_cache:
	kmem_cache_destroy(backing_req_cache);
err:
	return ret;
}

void pcache_backing_exit(void)
{
	kmem_cache_destroy(backing_bvec_cache);
	kmem_cache_destroy(backing_req_cache);
}