tools/testing/selftests/mm/rmap.c

// SPDX-License-Identifier: GPL-2.0
/*
 * RMAP functional tests
 *
 * Author(s): Wei Yang <richard.weiyang@gmail.com>
 */

#include "kselftest_harness.h"
#include <strings.h>
#include <pthread.h>
#include <numa.h>
#include <numaif.h>
#include <sys/mman.h>
#include <sys/prctl.h>
#include <sys/types.h>
#include <signal.h>
#include <time.h>
#include <sys/sem.h>
#include <unistd.h>
#include <fcntl.h>

#include "vm_util.h"

#define TOTAL_LEVEL 5
#define MAX_CHILDREN 3

#define FAIL_ON_CHECK	(1 << 0)
#define FAIL_ON_WORK	(1 << 1)

struct sembuf sem_wait = {0, -1, 0};
struct sembuf sem_signal = {0, 1, 0};

enum backend_type {
	ANON,
	SHM,
	NORM_FILE,
};

#define PREFIX "kst_rmap"
#define MAX_FILENAME_LEN 256
const char *suffixes[] = {
	"",
	"_shm",
	"_file",
};

struct global_data;
typedef int (*work_fn)(struct global_data *data);
typedef int (*check_fn)(struct global_data *data);
typedef void (*prepare_fn)(struct global_data *data);

struct global_data {
	int worker_level;

	int semid;
	int pipefd[2];

	unsigned int mapsize;
	unsigned int rand_seed;
	char *region;

	prepare_fn do_prepare;
	work_fn do_work;
	check_fn do_check;

	enum backend_type backend;
	char filename[MAX_FILENAME_LEN];

	unsigned long *expected_pfn;
};

/*
 * Create a process tree with TOTAL_LEVEL height and at most MAX_CHILDREN
 * children for each.
 *
 * It will randomly select one process as 'worker' process which will
 * 'do_work' until all processes are created. And all other processes will
 * wait until 'worker' finish its work.
 */
void propagate_children(struct __test_metadata *_metadata, struct global_data *data)
{
	pid_t root_pid, pid;
	unsigned int num_child;
	int status;
	int ret = 0;
	int curr_child, worker_child;
	int curr_level = 1;
	bool is_worker = true;

	root_pid = getpid();
repeat:
	num_child = rand_r(&data->rand_seed) % MAX_CHILDREN + 1;
	worker_child = is_worker ? rand_r(&data->rand_seed) % num_child : -1;

	for (curr_child = 0; curr_child < num_child; curr_child++) {
		pid = fork();

		if (pid < 0) {
			perror("Error: fork\n");
		} else if (pid == 0) {
			curr_level++;

			if (curr_child != worker_child)
				is_worker = false;

			if (curr_level == TOTAL_LEVEL)
				break;

			data->rand_seed += curr_child;
			goto repeat;
		}
	}

	if (data->do_prepare)
		data->do_prepare(data);

	close(data->pipefd[1]);

	if (is_worker && curr_level == data->worker_level) {
		/* This is the worker process, first wait last process created */
		char buf;

		while (read(data->pipefd[0], &buf, 1) > 0)
			;

		if (data->do_work)
			ret = data->do_work(data);

		/* Kick others */
		semctl(data->semid, 0, IPC_RMID);
	} else {
		/* Wait worker finish */
		semop(data->semid, &sem_wait, 1);
		if (data->do_check)
			ret = data->do_check(data);
	}

	/* Wait all child to quit */
	while (wait(&status) > 0) {
		if (WIFEXITED(status))
			ret |= WEXITSTATUS(status);
	}

	if (getpid() == root_pid) {
		if (ret & FAIL_ON_WORK)
			SKIP(return, "Failed in worker");

		ASSERT_EQ(ret, 0);
	} else {
		exit(ret);
	}
}

FIXTURE(migrate)
{
	struct global_data data;
};

FIXTURE_SETUP(migrate)
{
	struct global_data *data = &self->data;

	if (numa_available() < 0)
		SKIP(return, "NUMA not available");
	if (numa_bitmask_weight(numa_all_nodes_ptr) <= 1)
		SKIP(return, "Not enough NUMA nodes available");

	data->mapsize = getpagesize();

	data->expected_pfn = mmap(0, sizeof(unsigned long),
				PROT_READ | PROT_WRITE,
				MAP_SHARED | MAP_ANONYMOUS, -1, 0);
	ASSERT_NE(data->expected_pfn, MAP_FAILED);

	/* Prepare semaphore */
	data->semid = semget(IPC_PRIVATE, 1, 0666 | IPC_CREAT);
	ASSERT_NE(data->semid, -1);
	ASSERT_NE(semctl(data->semid, 0, SETVAL, 0), -1);

	/* Prepare pipe */
	ASSERT_NE(pipe(data->pipefd), -1);

	data->rand_seed = time(NULL);
	srand(data->rand_seed);

	data->worker_level = rand() % TOTAL_LEVEL + 1;

	data->do_prepare = NULL;
	data->do_work = NULL;
	data->do_check = NULL;

	data->backend = ANON;
};

FIXTURE_TEARDOWN(migrate)
{
	struct global_data *data = &self->data;

	if (data->region != MAP_FAILED)
		munmap(data->region, data->mapsize);
	data->region = MAP_FAILED;
	if (data->expected_pfn != MAP_FAILED)
		munmap(data->expected_pfn, sizeof(unsigned long));
	data->expected_pfn = MAP_FAILED;
	semctl(data->semid, 0, IPC_RMID);
	data->semid = -1;

	close(data->pipefd[0]);

	switch (data->backend) {
	case ANON:
		break;
	case SHM:
		shm_unlink(data->filename);
		break;
	case NORM_FILE:
		unlink(data->filename);
		break;
	}
}

void access_region(struct global_data *data)
{
	/*
	 * Force read "region" to make sure page fault in.
	 */
	FORCE_READ(*data->region);
}

int try_to_move_page(char *region)
{
	int ret;
	int node;
	int status = 0;
	int failures = 0;

	ret = move_pages(0, 1, (void **)&region, NULL, &status, MPOL_MF_MOVE_ALL);
	if (ret != 0) {
		perror("Failed to get original numa");
		return FAIL_ON_WORK;
	}

	/* Pick up a different target node */
	for (node = 0; node <= numa_max_node(); node++) {
		if (numa_bitmask_isbitset(numa_all_nodes_ptr, node) && node != status)
			break;
	}

	if (node > numa_max_node()) {
		ksft_print_msg("Couldn't find available numa node for testing\n");
		return FAIL_ON_WORK;
	}

	while (1) {
		ret = move_pages(0, 1, (void **)&region, &node, &status, MPOL_MF_MOVE_ALL);

		/* migrate successfully */
		if (!ret)
			break;

		/* error happened */
		if (ret < 0) {
			ksft_perror("Failed to move pages");
			return FAIL_ON_WORK;
		}

		/* migration is best effort; try again */
		if (++failures >= 100)
			return FAIL_ON_WORK;
	}

	return 0;
}

int move_region(struct global_data *data)
{
	int ret;
	int pagemap_fd;

	ret = try_to_move_page(data->region);
	if (ret != 0)
		return ret;

	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
	if (pagemap_fd == -1)
		return FAIL_ON_WORK;
	*data->expected_pfn = pagemap_get_pfn(pagemap_fd, data->region);

	return 0;
}

int has_same_pfn(struct global_data *data)
{
	unsigned long pfn;
	int pagemap_fd;

	if (data->region == MAP_FAILED)
		return 0;

	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
	if (pagemap_fd == -1)
		return FAIL_ON_CHECK;

	pfn = pagemap_get_pfn(pagemap_fd, data->region);
	if (pfn != *data->expected_pfn)
		return FAIL_ON_CHECK;

	return 0;
}

TEST_F(migrate, anon)
{
	struct global_data *data = &self->data;

	/* Map an area and fault in */
	data->region = mmap(0, data->mapsize, PROT_READ | PROT_WRITE,
				MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
	ASSERT_NE(data->region, MAP_FAILED);
	memset(data->region, 0xcf, data->mapsize);

	data->do_prepare = access_region;
	data->do_work = move_region;
	data->do_check = has_same_pfn;

	propagate_children(_metadata, data);
}

TEST_F(migrate, shm)
{
	int shm_fd;
	struct global_data *data = &self->data;

	snprintf(data->filename, MAX_FILENAME_LEN, "%s%s", PREFIX, suffixes[SHM]);
	shm_fd = shm_open(data->filename, O_CREAT | O_RDWR, 0666);
	ASSERT_NE(shm_fd, -1);
	ftruncate(shm_fd, data->mapsize);
	data->backend = SHM;

	/* Map a shared area and fault in */
	data->region = mmap(0, data->mapsize, PROT_READ | PROT_WRITE,
				MAP_SHARED, shm_fd, 0);
	ASSERT_NE(data->region, MAP_FAILED);
	memset(data->region, 0xcf, data->mapsize);
	close(shm_fd);

	data->do_prepare = access_region;
	data->do_work = move_region;
	data->do_check = has_same_pfn;

	propagate_children(_metadata, data);
}

TEST_F(migrate, file)
{
	int fd;
	struct global_data *data = &self->data;

	snprintf(data->filename, MAX_FILENAME_LEN, "%s%s", PREFIX, suffixes[NORM_FILE]);
	fd = open(data->filename, O_CREAT | O_RDWR | O_EXCL, 0666);
	ASSERT_NE(fd, -1);
	ftruncate(fd, data->mapsize);
	data->backend = NORM_FILE;

	/* Map a shared area and fault in */
	data->region = mmap(0, data->mapsize, PROT_READ | PROT_WRITE,
				MAP_SHARED, fd, 0);
	ASSERT_NE(data->region, MAP_FAILED);
	memset(data->region, 0xcf, data->mapsize);
	close(fd);

	data->do_prepare = access_region;
	data->do_work = move_region;
	data->do_check = has_same_pfn;

	propagate_children(_metadata, data);
}

void prepare_local_region(struct global_data *data)
{
	/* Allocate range and set the same data */
	data->region = mmap(NULL, data->mapsize, PROT_READ|PROT_WRITE,
			   MAP_PRIVATE|MAP_ANON, -1, 0);
	if (data->region == MAP_FAILED)
		return;

	memset(data->region, 0xcf, data->mapsize);
}

int merge_and_migrate(struct global_data *data)
{
	int pagemap_fd;
	int ret = 0;

	if (data->region == MAP_FAILED)
		return FAIL_ON_WORK;

	if (ksm_start() < 0)
		return FAIL_ON_WORK;

	ret = try_to_move_page(data->region);

	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
	if (pagemap_fd == -1)
		return FAIL_ON_WORK;
	*data->expected_pfn = pagemap_get_pfn(pagemap_fd, data->region);

	return ret;
}

TEST_F(migrate, ksm)
{
	int ret;
	struct global_data *data = &self->data;

	if (ksm_stop() < 0)
		SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\") failed");
	if (ksm_get_full_scans() < 0)
		SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\") failed");

	ret = prctl(PR_SET_MEMORY_MERGE, 1, 0, 0, 0);
	if (ret < 0 && errno == EINVAL)
		SKIP(return, "PR_SET_MEMORY_MERGE not supported");
	else if (ret)
		ksft_exit_fail_perror("PR_SET_MEMORY_MERGE=1 failed");

	data->do_prepare = prepare_local_region;
	data->do_work = merge_and_migrate;
	data->do_check = has_same_pfn;

	propagate_children(_metadata, data);
}

static bool range_maps_the_same_pfn(int pagemap_fd, void *region, int nr_pages)
{
	int i;
	int retries = 0;
	unsigned long first_pfn;

retry:
	if (retries > 10)
		return false;

	first_pfn = pagemap_get_pfn(pagemap_fd, region);
	for (i = 0; i < nr_pages; i++) {
		if (pagemap_get_pfn(pagemap_fd, region + i * getpagesize()) != first_pfn) {
			/*
			 * Retry up to 10 times at most in case of the low chance of page
			 * compaction migrating the page while we check for pfn.
			 */
			retries++;
			goto retry;
		}
	}

	return true;
}

TEST_F(migrate, ksm_and_mremap)
{
	unsigned long old_pfn, new_pfn;
	void *region, *mremap_region;
	const int nr_pages = 16;
	size_t mmap_size;
	int pagemap_fd;

	/* Skip if KSM is not available */
	if (ksm_stop() < 0)
		SKIP(return, "accessing \"/sys/kernel/mm/ksm/run\" failed");
	if (ksm_get_full_scans() < 0)
		SKIP(return, "accessing \"/sys/kernel/mm/ksm/full_scan\" failed");

	pagemap_fd = open("/proc/self/pagemap", O_RDONLY);
	if (pagemap_fd < 0)
		SKIP(return, "opening pagemap failed");

	/* Allocate and populate twice the anon pages initially. */
	mmap_size = 2 * nr_pages * getpagesize();
	region = mmap(NULL, mmap_size, PROT_READ | PROT_WRITE,
		      MAP_PRIVATE | MAP_ANON, -1, 0);
	ASSERT_NE(region, MAP_FAILED);
	memset(region, 0x77, mmap_size);

	/* mremap the second half over the first half, to stress rmap handling */
	mmap_size /= 2;
	mremap_region = mremap(region + mmap_size, mmap_size, mmap_size,
			       MREMAP_MAYMOVE | MREMAP_FIXED, region);
	ASSERT_EQ(mremap_region, region);

	/* Merge all pages into a single KSM page. */
	madvise(region, mmap_size, MADV_MERGEABLE);
	ASSERT_EQ(ksm_start(), 0);

	/* The whole range should map the same KSM page. */
	old_pfn = pagemap_get_pfn(pagemap_fd, region);
	if (old_pfn == -1ul)
		SKIP(return, "Obtaining PFN failed");
	ksm_start();
	ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages));

	/*
	 * Migrate the KSM page; the whole range should map the new (migrated)
	 * KSM page.
	 */
	ASSERT_EQ(try_to_move_page(region), 0);

	new_pfn = pagemap_get_pfn(pagemap_fd, region);
	if (new_pfn == -1ul)
		SKIP(return, "Obtaining PFN failed");
	ASSERT_NE(new_pfn, old_pfn);
	ASSERT_TRUE(range_maps_the_same_pfn(pagemap_fd, region, nr_pages));
}


TEST_HARNESS_MAIN