| // SPDX-License-Identifier: GPL-2.0-only | |
| /* | |
| * Userfaultfd unit tests. | |
| * | |
| * Copyright (C) 2015-2023 Red Hat, Inc. | |
| */ | |
| #include "uffd-common.h" | |
| #include "../../../../mm/gup_test.h" | |
| #ifdef __NR_userfaultfd | |
| /* The unit test doesn't need a large or random size, make it 32MB for now */ | |
| #define UFFD_TEST_MEM_SIZE (32UL << 20) | |
| #define MEM_ANON BIT_ULL(0) | |
| #define MEM_SHMEM BIT_ULL(1) | |
| #define MEM_SHMEM_PRIVATE BIT_ULL(2) | |
| #define MEM_HUGETLB BIT_ULL(3) | |
| #define MEM_HUGETLB_PRIVATE BIT_ULL(4) | |
| #define MEM_ALL (MEM_ANON | MEM_SHMEM | MEM_SHMEM_PRIVATE | \ | |
| MEM_HUGETLB | MEM_HUGETLB_PRIVATE) | |
| #define ALIGN_UP(x, align_to) \ | |
| ((__typeof__(x))((((unsigned long)(x)) + ((align_to)-1)) & ~((align_to)-1))) | |
| #define MAX(a, b) (((a) > (b)) ? (a) : (b)) | |
| struct mem_type { | |
| const char *name; | |
| unsigned int mem_flag; | |
| uffd_test_ops_t *mem_ops; | |
| bool shared; | |
| }; | |
| typedef struct mem_type mem_type_t; | |
| mem_type_t mem_types[] = { | |
| { | |
| .name = "anon", | |
| .mem_flag = MEM_ANON, | |
| .mem_ops = &anon_uffd_test_ops, | |
| .shared = false, | |
| }, | |
| { | |
| .name = "shmem", | |
| .mem_flag = MEM_SHMEM, | |
| .mem_ops = &shmem_uffd_test_ops, | |
| .shared = true, | |
| }, | |
| { | |
| .name = "shmem-private", | |
| .mem_flag = MEM_SHMEM_PRIVATE, | |
| .mem_ops = &shmem_uffd_test_ops, | |
| .shared = false, | |
| }, | |
| { | |
| .name = "hugetlb", | |
| .mem_flag = MEM_HUGETLB, | |
| .mem_ops = &hugetlb_uffd_test_ops, | |
| .shared = true, | |
| }, | |
| { | |
| .name = "hugetlb-private", | |
| .mem_flag = MEM_HUGETLB_PRIVATE, | |
| .mem_ops = &hugetlb_uffd_test_ops, | |
| .shared = false, | |
| }, | |
| }; | |
| /* Arguments to be passed over to each uffd unit test */ | |
| struct uffd_test_args { | |
| mem_type_t *mem_type; | |
| }; | |
| typedef struct uffd_test_args uffd_test_args_t; | |
| /* Returns: UFFD_TEST_* */ | |
| typedef void (*uffd_test_fn)(uffd_global_test_opts_t *, uffd_test_args_t *); | |
| typedef struct { | |
| const char *name; | |
| uffd_test_fn uffd_fn; | |
| unsigned int mem_targets; | |
| uint64_t uffd_feature_required; | |
| uffd_test_case_ops_t *test_case_ops; | |
| } uffd_test_case_t; | |
| static void uffd_test_report(void) | |
| { | |
| printf("Userfaults unit tests: pass=%u, skip=%u, fail=%u (total=%u)\n", | |
| ksft_get_pass_cnt(), | |
| ksft_get_xskip_cnt(), | |
| ksft_get_fail_cnt(), | |
| ksft_test_num()); | |
| } | |
| static void uffd_test_pass(void) | |
| { | |
| printf("done\n"); | |
| ksft_inc_pass_cnt(); | |
| } | |
| #define uffd_test_start(...) do { \ | |
| printf("Testing "); \ | |
| printf(__VA_ARGS__); \ | |
| printf("... "); \ | |
| fflush(stdout); \ | |
| } while (0) | |
| #define uffd_test_fail(...) do { \ | |
| printf("failed [reason: "); \ | |
| printf(__VA_ARGS__); \ | |
| printf("]\n"); \ | |
| ksft_inc_fail_cnt(); \ | |
| } while (0) | |
| static void uffd_test_skip(const char *message) | |
| { | |
| printf("skipped [reason: %s]\n", message); | |
| ksft_inc_xskip_cnt(); | |
| } | |
| /* | |
| * Returns 1 if specific userfaultfd supported, 0 otherwise. Note, we'll | |
| * return 1 even if some test failed as long as uffd supported, because in | |
| * that case we still want to proceed with the rest uffd unit tests. | |
| */ | |
| static int test_uffd_api(bool use_dev) | |
| { | |
| struct uffdio_api uffdio_api; | |
| int uffd; | |
| uffd_test_start("UFFDIO_API (with %s)", | |
| use_dev ? "/dev/userfaultfd" : "syscall"); | |
| if (use_dev) | |
| uffd = uffd_open_dev(UFFD_FLAGS); | |
| else | |
| uffd = uffd_open_sys(UFFD_FLAGS); | |
| if (uffd < 0) { | |
| uffd_test_skip("cannot open userfaultfd handle"); | |
| return 0; | |
| } | |
| /* Test wrong UFFD_API */ | |
| uffdio_api.api = 0xab; | |
| uffdio_api.features = 0; | |
| if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { | |
| uffd_test_fail("UFFDIO_API should fail with wrong api but didn't"); | |
| goto out; | |
| } | |
| /* Test wrong feature bit */ | |
| uffdio_api.api = UFFD_API; | |
| uffdio_api.features = BIT_ULL(63); | |
| if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { | |
| uffd_test_fail("UFFDIO_API should fail with wrong feature but didn't"); | |
| goto out; | |
| } | |
| /* Test normal UFFDIO_API */ | |
| uffdio_api.api = UFFD_API; | |
| uffdio_api.features = 0; | |
| if (ioctl(uffd, UFFDIO_API, &uffdio_api)) { | |
| uffd_test_fail("UFFDIO_API should succeed but failed"); | |
| goto out; | |
| } | |
| /* Test double requests of UFFDIO_API with a random feature set */ | |
| uffdio_api.features = BIT_ULL(0); | |
| if (ioctl(uffd, UFFDIO_API, &uffdio_api) == 0) { | |
| uffd_test_fail("UFFDIO_API should reject initialized uffd"); | |
| goto out; | |
| } | |
| uffd_test_pass(); | |
| out: | |
| close(uffd); | |
| /* We have a valid uffd handle */ | |
| return 1; | |
| } | |
| static bool uffd_feature_supported(uffd_test_case_t *test) | |
| { | |
| uint64_t features; | |
| if (uffd_get_features(&features)) | |
| return false; | |
| return (features & test->uffd_feature_required) == | |
| test->uffd_feature_required; | |
| } | |
| static int pagemap_open(void) | |
| { | |
| int fd = open("/proc/self/pagemap", O_RDONLY); | |
| if (fd < 0) | |
| err("open pagemap"); | |
| return fd; | |
| } | |
| /* This macro let __LINE__ works in err() */ | |
| #define pagemap_check_wp(value, wp) do { \ | |
| if (!!(value & PM_UFFD_WP) != wp) \ | |
| err("pagemap uffd-wp bit error: 0x%"PRIx64, value); \ | |
| } while (0) | |
| typedef struct { | |
| uffd_global_test_opts_t *gopts; | |
| int child_uffd; | |
| } fork_event_args; | |
| static void *fork_event_consumer(void *data) | |
| { | |
| fork_event_args *args = data; | |
| struct uffd_msg msg = { 0 }; | |
| args->gopts->ready_for_fork = true; | |
| /* Read until a full msg received */ | |
| while (uffd_read_msg(args->gopts, &msg)); | |
| if (msg.event != UFFD_EVENT_FORK) | |
| err("wrong message: %u\n", msg.event); | |
| /* Just to be properly freed later */ | |
| args->child_uffd = msg.arg.fork.ufd; | |
| return NULL; | |
| } | |
| typedef struct { | |
| int gup_fd; | |
| bool pinned; | |
| } pin_args; | |
| /* | |
| * Returns 0 if succeed, <0 for errors. pin_pages() needs to be paired | |
| * with unpin_pages(). Currently it needs to be RO longterm pin to satisfy | |
| * all needs of the test cases (e.g., trigger unshare, trigger fork() early | |
| * CoW, etc.). | |
| */ | |
| static int pin_pages(pin_args *args, void *buffer, size_t size) | |
| { | |
| struct pin_longterm_test test = { | |
| .addr = (uintptr_t)buffer, | |
| .size = size, | |
| /* Read-only pins */ | |
| .flags = 0, | |
| }; | |
| if (args->pinned) | |
| err("already pinned"); | |
| args->gup_fd = open("/sys/kernel/debug/gup_test", O_RDWR); | |
| if (args->gup_fd < 0) | |
| return -errno; | |
| if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_START, &test)) { | |
| /* Even if gup_test existed, can be an old gup_test / kernel */ | |
| close(args->gup_fd); | |
| return -errno; | |
| } | |
| args->pinned = true; | |
| return 0; | |
| } | |
| static void unpin_pages(pin_args *args) | |
| { | |
| if (!args->pinned) | |
| err("unpin without pin first"); | |
| if (ioctl(args->gup_fd, PIN_LONGTERM_TEST_STOP)) | |
| err("PIN_LONGTERM_TEST_STOP"); | |
| close(args->gup_fd); | |
| args->pinned = false; | |
| } | |
| static int pagemap_test_fork(uffd_global_test_opts_t *gopts, bool with_event, bool test_pin) | |
| { | |
| fork_event_args args = { .gopts = gopts, .child_uffd = -1 }; | |
| pthread_t thread; | |
| pid_t child; | |
| uint64_t value; | |
| int fd, result; | |
| /* Prepare a thread to resolve EVENT_FORK */ | |
| if (with_event) { | |
| gopts->ready_for_fork = false; | |
| if (pthread_create(&thread, NULL, fork_event_consumer, &args)) | |
| err("pthread_create()"); | |
| while (!gopts->ready_for_fork) | |
| ; /* Wait for the poll_thread to start executing before forking */ | |
| } | |
| child = fork(); | |
| if (!child) { | |
| /* Open the pagemap fd of the child itself */ | |
| pin_args args = {}; | |
| fd = pagemap_open(); | |
| if (test_pin && pin_pages(&args, gopts->area_dst, gopts->page_size)) | |
| /* | |
| * Normally when reach here we have pinned in | |
| * previous tests, so shouldn't fail anymore | |
| */ | |
| err("pin page failed in child"); | |
| value = pagemap_get_entry(fd, gopts->area_dst); | |
| /* | |
| * After fork(), we should handle uffd-wp bit differently: | |
| * | |
| * (1) when with EVENT_FORK, it should persist | |
| * (2) when without EVENT_FORK, it should be dropped | |
| */ | |
| pagemap_check_wp(value, with_event); | |
| if (test_pin) | |
| unpin_pages(&args); | |
| /* Succeed */ | |
| exit(0); | |
| } | |
| waitpid(child, &result, 0); | |
| if (with_event) { | |
| if (pthread_join(thread, NULL)) | |
| err("pthread_join()"); | |
| if (args.child_uffd < 0) | |
| err("Didn't receive child uffd"); | |
| close(args.child_uffd); | |
| } | |
| return result; | |
| } | |
| static void uffd_wp_unpopulated_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uint64_t value; | |
| int pagemap_fd; | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| false, true, false)) | |
| err("register failed"); | |
| pagemap_fd = pagemap_open(); | |
| /* Test applying pte marker to anon unpopulated */ | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, true); | |
| /* Test unprotect on anon pte marker */ | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false); | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, false); | |
| /* Test zap on anon marker */ | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); | |
| if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) | |
| err("madvise(MADV_DONTNEED) failed"); | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, false); | |
| /* Test fault in after marker removed */ | |
| *gopts->area_dst = 1; | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, false); | |
| /* Drop it to make pte none again */ | |
| if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) | |
| err("madvise(MADV_DONTNEED) failed"); | |
| /* Test read-zero-page upon pte marker */ | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); | |
| *(volatile char *)gopts->area_dst; | |
| /* Drop it to make pte none again */ | |
| if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) | |
| err("madvise(MADV_DONTNEED) failed"); | |
| uffd_test_pass(); | |
| } | |
| static void uffd_wp_fork_test_common(uffd_global_test_opts_t *gopts, uffd_test_args_t *args, | |
| bool with_event) | |
| { | |
| int pagemap_fd; | |
| uint64_t value; | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| false, true, false)) | |
| err("register failed"); | |
| pagemap_fd = pagemap_open(); | |
| /* Touch the page */ | |
| *gopts->area_dst = 1; | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, true); | |
| if (pagemap_test_fork(gopts, with_event, false)) { | |
| uffd_test_fail("Detected %s uffd-wp bit in child in present pte", | |
| with_event ? "missing" : "stall"); | |
| goto out; | |
| } | |
| /* | |
| * This is an attempt for zapping the pgtable so as to test the | |
| * markers. | |
| * | |
| * For private mappings, PAGEOUT will only work on exclusive ptes | |
| * (PM_MMAP_EXCLUSIVE) which we should satisfy. | |
| * | |
| * For shared, PAGEOUT may not work. Use DONTNEED instead which | |
| * plays a similar role of zapping (rather than freeing the page) | |
| * to expose pte markers. | |
| */ | |
| if (args->mem_type->shared) { | |
| if (madvise(gopts->area_dst, gopts->page_size, MADV_DONTNEED)) | |
| err("MADV_DONTNEED"); | |
| } else { | |
| /* | |
| * NOTE: ignore retval because private-hugetlb doesn't yet | |
| * support swapping, so it could fail. | |
| */ | |
| madvise(gopts->area_dst, gopts->page_size, MADV_PAGEOUT); | |
| } | |
| /* Uffd-wp should persist even swapped out */ | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, true); | |
| if (pagemap_test_fork(gopts, with_event, false)) { | |
| uffd_test_fail("Detected %s uffd-wp bit in child in zapped pte", | |
| with_event ? "missing" : "stall"); | |
| goto out; | |
| } | |
| /* Unprotect; this tests swap pte modifications */ | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, false); | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, false); | |
| /* Fault in the page from disk */ | |
| *gopts->area_dst = 2; | |
| value = pagemap_get_entry(pagemap_fd, gopts->area_dst); | |
| pagemap_check_wp(value, false); | |
| uffd_test_pass(); | |
| out: | |
| if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size)) | |
| err("unregister failed"); | |
| close(pagemap_fd); | |
| } | |
| static void uffd_wp_fork_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_wp_fork_test_common(gopts, args, false); | |
| } | |
| static void uffd_wp_fork_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_wp_fork_test_common(gopts, args, true); | |
| } | |
| static void uffd_wp_fork_pin_test_common(uffd_global_test_opts_t *gopts, | |
| uffd_test_args_t *args, | |
| bool with_event) | |
| { | |
| int pagemap_fd; | |
| pin_args pin_args = {}; | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->page_size, false, true, false)) | |
| err("register failed"); | |
| pagemap_fd = pagemap_open(); | |
| /* Touch the page */ | |
| *gopts->area_dst = 1; | |
| wp_range(gopts->uffd, (uint64_t)gopts->area_dst, gopts->page_size, true); | |
| /* | |
| * 1. First pin, then fork(). This tests fork() special path when | |
| * doing early CoW if the page is private. | |
| */ | |
| if (pin_pages(&pin_args, gopts->area_dst, gopts->page_size)) { | |
| uffd_test_skip("Possibly CONFIG_GUP_TEST missing " | |
| "or unprivileged"); | |
| close(pagemap_fd); | |
| uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size); | |
| return; | |
| } | |
| if (pagemap_test_fork(gopts, with_event, false)) { | |
| uffd_test_fail("Detected %s uffd-wp bit in early CoW of fork()", | |
| with_event ? "missing" : "stall"); | |
| unpin_pages(&pin_args); | |
| goto out; | |
| } | |
| unpin_pages(&pin_args); | |
| /* | |
| * 2. First fork(), then pin (in the child, where test_pin==true). | |
| * This tests COR, aka, page unsharing on private memories. | |
| */ | |
| if (pagemap_test_fork(gopts, with_event, true)) { | |
| uffd_test_fail("Detected %s uffd-wp bit when RO pin", | |
| with_event ? "missing" : "stall"); | |
| goto out; | |
| } | |
| uffd_test_pass(); | |
| out: | |
| if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size)) | |
| err("register failed"); | |
| close(pagemap_fd); | |
| } | |
| static void uffd_wp_fork_pin_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_wp_fork_pin_test_common(gopts, args, false); | |
| } | |
| static void uffd_wp_fork_pin_with_event_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_wp_fork_pin_test_common(gopts, args, true); | |
| } | |
| static void check_memory_contents(uffd_global_test_opts_t *gopts, char *p) | |
| { | |
| unsigned long i, j; | |
| uint8_t expected_byte; | |
| for (i = 0; i < gopts->nr_pages; ++i) { | |
| expected_byte = ~((uint8_t)(i % ((uint8_t)-1))); | |
| for (j = 0; j < gopts->page_size; j++) { | |
| uint8_t v = *(uint8_t *)(p + (i * gopts->page_size) + j); | |
| if (v != expected_byte) | |
| err("unexpected page contents"); | |
| } | |
| } | |
| } | |
| static void uffd_minor_test_common(uffd_global_test_opts_t *gopts, bool test_collapse, bool test_wp) | |
| { | |
| unsigned long p; | |
| pthread_t uffd_mon; | |
| char c = '\0'; | |
| struct uffd_args args = { 0 }; | |
| args.gopts = gopts; | |
| /* | |
| * NOTE: MADV_COLLAPSE is not yet compatible with WP, so testing | |
| * both do not make much sense. | |
| */ | |
| assert(!(test_collapse && test_wp)); | |
| if (uffd_register(gopts->uffd, gopts->area_dst_alias, gopts->nr_pages * gopts->page_size, | |
| /* NOTE! MADV_COLLAPSE may not work with uffd-wp */ | |
| false, test_wp, true)) | |
| err("register failure"); | |
| /* | |
| * After registering with UFFD, populate the non-UFFD-registered side of | |
| * the shared mapping. This should *not* trigger any UFFD minor faults. | |
| */ | |
| for (p = 0; p < gopts->nr_pages; ++p) | |
| memset(gopts->area_dst + (p * gopts->page_size), p % ((uint8_t)-1), | |
| gopts->page_size); | |
| args.apply_wp = test_wp; | |
| if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) | |
| err("uffd_poll_thread create"); | |
| /* | |
| * Read each of the pages back using the UFFD-registered mapping. We | |
| * expect that the first time we touch a page, it will result in a minor | |
| * fault. uffd_poll_thread will resolve the fault by bit-flipping the | |
| * page's contents, and then issuing a CONTINUE ioctl. | |
| */ | |
| check_memory_contents(gopts, gopts->area_dst_alias); | |
| if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) | |
| err("pipe write"); | |
| if (pthread_join(uffd_mon, NULL)) | |
| err("join() failed"); | |
| if (test_collapse) { | |
| if (madvise(gopts->area_dst_alias, gopts->nr_pages * gopts->page_size, | |
| MADV_COLLAPSE)) { | |
| /* It's fine to fail for this one... */ | |
| uffd_test_skip("MADV_COLLAPSE failed"); | |
| return; | |
| } | |
| uffd_test_ops->check_pmd_mapping(gopts, | |
| gopts->area_dst, | |
| gopts->nr_pages * gopts->page_size / | |
| read_pmd_pagesize()); | |
| /* | |
| * This won't cause uffd-fault - it purely just makes sure there | |
| * was no corruption. | |
| */ | |
| check_memory_contents(gopts, gopts->area_dst_alias); | |
| } | |
| if (args.missing_faults != 0 || args.minor_faults != gopts->nr_pages) | |
| uffd_test_fail("stats check error"); | |
| else | |
| uffd_test_pass(); | |
| } | |
| void uffd_minor_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_minor_test_common(gopts, false, false); | |
| } | |
| void uffd_minor_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_minor_test_common(gopts, false, true); | |
| } | |
| void uffd_minor_collapse_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_minor_test_common(gopts, true, false); | |
| } | |
| static sigjmp_buf jbuf, *sigbuf; | |
| static void sighndl(int sig, siginfo_t *siginfo, void *ptr) | |
| { | |
| if (sig == SIGBUS) { | |
| if (sigbuf) | |
| siglongjmp(*sigbuf, 1); | |
| abort(); | |
| } | |
| } | |
| /* | |
| * For non-cooperative userfaultfd test we fork() a process that will | |
| * generate pagefaults, will mremap the area monitored by the | |
| * userfaultfd and at last this process will release the monitored | |
| * area. | |
| * For the anonymous and shared memory the area is divided into two | |
| * parts, the first part is accessed before mremap, and the second | |
| * part is accessed after mremap. Since hugetlbfs does not support | |
| * mremap, the entire monitored area is accessed in a single pass for | |
| * HUGETLB_TEST. | |
| * The release of the pages currently generates event for shmem and | |
| * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked | |
| * for hugetlb. | |
| * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register | |
| * monitored area, generate pagefaults and test that signal is delivered. | |
| * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2 | |
| * test robustness use case - we release monitored area, fork a process | |
| * that will generate pagefaults and verify signal is generated. | |
| * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal | |
| * feature. Using monitor thread, verify no userfault events are generated. | |
| */ | |
| static int faulting_process(uffd_global_test_opts_t *gopts, int signal_test, bool wp) | |
| { | |
| unsigned long nr, i; | |
| unsigned long long count; | |
| unsigned long split_nr_pages; | |
| unsigned long lastnr; | |
| struct sigaction act; | |
| volatile unsigned long signalled = 0; | |
| split_nr_pages = (gopts->nr_pages + 1) / 2; | |
| if (signal_test) { | |
| sigbuf = &jbuf; | |
| memset(&act, 0, sizeof(act)); | |
| act.sa_sigaction = sighndl; | |
| act.sa_flags = SA_SIGINFO; | |
| if (sigaction(SIGBUS, &act, 0)) | |
| err("sigaction"); | |
| lastnr = (unsigned long)-1; | |
| } | |
| for (nr = 0; nr < split_nr_pages; nr++) { | |
| volatile int steps = 1; | |
| unsigned long offset = nr * gopts->page_size; | |
| if (signal_test) { | |
| if (sigsetjmp(*sigbuf, 1) != 0) { | |
| if (steps == 1 && nr == lastnr) | |
| err("Signal repeated"); | |
| lastnr = nr; | |
| if (signal_test == 1) { | |
| if (steps == 1) { | |
| /* This is a MISSING request */ | |
| steps++; | |
| if (copy_page(gopts, offset, wp)) | |
| signalled++; | |
| } else { | |
| /* This is a WP request */ | |
| assert(steps == 2); | |
| wp_range(gopts->uffd, | |
| (__u64)gopts->area_dst + | |
| offset, | |
| gopts->page_size, false); | |
| } | |
| } else { | |
| signalled++; | |
| continue; | |
| } | |
| } | |
| } | |
| count = *area_count(gopts->area_dst, nr, gopts); | |
| if (count != gopts->count_verify[nr]) | |
| err("nr %lu memory corruption %llu %llu\n", | |
| nr, count, gopts->count_verify[nr]); | |
| /* | |
| * Trigger write protection if there is by writing | |
| * the same value back. | |
| */ | |
| *area_count(gopts->area_dst, nr, gopts) = count; | |
| } | |
| if (signal_test) | |
| return signalled != split_nr_pages; | |
| gopts->area_dst = mremap(gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| gopts->nr_pages * gopts->page_size, | |
| MREMAP_MAYMOVE | MREMAP_FIXED, | |
| gopts->area_src); | |
| if (gopts->area_dst == MAP_FAILED) | |
| err("mremap"); | |
| /* Reset area_src since we just clobbered it */ | |
| gopts->area_src = NULL; | |
| for (; nr < gopts->nr_pages; nr++) { | |
| count = *area_count(gopts->area_dst, nr, gopts); | |
| if (count != gopts->count_verify[nr]) { | |
| err("nr %lu memory corruption %llu %llu\n", | |
| nr, count, gopts->count_verify[nr]); | |
| } | |
| /* | |
| * Trigger write protection if there is by writing | |
| * the same value back. | |
| */ | |
| *area_count(gopts->area_dst, nr, gopts) = count; | |
| } | |
| uffd_test_ops->release_pages(gopts, gopts->area_dst); | |
| for (nr = 0; nr < gopts->nr_pages; nr++) | |
| for (i = 0; i < gopts->page_size; i++) | |
| if (*(gopts->area_dst + nr * gopts->page_size + i) != 0) | |
| err("page %lu offset %lu is not zero", nr, i); | |
| return 0; | |
| } | |
| static void uffd_sigbus_test_common(uffd_global_test_opts_t *gopts, bool wp) | |
| { | |
| unsigned long userfaults; | |
| pthread_t uffd_mon; | |
| pid_t pid; | |
| int err; | |
| char c = '\0'; | |
| struct uffd_args args = { 0 }; | |
| args.gopts = gopts; | |
| gopts->ready_for_fork = false; | |
| fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| true, wp, false)) | |
| err("register failure"); | |
| if (faulting_process(gopts, 1, wp)) | |
| err("faulting process failed"); | |
| uffd_test_ops->release_pages(gopts, gopts->area_dst); | |
| args.apply_wp = wp; | |
| if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) | |
| err("uffd_poll_thread create"); | |
| while (!gopts->ready_for_fork) | |
| ; /* Wait for the poll_thread to start executing before forking */ | |
| pid = fork(); | |
| if (pid < 0) | |
| err("fork"); | |
| if (!pid) | |
| exit(faulting_process(gopts, 2, wp)); | |
| waitpid(pid, &err, 0); | |
| if (err) | |
| err("faulting process failed"); | |
| if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) | |
| err("pipe write"); | |
| if (pthread_join(uffd_mon, (void **)&userfaults)) | |
| err("pthread_join()"); | |
| if (userfaults) | |
| uffd_test_fail("Signal test failed, userfaults: %ld", userfaults); | |
| else | |
| uffd_test_pass(); | |
| } | |
| static void uffd_sigbus_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_sigbus_test_common(gopts, false); | |
| } | |
| static void uffd_sigbus_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_sigbus_test_common(gopts, true); | |
| } | |
| static void uffd_events_test_common(uffd_global_test_opts_t *gopts, bool wp) | |
| { | |
| pthread_t uffd_mon; | |
| pid_t pid; | |
| int err; | |
| char c = '\0'; | |
| struct uffd_args args = { 0 }; | |
| args.gopts = gopts; | |
| gopts->ready_for_fork = false; | |
| fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| true, wp, false)) | |
| err("register failure"); | |
| args.apply_wp = wp; | |
| if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) | |
| err("uffd_poll_thread create"); | |
| while (!gopts->ready_for_fork) | |
| ; /* Wait for the poll_thread to start executing before forking */ | |
| pid = fork(); | |
| if (pid < 0) | |
| err("fork"); | |
| if (!pid) | |
| exit(faulting_process(gopts, 0, wp)); | |
| waitpid(pid, &err, 0); | |
| if (err) | |
| err("faulting process failed"); | |
| if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) | |
| err("pipe write"); | |
| if (pthread_join(uffd_mon, NULL)) | |
| err("pthread_join()"); | |
| if (args.missing_faults != gopts->nr_pages) | |
| uffd_test_fail("Fault counts wrong"); | |
| else | |
| uffd_test_pass(); | |
| } | |
| static void uffd_events_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_events_test_common(gopts, false); | |
| } | |
| static void uffd_events_wp_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| uffd_events_test_common(gopts, true); | |
| } | |
| static void retry_uffdio_zeropage(uffd_global_test_opts_t *gopts, | |
| struct uffdio_zeropage *uffdio_zeropage) | |
| { | |
| uffd_test_ops->alias_mapping(gopts, &uffdio_zeropage->range.start, | |
| uffdio_zeropage->range.len, | |
| 0); | |
| if (ioctl(gopts->uffd, UFFDIO_ZEROPAGE, uffdio_zeropage)) { | |
| if (uffdio_zeropage->zeropage != -EEXIST) | |
| err("UFFDIO_ZEROPAGE error: %"PRId64, | |
| (int64_t)uffdio_zeropage->zeropage); | |
| } else { | |
| err("UFFDIO_ZEROPAGE error: %"PRId64, | |
| (int64_t)uffdio_zeropage->zeropage); | |
| } | |
| } | |
| static bool do_uffdio_zeropage(uffd_global_test_opts_t *gopts, bool has_zeropage) | |
| { | |
| struct uffdio_zeropage uffdio_zeropage = { 0 }; | |
| int ret; | |
| __s64 res; | |
| uffdio_zeropage.range.start = (unsigned long) gopts->area_dst; | |
| uffdio_zeropage.range.len = gopts->page_size; | |
| uffdio_zeropage.mode = 0; | |
| ret = ioctl(gopts->uffd, UFFDIO_ZEROPAGE, &uffdio_zeropage); | |
| res = uffdio_zeropage.zeropage; | |
| if (ret) { | |
| /* real retval in ufdio_zeropage.zeropage */ | |
| if (has_zeropage) | |
| err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res); | |
| else if (res != -EINVAL) | |
| err("UFFDIO_ZEROPAGE not -EINVAL"); | |
| } else if (has_zeropage) { | |
| if (res != gopts->page_size) | |
| err("UFFDIO_ZEROPAGE unexpected size"); | |
| else | |
| retry_uffdio_zeropage(gopts, &uffdio_zeropage); | |
| return true; | |
| } else | |
| err("UFFDIO_ZEROPAGE succeeded"); | |
| return false; | |
| } | |
| /* | |
| * Registers a range with MISSING mode only for zeropage test. Return true | |
| * if UFFDIO_ZEROPAGE supported, false otherwise. Can't use uffd_register() | |
| * because we want to detect .ioctls along the way. | |
| */ | |
| static bool | |
| uffd_register_detect_zeropage(int uffd, void *addr, uint64_t len) | |
| { | |
| uint64_t ioctls = 0; | |
| if (uffd_register_with_ioctls(uffd, addr, len, true, | |
| false, false, &ioctls)) | |
| err("zeropage register fail"); | |
| return ioctls & (1 << _UFFDIO_ZEROPAGE); | |
| } | |
| /* exercise UFFDIO_ZEROPAGE */ | |
| static void uffd_zeropage_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| bool has_zeropage; | |
| int i; | |
| has_zeropage = uffd_register_detect_zeropage(gopts->uffd, | |
| gopts->area_dst, | |
| gopts->page_size); | |
| if (gopts->area_dst_alias) | |
| /* Ignore the retval; we already have it */ | |
| uffd_register_detect_zeropage(gopts->uffd, gopts->area_dst_alias, gopts->page_size); | |
| if (do_uffdio_zeropage(gopts, has_zeropage)) | |
| for (i = 0; i < gopts->page_size; i++) | |
| if (gopts->area_dst[i] != 0) | |
| err("data non-zero at offset %d\n", i); | |
| if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size)) | |
| err("unregister"); | |
| if (gopts->area_dst_alias && uffd_unregister(gopts->uffd, | |
| gopts->area_dst_alias, | |
| gopts->page_size)) | |
| err("unregister"); | |
| uffd_test_pass(); | |
| } | |
| static void uffd_register_poison(int uffd, void *addr, uint64_t len) | |
| { | |
| uint64_t ioctls = 0; | |
| uint64_t expected = (1 << _UFFDIO_COPY) | (1 << _UFFDIO_POISON); | |
| if (uffd_register_with_ioctls(uffd, addr, len, true, | |
| false, false, &ioctls)) | |
| err("poison register fail"); | |
| if ((ioctls & expected) != expected) | |
| err("registered area doesn't support COPY and POISON ioctls"); | |
| } | |
| static void do_uffdio_poison(uffd_global_test_opts_t *gopts, unsigned long offset) | |
| { | |
| struct uffdio_poison uffdio_poison = { 0 }; | |
| int ret; | |
| __s64 res; | |
| uffdio_poison.range.start = (unsigned long) gopts->area_dst + offset; | |
| uffdio_poison.range.len = gopts->page_size; | |
| uffdio_poison.mode = 0; | |
| ret = ioctl(gopts->uffd, UFFDIO_POISON, &uffdio_poison); | |
| res = uffdio_poison.updated; | |
| if (ret) | |
| err("UFFDIO_POISON error: %"PRId64, (int64_t)res); | |
| else if (res != gopts->page_size) | |
| err("UFFDIO_POISON unexpected size: %"PRId64, (int64_t)res); | |
| } | |
| static void uffd_poison_handle_fault(uffd_global_test_opts_t *gopts, | |
| struct uffd_msg *msg, | |
| struct uffd_args *args) | |
| { | |
| unsigned long offset; | |
| if (msg->event != UFFD_EVENT_PAGEFAULT) | |
| err("unexpected msg event %u", msg->event); | |
| if (msg->arg.pagefault.flags & | |
| (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR)) | |
| err("unexpected fault type %llu", msg->arg.pagefault.flags); | |
| offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst; | |
| offset &= ~(gopts->page_size-1); | |
| /* Odd pages -> copy zeroed page; even pages -> poison. */ | |
| if (offset & gopts->page_size) | |
| copy_page(gopts, offset, false); | |
| else | |
| do_uffdio_poison(gopts, offset); | |
| } | |
| /* Make sure to cover odd/even, and minimum duplications */ | |
| #define UFFD_POISON_TEST_NPAGES 4 | |
| static void uffd_poison_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) | |
| { | |
| pthread_t uffd_mon; | |
| char c; | |
| struct uffd_args args = { 0 }; | |
| struct sigaction act = { 0 }; | |
| unsigned long nr_sigbus = 0; | |
| unsigned long nr, poison_pages = UFFD_POISON_TEST_NPAGES; | |
| if (gopts->nr_pages < poison_pages) { | |
| uffd_test_skip("Too less pages for POISON test"); | |
| return; | |
| } | |
| args.gopts = gopts; | |
| fcntl(gopts->uffd, F_SETFL, gopts->uffd_flags | O_NONBLOCK); | |
| uffd_register_poison(gopts->uffd, gopts->area_dst, poison_pages * gopts->page_size); | |
| memset(gopts->area_src, 0, poison_pages * gopts->page_size); | |
| args.handle_fault = uffd_poison_handle_fault; | |
| if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) | |
| err("uffd_poll_thread create"); | |
| sigbuf = &jbuf; | |
| act.sa_sigaction = sighndl; | |
| act.sa_flags = SA_SIGINFO; | |
| if (sigaction(SIGBUS, &act, 0)) | |
| err("sigaction"); | |
| for (nr = 0; nr < poison_pages; ++nr) { | |
| unsigned long offset = nr * gopts->page_size; | |
| const char *bytes = (const char *) gopts->area_dst + offset; | |
| const char *i; | |
| if (sigsetjmp(*sigbuf, 1)) { | |
| /* | |
| * Access below triggered a SIGBUS, which was caught by | |
| * sighndl, which then jumped here. Count this SIGBUS, | |
| * and move on to next page. | |
| */ | |
| ++nr_sigbus; | |
| continue; | |
| } | |
| for (i = bytes; i < bytes + gopts->page_size; ++i) { | |
| if (*i) | |
| err("nonzero byte in area_dst (%p) at %p: %u", | |
| gopts->area_dst, i, *i); | |
| } | |
| } | |
| if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) | |
| err("pipe write"); | |
| if (pthread_join(uffd_mon, NULL)) | |
| err("pthread_join()"); | |
| if (nr_sigbus != poison_pages / 2) | |
| err("expected to receive %lu SIGBUS, actually received %lu", | |
| poison_pages / 2, nr_sigbus); | |
| uffd_test_pass(); | |
| } | |
| static void | |
| uffd_move_handle_fault_common(uffd_global_test_opts_t *gopts, | |
| struct uffd_msg *msg, | |
| struct uffd_args *args, | |
| unsigned long len) | |
| { | |
| unsigned long offset; | |
| if (msg->event != UFFD_EVENT_PAGEFAULT) | |
| err("unexpected msg event %u", msg->event); | |
| if (msg->arg.pagefault.flags & | |
| (UFFD_PAGEFAULT_FLAG_WP | UFFD_PAGEFAULT_FLAG_MINOR | UFFD_PAGEFAULT_FLAG_WRITE)) | |
| err("unexpected fault type %llu", msg->arg.pagefault.flags); | |
| offset = (char *)(unsigned long)msg->arg.pagefault.address - gopts->area_dst; | |
| offset &= ~(len-1); | |
| if (move_page(gopts, offset, len)) | |
| args->missing_faults++; | |
| } | |
| static void uffd_move_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg, | |
| struct uffd_args *args) | |
| { | |
| uffd_move_handle_fault_common(gopts, msg, args, gopts->page_size); | |
| } | |
| static void uffd_move_pmd_handle_fault(uffd_global_test_opts_t *gopts, struct uffd_msg *msg, | |
| struct uffd_args *args) | |
| { | |
| uffd_move_handle_fault_common(gopts, msg, args, read_pmd_pagesize()); | |
| } | |
| static void | |
| uffd_move_test_common(uffd_global_test_opts_t *gopts, | |
| uffd_test_args_t *targs, | |
| unsigned long chunk_size, | |
| void (*handle_fault)(struct uffd_global_test_opts *gopts, | |
| struct uffd_msg *msg, struct uffd_args *args) | |
| ) | |
| { | |
| unsigned long nr; | |
| pthread_t uffd_mon; | |
| char c = '\0'; | |
| unsigned long long count; | |
| struct uffd_args args = { 0 }; | |
| char *orig_area_src = NULL, *orig_area_dst = NULL; | |
| unsigned long step_size, step_count; | |
| unsigned long src_offs = 0; | |
| unsigned long dst_offs = 0; | |
| args.gopts = gopts; | |
| /* Prevent source pages from being mapped more than once */ | |
| if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_DONTFORK)) | |
| err("madvise(MADV_DONTFORK) failure"); | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| true, false, false)) | |
| err("register failure"); | |
| args.handle_fault = handle_fault; | |
| if (pthread_create(&uffd_mon, NULL, uffd_poll_thread, &args)) | |
| err("uffd_poll_thread create"); | |
| step_size = chunk_size / gopts->page_size; | |
| step_count = gopts->nr_pages / step_size; | |
| if (chunk_size > gopts->page_size) { | |
| char *aligned_src = ALIGN_UP(gopts->area_src, chunk_size); | |
| char *aligned_dst = ALIGN_UP(gopts->area_dst, chunk_size); | |
| if (aligned_src != gopts->area_src || aligned_dst != gopts->area_dst) { | |
| src_offs = (aligned_src - gopts->area_src) / gopts->page_size; | |
| dst_offs = (aligned_dst - gopts->area_dst) / gopts->page_size; | |
| step_count--; | |
| } | |
| orig_area_src = gopts->area_src; | |
| orig_area_dst = gopts->area_dst; | |
| gopts->area_src = aligned_src; | |
| gopts->area_dst = aligned_dst; | |
| } | |
| /* | |
| * Read each of the pages back using the UFFD-registered mapping. We | |
| * expect that the first time we touch a page, it will result in a missing | |
| * fault. uffd_poll_thread will resolve the fault by moving source | |
| * page to destination. | |
| */ | |
| for (nr = 0; nr < step_count * step_size; nr += step_size) { | |
| unsigned long i; | |
| /* Check area_src content */ | |
| for (i = 0; i < step_size; i++) { | |
| count = *area_count(gopts->area_src, nr + i, gopts); | |
| if (count != gopts->count_verify[src_offs + nr + i]) | |
| err("nr %lu source memory invalid %llu %llu\n", | |
| nr + i, count, gopts->count_verify[src_offs + nr + i]); | |
| } | |
| /* Faulting into area_dst should move the page or the huge page */ | |
| for (i = 0; i < step_size; i++) { | |
| count = *area_count(gopts->area_dst, nr + i, gopts); | |
| if (count != gopts->count_verify[dst_offs + nr + i]) | |
| err("nr %lu memory corruption %llu %llu\n", | |
| nr, count, gopts->count_verify[dst_offs + nr + i]); | |
| } | |
| /* Re-check area_src content which should be empty */ | |
| for (i = 0; i < step_size; i++) { | |
| count = *area_count(gopts->area_src, nr + i, gopts); | |
| if (count != 0) | |
| err("nr %lu move failed %llu %llu\n", | |
| nr, count, gopts->count_verify[src_offs + nr + i]); | |
| } | |
| } | |
| if (chunk_size > gopts->page_size) { | |
| gopts->area_src = orig_area_src; | |
| gopts->area_dst = orig_area_dst; | |
| } | |
| if (write(gopts->pipefd[1], &c, sizeof(c)) != sizeof(c)) | |
| err("pipe write"); | |
| if (pthread_join(uffd_mon, NULL)) | |
| err("join() failed"); | |
| if (args.missing_faults != step_count || args.minor_faults != 0) | |
| uffd_test_fail("stats check error"); | |
| else | |
| uffd_test_pass(); | |
| } | |
| static void uffd_move_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) | |
| { | |
| uffd_move_test_common(gopts, targs, gopts->page_size, uffd_move_handle_fault); | |
| } | |
| static void uffd_move_pmd_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) | |
| { | |
| if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) | |
| err("madvise(MADV_HUGEPAGE) failure"); | |
| uffd_move_test_common(gopts, targs, read_pmd_pagesize(), | |
| uffd_move_pmd_handle_fault); | |
| } | |
| static void uffd_move_pmd_split_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) | |
| { | |
| if (madvise(gopts->area_dst, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) | |
| err("madvise(MADV_NOHUGEPAGE) failure"); | |
| uffd_move_test_common(gopts, targs, read_pmd_pagesize(), | |
| uffd_move_pmd_handle_fault); | |
| } | |
| static bool | |
| uffdio_verify_results(const char *name, int ret, int error, long result) | |
| { | |
| /* | |
| * Should always return -1 with errno=EAGAIN, with corresponding | |
| * result field updated in ioctl() args to be -EAGAIN too | |
| * (e.g. copy.copy field for UFFDIO_COPY). | |
| */ | |
| if (ret != -1) { | |
| uffd_test_fail("%s should have returned -1", name); | |
| return false; | |
| } | |
| if (error != EAGAIN) { | |
| uffd_test_fail("%s should have errno==EAGAIN", name); | |
| return false; | |
| } | |
| if (result != -EAGAIN) { | |
| uffd_test_fail("%s should have been updated for -EAGAIN", | |
| name); | |
| return false; | |
| } | |
| return true; | |
| } | |
| /* | |
| * This defines a function to test one ioctl. Note that here "field" can | |
| * be 1 or anything not -EAGAIN. With that initial value set, we can | |
| * verify later that it should be updated by kernel (when -EAGAIN | |
| * returned), by checking whether it is also updated to -EAGAIN. | |
| */ | |
| #define DEFINE_MMAP_CHANGING_TEST(name, ioctl_name, field) \ | |
| static bool uffdio_mmap_changing_test_##name(int fd) \ | |
| { \ | |
| int ret; \ | |
| struct uffdio_##name args = { \ | |
| .field = 1, \ | |
| }; \ | |
| ret = ioctl(fd, ioctl_name, &args); \ | |
| return uffdio_verify_results(#ioctl_name, ret, errno, args.field); \ | |
| } | |
| DEFINE_MMAP_CHANGING_TEST(zeropage, UFFDIO_ZEROPAGE, zeropage) | |
| DEFINE_MMAP_CHANGING_TEST(copy, UFFDIO_COPY, copy) | |
| DEFINE_MMAP_CHANGING_TEST(move, UFFDIO_MOVE, move) | |
| DEFINE_MMAP_CHANGING_TEST(poison, UFFDIO_POISON, updated) | |
| DEFINE_MMAP_CHANGING_TEST(continue, UFFDIO_CONTINUE, mapped) | |
| typedef enum { | |
| /* We actually do not care about any state except UNINTERRUPTIBLE.. */ | |
| THR_STATE_UNKNOWN = 0, | |
| THR_STATE_UNINTERRUPTIBLE, | |
| } thread_state; | |
| typedef struct { | |
| uffd_global_test_opts_t *gopts; | |
| volatile pid_t *pid; | |
| } mmap_changing_thread_args; | |
| static void sleep_short(void) | |
| { | |
| usleep(1000); | |
| } | |
| static thread_state thread_state_get(pid_t tid) | |
| { | |
| const char *header = "State:\t"; | |
| char tmp[256], *p, c; | |
| FILE *fp; | |
| snprintf(tmp, sizeof(tmp), "/proc/%d/status", tid); | |
| fp = fopen(tmp, "r"); | |
| if (!fp) | |
| return THR_STATE_UNKNOWN; | |
| while (fgets(tmp, sizeof(tmp), fp)) { | |
| p = strstr(tmp, header); | |
| if (p) { | |
| /* For example, "State:\tD (disk sleep)" */ | |
| c = *(p + strlen(header)); | |
| return c == 'D' ? | |
| THR_STATE_UNINTERRUPTIBLE : THR_STATE_UNKNOWN; | |
| } | |
| } | |
| return THR_STATE_UNKNOWN; | |
| } | |
| static void thread_state_until(pid_t tid, thread_state state) | |
| { | |
| thread_state s; | |
| do { | |
| s = thread_state_get(tid); | |
| sleep_short(); | |
| } while (s != state); | |
| } | |
| static void *uffd_mmap_changing_thread(void *opaque) | |
| { | |
| mmap_changing_thread_args *args = opaque; | |
| uffd_global_test_opts_t *gopts = args->gopts; | |
| volatile pid_t *pid = args->pid; | |
| int ret; | |
| /* Unfortunately, it's only fetch-able from the thread itself.. */ | |
| assert(*pid == 0); | |
| *pid = syscall(SYS_gettid); | |
| /* Inject an event, this will hang solid until the event read */ | |
| ret = madvise(gopts->area_dst, gopts->page_size, MADV_REMOVE); | |
| if (ret) | |
| err("madvise(MADV_REMOVE) failed"); | |
| return NULL; | |
| } | |
| static void uffd_consume_message(uffd_global_test_opts_t *gopts) | |
| { | |
| struct uffd_msg msg = { 0 }; | |
| while (uffd_read_msg(gopts, &msg)); | |
| } | |
| static void uffd_mmap_changing_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *targs) | |
| { | |
| /* | |
| * This stores the real PID (which can be different from how tid is | |
| * defined..) for the child thread, 0 means not initialized. | |
| */ | |
| pid_t pid = 0; | |
| pthread_t tid; | |
| int ret; | |
| mmap_changing_thread_args args = { gopts, &pid }; | |
| if (uffd_register(gopts->uffd, gopts->area_dst, gopts->nr_pages * gopts->page_size, | |
| true, false, false)) | |
| err("uffd_register() failed"); | |
| /* Create a thread to generate the racy event */ | |
| ret = pthread_create(&tid, NULL, uffd_mmap_changing_thread, &args); | |
| if (ret) | |
| err("pthread_create() failed"); | |
| /* | |
| * Wait until the thread setup the pid. Use volatile to make sure | |
| * it reads from RAM not regs. | |
| */ | |
| while (!(volatile pid_t)pid) | |
| sleep_short(); | |
| /* Wait until the thread hangs at REMOVE event */ | |
| thread_state_until(pid, THR_STATE_UNINTERRUPTIBLE); | |
| if (!uffdio_mmap_changing_test_copy(gopts->uffd)) | |
| return; | |
| if (!uffdio_mmap_changing_test_zeropage(gopts->uffd)) | |
| return; | |
| if (!uffdio_mmap_changing_test_move(gopts->uffd)) | |
| return; | |
| if (!uffdio_mmap_changing_test_poison(gopts->uffd)) | |
| return; | |
| if (!uffdio_mmap_changing_test_continue(gopts->uffd)) | |
| return; | |
| /* | |
| * All succeeded above! Recycle everything. Start by reading the | |
| * event so as to kick the thread roll again.. | |
| */ | |
| uffd_consume_message(gopts); | |
| ret = pthread_join(tid, NULL); | |
| assert(ret == 0); | |
| uffd_test_pass(); | |
| } | |
| static int prevent_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg) | |
| { | |
| /* This should be done before source area is populated */ | |
| if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_NOHUGEPAGE)) { | |
| /* Ignore only if CONFIG_TRANSPARENT_HUGEPAGE=n */ | |
| if (errno != EINVAL) { | |
| if (errmsg) | |
| *errmsg = "madvise(MADV_NOHUGEPAGE) failed"; | |
| return -errno; | |
| } | |
| } | |
| return 0; | |
| } | |
| static int request_hugepages(uffd_global_test_opts_t *gopts, const char **errmsg) | |
| { | |
| /* This should be done before source area is populated */ | |
| if (madvise(gopts->area_src, gopts->nr_pages * gopts->page_size, MADV_HUGEPAGE)) { | |
| if (errmsg) { | |
| *errmsg = (errno == EINVAL) ? | |
| "CONFIG_TRANSPARENT_HUGEPAGE is not set" : | |
| "madvise(MADV_HUGEPAGE) failed"; | |
| } | |
| return -errno; | |
| } | |
| return 0; | |
| } | |
| struct uffd_test_case_ops uffd_move_test_case_ops = { | |
| .post_alloc = prevent_hugepages, | |
| }; | |
| struct uffd_test_case_ops uffd_move_test_pmd_case_ops = { | |
| .post_alloc = request_hugepages, | |
| }; | |
| /* | |
| * Test the returned uffdio_register.ioctls with different register modes. | |
| * Note that _UFFDIO_ZEROPAGE is tested separately in the zeropage test. | |
| */ | |
| static void | |
| do_register_ioctls_test(uffd_global_test_opts_t *gopts, | |
| uffd_test_args_t *args, | |
| bool miss, | |
| bool wp, | |
| bool minor) | |
| { | |
| uint64_t ioctls = 0, expected = BIT_ULL(_UFFDIO_WAKE); | |
| mem_type_t *mem_type = args->mem_type; | |
| int ret; | |
| ret = uffd_register_with_ioctls(gopts->uffd, gopts->area_dst, gopts->page_size, | |
| miss, wp, minor, &ioctls); | |
| /* | |
| * Handle special cases of UFFDIO_REGISTER here where it should | |
| * just fail with -EINVAL first.. | |
| * | |
| * Case 1: register MINOR on anon | |
| * Case 2: register with no mode selected | |
| */ | |
| if ((minor && (mem_type->mem_flag == MEM_ANON)) || | |
| (!miss && !wp && !minor)) { | |
| if (ret != -EINVAL) | |
| err("register (miss=%d, wp=%d, minor=%d) failed " | |
| "with wrong errno=%d", miss, wp, minor, ret); | |
| return; | |
| } | |
| /* UFFDIO_REGISTER should succeed, then check ioctls returned */ | |
| if (miss) | |
| expected |= BIT_ULL(_UFFDIO_COPY); | |
| if (wp) | |
| expected |= BIT_ULL(_UFFDIO_WRITEPROTECT); | |
| if (minor) | |
| expected |= BIT_ULL(_UFFDIO_CONTINUE); | |
| if ((ioctls & expected) != expected) | |
| err("unexpected uffdio_register.ioctls " | |
| "(miss=%d, wp=%d, minor=%d): expected=0x%"PRIx64", " | |
| "returned=0x%"PRIx64, miss, wp, minor, expected, ioctls); | |
| if (uffd_unregister(gopts->uffd, gopts->area_dst, gopts->page_size)) | |
| err("unregister"); | |
| } | |
| static void uffd_register_ioctls_test(uffd_global_test_opts_t *gopts, uffd_test_args_t *args) | |
| { | |
| int miss, wp, minor; | |
| for (miss = 0; miss <= 1; miss++) | |
| for (wp = 0; wp <= 1; wp++) | |
| for (minor = 0; minor <= 1; minor++) | |
| do_register_ioctls_test(gopts, args, miss, wp, minor); | |
| uffd_test_pass(); | |
| } | |
| uffd_test_case_t uffd_tests[] = { | |
| { | |
| /* Test returned uffdio_register.ioctls. */ | |
| .name = "register-ioctls", | |
| .uffd_fn = uffd_register_ioctls_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_MISSING_HUGETLBFS | | |
| UFFD_FEATURE_MISSING_SHMEM | | |
| UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM | | |
| UFFD_FEATURE_MINOR_HUGETLBFS | | |
| UFFD_FEATURE_MINOR_SHMEM, | |
| }, | |
| { | |
| .name = "zeropage", | |
| .uffd_fn = uffd_zeropage_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = 0, | |
| }, | |
| { | |
| .name = "move", | |
| .uffd_fn = uffd_move_test, | |
| .mem_targets = MEM_ANON, | |
| .uffd_feature_required = UFFD_FEATURE_MOVE, | |
| .test_case_ops = &uffd_move_test_case_ops, | |
| }, | |
| { | |
| .name = "move-pmd", | |
| .uffd_fn = uffd_move_pmd_test, | |
| .mem_targets = MEM_ANON, | |
| .uffd_feature_required = UFFD_FEATURE_MOVE, | |
| .test_case_ops = &uffd_move_test_pmd_case_ops, | |
| }, | |
| { | |
| .name = "move-pmd-split", | |
| .uffd_fn = uffd_move_pmd_split_test, | |
| .mem_targets = MEM_ANON, | |
| .uffd_feature_required = UFFD_FEATURE_MOVE, | |
| .test_case_ops = &uffd_move_test_pmd_case_ops, | |
| }, | |
| { | |
| .name = "wp-fork", | |
| .uffd_fn = uffd_wp_fork_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM, | |
| }, | |
| { | |
| .name = "wp-fork-with-event", | |
| .uffd_fn = uffd_wp_fork_with_event_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM | | |
| /* when set, child process should inherit uffd-wp bits */ | |
| UFFD_FEATURE_EVENT_FORK, | |
| }, | |
| { | |
| .name = "wp-fork-pin", | |
| .uffd_fn = uffd_wp_fork_pin_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM, | |
| }, | |
| { | |
| .name = "wp-fork-pin-with-event", | |
| .uffd_fn = uffd_wp_fork_pin_with_event_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM | | |
| /* when set, child process should inherit uffd-wp bits */ | |
| UFFD_FEATURE_EVENT_FORK, | |
| }, | |
| { | |
| .name = "wp-unpopulated", | |
| .uffd_fn = uffd_wp_unpopulated_test, | |
| .mem_targets = MEM_ANON, | |
| .uffd_feature_required = | |
| UFFD_FEATURE_PAGEFAULT_FLAG_WP | UFFD_FEATURE_WP_UNPOPULATED, | |
| }, | |
| { | |
| .name = "minor", | |
| .uffd_fn = uffd_minor_test, | |
| .mem_targets = MEM_SHMEM | MEM_HUGETLB, | |
| .uffd_feature_required = | |
| UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM, | |
| }, | |
| { | |
| .name = "minor-wp", | |
| .uffd_fn = uffd_minor_wp_test, | |
| .mem_targets = MEM_SHMEM | MEM_HUGETLB, | |
| .uffd_feature_required = | |
| UFFD_FEATURE_MINOR_HUGETLBFS | UFFD_FEATURE_MINOR_SHMEM | | |
| UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| /* | |
| * HACK: here we leveraged WP_UNPOPULATED to detect whether | |
| * minor mode supports wr-protect. There's no feature flag | |
| * for it so this is the best we can test against. | |
| */ | |
| UFFD_FEATURE_WP_UNPOPULATED, | |
| }, | |
| { | |
| .name = "minor-collapse", | |
| .uffd_fn = uffd_minor_collapse_test, | |
| /* MADV_COLLAPSE only works with shmem */ | |
| .mem_targets = MEM_SHMEM, | |
| /* We can't test MADV_COLLAPSE, so try our luck */ | |
| .uffd_feature_required = UFFD_FEATURE_MINOR_SHMEM, | |
| }, | |
| { | |
| .name = "sigbus", | |
| .uffd_fn = uffd_sigbus_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_SIGBUS | | |
| UFFD_FEATURE_EVENT_FORK, | |
| }, | |
| { | |
| .name = "sigbus-wp", | |
| .uffd_fn = uffd_sigbus_wp_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_SIGBUS | | |
| UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM, | |
| }, | |
| { | |
| .name = "events", | |
| .uffd_fn = uffd_events_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_EVENT_FORK | | |
| UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE, | |
| }, | |
| { | |
| .name = "events-wp", | |
| .uffd_fn = uffd_events_wp_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_EVENT_FORK | | |
| UFFD_FEATURE_EVENT_REMAP | UFFD_FEATURE_EVENT_REMOVE | | |
| UFFD_FEATURE_PAGEFAULT_FLAG_WP | | |
| UFFD_FEATURE_WP_HUGETLBFS_SHMEM, | |
| }, | |
| { | |
| .name = "poison", | |
| .uffd_fn = uffd_poison_test, | |
| .mem_targets = MEM_ALL, | |
| .uffd_feature_required = UFFD_FEATURE_POISON, | |
| }, | |
| { | |
| .name = "mmap-changing", | |
| .uffd_fn = uffd_mmap_changing_test, | |
| /* | |
| * There's no point running this test over all mem types as | |
| * they share the same code paths. | |
| * | |
| * Choose shmem for simplicity, because (1) shmem supports | |
| * MINOR mode to cover UFFDIO_CONTINUE, and (2) shmem is | |
| * almost always available (unlike hugetlb). Here we | |
| * abused SHMEM for UFFDIO_MOVE, but the test we want to | |
| * cover doesn't yet need the correct memory type.. | |
| */ | |
| .mem_targets = MEM_SHMEM, | |
| /* | |
| * Any UFFD_FEATURE_EVENT_* should work to trigger the | |
| * race logically, but choose the simplest (REMOVE). | |
| * | |
| * Meanwhile, since we'll cover quite a few new ioctl()s | |
| * (CONTINUE, POISON, MOVE), skip this test for old kernels | |
| * by choosing all of them. | |
| */ | |
| .uffd_feature_required = UFFD_FEATURE_EVENT_REMOVE | | |
| UFFD_FEATURE_MOVE | UFFD_FEATURE_POISON | | |
| UFFD_FEATURE_MINOR_SHMEM, | |
| }, | |
| }; | |
| static void usage(const char *prog) | |
| { | |
| printf("usage: %s [-f TESTNAME]\n", prog); | |
| puts(""); | |
| puts(" -f: test name to filter (e.g., event)"); | |
| puts(" -h: show the help msg"); | |
| puts(" -l: list tests only"); | |
| puts(""); | |
| exit(KSFT_FAIL); | |
| } | |
| int main(int argc, char *argv[]) | |
| { | |
| int n_tests = sizeof(uffd_tests) / sizeof(uffd_test_case_t); | |
| int n_mems = sizeof(mem_types) / sizeof(mem_type_t); | |
| const char *test_filter = NULL; | |
| bool list_only = false; | |
| uffd_test_case_t *test; | |
| mem_type_t *mem_type; | |
| uffd_test_args_t args; | |
| const char *errmsg; | |
| int has_uffd, opt; | |
| int i, j; | |
| while ((opt = getopt(argc, argv, "f:hl")) != -1) { | |
| switch (opt) { | |
| case 'f': | |
| test_filter = optarg; | |
| break; | |
| case 'l': | |
| list_only = true; | |
| break; | |
| case 'h': | |
| default: | |
| /* Unknown */ | |
| usage(argv[0]); | |
| break; | |
| } | |
| } | |
| if (!test_filter && !list_only) { | |
| has_uffd = test_uffd_api(false); | |
| has_uffd |= test_uffd_api(true); | |
| if (!has_uffd) { | |
| printf("Userfaultfd not supported or unprivileged, skip all tests\n"); | |
| exit(KSFT_SKIP); | |
| } | |
| } | |
| for (i = 0; i < n_tests; i++) { | |
| test = &uffd_tests[i]; | |
| if (test_filter && !strstr(test->name, test_filter)) | |
| continue; | |
| if (list_only) { | |
| printf("%s\n", test->name); | |
| continue; | |
| } | |
| for (j = 0; j < n_mems; j++) { | |
| mem_type = &mem_types[j]; | |
| /* Initialize global test options */ | |
| uffd_global_test_opts_t gopts = { 0 }; | |
| gopts.map_shared = mem_type->shared; | |
| uffd_test_ops = mem_type->mem_ops; | |
| uffd_test_case_ops = test->test_case_ops; | |
| if (mem_type->mem_flag & (MEM_HUGETLB_PRIVATE | MEM_HUGETLB)) { | |
| gopts.page_size = default_huge_page_size(); | |
| if (gopts.page_size == 0) { | |
| uffd_test_skip("huge page size is 0, feature missing?"); | |
| continue; | |
| } | |
| } else { | |
| gopts.page_size = psize(); | |
| } | |
| /* Ensure we have at least 2 pages */ | |
| gopts.nr_pages = MAX(UFFD_TEST_MEM_SIZE, gopts.page_size * 2) | |
| / gopts.page_size; | |
| gopts.nr_parallel = 1; | |
| /* Initialize test arguments */ | |
| args.mem_type = mem_type; | |
| if (!(test->mem_targets & mem_type->mem_flag)) | |
| continue; | |
| uffd_test_start("%s on %s", test->name, mem_type->name); | |
| if (!uffd_feature_supported(test)) { | |
| uffd_test_skip("feature missing"); | |
| continue; | |
| } | |
| if (uffd_test_ctx_init(&gopts, test->uffd_feature_required, &errmsg)) { | |
| uffd_test_skip(errmsg); | |
| continue; | |
| } | |
| test->uffd_fn(&gopts, &args); | |
| uffd_test_ctx_clear(&gopts); | |
| } | |
| } | |
| if (!list_only) | |
| uffd_test_report(); | |
| return ksft_get_fail_cnt() ? KSFT_FAIL : KSFT_PASS; | |
| } | |
| #else /* __NR_userfaultfd */ | |
| #warning "missing __NR_userfaultfd definition" | |
| int main(void) | |
| { | |
| printf("Skipping %s (missing __NR_userfaultfd)\n", __file__); | |
| return KSFT_SKIP; | |
| } | |
| #endif /* __NR_userfaultfd */ |