blob: 108c37fca78279c06e896afcc411220996872760 [file] [log] [blame]
Dan Williamsab68f262016-05-18 09:15:08 -07001/*
Dan Williams7b6be842017-04-11 09:49:49 -07002 * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved.
Dan Williamsab68f262016-05-18 09:15:08 -07003 *
4 * This program is free software; you can redistribute it and/or modify
5 * it under the terms of version 2 of the GNU General Public License as
6 * published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 */
13#include <linux/pagemap.h>
14#include <linux/module.h>
15#include <linux/device.h>
16#include <linux/pfn_t.h>
Dan Williamsba09c012016-07-24 15:55:42 -070017#include <linux/cdev.h>
Dan Williamsab68f262016-05-18 09:15:08 -070018#include <linux/slab.h>
19#include <linux/dax.h>
20#include <linux/fs.h>
21#include <linux/mm.h>
Dave Jiangef842302018-04-19 13:39:43 -070022#include <linux/mman.h>
Dan Williams73616362017-05-04 23:38:43 -070023#include "dax-private.h"
Dan Williamsccdb07f2016-08-06 16:05:06 -070024#include "dax.h"
Dan Williamsab68f262016-05-18 09:15:08 -070025
Dan Williamsab68f262016-05-18 09:15:08 -070026static struct class *dax_class;
Dan Williamsab68f262016-05-18 09:15:08 -070027
Dan Williams73616362017-05-04 23:38:43 -070028/*
29 * Rely on the fact that drvdata is set before the attributes are
30 * registered, and that the attributes are unregistered before drvdata
31 * is cleared to assume that drvdata is always valid.
Dan Williamsab68f262016-05-18 09:15:08 -070032 */
Dan Williamsd7fe1a62016-12-17 14:50:04 -080033static ssize_t id_show(struct device *dev,
34 struct device_attribute *attr, char *buf)
35{
Dan Williams73616362017-05-04 23:38:43 -070036 struct dax_region *dax_region = dev_get_drvdata(dev);
Dan Williamsd7fe1a62016-12-17 14:50:04 -080037
Dan Williams73616362017-05-04 23:38:43 -070038 return sprintf(buf, "%d\n", dax_region->id);
Dan Williamsd7fe1a62016-12-17 14:50:04 -080039}
40static DEVICE_ATTR_RO(id);
41
42static ssize_t region_size_show(struct device *dev,
43 struct device_attribute *attr, char *buf)
44{
Dan Williams73616362017-05-04 23:38:43 -070045 struct dax_region *dax_region = dev_get_drvdata(dev);
Dan Williamsd7fe1a62016-12-17 14:50:04 -080046
Dan Williams73616362017-05-04 23:38:43 -070047 return sprintf(buf, "%llu\n", (unsigned long long)
48 resource_size(&dax_region->res));
Dan Williamsd7fe1a62016-12-17 14:50:04 -080049}
50static struct device_attribute dev_attr_region_size = __ATTR(size, 0444,
51 region_size_show, NULL);
52
53static ssize_t align_show(struct device *dev,
54 struct device_attribute *attr, char *buf)
55{
Dan Williams73616362017-05-04 23:38:43 -070056 struct dax_region *dax_region = dev_get_drvdata(dev);
Dan Williamsd7fe1a62016-12-17 14:50:04 -080057
Dan Williams73616362017-05-04 23:38:43 -070058 return sprintf(buf, "%u\n", dax_region->align);
Dan Williamsd7fe1a62016-12-17 14:50:04 -080059}
60static DEVICE_ATTR_RO(align);
61
62static struct attribute *dax_region_attributes[] = {
63 &dev_attr_region_size.attr,
64 &dev_attr_align.attr,
65 &dev_attr_id.attr,
66 NULL,
67};
68
69static const struct attribute_group dax_region_attribute_group = {
70 .name = "dax_region",
71 .attrs = dax_region_attributes,
72};
73
74static const struct attribute_group *dax_region_attribute_groups[] = {
75 &dax_region_attribute_group,
76 NULL,
77};
78
Dan Williamsab68f262016-05-18 09:15:08 -070079static void dax_region_free(struct kref *kref)
80{
81 struct dax_region *dax_region;
82
83 dax_region = container_of(kref, struct dax_region, kref);
84 kfree(dax_region);
85}
86
87void dax_region_put(struct dax_region *dax_region)
88{
89 kref_put(&dax_region->kref, dax_region_free);
90}
91EXPORT_SYMBOL_GPL(dax_region_put);
92
Dan Williamsd7fe1a62016-12-17 14:50:04 -080093static void dax_region_unregister(void *region)
94{
95 struct dax_region *dax_region = region;
96
97 sysfs_remove_groups(&dax_region->dev->kobj,
98 dax_region_attribute_groups);
99 dax_region_put(dax_region);
100}
101
Dan Williamsab68f262016-05-18 09:15:08 -0700102struct dax_region *alloc_dax_region(struct device *parent, int region_id,
103 struct resource *res, unsigned int align, void *addr,
104 unsigned long pfn_flags)
105{
106 struct dax_region *dax_region;
107
Dan Williamsd7fe1a62016-12-17 14:50:04 -0800108 /*
109 * The DAX core assumes that it can store its private data in
110 * parent->driver_data. This WARN is a reminder / safeguard for
111 * developers of device-dax drivers.
112 */
113 if (dev_get_drvdata(parent)) {
114 dev_WARN(parent, "dax core failed to setup private data\n");
115 return NULL;
116 }
117
Dan Williams9d2d01a2016-07-19 16:17:58 -0700118 if (!IS_ALIGNED(res->start, align)
119 || !IS_ALIGNED(resource_size(res), align))
120 return NULL;
Dan Williamsab68f262016-05-18 09:15:08 -0700121
Dan Williams9d2d01a2016-07-19 16:17:58 -0700122 dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL);
Dan Williamsab68f262016-05-18 09:15:08 -0700123 if (!dax_region)
124 return NULL;
125
Dan Williamsd7fe1a62016-12-17 14:50:04 -0800126 dev_set_drvdata(parent, dax_region);
Dan Williamsab68f262016-05-18 09:15:08 -0700127 memcpy(&dax_region->res, res, sizeof(*res));
128 dax_region->pfn_flags = pfn_flags;
129 kref_init(&dax_region->kref);
130 dax_region->id = region_id;
131 ida_init(&dax_region->ida);
132 dax_region->align = align;
133 dax_region->dev = parent;
134 dax_region->base = addr;
Dan Williamsd7fe1a62016-12-17 14:50:04 -0800135 if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) {
136 kfree(dax_region);
Luis de Bethencourt1c47a642018-01-23 11:36:32 +0000137 return NULL;
Dan Williamsd7fe1a62016-12-17 14:50:04 -0800138 }
Dan Williamsab68f262016-05-18 09:15:08 -0700139
Dan Williamsd7fe1a62016-12-17 14:50:04 -0800140 kref_get(&dax_region->kref);
141 if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region))
142 return NULL;
Dan Williamsab68f262016-05-18 09:15:08 -0700143 return dax_region;
144}
145EXPORT_SYMBOL_GPL(alloc_dax_region);
146
Dan Williams5f0694b2017-01-30 21:43:10 -0800147static struct dev_dax *to_dev_dax(struct device *dev)
Dan Williamsebd84d72016-08-11 00:41:51 -0700148{
Dan Williams5f0694b2017-01-30 21:43:10 -0800149 return container_of(dev, struct dev_dax, dev);
Dan Williamsebd84d72016-08-11 00:41:51 -0700150}
151
Dan Williamsab68f262016-05-18 09:15:08 -0700152static ssize_t size_show(struct device *dev,
153 struct device_attribute *attr, char *buf)
154{
Dan Williams5f0694b2017-01-30 21:43:10 -0800155 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williamsab68f262016-05-18 09:15:08 -0700156 unsigned long long size = 0;
157 int i;
158
Dan Williams5f0694b2017-01-30 21:43:10 -0800159 for (i = 0; i < dev_dax->num_resources; i++)
160 size += resource_size(&dev_dax->res[i]);
Dan Williamsab68f262016-05-18 09:15:08 -0700161
162 return sprintf(buf, "%llu\n", size);
163}
164static DEVICE_ATTR_RO(size);
165
Dan Williams5f0694b2017-01-30 21:43:10 -0800166static struct attribute *dev_dax_attributes[] = {
Dan Williamsab68f262016-05-18 09:15:08 -0700167 &dev_attr_size.attr,
168 NULL,
169};
170
Dan Williams5f0694b2017-01-30 21:43:10 -0800171static const struct attribute_group dev_dax_attribute_group = {
172 .attrs = dev_dax_attributes,
Dan Williamsab68f262016-05-18 09:15:08 -0700173};
174
175static const struct attribute_group *dax_attribute_groups[] = {
Dan Williams5f0694b2017-01-30 21:43:10 -0800176 &dev_dax_attribute_group,
Dan Williamsab68f262016-05-18 09:15:08 -0700177 NULL,
178};
179
Dan Williams5f0694b2017-01-30 21:43:10 -0800180static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
Dan Williamsdee41072016-05-14 12:20:44 -0700181 const char *func)
182{
Dan Williams5f0694b2017-01-30 21:43:10 -0800183 struct dax_region *dax_region = dev_dax->region;
184 struct device *dev = &dev_dax->dev;
Dan Williamsdee41072016-05-14 12:20:44 -0700185 unsigned long mask;
186
Dan Williams7b6be842017-04-11 09:49:49 -0700187 if (!dax_alive(dev_dax->dax_dev))
Dan Williamsdee41072016-05-14 12:20:44 -0700188 return -ENXIO;
189
Dan Williams4cb19352016-11-16 09:00:38 -0800190 /* prevent private mappings from being established */
Dan Williams325896f2016-12-06 17:03:35 -0800191 if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
Jeff Moyer5a14e912018-06-27 11:43:58 -0400192 dev_info_ratelimited(dev,
193 "%s: %s: fail, attempted private mapping\n",
Dan Williamsdee41072016-05-14 12:20:44 -0700194 current->comm, func);
195 return -EINVAL;
196 }
197
198 mask = dax_region->align - 1;
199 if (vma->vm_start & mask || vma->vm_end & mask) {
Jeff Moyer5a14e912018-06-27 11:43:58 -0400200 dev_info_ratelimited(dev,
201 "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
Dan Williamsdee41072016-05-14 12:20:44 -0700202 current->comm, func, vma->vm_start, vma->vm_end,
203 mask);
204 return -EINVAL;
205 }
206
207 if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV
208 && (vma->vm_flags & VM_DONTCOPY) == 0) {
Jeff Moyer5a14e912018-06-27 11:43:58 -0400209 dev_info_ratelimited(dev,
210 "%s: %s: fail, dax range requires MADV_DONTFORK\n",
Dan Williamsdee41072016-05-14 12:20:44 -0700211 current->comm, func);
212 return -EINVAL;
213 }
214
215 if (!vma_is_dax(vma)) {
Jeff Moyer5a14e912018-06-27 11:43:58 -0400216 dev_info_ratelimited(dev,
217 "%s: %s: fail, vma is not DAX capable\n",
Dan Williamsdee41072016-05-14 12:20:44 -0700218 current->comm, func);
219 return -EINVAL;
220 }
221
222 return 0;
223}
224
Dan Williams73616362017-05-04 23:38:43 -0700225/* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
226__weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
Dan Williamsdee41072016-05-14 12:20:44 -0700227 unsigned long size)
228{
229 struct resource *res;
Ross Zwisler0a3ff782017-10-18 12:21:55 -0600230 /* gcc-4.6.3-nolibc for i386 complains that this is uninitialized */
231 phys_addr_t uninitialized_var(phys);
Dan Williamsdee41072016-05-14 12:20:44 -0700232 int i;
233
Dan Williams5f0694b2017-01-30 21:43:10 -0800234 for (i = 0; i < dev_dax->num_resources; i++) {
235 res = &dev_dax->res[i];
Dan Williamsdee41072016-05-14 12:20:44 -0700236 phys = pgoff * PAGE_SIZE + res->start;
237 if (phys >= res->start && phys <= res->end)
238 break;
239 pgoff -= PHYS_PFN(resource_size(res));
240 }
241
Dan Williams5f0694b2017-01-30 21:43:10 -0800242 if (i < dev_dax->num_resources) {
243 res = &dev_dax->res[i];
Dan Williamsdee41072016-05-14 12:20:44 -0700244 if (phys + size - 1 <= res->end)
245 return phys;
246 }
247
248 return -1;
249}
250
Dan Williams5f0694b2017-01-30 21:43:10 -0800251static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
Dan Williamsdee41072016-05-14 12:20:44 -0700252{
Dan Williams5f0694b2017-01-30 21:43:10 -0800253 struct device *dev = &dev_dax->dev;
Dan Williamsdee41072016-05-14 12:20:44 -0700254 struct dax_region *dax_region;
255 int rc = VM_FAULT_SIGBUS;
256 phys_addr_t phys;
257 pfn_t pfn;
Dave Jiang0134ed42017-03-10 13:24:22 -0700258 unsigned int fault_size = PAGE_SIZE;
Dan Williamsdee41072016-05-14 12:20:44 -0700259
Dan Williams5f0694b2017-01-30 21:43:10 -0800260 if (check_vma(dev_dax, vmf->vma, __func__))
Dan Williamsdee41072016-05-14 12:20:44 -0700261 return VM_FAULT_SIGBUS;
262
Dan Williams5f0694b2017-01-30 21:43:10 -0800263 dax_region = dev_dax->region;
Dan Williamsdee41072016-05-14 12:20:44 -0700264 if (dax_region->align > PAGE_SIZE) {
Dan Williams6daaca52018-03-05 16:40:05 -0800265 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
266 dax_region->align, fault_size);
Dan Williamsdee41072016-05-14 12:20:44 -0700267 return VM_FAULT_SIGBUS;
268 }
269
Dave Jiang0134ed42017-03-10 13:24:22 -0700270 if (fault_size != dax_region->align)
271 return VM_FAULT_SIGBUS;
272
Dan Williams73616362017-05-04 23:38:43 -0700273 phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
Dan Williamsdee41072016-05-14 12:20:44 -0700274 if (phys == -1) {
Dan Williams6daaca52018-03-05 16:40:05 -0800275 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
Dan Williamsdee41072016-05-14 12:20:44 -0700276 return VM_FAULT_SIGBUS;
277 }
278
279 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
280
Dave Jiang11bac802017-02-24 14:56:41 -0800281 rc = vm_insert_mixed(vmf->vma, vmf->address, pfn);
Dan Williamsdee41072016-05-14 12:20:44 -0700282
283 if (rc == -ENOMEM)
284 return VM_FAULT_OOM;
285 if (rc < 0 && rc != -EBUSY)
286 return VM_FAULT_SIGBUS;
287
288 return VM_FAULT_NOPAGE;
289}
290
Dan Williams5f0694b2017-01-30 21:43:10 -0800291static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
Dan Williamsdee41072016-05-14 12:20:44 -0700292{
Dave Jiangd8a849e2017-02-22 15:40:03 -0800293 unsigned long pmd_addr = vmf->address & PMD_MASK;
Dan Williams5f0694b2017-01-30 21:43:10 -0800294 struct device *dev = &dev_dax->dev;
Dan Williamsdee41072016-05-14 12:20:44 -0700295 struct dax_region *dax_region;
296 phys_addr_t phys;
297 pgoff_t pgoff;
298 pfn_t pfn;
Dave Jiang0134ed42017-03-10 13:24:22 -0700299 unsigned int fault_size = PMD_SIZE;
Dan Williamsdee41072016-05-14 12:20:44 -0700300
Dan Williams5f0694b2017-01-30 21:43:10 -0800301 if (check_vma(dev_dax, vmf->vma, __func__))
Dan Williamsdee41072016-05-14 12:20:44 -0700302 return VM_FAULT_SIGBUS;
303
Dan Williams5f0694b2017-01-30 21:43:10 -0800304 dax_region = dev_dax->region;
Dan Williamsdee41072016-05-14 12:20:44 -0700305 if (dax_region->align > PMD_SIZE) {
Dan Williams6daaca52018-03-05 16:40:05 -0800306 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
307 dax_region->align, fault_size);
Dan Williamsdee41072016-05-14 12:20:44 -0700308 return VM_FAULT_SIGBUS;
309 }
310
311 /* dax pmd mappings require pfn_t_devmap() */
312 if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
Dan Williams6daaca52018-03-05 16:40:05 -0800313 dev_dbg(dev, "region lacks devmap flags\n");
Dan Williamsdee41072016-05-14 12:20:44 -0700314 return VM_FAULT_SIGBUS;
315 }
316
Dave Jiang0134ed42017-03-10 13:24:22 -0700317 if (fault_size < dax_region->align)
318 return VM_FAULT_SIGBUS;
319 else if (fault_size > dax_region->align)
320 return VM_FAULT_FALLBACK;
321
322 /* if we are outside of the VMA */
323 if (pmd_addr < vmf->vma->vm_start ||
324 (pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
325 return VM_FAULT_SIGBUS;
326
Dave Jiangf4200392017-02-22 15:40:06 -0800327 pgoff = linear_page_index(vmf->vma, pmd_addr);
Dan Williams73616362017-05-04 23:38:43 -0700328 phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
Dan Williamsdee41072016-05-14 12:20:44 -0700329 if (phys == -1) {
Dan Williams6daaca52018-03-05 16:40:05 -0800330 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
Dan Williamsdee41072016-05-14 12:20:44 -0700331 return VM_FAULT_SIGBUS;
332 }
333
334 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
335
Dave Jiangf4200392017-02-22 15:40:06 -0800336 return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn,
Dave Jiangd8a849e2017-02-22 15:40:03 -0800337 vmf->flags & FAULT_FLAG_WRITE);
Dan Williamsdee41072016-05-14 12:20:44 -0700338}
339
Dave Jiang9557fee2017-02-24 14:57:05 -0800340#ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
Dan Williams5f0694b2017-01-30 21:43:10 -0800341static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
Dave Jiang9557fee2017-02-24 14:57:05 -0800342{
343 unsigned long pud_addr = vmf->address & PUD_MASK;
Dan Williams5f0694b2017-01-30 21:43:10 -0800344 struct device *dev = &dev_dax->dev;
Dave Jiang9557fee2017-02-24 14:57:05 -0800345 struct dax_region *dax_region;
346 phys_addr_t phys;
347 pgoff_t pgoff;
348 pfn_t pfn;
Dave Jiang70b085b2017-03-10 13:24:27 -0700349 unsigned int fault_size = PUD_SIZE;
350
Dave Jiang9557fee2017-02-24 14:57:05 -0800351
Dan Williams5f0694b2017-01-30 21:43:10 -0800352 if (check_vma(dev_dax, vmf->vma, __func__))
Dave Jiang9557fee2017-02-24 14:57:05 -0800353 return VM_FAULT_SIGBUS;
354
Dan Williams5f0694b2017-01-30 21:43:10 -0800355 dax_region = dev_dax->region;
Dave Jiang9557fee2017-02-24 14:57:05 -0800356 if (dax_region->align > PUD_SIZE) {
Dan Williams6daaca52018-03-05 16:40:05 -0800357 dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
358 dax_region->align, fault_size);
Dave Jiang9557fee2017-02-24 14:57:05 -0800359 return VM_FAULT_SIGBUS;
360 }
361
362 /* dax pud mappings require pfn_t_devmap() */
363 if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) {
Dan Williams6daaca52018-03-05 16:40:05 -0800364 dev_dbg(dev, "region lacks devmap flags\n");
Dave Jiang9557fee2017-02-24 14:57:05 -0800365 return VM_FAULT_SIGBUS;
366 }
367
Dave Jiang70b085b2017-03-10 13:24:27 -0700368 if (fault_size < dax_region->align)
369 return VM_FAULT_SIGBUS;
370 else if (fault_size > dax_region->align)
371 return VM_FAULT_FALLBACK;
372
373 /* if we are outside of the VMA */
374 if (pud_addr < vmf->vma->vm_start ||
375 (pud_addr + PUD_SIZE) > vmf->vma->vm_end)
376 return VM_FAULT_SIGBUS;
377
Dave Jiang9557fee2017-02-24 14:57:05 -0800378 pgoff = linear_page_index(vmf->vma, pud_addr);
Dan Williams73616362017-05-04 23:38:43 -0700379 phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
Dave Jiang9557fee2017-02-24 14:57:05 -0800380 if (phys == -1) {
Dan Williams6daaca52018-03-05 16:40:05 -0800381 dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
Dave Jiang9557fee2017-02-24 14:57:05 -0800382 return VM_FAULT_SIGBUS;
383 }
384
385 pfn = phys_to_pfn_t(phys, dax_region->pfn_flags);
386
387 return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn,
388 vmf->flags & FAULT_FLAG_WRITE);
389}
390#else
Dan Williams5f0694b2017-01-30 21:43:10 -0800391static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf)
Dave Jiang9557fee2017-02-24 14:57:05 -0800392{
393 return VM_FAULT_FALLBACK;
394}
395#endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
396
Dan Williams5f0694b2017-01-30 21:43:10 -0800397static int dev_dax_huge_fault(struct vm_fault *vmf,
Dave Jiangc791ace2017-02-24 14:57:08 -0800398 enum page_entry_size pe_size)
Dan Williamsdee41072016-05-14 12:20:44 -0700399{
Dan Williams956a4cd2017-04-07 16:42:08 -0700400 int rc, id;
Dave Jiangf4200392017-02-22 15:40:06 -0800401 struct file *filp = vmf->vma->vm_file;
Dan Williams5f0694b2017-01-30 21:43:10 -0800402 struct dev_dax *dev_dax = filp->private_data;
Dan Williamsdee41072016-05-14 12:20:44 -0700403
Dan Williams6daaca52018-03-05 16:40:05 -0800404 dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
405 (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
Oliver O'Halloran76202622017-04-12 01:59:36 +1000406 vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
Dan Williamsdee41072016-05-14 12:20:44 -0700407
Dan Williams7b6be842017-04-11 09:49:49 -0700408 id = dax_read_lock();
Dave Jiangc791ace2017-02-24 14:57:08 -0800409 switch (pe_size) {
410 case PE_SIZE_PTE:
Dan Williams5f0694b2017-01-30 21:43:10 -0800411 rc = __dev_dax_pte_fault(dev_dax, vmf);
Dave Jianga2d58162017-02-24 14:56:59 -0800412 break;
Dave Jiangc791ace2017-02-24 14:57:08 -0800413 case PE_SIZE_PMD:
Dan Williams5f0694b2017-01-30 21:43:10 -0800414 rc = __dev_dax_pmd_fault(dev_dax, vmf);
Dave Jianga2d58162017-02-24 14:56:59 -0800415 break;
Dave Jiangc791ace2017-02-24 14:57:08 -0800416 case PE_SIZE_PUD:
Dan Williams5f0694b2017-01-30 21:43:10 -0800417 rc = __dev_dax_pud_fault(dev_dax, vmf);
Dave Jiang9557fee2017-02-24 14:57:05 -0800418 break;
Dave Jianga2d58162017-02-24 14:56:59 -0800419 default:
Pushkar Jambhlekar54eafcc2017-04-11 09:12:25 -0700420 rc = VM_FAULT_SIGBUS;
Dave Jianga2d58162017-02-24 14:56:59 -0800421 }
Dan Williams7b6be842017-04-11 09:49:49 -0700422 dax_read_unlock(id);
Dan Williamsdee41072016-05-14 12:20:44 -0700423
424 return rc;
425}
426
Dan Williams5f0694b2017-01-30 21:43:10 -0800427static int dev_dax_fault(struct vm_fault *vmf)
Dave Jiangc791ace2017-02-24 14:57:08 -0800428{
Dan Williams5f0694b2017-01-30 21:43:10 -0800429 return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
Dave Jiangc791ace2017-02-24 14:57:08 -0800430}
431
Dan Williams9702cff2017-11-29 16:10:32 -0800432static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
433{
434 struct file *filp = vma->vm_file;
435 struct dev_dax *dev_dax = filp->private_data;
436 struct dax_region *dax_region = dev_dax->region;
437
438 if (!IS_ALIGNED(addr, dax_region->align))
439 return -EINVAL;
440 return 0;
441}
442
Dan Williamsc1d53b92018-04-05 16:24:28 -0700443static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
444{
445 struct file *filp = vma->vm_file;
446 struct dev_dax *dev_dax = filp->private_data;
447 struct dax_region *dax_region = dev_dax->region;
448
449 return dax_region->align;
450}
451
Dan Williams5f0694b2017-01-30 21:43:10 -0800452static const struct vm_operations_struct dax_vm_ops = {
453 .fault = dev_dax_fault,
454 .huge_fault = dev_dax_huge_fault,
Dan Williams9702cff2017-11-29 16:10:32 -0800455 .split = dev_dax_split,
Dan Williamsc1d53b92018-04-05 16:24:28 -0700456 .pagesize = dev_dax_pagesize,
Dan Williamsdee41072016-05-14 12:20:44 -0700457};
458
Dan Williamsaf69f512016-08-11 00:38:03 -0700459static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
Dan Williamsdee41072016-05-14 12:20:44 -0700460{
Dan Williams5f0694b2017-01-30 21:43:10 -0800461 struct dev_dax *dev_dax = filp->private_data;
Dan Williams7b6be842017-04-11 09:49:49 -0700462 int rc, id;
Dan Williamsdee41072016-05-14 12:20:44 -0700463
Dan Williams6daaca52018-03-05 16:40:05 -0800464 dev_dbg(&dev_dax->dev, "trace\n");
Dan Williamsdee41072016-05-14 12:20:44 -0700465
Dan Williams7b6be842017-04-11 09:49:49 -0700466 /*
467 * We lock to check dax_dev liveness and will re-check at
468 * fault time.
469 */
470 id = dax_read_lock();
Dan Williams5f0694b2017-01-30 21:43:10 -0800471 rc = check_vma(dev_dax, vma, __func__);
Dan Williams7b6be842017-04-11 09:49:49 -0700472 dax_read_unlock(id);
Dan Williamsdee41072016-05-14 12:20:44 -0700473 if (rc)
474 return rc;
475
Dan Williams5f0694b2017-01-30 21:43:10 -0800476 vma->vm_ops = &dax_vm_ops;
Dan Williamsdee41072016-05-14 12:20:44 -0700477 vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
478 return 0;
Dan Williams043a9252016-08-07 08:23:56 -0700479}
Dan Williamsdee41072016-05-14 12:20:44 -0700480
Dan Williams043a9252016-08-07 08:23:56 -0700481/* return an unmapped area aligned to the dax region specified alignment */
Dan Williamsaf69f512016-08-11 00:38:03 -0700482static unsigned long dax_get_unmapped_area(struct file *filp,
Dan Williams043a9252016-08-07 08:23:56 -0700483 unsigned long addr, unsigned long len, unsigned long pgoff,
484 unsigned long flags)
485{
486 unsigned long off, off_end, off_align, len_align, addr_align, align;
Dan Williams5f0694b2017-01-30 21:43:10 -0800487 struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
Dan Williams043a9252016-08-07 08:23:56 -0700488 struct dax_region *dax_region;
489
Dan Williams5f0694b2017-01-30 21:43:10 -0800490 if (!dev_dax || addr)
Dan Williams043a9252016-08-07 08:23:56 -0700491 goto out;
492
Dan Williams5f0694b2017-01-30 21:43:10 -0800493 dax_region = dev_dax->region;
Dan Williams043a9252016-08-07 08:23:56 -0700494 align = dax_region->align;
495 off = pgoff << PAGE_SHIFT;
496 off_end = off + len;
497 off_align = round_up(off, align);
498
499 if ((off_end <= off_align) || ((off_end - off_align) < align))
500 goto out;
501
502 len_align = len + align;
503 if ((off + len_align) < off)
504 goto out;
505
506 addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
507 pgoff, flags);
508 if (!IS_ERR_VALUE(addr_align)) {
509 addr_align += (off - addr_align) & (align - 1);
510 return addr_align;
511 }
512 out:
513 return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
514}
515
Dan Williamsaf69f512016-08-11 00:38:03 -0700516static int dax_open(struct inode *inode, struct file *filp)
Dan Williams043a9252016-08-07 08:23:56 -0700517{
Dan Williams7b6be842017-04-11 09:49:49 -0700518 struct dax_device *dax_dev = inode_dax(inode);
519 struct inode *__dax_inode = dax_inode(dax_dev);
520 struct dev_dax *dev_dax = dax_get_private(dax_dev);
Dan Williams043a9252016-08-07 08:23:56 -0700521
Dan Williams6daaca52018-03-05 16:40:05 -0800522 dev_dbg(&dev_dax->dev, "trace\n");
Dan Williams7b6be842017-04-11 09:49:49 -0700523 inode->i_mapping = __dax_inode->i_mapping;
524 inode->i_mapping->host = __dax_inode;
Dan Williams3bc52c42016-07-24 21:55:45 -0700525 filp->f_mapping = inode->i_mapping;
Jeff Layton5660e132017-07-06 07:02:25 -0400526 filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
Dan Williams5f0694b2017-01-30 21:43:10 -0800527 filp->private_data = dev_dax;
Dan Williamsebd84d72016-08-11 00:41:51 -0700528 inode->i_flags = S_DAX;
Dan Williams043a9252016-08-07 08:23:56 -0700529
Dan Williams043a9252016-08-07 08:23:56 -0700530 return 0;
531}
532
Dan Williamsaf69f512016-08-11 00:38:03 -0700533static int dax_release(struct inode *inode, struct file *filp)
Dan Williams043a9252016-08-07 08:23:56 -0700534{
Dan Williams5f0694b2017-01-30 21:43:10 -0800535 struct dev_dax *dev_dax = filp->private_data;
Dan Williams043a9252016-08-07 08:23:56 -0700536
Dan Williams6daaca52018-03-05 16:40:05 -0800537 dev_dbg(&dev_dax->dev, "trace\n");
Dan Williams043a9252016-08-07 08:23:56 -0700538 return 0;
Dan Williamsdee41072016-05-14 12:20:44 -0700539}
540
Dan Williamsab68f262016-05-18 09:15:08 -0700541static const struct file_operations dax_fops = {
542 .llseek = noop_llseek,
543 .owner = THIS_MODULE,
Dan Williamsaf69f512016-08-11 00:38:03 -0700544 .open = dax_open,
545 .release = dax_release,
546 .get_unmapped_area = dax_get_unmapped_area,
547 .mmap = dax_mmap,
Dave Jiangef842302018-04-19 13:39:43 -0700548 .mmap_supported_flags = MAP_SYNC,
Dan Williamsab68f262016-05-18 09:15:08 -0700549};
550
Dan Williams5f0694b2017-01-30 21:43:10 -0800551static void dev_dax_release(struct device *dev)
Dan Williams043a9252016-08-07 08:23:56 -0700552{
Dan Williams5f0694b2017-01-30 21:43:10 -0800553 struct dev_dax *dev_dax = to_dev_dax(dev);
554 struct dax_region *dax_region = dev_dax->region;
Dan Williams7b6be842017-04-11 09:49:49 -0700555 struct dax_device *dax_dev = dev_dax->dax_dev;
Dan Williams043a9252016-08-07 08:23:56 -0700556
Dan Williamsbbb3be12017-07-18 17:49:14 -0700557 if (dev_dax->id >= 0)
558 ida_simple_remove(&dax_region->ida, dev_dax->id);
Dan Williamsebd84d72016-08-11 00:41:51 -0700559 dax_region_put(dax_region);
Dan Williams7b6be842017-04-11 09:49:49 -0700560 put_dax(dax_dev);
Dan Williams5f0694b2017-01-30 21:43:10 -0800561 kfree(dev_dax);
Dan Williamsebd84d72016-08-11 00:41:51 -0700562}
563
Dan Williams5f0694b2017-01-30 21:43:10 -0800564static void kill_dev_dax(struct dev_dax *dev_dax)
Dan Williamsebd84d72016-08-11 00:41:51 -0700565{
Dan Williams7b6be842017-04-11 09:49:49 -0700566 struct dax_device *dax_dev = dev_dax->dax_dev;
567 struct inode *inode = dax_inode(dax_dev);
568
569 kill_dax(dax_dev);
570 unmap_mapping_range(inode->i_mapping, 0, 0, 1);
Dan Williamsed01e502017-03-17 12:48:09 -0600571}
572
Dan Williams5f0694b2017-01-30 21:43:10 -0800573static void unregister_dev_dax(void *dev)
Dan Williamsed01e502017-03-17 12:48:09 -0600574{
Dan Williams5f0694b2017-01-30 21:43:10 -0800575 struct dev_dax *dev_dax = to_dev_dax(dev);
Dan Williams7b6be842017-04-11 09:49:49 -0700576 struct dax_device *dax_dev = dev_dax->dax_dev;
577 struct inode *inode = dax_inode(dax_dev);
578 struct cdev *cdev = inode->i_cdev;
Dan Williamsed01e502017-03-17 12:48:09 -0600579
Dan Williams6daaca52018-03-05 16:40:05 -0800580 dev_dbg(dev, "trace\n");
Dan Williamsed01e502017-03-17 12:48:09 -0600581
Dan Williams5f0694b2017-01-30 21:43:10 -0800582 kill_dev_dax(dev_dax);
Dan Williams7b6be842017-04-11 09:49:49 -0700583 cdev_device_del(cdev, dev);
Logan Gunthorpe92a3fa02017-03-17 12:48:10 -0600584 put_device(dev);
Dan Williams043a9252016-08-07 08:23:56 -0700585}
586
Dan Williams5f0694b2017-01-30 21:43:10 -0800587struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region,
Dan Williamsbbb3be12017-07-18 17:49:14 -0700588 int id, struct resource *res, int count)
Dan Williams043a9252016-08-07 08:23:56 -0700589{
590 struct device *parent = dax_region->dev;
Dan Williams7b6be842017-04-11 09:49:49 -0700591 struct dax_device *dax_dev;
Dan Williams5f0694b2017-01-30 21:43:10 -0800592 struct dev_dax *dev_dax;
Dan Williams7b6be842017-04-11 09:49:49 -0700593 struct inode *inode;
Dan Williams043a9252016-08-07 08:23:56 -0700594 struct device *dev;
Dan Williamsba09c012016-07-24 15:55:42 -0700595 struct cdev *cdev;
Dan Williams43fe51e2017-07-12 13:42:37 -0700596 int rc, i;
597
598 if (!count)
599 return ERR_PTR(-EINVAL);
Dan Williams043a9252016-08-07 08:23:56 -0700600
Kees Cookacafe7e2018-05-08 13:45:50 -0700601 dev_dax = kzalloc(struct_size(dev_dax, res, count), GFP_KERNEL);
Dan Williams5f0694b2017-01-30 21:43:10 -0800602 if (!dev_dax)
Dan Williamsd76911e2016-07-19 17:51:40 -0700603 return ERR_PTR(-ENOMEM);
Dan Williams043a9252016-08-07 08:23:56 -0700604
Dan Williams9d2d01a2016-07-19 16:17:58 -0700605 for (i = 0; i < count; i++) {
606 if (!IS_ALIGNED(res[i].start, dax_region->align)
607 || !IS_ALIGNED(resource_size(&res[i]),
608 dax_region->align)) {
609 rc = -EINVAL;
610 break;
611 }
Dan Williams5f0694b2017-01-30 21:43:10 -0800612 dev_dax->res[i].start = res[i].start;
613 dev_dax->res[i].end = res[i].end;
Dan Williams9d2d01a2016-07-19 16:17:58 -0700614 }
615
616 if (i < count)
617 goto err_id;
618
Dan Williamsbbb3be12017-07-18 17:49:14 -0700619 if (id < 0) {
620 id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL);
621 dev_dax->id = id;
622 if (id < 0) {
623 rc = id;
624 goto err_id;
625 }
626 } else {
627 /* region provider owns @id lifetime */
628 dev_dax->id = -1;
Dan Williams043a9252016-08-07 08:23:56 -0700629 }
630
Dan Williams6568b082017-01-24 18:44:18 -0800631 /*
632 * No 'host' or dax_operations since there is no access to this
633 * device outside of mmap of the resulting character device.
634 */
635 dax_dev = alloc_dax(dev_dax, NULL, NULL);
Dan Williams43fe51e2017-07-12 13:42:37 -0700636 if (!dax_dev) {
637 rc = -ENOMEM;
Dan Williams7b6be842017-04-11 09:49:49 -0700638 goto err_dax;
Dan Williams43fe51e2017-07-12 13:42:37 -0700639 }
Dan Williams043a9252016-08-07 08:23:56 -0700640
Dan Williams7b6be842017-04-11 09:49:49 -0700641 /* from here on we're committed to teardown via dax_dev_release() */
Dan Williams5f0694b2017-01-30 21:43:10 -0800642 dev = &dev_dax->dev;
Dan Williamsebd84d72016-08-11 00:41:51 -0700643 device_initialize(dev);
Dan Williamsba09c012016-07-24 15:55:42 -0700644
Dan Williams7b6be842017-04-11 09:49:49 -0700645 inode = dax_inode(dax_dev);
646 cdev = inode->i_cdev;
Dan Williamsba09c012016-07-24 15:55:42 -0700647 cdev_init(cdev, &dax_fops);
648 cdev->owner = parent->driver->owner;
Dan Williamsba09c012016-07-24 15:55:42 -0700649
Dan Williams5f0694b2017-01-30 21:43:10 -0800650 dev_dax->num_resources = count;
Dan Williams7b6be842017-04-11 09:49:49 -0700651 dev_dax->dax_dev = dax_dev;
Dan Williams5f0694b2017-01-30 21:43:10 -0800652 dev_dax->region = dax_region;
Dan Williamsba09c012016-07-24 15:55:42 -0700653 kref_get(&dax_region->kref);
654
Dan Williams7b6be842017-04-11 09:49:49 -0700655 dev->devt = inode->i_rdev;
Dan Williamsebd84d72016-08-11 00:41:51 -0700656 dev->class = dax_class;
657 dev->parent = parent;
658 dev->groups = dax_attribute_groups;
Dan Williams5f0694b2017-01-30 21:43:10 -0800659 dev->release = dev_dax_release;
Dan Williamsbbb3be12017-07-18 17:49:14 -0700660 dev_set_name(dev, "dax%d.%d", dax_region->id, id);
Logan Gunthorpe92a3fa02017-03-17 12:48:10 -0600661
662 rc = cdev_device_add(cdev, dev);
Dan Williamsebd84d72016-08-11 00:41:51 -0700663 if (rc) {
Dan Williams5f0694b2017-01-30 21:43:10 -0800664 kill_dev_dax(dev_dax);
Dan Williamsebd84d72016-08-11 00:41:51 -0700665 put_device(dev);
Dan Williamsd76911e2016-07-19 17:51:40 -0700666 return ERR_PTR(rc);
Dan Williamsebd84d72016-08-11 00:41:51 -0700667 }
Dan Williams043a9252016-08-07 08:23:56 -0700668
Dan Williams5f0694b2017-01-30 21:43:10 -0800669 rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev);
Dan Williamsd76911e2016-07-19 17:51:40 -0700670 if (rc)
671 return ERR_PTR(rc);
672
Dan Williams5f0694b2017-01-30 21:43:10 -0800673 return dev_dax;
Dan Williams043a9252016-08-07 08:23:56 -0700674
Dan Williams7b6be842017-04-11 09:49:49 -0700675 err_dax:
Dan Williamsbbb3be12017-07-18 17:49:14 -0700676 if (dev_dax->id >= 0)
677 ida_simple_remove(&dax_region->ida, dev_dax->id);
Dan Williams043a9252016-08-07 08:23:56 -0700678 err_id:
Dan Williams5f0694b2017-01-30 21:43:10 -0800679 kfree(dev_dax);
Dan Williams043a9252016-08-07 08:23:56 -0700680
Dan Williamsd76911e2016-07-19 17:51:40 -0700681 return ERR_PTR(rc);
Dan Williams043a9252016-08-07 08:23:56 -0700682}
Dan Williams5f0694b2017-01-30 21:43:10 -0800683EXPORT_SYMBOL_GPL(devm_create_dev_dax);
Dan Williams043a9252016-08-07 08:23:56 -0700684
Dan Williamsab68f262016-05-18 09:15:08 -0700685static int __init dax_init(void)
686{
Dan Williamsab68f262016-05-18 09:15:08 -0700687 dax_class = class_create(THIS_MODULE, "dax");
Dan Williams7b6be842017-04-11 09:49:49 -0700688 return PTR_ERR_OR_ZERO(dax_class);
Dan Williamsab68f262016-05-18 09:15:08 -0700689}
690
691static void __exit dax_exit(void)
692{
693 class_destroy(dax_class);
Dan Williamsab68f262016-05-18 09:15:08 -0700694}
695
696MODULE_AUTHOR("Intel Corporation");
697MODULE_LICENSE("GPL v2");
698subsys_initcall(dax_init);
699module_exit(dax_exit);