]>
Commit | Line | Data |
---|---|---|
ab68f262 | 1 | /* |
7b6be844 | 2 | * Copyright(c) 2016 - 2017 Intel Corporation. All rights reserved. |
ab68f262 DW |
3 | * |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of version 2 of the GNU General Public License as | |
6 | * published by the Free Software Foundation. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
11 | * General Public License for more details. | |
12 | */ | |
13 | #include <linux/pagemap.h> | |
14 | #include <linux/module.h> | |
15 | #include <linux/device.h> | |
16 | #include <linux/pfn_t.h> | |
ba09c01d | 17 | #include <linux/cdev.h> |
ab68f262 DW |
18 | #include <linux/slab.h> |
19 | #include <linux/dax.h> | |
20 | #include <linux/fs.h> | |
21 | #include <linux/mm.h> | |
ef842302 | 22 | #include <linux/mman.h> |
efebc711 | 23 | #include "dax-private.h" |
ccdb07f6 | 24 | #include "dax.h" |
ab68f262 | 25 | |
ab68f262 | 26 | static struct class *dax_class; |
ab68f262 | 27 | |
565851c9 DW |
28 | /* |
29 | * Rely on the fact that drvdata is set before the attributes are | |
30 | * registered, and that the attributes are unregistered before drvdata | |
31 | * is cleared to assume that drvdata is always valid. | |
32 | */ | |
d7fe1a67 DW |
33 | static ssize_t id_show(struct device *dev, |
34 | struct device_attribute *attr, char *buf) | |
35 | { | |
565851c9 | 36 | struct dax_region *dax_region = dev_get_drvdata(dev); |
d7fe1a67 | 37 | |
565851c9 | 38 | return sprintf(buf, "%d\n", dax_region->id); |
d7fe1a67 DW |
39 | } |
40 | static DEVICE_ATTR_RO(id); | |
41 | ||
42 | static ssize_t region_size_show(struct device *dev, | |
43 | struct device_attribute *attr, char *buf) | |
44 | { | |
565851c9 | 45 | struct dax_region *dax_region = dev_get_drvdata(dev); |
d7fe1a67 | 46 | |
565851c9 DW |
47 | return sprintf(buf, "%llu\n", (unsigned long long) |
48 | resource_size(&dax_region->res)); | |
d7fe1a67 DW |
49 | } |
50 | static struct device_attribute dev_attr_region_size = __ATTR(size, 0444, | |
51 | region_size_show, NULL); | |
52 | ||
53 | static ssize_t align_show(struct device *dev, | |
54 | struct device_attribute *attr, char *buf) | |
55 | { | |
565851c9 | 56 | struct dax_region *dax_region = dev_get_drvdata(dev); |
d7fe1a67 | 57 | |
565851c9 | 58 | return sprintf(buf, "%u\n", dax_region->align); |
d7fe1a67 DW |
59 | } |
60 | static DEVICE_ATTR_RO(align); | |
61 | ||
62 | static struct attribute *dax_region_attributes[] = { | |
63 | &dev_attr_region_size.attr, | |
64 | &dev_attr_align.attr, | |
65 | &dev_attr_id.attr, | |
66 | NULL, | |
67 | }; | |
68 | ||
69 | static const struct attribute_group dax_region_attribute_group = { | |
70 | .name = "dax_region", | |
71 | .attrs = dax_region_attributes, | |
72 | }; | |
73 | ||
74 | static const struct attribute_group *dax_region_attribute_groups[] = { | |
75 | &dax_region_attribute_group, | |
76 | NULL, | |
77 | }; | |
78 | ||
ab68f262 DW |
79 | static void dax_region_free(struct kref *kref) |
80 | { | |
81 | struct dax_region *dax_region; | |
82 | ||
83 | dax_region = container_of(kref, struct dax_region, kref); | |
84 | kfree(dax_region); | |
85 | } | |
86 | ||
87 | void dax_region_put(struct dax_region *dax_region) | |
ab68f262 | 88 | { |
ab68f262 | 89 | kref_put(&dax_region->kref, dax_region_free); |
ab68f262 | 90 | } |
ab68f262 | 91 | EXPORT_SYMBOL_GPL(dax_region_put); |
ab68f262 | 92 | |
d7fe1a67 DW |
93 | static void dax_region_unregister(void *region) |
94 | { | |
95 | struct dax_region *dax_region = region; | |
96 | ||
97 | sysfs_remove_groups(&dax_region->dev->kobj, | |
98 | dax_region_attribute_groups); | |
99 | dax_region_put(dax_region); | |
100 | } | |
101 | ||
ab68f262 DW |
102 | struct dax_region *alloc_dax_region(struct device *parent, int region_id, |
103 | struct resource *res, unsigned int align, void *addr, | |
104 | unsigned long pfn_flags) | |
105 | { | |
106 | struct dax_region *dax_region; | |
107 | ||
d7fe1a67 DW |
108 | /* |
109 | * The DAX core assumes that it can store its private data in | |
110 | * parent->driver_data. This WARN is a reminder / safeguard for | |
111 | * developers of device-dax drivers. | |
112 | */ | |
113 | if (dev_get_drvdata(parent)) { | |
114 | dev_WARN(parent, "dax core failed to setup private data\n"); | |
115 | return NULL; | |
116 | } | |
117 | ||
9d2d01a0 DW |
118 | if (!IS_ALIGNED(res->start, align) |
119 | || !IS_ALIGNED(resource_size(res), align)) | |
120 | return NULL; | |
ab68f262 | 121 | |
9d2d01a0 | 122 | dax_region = kzalloc(sizeof(*dax_region), GFP_KERNEL); |
ab68f262 DW |
123 | if (!dax_region) |
124 | return NULL; | |
125 | ||
d7fe1a67 | 126 | dev_set_drvdata(parent, dax_region); |
ab68f262 DW |
127 | memcpy(&dax_region->res, res, sizeof(*res)); |
128 | dax_region->pfn_flags = pfn_flags; | |
129 | kref_init(&dax_region->kref); | |
130 | dax_region->id = region_id; | |
131 | ida_init(&dax_region->ida); | |
132 | dax_region->align = align; | |
133 | dax_region->dev = parent; | |
134 | dax_region->base = addr; | |
d7fe1a67 DW |
135 | if (sysfs_create_groups(&parent->kobj, dax_region_attribute_groups)) { |
136 | kfree(dax_region); | |
1c47a645 | 137 | return NULL; |
d7fe1a67 | 138 | } |
ab68f262 | 139 | |
d7fe1a67 DW |
140 | kref_get(&dax_region->kref); |
141 | if (devm_add_action_or_reset(parent, dax_region_unregister, dax_region)) | |
142 | return NULL; | |
ab68f262 DW |
143 | return dax_region; |
144 | } | |
145 | EXPORT_SYMBOL_GPL(alloc_dax_region); | |
146 | ||
5f0694b3 | 147 | static struct dev_dax *to_dev_dax(struct device *dev) |
ebd84d72 | 148 | { |
5f0694b3 | 149 | return container_of(dev, struct dev_dax, dev); |
ebd84d72 DW |
150 | } |
151 | ||
ab68f262 DW |
152 | static ssize_t size_show(struct device *dev, |
153 | struct device_attribute *attr, char *buf) | |
154 | { | |
5f0694b3 | 155 | struct dev_dax *dev_dax = to_dev_dax(dev); |
ab68f262 DW |
156 | unsigned long long size = 0; |
157 | int i; | |
158 | ||
5f0694b3 DW |
159 | for (i = 0; i < dev_dax->num_resources; i++) |
160 | size += resource_size(&dev_dax->res[i]); | |
ab68f262 DW |
161 | |
162 | return sprintf(buf, "%llu\n", size); | |
163 | } | |
164 | static DEVICE_ATTR_RO(size); | |
165 | ||
5f0694b3 | 166 | static struct attribute *dev_dax_attributes[] = { |
ab68f262 DW |
167 | &dev_attr_size.attr, |
168 | NULL, | |
169 | }; | |
170 | ||
5f0694b3 DW |
171 | static const struct attribute_group dev_dax_attribute_group = { |
172 | .attrs = dev_dax_attributes, | |
ab68f262 DW |
173 | }; |
174 | ||
175 | static const struct attribute_group *dax_attribute_groups[] = { | |
5f0694b3 | 176 | &dev_dax_attribute_group, |
ab68f262 DW |
177 | NULL, |
178 | }; | |
179 | ||
5f0694b3 | 180 | static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma, |
dee41079 DW |
181 | const char *func) |
182 | { | |
5f0694b3 DW |
183 | struct dax_region *dax_region = dev_dax->region; |
184 | struct device *dev = &dev_dax->dev; | |
dee41079 DW |
185 | unsigned long mask; |
186 | ||
7b6be844 | 187 | if (!dax_alive(dev_dax->dax_dev)) |
dee41079 DW |
188 | return -ENXIO; |
189 | ||
4cb19355 | 190 | /* prevent private mappings from being established */ |
325896ff | 191 | if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) { |
5a14e91d JM |
192 | dev_info_ratelimited(dev, |
193 | "%s: %s: fail, attempted private mapping\n", | |
dee41079 DW |
194 | current->comm, func); |
195 | return -EINVAL; | |
196 | } | |
197 | ||
198 | mask = dax_region->align - 1; | |
199 | if (vma->vm_start & mask || vma->vm_end & mask) { | |
5a14e91d JM |
200 | dev_info_ratelimited(dev, |
201 | "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n", | |
dee41079 DW |
202 | current->comm, func, vma->vm_start, vma->vm_end, |
203 | mask); | |
204 | return -EINVAL; | |
205 | } | |
206 | ||
207 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) == PFN_DEV | |
208 | && (vma->vm_flags & VM_DONTCOPY) == 0) { | |
5a14e91d JM |
209 | dev_info_ratelimited(dev, |
210 | "%s: %s: fail, dax range requires MADV_DONTFORK\n", | |
dee41079 DW |
211 | current->comm, func); |
212 | return -EINVAL; | |
213 | } | |
214 | ||
215 | if (!vma_is_dax(vma)) { | |
5a14e91d JM |
216 | dev_info_ratelimited(dev, |
217 | "%s: %s: fail, vma is not DAX capable\n", | |
dee41079 DW |
218 | current->comm, func); |
219 | return -EINVAL; | |
220 | } | |
221 | ||
222 | return 0; | |
223 | } | |
224 | ||
efebc711 | 225 | /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */ |
73616367 | 226 | __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff, |
dee41079 DW |
227 | unsigned long size) |
228 | { | |
229 | struct resource *res; | |
0a3ff786 RZ |
230 | /* gcc-4.6.3-nolibc for i386 complains that this is uninitialized */ |
231 | phys_addr_t uninitialized_var(phys); | |
dee41079 DW |
232 | int i; |
233 | ||
5f0694b3 DW |
234 | for (i = 0; i < dev_dax->num_resources; i++) { |
235 | res = &dev_dax->res[i]; | |
dee41079 DW |
236 | phys = pgoff * PAGE_SIZE + res->start; |
237 | if (phys >= res->start && phys <= res->end) | |
238 | break; | |
239 | pgoff -= PHYS_PFN(resource_size(res)); | |
240 | } | |
241 | ||
5f0694b3 DW |
242 | if (i < dev_dax->num_resources) { |
243 | res = &dev_dax->res[i]; | |
dee41079 DW |
244 | if (phys + size - 1 <= res->end) |
245 | return phys; | |
246 | } | |
247 | ||
248 | return -1; | |
249 | } | |
250 | ||
5f0694b3 | 251 | static int __dev_dax_pte_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) |
dee41079 | 252 | { |
5f0694b3 | 253 | struct device *dev = &dev_dax->dev; |
dee41079 DW |
254 | struct dax_region *dax_region; |
255 | int rc = VM_FAULT_SIGBUS; | |
256 | phys_addr_t phys; | |
257 | pfn_t pfn; | |
0134ed4f | 258 | unsigned int fault_size = PAGE_SIZE; |
dee41079 | 259 | |
5f0694b3 | 260 | if (check_vma(dev_dax, vmf->vma, __func__)) |
dee41079 DW |
261 | return VM_FAULT_SIGBUS; |
262 | ||
5f0694b3 | 263 | dax_region = dev_dax->region; |
dee41079 | 264 | if (dax_region->align > PAGE_SIZE) { |
6daaca52 DW |
265 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
266 | dax_region->align, fault_size); | |
dee41079 DW |
267 | return VM_FAULT_SIGBUS; |
268 | } | |
269 | ||
0134ed4f DJ |
270 | if (fault_size != dax_region->align) |
271 | return VM_FAULT_SIGBUS; | |
272 | ||
73616367 | 273 | phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE); |
dee41079 | 274 | if (phys == -1) { |
6daaca52 | 275 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff); |
dee41079 DW |
276 | return VM_FAULT_SIGBUS; |
277 | } | |
278 | ||
279 | pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); | |
280 | ||
11bac800 | 281 | rc = vm_insert_mixed(vmf->vma, vmf->address, pfn); |
dee41079 DW |
282 | |
283 | if (rc == -ENOMEM) | |
284 | return VM_FAULT_OOM; | |
285 | if (rc < 0 && rc != -EBUSY) | |
286 | return VM_FAULT_SIGBUS; | |
287 | ||
288 | return VM_FAULT_NOPAGE; | |
289 | } | |
290 | ||
5f0694b3 | 291 | static int __dev_dax_pmd_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) |
dee41079 | 292 | { |
d8a849e1 | 293 | unsigned long pmd_addr = vmf->address & PMD_MASK; |
5f0694b3 | 294 | struct device *dev = &dev_dax->dev; |
dee41079 DW |
295 | struct dax_region *dax_region; |
296 | phys_addr_t phys; | |
297 | pgoff_t pgoff; | |
298 | pfn_t pfn; | |
0134ed4f | 299 | unsigned int fault_size = PMD_SIZE; |
dee41079 | 300 | |
5f0694b3 | 301 | if (check_vma(dev_dax, vmf->vma, __func__)) |
dee41079 DW |
302 | return VM_FAULT_SIGBUS; |
303 | ||
5f0694b3 | 304 | dax_region = dev_dax->region; |
dee41079 | 305 | if (dax_region->align > PMD_SIZE) { |
6daaca52 DW |
306 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
307 | dax_region->align, fault_size); | |
dee41079 DW |
308 | return VM_FAULT_SIGBUS; |
309 | } | |
310 | ||
311 | /* dax pmd mappings require pfn_t_devmap() */ | |
312 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) { | |
6daaca52 | 313 | dev_dbg(dev, "region lacks devmap flags\n"); |
dee41079 DW |
314 | return VM_FAULT_SIGBUS; |
315 | } | |
316 | ||
0134ed4f DJ |
317 | if (fault_size < dax_region->align) |
318 | return VM_FAULT_SIGBUS; | |
319 | else if (fault_size > dax_region->align) | |
320 | return VM_FAULT_FALLBACK; | |
321 | ||
322 | /* if we are outside of the VMA */ | |
323 | if (pmd_addr < vmf->vma->vm_start || | |
324 | (pmd_addr + PMD_SIZE) > vmf->vma->vm_end) | |
325 | return VM_FAULT_SIGBUS; | |
326 | ||
f4200391 | 327 | pgoff = linear_page_index(vmf->vma, pmd_addr); |
73616367 | 328 | phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE); |
dee41079 | 329 | if (phys == -1) { |
6daaca52 | 330 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); |
dee41079 DW |
331 | return VM_FAULT_SIGBUS; |
332 | } | |
333 | ||
334 | pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); | |
335 | ||
f4200391 | 336 | return vmf_insert_pfn_pmd(vmf->vma, vmf->address, vmf->pmd, pfn, |
d8a849e1 | 337 | vmf->flags & FAULT_FLAG_WRITE); |
dee41079 DW |
338 | } |
339 | ||
9557feee | 340 | #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD |
5f0694b3 | 341 | static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) |
9557feee DJ |
342 | { |
343 | unsigned long pud_addr = vmf->address & PUD_MASK; | |
5f0694b3 | 344 | struct device *dev = &dev_dax->dev; |
9557feee DJ |
345 | struct dax_region *dax_region; |
346 | phys_addr_t phys; | |
347 | pgoff_t pgoff; | |
348 | pfn_t pfn; | |
70b085b0 DJ |
349 | unsigned int fault_size = PUD_SIZE; |
350 | ||
9557feee | 351 | |
5f0694b3 | 352 | if (check_vma(dev_dax, vmf->vma, __func__)) |
9557feee DJ |
353 | return VM_FAULT_SIGBUS; |
354 | ||
5f0694b3 | 355 | dax_region = dev_dax->region; |
9557feee | 356 | if (dax_region->align > PUD_SIZE) { |
6daaca52 DW |
357 | dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n", |
358 | dax_region->align, fault_size); | |
9557feee DJ |
359 | return VM_FAULT_SIGBUS; |
360 | } | |
361 | ||
362 | /* dax pud mappings require pfn_t_devmap() */ | |
363 | if ((dax_region->pfn_flags & (PFN_DEV|PFN_MAP)) != (PFN_DEV|PFN_MAP)) { | |
6daaca52 | 364 | dev_dbg(dev, "region lacks devmap flags\n"); |
9557feee DJ |
365 | return VM_FAULT_SIGBUS; |
366 | } | |
367 | ||
70b085b0 DJ |
368 | if (fault_size < dax_region->align) |
369 | return VM_FAULT_SIGBUS; | |
370 | else if (fault_size > dax_region->align) | |
371 | return VM_FAULT_FALLBACK; | |
372 | ||
373 | /* if we are outside of the VMA */ | |
374 | if (pud_addr < vmf->vma->vm_start || | |
375 | (pud_addr + PUD_SIZE) > vmf->vma->vm_end) | |
376 | return VM_FAULT_SIGBUS; | |
377 | ||
9557feee | 378 | pgoff = linear_page_index(vmf->vma, pud_addr); |
73616367 | 379 | phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE); |
9557feee | 380 | if (phys == -1) { |
6daaca52 | 381 | dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff); |
9557feee DJ |
382 | return VM_FAULT_SIGBUS; |
383 | } | |
384 | ||
385 | pfn = phys_to_pfn_t(phys, dax_region->pfn_flags); | |
386 | ||
387 | return vmf_insert_pfn_pud(vmf->vma, vmf->address, vmf->pud, pfn, | |
388 | vmf->flags & FAULT_FLAG_WRITE); | |
389 | } | |
390 | #else | |
5f0694b3 | 391 | static int __dev_dax_pud_fault(struct dev_dax *dev_dax, struct vm_fault *vmf) |
9557feee DJ |
392 | { |
393 | return VM_FAULT_FALLBACK; | |
394 | } | |
395 | #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */ | |
396 | ||
5f0694b3 | 397 | static int dev_dax_huge_fault(struct vm_fault *vmf, |
c791ace1 | 398 | enum page_entry_size pe_size) |
dee41079 | 399 | { |
956a4cd2 | 400 | int rc, id; |
f4200391 | 401 | struct file *filp = vmf->vma->vm_file; |
5f0694b3 | 402 | struct dev_dax *dev_dax = filp->private_data; |
dee41079 | 403 | |
6daaca52 DW |
404 | dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm, |
405 | (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read", | |
76202620 | 406 | vmf->vma->vm_start, vmf->vma->vm_end, pe_size); |
dee41079 | 407 | |
7b6be844 | 408 | id = dax_read_lock(); |
c791ace1 DJ |
409 | switch (pe_size) { |
410 | case PE_SIZE_PTE: | |
5f0694b3 | 411 | rc = __dev_dax_pte_fault(dev_dax, vmf); |
a2d58167 | 412 | break; |
c791ace1 | 413 | case PE_SIZE_PMD: |
5f0694b3 | 414 | rc = __dev_dax_pmd_fault(dev_dax, vmf); |
9557feee | 415 | break; |
c791ace1 | 416 | case PE_SIZE_PUD: |
5f0694b3 | 417 | rc = __dev_dax_pud_fault(dev_dax, vmf); |
a2d58167 DJ |
418 | break; |
419 | default: | |
54eafcc9 | 420 | rc = VM_FAULT_SIGBUS; |
a2d58167 | 421 | } |
7b6be844 | 422 | dax_read_unlock(id); |
dee41079 DW |
423 | |
424 | return rc; | |
425 | } | |
426 | ||
5f0694b3 | 427 | static int dev_dax_fault(struct vm_fault *vmf) |
c791ace1 | 428 | { |
5f0694b3 | 429 | return dev_dax_huge_fault(vmf, PE_SIZE_PTE); |
c791ace1 DJ |
430 | } |
431 | ||
9702cffd DW |
432 | static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr) |
433 | { | |
434 | struct file *filp = vma->vm_file; | |
435 | struct dev_dax *dev_dax = filp->private_data; | |
436 | struct dax_region *dax_region = dev_dax->region; | |
437 | ||
438 | if (!IS_ALIGNED(addr, dax_region->align)) | |
439 | return -EINVAL; | |
440 | return 0; | |
441 | } | |
442 | ||
c1d53b92 DW |
443 | static unsigned long dev_dax_pagesize(struct vm_area_struct *vma) |
444 | { | |
445 | struct file *filp = vma->vm_file; | |
446 | struct dev_dax *dev_dax = filp->private_data; | |
447 | struct dax_region *dax_region = dev_dax->region; | |
448 | ||
449 | return dax_region->align; | |
450 | } | |
451 | ||
5f0694b3 DW |
452 | static const struct vm_operations_struct dax_vm_ops = { |
453 | .fault = dev_dax_fault, | |
454 | .huge_fault = dev_dax_huge_fault, | |
9702cffd | 455 | .split = dev_dax_split, |
c1d53b92 | 456 | .pagesize = dev_dax_pagesize, |
dee41079 DW |
457 | }; |
458 | ||
af69f51e | 459 | static int dax_mmap(struct file *filp, struct vm_area_struct *vma) |
dee41079 | 460 | { |
5f0694b3 | 461 | struct dev_dax *dev_dax = filp->private_data; |
7b6be844 | 462 | int rc, id; |
dee41079 | 463 | |
6daaca52 | 464 | dev_dbg(&dev_dax->dev, "trace\n"); |
dee41079 | 465 | |
7b6be844 DW |
466 | /* |
467 | * We lock to check dax_dev liveness and will re-check at | |
468 | * fault time. | |
469 | */ | |
470 | id = dax_read_lock(); | |
5f0694b3 | 471 | rc = check_vma(dev_dax, vma, __func__); |
7b6be844 | 472 | dax_read_unlock(id); |
dee41079 DW |
473 | if (rc) |
474 | return rc; | |
475 | ||
5f0694b3 | 476 | vma->vm_ops = &dax_vm_ops; |
e1fb4a08 | 477 | vma->vm_flags |= VM_HUGEPAGE; |
dee41079 | 478 | return 0; |
043a9255 DW |
479 | } |
480 | ||
481 | /* return an unmapped area aligned to the dax region specified alignment */ | |
af69f51e | 482 | static unsigned long dax_get_unmapped_area(struct file *filp, |
043a9255 DW |
483 | unsigned long addr, unsigned long len, unsigned long pgoff, |
484 | unsigned long flags) | |
485 | { | |
486 | unsigned long off, off_end, off_align, len_align, addr_align, align; | |
5f0694b3 | 487 | struct dev_dax *dev_dax = filp ? filp->private_data : NULL; |
043a9255 DW |
488 | struct dax_region *dax_region; |
489 | ||
5f0694b3 | 490 | if (!dev_dax || addr) |
043a9255 DW |
491 | goto out; |
492 | ||
5f0694b3 | 493 | dax_region = dev_dax->region; |
043a9255 DW |
494 | align = dax_region->align; |
495 | off = pgoff << PAGE_SHIFT; | |
496 | off_end = off + len; | |
497 | off_align = round_up(off, align); | |
498 | ||
499 | if ((off_end <= off_align) || ((off_end - off_align) < align)) | |
500 | goto out; | |
501 | ||
502 | len_align = len + align; | |
503 | if ((off + len_align) < off) | |
504 | goto out; | |
dee41079 | 505 | |
043a9255 DW |
506 | addr_align = current->mm->get_unmapped_area(filp, addr, len_align, |
507 | pgoff, flags); | |
508 | if (!IS_ERR_VALUE(addr_align)) { | |
509 | addr_align += (off - addr_align) & (align - 1); | |
510 | return addr_align; | |
511 | } | |
512 | out: | |
513 | return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags); | |
514 | } | |
515 | ||
af69f51e | 516 | static int dax_open(struct inode *inode, struct file *filp) |
043a9255 | 517 | { |
7b6be844 DW |
518 | struct dax_device *dax_dev = inode_dax(inode); |
519 | struct inode *__dax_inode = dax_inode(dax_dev); | |
520 | struct dev_dax *dev_dax = dax_get_private(dax_dev); | |
043a9255 | 521 | |
6daaca52 | 522 | dev_dbg(&dev_dax->dev, "trace\n"); |
7b6be844 DW |
523 | inode->i_mapping = __dax_inode->i_mapping; |
524 | inode->i_mapping->host = __dax_inode; | |
3bc52c45 | 525 | filp->f_mapping = inode->i_mapping; |
5660e13d | 526 | filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping); |
5f0694b3 | 527 | filp->private_data = dev_dax; |
ebd84d72 | 528 | inode->i_flags = S_DAX; |
043a9255 | 529 | |
043a9255 DW |
530 | return 0; |
531 | } | |
dee41079 | 532 | |
af69f51e | 533 | static int dax_release(struct inode *inode, struct file *filp) |
043a9255 | 534 | { |
5f0694b3 | 535 | struct dev_dax *dev_dax = filp->private_data; |
043a9255 | 536 | |
6daaca52 | 537 | dev_dbg(&dev_dax->dev, "trace\n"); |
043a9255 | 538 | return 0; |
dee41079 DW |
539 | } |
540 | ||
ab68f262 DW |
541 | static const struct file_operations dax_fops = { |
542 | .llseek = noop_llseek, | |
543 | .owner = THIS_MODULE, | |
af69f51e DW |
544 | .open = dax_open, |
545 | .release = dax_release, | |
546 | .get_unmapped_area = dax_get_unmapped_area, | |
547 | .mmap = dax_mmap, | |
ef842302 | 548 | .mmap_supported_flags = MAP_SYNC, |
ab68f262 DW |
549 | }; |
550 | ||
5f0694b3 | 551 | static void dev_dax_release(struct device *dev) |
043a9255 | 552 | { |
5f0694b3 DW |
553 | struct dev_dax *dev_dax = to_dev_dax(dev); |
554 | struct dax_region *dax_region = dev_dax->region; | |
7b6be844 | 555 | struct dax_device *dax_dev = dev_dax->dax_dev; |
043a9255 | 556 | |
bbb3be17 DW |
557 | if (dev_dax->id >= 0) |
558 | ida_simple_remove(&dax_region->ida, dev_dax->id); | |
ebd84d72 | 559 | dax_region_put(dax_region); |
7b6be844 | 560 | put_dax(dax_dev); |
5f0694b3 | 561 | kfree(dev_dax); |
ebd84d72 DW |
562 | } |
563 | ||
5f0694b3 | 564 | static void kill_dev_dax(struct dev_dax *dev_dax) |
ebd84d72 | 565 | { |
7b6be844 DW |
566 | struct dax_device *dax_dev = dev_dax->dax_dev; |
567 | struct inode *inode = dax_inode(dax_dev); | |
568 | ||
569 | kill_dax(dax_dev); | |
570 | unmap_mapping_range(inode->i_mapping, 0, 0, 1); | |
ebd84d72 DW |
571 | } |
572 | ||
5f0694b3 | 573 | static void unregister_dev_dax(void *dev) |
ebd84d72 | 574 | { |
5f0694b3 | 575 | struct dev_dax *dev_dax = to_dev_dax(dev); |
7b6be844 DW |
576 | struct dax_device *dax_dev = dev_dax->dax_dev; |
577 | struct inode *inode = dax_inode(dax_dev); | |
578 | struct cdev *cdev = inode->i_cdev; | |
ebd84d72 | 579 | |
6daaca52 | 580 | dev_dbg(dev, "trace\n"); |
043a9255 | 581 | |
5f0694b3 | 582 | kill_dev_dax(dev_dax); |
7b6be844 | 583 | cdev_device_del(cdev, dev); |
92a3fa07 | 584 | put_device(dev); |
043a9255 DW |
585 | } |
586 | ||
5f0694b3 | 587 | struct dev_dax *devm_create_dev_dax(struct dax_region *dax_region, |
bbb3be17 | 588 | int id, struct resource *res, int count) |
043a9255 DW |
589 | { |
590 | struct device *parent = dax_region->dev; | |
7b6be844 | 591 | struct dax_device *dax_dev; |
5f0694b3 | 592 | struct dev_dax *dev_dax; |
7b6be844 | 593 | struct inode *inode; |
043a9255 | 594 | struct device *dev; |
ba09c01d | 595 | struct cdev *cdev; |
43fe51e1 DW |
596 | int rc, i; |
597 | ||
598 | if (!count) | |
599 | return ERR_PTR(-EINVAL); | |
043a9255 | 600 | |
acafe7e3 | 601 | dev_dax = kzalloc(struct_size(dev_dax, res, count), GFP_KERNEL); |
5f0694b3 | 602 | if (!dev_dax) |
d76911ee | 603 | return ERR_PTR(-ENOMEM); |
043a9255 | 604 | |
9d2d01a0 DW |
605 | for (i = 0; i < count; i++) { |
606 | if (!IS_ALIGNED(res[i].start, dax_region->align) | |
607 | || !IS_ALIGNED(resource_size(&res[i]), | |
608 | dax_region->align)) { | |
609 | rc = -EINVAL; | |
610 | break; | |
611 | } | |
5f0694b3 DW |
612 | dev_dax->res[i].start = res[i].start; |
613 | dev_dax->res[i].end = res[i].end; | |
9d2d01a0 DW |
614 | } |
615 | ||
616 | if (i < count) | |
617 | goto err_id; | |
618 | ||
bbb3be17 DW |
619 | if (id < 0) { |
620 | id = ida_simple_get(&dax_region->ida, 0, 0, GFP_KERNEL); | |
621 | dev_dax->id = id; | |
622 | if (id < 0) { | |
623 | rc = id; | |
624 | goto err_id; | |
625 | } | |
626 | } else { | |
627 | /* region provider owns @id lifetime */ | |
628 | dev_dax->id = -1; | |
043a9255 DW |
629 | } |
630 | ||
6568b08b DW |
631 | /* |
632 | * No 'host' or dax_operations since there is no access to this | |
633 | * device outside of mmap of the resulting character device. | |
634 | */ | |
635 | dax_dev = alloc_dax(dev_dax, NULL, NULL); | |
43fe51e1 DW |
636 | if (!dax_dev) { |
637 | rc = -ENOMEM; | |
7b6be844 | 638 | goto err_dax; |
43fe51e1 | 639 | } |
3bc52c45 | 640 | |
7b6be844 | 641 | /* from here on we're committed to teardown via dax_dev_release() */ |
5f0694b3 | 642 | dev = &dev_dax->dev; |
ebd84d72 | 643 | device_initialize(dev); |
ba09c01d | 644 | |
7b6be844 DW |
645 | inode = dax_inode(dax_dev); |
646 | cdev = inode->i_cdev; | |
ba09c01d DW |
647 | cdev_init(cdev, &dax_fops); |
648 | cdev->owner = parent->driver->owner; | |
ba09c01d | 649 | |
5f0694b3 | 650 | dev_dax->num_resources = count; |
7b6be844 | 651 | dev_dax->dax_dev = dax_dev; |
5f0694b3 | 652 | dev_dax->region = dax_region; |
ba09c01d DW |
653 | kref_get(&dax_region->kref); |
654 | ||
7b6be844 | 655 | dev->devt = inode->i_rdev; |
ebd84d72 DW |
656 | dev->class = dax_class; |
657 | dev->parent = parent; | |
658 | dev->groups = dax_attribute_groups; | |
5f0694b3 | 659 | dev->release = dev_dax_release; |
bbb3be17 | 660 | dev_set_name(dev, "dax%d.%d", dax_region->id, id); |
92a3fa07 LG |
661 | |
662 | rc = cdev_device_add(cdev, dev); | |
ebd84d72 | 663 | if (rc) { |
5f0694b3 | 664 | kill_dev_dax(dev_dax); |
ebd84d72 | 665 | put_device(dev); |
d76911ee | 666 | return ERR_PTR(rc); |
ebd84d72 | 667 | } |
043a9255 | 668 | |
5f0694b3 | 669 | rc = devm_add_action_or_reset(dax_region->dev, unregister_dev_dax, dev); |
d76911ee DW |
670 | if (rc) |
671 | return ERR_PTR(rc); | |
672 | ||
5f0694b3 | 673 | return dev_dax; |
043a9255 | 674 | |
7b6be844 | 675 | err_dax: |
bbb3be17 DW |
676 | if (dev_dax->id >= 0) |
677 | ida_simple_remove(&dax_region->ida, dev_dax->id); | |
043a9255 | 678 | err_id: |
5f0694b3 | 679 | kfree(dev_dax); |
043a9255 | 680 | |
d76911ee | 681 | return ERR_PTR(rc); |
043a9255 | 682 | } |
5f0694b3 | 683 | EXPORT_SYMBOL_GPL(devm_create_dev_dax); |
043a9255 | 684 | |
ab68f262 DW |
685 | static int __init dax_init(void) |
686 | { | |
ab68f262 | 687 | dax_class = class_create(THIS_MODULE, "dax"); |
7b6be844 | 688 | return PTR_ERR_OR_ZERO(dax_class); |
ab68f262 DW |
689 | } |
690 | ||
691 | static void __exit dax_exit(void) | |
692 | { | |
693 | class_destroy(dax_class); | |
ab68f262 DW |
694 | } |
695 | ||
696 | MODULE_AUTHOR("Intel Corporation"); | |
697 | MODULE_LICENSE("GPL v2"); | |
698 | subsys_initcall(dax_init); | |
699 | module_exit(dax_exit); |