1 /* SPDX-License-Identifier: GPL-2.0 */
2 /* Copyright(c) 2015 Intel Corporation. All rights reserved. */
3 #include <linux/device.h>
5 #include <linux/kasan.h>
6 #include <linux/memory_hotplug.h>
8 #include <linux/pfn_t.h>
9 #include <linux/swap.h>
10 #include <linux/swapops.h>
11 #include <linux/types.h>
12 #include <linux/wait_bit.h>
13 #include <linux/xarray.h>
14 #include <linux/hmm.h>
16 static DEFINE_XARRAY(pgmap_array
);
17 #define SECTION_MASK ~((1UL << PA_SECTION_SHIFT) - 1)
18 #define SECTION_SIZE (1UL << PA_SECTION_SHIFT)
20 #if IS_ENABLED(CONFIG_DEVICE_PRIVATE)
21 vm_fault_t
device_private_entry_fault(struct vm_area_struct
*vma
,
27 struct page
*page
= device_private_entry_to_page(entry
);
28 struct hmm_devmem
*devmem
;
30 devmem
= container_of(page
->pgmap
, typeof(*devmem
), pagemap
);
33 * The page_fault() callback must migrate page back to system memory
34 * so that CPU can access it. This might fail for various reasons
35 * (device issue, device was unsafely unplugged, ...). When such
36 * error conditions happen, the callback must return VM_FAULT_SIGBUS.
38 * Note that because memory cgroup charges are accounted to the device
39 * memory, this should never fail because of memory restrictions (but
40 * allocation of regular system page might still fail because we are
43 * There is a more in-depth description of what that callback can and
44 * cannot do, in include/linux/memremap.h
46 return devmem
->page_fault(vma
, addr
, page
, flags
, pmdp
);
48 #endif /* CONFIG_DEVICE_PRIVATE */
50 static void pgmap_array_delete(struct resource
*res
)
52 xa_store_range(&pgmap_array
, PHYS_PFN(res
->start
), PHYS_PFN(res
->end
),
57 static unsigned long pfn_first(struct dev_pagemap
*pgmap
)
59 const struct resource
*res
= &pgmap
->res
;
60 struct vmem_altmap
*altmap
= &pgmap
->altmap
;
63 pfn
= res
->start
>> PAGE_SHIFT
;
64 if (pgmap
->altmap_valid
)
65 pfn
+= vmem_altmap_offset(altmap
);
69 static unsigned long pfn_end(struct dev_pagemap
*pgmap
)
71 const struct resource
*res
= &pgmap
->res
;
73 return (res
->start
+ resource_size(res
)) >> PAGE_SHIFT
;
76 static unsigned long pfn_next(unsigned long pfn
)
83 #define for_each_device_pfn(pfn, map) \
84 for (pfn = pfn_first(map); pfn < pfn_end(map); pfn = pfn_next(pfn))
86 static void devm_memremap_pages_release(void *data
)
88 struct dev_pagemap
*pgmap
= data
;
89 struct device
*dev
= pgmap
->dev
;
90 struct resource
*res
= &pgmap
->res
;
91 resource_size_t align_start
, align_size
;
95 pgmap
->kill(pgmap
->ref
);
96 for_each_device_pfn(pfn
, pgmap
)
97 put_page(pfn_to_page(pfn
));
99 /* pages are dead and unused, undo the arch mapping */
100 align_start
= res
->start
& ~(SECTION_SIZE
- 1);
101 align_size
= ALIGN(res
->start
+ resource_size(res
), SECTION_SIZE
)
104 nid
= page_to_nid(pfn_to_page(align_start
>> PAGE_SHIFT
));
107 if (pgmap
->type
== MEMORY_DEVICE_PRIVATE
) {
108 pfn
= align_start
>> PAGE_SHIFT
;
109 __remove_pages(page_zone(pfn_to_page(pfn
)), pfn
,
110 align_size
>> PAGE_SHIFT
, NULL
);
112 arch_remove_memory(nid
, align_start
, align_size
,
113 pgmap
->altmap_valid
? &pgmap
->altmap
: NULL
);
114 kasan_remove_zero_shadow(__va(align_start
), align_size
);
118 untrack_pfn(NULL
, PHYS_PFN(align_start
), align_size
);
119 pgmap_array_delete(res
);
120 dev_WARN_ONCE(dev
, pgmap
->altmap
.alloc
,
121 "%s: failed to free all reserved pages\n", __func__
);
125 * devm_memremap_pages - remap and provide memmap backing for the given resource
126 * @dev: hosting device for @res
127 * @pgmap: pointer to a struct dev_pagemap
130 * 1/ At a minimum the res, ref and type members of @pgmap must be initialized
131 * by the caller before passing it to this function
133 * 2/ The altmap field may optionally be initialized, in which case altmap_valid
134 * must be set to true
136 * 3/ pgmap->ref must be 'live' on entry and will be killed at
137 * devm_memremap_pages_release() time, or if this routine fails.
139 * 4/ res is expected to be a host memory range that could feasibly be
140 * treated as a "System RAM" range, i.e. not a device mmio range, but
141 * this is not enforced.
143 void *devm_memremap_pages(struct device
*dev
, struct dev_pagemap
*pgmap
)
145 resource_size_t align_start
, align_size
, align_end
;
146 struct vmem_altmap
*altmap
= pgmap
->altmap_valid
?
147 &pgmap
->altmap
: NULL
;
148 struct resource
*res
= &pgmap
->res
;
149 struct dev_pagemap
*conflict_pgmap
;
150 struct mhp_restrictions restrictions
= {
152 * We do not want any optional features only our own memmap
156 pgprot_t pgprot
= PAGE_KERNEL
;
157 int error
, nid
, is_ram
;
159 if (!pgmap
->ref
|| !pgmap
->kill
)
160 return ERR_PTR(-EINVAL
);
162 align_start
= res
->start
& ~(SECTION_SIZE
- 1);
163 align_size
= ALIGN(res
->start
+ resource_size(res
), SECTION_SIZE
)
165 align_end
= align_start
+ align_size
- 1;
167 conflict_pgmap
= get_dev_pagemap(PHYS_PFN(align_start
), NULL
);
168 if (conflict_pgmap
) {
169 dev_WARN(dev
, "Conflicting mapping in same section\n");
170 put_dev_pagemap(conflict_pgmap
);
171 return ERR_PTR(-ENOMEM
);
174 conflict_pgmap
= get_dev_pagemap(PHYS_PFN(align_end
), NULL
);
175 if (conflict_pgmap
) {
176 dev_WARN(dev
, "Conflicting mapping in same section\n");
177 put_dev_pagemap(conflict_pgmap
);
178 return ERR_PTR(-ENOMEM
);
181 is_ram
= region_intersects(align_start
, align_size
,
182 IORESOURCE_SYSTEM_RAM
, IORES_DESC_NONE
);
184 if (is_ram
!= REGION_DISJOINT
) {
185 WARN_ONCE(1, "%s attempted on %s region %pr\n", __func__
,
186 is_ram
== REGION_MIXED
? "mixed" : "ram", res
);
193 error
= xa_err(xa_store_range(&pgmap_array
, PHYS_PFN(res
->start
),
194 PHYS_PFN(res
->end
), pgmap
, GFP_KERNEL
));
198 nid
= dev_to_node(dev
);
202 error
= track_pfn_remap(NULL
, &pgprot
, PHYS_PFN(align_start
), 0,
210 * For device private memory we call add_pages() as we only need to
211 * allocate and initialize struct page for the device memory. More-
212 * over the device memory is un-accessible thus we do not want to
213 * create a linear mapping for the memory like arch_add_memory()
216 * For all other device memory types, which are accessible by
217 * the CPU, we do want the linear mapping and thus use
220 if (pgmap
->type
== MEMORY_DEVICE_PRIVATE
) {
221 error
= add_pages(nid
, align_start
>> PAGE_SHIFT
,
222 align_size
>> PAGE_SHIFT
, &restrictions
);
224 error
= kasan_add_zero_shadow(__va(align_start
), align_size
);
230 error
= arch_add_memory(nid
, align_start
, align_size
,
237 zone
= &NODE_DATA(nid
)->node_zones
[ZONE_DEVICE
];
238 move_pfn_range_to_zone(zone
, align_start
>> PAGE_SHIFT
,
239 align_size
>> PAGE_SHIFT
, altmap
);
247 * Initialization of the pages has been deferred until now in order
248 * to allow us to do the work while not holding the hotplug lock.
250 memmap_init_zone_device(&NODE_DATA(nid
)->node_zones
[ZONE_DEVICE
],
251 align_start
>> PAGE_SHIFT
,
252 align_size
>> PAGE_SHIFT
, pgmap
);
253 percpu_ref_get_many(pgmap
->ref
, pfn_end(pgmap
) - pfn_first(pgmap
));
255 error
= devm_add_action_or_reset(dev
, devm_memremap_pages_release
,
258 return ERR_PTR(error
);
260 return __va(res
->start
);
263 kasan_remove_zero_shadow(__va(align_start
), align_size
);
265 untrack_pfn(NULL
, PHYS_PFN(align_start
), align_size
);
267 pgmap_array_delete(res
);
269 pgmap
->kill(pgmap
->ref
);
270 return ERR_PTR(error
);
272 EXPORT_SYMBOL_GPL(devm_memremap_pages
);
274 unsigned long vmem_altmap_offset(struct vmem_altmap
*altmap
)
276 /* number of pfns from base where pfn_to_page() is valid */
277 return altmap
->reserve
+ altmap
->free
;
280 void vmem_altmap_free(struct vmem_altmap
*altmap
, unsigned long nr_pfns
)
282 altmap
->alloc
-= nr_pfns
;
286 * get_dev_pagemap() - take a new live reference on the dev_pagemap for @pfn
287 * @pfn: page frame number to lookup page_map
288 * @pgmap: optional known pgmap that already has a reference
290 * If @pgmap is non-NULL and covers @pfn it will be returned as-is. If @pgmap
291 * is non-NULL but does not cover @pfn the reference to it will be released.
293 struct dev_pagemap
*get_dev_pagemap(unsigned long pfn
,
294 struct dev_pagemap
*pgmap
)
296 resource_size_t phys
= PFN_PHYS(pfn
);
299 * In the cached case we're already holding a live reference.
302 if (phys
>= pgmap
->res
.start
&& phys
<= pgmap
->res
.end
)
304 put_dev_pagemap(pgmap
);
307 /* fall back to slow path lookup */
309 pgmap
= xa_load(&pgmap_array
, PHYS_PFN(phys
));
310 if (pgmap
&& !percpu_ref_tryget_live(pgmap
->ref
))
316 EXPORT_SYMBOL_GPL(get_dev_pagemap
);
318 #ifdef CONFIG_DEV_PAGEMAP_OPS
319 DEFINE_STATIC_KEY_FALSE(devmap_managed_key
);
320 EXPORT_SYMBOL(devmap_managed_key
);
321 static atomic_t devmap_enable
;
324 * Toggle the static key for ->page_free() callbacks when dev_pagemap
327 void dev_pagemap_get_ops(void)
329 if (atomic_inc_return(&devmap_enable
) == 1)
330 static_branch_enable(&devmap_managed_key
);
332 EXPORT_SYMBOL_GPL(dev_pagemap_get_ops
);
334 void dev_pagemap_put_ops(void)
336 if (atomic_dec_and_test(&devmap_enable
))
337 static_branch_disable(&devmap_managed_key
);
339 EXPORT_SYMBOL_GPL(dev_pagemap_put_ops
);
341 void __put_devmap_managed_page(struct page
*page
)
343 int count
= page_ref_dec_return(page
);
346 * If refcount is 1 then page is freed and refcount is stable as nobody
347 * holds a reference on the page.
350 /* Clear Active bit in case of parallel mark_page_accessed */
351 __ClearPageActive(page
);
352 __ClearPageWaiters(page
);
354 mem_cgroup_uncharge(page
);
356 page
->pgmap
->page_free(page
, page
->pgmap
->data
);
360 EXPORT_SYMBOL(__put_devmap_managed_page
);
361 #endif /* CONFIG_DEV_PAGEMAP_OPS */