]>
Commit | Line | Data |
---|---|---|
133ff0ea JG |
1 | /* |
2 | * Copyright 2013 Red Hat Inc. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or modify | |
5 | * it under the terms of the GNU General Public License as published by | |
6 | * the Free Software Foundation; either version 2 of the License, or | |
7 | * (at your option) any later version. | |
8 | * | |
9 | * This program is distributed in the hope that it will be useful, | |
10 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | * GNU General Public License for more details. | |
13 | * | |
14 | * Authors: Jérôme Glisse <jglisse@redhat.com> | |
15 | */ | |
16 | /* | |
17 | * Heterogeneous Memory Management (HMM) | |
18 | * | |
19 | * See Documentation/vm/hmm.txt for reasons and overview of what HMM is and it | |
20 | * is for. Here we focus on the HMM API description, with some explanation of | |
21 | * the underlying implementation. | |
22 | * | |
23 | * Short description: HMM provides a set of helpers to share a virtual address | |
24 | * space between CPU and a device, so that the device can access any valid | |
25 | * address of the process (while still obeying memory protection). HMM also | |
26 | * provides helpers to migrate process memory to device memory, and back. Each | |
27 | * set of functionality (address space mirroring, and migration to and from | |
28 | * device memory) can be used independently of the other. | |
29 | * | |
30 | * | |
31 | * HMM address space mirroring API: | |
32 | * | |
33 | * Use HMM address space mirroring if you want to mirror range of the CPU page | |
34 | * table of a process into a device page table. Here, "mirror" means "keep | |
35 | * synchronized". Prerequisites: the device must provide the ability to write- | |
36 | * protect its page tables (at PAGE_SIZE granularity), and must be able to | |
37 | * recover from the resulting potential page faults. | |
38 | * | |
39 | * HMM guarantees that at any point in time, a given virtual address points to | |
40 | * either the same memory in both CPU and device page tables (that is: CPU and | |
41 | * device page tables each point to the same pages), or that one page table (CPU | |
42 | * or device) points to no entry, while the other still points to the old page | |
43 | * for the address. The latter case happens when the CPU page table update | |
44 | * happens first, and then the update is mirrored over to the device page table. | |
45 | * This does not cause any issue, because the CPU page table cannot start | |
46 | * pointing to a new page until the device page table is invalidated. | |
47 | * | |
48 | * HMM uses mmu_notifiers to monitor the CPU page tables, and forwards any | |
49 | * updates to each device driver that has registered a mirror. It also provides | |
50 | * some API calls to help with taking a snapshot of the CPU page table, and to | |
51 | * synchronize with any updates that might happen concurrently. | |
52 | * | |
53 | * | |
54 | * HMM migration to and from device memory: | |
55 | * | |
56 | * HMM provides a set of helpers to hotplug device memory as ZONE_DEVICE, with | |
57 | * a new MEMORY_DEVICE_PRIVATE type. This provides a struct page for each page | |
58 | * of the device memory, and allows the device driver to manage its memory | |
59 | * using those struct pages. Having struct pages for device memory makes | |
60 | * migration easier. Because that memory is not addressable by the CPU it must | |
61 | * never be pinned to the device; in other words, any CPU page fault can always | |
62 | * cause the device memory to be migrated (copied/moved) back to regular memory. | |
63 | * | |
64 | * A new migrate helper (migrate_vma()) has been added (see mm/migrate.c) that | |
65 | * allows use of a device DMA engine to perform the copy operation between | |
66 | * regular system memory and device memory. | |
67 | */ | |
68 | #ifndef LINUX_HMM_H | |
69 | #define LINUX_HMM_H | |
70 | ||
71 | #include <linux/kconfig.h> | |
72 | ||
73 | #if IS_ENABLED(CONFIG_HMM) | |
74 | ||
c0b12405 | 75 | struct hmm; |
133ff0ea JG |
76 | |
77 | /* | |
78 | * hmm_pfn_t - HMM uses its own pfn type to keep several flags per page | |
79 | * | |
80 | * Flags: | |
81 | * HMM_PFN_VALID: pfn is valid | |
82 | * HMM_PFN_WRITE: CPU page table has write permission set | |
83 | */ | |
84 | typedef unsigned long hmm_pfn_t; | |
85 | ||
86 | #define HMM_PFN_VALID (1 << 0) | |
87 | #define HMM_PFN_WRITE (1 << 1) | |
88 | #define HMM_PFN_SHIFT 2 | |
89 | ||
90 | /* | |
91 | * hmm_pfn_t_to_page() - return struct page pointed to by a valid hmm_pfn_t | |
92 | * @pfn: hmm_pfn_t to convert to struct page | |
93 | * Returns: struct page pointer if pfn is a valid hmm_pfn_t, NULL otherwise | |
94 | * | |
95 | * If the hmm_pfn_t is valid (ie valid flag set) then return the struct page | |
96 | * matching the pfn value stored in the hmm_pfn_t. Otherwise return NULL. | |
97 | */ | |
98 | static inline struct page *hmm_pfn_t_to_page(hmm_pfn_t pfn) | |
99 | { | |
100 | if (!(pfn & HMM_PFN_VALID)) | |
101 | return NULL; | |
102 | return pfn_to_page(pfn >> HMM_PFN_SHIFT); | |
103 | } | |
104 | ||
105 | /* | |
106 | * hmm_pfn_t_to_pfn() - return pfn value store in a hmm_pfn_t | |
107 | * @pfn: hmm_pfn_t to extract pfn from | |
108 | * Returns: pfn value if hmm_pfn_t is valid, -1UL otherwise | |
109 | */ | |
110 | static inline unsigned long hmm_pfn_t_to_pfn(hmm_pfn_t pfn) | |
111 | { | |
112 | if (!(pfn & HMM_PFN_VALID)) | |
113 | return -1UL; | |
114 | return (pfn >> HMM_PFN_SHIFT); | |
115 | } | |
116 | ||
117 | /* | |
118 | * hmm_pfn_t_from_page() - create a valid hmm_pfn_t value from struct page | |
119 | * @page: struct page pointer for which to create the hmm_pfn_t | |
120 | * Returns: valid hmm_pfn_t for the page | |
121 | */ | |
122 | static inline hmm_pfn_t hmm_pfn_t_from_page(struct page *page) | |
123 | { | |
124 | return (page_to_pfn(page) << HMM_PFN_SHIFT) | HMM_PFN_VALID; | |
125 | } | |
126 | ||
127 | /* | |
128 | * hmm_pfn_t_from_pfn() - create a valid hmm_pfn_t value from pfn | |
129 | * @pfn: pfn value for which to create the hmm_pfn_t | |
130 | * Returns: valid hmm_pfn_t for the pfn | |
131 | */ | |
132 | static inline hmm_pfn_t hmm_pfn_t_from_pfn(unsigned long pfn) | |
133 | { | |
134 | return (pfn << HMM_PFN_SHIFT) | HMM_PFN_VALID; | |
135 | } | |
136 | ||
137 | ||
c0b12405 JG |
138 | #if IS_ENABLED(CONFIG_HMM_MIRROR) |
139 | /* | |
140 | * Mirroring: how to synchronize device page table with CPU page table. | |
141 | * | |
142 | * A device driver that is participating in HMM mirroring must always | |
143 | * synchronize with CPU page table updates. For this, device drivers can either | |
144 | * directly use mmu_notifier APIs or they can use the hmm_mirror API. Device | |
145 | * drivers can decide to register one mirror per device per process, or just | |
146 | * one mirror per process for a group of devices. The pattern is: | |
147 | * | |
148 | * int device_bind_address_space(..., struct mm_struct *mm, ...) | |
149 | * { | |
150 | * struct device_address_space *das; | |
151 | * | |
152 | * // Device driver specific initialization, and allocation of das | |
153 | * // which contains an hmm_mirror struct as one of its fields. | |
154 | * ... | |
155 | * | |
156 | * ret = hmm_mirror_register(&das->mirror, mm, &device_mirror_ops); | |
157 | * if (ret) { | |
158 | * // Cleanup on error | |
159 | * return ret; | |
160 | * } | |
161 | * | |
162 | * // Other device driver specific initialization | |
163 | * ... | |
164 | * } | |
165 | * | |
166 | * Once an hmm_mirror is registered for an address space, the device driver | |
167 | * will get callbacks through sync_cpu_device_pagetables() operation (see | |
168 | * hmm_mirror_ops struct). | |
169 | * | |
170 | * Device driver must not free the struct containing the hmm_mirror struct | |
171 | * before calling hmm_mirror_unregister(). The expected usage is to do that when | |
172 | * the device driver is unbinding from an address space. | |
173 | * | |
174 | * | |
175 | * void device_unbind_address_space(struct device_address_space *das) | |
176 | * { | |
177 | * // Device driver specific cleanup | |
178 | * ... | |
179 | * | |
180 | * hmm_mirror_unregister(&das->mirror); | |
181 | * | |
182 | * // Other device driver specific cleanup, and now das can be freed | |
183 | * ... | |
184 | * } | |
185 | */ | |
186 | ||
187 | struct hmm_mirror; | |
188 | ||
189 | /* | |
190 | * enum hmm_update_type - type of update | |
191 | * @HMM_UPDATE_INVALIDATE: invalidate range (no indication as to why) | |
192 | */ | |
193 | enum hmm_update_type { | |
194 | HMM_UPDATE_INVALIDATE, | |
195 | }; | |
196 | ||
197 | /* | |
198 | * struct hmm_mirror_ops - HMM mirror device operations callback | |
199 | * | |
200 | * @update: callback to update range on a device | |
201 | */ | |
202 | struct hmm_mirror_ops { | |
203 | /* sync_cpu_device_pagetables() - synchronize page tables | |
204 | * | |
205 | * @mirror: pointer to struct hmm_mirror | |
206 | * @update_type: type of update that occurred to the CPU page table | |
207 | * @start: virtual start address of the range to update | |
208 | * @end: virtual end address of the range to update | |
209 | * | |
210 | * This callback ultimately originates from mmu_notifiers when the CPU | |
211 | * page table is updated. The device driver must update its page table | |
212 | * in response to this callback. The update argument tells what action | |
213 | * to perform. | |
214 | * | |
215 | * The device driver must not return from this callback until the device | |
216 | * page tables are completely updated (TLBs flushed, etc); this is a | |
217 | * synchronous call. | |
218 | */ | |
219 | void (*sync_cpu_device_pagetables)(struct hmm_mirror *mirror, | |
220 | enum hmm_update_type update_type, | |
221 | unsigned long start, | |
222 | unsigned long end); | |
223 | }; | |
224 | ||
225 | /* | |
226 | * struct hmm_mirror - mirror struct for a device driver | |
227 | * | |
228 | * @hmm: pointer to struct hmm (which is unique per mm_struct) | |
229 | * @ops: device driver callback for HMM mirror operations | |
230 | * @list: for list of mirrors of a given mm | |
231 | * | |
232 | * Each address space (mm_struct) being mirrored by a device must register one | |
233 | * instance of an hmm_mirror struct with HMM. HMM will track the list of all | |
234 | * mirrors for each mm_struct. | |
235 | */ | |
236 | struct hmm_mirror { | |
237 | struct hmm *hmm; | |
238 | const struct hmm_mirror_ops *ops; | |
239 | struct list_head list; | |
240 | }; | |
241 | ||
242 | int hmm_mirror_register(struct hmm_mirror *mirror, struct mm_struct *mm); | |
243 | void hmm_mirror_unregister(struct hmm_mirror *mirror); | |
244 | #endif /* IS_ENABLED(CONFIG_HMM_MIRROR) */ | |
245 | ||
246 | ||
133ff0ea JG |
247 | /* Below are for HMM internal use only! Not to be used by device driver! */ |
248 | void hmm_mm_destroy(struct mm_struct *mm); | |
249 | ||
250 | static inline void hmm_mm_init(struct mm_struct *mm) | |
251 | { | |
252 | mm->hmm = NULL; | |
253 | } | |
254 | ||
255 | #else /* IS_ENABLED(CONFIG_HMM) */ | |
256 | ||
257 | /* Below are for HMM internal use only! Not to be used by device driver! */ | |
258 | static inline void hmm_mm_destroy(struct mm_struct *mm) {} | |
259 | static inline void hmm_mm_init(struct mm_struct *mm) {} | |
260 | ||
261 | #endif /* IS_ENABLED(CONFIG_HMM) */ | |
262 | #endif /* LINUX_HMM_H */ |