]>
Commit | Line | Data |
---|---|---|
06e0ffa6 MH |
1 | /* |
2 | * Copyright(c) 2016 Intel Corporation. | |
3 | * | |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
20 | * Redistribution and use in source and binary forms, with or without | |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | #include <linux/list.h> | |
67caea1f | 48 | #include <linux/rculist.h> |
06e0ffa6 | 49 | #include <linux/mmu_notifier.h> |
df5a00f8 | 50 | #include <linux/interval_tree_generic.h> |
06e0ffa6 MH |
51 | |
52 | #include "mmu_rb.h" | |
53 | #include "trace.h" | |
54 | ||
55 | struct mmu_rb_handler { | |
06e0ffa6 | 56 | struct mmu_notifier mn; |
e0b09ac5 DL |
57 | struct rb_root root; |
58 | void *ops_arg; | |
06e0ffa6 MH |
59 | spinlock_t lock; /* protect the RB tree */ |
60 | struct mmu_rb_ops *ops; | |
3faa3d9a | 61 | struct mm_struct *mm; |
0636e9ab | 62 | struct list_head lru_list; |
b85ced91 DL |
63 | struct work_struct del_work; |
64 | struct list_head del_list; | |
65 | struct workqueue_struct *wq; | |
06e0ffa6 MH |
66 | }; |
67 | ||
df5a00f8 MH |
68 | static unsigned long mmu_node_start(struct mmu_rb_node *); |
69 | static unsigned long mmu_node_last(struct mmu_rb_node *); | |
06e0ffa6 MH |
70 | static inline void mmu_notifier_range_start(struct mmu_notifier *, |
71 | struct mm_struct *, | |
72 | unsigned long, unsigned long); | |
73 | static void mmu_notifier_mem_invalidate(struct mmu_notifier *, | |
f19bd643 | 74 | struct mm_struct *, |
06e0ffa6 MH |
75 | unsigned long, unsigned long); |
76 | static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *, | |
77 | unsigned long, unsigned long); | |
b85ced91 DL |
78 | static void do_remove(struct mmu_rb_handler *handler, |
79 | struct list_head *del_list); | |
80 | static void handle_remove(struct work_struct *work); | |
06e0ffa6 | 81 | |
0fc859a6 | 82 | static const struct mmu_notifier_ops mn_opts = { |
06e0ffa6 MH |
83 | .invalidate_range_start = mmu_notifier_range_start, |
84 | }; | |
85 | ||
df5a00f8 MH |
86 | INTERVAL_TREE_DEFINE(struct mmu_rb_node, node, unsigned long, __last, |
87 | mmu_node_start, mmu_node_last, static, __mmu_int_rb); | |
88 | ||
89 | static unsigned long mmu_node_start(struct mmu_rb_node *node) | |
90 | { | |
91 | return node->addr & PAGE_MASK; | |
92 | } | |
93 | ||
94 | static unsigned long mmu_node_last(struct mmu_rb_node *node) | |
95 | { | |
de79093b | 96 | return PAGE_ALIGN(node->addr + node->len) - 1; |
df5a00f8 MH |
97 | } |
98 | ||
e0b09ac5 DL |
99 | int hfi1_mmu_rb_register(void *ops_arg, struct mm_struct *mm, |
100 | struct mmu_rb_ops *ops, | |
b85ced91 | 101 | struct workqueue_struct *wq, |
e0b09ac5 | 102 | struct mmu_rb_handler **handler) |
06e0ffa6 MH |
103 | { |
104 | struct mmu_rb_handler *handlr; | |
3faa3d9a | 105 | int ret; |
06e0ffa6 | 106 | |
06e0ffa6 MH |
107 | handlr = kmalloc(sizeof(*handlr), GFP_KERNEL); |
108 | if (!handlr) | |
109 | return -ENOMEM; | |
110 | ||
e0b09ac5 | 111 | handlr->root = RB_ROOT; |
06e0ffa6 | 112 | handlr->ops = ops; |
e0b09ac5 | 113 | handlr->ops_arg = ops_arg; |
06e0ffa6 MH |
114 | INIT_HLIST_NODE(&handlr->mn.hlist); |
115 | spin_lock_init(&handlr->lock); | |
116 | handlr->mn.ops = &mn_opts; | |
3faa3d9a | 117 | handlr->mm = mm; |
b85ced91 DL |
118 | INIT_WORK(&handlr->del_work, handle_remove); |
119 | INIT_LIST_HEAD(&handlr->del_list); | |
0636e9ab | 120 | INIT_LIST_HEAD(&handlr->lru_list); |
b85ced91 | 121 | handlr->wq = wq; |
3faa3d9a IW |
122 | |
123 | ret = mmu_notifier_register(&handlr->mn, handlr->mm); | |
124 | if (ret) { | |
125 | kfree(handlr); | |
126 | return ret; | |
127 | } | |
128 | ||
e0b09ac5 DL |
129 | *handler = handlr; |
130 | return 0; | |
06e0ffa6 MH |
131 | } |
132 | ||
e0b09ac5 | 133 | void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler) |
06e0ffa6 | 134 | { |
20a42d08 DL |
135 | struct mmu_rb_node *rbnode; |
136 | struct rb_node *node; | |
c81e1f64 | 137 | unsigned long flags; |
b85ced91 | 138 | struct list_head del_list; |
06e0ffa6 | 139 | |
782f6697 | 140 | /* Unregister first so we don't get any more notifications. */ |
3faa3d9a | 141 | mmu_notifier_unregister(&handler->mn, handler->mm); |
782f6697 | 142 | |
b85ced91 DL |
143 | /* |
144 | * Make sure the wq delete handler is finished running. It will not | |
145 | * be triggered once the mmu notifiers are unregistered above. | |
146 | */ | |
147 | flush_work(&handler->del_work); | |
148 | ||
149 | INIT_LIST_HEAD(&del_list); | |
150 | ||
782f6697 | 151 | spin_lock_irqsave(&handler->lock, flags); |
e0b09ac5 | 152 | while ((node = rb_first(&handler->root))) { |
20a42d08 | 153 | rbnode = rb_entry(node, struct mmu_rb_node, node); |
e0b09ac5 | 154 | rb_erase(node, &handler->root); |
0636e9ab DL |
155 | /* move from LRU list to delete list */ |
156 | list_move(&rbnode->list, &del_list); | |
06e0ffa6 | 157 | } |
782f6697 | 158 | spin_unlock_irqrestore(&handler->lock, flags); |
06e0ffa6 | 159 | |
b85ced91 DL |
160 | do_remove(handler, &del_list); |
161 | ||
06e0ffa6 MH |
162 | kfree(handler); |
163 | } | |
164 | ||
e0b09ac5 DL |
165 | int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler, |
166 | struct mmu_rb_node *mnode) | |
06e0ffa6 | 167 | { |
df5a00f8 | 168 | struct mmu_rb_node *node; |
c81e1f64 | 169 | unsigned long flags; |
df5a00f8 | 170 | int ret = 0; |
06e0ffa6 | 171 | |
c81e1f64 | 172 | spin_lock_irqsave(&handler->lock, flags); |
353b71c7 MH |
173 | hfi1_cdbg(MMU, "Inserting node addr 0x%llx, len %u", mnode->addr, |
174 | mnode->len); | |
df5a00f8 MH |
175 | node = __mmu_rb_search(handler, mnode->addr, mnode->len); |
176 | if (node) { | |
177 | ret = -EINVAL; | |
178 | goto unlock; | |
06e0ffa6 | 179 | } |
e0b09ac5 | 180 | __mmu_int_rb_insert(mnode, &handler->root); |
0636e9ab | 181 | list_add(&mnode->list, &handler->lru_list); |
06e0ffa6 | 182 | |
e0b09ac5 | 183 | ret = handler->ops->insert(handler->ops_arg, mnode); |
0636e9ab | 184 | if (ret) { |
e0b09ac5 | 185 | __mmu_int_rb_remove(mnode, &handler->root); |
0636e9ab DL |
186 | list_del(&mnode->list); /* remove from LRU list */ |
187 | } | |
06e0ffa6 | 188 | unlock: |
c81e1f64 | 189 | spin_unlock_irqrestore(&handler->lock, flags); |
06e0ffa6 MH |
190 | return ret; |
191 | } | |
192 | ||
de82bdff | 193 | /* Caller must hold handler lock */ |
06e0ffa6 MH |
194 | static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler, |
195 | unsigned long addr, | |
196 | unsigned long len) | |
197 | { | |
0f310a00 | 198 | struct mmu_rb_node *node = NULL; |
df5a00f8 | 199 | |
353b71c7 | 200 | hfi1_cdbg(MMU, "Searching for addr 0x%llx, len %u", addr, len); |
0f310a00 | 201 | if (!handler->ops->filter) { |
e0b09ac5 | 202 | node = __mmu_int_rb_iter_first(&handler->root, addr, |
0f310a00 MH |
203 | (addr + len) - 1); |
204 | } else { | |
e0b09ac5 | 205 | for (node = __mmu_int_rb_iter_first(&handler->root, addr, |
0f310a00 MH |
206 | (addr + len) - 1); |
207 | node; | |
208 | node = __mmu_int_rb_iter_next(node, addr, | |
209 | (addr + len) - 1)) { | |
210 | if (handler->ops->filter(node, addr, len)) | |
211 | return node; | |
212 | } | |
213 | } | |
df5a00f8 | 214 | return node; |
06e0ffa6 MH |
215 | } |
216 | ||
e0b09ac5 | 217 | struct mmu_rb_node *hfi1_mmu_rb_extract(struct mmu_rb_handler *handler, |
f53af85e MH |
218 | unsigned long addr, unsigned long len) |
219 | { | |
f53af85e MH |
220 | struct mmu_rb_node *node; |
221 | unsigned long flags; | |
222 | ||
f53af85e MH |
223 | spin_lock_irqsave(&handler->lock, flags); |
224 | node = __mmu_rb_search(handler, addr, len); | |
0636e9ab | 225 | if (node) { |
e0b09ac5 | 226 | __mmu_int_rb_remove(node, &handler->root); |
0636e9ab DL |
227 | list_del(&node->list); /* remove from LRU list */ |
228 | } | |
f53af85e MH |
229 | spin_unlock_irqrestore(&handler->lock, flags); |
230 | ||
231 | return node; | |
232 | } | |
233 | ||
10345998 DL |
234 | void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg) |
235 | { | |
0636e9ab | 236 | struct mmu_rb_node *rbnode, *ptr; |
10345998 DL |
237 | struct list_head del_list; |
238 | unsigned long flags; | |
239 | bool stop = false; | |
240 | ||
241 | INIT_LIST_HEAD(&del_list); | |
242 | ||
243 | spin_lock_irqsave(&handler->lock, flags); | |
0636e9ab DL |
244 | list_for_each_entry_safe_reverse(rbnode, ptr, &handler->lru_list, |
245 | list) { | |
10345998 DL |
246 | if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg, |
247 | &stop)) { | |
248 | __mmu_int_rb_remove(rbnode, &handler->root); | |
0636e9ab DL |
249 | /* move from LRU list to delete list */ |
250 | list_move(&rbnode->list, &del_list); | |
10345998 DL |
251 | } |
252 | if (stop) | |
253 | break; | |
254 | } | |
255 | spin_unlock_irqrestore(&handler->lock, flags); | |
256 | ||
10345998 DL |
257 | while (!list_empty(&del_list)) { |
258 | rbnode = list_first_entry(&del_list, struct mmu_rb_node, list); | |
259 | list_del(&rbnode->list); | |
082b3532 | 260 | handler->ops->remove(handler->ops_arg, rbnode); |
10345998 | 261 | } |
10345998 DL |
262 | } |
263 | ||
b85ced91 DL |
264 | /* |
265 | * It is up to the caller to ensure that this function does not race with the | |
266 | * mmu invalidate notifier which may be calling the users remove callback on | |
267 | * 'node'. | |
268 | */ | |
e0b09ac5 DL |
269 | void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler, |
270 | struct mmu_rb_node *node) | |
06e0ffa6 | 271 | { |
3c1091aa | 272 | unsigned long flags; |
06e0ffa6 | 273 | |
3c1091aa IW |
274 | /* Validity of handler and node pointers has been checked by caller. */ |
275 | hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr, | |
276 | node->len); | |
277 | spin_lock_irqsave(&handler->lock, flags); | |
e0b09ac5 | 278 | __mmu_int_rb_remove(node, &handler->root); |
0636e9ab | 279 | list_del(&node->list); /* remove from LRU list */ |
3c1091aa IW |
280 | spin_unlock_irqrestore(&handler->lock, flags); |
281 | ||
082b3532 | 282 | handler->ops->remove(handler->ops_arg, node); |
06e0ffa6 MH |
283 | } |
284 | ||
06e0ffa6 MH |
285 | static inline void mmu_notifier_range_start(struct mmu_notifier *mn, |
286 | struct mm_struct *mm, | |
287 | unsigned long start, | |
288 | unsigned long end) | |
289 | { | |
f19bd643 | 290 | mmu_notifier_mem_invalidate(mn, mm, start, end); |
06e0ffa6 MH |
291 | } |
292 | ||
293 | static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn, | |
f19bd643 | 294 | struct mm_struct *mm, |
06e0ffa6 MH |
295 | unsigned long start, unsigned long end) |
296 | { | |
297 | struct mmu_rb_handler *handler = | |
298 | container_of(mn, struct mmu_rb_handler, mn); | |
e0b09ac5 | 299 | struct rb_root *root = &handler->root; |
f19bd643 | 300 | struct mmu_rb_node *node, *ptr = NULL; |
df5a00f8 | 301 | unsigned long flags; |
b85ced91 | 302 | bool added = false; |
06e0ffa6 | 303 | |
c81e1f64 | 304 | spin_lock_irqsave(&handler->lock, flags); |
f19bd643 MH |
305 | for (node = __mmu_int_rb_iter_first(root, start, end - 1); |
306 | node; node = ptr) { | |
307 | /* Guard against node removal. */ | |
308 | ptr = __mmu_int_rb_iter_next(node, start, end - 1); | |
353b71c7 MH |
309 | hfi1_cdbg(MMU, "Invalidating node addr 0x%llx, len %u", |
310 | node->addr, node->len); | |
e0b09ac5 | 311 | if (handler->ops->invalidate(handler->ops_arg, node)) { |
e88c9271 | 312 | __mmu_int_rb_remove(node, root); |
0636e9ab DL |
313 | /* move from LRU list to delete list */ |
314 | list_move(&node->list, &handler->del_list); | |
b85ced91 | 315 | added = true; |
de82bdff | 316 | } |
06e0ffa6 | 317 | } |
c81e1f64 | 318 | spin_unlock_irqrestore(&handler->lock, flags); |
b85ced91 DL |
319 | |
320 | if (added) | |
321 | queue_work(handler->wq, &handler->del_work); | |
322 | } | |
323 | ||
324 | /* | |
325 | * Call the remove function for the given handler and the list. This | |
326 | * is expected to be called with a delete list extracted from handler. | |
327 | * The caller should not be holding the handler lock. | |
328 | */ | |
329 | static void do_remove(struct mmu_rb_handler *handler, | |
330 | struct list_head *del_list) | |
331 | { | |
332 | struct mmu_rb_node *node; | |
333 | ||
334 | while (!list_empty(del_list)) { | |
335 | node = list_first_entry(del_list, struct mmu_rb_node, list); | |
336 | list_del(&node->list); | |
082b3532 | 337 | handler->ops->remove(handler->ops_arg, node); |
b85ced91 DL |
338 | } |
339 | } | |
340 | ||
341 | /* | |
342 | * Work queue function to remove all nodes that have been queued up to | |
343 | * be removed. The key feature is that mm->mmap_sem is not being held | |
344 | * and the remove callback can sleep while taking it, if needed. | |
345 | */ | |
346 | static void handle_remove(struct work_struct *work) | |
347 | { | |
348 | struct mmu_rb_handler *handler = container_of(work, | |
349 | struct mmu_rb_handler, | |
350 | del_work); | |
351 | struct list_head del_list; | |
352 | unsigned long flags; | |
353 | ||
354 | /* remove anything that is queued to get removed */ | |
355 | spin_lock_irqsave(&handler->lock, flags); | |
356 | list_replace_init(&handler->del_list, &del_list); | |
357 | spin_unlock_irqrestore(&handler->lock, flags); | |
358 | ||
359 | do_remove(handler, &del_list); | |
06e0ffa6 | 360 | } |