]>
Commit | Line | Data |
---|---|---|
06e0ffa6 MH |
1 | /* |
2 | * Copyright(c) 2016 Intel Corporation. | |
3 | * | |
4 | * This file is provided under a dual BSD/GPLv2 license. When using or | |
5 | * redistributing this file, you may do so under either license. | |
6 | * | |
7 | * GPL LICENSE SUMMARY | |
8 | * | |
9 | * This program is free software; you can redistribute it and/or modify | |
10 | * it under the terms of version 2 of the GNU General Public License as | |
11 | * published by the Free Software Foundation. | |
12 | * | |
13 | * This program is distributed in the hope that it will be useful, but | |
14 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
15 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU | |
16 | * General Public License for more details. | |
17 | * | |
18 | * BSD LICENSE | |
19 | * | |
20 | * Redistribution and use in source and binary forms, with or without | |
21 | * modification, are permitted provided that the following conditions | |
22 | * are met: | |
23 | * | |
24 | * - Redistributions of source code must retain the above copyright | |
25 | * notice, this list of conditions and the following disclaimer. | |
26 | * - Redistributions in binary form must reproduce the above copyright | |
27 | * notice, this list of conditions and the following disclaimer in | |
28 | * the documentation and/or other materials provided with the | |
29 | * distribution. | |
30 | * - Neither the name of Intel Corporation nor the names of its | |
31 | * contributors may be used to endorse or promote products derived | |
32 | * from this software without specific prior written permission. | |
33 | * | |
34 | * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS | |
35 | * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT | |
36 | * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR | |
37 | * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT | |
38 | * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, | |
39 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT | |
40 | * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, | |
41 | * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY | |
42 | * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT | |
43 | * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE | |
44 | * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. | |
45 | * | |
46 | */ | |
47 | #include <linux/list.h> | |
67caea1f | 48 | #include <linux/rculist.h> |
06e0ffa6 | 49 | #include <linux/mmu_notifier.h> |
df5a00f8 | 50 | #include <linux/interval_tree_generic.h> |
06e0ffa6 MH |
51 | |
52 | #include "mmu_rb.h" | |
53 | #include "trace.h" | |
54 | ||
55 | struct mmu_rb_handler { | |
06e0ffa6 | 56 | struct mmu_notifier mn; |
e0b09ac5 DL |
57 | struct rb_root root; |
58 | void *ops_arg; | |
06e0ffa6 MH |
59 | spinlock_t lock; /* protect the RB tree */ |
60 | struct mmu_rb_ops *ops; | |
3faa3d9a | 61 | struct mm_struct *mm; |
06e0ffa6 MH |
62 | }; |
63 | ||
df5a00f8 MH |
64 | static unsigned long mmu_node_start(struct mmu_rb_node *); |
65 | static unsigned long mmu_node_last(struct mmu_rb_node *); | |
06e0ffa6 MH |
66 | static inline void mmu_notifier_page(struct mmu_notifier *, struct mm_struct *, |
67 | unsigned long); | |
68 | static inline void mmu_notifier_range_start(struct mmu_notifier *, | |
69 | struct mm_struct *, | |
70 | unsigned long, unsigned long); | |
71 | static void mmu_notifier_mem_invalidate(struct mmu_notifier *, | |
f19bd643 | 72 | struct mm_struct *, |
06e0ffa6 MH |
73 | unsigned long, unsigned long); |
74 | static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *, | |
75 | unsigned long, unsigned long); | |
76 | ||
77 | static struct mmu_notifier_ops mn_opts = { | |
78 | .invalidate_page = mmu_notifier_page, | |
79 | .invalidate_range_start = mmu_notifier_range_start, | |
80 | }; | |
81 | ||
df5a00f8 MH |
82 | INTERVAL_TREE_DEFINE(struct mmu_rb_node, node, unsigned long, __last, |
83 | mmu_node_start, mmu_node_last, static, __mmu_int_rb); | |
84 | ||
85 | static unsigned long mmu_node_start(struct mmu_rb_node *node) | |
86 | { | |
87 | return node->addr & PAGE_MASK; | |
88 | } | |
89 | ||
90 | static unsigned long mmu_node_last(struct mmu_rb_node *node) | |
91 | { | |
de79093b | 92 | return PAGE_ALIGN(node->addr + node->len) - 1; |
df5a00f8 MH |
93 | } |
94 | ||
e0b09ac5 DL |
95 | int hfi1_mmu_rb_register(void *ops_arg, struct mm_struct *mm, |
96 | struct mmu_rb_ops *ops, | |
97 | struct mmu_rb_handler **handler) | |
06e0ffa6 MH |
98 | { |
99 | struct mmu_rb_handler *handlr; | |
3faa3d9a | 100 | int ret; |
06e0ffa6 | 101 | |
06e0ffa6 MH |
102 | handlr = kmalloc(sizeof(*handlr), GFP_KERNEL); |
103 | if (!handlr) | |
104 | return -ENOMEM; | |
105 | ||
e0b09ac5 | 106 | handlr->root = RB_ROOT; |
06e0ffa6 | 107 | handlr->ops = ops; |
e0b09ac5 | 108 | handlr->ops_arg = ops_arg; |
06e0ffa6 MH |
109 | INIT_HLIST_NODE(&handlr->mn.hlist); |
110 | spin_lock_init(&handlr->lock); | |
111 | handlr->mn.ops = &mn_opts; | |
3faa3d9a IW |
112 | handlr->mm = mm; |
113 | ||
114 | ret = mmu_notifier_register(&handlr->mn, handlr->mm); | |
115 | if (ret) { | |
116 | kfree(handlr); | |
117 | return ret; | |
118 | } | |
119 | ||
e0b09ac5 DL |
120 | *handler = handlr; |
121 | return 0; | |
06e0ffa6 MH |
122 | } |
123 | ||
e0b09ac5 | 124 | void hfi1_mmu_rb_unregister(struct mmu_rb_handler *handler) |
06e0ffa6 | 125 | { |
20a42d08 DL |
126 | struct mmu_rb_node *rbnode; |
127 | struct rb_node *node; | |
c81e1f64 | 128 | unsigned long flags; |
06e0ffa6 | 129 | |
782f6697 | 130 | /* Unregister first so we don't get any more notifications. */ |
3faa3d9a | 131 | mmu_notifier_unregister(&handler->mn, handler->mm); |
782f6697 | 132 | |
782f6697 | 133 | spin_lock_irqsave(&handler->lock, flags); |
e0b09ac5 | 134 | while ((node = rb_first(&handler->root))) { |
20a42d08 | 135 | rbnode = rb_entry(node, struct mmu_rb_node, node); |
e0b09ac5 DL |
136 | rb_erase(node, &handler->root); |
137 | handler->ops->remove(handler->ops_arg, rbnode, | |
138 | NULL); | |
06e0ffa6 | 139 | } |
782f6697 | 140 | spin_unlock_irqrestore(&handler->lock, flags); |
06e0ffa6 | 141 | |
06e0ffa6 MH |
142 | kfree(handler); |
143 | } | |
144 | ||
e0b09ac5 DL |
145 | int hfi1_mmu_rb_insert(struct mmu_rb_handler *handler, |
146 | struct mmu_rb_node *mnode) | |
06e0ffa6 | 147 | { |
df5a00f8 | 148 | struct mmu_rb_node *node; |
c81e1f64 | 149 | unsigned long flags; |
df5a00f8 | 150 | int ret = 0; |
06e0ffa6 | 151 | |
c81e1f64 | 152 | spin_lock_irqsave(&handler->lock, flags); |
353b71c7 MH |
153 | hfi1_cdbg(MMU, "Inserting node addr 0x%llx, len %u", mnode->addr, |
154 | mnode->len); | |
df5a00f8 MH |
155 | node = __mmu_rb_search(handler, mnode->addr, mnode->len); |
156 | if (node) { | |
157 | ret = -EINVAL; | |
158 | goto unlock; | |
06e0ffa6 | 159 | } |
e0b09ac5 | 160 | __mmu_int_rb_insert(mnode, &handler->root); |
06e0ffa6 | 161 | |
e0b09ac5 | 162 | ret = handler->ops->insert(handler->ops_arg, mnode); |
c0946642 | 163 | if (ret) |
e0b09ac5 | 164 | __mmu_int_rb_remove(mnode, &handler->root); |
06e0ffa6 | 165 | unlock: |
c81e1f64 | 166 | spin_unlock_irqrestore(&handler->lock, flags); |
06e0ffa6 MH |
167 | return ret; |
168 | } | |
169 | ||
de82bdff | 170 | /* Caller must hold handler lock */ |
06e0ffa6 MH |
171 | static struct mmu_rb_node *__mmu_rb_search(struct mmu_rb_handler *handler, |
172 | unsigned long addr, | |
173 | unsigned long len) | |
174 | { | |
0f310a00 | 175 | struct mmu_rb_node *node = NULL; |
df5a00f8 | 176 | |
353b71c7 | 177 | hfi1_cdbg(MMU, "Searching for addr 0x%llx, len %u", addr, len); |
0f310a00 | 178 | if (!handler->ops->filter) { |
e0b09ac5 | 179 | node = __mmu_int_rb_iter_first(&handler->root, addr, |
0f310a00 MH |
180 | (addr + len) - 1); |
181 | } else { | |
e0b09ac5 | 182 | for (node = __mmu_int_rb_iter_first(&handler->root, addr, |
0f310a00 MH |
183 | (addr + len) - 1); |
184 | node; | |
185 | node = __mmu_int_rb_iter_next(node, addr, | |
186 | (addr + len) - 1)) { | |
187 | if (handler->ops->filter(node, addr, len)) | |
188 | return node; | |
189 | } | |
190 | } | |
df5a00f8 | 191 | return node; |
06e0ffa6 MH |
192 | } |
193 | ||
e0b09ac5 | 194 | struct mmu_rb_node *hfi1_mmu_rb_extract(struct mmu_rb_handler *handler, |
f53af85e MH |
195 | unsigned long addr, unsigned long len) |
196 | { | |
f53af85e MH |
197 | struct mmu_rb_node *node; |
198 | unsigned long flags; | |
199 | ||
f53af85e MH |
200 | spin_lock_irqsave(&handler->lock, flags); |
201 | node = __mmu_rb_search(handler, addr, len); | |
202 | if (node) | |
e0b09ac5 | 203 | __mmu_int_rb_remove(node, &handler->root); |
f53af85e MH |
204 | spin_unlock_irqrestore(&handler->lock, flags); |
205 | ||
206 | return node; | |
207 | } | |
208 | ||
10345998 DL |
209 | void hfi1_mmu_rb_evict(struct mmu_rb_handler *handler, void *evict_arg) |
210 | { | |
211 | struct mmu_rb_node *rbnode; | |
212 | struct rb_node *node, *next; | |
213 | struct list_head del_list; | |
214 | unsigned long flags; | |
215 | bool stop = false; | |
216 | ||
217 | INIT_LIST_HEAD(&del_list); | |
218 | ||
219 | spin_lock_irqsave(&handler->lock, flags); | |
220 | for (node = rb_first(&handler->root); node; node = next) { | |
221 | next = rb_next(node); | |
222 | rbnode = rb_entry(node, struct mmu_rb_node, node); | |
223 | if (handler->ops->evict(handler->ops_arg, rbnode, evict_arg, | |
224 | &stop)) { | |
225 | __mmu_int_rb_remove(rbnode, &handler->root); | |
226 | list_add(&rbnode->list, &del_list); | |
227 | } | |
228 | if (stop) | |
229 | break; | |
230 | } | |
231 | spin_unlock_irqrestore(&handler->lock, flags); | |
232 | ||
233 | down_write(&handler->mm->mmap_sem); | |
234 | while (!list_empty(&del_list)) { | |
235 | rbnode = list_first_entry(&del_list, struct mmu_rb_node, list); | |
236 | list_del(&rbnode->list); | |
237 | handler->ops->remove(handler->ops_arg, rbnode, | |
238 | handler->mm); | |
239 | } | |
240 | up_write(&handler->mm->mmap_sem); | |
241 | } | |
242 | ||
e0b09ac5 DL |
243 | void hfi1_mmu_rb_remove(struct mmu_rb_handler *handler, |
244 | struct mmu_rb_node *node) | |
06e0ffa6 | 245 | { |
3c1091aa | 246 | unsigned long flags; |
06e0ffa6 | 247 | |
3c1091aa IW |
248 | /* Validity of handler and node pointers has been checked by caller. */ |
249 | hfi1_cdbg(MMU, "Removing node addr 0x%llx, len %u", node->addr, | |
250 | node->len); | |
251 | spin_lock_irqsave(&handler->lock, flags); | |
e0b09ac5 | 252 | __mmu_int_rb_remove(node, &handler->root); |
3c1091aa IW |
253 | spin_unlock_irqrestore(&handler->lock, flags); |
254 | ||
e0b09ac5 | 255 | handler->ops->remove(handler->ops_arg, node, NULL); |
06e0ffa6 MH |
256 | } |
257 | ||
258 | static inline void mmu_notifier_page(struct mmu_notifier *mn, | |
259 | struct mm_struct *mm, unsigned long addr) | |
260 | { | |
f19bd643 | 261 | mmu_notifier_mem_invalidate(mn, mm, addr, addr + PAGE_SIZE); |
06e0ffa6 MH |
262 | } |
263 | ||
264 | static inline void mmu_notifier_range_start(struct mmu_notifier *mn, | |
265 | struct mm_struct *mm, | |
266 | unsigned long start, | |
267 | unsigned long end) | |
268 | { | |
f19bd643 | 269 | mmu_notifier_mem_invalidate(mn, mm, start, end); |
06e0ffa6 MH |
270 | } |
271 | ||
272 | static void mmu_notifier_mem_invalidate(struct mmu_notifier *mn, | |
f19bd643 | 273 | struct mm_struct *mm, |
06e0ffa6 MH |
274 | unsigned long start, unsigned long end) |
275 | { | |
276 | struct mmu_rb_handler *handler = | |
277 | container_of(mn, struct mmu_rb_handler, mn); | |
e0b09ac5 | 278 | struct rb_root *root = &handler->root; |
f19bd643 | 279 | struct mmu_rb_node *node, *ptr = NULL; |
df5a00f8 | 280 | unsigned long flags; |
06e0ffa6 | 281 | |
c81e1f64 | 282 | spin_lock_irqsave(&handler->lock, flags); |
f19bd643 MH |
283 | for (node = __mmu_int_rb_iter_first(root, start, end - 1); |
284 | node; node = ptr) { | |
285 | /* Guard against node removal. */ | |
286 | ptr = __mmu_int_rb_iter_next(node, start, end - 1); | |
353b71c7 MH |
287 | hfi1_cdbg(MMU, "Invalidating node addr 0x%llx, len %u", |
288 | node->addr, node->len); | |
e0b09ac5 | 289 | if (handler->ops->invalidate(handler->ops_arg, node)) { |
e88c9271 | 290 | __mmu_int_rb_remove(node, root); |
e0b09ac5 | 291 | handler->ops->remove(handler->ops_arg, node, mm); |
de82bdff | 292 | } |
06e0ffa6 | 293 | } |
c81e1f64 | 294 | spin_unlock_irqrestore(&handler->lock, flags); |
06e0ffa6 | 295 | } |