]>
Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * lib/prio_tree.c - priority search tree | |
3 | * | |
4 | * Copyright (C) 2004, Rajesh Venkatasubramanian <vrajesh@umich.edu> | |
5 | * | |
6 | * This file is released under the GPL v2. | |
7 | * | |
8 | * Based on the radix priority search tree proposed by Edward M. McCreight | |
9 | * SIAM Journal of Computing, vol. 14, no.2, pages 257-276, May 1985 | |
10 | * | |
11 | * 02Feb2004 Initial version | |
12 | */ | |
13 | ||
14 | #include <linux/init.h> | |
15 | #include <linux/mm.h> | |
16 | #include <linux/prio_tree.h> | |
fff3fd8a | 17 | #include <linux/export.h> |
1da177e4 LT |
18 | |
19 | /* | |
20 | * A clever mix of heap and radix trees forms a radix priority search tree (PST) | |
21 | * which is useful for storing intervals, e.g, we can consider a vma as a closed | |
22 | * interval of file pages [offset_begin, offset_end], and store all vmas that | |
23 | * map a file in a PST. Then, using the PST, we can answer a stabbing query, | |
24 | * i.e., selecting a set of stored intervals (vmas) that overlap with (map) a | |
25 | * given input interval X (a set of consecutive file pages), in "O(log n + m)" | |
26 | * time where 'log n' is the height of the PST, and 'm' is the number of stored | |
27 | * intervals (vmas) that overlap (map) with the input interval X (the set of | |
28 | * consecutive file pages). | |
29 | * | |
30 | * In our implementation, we store closed intervals of the form [radix_index, | |
31 | * heap_index]. We assume that always radix_index <= heap_index. McCreight's PST | |
32 | * is designed for storing intervals with unique radix indices, i.e., each | |
33 | * interval have different radix_index. However, this limitation can be easily | |
34 | * overcome by using the size, i.e., heap_index - radix_index, as part of the | |
35 | * index, so we index the tree using [(radix_index,size), heap_index]. | |
36 | * | |
37 | * When the above-mentioned indexing scheme is used, theoretically, in a 32 bit | |
38 | * machine, the maximum height of a PST can be 64. We can use a balanced version | |
39 | * of the priority search tree to optimize the tree height, but the balanced | |
40 | * tree proposed by McCreight is too complex and memory-hungry for our purpose. | |
41 | */ | |
42 | ||
43 | /* | |
44 | * The following macros are used for implementing prio_tree for i_mmap | |
45 | */ | |
46 | ||
47 | #define RADIX_INDEX(vma) ((vma)->vm_pgoff) | |
48 | #define VMA_SIZE(vma) (((vma)->vm_end - (vma)->vm_start) >> PAGE_SHIFT) | |
49 | /* avoid overflow */ | |
50 | #define HEAP_INDEX(vma) ((vma)->vm_pgoff + (VMA_SIZE(vma) - 1)) | |
51 | ||
52 | ||
53 | static void get_index(const struct prio_tree_root *root, | |
54 | const struct prio_tree_node *node, | |
55 | unsigned long *radix, unsigned long *heap) | |
56 | { | |
57 | if (root->raw) { | |
58 | struct vm_area_struct *vma = prio_tree_entry( | |
59 | node, struct vm_area_struct, shared.prio_tree_node); | |
60 | ||
61 | *radix = RADIX_INDEX(vma); | |
62 | *heap = HEAP_INDEX(vma); | |
63 | } | |
64 | else { | |
65 | *radix = node->start; | |
66 | *heap = node->last; | |
67 | } | |
68 | } | |
69 | ||
70 | static unsigned long index_bits_to_maxindex[BITS_PER_LONG]; | |
71 | ||
72 | void __init prio_tree_init(void) | |
73 | { | |
74 | unsigned int i; | |
75 | ||
76 | for (i = 0; i < ARRAY_SIZE(index_bits_to_maxindex) - 1; i++) | |
77 | index_bits_to_maxindex[i] = (1UL << (i + 1)) - 1; | |
78 | index_bits_to_maxindex[ARRAY_SIZE(index_bits_to_maxindex) - 1] = ~0UL; | |
79 | } | |
80 | ||
81 | /* | |
82 | * Maximum heap_index that can be stored in a PST with index_bits bits | |
83 | */ | |
84 | static inline unsigned long prio_tree_maxindex(unsigned int bits) | |
85 | { | |
86 | return index_bits_to_maxindex[bits - 1]; | |
87 | } | |
88 | ||
97e834c5 XG |
89 | static void prio_set_parent(struct prio_tree_node *parent, |
90 | struct prio_tree_node *child, bool left) | |
91 | { | |
92 | if (left) | |
93 | parent->left = child; | |
94 | else | |
95 | parent->right = child; | |
96 | ||
97 | child->parent = parent; | |
98 | } | |
99 | ||
1da177e4 LT |
100 | /* |
101 | * Extend a priority search tree so that it can store a node with heap_index | |
102 | * max_heap_index. In the worst case, this algorithm takes O((log n)^2). | |
103 | * However, this function is used rarely and the common case performance is | |
104 | * not bad. | |
105 | */ | |
106 | static struct prio_tree_node *prio_tree_expand(struct prio_tree_root *root, | |
107 | struct prio_tree_node *node, unsigned long max_heap_index) | |
108 | { | |
742245d5 | 109 | struct prio_tree_node *prev; |
1da177e4 LT |
110 | |
111 | if (max_heap_index > prio_tree_maxindex(root->index_bits)) | |
112 | root->index_bits++; | |
113 | ||
742245d5 XG |
114 | prev = node; |
115 | INIT_PRIO_TREE_NODE(node); | |
116 | ||
1da177e4 | 117 | while (max_heap_index > prio_tree_maxindex(root->index_bits)) { |
742245d5 XG |
118 | struct prio_tree_node *tmp = root->prio_tree_node; |
119 | ||
1da177e4 LT |
120 | root->index_bits++; |
121 | ||
122 | if (prio_tree_empty(root)) | |
123 | continue; | |
124 | ||
742245d5 XG |
125 | prio_tree_remove(root, root->prio_tree_node); |
126 | INIT_PRIO_TREE_NODE(tmp); | |
1da177e4 | 127 | |
97e834c5 | 128 | prio_set_parent(prev, tmp, true); |
742245d5 XG |
129 | prev = tmp; |
130 | } | |
1da177e4 | 131 | |
97e834c5 XG |
132 | if (!prio_tree_empty(root)) |
133 | prio_set_parent(prev, root->prio_tree_node, true); | |
1da177e4 LT |
134 | |
135 | root->prio_tree_node = node; | |
136 | return node; | |
137 | } | |
138 | ||
139 | /* | |
140 | * Replace a prio_tree_node with a new node and return the old node | |
141 | */ | |
142 | struct prio_tree_node *prio_tree_replace(struct prio_tree_root *root, | |
143 | struct prio_tree_node *old, struct prio_tree_node *node) | |
144 | { | |
145 | INIT_PRIO_TREE_NODE(node); | |
146 | ||
147 | if (prio_tree_root(old)) { | |
148 | BUG_ON(root->prio_tree_node != old); | |
149 | /* | |
150 | * We can reduce root->index_bits here. However, it is complex | |
151 | * and does not help much to improve performance (IMO). | |
152 | */ | |
1da177e4 | 153 | root->prio_tree_node = node; |
97e834c5 XG |
154 | } else |
155 | prio_set_parent(old->parent, node, old->parent->left == old); | |
1da177e4 | 156 | |
97e834c5 XG |
157 | if (!prio_tree_left_empty(old)) |
158 | prio_set_parent(node, old->left, true); | |
1da177e4 | 159 | |
97e834c5 XG |
160 | if (!prio_tree_right_empty(old)) |
161 | prio_set_parent(node, old->right, false); | |
1da177e4 LT |
162 | |
163 | return old; | |
164 | } | |
165 | ||
166 | /* | |
167 | * Insert a prio_tree_node @node into a radix priority search tree @root. The | |
168 | * algorithm typically takes O(log n) time where 'log n' is the number of bits | |
169 | * required to represent the maximum heap_index. In the worst case, the algo | |
170 | * can take O((log n)^2) - check prio_tree_expand. | |
171 | * | |
172 | * If a prior node with same radix_index and heap_index is already found in | |
173 | * the tree, then returns the address of the prior node. Otherwise, inserts | |
174 | * @node into the tree and returns @node. | |
175 | */ | |
176 | struct prio_tree_node *prio_tree_insert(struct prio_tree_root *root, | |
177 | struct prio_tree_node *node) | |
178 | { | |
179 | struct prio_tree_node *cur, *res = node; | |
180 | unsigned long radix_index, heap_index; | |
181 | unsigned long r_index, h_index, index, mask; | |
182 | int size_flag = 0; | |
183 | ||
184 | get_index(root, node, &radix_index, &heap_index); | |
185 | ||
186 | if (prio_tree_empty(root) || | |
187 | heap_index > prio_tree_maxindex(root->index_bits)) | |
188 | return prio_tree_expand(root, node, heap_index); | |
189 | ||
190 | cur = root->prio_tree_node; | |
191 | mask = 1UL << (root->index_bits - 1); | |
192 | ||
193 | while (mask) { | |
194 | get_index(root, cur, &r_index, &h_index); | |
195 | ||
196 | if (r_index == radix_index && h_index == heap_index) | |
197 | return cur; | |
198 | ||
199 | if (h_index < heap_index || | |
200 | (h_index == heap_index && r_index > radix_index)) { | |
201 | struct prio_tree_node *tmp = node; | |
202 | node = prio_tree_replace(root, cur, node); | |
203 | cur = tmp; | |
204 | /* swap indices */ | |
205 | index = r_index; | |
206 | r_index = radix_index; | |
207 | radix_index = index; | |
208 | index = h_index; | |
209 | h_index = heap_index; | |
210 | heap_index = index; | |
211 | } | |
212 | ||
213 | if (size_flag) | |
214 | index = heap_index - radix_index; | |
215 | else | |
216 | index = radix_index; | |
217 | ||
218 | if (index & mask) { | |
219 | if (prio_tree_right_empty(cur)) { | |
220 | INIT_PRIO_TREE_NODE(node); | |
97e834c5 | 221 | prio_set_parent(cur, node, false); |
1da177e4 LT |
222 | return res; |
223 | } else | |
224 | cur = cur->right; | |
225 | } else { | |
226 | if (prio_tree_left_empty(cur)) { | |
227 | INIT_PRIO_TREE_NODE(node); | |
97e834c5 | 228 | prio_set_parent(cur, node, true); |
1da177e4 LT |
229 | return res; |
230 | } else | |
231 | cur = cur->left; | |
232 | } | |
233 | ||
234 | mask >>= 1; | |
235 | ||
236 | if (!mask) { | |
237 | mask = 1UL << (BITS_PER_LONG - 1); | |
238 | size_flag = 1; | |
239 | } | |
240 | } | |
241 | /* Should not reach here */ | |
242 | BUG(); | |
243 | return NULL; | |
244 | } | |
fff3fd8a | 245 | EXPORT_SYMBOL(prio_tree_insert); |
1da177e4 LT |
246 | |
247 | /* | |
248 | * Remove a prio_tree_node @node from a radix priority search tree @root. The | |
249 | * algorithm takes O(log n) time where 'log n' is the number of bits required | |
250 | * to represent the maximum heap_index. | |
251 | */ | |
252 | void prio_tree_remove(struct prio_tree_root *root, struct prio_tree_node *node) | |
253 | { | |
254 | struct prio_tree_node *cur; | |
255 | unsigned long r_index, h_index_right, h_index_left; | |
256 | ||
257 | cur = node; | |
258 | ||
259 | while (!prio_tree_left_empty(cur) || !prio_tree_right_empty(cur)) { | |
260 | if (!prio_tree_left_empty(cur)) | |
261 | get_index(root, cur->left, &r_index, &h_index_left); | |
262 | else { | |
263 | cur = cur->right; | |
264 | continue; | |
265 | } | |
266 | ||
267 | if (!prio_tree_right_empty(cur)) | |
268 | get_index(root, cur->right, &r_index, &h_index_right); | |
269 | else { | |
270 | cur = cur->left; | |
271 | continue; | |
272 | } | |
273 | ||
274 | /* both h_index_left and h_index_right cannot be 0 */ | |
275 | if (h_index_left >= h_index_right) | |
276 | cur = cur->left; | |
277 | else | |
278 | cur = cur->right; | |
279 | } | |
280 | ||
281 | if (prio_tree_root(cur)) { | |
282 | BUG_ON(root->prio_tree_node != cur); | |
283 | __INIT_PRIO_TREE_ROOT(root, root->raw); | |
284 | return; | |
285 | } | |
286 | ||
287 | if (cur->parent->right == cur) | |
288 | cur->parent->right = cur->parent; | |
289 | else | |
290 | cur->parent->left = cur->parent; | |
291 | ||
292 | while (cur != node) | |
293 | cur = prio_tree_replace(root, cur->parent, cur); | |
294 | } | |
fff3fd8a | 295 | EXPORT_SYMBOL(prio_tree_remove); |
1da177e4 | 296 | |
f35368dd XG |
297 | static void iter_walk_down(struct prio_tree_iter *iter) |
298 | { | |
299 | iter->mask >>= 1; | |
300 | if (iter->mask) { | |
301 | if (iter->size_level) | |
302 | iter->size_level++; | |
303 | return; | |
304 | } | |
305 | ||
306 | if (iter->size_level) { | |
307 | BUG_ON(!prio_tree_left_empty(iter->cur)); | |
308 | BUG_ON(!prio_tree_right_empty(iter->cur)); | |
309 | iter->size_level++; | |
310 | iter->mask = ULONG_MAX; | |
311 | } else { | |
312 | iter->size_level = 1; | |
313 | iter->mask = 1UL << (BITS_PER_LONG - 1); | |
314 | } | |
315 | } | |
316 | ||
317 | static void iter_walk_up(struct prio_tree_iter *iter) | |
318 | { | |
319 | if (iter->mask == ULONG_MAX) | |
320 | iter->mask = 1UL; | |
321 | else if (iter->size_level == 1) | |
322 | iter->mask = 1UL; | |
323 | else | |
324 | iter->mask <<= 1; | |
325 | if (iter->size_level) | |
326 | iter->size_level--; | |
327 | if (!iter->size_level && (iter->value & iter->mask)) | |
328 | iter->value ^= iter->mask; | |
329 | } | |
330 | ||
1da177e4 LT |
331 | /* |
332 | * Following functions help to enumerate all prio_tree_nodes in the tree that | |
333 | * overlap with the input interval X [radix_index, heap_index]. The enumeration | |
334 | * takes O(log n + m) time where 'log n' is the height of the tree (which is | |
335 | * proportional to # of bits required to represent the maximum heap_index) and | |
336 | * 'm' is the number of prio_tree_nodes that overlap the interval X. | |
337 | */ | |
338 | ||
339 | static struct prio_tree_node *prio_tree_left(struct prio_tree_iter *iter, | |
340 | unsigned long *r_index, unsigned long *h_index) | |
341 | { | |
342 | if (prio_tree_left_empty(iter->cur)) | |
343 | return NULL; | |
344 | ||
345 | get_index(iter->root, iter->cur->left, r_index, h_index); | |
346 | ||
347 | if (iter->r_index <= *h_index) { | |
348 | iter->cur = iter->cur->left; | |
f35368dd | 349 | iter_walk_down(iter); |
1da177e4 LT |
350 | return iter->cur; |
351 | } | |
352 | ||
353 | return NULL; | |
354 | } | |
355 | ||
356 | static struct prio_tree_node *prio_tree_right(struct prio_tree_iter *iter, | |
357 | unsigned long *r_index, unsigned long *h_index) | |
358 | { | |
359 | unsigned long value; | |
360 | ||
361 | if (prio_tree_right_empty(iter->cur)) | |
362 | return NULL; | |
363 | ||
364 | if (iter->size_level) | |
365 | value = iter->value; | |
366 | else | |
367 | value = iter->value | iter->mask; | |
368 | ||
369 | if (iter->h_index < value) | |
370 | return NULL; | |
371 | ||
372 | get_index(iter->root, iter->cur->right, r_index, h_index); | |
373 | ||
374 | if (iter->r_index <= *h_index) { | |
375 | iter->cur = iter->cur->right; | |
f35368dd | 376 | iter_walk_down(iter); |
1da177e4 LT |
377 | return iter->cur; |
378 | } | |
379 | ||
380 | return NULL; | |
381 | } | |
382 | ||
383 | static struct prio_tree_node *prio_tree_parent(struct prio_tree_iter *iter) | |
384 | { | |
385 | iter->cur = iter->cur->parent; | |
f35368dd | 386 | iter_walk_up(iter); |
1da177e4 LT |
387 | return iter->cur; |
388 | } | |
389 | ||
390 | static inline int overlap(struct prio_tree_iter *iter, | |
391 | unsigned long r_index, unsigned long h_index) | |
392 | { | |
393 | return iter->h_index >= r_index && iter->r_index <= h_index; | |
394 | } | |
395 | ||
396 | /* | |
397 | * prio_tree_first: | |
398 | * | |
399 | * Get the first prio_tree_node that overlaps with the interval [radix_index, | |
400 | * heap_index]. Note that always radix_index <= heap_index. We do a pre-order | |
401 | * traversal of the tree. | |
402 | */ | |
403 | static struct prio_tree_node *prio_tree_first(struct prio_tree_iter *iter) | |
404 | { | |
405 | struct prio_tree_root *root; | |
406 | unsigned long r_index, h_index; | |
407 | ||
408 | INIT_PRIO_TREE_ITER(iter); | |
409 | ||
410 | root = iter->root; | |
411 | if (prio_tree_empty(root)) | |
412 | return NULL; | |
413 | ||
414 | get_index(root, root->prio_tree_node, &r_index, &h_index); | |
415 | ||
416 | if (iter->r_index > h_index) | |
417 | return NULL; | |
418 | ||
419 | iter->mask = 1UL << (root->index_bits - 1); | |
420 | iter->cur = root->prio_tree_node; | |
421 | ||
422 | while (1) { | |
423 | if (overlap(iter, r_index, h_index)) | |
424 | return iter->cur; | |
425 | ||
426 | if (prio_tree_left(iter, &r_index, &h_index)) | |
427 | continue; | |
428 | ||
429 | if (prio_tree_right(iter, &r_index, &h_index)) | |
430 | continue; | |
431 | ||
432 | break; | |
433 | } | |
434 | return NULL; | |
435 | } | |
436 | ||
437 | /* | |
438 | * prio_tree_next: | |
439 | * | |
440 | * Get the next prio_tree_node that overlaps with the input interval in iter | |
441 | */ | |
442 | struct prio_tree_node *prio_tree_next(struct prio_tree_iter *iter) | |
443 | { | |
444 | unsigned long r_index, h_index; | |
445 | ||
446 | if (iter->cur == NULL) | |
447 | return prio_tree_first(iter); | |
448 | ||
449 | repeat: | |
450 | while (prio_tree_left(iter, &r_index, &h_index)) | |
451 | if (overlap(iter, r_index, h_index)) | |
452 | return iter->cur; | |
453 | ||
454 | while (!prio_tree_right(iter, &r_index, &h_index)) { | |
455 | while (!prio_tree_root(iter->cur) && | |
456 | iter->cur->parent->right == iter->cur) | |
457 | prio_tree_parent(iter); | |
458 | ||
459 | if (prio_tree_root(iter->cur)) | |
460 | return NULL; | |
461 | ||
462 | prio_tree_parent(iter); | |
463 | } | |
464 | ||
465 | if (overlap(iter, r_index, h_index)) | |
466 | return iter->cur; | |
467 | ||
468 | goto repeat; | |
469 | } | |
fff3fd8a | 470 | EXPORT_SYMBOL(prio_tree_next); |