2 * multiorder.c: Multi-order radix tree entry testing
3 * Copyright (c) 2016 Intel Corporation
4 * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
5 * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
16 #include <linux/radix-tree.h>
17 #include <linux/slab.h>
18 #include <linux/errno.h>
22 #define for_each_index(i, base, order) \
23 for (i = base; i < base + (1 << order); i++)
25 static void __multiorder_tag_test(int index
, int order
)
27 RADIX_TREE(tree
, GFP_KERNEL
);
30 /* our canonical entry */
31 base
= index
& ~((1 << order
) - 1);
33 printf("Multiorder tag test with index %d, canonical entry %d\n",
36 err
= item_insert_order(&tree
, index
, order
);
40 * Verify we get collisions for covered indices. We try and fail to
41 * insert an exceptional entry so we don't leak memory via
42 * item_insert_order().
44 for_each_index(i
, base
, order
) {
45 err
= __radix_tree_insert(&tree
, i
, order
,
46 (void *)(0xA0 | RADIX_TREE_EXCEPTIONAL_ENTRY
));
47 assert(err
== -EEXIST
);
50 for_each_index(i
, base
, order
) {
51 assert(!radix_tree_tag_get(&tree
, i
, 0));
52 assert(!radix_tree_tag_get(&tree
, i
, 1));
55 assert(radix_tree_tag_set(&tree
, index
, 0));
57 for_each_index(i
, base
, order
) {
58 assert(radix_tree_tag_get(&tree
, i
, 0));
59 assert(!radix_tree_tag_get(&tree
, i
, 1));
62 assert(tag_tagged_items(&tree
, NULL
, 0, ~0UL, 10, 0, 1) == 1);
63 assert(radix_tree_tag_clear(&tree
, index
, 0));
65 for_each_index(i
, base
, order
) {
66 assert(!radix_tree_tag_get(&tree
, i
, 0));
67 assert(radix_tree_tag_get(&tree
, i
, 1));
70 assert(radix_tree_tag_clear(&tree
, index
, 1));
72 assert(!radix_tree_tagged(&tree
, 0));
73 assert(!radix_tree_tagged(&tree
, 1));
75 item_kill_tree(&tree
);
78 static void multiorder_tag_tests(void)
80 /* test multi-order entry for indices 0-7 with no sibling pointers */
81 __multiorder_tag_test(0, 3);
82 __multiorder_tag_test(5, 3);
84 /* test multi-order entry for indices 8-15 with no sibling pointers */
85 __multiorder_tag_test(8, 3);
86 __multiorder_tag_test(15, 3);
89 * Our order 5 entry covers indices 0-31 in a tree with height=2.
90 * This is broken up as follows:
91 * 0-7: canonical entry
96 __multiorder_tag_test(0, 5);
97 __multiorder_tag_test(29, 5);
99 /* same test, but with indices 32-63 */
100 __multiorder_tag_test(32, 5);
101 __multiorder_tag_test(44, 5);
104 * Our order 8 entry covers indices 0-255 in a tree with height=3.
105 * This is broken up as follows:
106 * 0-63: canonical entry
111 __multiorder_tag_test(0, 8);
112 __multiorder_tag_test(190, 8);
114 /* same test, but with indices 256-511 */
115 __multiorder_tag_test(256, 8);
116 __multiorder_tag_test(300, 8);
118 __multiorder_tag_test(0x12345678UL
, 8);
121 static void multiorder_check(unsigned long index
, int order
)
124 unsigned long min
= index
& ~((1UL << order
) - 1);
125 unsigned long max
= min
+ (1UL << order
);
127 struct item
*item2
= item_create(min
, order
);
128 RADIX_TREE(tree
, GFP_KERNEL
);
130 printf("Multiorder index %ld, order %d\n", index
, order
);
132 assert(item_insert_order(&tree
, index
, order
) == 0);
134 for (i
= min
; i
< max
; i
++) {
135 struct item
*item
= item_lookup(&tree
, i
);
137 assert(item
->index
== index
);
139 for (i
= 0; i
< min
; i
++)
140 item_check_absent(&tree
, i
);
141 for (i
= max
; i
< 2*max
; i
++)
142 item_check_absent(&tree
, i
);
143 for (i
= min
; i
< max
; i
++)
144 assert(radix_tree_insert(&tree
, i
, item2
) == -EEXIST
);
146 slot
= radix_tree_lookup_slot(&tree
, index
);
148 radix_tree_replace_slot(&tree
, slot
, item2
);
149 for (i
= min
; i
< max
; i
++) {
150 struct item
*item
= item_lookup(&tree
, i
);
152 assert(item
->index
== min
);
155 assert(item_delete(&tree
, min
) != 0);
157 for (i
= 0; i
< 2*max
; i
++)
158 item_check_absent(&tree
, i
);
161 static void multiorder_shrink(unsigned long index
, int order
)
164 unsigned long max
= 1 << order
;
165 RADIX_TREE(tree
, GFP_KERNEL
);
166 struct radix_tree_node
*node
;
168 printf("Multiorder shrink index %ld, order %d\n", index
, order
);
170 assert(item_insert_order(&tree
, 0, order
) == 0);
174 assert(item_insert(&tree
, index
) == 0);
175 assert(node
!= tree
.rnode
);
177 assert(item_delete(&tree
, index
) != 0);
178 assert(node
== tree
.rnode
);
180 for (i
= 0; i
< max
; i
++) {
181 struct item
*item
= item_lookup(&tree
, i
);
183 assert(item
->index
== 0);
185 for (i
= max
; i
< 2*max
; i
++)
186 item_check_absent(&tree
, i
);
188 if (!item_delete(&tree
, 0)) {
189 printf("failed to delete index %ld (order %d)\n", index
, order
); abort();
192 for (i
= 0; i
< 2*max
; i
++)
193 item_check_absent(&tree
, i
);
196 static void multiorder_insert_bug(void)
198 RADIX_TREE(tree
, GFP_KERNEL
);
200 item_insert(&tree
, 0);
201 radix_tree_tag_set(&tree
, 0, 0);
202 item_insert_order(&tree
, 3 << 6, 6);
204 item_kill_tree(&tree
);
207 void multiorder_iteration(void)
209 RADIX_TREE(tree
, GFP_KERNEL
);
210 struct radix_tree_iter iter
;
214 printf("Multiorder iteration test\n");
216 #define NUM_ENTRIES 11
217 int index
[NUM_ENTRIES
] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128};
218 int order
[NUM_ENTRIES
] = {1, 1, 2, 3, 4, 1, 0, 1, 3, 0, 7};
220 for (i
= 0; i
< NUM_ENTRIES
; i
++) {
221 err
= item_insert_order(&tree
, index
[i
], order
[i
]);
225 for (j
= 0; j
< 256; j
++) {
226 for (i
= 0; i
< NUM_ENTRIES
; i
++)
227 if (j
<= (index
[i
] | ((1 << order
[i
]) - 1)))
230 radix_tree_for_each_slot(slot
, &tree
, &iter
, j
) {
231 int height
= order
[i
] / RADIX_TREE_MAP_SHIFT
;
232 int shift
= height
* RADIX_TREE_MAP_SHIFT
;
233 unsigned long mask
= (1UL << order
[i
]) - 1;
234 struct item
*item
= *slot
;
236 assert((iter
.index
| mask
) == (index
[i
] | mask
));
237 assert(iter
.shift
== shift
);
238 assert(!radix_tree_is_internal_node(item
));
239 assert((item
->index
| mask
) == (index
[i
] | mask
));
240 assert(item
->order
== order
[i
]);
245 item_kill_tree(&tree
);
248 void multiorder_tagged_iteration(void)
250 RADIX_TREE(tree
, GFP_KERNEL
);
251 struct radix_tree_iter iter
;
255 printf("Multiorder tagged iteration test\n");
257 #define MT_NUM_ENTRIES 9
258 int index
[MT_NUM_ENTRIES
] = {0, 2, 4, 16, 32, 40, 64, 72, 128};
259 int order
[MT_NUM_ENTRIES
] = {1, 0, 2, 4, 3, 1, 3, 0, 7};
261 #define TAG_ENTRIES 7
262 int tag_index
[TAG_ENTRIES
] = {0, 4, 16, 40, 64, 72, 128};
264 for (i
= 0; i
< MT_NUM_ENTRIES
; i
++)
265 assert(!item_insert_order(&tree
, index
[i
], order
[i
]));
267 assert(!radix_tree_tagged(&tree
, 1));
269 for (i
= 0; i
< TAG_ENTRIES
; i
++)
270 assert(radix_tree_tag_set(&tree
, tag_index
[i
], 1));
272 for (j
= 0; j
< 256; j
++) {
275 for (i
= 0; i
< TAG_ENTRIES
; i
++) {
276 for (k
= i
; index
[k
] < tag_index
[i
]; k
++)
278 if (j
<= (index
[k
] | ((1 << order
[k
]) - 1)))
282 radix_tree_for_each_tagged(slot
, &tree
, &iter
, j
, 1) {
284 struct item
*item
= *slot
;
285 for (k
= i
; index
[k
] < tag_index
[i
]; k
++)
287 mask
= (1UL << order
[k
]) - 1;
289 assert((iter
.index
| mask
) == (tag_index
[i
] | mask
));
290 assert(!radix_tree_is_internal_node(item
));
291 assert((item
->index
| mask
) == (tag_index
[i
] | mask
));
292 assert(item
->order
== order
[k
]);
297 assert(tag_tagged_items(&tree
, NULL
, 0, ~0UL, TAG_ENTRIES
, 1, 2) ==
300 for (j
= 0; j
< 256; j
++) {
303 for (i
= 0; i
< TAG_ENTRIES
; i
++) {
304 for (k
= i
; index
[k
] < tag_index
[i
]; k
++)
306 if (j
<= (index
[k
] | ((1 << order
[k
]) - 1)))
310 radix_tree_for_each_tagged(slot
, &tree
, &iter
, j
, 2) {
311 struct item
*item
= *slot
;
312 for (k
= i
; index
[k
] < tag_index
[i
]; k
++)
314 mask
= (1 << order
[k
]) - 1;
316 assert((iter
.index
| mask
) == (tag_index
[i
] | mask
));
317 assert(!radix_tree_is_internal_node(item
));
318 assert((item
->index
| mask
) == (tag_index
[i
] | mask
));
319 assert(item
->order
== order
[k
]);
324 assert(tag_tagged_items(&tree
, NULL
, 1, ~0UL, MT_NUM_ENTRIES
* 2, 1, 0)
327 radix_tree_for_each_tagged(slot
, &tree
, &iter
, 0, 0) {
328 assert(iter
.index
== tag_index
[i
]);
332 item_kill_tree(&tree
);
335 static void __multiorder_join(unsigned long index
,
336 unsigned order1
, unsigned order2
)
339 void *item
, *item2
= item_create(index
+ 1, order1
);
340 RADIX_TREE(tree
, GFP_KERNEL
);
342 item_insert_order(&tree
, index
, order2
);
343 item
= radix_tree_lookup(&tree
, index
);
344 radix_tree_join(&tree
, index
+ 1, order1
, item2
);
345 loc
= find_item(&tree
, item
);
348 item
= radix_tree_lookup(&tree
, index
+ 1);
349 assert(item
== item2
);
350 item_kill_tree(&tree
);
353 static void __multiorder_join2(unsigned order1
, unsigned order2
)
355 RADIX_TREE(tree
, GFP_KERNEL
);
356 struct radix_tree_node
*node
;
357 void *item1
= item_create(0, order1
);
360 item_insert_order(&tree
, 0, order2
);
361 radix_tree_insert(&tree
, 1 << order2
, (void *)0x12UL
);
362 item2
= __radix_tree_lookup(&tree
, 1 << order2
, &node
, NULL
);
363 assert(item2
== (void *)0x12UL
);
364 assert(node
->exceptional
== 1);
366 radix_tree_join(&tree
, 0, order1
, item1
);
367 item2
= __radix_tree_lookup(&tree
, 1 << order2
, &node
, NULL
);
368 assert(item2
== item1
);
369 assert(node
->exceptional
== 0);
370 item_kill_tree(&tree
);
373 static void multiorder_join(void)
377 for (idx
= 0; idx
< 1024; idx
= idx
* 2 + 3) {
378 for (i
= 1; i
< 15; i
++) {
379 for (j
= 0; j
< i
; j
++) {
380 __multiorder_join(idx
, i
, j
);
385 for (i
= 1; i
< 15; i
++) {
386 for (j
= 0; j
< i
; j
++) {
387 __multiorder_join2(i
, j
);
392 static void check_mem(unsigned old_order
, unsigned new_order
, unsigned alloc
)
394 struct radix_tree_preload
*rtp
= &radix_tree_preloads
;
396 printf("split(%u %u) remaining %u\n", old_order
, new_order
,
399 * Can't check for equality here as some nodes may have been
400 * RCU-freed while we ran. But we should never finish with more
401 * nodes allocated since they should have all been preloaded.
403 if (nr_allocated
> alloc
)
404 printf("split(%u %u) allocated %u %u\n", old_order
, new_order
,
405 alloc
, nr_allocated
);
408 static void __multiorder_split(int old_order
, int new_order
)
410 RADIX_TREE(tree
, GFP_ATOMIC
);
412 struct radix_tree_iter iter
;
415 radix_tree_preload(GFP_KERNEL
);
416 assert(item_insert_order(&tree
, 0, old_order
) == 0);
417 radix_tree_preload_end();
419 /* Wipe out the preloaded cache or it'll confuse check_mem() */
420 radix_tree_cpu_dead(0);
422 radix_tree_tag_set(&tree
, 0, 2);
424 radix_tree_split_preload(old_order
, new_order
, GFP_KERNEL
);
425 alloc
= nr_allocated
;
426 radix_tree_split(&tree
, 0, new_order
);
427 check_mem(old_order
, new_order
, alloc
);
428 radix_tree_for_each_slot(slot
, &tree
, &iter
, 0) {
429 radix_tree_iter_replace(&tree
, &iter
, slot
,
430 item_create(iter
.index
, new_order
));
432 radix_tree_preload_end();
434 item_kill_tree(&tree
);
437 static void __multiorder_split2(int old_order
, int new_order
)
439 RADIX_TREE(tree
, GFP_KERNEL
);
441 struct radix_tree_iter iter
;
442 struct radix_tree_node
*node
;
445 __radix_tree_insert(&tree
, 0, old_order
, (void *)0x12);
447 item
= __radix_tree_lookup(&tree
, 0, &node
, NULL
);
448 assert(item
== (void *)0x12);
449 assert(node
->exceptional
> 0);
451 radix_tree_split(&tree
, 0, new_order
);
452 radix_tree_for_each_slot(slot
, &tree
, &iter
, 0) {
453 radix_tree_iter_replace(&tree
, &iter
, slot
,
454 item_create(iter
.index
, new_order
));
457 item
= __radix_tree_lookup(&tree
, 0, &node
, NULL
);
458 assert(item
!= (void *)0x12);
459 assert(node
->exceptional
== 0);
461 item_kill_tree(&tree
);
464 static void __multiorder_split3(int old_order
, int new_order
)
466 RADIX_TREE(tree
, GFP_KERNEL
);
468 struct radix_tree_iter iter
;
469 struct radix_tree_node
*node
;
472 __radix_tree_insert(&tree
, 0, old_order
, (void *)0x12);
474 item
= __radix_tree_lookup(&tree
, 0, &node
, NULL
);
475 assert(item
== (void *)0x12);
476 assert(node
->exceptional
> 0);
478 radix_tree_split(&tree
, 0, new_order
);
479 radix_tree_for_each_slot(slot
, &tree
, &iter
, 0) {
480 radix_tree_iter_replace(&tree
, &iter
, slot
, (void *)0x16);
483 item
= __radix_tree_lookup(&tree
, 0, &node
, NULL
);
484 assert(item
== (void *)0x16);
485 assert(node
->exceptional
> 0);
487 item_kill_tree(&tree
);
489 __radix_tree_insert(&tree
, 0, old_order
, (void *)0x12);
491 item
= __radix_tree_lookup(&tree
, 0, &node
, NULL
);
492 assert(item
== (void *)0x12);
493 assert(node
->exceptional
> 0);
495 radix_tree_split(&tree
, 0, new_order
);
496 radix_tree_for_each_slot(slot
, &tree
, &iter
, 0) {
497 if (iter
.index
== (1 << new_order
))
498 radix_tree_iter_replace(&tree
, &iter
, slot
,
501 radix_tree_iter_replace(&tree
, &iter
, slot
, NULL
);
504 item
= __radix_tree_lookup(&tree
, 1 << new_order
, &node
, NULL
);
505 assert(item
== (void *)0x16);
506 assert(node
->count
== node
->exceptional
);
511 assert(node
->count
== 1);
512 assert(node
->exceptional
== 0);
515 item_kill_tree(&tree
);
518 static void multiorder_split(void)
522 for (i
= 3; i
< 11; i
++)
523 for (j
= 0; j
< i
; j
++) {
524 __multiorder_split(i
, j
);
525 __multiorder_split2(i
, j
);
526 __multiorder_split3(i
, j
);
530 static void multiorder_account(void)
532 RADIX_TREE(tree
, GFP_KERNEL
);
533 struct radix_tree_node
*node
;
536 item_insert_order(&tree
, 0, 5);
538 __radix_tree_insert(&tree
, 1 << 5, 5, (void *)0x12);
539 __radix_tree_lookup(&tree
, 0, &node
, NULL
);
540 assert(node
->count
== node
->exceptional
* 2);
541 radix_tree_delete(&tree
, 1 << 5);
542 assert(node
->exceptional
== 0);
544 __radix_tree_insert(&tree
, 1 << 5, 5, (void *)0x12);
545 __radix_tree_lookup(&tree
, 1 << 5, &node
, &slot
);
546 assert(node
->count
== node
->exceptional
* 2);
547 __radix_tree_replace(&tree
, node
, slot
, NULL
, NULL
, NULL
);
548 assert(node
->exceptional
== 0);
550 item_kill_tree(&tree
);
553 void multiorder_checks(void)
557 for (i
= 0; i
< 20; i
++) {
558 multiorder_check(200, i
);
559 multiorder_check(0, i
);
560 multiorder_check((1UL << i
) + 1, i
);
563 for (i
= 0; i
< 15; i
++)
564 multiorder_shrink((1UL << (i
+ RADIX_TREE_MAP_SHIFT
)), i
);
566 multiorder_insert_bug();
567 multiorder_tag_tests();
568 multiorder_iteration();
569 multiorder_tagged_iteration();
572 multiorder_account();
574 radix_tree_cpu_dead(0);