tools/testing/radix-tree/multiorder.c

   1 /*
   2  * multiorder.c: Multi-order radix tree entry testing
   3  * Copyright (c) 2016 Intel Corporation
   4  * Author: Ross Zwisler <ross.zwisler@linux.intel.com>
   5  * Author: Matthew Wilcox <matthew.r.wilcox@intel.com>
   6  *
   7  * This program is free software; you can redistribute it and/or modify it
   8  * under the terms and conditions of the GNU General Public License,
   9  * version 2, as published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  14  * more details.
  15  */
  16 #include <linux/radix-tree.h>
  17 #include <linux/slab.h>
  18 #include <linux/errno.h>
  19 #include <pthread.h>
  20
  21 #include "test.h"
  22
  23 #define for_each_index(i, base, order) \
  24         for (i = base; i < base + (1 << order); i++)
  25
  26 static void __multiorder_tag_test(int index, int order)
  27 {
  28         RADIX_TREE(tree, GFP_KERNEL);
  29         int base, err, i;
  30
  31         /* our canonical entry */
  32         base = index & ~((1 << order) - 1);
  33
  34         printv(2, "Multiorder tag test with index %d, canonical entry %d\n",
  35                         index, base);
  36
  37         err = item_insert_order(&tree, index, order);
  38         assert(!err);
  39
  40         /*
  41          * Verify we get collisions for covered indices.  We try and fail to
  42          * insert a value entry so we don't leak memory via
  43          * item_insert_order().
  44          */
  45         for_each_index(i, base, order) {
  46                 err = __radix_tree_insert(&tree, i, order, xa_mk_value(0xA0));
  47                 assert(err == -EEXIST);
  48         }
  49
  50         for_each_index(i, base, order) {
  51                 assert(!radix_tree_tag_get(&tree, i, 0));
  52                 assert(!radix_tree_tag_get(&tree, i, 1));
  53         }
  54
  55         assert(radix_tree_tag_set(&tree, index, 0));
  56
  57         for_each_index(i, base, order) {
  58                 assert(radix_tree_tag_get(&tree, i, 0));
  59                 assert(!radix_tree_tag_get(&tree, i, 1));
  60         }
  61
  62         assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 1);
  63         assert(radix_tree_tag_clear(&tree, index, 0));
  64
  65         for_each_index(i, base, order) {
  66                 assert(!radix_tree_tag_get(&tree, i, 0));
  67                 assert(radix_tree_tag_get(&tree, i, 1));
  68         }
  69
  70         assert(radix_tree_tag_clear(&tree, index, 1));
  71
  72         assert(!radix_tree_tagged(&tree, 0));
  73         assert(!radix_tree_tagged(&tree, 1));
  74
  75         item_kill_tree(&tree);
  76 }
  77
  78 static void __multiorder_tag_test2(unsigned order, unsigned long index2)
  79 {
  80         RADIX_TREE(tree, GFP_KERNEL);
  81         unsigned long index = (1 << order);
  82         index2 += index;
  83
  84         assert(item_insert_order(&tree, 0, order) == 0);
  85         assert(item_insert(&tree, index2) == 0);
  86
  87         assert(radix_tree_tag_set(&tree, 0, 0));
  88         assert(radix_tree_tag_set(&tree, index2, 0));
  89
  90         assert(tag_tagged_items(&tree, NULL, 0, ~0UL, 10, 0, 1) == 2);
  91
  92         item_kill_tree(&tree);
  93 }
  94
  95 static void multiorder_tag_tests(void)
  96 {
  97         int i, j;
  98
  99         /* test multi-order entry for indices 0-7 with no sibling pointers */
 100         __multiorder_tag_test(0, 3);
 101         __multiorder_tag_test(5, 3);
 102
 103         /* test multi-order entry for indices 8-15 with no sibling pointers */
 104         __multiorder_tag_test(8, 3);
 105         __multiorder_tag_test(15, 3);
 106
 107         /*
 108          * Our order 5 entry covers indices 0-31 in a tree with height=2.
 109          * This is broken up as follows:
 110          * 0-7:         canonical entry
 111          * 8-15:        sibling 1
 112          * 16-23:       sibling 2
 113          * 24-31:       sibling 3
 114          */
 115         __multiorder_tag_test(0, 5);
 116         __multiorder_tag_test(29, 5);
 117
 118         /* same test, but with indices 32-63 */
 119         __multiorder_tag_test(32, 5);
 120         __multiorder_tag_test(44, 5);
 121
 122         /*
 123          * Our order 8 entry covers indices 0-255 in a tree with height=3.
 124          * This is broken up as follows:
 125          * 0-63:        canonical entry
 126          * 64-127:      sibling 1
 127          * 128-191:     sibling 2
 128          * 192-255:     sibling 3
 129          */
 130         __multiorder_tag_test(0, 8);
 131         __multiorder_tag_test(190, 8);
 132
 133         /* same test, but with indices 256-511 */
 134         __multiorder_tag_test(256, 8);
 135         __multiorder_tag_test(300, 8);
 136
 137         __multiorder_tag_test(0x12345678UL, 8);
 138
 139         for (i = 1; i < 10; i++)
 140                 for (j = 0; j < (10 << i); j++)
 141                         __multiorder_tag_test2(i, j);
 142 }
 143
 144 static void multiorder_check(unsigned long index, int order)
 145 {
 146         unsigned long i;
 147         unsigned long min = index & ~((1UL << order) - 1);
 148         unsigned long max = min + (1UL << order);
 149         void **slot;
 150         struct item *item2 = item_create(min, order);
 151         RADIX_TREE(tree, GFP_KERNEL);
 152
 153         printv(2, "Multiorder index %ld, order %d\n", index, order);
 154
 155         assert(item_insert_order(&tree, index, order) == 0);
 156
 157         for (i = min; i < max; i++) {
 158                 struct item *item = item_lookup(&tree, i);
 159                 assert(item != 0);
 160                 assert(item->index == index);
 161         }
 162         for (i = 0; i < min; i++)
 163                 item_check_absent(&tree, i);
 164         for (i = max; i < 2*max; i++)
 165                 item_check_absent(&tree, i);
 166         for (i = min; i < max; i++)
 167                 assert(radix_tree_insert(&tree, i, item2) == -EEXIST);
 168
 169         slot = radix_tree_lookup_slot(&tree, index);
 170         free(*slot);
 171         radix_tree_replace_slot(&tree, slot, item2);
 172         for (i = min; i < max; i++) {
 173                 struct item *item = item_lookup(&tree, i);
 174                 assert(item != 0);
 175                 assert(item->index == min);
 176         }
 177
 178         assert(item_delete(&tree, min) != 0);
 179
 180         for (i = 0; i < 2*max; i++)
 181                 item_check_absent(&tree, i);
 182 }
 183
 184 static void multiorder_shrink(unsigned long index, int order)
 185 {
 186         unsigned long i;
 187         unsigned long max = 1 << order;
 188         RADIX_TREE(tree, GFP_KERNEL);
 189         struct radix_tree_node *node;
 190
 191         printv(2, "Multiorder shrink index %ld, order %d\n", index, order);
 192
 193         assert(item_insert_order(&tree, 0, order) == 0);
 194
 195         node = tree.xa_head;
 196
 197         assert(item_insert(&tree, index) == 0);
 198         assert(node != tree.xa_head);
 199
 200         assert(item_delete(&tree, index) != 0);
 201         assert(node == tree.xa_head);
 202
 203         for (i = 0; i < max; i++) {
 204                 struct item *item = item_lookup(&tree, i);
 205                 assert(item != 0);
 206                 assert(item->index == 0);
 207         }
 208         for (i = max; i < 2*max; i++)
 209                 item_check_absent(&tree, i);
 210
 211         if (!item_delete(&tree, 0)) {
 212                 printv(2, "failed to delete index %ld (order %d)\n", index, order);
 213                 abort();
 214         }
 215
 216         for (i = 0; i < 2*max; i++)
 217                 item_check_absent(&tree, i);
 218 }
 219
 220 static void multiorder_insert_bug(void)
 221 {
 222         RADIX_TREE(tree, GFP_KERNEL);
 223
 224         item_insert(&tree, 0);
 225         radix_tree_tag_set(&tree, 0, 0);
 226         item_insert_order(&tree, 3 << 6, 6);
 227
 228         item_kill_tree(&tree);
 229 }
 230
 231 void multiorder_iteration(void)
 232 {
 233         RADIX_TREE(tree, GFP_KERNEL);
 234         struct radix_tree_iter iter;
 235         void **slot;
 236         int i, j, err;
 237
 238         printv(1, "Multiorder iteration test\n");
 239
 240 #define NUM_ENTRIES 11
 241         int index[NUM_ENTRIES] = {0, 2, 4, 8, 16, 32, 34, 36, 64, 72, 128};
 242         int order[NUM_ENTRIES] = {1, 1, 2, 3,  4,  1,  0,  1,  3,  0, 7};
 243
 244         for (i = 0; i < NUM_ENTRIES; i++) {
 245                 err = item_insert_order(&tree, index[i], order[i]);
 246                 assert(!err);
 247         }
 248
 249         for (j = 0; j < 256; j++) {
 250                 for (i = 0; i < NUM_ENTRIES; i++)
 251                         if (j <= (index[i] | ((1 << order[i]) - 1)))
 252                                 break;
 253
 254                 radix_tree_for_each_slot(slot, &tree, &iter, j) {
 255                         int height = order[i] / RADIX_TREE_MAP_SHIFT;
 256                         int shift = height * RADIX_TREE_MAP_SHIFT;
 257                         unsigned long mask = (1UL << order[i]) - 1;
 258                         struct item *item = *slot;
 259
 260                         assert((iter.index | mask) == (index[i] | mask));
 261                         assert(iter.shift == shift);
 262                         assert(!radix_tree_is_internal_node(item));
 263                         assert((item->index | mask) == (index[i] | mask));
 264                         assert(item->order == order[i]);
 265                         i++;
 266                 }
 267         }
 268
 269         item_kill_tree(&tree);
 270 }
 271
 272 void multiorder_tagged_iteration(void)
 273 {
 274         RADIX_TREE(tree, GFP_KERNEL);
 275         struct radix_tree_iter iter;
 276         void **slot;
 277         int i, j;
 278
 279         printv(1, "Multiorder tagged iteration test\n");
 280
 281 #define MT_NUM_ENTRIES 9
 282         int index[MT_NUM_ENTRIES] = {0, 2, 4, 16, 32, 40, 64, 72, 128};
 283         int order[MT_NUM_ENTRIES] = {1, 0, 2, 4,  3,  1,  3,  0,   7};
 284
 285 #define TAG_ENTRIES 7
 286         int tag_index[TAG_ENTRIES] = {0, 4, 16, 40, 64, 72, 128};
 287
 288         for (i = 0; i < MT_NUM_ENTRIES; i++)
 289                 assert(!item_insert_order(&tree, index[i], order[i]));
 290
 291         assert(!radix_tree_tagged(&tree, 1));
 292
 293         for (i = 0; i < TAG_ENTRIES; i++)
 294                 assert(radix_tree_tag_set(&tree, tag_index[i], 1));
 295
 296         for (j = 0; j < 256; j++) {
 297                 int k;
 298
 299                 for (i = 0; i < TAG_ENTRIES; i++) {
 300                         for (k = i; index[k] < tag_index[i]; k++)
 301                                 ;
 302                         if (j <= (index[k] | ((1 << order[k]) - 1)))
 303                                 break;
 304                 }
 305
 306                 radix_tree_for_each_tagged(slot, &tree, &iter, j, 1) {
 307                         unsigned long mask;
 308                         struct item *item = *slot;
 309                         for (k = i; index[k] < tag_index[i]; k++)
 310                                 ;
 311                         mask = (1UL << order[k]) - 1;
 312
 313                         assert((iter.index | mask) == (tag_index[i] | mask));
 314                         assert(!radix_tree_is_internal_node(item));
 315                         assert((item->index | mask) == (tag_index[i] | mask));
 316                         assert(item->order == order[k]);
 317                         i++;
 318                 }
 319         }
 320
 321         assert(tag_tagged_items(&tree, NULL, 0, ~0UL, TAG_ENTRIES, 1, 2) ==
 322                                 TAG_ENTRIES);
 323
 324         for (j = 0; j < 256; j++) {
 325                 int mask, k;
 326
 327                 for (i = 0; i < TAG_ENTRIES; i++) {
 328                         for (k = i; index[k] < tag_index[i]; k++)
 329                                 ;
 330                         if (j <= (index[k] | ((1 << order[k]) - 1)))
 331                                 break;
 332                 }
 333
 334                 radix_tree_for_each_tagged(slot, &tree, &iter, j, 2) {
 335                         struct item *item = *slot;
 336                         for (k = i; index[k] < tag_index[i]; k++)
 337                                 ;
 338                         mask = (1 << order[k]) - 1;
 339
 340                         assert((iter.index | mask) == (tag_index[i] | mask));
 341                         assert(!radix_tree_is_internal_node(item));
 342                         assert((item->index | mask) == (tag_index[i] | mask));
 343                         assert(item->order == order[k]);
 344                         i++;
 345                 }
 346         }
 347
 348         assert(tag_tagged_items(&tree, NULL, 1, ~0UL, MT_NUM_ENTRIES * 2, 1, 0)
 349                         == TAG_ENTRIES);
 350         i = 0;
 351         radix_tree_for_each_tagged(slot, &tree, &iter, 0, 0) {
 352                 assert(iter.index == tag_index[i]);
 353                 i++;
 354         }
 355
 356         item_kill_tree(&tree);
 357 }
 358
 359 static void multiorder_account(void)
 360 {
 361         RADIX_TREE(tree, GFP_KERNEL);
 362         struct radix_tree_node *node;
 363         void **slot;
 364
 365         item_insert_order(&tree, 0, 5);
 366
 367         __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5));
 368         __radix_tree_lookup(&tree, 0, &node, NULL);
 369         assert(node->count == node->nr_values * 2);
 370         radix_tree_delete(&tree, 1 << 5);
 371         assert(node->nr_values == 0);
 372
 373         __radix_tree_insert(&tree, 1 << 5, 5, xa_mk_value(5));
 374         __radix_tree_lookup(&tree, 1 << 5, &node, &slot);
 375         assert(node->count == node->nr_values * 2);
 376         __radix_tree_replace(&tree, node, slot, NULL);
 377         assert(node->nr_values == 0);
 378
 379         item_kill_tree(&tree);
 380 }
 381
 382 bool stop_iteration = false;
 383
 384 static void *creator_func(void *ptr)
 385 {
 386         /* 'order' is set up to ensure we have sibling entries */
 387         unsigned int order = RADIX_TREE_MAP_SHIFT - 1;
 388         struct radix_tree_root *tree = ptr;
 389         int i;
 390
 391         for (i = 0; i < 10000; i++) {
 392                 item_insert_order(tree, 0, order);
 393                 item_delete_rcu(tree, 0);
 394         }
 395
 396         stop_iteration = true;
 397         return NULL;
 398 }
 399
 400 static void *iterator_func(void *ptr)
 401 {
 402         struct radix_tree_root *tree = ptr;
 403         struct radix_tree_iter iter;
 404         struct item *item;
 405         void **slot;
 406
 407         while (!stop_iteration) {
 408                 rcu_read_lock();
 409                 radix_tree_for_each_slot(slot, tree, &iter, 0) {
 410                         item = radix_tree_deref_slot(slot);
 411
 412                         if (!item)
 413                                 continue;
 414                         if (radix_tree_deref_retry(item)) {
 415                                 slot = radix_tree_iter_retry(&iter);
 416                                 continue;
 417                         }
 418
 419                         item_sanity(item, iter.index);
 420                 }
 421                 rcu_read_unlock();
 422         }
 423         return NULL;
 424 }
 425
 426 static void multiorder_iteration_race(void)
 427 {
 428         const int num_threads = sysconf(_SC_NPROCESSORS_ONLN);
 429         pthread_t worker_thread[num_threads];
 430         RADIX_TREE(tree, GFP_KERNEL);
 431         int i;
 432
 433         pthread_create(&worker_thread[0], NULL, &creator_func, &tree);
 434         for (i = 1; i < num_threads; i++)
 435                 pthread_create(&worker_thread[i], NULL, &iterator_func, &tree);
 436
 437         for (i = 0; i < num_threads; i++)
 438                 pthread_join(worker_thread[i], NULL);
 439
 440         item_kill_tree(&tree);
 441 }
 442
 443 void multiorder_checks(void)
 444 {
 445         int i;
 446
 447         for (i = 0; i < 20; i++) {
 448                 multiorder_check(200, i);
 449                 multiorder_check(0, i);
 450                 multiorder_check((1UL << i) + 1, i);
 451         }
 452
 453         for (i = 0; i < 15; i++)
 454                 multiorder_shrink((1UL << (i + RADIX_TREE_MAP_SHIFT)), i);
 455
 456         multiorder_insert_bug();
 457         multiorder_tag_tests();
 458         multiorder_iteration();
 459         multiorder_tagged_iteration();
 460         multiorder_account();
 461         multiorder_iteration_race();
 462
 463         radix_tree_cpu_dead(0);
 464 }
 465
 466 int __weak main(void)
 467 {
 468         radix_tree_init();
 469         multiorder_checks();
 470         return 0;
 471 }