]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/app/test/test_hash_readwrite.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
import 15.2.0 Octopus source
[ceph.git] / ceph / src / spdk / dpdk / app / test / test_hash_readwrite.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 #include <inttypes.h>
6 #include <locale.h>
7
8 #include <rte_cycles.h>
9 #include <rte_hash.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
16
17 #include "test.h"
18
19 #define RTE_RWTEST_FAIL 0
20
21 #define TOTAL_ENTRY (5*1024*1024)
22 #define TOTAL_INSERT (4.5*1024*1024)
23 #define TOTAL_INSERT_EXT (5*1024*1024)
24
25 #define NUM_TEST 3
26 unsigned int core_cnt[NUM_TEST] = {2, 4, 8};
27
28 unsigned int slave_core_ids[RTE_MAX_LCORE];
29 struct perf {
30 uint32_t single_read;
31 uint32_t single_write;
32 uint32_t read_only[NUM_TEST];
33 uint32_t write_only[NUM_TEST];
34 uint32_t read_write_r[NUM_TEST];
35 uint32_t read_write_w[NUM_TEST];
36 };
37
38 static struct perf htm_results, non_htm_results;
39
40 struct {
41 uint32_t *keys;
42 uint8_t *found;
43 uint32_t num_insert;
44 uint32_t rounded_tot_insert;
45 struct rte_hash *h;
46 } tbl_rw_test_param;
47
48 static rte_atomic64_t gcycles;
49 static rte_atomic64_t ginsertions;
50
51 static rte_atomic64_t gread_cycles;
52 static rte_atomic64_t gwrite_cycles;
53
54 static rte_atomic64_t greads;
55 static rte_atomic64_t gwrites;
56
57 static int
58 test_hash_readwrite_worker(__attribute__((unused)) void *arg)
59 {
60 uint64_t i, offset;
61 uint32_t lcore_id = rte_lcore_id();
62 uint64_t begin, cycles;
63 int *ret;
64
65 ret = rte_malloc(NULL, sizeof(int) *
66 tbl_rw_test_param.num_insert, 0);
67 for (i = 0; i < rte_lcore_count(); i++) {
68 if (slave_core_ids[i] == lcore_id)
69 break;
70 }
71 offset = tbl_rw_test_param.num_insert * i;
72
73 printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n",
74 lcore_id, tbl_rw_test_param.num_insert,
75 offset, offset + tbl_rw_test_param.num_insert - 1);
76
77 begin = rte_rdtsc_precise();
78
79 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
80
81 if (rte_hash_lookup(tbl_rw_test_param.h,
82 tbl_rw_test_param.keys + i) > 0)
83 break;
84
85 ret[i - offset] = rte_hash_add_key(tbl_rw_test_param.h,
86 tbl_rw_test_param.keys + i);
87 if (ret[i - offset] < 0)
88 break;
89
90 /* lookup a random key */
91 uint32_t rand = rte_rand() % (i + 1 - offset);
92
93 if (rte_hash_lookup(tbl_rw_test_param.h,
94 tbl_rw_test_param.keys + rand) != ret[rand])
95 break;
96
97
98 if (rte_hash_del_key(tbl_rw_test_param.h,
99 tbl_rw_test_param.keys + rand) != ret[rand])
100 break;
101
102 ret[rand] = rte_hash_add_key(tbl_rw_test_param.h,
103 tbl_rw_test_param.keys + rand);
104 if (ret[rand] < 0)
105 break;
106
107 if (rte_hash_lookup(tbl_rw_test_param.h,
108 tbl_rw_test_param.keys + rand) != ret[rand])
109 break;
110 }
111
112 cycles = rte_rdtsc_precise() - begin;
113 rte_atomic64_add(&gcycles, cycles);
114 rte_atomic64_add(&ginsertions, i - offset);
115
116 for (; i < offset + tbl_rw_test_param.num_insert; i++)
117 tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL;
118
119 rte_free(ret);
120 return 0;
121 }
122
123 static int
124 init_params(int use_ext, int use_htm, int use_jhash)
125 {
126 unsigned int i;
127
128 uint32_t *keys = NULL;
129 uint8_t *found = NULL;
130 struct rte_hash *handle;
131
132 struct rte_hash_parameters hash_params = {
133 .entries = TOTAL_ENTRY,
134 .key_len = sizeof(uint32_t),
135 .hash_func_init_val = 0,
136 .socket_id = rte_socket_id(),
137 };
138 if (use_jhash)
139 hash_params.hash_func = rte_jhash;
140 else
141 hash_params.hash_func = rte_hash_crc;
142
143 if (use_htm)
144 hash_params.extra_flag =
145 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
146 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
147 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
148 else
149 hash_params.extra_flag =
150 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY |
151 RTE_HASH_EXTRA_FLAGS_MULTI_WRITER_ADD;
152
153 if (use_ext)
154 hash_params.extra_flag |=
155 RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
156 else
157 hash_params.extra_flag &=
158 ~RTE_HASH_EXTRA_FLAGS_EXT_TABLE;
159
160 hash_params.name = "tests";
161
162 handle = rte_hash_create(&hash_params);
163 if (handle == NULL) {
164 printf("hash creation failed");
165 return -1;
166 }
167
168 tbl_rw_test_param.h = handle;
169 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
170
171 if (keys == NULL) {
172 printf("RTE_MALLOC failed\n");
173 goto err;
174 }
175
176 found = rte_zmalloc(NULL, sizeof(uint8_t) * TOTAL_ENTRY, 0);
177 if (found == NULL) {
178 printf("RTE_ZMALLOC failed\n");
179 goto err;
180 }
181
182 tbl_rw_test_param.keys = keys;
183 tbl_rw_test_param.found = found;
184
185 for (i = 0; i < TOTAL_ENTRY; i++)
186 keys[i] = i;
187
188 return 0;
189
190 err:
191 rte_free(keys);
192 rte_hash_free(handle);
193
194 return -1;
195 }
196
197 static int
198 test_hash_readwrite_functional(int use_ext, int use_htm)
199 {
200 unsigned int i;
201 const void *next_key;
202 void *next_data;
203 uint32_t iter = 0;
204
205 uint32_t duplicated_keys = 0;
206 uint32_t lost_keys = 0;
207 int use_jhash = 1;
208 int slave_cnt = rte_lcore_count() - 1;
209 uint32_t tot_insert = 0;
210
211 rte_atomic64_init(&gcycles);
212 rte_atomic64_clear(&gcycles);
213
214 rte_atomic64_init(&ginsertions);
215 rte_atomic64_clear(&ginsertions);
216
217 if (init_params(use_ext, use_htm, use_jhash) != 0)
218 goto err;
219
220 if (use_ext)
221 tot_insert = TOTAL_INSERT_EXT;
222 else
223 tot_insert = TOTAL_INSERT;
224
225 tbl_rw_test_param.num_insert =
226 tot_insert / slave_cnt;
227
228 tbl_rw_test_param.rounded_tot_insert =
229 tbl_rw_test_param.num_insert
230 * slave_cnt;
231
232 printf("++++++++Start function tests:+++++++++\n");
233
234 /* Fire all threads. */
235 rte_eal_mp_remote_launch(test_hash_readwrite_worker,
236 NULL, SKIP_MASTER);
237 rte_eal_mp_wait_lcore();
238
239 while (rte_hash_iterate(tbl_rw_test_param.h, &next_key,
240 &next_data, &iter) >= 0) {
241 /* Search for the key in the list of keys added .*/
242 i = *(const uint32_t *)next_key;
243 tbl_rw_test_param.found[i]++;
244 }
245
246 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
247 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
248 if (tbl_rw_test_param.found[i] > 1) {
249 duplicated_keys++;
250 break;
251 }
252 if (tbl_rw_test_param.found[i] == 0) {
253 lost_keys++;
254 printf("key %d is lost\n", i);
255 break;
256 }
257 }
258 }
259
260 if (duplicated_keys > 0) {
261 printf("%d key duplicated\n", duplicated_keys);
262 goto err_free;
263 }
264
265 if (lost_keys > 0) {
266 printf("%d key lost\n", lost_keys);
267 goto err_free;
268 }
269
270 printf("No key corrupted during read-write test.\n");
271
272 unsigned long long int cycles_per_insertion =
273 rte_atomic64_read(&gcycles) /
274 rte_atomic64_read(&ginsertions);
275
276 printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion);
277
278 rte_free(tbl_rw_test_param.found);
279 rte_free(tbl_rw_test_param.keys);
280 rte_hash_free(tbl_rw_test_param.h);
281 printf("+++++++++Complete function tests+++++++++\n");
282 return 0;
283
284 err_free:
285 rte_free(tbl_rw_test_param.found);
286 rte_free(tbl_rw_test_param.keys);
287 rte_hash_free(tbl_rw_test_param.h);
288 err:
289 return -1;
290 }
291
292 static int
293 test_rw_reader(void *arg)
294 {
295 uint64_t i;
296 uint64_t begin, cycles;
297 uint64_t read_cnt = (uint64_t)((uintptr_t)arg);
298
299 begin = rte_rdtsc_precise();
300 for (i = 0; i < read_cnt; i++) {
301 void *data;
302 rte_hash_lookup_data(tbl_rw_test_param.h,
303 tbl_rw_test_param.keys + i,
304 &data);
305 if (i != (uint64_t)(uintptr_t)data) {
306 printf("lookup find wrong value %"PRIu64","
307 "%"PRIu64"\n", i,
308 (uint64_t)(uintptr_t)data);
309 break;
310 }
311 }
312
313 cycles = rte_rdtsc_precise() - begin;
314 rte_atomic64_add(&gread_cycles, cycles);
315 rte_atomic64_add(&greads, i);
316 return 0;
317 }
318
319 static int
320 test_rw_writer(void *arg)
321 {
322 uint64_t i;
323 uint32_t lcore_id = rte_lcore_id();
324 uint64_t begin, cycles;
325 int ret;
326 uint64_t start_coreid = (uint64_t)(uintptr_t)arg;
327 uint64_t offset;
328
329 for (i = 0; i < rte_lcore_count(); i++) {
330 if (slave_core_ids[i] == lcore_id)
331 break;
332 }
333
334 offset = TOTAL_INSERT / 2 + (i - (start_coreid)) *
335 tbl_rw_test_param.num_insert;
336 begin = rte_rdtsc_precise();
337 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
338 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
339 tbl_rw_test_param.keys + i,
340 (void *)((uintptr_t)i));
341 if (ret < 0) {
342 printf("writer failed %"PRIu64"\n", i);
343 break;
344 }
345 }
346
347 cycles = rte_rdtsc_precise() - begin;
348 rte_atomic64_add(&gwrite_cycles, cycles);
349 rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert);
350 return 0;
351 }
352
353 static int
354 test_hash_readwrite_perf(struct perf *perf_results, int use_htm,
355 int reader_faster)
356 {
357 unsigned int n;
358 int ret;
359 int start_coreid;
360 uint64_t i, read_cnt;
361
362 const void *next_key;
363 void *next_data;
364 uint32_t iter;
365 int use_jhash = 0;
366
367 uint32_t duplicated_keys = 0;
368 uint32_t lost_keys = 0;
369
370 uint64_t start = 0, end = 0;
371
372 rte_atomic64_init(&greads);
373 rte_atomic64_init(&gwrites);
374 rte_atomic64_clear(&gwrites);
375 rte_atomic64_clear(&greads);
376
377 rte_atomic64_init(&gread_cycles);
378 rte_atomic64_clear(&gread_cycles);
379 rte_atomic64_init(&gwrite_cycles);
380 rte_atomic64_clear(&gwrite_cycles);
381
382 if (init_params(0, use_htm, use_jhash) != 0)
383 goto err;
384
385 /*
386 * Do a readers finish faster or writers finish faster test.
387 * When readers finish faster, we timing the readers, and when writers
388 * finish faster, we timing the writers.
389 * Divided by 10 or 2 is just experimental values to vary the workload
390 * of readers.
391 */
392 if (reader_faster) {
393 printf("++++++Start perf test: reader++++++++\n");
394 read_cnt = TOTAL_INSERT / 10;
395 } else {
396 printf("++++++Start perf test: writer++++++++\n");
397 read_cnt = TOTAL_INSERT / 2;
398 }
399
400 /* We first test single thread performance */
401 start = rte_rdtsc_precise();
402 /* Insert half of the keys */
403 for (i = 0; i < TOTAL_INSERT / 2; i++) {
404 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
405 tbl_rw_test_param.keys + i,
406 (void *)((uintptr_t)i));
407 if (ret < 0) {
408 printf("Failed to insert half of keys\n");
409 goto err_free;
410 }
411 }
412 end = rte_rdtsc_precise() - start;
413 perf_results->single_write = end / i;
414
415 start = rte_rdtsc_precise();
416
417 for (i = 0; i < read_cnt; i++) {
418 void *data;
419 rte_hash_lookup_data(tbl_rw_test_param.h,
420 tbl_rw_test_param.keys + i,
421 &data);
422 if (i != (uint64_t)(uintptr_t)data) {
423 printf("lookup find wrong value"
424 " %"PRIu64",%"PRIu64"\n", i,
425 (uint64_t)(uintptr_t)data);
426 break;
427 }
428 }
429 end = rte_rdtsc_precise() - start;
430 perf_results->single_read = end / i;
431
432 for (n = 0; n < NUM_TEST; n++) {
433 unsigned int tot_slave_lcore = rte_lcore_count() - 1;
434 if (tot_slave_lcore < core_cnt[n] * 2)
435 goto finish;
436
437 rte_atomic64_clear(&greads);
438 rte_atomic64_clear(&gread_cycles);
439 rte_atomic64_clear(&gwrites);
440 rte_atomic64_clear(&gwrite_cycles);
441
442 rte_hash_reset(tbl_rw_test_param.h);
443
444 tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n];
445 tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 +
446 tbl_rw_test_param.num_insert *
447 core_cnt[n];
448
449 for (i = 0; i < TOTAL_INSERT / 2; i++) {
450 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
451 tbl_rw_test_param.keys + i,
452 (void *)((uintptr_t)i));
453 if (ret < 0) {
454 printf("Failed to insert half of keys\n");
455 goto err_free;
456 }
457 }
458
459 /* Then test multiple thread case but only all reads or
460 * all writes
461 */
462
463 /* Test only reader cases */
464 for (i = 0; i < core_cnt[n]; i++)
465 rte_eal_remote_launch(test_rw_reader,
466 (void *)(uintptr_t)read_cnt,
467 slave_core_ids[i]);
468
469 rte_eal_mp_wait_lcore();
470
471 start_coreid = i;
472 /* Test only writer cases */
473 for (; i < core_cnt[n] * 2; i++)
474 rte_eal_remote_launch(test_rw_writer,
475 (void *)((uintptr_t)start_coreid),
476 slave_core_ids[i]);
477
478 rte_eal_mp_wait_lcore();
479
480 if (reader_faster) {
481 unsigned long long int cycles_per_insertion =
482 rte_atomic64_read(&gread_cycles) /
483 rte_atomic64_read(&greads);
484 perf_results->read_only[n] = cycles_per_insertion;
485 printf("Reader only: cycles per lookup: %llu\n",
486 cycles_per_insertion);
487 }
488
489 else {
490 unsigned long long int cycles_per_insertion =
491 rte_atomic64_read(&gwrite_cycles) /
492 rte_atomic64_read(&gwrites);
493 perf_results->write_only[n] = cycles_per_insertion;
494 printf("Writer only: cycles per writes: %llu\n",
495 cycles_per_insertion);
496 }
497
498 rte_atomic64_clear(&greads);
499 rte_atomic64_clear(&gread_cycles);
500 rte_atomic64_clear(&gwrites);
501 rte_atomic64_clear(&gwrite_cycles);
502
503 rte_hash_reset(tbl_rw_test_param.h);
504
505 for (i = 0; i < TOTAL_INSERT / 2; i++) {
506 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
507 tbl_rw_test_param.keys + i,
508 (void *)((uintptr_t)i));
509 if (ret < 0) {
510 printf("Failed to insert half of keys\n");
511 goto err_free;
512 }
513 }
514
515 start_coreid = core_cnt[n];
516
517 if (reader_faster) {
518 for (i = core_cnt[n]; i < core_cnt[n] * 2; i++)
519 rte_eal_remote_launch(test_rw_writer,
520 (void *)((uintptr_t)start_coreid),
521 slave_core_ids[i]);
522 for (i = 0; i < core_cnt[n]; i++)
523 rte_eal_remote_launch(test_rw_reader,
524 (void *)(uintptr_t)read_cnt,
525 slave_core_ids[i]);
526 } else {
527 for (i = 0; i < core_cnt[n]; i++)
528 rte_eal_remote_launch(test_rw_reader,
529 (void *)(uintptr_t)read_cnt,
530 slave_core_ids[i]);
531 for (; i < core_cnt[n] * 2; i++)
532 rte_eal_remote_launch(test_rw_writer,
533 (void *)((uintptr_t)start_coreid),
534 slave_core_ids[i]);
535 }
536
537 rte_eal_mp_wait_lcore();
538
539 iter = 0;
540 memset(tbl_rw_test_param.found, 0, TOTAL_ENTRY);
541 while (rte_hash_iterate(tbl_rw_test_param.h,
542 &next_key, &next_data, &iter) >= 0) {
543 /* Search for the key in the list of keys added .*/
544 i = *(const uint32_t *)next_key;
545 tbl_rw_test_param.found[i]++;
546 }
547
548 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
549 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
550 if (tbl_rw_test_param.found[i] > 1) {
551 duplicated_keys++;
552 break;
553 }
554 if (tbl_rw_test_param.found[i] == 0) {
555 lost_keys++;
556 printf("key %"PRIu64" is lost\n", i);
557 break;
558 }
559 }
560 }
561
562 if (duplicated_keys > 0) {
563 printf("%d key duplicated\n", duplicated_keys);
564 goto err_free;
565 }
566
567 if (lost_keys > 0) {
568 printf("%d key lost\n", lost_keys);
569 goto err_free;
570 }
571
572 printf("No key corrupted during read-write test.\n");
573
574 if (reader_faster) {
575 unsigned long long int cycles_per_insertion =
576 rte_atomic64_read(&gread_cycles) /
577 rte_atomic64_read(&greads);
578 perf_results->read_write_r[n] = cycles_per_insertion;
579 printf("Read-write cycles per lookup: %llu\n",
580 cycles_per_insertion);
581 }
582
583 else {
584 unsigned long long int cycles_per_insertion =
585 rte_atomic64_read(&gwrite_cycles) /
586 rte_atomic64_read(&gwrites);
587 perf_results->read_write_w[n] = cycles_per_insertion;
588 printf("Read-write cycles per writes: %llu\n",
589 cycles_per_insertion);
590 }
591 }
592
593 finish:
594 rte_free(tbl_rw_test_param.found);
595 rte_free(tbl_rw_test_param.keys);
596 rte_hash_free(tbl_rw_test_param.h);
597 return 0;
598
599 err_free:
600 rte_free(tbl_rw_test_param.found);
601 rte_free(tbl_rw_test_param.keys);
602 rte_hash_free(tbl_rw_test_param.h);
603
604 err:
605 return -1;
606 }
607
608 static int
609 test_hash_readwrite_main(void)
610 {
611 /*
612 * Variables used to choose different tests.
613 * use_htm indicates if hardware transactional memory should be used.
614 * reader_faster indicates if the reader threads should finish earlier
615 * than writer threads. This is to timing either reader threads or
616 * writer threads for performance numbers.
617 */
618 int use_htm, use_ext, reader_faster;
619 unsigned int i = 0, core_id = 0;
620
621 if (rte_lcore_count() <= 2) {
622 printf("More than two lcores are required "
623 "to do read write test\n");
624 return -1;
625 }
626
627 RTE_LCORE_FOREACH_SLAVE(core_id) {
628 slave_core_ids[i] = core_id;
629 i++;
630 }
631
632 setlocale(LC_NUMERIC, "");
633
634 if (rte_tm_supported()) {
635 printf("Hardware transactional memory (lock elision) "
636 "is supported\n");
637
638 printf("Test read-write with Hardware transactional memory\n");
639
640 use_htm = 1;
641 use_ext = 0;
642
643 if (test_hash_readwrite_functional(use_ext, use_htm) < 0)
644 return -1;
645
646 use_ext = 1;
647 if (test_hash_readwrite_functional(use_ext, use_htm) < 0)
648 return -1;
649
650 reader_faster = 1;
651 if (test_hash_readwrite_perf(&htm_results, use_htm,
652 reader_faster) < 0)
653 return -1;
654
655 reader_faster = 0;
656 if (test_hash_readwrite_perf(&htm_results, use_htm,
657 reader_faster) < 0)
658 return -1;
659 } else {
660 printf("Hardware transactional memory (lock elision) "
661 "is NOT supported\n");
662 }
663
664 printf("Test read-write without Hardware transactional memory\n");
665 use_htm = 0;
666 use_ext = 0;
667 if (test_hash_readwrite_functional(use_ext, use_htm) < 0)
668 return -1;
669
670 use_ext = 1;
671 if (test_hash_readwrite_functional(use_ext, use_htm) < 0)
672 return -1;
673
674 reader_faster = 1;
675 if (test_hash_readwrite_perf(&non_htm_results, use_htm,
676 reader_faster) < 0)
677 return -1;
678 reader_faster = 0;
679 if (test_hash_readwrite_perf(&non_htm_results, use_htm,
680 reader_faster) < 0)
681 return -1;
682
683 printf("================\n");
684 printf("Results summary:\n");
685 printf("================\n");
686
687 printf("single read: %u\n", htm_results.single_read);
688 printf("single write: %u\n", htm_results.single_write);
689 for (i = 0; i < NUM_TEST; i++) {
690 printf("+++ core_cnt: %u +++\n", core_cnt[i]);
691 printf("HTM:\n");
692 printf(" read only: %u\n", htm_results.read_only[i]);
693 printf(" write only: %u\n", htm_results.write_only[i]);
694 printf(" read-write read: %u\n", htm_results.read_write_r[i]);
695 printf(" read-write write: %u\n", htm_results.read_write_w[i]);
696
697 printf("non HTM:\n");
698 printf(" read only: %u\n", non_htm_results.read_only[i]);
699 printf(" write only: %u\n", non_htm_results.write_only[i]);
700 printf(" read-write read: %u\n",
701 non_htm_results.read_write_r[i]);
702 printf(" read-write write: %u\n",
703 non_htm_results.read_write_w[i]);
704 }
705
706 return 0;
707 }
708
709 REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main);
Ceph