]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/test/test/test_hash_readwrite.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update sources to ceph Nautilus 14.2.1
[ceph.git] / ceph / src / spdk / dpdk / test / test / test_hash_readwrite.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2018 Intel Corporation
3 */
4
5 #include <inttypes.h>
6 #include <locale.h>
7
8 #include <rte_cycles.h>
9 #include <rte_hash.h>
10 #include <rte_hash_crc.h>
11 #include <rte_jhash.h>
12 #include <rte_launch.h>
13 #include <rte_malloc.h>
14 #include <rte_random.h>
15 #include <rte_spinlock.h>
16
17 #include "test.h"
18
19 #define RTE_RWTEST_FAIL 0
20
21 #define TOTAL_ENTRY (16*1024*1024)
22 #define TOTAL_INSERT (15*1024*1024)
23
24 #define NUM_TEST 3
25 unsigned int core_cnt[NUM_TEST] = {2, 4, 8};
26
27 struct perf {
28 uint32_t single_read;
29 uint32_t single_write;
30 uint32_t read_only[NUM_TEST];
31 uint32_t write_only[NUM_TEST];
32 uint32_t read_write_r[NUM_TEST];
33 uint32_t read_write_w[NUM_TEST];
34 };
35
36 static struct perf htm_results, non_htm_results;
37
38 struct {
39 uint32_t *keys;
40 uint32_t *found;
41 uint32_t num_insert;
42 uint32_t rounded_tot_insert;
43 struct rte_hash *h;
44 } tbl_rw_test_param;
45
46 static rte_atomic64_t gcycles;
47 static rte_atomic64_t ginsertions;
48
49 static rte_atomic64_t gread_cycles;
50 static rte_atomic64_t gwrite_cycles;
51
52 static rte_atomic64_t greads;
53 static rte_atomic64_t gwrites;
54
55 static int
56 test_hash_readwrite_worker(__attribute__((unused)) void *arg)
57 {
58 uint64_t i, offset;
59 uint32_t lcore_id = rte_lcore_id();
60 uint64_t begin, cycles;
61 int ret;
62
63 offset = (lcore_id - rte_get_master_lcore())
64 * tbl_rw_test_param.num_insert;
65
66 printf("Core #%d inserting and reading %d: %'"PRId64" - %'"PRId64"\n",
67 lcore_id, tbl_rw_test_param.num_insert,
68 offset, offset + tbl_rw_test_param.num_insert);
69
70 begin = rte_rdtsc_precise();
71
72 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
73
74 if (rte_hash_lookup(tbl_rw_test_param.h,
75 tbl_rw_test_param.keys + i) > 0)
76 break;
77
78 ret = rte_hash_add_key(tbl_rw_test_param.h,
79 tbl_rw_test_param.keys + i);
80 if (ret < 0)
81 break;
82
83 if (rte_hash_lookup(tbl_rw_test_param.h,
84 tbl_rw_test_param.keys + i) != ret)
85 break;
86 }
87
88 cycles = rte_rdtsc_precise() - begin;
89 rte_atomic64_add(&gcycles, cycles);
90 rte_atomic64_add(&ginsertions, i - offset);
91
92 for (; i < offset + tbl_rw_test_param.num_insert; i++)
93 tbl_rw_test_param.keys[i] = RTE_RWTEST_FAIL;
94
95 return 0;
96 }
97
98 static int
99 init_params(int use_htm, int use_jhash)
100 {
101 unsigned int i;
102
103 uint32_t *keys = NULL;
104 uint32_t *found = NULL;
105 struct rte_hash *handle;
106
107 struct rte_hash_parameters hash_params = {
108 .entries = TOTAL_ENTRY,
109 .key_len = sizeof(uint32_t),
110 .hash_func_init_val = 0,
111 .socket_id = rte_socket_id(),
112 };
113 if (use_jhash)
114 hash_params.hash_func = rte_jhash;
115 else
116 hash_params.hash_func = rte_hash_crc;
117
118 if (use_htm)
119 hash_params.extra_flag =
120 RTE_HASH_EXTRA_FLAGS_TRANS_MEM_SUPPORT |
121 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
122 else
123 hash_params.extra_flag =
124 RTE_HASH_EXTRA_FLAGS_RW_CONCURRENCY;
125
126 hash_params.name = "tests";
127
128 handle = rte_hash_create(&hash_params);
129 if (handle == NULL) {
130 printf("hash creation failed");
131 return -1;
132 }
133
134 tbl_rw_test_param.h = handle;
135 keys = rte_malloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
136
137 if (keys == NULL) {
138 printf("RTE_MALLOC failed\n");
139 goto err;
140 }
141
142 found = rte_zmalloc(NULL, sizeof(uint32_t) * TOTAL_ENTRY, 0);
143 if (found == NULL) {
144 printf("RTE_ZMALLOC failed\n");
145 goto err;
146 }
147
148 tbl_rw_test_param.keys = keys;
149 tbl_rw_test_param.found = found;
150
151 for (i = 0; i < TOTAL_ENTRY; i++)
152 keys[i] = i;
153
154 return 0;
155
156 err:
157 rte_free(keys);
158 rte_hash_free(handle);
159
160 return -1;
161 }
162
163 static int
164 test_hash_readwrite_functional(int use_htm)
165 {
166 unsigned int i;
167 const void *next_key;
168 void *next_data;
169 uint32_t iter = 0;
170
171 uint32_t duplicated_keys = 0;
172 uint32_t lost_keys = 0;
173 int use_jhash = 1;
174
175 rte_atomic64_init(&gcycles);
176 rte_atomic64_clear(&gcycles);
177
178 rte_atomic64_init(&ginsertions);
179 rte_atomic64_clear(&ginsertions);
180
181 if (init_params(use_htm, use_jhash) != 0)
182 goto err;
183
184 tbl_rw_test_param.num_insert =
185 TOTAL_INSERT / rte_lcore_count();
186
187 tbl_rw_test_param.rounded_tot_insert =
188 tbl_rw_test_param.num_insert
189 * rte_lcore_count();
190
191 printf("++++++++Start function tests:+++++++++\n");
192
193 /* Fire all threads. */
194 rte_eal_mp_remote_launch(test_hash_readwrite_worker,
195 NULL, CALL_MASTER);
196 rte_eal_mp_wait_lcore();
197
198 while (rte_hash_iterate(tbl_rw_test_param.h, &next_key,
199 &next_data, &iter) >= 0) {
200 /* Search for the key in the list of keys added .*/
201 i = *(const uint32_t *)next_key;
202 tbl_rw_test_param.found[i]++;
203 }
204
205 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
206 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
207 if (tbl_rw_test_param.found[i] > 1) {
208 duplicated_keys++;
209 break;
210 }
211 if (tbl_rw_test_param.found[i] == 0) {
212 lost_keys++;
213 printf("key %d is lost\n", i);
214 break;
215 }
216 }
217 }
218
219 if (duplicated_keys > 0) {
220 printf("%d key duplicated\n", duplicated_keys);
221 goto err_free;
222 }
223
224 if (lost_keys > 0) {
225 printf("%d key lost\n", lost_keys);
226 goto err_free;
227 }
228
229 printf("No key corrupted during read-write test.\n");
230
231 unsigned long long int cycles_per_insertion =
232 rte_atomic64_read(&gcycles) /
233 rte_atomic64_read(&ginsertions);
234
235 printf("cycles per insertion and lookup: %llu\n", cycles_per_insertion);
236
237 rte_free(tbl_rw_test_param.found);
238 rte_free(tbl_rw_test_param.keys);
239 rte_hash_free(tbl_rw_test_param.h);
240 printf("+++++++++Complete function tests+++++++++\n");
241 return 0;
242
243 err_free:
244 rte_free(tbl_rw_test_param.found);
245 rte_free(tbl_rw_test_param.keys);
246 rte_hash_free(tbl_rw_test_param.h);
247 err:
248 return -1;
249 }
250
251 static int
252 test_rw_reader(__attribute__((unused)) void *arg)
253 {
254 uint64_t i;
255 uint64_t begin, cycles;
256 uint64_t read_cnt = (uint64_t)((uintptr_t)arg);
257
258 begin = rte_rdtsc_precise();
259 for (i = 0; i < read_cnt; i++) {
260 void *data;
261 rte_hash_lookup_data(tbl_rw_test_param.h,
262 tbl_rw_test_param.keys + i,
263 &data);
264 if (i != (uint64_t)(uintptr_t)data) {
265 printf("lookup find wrong value %"PRIu64","
266 "%"PRIu64"\n", i,
267 (uint64_t)(uintptr_t)data);
268 break;
269 }
270 }
271
272 cycles = rte_rdtsc_precise() - begin;
273 rte_atomic64_add(&gread_cycles, cycles);
274 rte_atomic64_add(&greads, i);
275 return 0;
276 }
277
278 static int
279 test_rw_writer(__attribute__((unused)) void *arg)
280 {
281 uint64_t i;
282 uint32_t lcore_id = rte_lcore_id();
283 uint64_t begin, cycles;
284 int ret;
285 uint64_t start_coreid = (uint64_t)(uintptr_t)arg;
286 uint64_t offset;
287
288 offset = TOTAL_INSERT / 2 + (lcore_id - start_coreid)
289 * tbl_rw_test_param.num_insert;
290 begin = rte_rdtsc_precise();
291 for (i = offset; i < offset + tbl_rw_test_param.num_insert; i++) {
292 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
293 tbl_rw_test_param.keys + i,
294 (void *)((uintptr_t)i));
295 if (ret < 0) {
296 printf("writer failed %"PRIu64"\n", i);
297 break;
298 }
299 }
300
301 cycles = rte_rdtsc_precise() - begin;
302 rte_atomic64_add(&gwrite_cycles, cycles);
303 rte_atomic64_add(&gwrites, tbl_rw_test_param.num_insert);
304 return 0;
305 }
306
307 static int
308 test_hash_readwrite_perf(struct perf *perf_results, int use_htm,
309 int reader_faster)
310 {
311 unsigned int n;
312 int ret;
313 int start_coreid;
314 uint64_t i, read_cnt;
315
316 const void *next_key;
317 void *next_data;
318 uint32_t iter = 0;
319 int use_jhash = 0;
320
321 uint32_t duplicated_keys = 0;
322 uint32_t lost_keys = 0;
323
324 uint64_t start = 0, end = 0;
325
326 rte_atomic64_init(&greads);
327 rte_atomic64_init(&gwrites);
328 rte_atomic64_clear(&gwrites);
329 rte_atomic64_clear(&greads);
330
331 rte_atomic64_init(&gread_cycles);
332 rte_atomic64_clear(&gread_cycles);
333 rte_atomic64_init(&gwrite_cycles);
334 rte_atomic64_clear(&gwrite_cycles);
335
336 if (init_params(use_htm, use_jhash) != 0)
337 goto err;
338
339 /*
340 * Do a readers finish faster or writers finish faster test.
341 * When readers finish faster, we timing the readers, and when writers
342 * finish faster, we timing the writers.
343 * Divided by 10 or 2 is just experimental values to vary the workload
344 * of readers.
345 */
346 if (reader_faster) {
347 printf("++++++Start perf test: reader++++++++\n");
348 read_cnt = TOTAL_INSERT / 10;
349 } else {
350 printf("++++++Start perf test: writer++++++++\n");
351 read_cnt = TOTAL_INSERT / 2;
352 }
353
354 /* We first test single thread performance */
355 start = rte_rdtsc_precise();
356 /* Insert half of the keys */
357 for (i = 0; i < TOTAL_INSERT / 2; i++) {
358 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
359 tbl_rw_test_param.keys + i,
360 (void *)((uintptr_t)i));
361 if (ret < 0) {
362 printf("Failed to insert half of keys\n");
363 goto err_free;
364 }
365 }
366 end = rte_rdtsc_precise() - start;
367 perf_results->single_write = end / i;
368
369 start = rte_rdtsc_precise();
370
371 for (i = 0; i < read_cnt; i++) {
372 void *data;
373 rte_hash_lookup_data(tbl_rw_test_param.h,
374 tbl_rw_test_param.keys + i,
375 &data);
376 if (i != (uint64_t)(uintptr_t)data) {
377 printf("lookup find wrong value"
378 " %"PRIu64",%"PRIu64"\n", i,
379 (uint64_t)(uintptr_t)data);
380 break;
381 }
382 }
383 end = rte_rdtsc_precise() - start;
384 perf_results->single_read = end / i;
385
386 for (n = 0; n < NUM_TEST; n++) {
387 unsigned int tot_lcore = rte_lcore_count();
388 if (tot_lcore < core_cnt[n] * 2 + 1)
389 goto finish;
390
391 rte_atomic64_clear(&greads);
392 rte_atomic64_clear(&gread_cycles);
393 rte_atomic64_clear(&gwrites);
394 rte_atomic64_clear(&gwrite_cycles);
395
396 rte_hash_reset(tbl_rw_test_param.h);
397
398 tbl_rw_test_param.num_insert = TOTAL_INSERT / 2 / core_cnt[n];
399 tbl_rw_test_param.rounded_tot_insert = TOTAL_INSERT / 2 +
400 tbl_rw_test_param.num_insert *
401 core_cnt[n];
402
403 for (i = 0; i < TOTAL_INSERT / 2; i++) {
404 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
405 tbl_rw_test_param.keys + i,
406 (void *)((uintptr_t)i));
407 if (ret < 0) {
408 printf("Failed to insert half of keys\n");
409 goto err_free;
410 }
411 }
412
413 /* Then test multiple thread case but only all reads or
414 * all writes
415 */
416
417 /* Test only reader cases */
418 for (i = 1; i <= core_cnt[n]; i++)
419 rte_eal_remote_launch(test_rw_reader,
420 (void *)(uintptr_t)read_cnt, i);
421
422 rte_eal_mp_wait_lcore();
423
424 start_coreid = i;
425 /* Test only writer cases */
426 for (; i <= core_cnt[n] * 2; i++)
427 rte_eal_remote_launch(test_rw_writer,
428 (void *)((uintptr_t)start_coreid), i);
429
430 rte_eal_mp_wait_lcore();
431
432 if (reader_faster) {
433 unsigned long long int cycles_per_insertion =
434 rte_atomic64_read(&gread_cycles) /
435 rte_atomic64_read(&greads);
436 perf_results->read_only[n] = cycles_per_insertion;
437 printf("Reader only: cycles per lookup: %llu\n",
438 cycles_per_insertion);
439 }
440
441 else {
442 unsigned long long int cycles_per_insertion =
443 rte_atomic64_read(&gwrite_cycles) /
444 rte_atomic64_read(&gwrites);
445 perf_results->write_only[n] = cycles_per_insertion;
446 printf("Writer only: cycles per writes: %llu\n",
447 cycles_per_insertion);
448 }
449
450 rte_atomic64_clear(&greads);
451 rte_atomic64_clear(&gread_cycles);
452 rte_atomic64_clear(&gwrites);
453 rte_atomic64_clear(&gwrite_cycles);
454
455 rte_hash_reset(tbl_rw_test_param.h);
456
457 for (i = 0; i < TOTAL_INSERT / 2; i++) {
458 ret = rte_hash_add_key_data(tbl_rw_test_param.h,
459 tbl_rw_test_param.keys + i,
460 (void *)((uintptr_t)i));
461 if (ret < 0) {
462 printf("Failed to insert half of keys\n");
463 goto err_free;
464 }
465 }
466
467 start_coreid = core_cnt[n] + 1;
468
469 if (reader_faster) {
470 for (i = core_cnt[n] + 1; i <= core_cnt[n] * 2; i++)
471 rte_eal_remote_launch(test_rw_writer,
472 (void *)((uintptr_t)start_coreid), i);
473 for (i = 1; i <= core_cnt[n]; i++)
474 rte_eal_remote_launch(test_rw_reader,
475 (void *)(uintptr_t)read_cnt, i);
476 } else {
477 for (i = 1; i <= core_cnt[n]; i++)
478 rte_eal_remote_launch(test_rw_reader,
479 (void *)(uintptr_t)read_cnt, i);
480 for (; i <= core_cnt[n] * 2; i++)
481 rte_eal_remote_launch(test_rw_writer,
482 (void *)((uintptr_t)start_coreid), i);
483 }
484
485 rte_eal_mp_wait_lcore();
486
487 while (rte_hash_iterate(tbl_rw_test_param.h,
488 &next_key, &next_data, &iter) >= 0) {
489 /* Search for the key in the list of keys added .*/
490 i = *(const uint32_t *)next_key;
491 tbl_rw_test_param.found[i]++;
492 }
493
494 for (i = 0; i < tbl_rw_test_param.rounded_tot_insert; i++) {
495 if (tbl_rw_test_param.keys[i] != RTE_RWTEST_FAIL) {
496 if (tbl_rw_test_param.found[i] > 1) {
497 duplicated_keys++;
498 break;
499 }
500 if (tbl_rw_test_param.found[i] == 0) {
501 lost_keys++;
502 printf("key %"PRIu64" is lost\n", i);
503 break;
504 }
505 }
506 }
507
508 if (duplicated_keys > 0) {
509 printf("%d key duplicated\n", duplicated_keys);
510 goto err_free;
511 }
512
513 if (lost_keys > 0) {
514 printf("%d key lost\n", lost_keys);
515 goto err_free;
516 }
517
518 printf("No key corrupted during read-write test.\n");
519
520 if (reader_faster) {
521 unsigned long long int cycles_per_insertion =
522 rte_atomic64_read(&gread_cycles) /
523 rte_atomic64_read(&greads);
524 perf_results->read_write_r[n] = cycles_per_insertion;
525 printf("Read-write cycles per lookup: %llu\n",
526 cycles_per_insertion);
527 }
528
529 else {
530 unsigned long long int cycles_per_insertion =
531 rte_atomic64_read(&gwrite_cycles) /
532 rte_atomic64_read(&gwrites);
533 perf_results->read_write_w[n] = cycles_per_insertion;
534 printf("Read-write cycles per writes: %llu\n",
535 cycles_per_insertion);
536 }
537 }
538
539 finish:
540 rte_free(tbl_rw_test_param.found);
541 rte_free(tbl_rw_test_param.keys);
542 rte_hash_free(tbl_rw_test_param.h);
543 return 0;
544
545 err_free:
546 rte_free(tbl_rw_test_param.found);
547 rte_free(tbl_rw_test_param.keys);
548 rte_hash_free(tbl_rw_test_param.h);
549
550 err:
551 return -1;
552 }
553
554 static int
555 test_hash_readwrite_main(void)
556 {
557 /*
558 * Variables used to choose different tests.
559 * use_htm indicates if hardware transactional memory should be used.
560 * reader_faster indicates if the reader threads should finish earlier
561 * than writer threads. This is to timing either reader threads or
562 * writer threads for performance numbers.
563 */
564 int use_htm, reader_faster;
565
566 if (rte_lcore_count() == 1) {
567 printf("More than one lcore is required "
568 "to do read write test\n");
569 return 0;
570 }
571
572 setlocale(LC_NUMERIC, "");
573
574 if (rte_tm_supported()) {
575 printf("Hardware transactional memory (lock elision) "
576 "is supported\n");
577
578 printf("Test read-write with Hardware transactional memory\n");
579
580 use_htm = 1;
581 if (test_hash_readwrite_functional(use_htm) < 0)
582 return -1;
583
584 reader_faster = 1;
585 if (test_hash_readwrite_perf(&htm_results, use_htm,
586 reader_faster) < 0)
587 return -1;
588
589 reader_faster = 0;
590 if (test_hash_readwrite_perf(&htm_results, use_htm,
591 reader_faster) < 0)
592 return -1;
593 } else {
594 printf("Hardware transactional memory (lock elision) "
595 "is NOT supported\n");
596 }
597
598 printf("Test read-write without Hardware transactional memory\n");
599 use_htm = 0;
600 if (test_hash_readwrite_functional(use_htm) < 0)
601 return -1;
602 reader_faster = 1;
603 if (test_hash_readwrite_perf(&non_htm_results, use_htm,
604 reader_faster) < 0)
605 return -1;
606 reader_faster = 0;
607 if (test_hash_readwrite_perf(&non_htm_results, use_htm,
608 reader_faster) < 0)
609 return -1;
610
611 printf("Results summary:\n");
612
613 int i;
614
615 printf("single read: %u\n", htm_results.single_read);
616 printf("single write: %u\n", htm_results.single_write);
617 for (i = 0; i < NUM_TEST; i++) {
618 printf("core_cnt: %u\n", core_cnt[i]);
619 printf("HTM:\n");
620 printf("read only: %u\n", htm_results.read_only[i]);
621 printf("write only: %u\n", htm_results.write_only[i]);
622 printf("read-write read: %u\n", htm_results.read_write_r[i]);
623 printf("read-write write: %u\n", htm_results.read_write_w[i]);
624
625 printf("non HTM:\n");
626 printf("read only: %u\n", non_htm_results.read_only[i]);
627 printf("write only: %u\n", non_htm_results.write_only[i]);
628 printf("read-write read: %u\n",
629 non_htm_results.read_write_r[i]);
630 printf("read-write write: %u\n",
631 non_htm_results.read_write_w[i]);
632 }
633
634 return 0;
635 }
636
637 REGISTER_TEST_COMMAND(hash_readwrite_autotest, test_hash_readwrite_main);
Ceph