]> git.proxmox.com Git - ceph.git/blob - ceph/src/spdk/dpdk/examples/ntb/ntb_fwd.c
projects / ceph.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / dpdk / examples / ntb / ntb_fwd.c
1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation
3 */
4 #include <stdint.h>
5 #include <stdio.h>
6 #include <inttypes.h>
7 #include <unistd.h>
8 #include <signal.h>
9 #include <string.h>
10 #include <getopt.h>
11
12 #include <cmdline_parse_string.h>
13 #include <cmdline_socket.h>
14 #include <cmdline.h>
15 #include <rte_common.h>
16 #include <rte_rawdev.h>
17 #include <rte_ethdev.h>
18 #include <rte_malloc.h>
19 #include <rte_lcore.h>
20 #include <rte_cycles.h>
21 #include <rte_pmd_ntb.h>
22 #include <rte_mbuf_pool_ops.h>
23
24 /* Per-port statistics struct */
25 struct ntb_port_statistics {
26 uint64_t tx;
27 uint64_t rx;
28 } __rte_cache_aligned;
29 /* Port 0: NTB dev, Port 1: ethdev when iofwd. */
30 struct ntb_port_statistics ntb_port_stats[2];
31
32 struct ntb_fwd_stream {
33 uint16_t tx_port;
34 uint16_t rx_port;
35 uint16_t qp_id;
36 uint8_t tx_ntb; /* If ntb device is tx port. */
37 };
38
39 struct ntb_fwd_lcore_conf {
40 uint16_t stream_id;
41 uint16_t nb_stream;
42 uint8_t stopped;
43 };
44
45 enum ntb_fwd_mode {
46 FILE_TRANS = 0,
47 RXONLY,
48 TXONLY,
49 IOFWD,
50 MAX_FWD_MODE,
51 };
52 static const char *const fwd_mode_s[] = {
53 "file-trans",
54 "rxonly",
55 "txonly",
56 "iofwd",
57 NULL,
58 };
59 static enum ntb_fwd_mode fwd_mode = MAX_FWD_MODE;
60
61 static struct ntb_fwd_lcore_conf fwd_lcore_conf[RTE_MAX_LCORE];
62 static struct ntb_fwd_stream *fwd_streams;
63
64 static struct rte_mempool *mbuf_pool;
65
66 #define NTB_DRV_NAME_LEN 7
67 #define MEMPOOL_CACHE_SIZE 256
68
69 static uint8_t in_test;
70 static uint8_t interactive = 1;
71 static uint16_t eth_port_id = RTE_MAX_ETHPORTS;
72 static uint16_t dev_id;
73
74 /* Number of queues, default set as 1 */
75 static uint16_t num_queues = 1;
76 static uint16_t ntb_buf_size = RTE_MBUF_DEFAULT_BUF_SIZE;
77
78 /* Configurable number of descriptors */
79 #define NTB_DEFAULT_NUM_DESCS 1024
80 static uint16_t nb_desc = NTB_DEFAULT_NUM_DESCS;
81
82 static uint16_t tx_free_thresh;
83
84 #define NTB_MAX_PKT_BURST 32
85 #define NTB_DFLT_PKT_BURST 32
86 static uint16_t pkt_burst = NTB_DFLT_PKT_BURST;
87
88 #define BURST_TX_RETRIES 64
89
90 static struct rte_eth_conf eth_port_conf = {
91 .rxmode = {
92 .mq_mode = ETH_MQ_RX_RSS,
93 .split_hdr_size = 0,
94 },
95 .rx_adv_conf = {
96 .rss_conf = {
97 .rss_key = NULL,
98 .rss_hf = ETH_RSS_IP,
99 },
100 },
101 .txmode = {
102 .mq_mode = ETH_MQ_TX_NONE,
103 },
104 };
105
106 /* *** Help command with introduction. *** */
107 struct cmd_help_result {
108 cmdline_fixed_string_t help;
109 };
110
111 static void
112 cmd_help_parsed(__rte_unused void *parsed_result,
113 struct cmdline *cl,
114 __rte_unused void *data)
115 {
116 cmdline_printf(
117 cl,
118 "\n"
119 "The following commands are currently available:\n\n"
120 "Control:\n"
121 " quit :"
122 " Quit the application.\n"
123 "\nTransmission:\n"
124 " send [path] :"
125 " Send [path] file. Only take effect in file-trans mode\n"
126 " start :"
127 " Start transmissions.\n"
128 " stop :"
129 " Stop transmissions.\n"
130 " clear/show port stats :"
131 " Clear/show port stats.\n"
132 " set fwd file-trans/rxonly/txonly/iofwd :"
133 " Set packet forwarding mode.\n"
134 );
135
136 }
137
138 cmdline_parse_token_string_t cmd_help_help =
139 TOKEN_STRING_INITIALIZER(struct cmd_help_result, help, "help");
140
141 cmdline_parse_inst_t cmd_help = {
142 .f = cmd_help_parsed,
143 .data = NULL,
144 .help_str = "show help",
145 .tokens = {
146 (void *)&cmd_help_help,
147 NULL,
148 },
149 };
150
151 /* *** QUIT *** */
152 struct cmd_quit_result {
153 cmdline_fixed_string_t quit;
154 };
155
156 static void
157 cmd_quit_parsed(__rte_unused void *parsed_result,
158 struct cmdline *cl,
159 __rte_unused void *data)
160 {
161 struct ntb_fwd_lcore_conf *conf;
162 uint32_t lcore_id;
163
164 /* Stop transmission first. */
165 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
166 conf = &fwd_lcore_conf[lcore_id];
167
168 if (!conf->nb_stream)
169 continue;
170
171 if (conf->stopped)
172 continue;
173
174 conf->stopped = 1;
175 }
176 printf("\nWaiting for lcores to finish...\n");
177 rte_eal_mp_wait_lcore();
178 in_test = 0;
179
180 /* Stop traffic and Close port. */
181 rte_rawdev_stop(dev_id);
182 rte_rawdev_close(dev_id);
183 if (eth_port_id < RTE_MAX_ETHPORTS && fwd_mode == IOFWD) {
184 rte_eth_dev_stop(eth_port_id);
185 rte_eth_dev_close(eth_port_id);
186 }
187
188 cmdline_quit(cl);
189 }
190
191 cmdline_parse_token_string_t cmd_quit_quit =
192 TOKEN_STRING_INITIALIZER(struct cmd_quit_result, quit, "quit");
193
194 cmdline_parse_inst_t cmd_quit = {
195 .f = cmd_quit_parsed,
196 .data = NULL,
197 .help_str = "exit application",
198 .tokens = {
199 (void *)&cmd_quit_quit,
200 NULL,
201 },
202 };
203
204 /* *** SEND FILE PARAMETERS *** */
205 struct cmd_sendfile_result {
206 cmdline_fixed_string_t send_string;
207 char filepath[];
208 };
209
210 static void
211 cmd_sendfile_parsed(void *parsed_result,
212 __rte_unused struct cmdline *cl,
213 __rte_unused void *data)
214 {
215 struct cmd_sendfile_result *res = parsed_result;
216 struct rte_rawdev_buf *pkts_send[NTB_MAX_PKT_BURST];
217 struct rte_mbuf *mbuf_send[NTB_MAX_PKT_BURST];
218 uint64_t size, count, i, j, nb_burst;
219 uint16_t nb_tx, buf_size;
220 unsigned int nb_pkt;
221 size_t queue_id = 0;
222 uint16_t retry = 0;
223 uint32_t val;
224 FILE *file;
225 int ret;
226
227 if (num_queues != 1) {
228 printf("File transmission only supports 1 queue.\n");
229 num_queues = 1;
230 }
231
232 file = fopen(res->filepath, "r");
233 if (file == NULL) {
234 printf("Fail to open the file.\n");
235 return;
236 }
237
238 if (fseek(file, 0, SEEK_END) < 0) {
239 printf("Fail to get file size.\n");
240 fclose(file);
241 return;
242 }
243 size = ftell(file);
244 if (fseek(file, 0, SEEK_SET) < 0) {
245 printf("Fail to get file size.\n");
246 fclose(file);
247 return;
248 }
249
250 /* Tell remote about the file size. */
251 val = size >> 32;
252 rte_rawdev_set_attr(dev_id, "spad_user_0", val);
253 val = size;
254 rte_rawdev_set_attr(dev_id, "spad_user_1", val);
255 printf("Sending file, size is %"PRIu64"\n", size);
256
257 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
258 pkts_send[i] = (struct rte_rawdev_buf *)
259 malloc(sizeof(struct rte_rawdev_buf));
260
261 buf_size = ntb_buf_size - RTE_PKTMBUF_HEADROOM;
262 count = (size + buf_size - 1) / buf_size;
263 nb_burst = (count + pkt_burst - 1) / pkt_burst;
264
265 for (i = 0; i < nb_burst; i++) {
266 val = RTE_MIN(count, pkt_burst);
267 if (rte_mempool_get_bulk(mbuf_pool, (void **)mbuf_send,
268 val) == 0) {
269 for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
270 mbuf_send[nb_pkt]->port = dev_id;
271 mbuf_send[nb_pkt]->data_len =
272 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
273 void *), 1, buf_size, file);
274 mbuf_send[nb_pkt]->pkt_len =
275 mbuf_send[nb_pkt]->data_len;
276 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
277 }
278 } else {
279 for (nb_pkt = 0; nb_pkt < val; nb_pkt++) {
280 mbuf_send[nb_pkt] =
281 rte_mbuf_raw_alloc(mbuf_pool);
282 if (mbuf_send[nb_pkt] == NULL)
283 break;
284 mbuf_send[nb_pkt]->port = dev_id;
285 mbuf_send[nb_pkt]->data_len =
286 fread(rte_pktmbuf_mtod(mbuf_send[nb_pkt],
287 void *), 1, buf_size, file);
288 mbuf_send[nb_pkt]->pkt_len =
289 mbuf_send[nb_pkt]->data_len;
290 pkts_send[nb_pkt]->buf_addr = mbuf_send[nb_pkt];
291 }
292 }
293
294 ret = rte_rawdev_enqueue_buffers(dev_id, pkts_send, nb_pkt,
295 (void *)queue_id);
296 if (ret < 0) {
297 printf("Enqueue failed with err %d\n", ret);
298 for (j = 0; j < nb_pkt; j++)
299 rte_pktmbuf_free(mbuf_send[j]);
300 goto clean;
301 }
302 nb_tx = ret;
303 while (nb_tx != nb_pkt && retry < BURST_TX_RETRIES) {
304 rte_delay_us(1);
305 ret = rte_rawdev_enqueue_buffers(dev_id,
306 &pkts_send[nb_tx], nb_pkt - nb_tx,
307 (void *)queue_id);
308 if (ret < 0) {
309 printf("Enqueue failed with err %d\n", ret);
310 for (j = nb_tx; j < nb_pkt; j++)
311 rte_pktmbuf_free(mbuf_send[j]);
312 goto clean;
313 }
314 nb_tx += ret;
315 }
316 count -= nb_pkt;
317 }
318
319 /* Clear register after file sending done. */
320 rte_rawdev_set_attr(dev_id, "spad_user_0", 0);
321 rte_rawdev_set_attr(dev_id, "spad_user_1", 0);
322 printf("Done sending file.\n");
323
324 clean:
325 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
326 free(pkts_send[i]);
327 fclose(file);
328 }
329
330 cmdline_parse_token_string_t cmd_send_file_send =
331 TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, send_string,
332 "send");
333 cmdline_parse_token_string_t cmd_send_file_filepath =
334 TOKEN_STRING_INITIALIZER(struct cmd_sendfile_result, filepath, NULL);
335
336
337 cmdline_parse_inst_t cmd_send_file = {
338 .f = cmd_sendfile_parsed,
339 .data = NULL,
340 .help_str = "send <file_path>",
341 .tokens = {
342 (void *)&cmd_send_file_send,
343 (void *)&cmd_send_file_filepath,
344 NULL,
345 },
346 };
347
348 #define RECV_FILE_LEN 30
349 static int
350 start_polling_recv_file(void *param)
351 {
352 struct rte_rawdev_buf *pkts_recv[NTB_MAX_PKT_BURST];
353 struct ntb_fwd_lcore_conf *conf = param;
354 struct rte_mbuf *mbuf;
355 char filepath[RECV_FILE_LEN];
356 uint64_t val, size, file_len;
357 uint16_t nb_rx, i, file_no;
358 size_t queue_id = 0;
359 FILE *file;
360 int ret;
361
362 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
363 pkts_recv[i] = (struct rte_rawdev_buf *)
364 malloc(sizeof(struct rte_rawdev_buf));
365
366 file_no = 0;
367 while (!conf->stopped) {
368 snprintf(filepath, RECV_FILE_LEN, "ntb_recv_file%d", file_no);
369 file = fopen(filepath, "w");
370 if (file == NULL) {
371 printf("Fail to open the file.\n");
372 return -EINVAL;
373 }
374
375 rte_rawdev_get_attr(dev_id, "spad_user_0", &val);
376 size = val << 32;
377 rte_rawdev_get_attr(dev_id, "spad_user_1", &val);
378 size |= val;
379
380 if (!size) {
381 fclose(file);
382 continue;
383 }
384
385 file_len = 0;
386 nb_rx = NTB_MAX_PKT_BURST;
387 while (file_len < size && !conf->stopped) {
388 ret = rte_rawdev_dequeue_buffers(dev_id, pkts_recv,
389 pkt_burst, (void *)queue_id);
390 if (ret < 0) {
391 printf("Dequeue failed with err %d\n", ret);
392 fclose(file);
393 goto clean;
394 }
395 nb_rx = ret;
396 ntb_port_stats[0].rx += nb_rx;
397 for (i = 0; i < nb_rx; i++) {
398 mbuf = pkts_recv[i]->buf_addr;
399 fwrite(rte_pktmbuf_mtod(mbuf, void *), 1,
400 mbuf->data_len, file);
401 file_len += mbuf->data_len;
402 rte_pktmbuf_free(mbuf);
403 pkts_recv[i]->buf_addr = NULL;
404 }
405 }
406
407 printf("Received file (size: %" PRIu64 ") from peer to %s.\n",
408 size, filepath);
409 fclose(file);
410 file_no++;
411 }
412
413 clean:
414 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
415 free(pkts_recv[i]);
416 return 0;
417 }
418
419 static int
420 start_iofwd_per_lcore(void *param)
421 {
422 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
423 struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
424 struct ntb_fwd_lcore_conf *conf = param;
425 struct ntb_fwd_stream fs;
426 uint16_t nb_rx, nb_tx;
427 int i, j, ret;
428
429 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
430 ntb_buf[i] = (struct rte_rawdev_buf *)
431 malloc(sizeof(struct rte_rawdev_buf));
432
433 while (!conf->stopped) {
434 for (i = 0; i < conf->nb_stream; i++) {
435 fs = fwd_streams[conf->stream_id + i];
436 if (fs.tx_ntb) {
437 nb_rx = rte_eth_rx_burst(fs.rx_port,
438 fs.qp_id, pkts_burst,
439 pkt_burst);
440 if (unlikely(nb_rx == 0))
441 continue;
442 for (j = 0; j < nb_rx; j++)
443 ntb_buf[j]->buf_addr = pkts_burst[j];
444 ret = rte_rawdev_enqueue_buffers(fs.tx_port,
445 ntb_buf, nb_rx,
446 (void *)(size_t)fs.qp_id);
447 if (ret < 0) {
448 printf("Enqueue failed with err %d\n",
449 ret);
450 for (j = 0; j < nb_rx; j++)
451 rte_pktmbuf_free(pkts_burst[j]);
452 goto clean;
453 }
454 nb_tx = ret;
455 ntb_port_stats[0].tx += nb_tx;
456 ntb_port_stats[1].rx += nb_rx;
457 } else {
458 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
459 ntb_buf, pkt_burst,
460 (void *)(size_t)fs.qp_id);
461 if (ret < 0) {
462 printf("Dequeue failed with err %d\n",
463 ret);
464 goto clean;
465 }
466 nb_rx = ret;
467 if (unlikely(nb_rx == 0))
468 continue;
469 for (j = 0; j < nb_rx; j++)
470 pkts_burst[j] = ntb_buf[j]->buf_addr;
471 nb_tx = rte_eth_tx_burst(fs.tx_port,
472 fs.qp_id, pkts_burst, nb_rx);
473 ntb_port_stats[1].tx += nb_tx;
474 ntb_port_stats[0].rx += nb_rx;
475 }
476 if (unlikely(nb_tx < nb_rx)) {
477 do {
478 rte_pktmbuf_free(pkts_burst[nb_tx]);
479 } while (++nb_tx < nb_rx);
480 }
481 }
482 }
483
484 clean:
485 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
486 free(ntb_buf[i]);
487
488 return 0;
489 }
490
491 static int
492 start_rxonly_per_lcore(void *param)
493 {
494 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
495 struct ntb_fwd_lcore_conf *conf = param;
496 struct ntb_fwd_stream fs;
497 uint16_t nb_rx;
498 int i, j, ret;
499
500 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
501 ntb_buf[i] = (struct rte_rawdev_buf *)
502 malloc(sizeof(struct rte_rawdev_buf));
503
504 while (!conf->stopped) {
505 for (i = 0; i < conf->nb_stream; i++) {
506 fs = fwd_streams[conf->stream_id + i];
507 ret = rte_rawdev_dequeue_buffers(fs.rx_port,
508 ntb_buf, pkt_burst, (void *)(size_t)fs.qp_id);
509 if (ret < 0) {
510 printf("Dequeue failed with err %d\n", ret);
511 goto clean;
512 }
513 nb_rx = ret;
514 if (unlikely(nb_rx == 0))
515 continue;
516 ntb_port_stats[0].rx += nb_rx;
517
518 for (j = 0; j < nb_rx; j++)
519 rte_pktmbuf_free(ntb_buf[j]->buf_addr);
520 }
521 }
522
523 clean:
524 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
525 free(ntb_buf[i]);
526
527 return 0;
528 }
529
530
531 static int
532 start_txonly_per_lcore(void *param)
533 {
534 struct rte_rawdev_buf *ntb_buf[NTB_MAX_PKT_BURST];
535 struct rte_mbuf *pkts_burst[NTB_MAX_PKT_BURST];
536 struct ntb_fwd_lcore_conf *conf = param;
537 struct ntb_fwd_stream fs;
538 uint16_t nb_pkt, nb_tx;
539 int i, j, ret;
540
541 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
542 ntb_buf[i] = (struct rte_rawdev_buf *)
543 malloc(sizeof(struct rte_rawdev_buf));
544
545 while (!conf->stopped) {
546 for (i = 0; i < conf->nb_stream; i++) {
547 fs = fwd_streams[conf->stream_id + i];
548 if (rte_mempool_get_bulk(mbuf_pool, (void **)pkts_burst,
549 pkt_burst) == 0) {
550 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
551 pkts_burst[nb_pkt]->port = dev_id;
552 pkts_burst[nb_pkt]->data_len =
553 pkts_burst[nb_pkt]->buf_len -
554 RTE_PKTMBUF_HEADROOM;
555 pkts_burst[nb_pkt]->pkt_len =
556 pkts_burst[nb_pkt]->data_len;
557 ntb_buf[nb_pkt]->buf_addr =
558 pkts_burst[nb_pkt];
559 }
560 } else {
561 for (nb_pkt = 0; nb_pkt < pkt_burst; nb_pkt++) {
562 pkts_burst[nb_pkt] =
563 rte_pktmbuf_alloc(mbuf_pool);
564 if (pkts_burst[nb_pkt] == NULL)
565 break;
566 pkts_burst[nb_pkt]->port = dev_id;
567 pkts_burst[nb_pkt]->data_len =
568 pkts_burst[nb_pkt]->buf_len -
569 RTE_PKTMBUF_HEADROOM;
570 pkts_burst[nb_pkt]->pkt_len =
571 pkts_burst[nb_pkt]->data_len;
572 ntb_buf[nb_pkt]->buf_addr =
573 pkts_burst[nb_pkt];
574 }
575 }
576 ret = rte_rawdev_enqueue_buffers(fs.tx_port, ntb_buf,
577 nb_pkt, (void *)(size_t)fs.qp_id);
578 if (ret < 0) {
579 printf("Enqueue failed with err %d\n", ret);
580 for (j = 0; j < nb_pkt; j++)
581 rte_pktmbuf_free(pkts_burst[j]);
582 goto clean;
583 }
584 nb_tx = ret;
585 ntb_port_stats[0].tx += nb_tx;
586 if (unlikely(nb_tx < nb_pkt)) {
587 do {
588 rte_pktmbuf_free(pkts_burst[nb_tx]);
589 } while (++nb_tx < nb_pkt);
590 }
591 }
592 }
593
594 clean:
595 for (i = 0; i < NTB_MAX_PKT_BURST; i++)
596 free(ntb_buf[i]);
597
598 return 0;
599 }
600
601 static int
602 ntb_fwd_config_setup(void)
603 {
604 uint16_t i;
605
606 /* Make sure iofwd has valid ethdev. */
607 if (fwd_mode == IOFWD && eth_port_id >= RTE_MAX_ETHPORTS) {
608 printf("No ethdev, cannot be in iofwd mode.");
609 return -EINVAL;
610 }
611
612 if (fwd_mode == IOFWD) {
613 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
614 sizeof(struct ntb_fwd_stream) * num_queues * 2,
615 RTE_CACHE_LINE_SIZE);
616 for (i = 0; i < num_queues; i++) {
617 fwd_streams[i * 2].qp_id = i;
618 fwd_streams[i * 2].tx_port = dev_id;
619 fwd_streams[i * 2].rx_port = eth_port_id;
620 fwd_streams[i * 2].tx_ntb = 1;
621
622 fwd_streams[i * 2 + 1].qp_id = i;
623 fwd_streams[i * 2 + 1].tx_port = eth_port_id;
624 fwd_streams[i * 2 + 1].rx_port = dev_id;
625 fwd_streams[i * 2 + 1].tx_ntb = 0;
626 }
627 return 0;
628 }
629
630 if (fwd_mode == RXONLY || fwd_mode == FILE_TRANS) {
631 /* Only support 1 queue in file-trans for in order. */
632 if (fwd_mode == FILE_TRANS)
633 num_queues = 1;
634
635 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
636 sizeof(struct ntb_fwd_stream) * num_queues,
637 RTE_CACHE_LINE_SIZE);
638 for (i = 0; i < num_queues; i++) {
639 fwd_streams[i].qp_id = i;
640 fwd_streams[i].tx_port = RTE_MAX_ETHPORTS;
641 fwd_streams[i].rx_port = dev_id;
642 fwd_streams[i].tx_ntb = 0;
643 }
644 return 0;
645 }
646
647 if (fwd_mode == TXONLY) {
648 fwd_streams = rte_zmalloc("ntb_fwd: fwd_streams",
649 sizeof(struct ntb_fwd_stream) * num_queues,
650 RTE_CACHE_LINE_SIZE);
651 for (i = 0; i < num_queues; i++) {
652 fwd_streams[i].qp_id = i;
653 fwd_streams[i].tx_port = dev_id;
654 fwd_streams[i].rx_port = RTE_MAX_ETHPORTS;
655 fwd_streams[i].tx_ntb = 1;
656 }
657 }
658 return 0;
659 }
660
661 static void
662 assign_stream_to_lcores(void)
663 {
664 struct ntb_fwd_lcore_conf *conf;
665 struct ntb_fwd_stream *fs;
666 uint16_t nb_streams, sm_per_lcore, sm_id, i;
667 uint32_t lcore_id;
668 uint8_t lcore_num, nb_extra;
669
670 lcore_num = rte_lcore_count();
671 /* Exclude master core */
672 lcore_num--;
673
674 nb_streams = (fwd_mode == IOFWD) ? num_queues * 2 : num_queues;
675
676 sm_per_lcore = nb_streams / lcore_num;
677 nb_extra = nb_streams % lcore_num;
678 sm_id = 0;
679 i = 0;
680
681 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
682 conf = &fwd_lcore_conf[lcore_id];
683
684 if (i < nb_extra) {
685 conf->nb_stream = sm_per_lcore + 1;
686 conf->stream_id = sm_id;
687 sm_id = sm_id + sm_per_lcore + 1;
688 } else {
689 conf->nb_stream = sm_per_lcore;
690 conf->stream_id = sm_id;
691 sm_id = sm_id + sm_per_lcore;
692 }
693
694 i++;
695 if (sm_id >= nb_streams)
696 break;
697 }
698
699 /* Print packet forwading config. */
700 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
701 conf = &fwd_lcore_conf[lcore_id];
702
703 if (!conf->nb_stream)
704 continue;
705
706 printf("Streams on Lcore %u :\n", lcore_id);
707 for (i = 0; i < conf->nb_stream; i++) {
708 fs = &fwd_streams[conf->stream_id + i];
709 if (fwd_mode == IOFWD)
710 printf(" + Stream %u : %s%u RX -> %s%u TX,"
711 " Q=%u\n", conf->stream_id + i,
712 fs->tx_ntb ? "Eth" : "NTB", fs->rx_port,
713 fs->tx_ntb ? "NTB" : "Eth", fs->tx_port,
714 fs->qp_id);
715 if (fwd_mode == FILE_TRANS || fwd_mode == RXONLY)
716 printf(" + Stream %u : %s%u RX only\n",
717 conf->stream_id, "NTB", fs->rx_port);
718 if (fwd_mode == TXONLY)
719 printf(" + Stream %u : %s%u TX only\n",
720 conf->stream_id, "NTB", fs->tx_port);
721 }
722 }
723 }
724
725 static void
726 start_pkt_fwd(void)
727 {
728 struct ntb_fwd_lcore_conf *conf;
729 struct rte_eth_link eth_link;
730 uint32_t lcore_id;
731 int ret, i;
732
733 ret = ntb_fwd_config_setup();
734 if (ret < 0) {
735 printf("Cannot start traffic. Please reset fwd mode.\n");
736 return;
737 }
738
739 /* If using iofwd, checking ethdev link status first. */
740 if (fwd_mode == IOFWD) {
741 printf("Checking eth link status...\n");
742 /* Wait for eth link up at most 100 times. */
743 for (i = 0; i < 100; i++) {
744 ret = rte_eth_link_get(eth_port_id, &eth_link);
745 if (ret < 0) {
746 printf("Link get failed with err %d\n", ret);
747 return;
748 }
749 if (eth_link.link_status) {
750 printf("Eth%u Link Up. Speed %u Mbps - %s\n",
751 eth_port_id, eth_link.link_speed,
752 (eth_link.link_duplex ==
753 ETH_LINK_FULL_DUPLEX) ?
754 ("full-duplex") : ("half-duplex"));
755 break;
756 }
757 }
758 if (!eth_link.link_status) {
759 printf("Eth%u link down. Cannot start traffic.\n",
760 eth_port_id);
761 return;
762 }
763 }
764
765 assign_stream_to_lcores();
766 in_test = 1;
767
768 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
769 conf = &fwd_lcore_conf[lcore_id];
770
771 if (!conf->nb_stream)
772 continue;
773
774 conf->stopped = 0;
775 if (fwd_mode == FILE_TRANS)
776 rte_eal_remote_launch(start_polling_recv_file,
777 conf, lcore_id);
778 else if (fwd_mode == IOFWD)
779 rte_eal_remote_launch(start_iofwd_per_lcore,
780 conf, lcore_id);
781 else if (fwd_mode == RXONLY)
782 rte_eal_remote_launch(start_rxonly_per_lcore,
783 conf, lcore_id);
784 else if (fwd_mode == TXONLY)
785 rte_eal_remote_launch(start_txonly_per_lcore,
786 conf, lcore_id);
787 }
788 }
789
790 /* *** START FWD PARAMETERS *** */
791 struct cmd_start_result {
792 cmdline_fixed_string_t start;
793 };
794
795 static void
796 cmd_start_parsed(__rte_unused void *parsed_result,
797 __rte_unused struct cmdline *cl,
798 __rte_unused void *data)
799 {
800 start_pkt_fwd();
801 }
802
803 cmdline_parse_token_string_t cmd_start_start =
804 TOKEN_STRING_INITIALIZER(struct cmd_start_result, start, "start");
805
806 cmdline_parse_inst_t cmd_start = {
807 .f = cmd_start_parsed,
808 .data = NULL,
809 .help_str = "start pkt fwd between ntb and ethdev",
810 .tokens = {
811 (void *)&cmd_start_start,
812 NULL,
813 },
814 };
815
816 /* *** STOP *** */
817 struct cmd_stop_result {
818 cmdline_fixed_string_t stop;
819 };
820
821 static void
822 cmd_stop_parsed(__rte_unused void *parsed_result,
823 __rte_unused struct cmdline *cl,
824 __rte_unused void *data)
825 {
826 struct ntb_fwd_lcore_conf *conf;
827 uint32_t lcore_id;
828
829 RTE_LCORE_FOREACH_SLAVE(lcore_id) {
830 conf = &fwd_lcore_conf[lcore_id];
831
832 if (!conf->nb_stream)
833 continue;
834
835 if (conf->stopped)
836 continue;
837
838 conf->stopped = 1;
839 }
840 printf("\nWaiting for lcores to finish...\n");
841 rte_eal_mp_wait_lcore();
842 in_test = 0;
843 printf("\nDone.\n");
844 }
845
846 cmdline_parse_token_string_t cmd_stop_stop =
847 TOKEN_STRING_INITIALIZER(struct cmd_stop_result, stop, "stop");
848
849 cmdline_parse_inst_t cmd_stop = {
850 .f = cmd_stop_parsed,
851 .data = NULL,
852 .help_str = "stop: Stop packet forwarding",
853 .tokens = {
854 (void *)&cmd_stop_stop,
855 NULL,
856 },
857 };
858
859 static void
860 ntb_stats_clear(void)
861 {
862 int nb_ids, i;
863 uint32_t *ids;
864
865 /* Clear NTB dev stats */
866 nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
867 if (nb_ids < 0) {
868 printf("Error: Cannot get count of xstats\n");
869 return;
870 }
871 ids = malloc(sizeof(uint32_t) * nb_ids);
872 for (i = 0; i < nb_ids; i++)
873 ids[i] = i;
874 rte_rawdev_xstats_reset(dev_id, ids, nb_ids);
875 printf("\n statistics for NTB port %d cleared\n", dev_id);
876
877 /* Clear Ethdev stats if have any */
878 if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
879 rte_eth_stats_reset(eth_port_id);
880 printf("\n statistics for ETH port %d cleared\n", eth_port_id);
881 }
882 }
883
884 static inline void
885 ntb_calculate_throughput(uint16_t port) {
886 uint64_t diff_pkts_rx, diff_pkts_tx, diff_cycles;
887 uint64_t mpps_rx, mpps_tx;
888 static uint64_t prev_pkts_rx[2];
889 static uint64_t prev_pkts_tx[2];
890 static uint64_t prev_cycles[2];
891
892 diff_cycles = prev_cycles[port];
893 prev_cycles[port] = rte_rdtsc();
894 if (diff_cycles > 0)
895 diff_cycles = prev_cycles[port] - diff_cycles;
896 diff_pkts_rx = (ntb_port_stats[port].rx > prev_pkts_rx[port]) ?
897 (ntb_port_stats[port].rx - prev_pkts_rx[port]) : 0;
898 diff_pkts_tx = (ntb_port_stats[port].tx > prev_pkts_tx[port]) ?
899 (ntb_port_stats[port].tx - prev_pkts_tx[port]) : 0;
900 prev_pkts_rx[port] = ntb_port_stats[port].rx;
901 prev_pkts_tx[port] = ntb_port_stats[port].tx;
902 mpps_rx = diff_cycles > 0 ?
903 diff_pkts_rx * rte_get_tsc_hz() / diff_cycles : 0;
904 mpps_tx = diff_cycles > 0 ?
905 diff_pkts_tx * rte_get_tsc_hz() / diff_cycles : 0;
906 printf(" Throughput (since last show)\n");
907 printf(" Rx-pps: %12"PRIu64"\n Tx-pps: %12"PRIu64"\n",
908 mpps_rx, mpps_tx);
909
910 }
911
912 static void
913 ntb_stats_display(void)
914 {
915 struct rte_rawdev_xstats_name *xstats_names;
916 struct rte_eth_stats stats;
917 uint64_t *values;
918 uint32_t *ids;
919 int nb_ids, i;
920
921 printf("###### statistics for NTB port %d #######\n", dev_id);
922
923 /* Get NTB dev stats and stats names */
924 nb_ids = rte_rawdev_xstats_names_get(dev_id, NULL, 0);
925 if (nb_ids < 0) {
926 printf("Error: Cannot get count of xstats\n");
927 return;
928 }
929 xstats_names = malloc(sizeof(struct rte_rawdev_xstats_name) * nb_ids);
930 if (xstats_names == NULL) {
931 printf("Cannot allocate memory for xstats lookup\n");
932 return;
933 }
934 if (nb_ids != rte_rawdev_xstats_names_get(
935 dev_id, xstats_names, nb_ids)) {
936 printf("Error: Cannot get xstats lookup\n");
937 free(xstats_names);
938 return;
939 }
940 ids = malloc(sizeof(uint32_t) * nb_ids);
941 for (i = 0; i < nb_ids; i++)
942 ids[i] = i;
943 values = malloc(sizeof(uint64_t) * nb_ids);
944 if (nb_ids != rte_rawdev_xstats_get(dev_id, ids, values, nb_ids)) {
945 printf("Error: Unable to get xstats\n");
946 free(xstats_names);
947 free(values);
948 free(ids);
949 return;
950 }
951
952 /* Display NTB dev stats */
953 for (i = 0; i < nb_ids; i++)
954 printf(" %s: %"PRIu64"\n", xstats_names[i].name, values[i]);
955 ntb_calculate_throughput(0);
956
957 /* Get Ethdev stats if have any */
958 if (fwd_mode == IOFWD && eth_port_id != RTE_MAX_ETHPORTS) {
959 printf("###### statistics for ETH port %d ######\n",
960 eth_port_id);
961 rte_eth_stats_get(eth_port_id, &stats);
962 printf(" RX-packets: %"PRIu64"\n", stats.ipackets);
963 printf(" RX-bytes: %"PRIu64"\n", stats.ibytes);
964 printf(" RX-errors: %"PRIu64"\n", stats.ierrors);
965 printf(" RX-missed: %"PRIu64"\n", stats.imissed);
966 printf(" TX-packets: %"PRIu64"\n", stats.opackets);
967 printf(" TX-bytes: %"PRIu64"\n", stats.obytes);
968 printf(" TX-errors: %"PRIu64"\n", stats.oerrors);
969 ntb_calculate_throughput(1);
970 }
971
972 free(xstats_names);
973 free(values);
974 free(ids);
975 }
976
977 /* *** SHOW/CLEAR PORT STATS *** */
978 struct cmd_stats_result {
979 cmdline_fixed_string_t show;
980 cmdline_fixed_string_t port;
981 cmdline_fixed_string_t stats;
982 };
983
984 static void
985 cmd_stats_parsed(void *parsed_result,
986 __rte_unused struct cmdline *cl,
987 __rte_unused void *data)
988 {
989 struct cmd_stats_result *res = parsed_result;
990 if (!strcmp(res->show, "clear"))
991 ntb_stats_clear();
992 else
993 ntb_stats_display();
994 }
995
996 cmdline_parse_token_string_t cmd_stats_show =
997 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, show, "show#clear");
998 cmdline_parse_token_string_t cmd_stats_port =
999 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, port, "port");
1000 cmdline_parse_token_string_t cmd_stats_stats =
1001 TOKEN_STRING_INITIALIZER(struct cmd_stats_result, stats, "stats");
1002
1003
1004 cmdline_parse_inst_t cmd_stats = {
1005 .f = cmd_stats_parsed,
1006 .data = NULL,
1007 .help_str = "show|clear port stats",
1008 .tokens = {
1009 (void *)&cmd_stats_show,
1010 (void *)&cmd_stats_port,
1011 (void *)&cmd_stats_stats,
1012 NULL,
1013 },
1014 };
1015
1016 /* *** SET FORWARDING MODE *** */
1017 struct cmd_set_fwd_mode_result {
1018 cmdline_fixed_string_t set;
1019 cmdline_fixed_string_t fwd;
1020 cmdline_fixed_string_t mode;
1021 };
1022
1023 static void
1024 cmd_set_fwd_mode_parsed(__rte_unused void *parsed_result,
1025 __rte_unused struct cmdline *cl,
1026 __rte_unused void *data)
1027 {
1028 struct cmd_set_fwd_mode_result *res = parsed_result;
1029 int i;
1030
1031 if (in_test) {
1032 printf("Please stop traffic first.\n");
1033 return;
1034 }
1035
1036 for (i = 0; i < MAX_FWD_MODE; i++) {
1037 if (!strcmp(res->mode, fwd_mode_s[i])) {
1038 fwd_mode = i;
1039 return;
1040 }
1041 }
1042 printf("Invalid %s packet forwarding mode.\n", res->mode);
1043 }
1044
1045 cmdline_parse_token_string_t cmd_setfwd_set =
1046 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, set, "set");
1047 cmdline_parse_token_string_t cmd_setfwd_fwd =
1048 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, fwd, "fwd");
1049 cmdline_parse_token_string_t cmd_setfwd_mode =
1050 TOKEN_STRING_INITIALIZER(struct cmd_set_fwd_mode_result, mode,
1051 "file-trans#iofwd#txonly#rxonly");
1052
1053 cmdline_parse_inst_t cmd_set_fwd_mode = {
1054 .f = cmd_set_fwd_mode_parsed,
1055 .data = NULL,
1056 .help_str = "set forwarding mode as file-trans|rxonly|txonly|iofwd",
1057 .tokens = {
1058 (void *)&cmd_setfwd_set,
1059 (void *)&cmd_setfwd_fwd,
1060 (void *)&cmd_setfwd_mode,
1061 NULL,
1062 },
1063 };
1064
1065 /* list of instructions */
1066 cmdline_parse_ctx_t main_ctx[] = {
1067 (cmdline_parse_inst_t *)&cmd_help,
1068 (cmdline_parse_inst_t *)&cmd_send_file,
1069 (cmdline_parse_inst_t *)&cmd_start,
1070 (cmdline_parse_inst_t *)&cmd_stop,
1071 (cmdline_parse_inst_t *)&cmd_stats,
1072 (cmdline_parse_inst_t *)&cmd_set_fwd_mode,
1073 (cmdline_parse_inst_t *)&cmd_quit,
1074 NULL,
1075 };
1076
1077 /* prompt function, called from main on MASTER lcore */
1078 static void
1079 prompt(void)
1080 {
1081 struct cmdline *cl;
1082
1083 cl = cmdline_stdin_new(main_ctx, "ntb> ");
1084 if (cl == NULL)
1085 return;
1086
1087 cmdline_interact(cl);
1088 cmdline_stdin_exit(cl);
1089 }
1090
1091 static void
1092 signal_handler(int signum)
1093 {
1094 if (signum == SIGINT || signum == SIGTERM) {
1095 printf("\nSignal %d received, preparing to exit...\n", signum);
1096 signal(signum, SIG_DFL);
1097 kill(getpid(), signum);
1098 }
1099 }
1100
1101 #define OPT_BUF_SIZE "buf-size"
1102 #define OPT_FWD_MODE "fwd-mode"
1103 #define OPT_NB_DESC "nb-desc"
1104 #define OPT_TXFREET "txfreet"
1105 #define OPT_BURST "burst"
1106 #define OPT_QP "qp"
1107
1108 enum {
1109 /* long options mapped to a short option */
1110 OPT_NO_ZERO_COPY_NUM = 1,
1111 OPT_BUF_SIZE_NUM,
1112 OPT_FWD_MODE_NUM,
1113 OPT_NB_DESC_NUM,
1114 OPT_TXFREET_NUM,
1115 OPT_BURST_NUM,
1116 OPT_QP_NUM,
1117 };
1118
1119 static const char short_options[] =
1120 "i" /* interactive mode */
1121 ;
1122
1123 static const struct option lgopts[] = {
1124 {OPT_BUF_SIZE, 1, NULL, OPT_BUF_SIZE_NUM },
1125 {OPT_FWD_MODE, 1, NULL, OPT_FWD_MODE_NUM },
1126 {OPT_NB_DESC, 1, NULL, OPT_NB_DESC_NUM },
1127 {OPT_TXFREET, 1, NULL, OPT_TXFREET_NUM },
1128 {OPT_BURST, 1, NULL, OPT_BURST_NUM },
1129 {OPT_QP, 1, NULL, OPT_QP_NUM },
1130 {0, 0, NULL, 0 }
1131 };
1132
1133 static void
1134 ntb_usage(const char *prgname)
1135 {
1136 printf("%s [EAL options] -- [options]\n"
1137 "-i: run in interactive mode.\n"
1138 "-qp=N: set number of queues as N (N > 0, default: 1).\n"
1139 "--fwd-mode=N: set fwd mode (N: file-trans | rxonly | "
1140 "txonly | iofwd, default: file-trans)\n"
1141 "--buf-size=N: set mbuf dataroom size as N (0 < N < 65535,"
1142 " default: 2048).\n"
1143 "--nb-desc=N: set number of descriptors as N (%u <= N <= %u,"
1144 " default: 1024).\n"
1145 "--txfreet=N: set tx free thresh for NTB driver as N. (N >= 0)\n"
1146 "--burst=N: set pkt burst as N (0 < N <= %u default: 32).\n",
1147 prgname, NTB_MIN_DESC_SIZE, NTB_MAX_DESC_SIZE,
1148 NTB_MAX_PKT_BURST);
1149 }
1150
1151 static void
1152 ntb_parse_args(int argc, char **argv)
1153 {
1154 char *prgname = argv[0], **argvopt = argv;
1155 int opt, opt_idx, n, i;
1156
1157 while ((opt = getopt_long(argc, argvopt, short_options,
1158 lgopts, &opt_idx)) != EOF) {
1159 switch (opt) {
1160 case 'i':
1161 printf("Interactive-mode selected.\n");
1162 interactive = 1;
1163 break;
1164 case OPT_QP_NUM:
1165 n = atoi(optarg);
1166 if (n > 0)
1167 num_queues = n;
1168 else
1169 rte_exit(EXIT_FAILURE, "q must be > 0.\n");
1170 break;
1171 case OPT_BUF_SIZE_NUM:
1172 n = atoi(optarg);
1173 if (n > RTE_PKTMBUF_HEADROOM && n <= 0xFFFF)
1174 ntb_buf_size = n;
1175 else
1176 rte_exit(EXIT_FAILURE, "buf-size must be > "
1177 "%u and < 65536.\n",
1178 RTE_PKTMBUF_HEADROOM);
1179 break;
1180 case OPT_FWD_MODE_NUM:
1181 for (i = 0; i < MAX_FWD_MODE; i++) {
1182 if (!strcmp(optarg, fwd_mode_s[i])) {
1183 fwd_mode = i;
1184 break;
1185 }
1186 }
1187 if (i == MAX_FWD_MODE)
1188 rte_exit(EXIT_FAILURE, "Unsupported mode. "
1189 "(Should be: file-trans | rxonly | txonly "
1190 "| iofwd)\n");
1191 break;
1192 case OPT_NB_DESC_NUM:
1193 n = atoi(optarg);
1194 if (n >= NTB_MIN_DESC_SIZE && n <= NTB_MAX_DESC_SIZE)
1195 nb_desc = n;
1196 else
1197 rte_exit(EXIT_FAILURE, "nb-desc must be within"
1198 " [%u, %u].\n", NTB_MIN_DESC_SIZE,
1199 NTB_MAX_DESC_SIZE);
1200 break;
1201 case OPT_TXFREET_NUM:
1202 n = atoi(optarg);
1203 if (n >= 0)
1204 tx_free_thresh = n;
1205 else
1206 rte_exit(EXIT_FAILURE, "txfreet must be"
1207 " >= 0\n");
1208 break;
1209 case OPT_BURST_NUM:
1210 n = atoi(optarg);
1211 if (n > 0 && n <= NTB_MAX_PKT_BURST)
1212 pkt_burst = n;
1213 else
1214 rte_exit(EXIT_FAILURE, "burst must be within "
1215 "(0, %u].\n", NTB_MAX_PKT_BURST);
1216 break;
1217
1218 default:
1219 ntb_usage(prgname);
1220 rte_exit(EXIT_FAILURE,
1221 "Command line is incomplete or incorrect.\n");
1222 break;
1223 }
1224 }
1225 }
1226
1227 static void
1228 ntb_mempool_mz_free(__rte_unused struct rte_mempool_memhdr *memhdr,
1229 void *opaque)
1230 {
1231 const struct rte_memzone *mz = opaque;
1232 rte_memzone_free(mz);
1233 }
1234
1235 static struct rte_mempool *
1236 ntb_mbuf_pool_create(uint16_t mbuf_seg_size, uint32_t nb_mbuf,
1237 struct ntb_dev_info ntb_info,
1238 struct ntb_dev_config *ntb_conf,
1239 unsigned int socket_id)
1240 {
1241 size_t mz_len, total_elt_sz, max_mz_len, left_sz;
1242 struct rte_pktmbuf_pool_private mbp_priv;
1243 char pool_name[RTE_MEMPOOL_NAMESIZE];
1244 char mz_name[RTE_MEMZONE_NAMESIZE];
1245 const struct rte_memzone *mz;
1246 struct rte_mempool *mp;
1247 uint64_t align;
1248 uint32_t mz_id;
1249 int ret;
1250
1251 snprintf(pool_name, sizeof(pool_name), "ntb_mbuf_pool_%u", socket_id);
1252 mp = rte_mempool_create_empty(pool_name, nb_mbuf,
1253 (mbuf_seg_size + sizeof(struct rte_mbuf)),
1254 MEMPOOL_CACHE_SIZE,
1255 sizeof(struct rte_pktmbuf_pool_private),
1256 socket_id, 0);
1257 if (mp == NULL)
1258 return NULL;
1259
1260 if (rte_mempool_set_ops_byname(mp, rte_mbuf_best_mempool_ops(), NULL)) {
1261 printf("error setting mempool handler\n");
1262 goto fail;
1263 }
1264
1265 memset(&mbp_priv, 0, sizeof(mbp_priv));
1266 mbp_priv.mbuf_data_room_size = mbuf_seg_size;
1267 mbp_priv.mbuf_priv_size = 0;
1268 rte_pktmbuf_pool_init(mp, &mbp_priv);
1269
1270 ntb_conf->mz_list = rte_zmalloc("ntb_memzone_list",
1271 sizeof(struct rte_memzone *) *
1272 ntb_info.mw_cnt, 0);
1273 if (ntb_conf->mz_list == NULL)
1274 goto fail;
1275
1276 /* Put ntb header on mw0. */
1277 if (ntb_info.mw_size[0] < ntb_info.ntb_hdr_size) {
1278 printf("mw0 (size: %" PRIu64 ") is not enough for ntb hdr"
1279 " (size: %u)\n", ntb_info.mw_size[0],
1280 ntb_info.ntb_hdr_size);
1281 goto fail;
1282 }
1283
1284 total_elt_sz = mp->header_size + mp->elt_size + mp->trailer_size;
1285 left_sz = total_elt_sz * nb_mbuf;
1286 for (mz_id = 0; mz_id < ntb_info.mw_cnt; mz_id++) {
1287 /* If populated mbuf is enough, no need to reserve extra mz. */
1288 if (!left_sz)
1289 break;
1290 snprintf(mz_name, sizeof(mz_name), "ntb_mw_%d", mz_id);
1291 align = ntb_info.mw_size_align ? ntb_info.mw_size[mz_id] :
1292 RTE_CACHE_LINE_SIZE;
1293 /* Reserve ntb header space on memzone 0. */
1294 max_mz_len = mz_id ? ntb_info.mw_size[mz_id] :
1295 ntb_info.mw_size[mz_id] - ntb_info.ntb_hdr_size;
1296 mz_len = left_sz <= max_mz_len ? left_sz :
1297 (max_mz_len / total_elt_sz * total_elt_sz);
1298 if (!mz_len)
1299 continue;
1300 mz = rte_memzone_reserve_aligned(mz_name, mz_len, socket_id,
1301 RTE_MEMZONE_IOVA_CONTIG, align);
1302 if (mz == NULL) {
1303 printf("Cannot allocate %" PRIu64 " aligned memzone"
1304 " %u\n", align, mz_id);
1305 goto fail;
1306 }
1307 left_sz -= mz_len;
1308
1309 /* Reserve ntb header space on memzone 0. */
1310 if (mz_id)
1311 ret = rte_mempool_populate_iova(mp, mz->addr, mz->iova,
1312 mz->len, ntb_mempool_mz_free,
1313 (void *)(uintptr_t)mz);
1314 else
1315 ret = rte_mempool_populate_iova(mp,
1316 (void *)((size_t)mz->addr +
1317 ntb_info.ntb_hdr_size),
1318 mz->iova + ntb_info.ntb_hdr_size,
1319 mz->len - ntb_info.ntb_hdr_size,
1320 ntb_mempool_mz_free,
1321 (void *)(uintptr_t)mz);
1322 if (ret <= 0) {
1323 rte_memzone_free(mz);
1324 rte_mempool_free(mp);
1325 return NULL;
1326 }
1327
1328 ntb_conf->mz_list[mz_id] = mz;
1329 }
1330 if (left_sz) {
1331 printf("mw space is not enough for mempool.\n");
1332 goto fail;
1333 }
1334
1335 ntb_conf->mz_num = mz_id;
1336 rte_mempool_obj_iter(mp, rte_pktmbuf_init, NULL);
1337
1338 return mp;
1339 fail:
1340 rte_mempool_free(mp);
1341 return NULL;
1342 }
1343
1344 int
1345 main(int argc, char **argv)
1346 {
1347 struct rte_eth_conf eth_pconf = eth_port_conf;
1348 struct rte_rawdev_info ntb_rawdev_conf;
1349 struct rte_rawdev_info ntb_rawdev_info;
1350 struct rte_eth_dev_info ethdev_info;
1351 struct rte_eth_rxconf eth_rx_conf;
1352 struct rte_eth_txconf eth_tx_conf;
1353 struct ntb_queue_conf ntb_q_conf;
1354 struct ntb_dev_config ntb_conf;
1355 struct ntb_dev_info ntb_info;
1356 uint64_t ntb_link_status;
1357 uint32_t nb_mbuf;
1358 int ret, i;
1359
1360 signal(SIGINT, signal_handler);
1361 signal(SIGTERM, signal_handler);
1362
1363 ret = rte_eal_init(argc, argv);
1364 if (ret < 0)
1365 rte_exit(EXIT_FAILURE, "Error with EAL initialization.\n");
1366
1367 if (rte_lcore_count() < 2)
1368 rte_exit(EXIT_FAILURE, "Need at least 2 cores\n");
1369
1370 /* Find 1st ntb rawdev. */
1371 for (i = 0; i < RTE_RAWDEV_MAX_DEVS; i++)
1372 if (rte_rawdevs[i].driver_name &&
1373 (strncmp(rte_rawdevs[i].driver_name, "raw_ntb",
1374 NTB_DRV_NAME_LEN) == 0) && (rte_rawdevs[i].attached == 1))
1375 break;
1376
1377 if (i == RTE_RAWDEV_MAX_DEVS)
1378 rte_exit(EXIT_FAILURE, "Cannot find any ntb device.\n");
1379
1380 dev_id = i;
1381
1382 argc -= ret;
1383 argv += ret;
1384
1385 ntb_parse_args(argc, argv);
1386
1387 rte_rawdev_set_attr(dev_id, NTB_QUEUE_SZ_NAME, nb_desc);
1388 printf("Set queue size as %u.\n", nb_desc);
1389 rte_rawdev_set_attr(dev_id, NTB_QUEUE_NUM_NAME, num_queues);
1390 printf("Set queue number as %u.\n", num_queues);
1391 ntb_rawdev_info.dev_private = (rte_rawdev_obj_t)(&ntb_info);
1392 rte_rawdev_info_get(dev_id, &ntb_rawdev_info);
1393
1394 nb_mbuf = nb_desc * num_queues * 2 * 2 + rte_lcore_count() *
1395 MEMPOOL_CACHE_SIZE;
1396 mbuf_pool = ntb_mbuf_pool_create(ntb_buf_size, nb_mbuf, ntb_info,
1397 &ntb_conf, rte_socket_id());
1398 if (mbuf_pool == NULL)
1399 rte_exit(EXIT_FAILURE, "Cannot create mbuf pool.\n");
1400
1401 ntb_conf.num_queues = num_queues;
1402 ntb_conf.queue_size = nb_desc;
1403 ntb_rawdev_conf.dev_private = (rte_rawdev_obj_t)(&ntb_conf);
1404 ret = rte_rawdev_configure(dev_id, &ntb_rawdev_conf);
1405 if (ret)
1406 rte_exit(EXIT_FAILURE, "Can't config ntb dev: err=%d, "
1407 "port=%u\n", ret, dev_id);
1408
1409 ntb_q_conf.tx_free_thresh = tx_free_thresh;
1410 ntb_q_conf.nb_desc = nb_desc;
1411 ntb_q_conf.rx_mp = mbuf_pool;
1412 for (i = 0; i < num_queues; i++) {
1413 /* Setup rawdev queue */
1414 ret = rte_rawdev_queue_setup(dev_id, i, &ntb_q_conf);
1415 if (ret < 0)
1416 rte_exit(EXIT_FAILURE,
1417 "Failed to setup ntb queue %u.\n", i);
1418 }
1419
1420 /* Waiting for peer dev up at most 100s.*/
1421 printf("Checking ntb link status...\n");
1422 for (i = 0; i < 1000; i++) {
1423 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME,
1424 &ntb_link_status);
1425 if (ntb_link_status) {
1426 printf("Peer dev ready, ntb link up.\n");
1427 break;
1428 }
1429 rte_delay_ms(100);
1430 }
1431 rte_rawdev_get_attr(dev_id, NTB_LINK_STATUS_NAME, &ntb_link_status);
1432 if (ntb_link_status == 0)
1433 printf("Expire 100s. Link is not up. Please restart app.\n");
1434
1435 ret = rte_rawdev_start(dev_id);
1436 if (ret < 0)
1437 rte_exit(EXIT_FAILURE, "rte_rawdev_start: err=%d, port=%u\n",
1438 ret, dev_id);
1439
1440 /* Find 1st ethdev */
1441 eth_port_id = rte_eth_find_next(0);
1442
1443 if (eth_port_id < RTE_MAX_ETHPORTS) {
1444 rte_eth_dev_info_get(eth_port_id, &ethdev_info);
1445 eth_pconf.rx_adv_conf.rss_conf.rss_hf &=
1446 ethdev_info.flow_type_rss_offloads;
1447 ret = rte_eth_dev_configure(eth_port_id, num_queues,
1448 num_queues, &eth_pconf);
1449 if (ret)
1450 rte_exit(EXIT_FAILURE, "Can't config ethdev: err=%d, "
1451 "port=%u\n", ret, eth_port_id);
1452 eth_rx_conf = ethdev_info.default_rxconf;
1453 eth_rx_conf.offloads = eth_pconf.rxmode.offloads;
1454 eth_tx_conf = ethdev_info.default_txconf;
1455 eth_tx_conf.offloads = eth_pconf.txmode.offloads;
1456
1457 /* Setup ethdev queue if ethdev exists */
1458 for (i = 0; i < num_queues; i++) {
1459 ret = rte_eth_rx_queue_setup(eth_port_id, i, nb_desc,
1460 rte_eth_dev_socket_id(eth_port_id),
1461 &eth_rx_conf, mbuf_pool);
1462 if (ret < 0)
1463 rte_exit(EXIT_FAILURE,
1464 "Failed to setup eth rxq %u.\n", i);
1465 ret = rte_eth_tx_queue_setup(eth_port_id, i, nb_desc,
1466 rte_eth_dev_socket_id(eth_port_id),
1467 &eth_tx_conf);
1468 if (ret < 0)
1469 rte_exit(EXIT_FAILURE,
1470 "Failed to setup eth txq %u.\n", i);
1471 }
1472
1473 ret = rte_eth_dev_start(eth_port_id);
1474 if (ret < 0)
1475 rte_exit(EXIT_FAILURE, "rte_eth_dev_start: err=%d, "
1476 "port=%u\n", ret, eth_port_id);
1477 }
1478
1479 /* initialize port stats */
1480 memset(&ntb_port_stats, 0, sizeof(ntb_port_stats));
1481
1482 /* Set default fwd mode if user doesn't set it. */
1483 if (fwd_mode == MAX_FWD_MODE && eth_port_id < RTE_MAX_ETHPORTS) {
1484 printf("Set default fwd mode as iofwd.\n");
1485 fwd_mode = IOFWD;
1486 }
1487 if (fwd_mode == MAX_FWD_MODE) {
1488 printf("Set default fwd mode as file-trans.\n");
1489 fwd_mode = FILE_TRANS;
1490 }
1491
1492 if (interactive) {
1493 sleep(1);
1494 prompt();
1495 } else {
1496 start_pkt_fwd();
1497 }
1498
1499 return 0;
1500 }
Ceph