]> git.proxmox.com Git - ceph.git/blame - ceph/src/spdk/dpdk/drivers/raw/ntb/ntb.c
projects / ceph.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
update source to Ceph Pacific 16.2.2
[ceph.git] / ceph / src / spdk / dpdk / drivers / raw / ntb / ntb.c
CommitLineData
f67539c2
TL		 1/* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2019 Intel Corporation.
3 */
4#include <stdint.h>
5#include <stdio.h>
6#include <string.h>
7#include <errno.h>
8
9#include <rte_common.h>
10#include <rte_lcore.h>
11#include <rte_cycles.h>
12#include <rte_eal.h>
13#include <rte_log.h>
14#include <rte_pci.h>
15#include <rte_mbuf.h>
16#include <rte_bus_pci.h>
17#include <rte_memzone.h>
18#include <rte_memcpy.h>
19#include <rte_rawdev.h>
20#include <rte_rawdev_pmd.h>
21
22#include "ntb_hw_intel.h"
23#include "rte_pmd_ntb.h"
24#include "ntb.h"
25
26int ntb_logtype;
27
28static const struct rte_pci_id pci_id_ntb_map[] = {
29 { RTE_PCI_DEVICE(NTB_INTEL_VENDOR_ID, NTB_INTEL_DEV_ID_B2B_SKX) },
30 { .vendor_id = 0, /* sentinel */ },
31};
32
33/* Align with enum ntb_xstats_idx */
34static struct rte_rawdev_xstats_name ntb_xstats_names[] = {
35 {"Tx-packets"},
36 {"Tx-bytes"},
37 {"Tx-errors"},
38 {"Rx-packets"},
39 {"Rx-bytes"},
40 {"Rx-missed"},
41};
42#define NTB_XSTATS_NUM RTE_DIM(ntb_xstats_names)
43
44static inline void
45ntb_link_cleanup(struct rte_rawdev *dev)
46{
47 struct ntb_hw *hw = dev->dev_private;
48 int status, i;
49
50 if (hw->ntb_ops->spad_write == NULL ||
51 hw->ntb_ops->mw_set_trans == NULL) {
52 NTB_LOG(ERR, "Not supported to clean up link.");
53 return;
54 }
55
56 /* Clean spad registers. */
57 for (i = 0; i < hw->spad_cnt; i++) {
58 status = (*hw->ntb_ops->spad_write)(dev, i, 0, 0);
59 if (status)
60 NTB_LOG(ERR, "Failed to clean local spad.");
61 }
62
63 /* Clear mw so that peer cannot access local memory.*/
64 for (i = 0; i < hw->used_mw_num; i++) {
65 status = (*hw->ntb_ops->mw_set_trans)(dev, i, 0, 0);
66 if (status)
67 NTB_LOG(ERR, "Failed to clean mw.");
68 }
69}
70
71static inline int
72ntb_handshake_work(const struct rte_rawdev *dev)
73{
74 struct ntb_hw *hw = dev->dev_private;
75 uint32_t val;
76 int ret, i;
77
78 if (hw->ntb_ops->spad_write == NULL ||
79 hw->ntb_ops->mw_set_trans == NULL) {
80 NTB_LOG(ERR, "Scratchpad/MW setting is not supported.");
81 return -ENOTSUP;
82 }
83
84 /* Tell peer the mw info of local side. */
85 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_MWS, 1, hw->mw_cnt);
86 if (ret < 0)
87 return ret;
88 for (i = 0; i < hw->mw_cnt; i++) {
89 NTB_LOG(INFO, "Local %u mw size: 0x%"PRIx64"", i,
90 hw->mw_size[i]);
91 val = hw->mw_size[i] >> 32;
92 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_H + 2 * i,
93 1, val);
94 if (ret < 0)
95 return ret;
96 val = hw->mw_size[i];
97 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_SZ_L + 2 * i,
98 1, val);
99 if (ret < 0)
100 return ret;
101 }
102
103 /* Tell peer about the queue info and map memory to the peer. */
104 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_Q_SZ, 1, hw->queue_size);
105 if (ret < 0)
106 return ret;
107 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_NUM_QPS, 1,
108 hw->queue_pairs);
109 if (ret < 0)
110 return ret;
111 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_USED_MWS, 1,
112 hw->used_mw_num);
113 if (ret < 0)
114 return ret;
115 for (i = 0; i < hw->used_mw_num; i++) {
116 val = (uint64_t)(size_t)(hw->mz[i]->addr) >> 32;
117 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_H + 2 * i,
118 1, val);
119 if (ret < 0)
120 return ret;
121 val = (uint64_t)(size_t)(hw->mz[i]->addr);
122 ret = (*hw->ntb_ops->spad_write)(dev, SPAD_MW0_BA_L + 2 * i,
123 1, val);
124 if (ret < 0)
125 return ret;
126 }
127
128 for (i = 0; i < hw->used_mw_num; i++) {
129 ret = (*hw->ntb_ops->mw_set_trans)(dev, i, hw->mz[i]->iova,
130 hw->mz[i]->len);
131 if (ret < 0)
132 return ret;
133 }
134
135 /* Ring doorbell 0 to tell peer the device is ready. */
136 ret = (*hw->ntb_ops->peer_db_set)(dev, 0);
137 if (ret < 0)
138 return ret;
139
140 return 0;
141}
142
143static void
144ntb_dev_intr_handler(void *param)
145{
146 struct rte_rawdev *dev = (struct rte_rawdev *)param;
147 struct ntb_hw *hw = dev->dev_private;
148 uint32_t val_h, val_l;
149 uint64_t peer_mw_size;
150 uint64_t db_bits = 0;
151 uint8_t peer_mw_cnt;
152 int i = 0;
153
154 if (hw->ntb_ops->db_read == NULL ||
155 hw->ntb_ops->db_clear == NULL ||
156 hw->ntb_ops->peer_db_set == NULL) {
157 NTB_LOG(ERR, "Doorbell is not supported.");
158 return;
159 }
160
161 db_bits = (*hw->ntb_ops->db_read)(dev);
162 if (!db_bits)
163 NTB_LOG(ERR, "No doorbells");
164
165 /* Doorbell 0 is for peer device ready. */
166 if (db_bits & 1) {
167 NTB_LOG(INFO, "DB0: Peer device is up.");
168 /* Clear received doorbell. */
169 (*hw->ntb_ops->db_clear)(dev, 1);
170
171 /**
172 * Peer dev is already up. All mw settings are already done.
173 * Skip them.
174 */
175 if (hw->peer_dev_up)
176 return;
177
178 if (hw->ntb_ops->spad_read == NULL) {
179 NTB_LOG(ERR, "Scratchpad read is not supported.");
180 return;
181 }
182
183 /* Check if mw setting on the peer is the same as local. */
184 peer_mw_cnt = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_MWS, 0);
185 if (peer_mw_cnt != hw->mw_cnt) {
186 NTB_LOG(ERR, "Both mw cnt must be the same.");
187 return;
188 }
189
190 for (i = 0; i < hw->mw_cnt; i++) {
191 val_h = (*hw->ntb_ops->spad_read)
192 (dev, SPAD_MW0_SZ_H + 2 * i, 0);
193 val_l = (*hw->ntb_ops->spad_read)
194 (dev, SPAD_MW0_SZ_L + 2 * i, 0);
195 peer_mw_size = ((uint64_t)val_h << 32) | val_l;
196 NTB_LOG(DEBUG, "Peer %u mw size: 0x%"PRIx64"", i,
197 peer_mw_size);
198 if (peer_mw_size != hw->mw_size[i]) {
199 NTB_LOG(ERR, "Mw config must be the same.");
200 return;
201 }
202 }
203
204 hw->peer_dev_up = 1;
205
206 /**
207 * Handshake with peer. Spad_write & mw_set_trans only works
208 * when both devices are up. So write spad again when db is
209 * received. And set db again for the later device who may miss
210 * the 1st db.
211 */
212 if (ntb_handshake_work(dev) < 0) {
213 NTB_LOG(ERR, "Handshake work failed.");
214 return;
215 }
216
217 /* To get the link info. */
218 if (hw->ntb_ops->get_link_status == NULL) {
219 NTB_LOG(ERR, "Not supported to get link status.");
220 return;
221 }
222 (*hw->ntb_ops->get_link_status)(dev);
223 NTB_LOG(INFO, "Link is up. Link speed: %u. Link width: %u",
224 hw->link_speed, hw->link_width);
225 return;
226 }
227
228 if (db_bits & (1 << 1)) {
229 NTB_LOG(INFO, "DB1: Peer device is down.");
230 /* Clear received doorbell. */
231 (*hw->ntb_ops->db_clear)(dev, 2);
232
233 /* Peer device will be down, So clean local side too. */
234 ntb_link_cleanup(dev);
235
236 hw->peer_dev_up = 0;
237 /* Response peer's dev_stop request. */
238 (*hw->ntb_ops->peer_db_set)(dev, 2);
239 return;
240 }
241
242 if (db_bits & (1 << 2)) {
243 NTB_LOG(INFO, "DB2: Peer device agrees dev to be down.");
244 /* Clear received doorbell. */
245 (*hw->ntb_ops->db_clear)(dev, (1 << 2));
246 hw->peer_dev_up = 0;
247 return;
248 }
249}
250
251static void
252ntb_queue_conf_get(struct rte_rawdev *dev,
253 uint16_t queue_id,
254 rte_rawdev_obj_t queue_conf)
255{
256 struct ntb_queue_conf *q_conf = queue_conf;
257 struct ntb_hw *hw = dev->dev_private;
258
259 q_conf->tx_free_thresh = hw->tx_queues[queue_id]->tx_free_thresh;
260 q_conf->nb_desc = hw->rx_queues[queue_id]->nb_rx_desc;
261 q_conf->rx_mp = hw->rx_queues[queue_id]->mpool;
262}
263
264static void
265ntb_rxq_release_mbufs(struct ntb_rx_queue *q)
266{
267 int i;
268
269 if (!q || !q->sw_ring) {
270 NTB_LOG(ERR, "Pointer to rxq or sw_ring is NULL");
271 return;
272 }
273
274 for (i = 0; i < q->nb_rx_desc; i++) {
275 if (q->sw_ring[i].mbuf) {
276 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf);
277 q->sw_ring[i].mbuf = NULL;
278 }
279 }
280}
281
282static void
283ntb_rxq_release(struct ntb_rx_queue *rxq)
284{
285 if (!rxq) {
286 NTB_LOG(ERR, "Pointer to rxq is NULL");
287 return;
288 }
289
290 ntb_rxq_release_mbufs(rxq);
291
292 rte_free(rxq->sw_ring);
293 rte_free(rxq);
294}
295
296static int
297ntb_rxq_setup(struct rte_rawdev *dev,
298 uint16_t qp_id,
299 rte_rawdev_obj_t queue_conf)
300{
301 struct ntb_queue_conf *rxq_conf = queue_conf;
302 struct ntb_hw *hw = dev->dev_private;
303 struct ntb_rx_queue *rxq;
304
305 /* Allocate the rx queue data structure */
306 rxq = rte_zmalloc_socket("ntb rx queue",
307 sizeof(struct ntb_rx_queue),
308 RTE_CACHE_LINE_SIZE,
309 dev->socket_id);
310 if (!rxq) {
311 NTB_LOG(ERR, "Failed to allocate memory for "
312 "rx queue data structure.");
313 return -ENOMEM;
314 }
315
316 if (rxq_conf->rx_mp == NULL) {
317 NTB_LOG(ERR, "Invalid null mempool pointer.");
318 return -EINVAL;
319 }
320 rxq->nb_rx_desc = rxq_conf->nb_desc;
321 rxq->mpool = rxq_conf->rx_mp;
322 rxq->port_id = dev->dev_id;
323 rxq->queue_id = qp_id;
324 rxq->hw = hw;
325
326 /* Allocate the software ring. */
327 rxq->sw_ring =
328 rte_zmalloc_socket("ntb rx sw ring",
329 sizeof(struct ntb_rx_entry) *
330 rxq->nb_rx_desc,
331 RTE_CACHE_LINE_SIZE,
332 dev->socket_id);
333 if (!rxq->sw_ring) {
334 ntb_rxq_release(rxq);
335 rxq = NULL;
336 NTB_LOG(ERR, "Failed to allocate memory for SW ring");
337 return -ENOMEM;
338 }
339
340 hw->rx_queues[qp_id] = rxq;
341
342 return 0;
343}
344
345static void
346ntb_txq_release_mbufs(struct ntb_tx_queue *q)
347{
348 int i;
349
350 if (!q || !q->sw_ring) {
351 NTB_LOG(ERR, "Pointer to txq or sw_ring is NULL");
352 return;
353 }
354
355 for (i = 0; i < q->nb_tx_desc; i++) {
356 if (q->sw_ring[i].mbuf) {
357 rte_pktmbuf_free_seg(q->sw_ring[i].mbuf);
358 q->sw_ring[i].mbuf = NULL;
359 }
360 }
361}
362
363static void
364ntb_txq_release(struct ntb_tx_queue *txq)
365{
366 if (!txq) {
367 NTB_LOG(ERR, "Pointer to txq is NULL");
368 return;
369 }
370
371 ntb_txq_release_mbufs(txq);
372
373 rte_free(txq->sw_ring);
374 rte_free(txq);
375}
376
377static int
378ntb_txq_setup(struct rte_rawdev *dev,
379 uint16_t qp_id,
380 rte_rawdev_obj_t queue_conf)
381{
382 struct ntb_queue_conf *txq_conf = queue_conf;
383 struct ntb_hw *hw = dev->dev_private;
384 struct ntb_tx_queue *txq;
385 uint16_t i, prev;
386
387 /* Allocate the TX queue data structure. */
388 txq = rte_zmalloc_socket("ntb tx queue",
389 sizeof(struct ntb_tx_queue),
390 RTE_CACHE_LINE_SIZE,
391 dev->socket_id);
392 if (!txq) {
393 NTB_LOG(ERR, "Failed to allocate memory for "
394 "tx queue structure");
395 return -ENOMEM;
396 }
397
398 txq->nb_tx_desc = txq_conf->nb_desc;
399 txq->port_id = dev->dev_id;
400 txq->queue_id = qp_id;
401 txq->hw = hw;
402
403 /* Allocate software ring */
404 txq->sw_ring =
405 rte_zmalloc_socket("ntb tx sw ring",
406 sizeof(struct ntb_tx_entry) *
407 txq->nb_tx_desc,
408 RTE_CACHE_LINE_SIZE,
409 dev->socket_id);
410 if (!txq->sw_ring) {
411 ntb_txq_release(txq);
412 txq = NULL;
413 NTB_LOG(ERR, "Failed to allocate memory for SW TX ring");
414 return -ENOMEM;
415 }
416
417 prev = txq->nb_tx_desc - 1;
418 for (i = 0; i < txq->nb_tx_desc; i++) {
419 txq->sw_ring[i].mbuf = NULL;
420 txq->sw_ring[i].last_id = i;
421 txq->sw_ring[prev].next_id = i;
422 prev = i;
423 }
424
425 txq->tx_free_thresh = txq_conf->tx_free_thresh ?
426 txq_conf->tx_free_thresh :
427 NTB_DFLT_TX_FREE_THRESH;
428 if (txq->tx_free_thresh >= txq->nb_tx_desc - 3) {
429 NTB_LOG(ERR, "tx_free_thresh must be less than nb_desc - 3. "
430 "(tx_free_thresh=%u qp_id=%u)", txq->tx_free_thresh,
431 qp_id);
432 return -EINVAL;
433 }
434
435 hw->tx_queues[qp_id] = txq;
436
437 return 0;
438}
439
440
441static int
442ntb_queue_setup(struct rte_rawdev *dev,
443 uint16_t queue_id,
444 rte_rawdev_obj_t queue_conf)
445{
446 struct ntb_hw *hw = dev->dev_private;
447 int ret;
448
449 if (queue_id >= hw->queue_pairs)
450 return -EINVAL;
451
452 ret = ntb_txq_setup(dev, queue_id, queue_conf);
453 if (ret < 0)
454 return ret;
455
456 ret = ntb_rxq_setup(dev, queue_id, queue_conf);
457
458 return ret;
459}
460
461static int
462ntb_queue_release(struct rte_rawdev *dev, uint16_t queue_id)
463{
464 struct ntb_hw *hw = dev->dev_private;
465
466 if (queue_id >= hw->queue_pairs)
467 return -EINVAL;
468
469 ntb_txq_release(hw->tx_queues[queue_id]);
470 hw->tx_queues[queue_id] = NULL;
471 ntb_rxq_release(hw->rx_queues[queue_id]);
472 hw->rx_queues[queue_id] = NULL;
473
474 return 0;
475}
476
477static uint16_t
478ntb_queue_count(struct rte_rawdev *dev)
479{
480 struct ntb_hw *hw = dev->dev_private;
481 return hw->queue_pairs;
482}
483
484static int
485ntb_queue_init(struct rte_rawdev *dev, uint16_t qp_id)
486{
487 struct ntb_hw *hw = dev->dev_private;
488 struct ntb_rx_queue *rxq = hw->rx_queues[qp_id];
489 struct ntb_tx_queue *txq = hw->tx_queues[qp_id];
490 volatile struct ntb_header *local_hdr;
491 struct ntb_header *remote_hdr;
492 uint16_t q_size = hw->queue_size;
493 uint32_t hdr_offset;
494 void *bar_addr;
495 uint16_t i;
496
497 if (hw->ntb_ops->get_peer_mw_addr == NULL) {
498 NTB_LOG(ERR, "Getting peer mw addr is not supported.");
499 return -EINVAL;
500 }
501
502 /* Put queue info into the start of shared memory. */
503 hdr_offset = hw->hdr_size_per_queue * qp_id;
504 local_hdr = (volatile struct ntb_header *)
505 ((size_t)hw->mz[0]->addr + hdr_offset);
506 bar_addr = (*hw->ntb_ops->get_peer_mw_addr)(dev, 0);
507 if (bar_addr == NULL)
508 return -EINVAL;
509 remote_hdr = (struct ntb_header *)
510 ((size_t)bar_addr + hdr_offset);
511
512 /* rxq init. */
513 rxq->rx_desc_ring = (struct ntb_desc *)
514 (&remote_hdr->desc_ring);
515 rxq->rx_used_ring = (volatile struct ntb_used *)
516 (&local_hdr->desc_ring[q_size]);
517 rxq->avail_cnt = &remote_hdr->avail_cnt;
518 rxq->used_cnt = &local_hdr->used_cnt;
519
520 for (i = 0; i < rxq->nb_rx_desc - 1; i++) {
521 struct rte_mbuf *mbuf = rte_mbuf_raw_alloc(rxq->mpool);
522 if (unlikely(!mbuf)) {
523 NTB_LOG(ERR, "Failed to allocate mbuf for RX");
524 return -ENOMEM;
525 }
526 mbuf->port = dev->dev_id;
527
528 rxq->sw_ring[i].mbuf = mbuf;
529
530 rxq->rx_desc_ring[i].addr = rte_pktmbuf_mtod(mbuf, size_t);
531 rxq->rx_desc_ring[i].len = mbuf->buf_len - RTE_PKTMBUF_HEADROOM;
532 }
533 rte_wmb();
534 *rxq->avail_cnt = rxq->nb_rx_desc - 1;
535 rxq->last_avail = rxq->nb_rx_desc - 1;
536 rxq->last_used = 0;
537
538 /* txq init */
539 txq->tx_desc_ring = (volatile struct ntb_desc *)
540 (&local_hdr->desc_ring);
541 txq->tx_used_ring = (struct ntb_used *)
542 (&remote_hdr->desc_ring[q_size]);
543 txq->avail_cnt = &local_hdr->avail_cnt;
544 txq->used_cnt = &remote_hdr->used_cnt;
545
546 rte_wmb();
547 *txq->used_cnt = 0;
548 txq->last_used = 0;
549 txq->last_avail = 0;
550 txq->nb_tx_free = txq->nb_tx_desc - 1;
551
552 /* Set per queue stats. */
553 for (i = 0; i < NTB_XSTATS_NUM; i++) {
554 hw->ntb_xstats[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0;
555 hw->ntb_xstats_off[i + NTB_XSTATS_NUM * (qp_id + 1)] = 0;
556 }
557
558 return 0;
559}
560
561static inline void
562ntb_enqueue_cleanup(struct ntb_tx_queue *txq)
563{
564 struct ntb_tx_entry *sw_ring = txq->sw_ring;
565 uint16_t tx_free = txq->last_avail;
566 uint16_t nb_to_clean, i;
567
568 /* avail_cnt + 1 represents where to rx next in the peer. */
569 nb_to_clean = (*txq->avail_cnt - txq->last_avail + 1 +
570 txq->nb_tx_desc) & (txq->nb_tx_desc - 1);
571 nb_to_clean = RTE_MIN(nb_to_clean, txq->tx_free_thresh);
572 for (i = 0; i < nb_to_clean; i++) {
573 if (sw_ring[tx_free].mbuf)
574 rte_pktmbuf_free_seg(sw_ring[tx_free].mbuf);
575 tx_free = (tx_free + 1) & (txq->nb_tx_desc - 1);
576 }
577
578 txq->nb_tx_free += nb_to_clean;
579 txq->last_avail = tx_free;
580}
581
582static int
583ntb_enqueue_bufs(struct rte_rawdev *dev,
584 struct rte_rawdev_buf **buffers,
585 unsigned int count,
586 rte_rawdev_obj_t context)
587{
588 struct ntb_hw *hw = dev->dev_private;
589 struct ntb_tx_queue *txq = hw->tx_queues[(size_t)context];
590 struct ntb_tx_entry *sw_ring = txq->sw_ring;
591 struct rte_mbuf *txm;
592 struct ntb_used tx_used[NTB_MAX_DESC_SIZE];
593 volatile struct ntb_desc *tx_item;
594 uint16_t tx_last, nb_segs, off, last_used, avail_cnt;
595 uint16_t nb_mbufs = 0;
596 uint16_t nb_tx = 0;
597 uint64_t bytes = 0;
598 void *buf_addr;
599 int i;
600
601 if (unlikely(hw->ntb_ops->ioremap == NULL)) {
602 NTB_LOG(ERR, "Ioremap not supported.");
603 return nb_tx;
604 }
605
606 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) {
607 NTB_LOG(DEBUG, "Link is not up.");
608 return nb_tx;
609 }
610
611 if (txq->nb_tx_free < txq->tx_free_thresh)
612 ntb_enqueue_cleanup(txq);
613
614 off = NTB_XSTATS_NUM * ((size_t)context + 1);
615 last_used = txq->last_used;
616 avail_cnt = *txq->avail_cnt;/* Where to alloc next. */
617 for (nb_tx = 0; nb_tx < count; nb_tx++) {
618 txm = (struct rte_mbuf *)(buffers[nb_tx]->buf_addr);
619 if (txm == NULL || txq->nb_tx_free < txm->nb_segs)
620 break;
621
622 tx_last = (txq->last_used + txm->nb_segs - 1) &
623 (txq->nb_tx_desc - 1);
624 nb_segs = txm->nb_segs;
625 for (i = 0; i < nb_segs; i++) {
626 /* Not enough ring space for tx. */
627 if (txq->last_used == avail_cnt)
628 goto end_of_tx;
629 sw_ring[txq->last_used].mbuf = txm;
630 tx_item = txq->tx_desc_ring + txq->last_used;
631
632 if (!tx_item->len) {
633 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++;
634 goto end_of_tx;
635 }
636 if (txm->data_len > tx_item->len) {
637 NTB_LOG(ERR, "Data length exceeds buf length."
638 " Only %u data would be transmitted.",
639 tx_item->len);
640 txm->data_len = tx_item->len;
641 }
642
643 /* translate remote virtual addr to bar virtual addr */
644 buf_addr = (*hw->ntb_ops->ioremap)(dev, tx_item->addr);
645 if (buf_addr == NULL) {
646 (hw->ntb_xstats[NTB_TX_ERRS_ID + off])++;
647 NTB_LOG(ERR, "Null remap addr.");
648 goto end_of_tx;
649 }
650 rte_memcpy(buf_addr, rte_pktmbuf_mtod(txm, void *),
651 txm->data_len);
652
653 tx_used[nb_mbufs].len = txm->data_len;
654 tx_used[nb_mbufs++].flags = (txq->last_used ==
655 tx_last) ?
656 NTB_FLAG_EOP : 0;
657
658 /* update stats */
659 bytes += txm->data_len;
660
661 txm = txm->next;
662
663 sw_ring[txq->last_used].next_id = (txq->last_used + 1) &
664 (txq->nb_tx_desc - 1);
665 sw_ring[txq->last_used].last_id = tx_last;
666 txq->last_used = (txq->last_used + 1) &
667 (txq->nb_tx_desc - 1);
668 }
669 txq->nb_tx_free -= nb_segs;
670 }
671
672end_of_tx:
673 if (nb_tx) {
674 uint16_t nb1, nb2;
675 if (nb_mbufs > txq->nb_tx_desc - last_used) {
676 nb1 = txq->nb_tx_desc - last_used;
677 nb2 = nb_mbufs - txq->nb_tx_desc + last_used;
678 } else {
679 nb1 = nb_mbufs;
680 nb2 = 0;
681 }
682 rte_memcpy(txq->tx_used_ring + last_used, tx_used,
683 sizeof(struct ntb_used) * nb1);
684 rte_memcpy(txq->tx_used_ring, tx_used + nb1,
685 sizeof(struct ntb_used) * nb2);
686 rte_wmb();
687 *txq->used_cnt = txq->last_used;
688
689 /* update queue stats */
690 hw->ntb_xstats[NTB_TX_BYTES_ID + off] += bytes;
691 hw->ntb_xstats[NTB_TX_PKTS_ID + off] += nb_tx;
692 }
693
694 return nb_tx;
695}
696
697static int
698ntb_dequeue_bufs(struct rte_rawdev *dev,
699 struct rte_rawdev_buf **buffers,
700 unsigned int count,
701 rte_rawdev_obj_t context)
702{
703 struct ntb_hw *hw = dev->dev_private;
704 struct ntb_rx_queue *rxq = hw->rx_queues[(size_t)context];
705 struct ntb_rx_entry *sw_ring = rxq->sw_ring;
706 struct ntb_desc rx_desc[NTB_MAX_DESC_SIZE];
707 struct rte_mbuf *first, *rxm_t;
708 struct rte_mbuf *prev = NULL;
709 volatile struct ntb_used *rx_item;
710 uint16_t nb_mbufs = 0;
711 uint16_t nb_rx = 0;
712 uint64_t bytes = 0;
713 uint16_t off, last_avail, used_cnt, used_nb;
714 int i;
715
716 if (unlikely(dev->started == 0 || hw->peer_dev_up == 0)) {
717 NTB_LOG(DEBUG, "Link is not up");
718 return nb_rx;
719 }
720
721 used_cnt = *rxq->used_cnt;
722
723 if (rxq->last_used == used_cnt)
724 return nb_rx;
725
726 last_avail = rxq->last_avail;
727 used_nb = (used_cnt - rxq->last_used) & (rxq->nb_rx_desc - 1);
728 count = RTE_MIN(count, used_nb);
729 for (nb_rx = 0; nb_rx < count; nb_rx++) {
730 i = 0;
731 while (true) {
732 rx_item = rxq->rx_used_ring + rxq->last_used;
733 rxm_t = sw_ring[rxq->last_used].mbuf;
734 rxm_t->data_len = rx_item->len;
735 rxm_t->data_off = RTE_PKTMBUF_HEADROOM;
736 rxm_t->port = rxq->port_id;
737
738 if (!i) {
739 rxm_t->nb_segs = 1;
740 first = rxm_t;
741 first->pkt_len = 0;
742 buffers[nb_rx]->buf_addr = rxm_t;
743 } else {
744 prev->next = rxm_t;
745 first->nb_segs++;
746 }
747
748 prev = rxm_t;
749 first->pkt_len += prev->data_len;
750 rxq->last_used = (rxq->last_used + 1) &
751 (rxq->nb_rx_desc - 1);
752
753 /* alloc new mbuf */
754 rxm_t = rte_mbuf_raw_alloc(rxq->mpool);
755 if (unlikely(rxm_t == NULL)) {
756 NTB_LOG(ERR, "recv alloc mbuf failed.");
757 goto end_of_rx;
758 }
759 rxm_t->port = rxq->port_id;
760 sw_ring[rxq->last_avail].mbuf = rxm_t;
761 i++;
762
763 /* fill new desc */
764 rx_desc[nb_mbufs].addr =
765 rte_pktmbuf_mtod(rxm_t, size_t);
766 rx_desc[nb_mbufs++].len = rxm_t->buf_len -
767 RTE_PKTMBUF_HEADROOM;
768 rxq->last_avail = (rxq->last_avail + 1) &
769 (rxq->nb_rx_desc - 1);
770
771 if (rx_item->flags & NTB_FLAG_EOP)
772 break;
773 }
774 /* update stats */
775 bytes += first->pkt_len;
776 }
777
778end_of_rx:
779 if (nb_rx) {
780 uint16_t nb1, nb2;
781 if (nb_mbufs > rxq->nb_rx_desc - last_avail) {
782 nb1 = rxq->nb_rx_desc - last_avail;
783 nb2 = nb_mbufs - rxq->nb_rx_desc + last_avail;
784 } else {
785 nb1 = nb_mbufs;
786 nb2 = 0;
787 }
788 rte_memcpy(rxq->rx_desc_ring + last_avail, rx_desc,
789 sizeof(struct ntb_desc) * nb1);
790 rte_memcpy(rxq->rx_desc_ring, rx_desc + nb1,
791 sizeof(struct ntb_desc) * nb2);
792 rte_wmb();
793 *rxq->avail_cnt = rxq->last_avail;
794
795 /* update queue stats */
796 off = NTB_XSTATS_NUM * ((size_t)context + 1);
797 hw->ntb_xstats[NTB_RX_BYTES_ID + off] += bytes;
798 hw->ntb_xstats[NTB_RX_PKTS_ID + off] += nb_rx;
799 hw->ntb_xstats[NTB_RX_MISS_ID + off] += (count - nb_rx);
800 }
801
802 return nb_rx;
803}
804
805static void
806ntb_dev_info_get(struct rte_rawdev *dev, rte_rawdev_obj_t dev_info)
807{
808 struct ntb_hw *hw = dev->dev_private;
809 struct ntb_dev_info *info = dev_info;
810
811 info->mw_cnt = hw->mw_cnt;
812 info->mw_size = hw->mw_size;
813
814 /**
815 * Intel hardware requires that mapped memory base address should be
816 * aligned with EMBARSZ and needs continuous memzone.
817 */
818 info->mw_size_align = (uint8_t)(hw->pci_dev->id.vendor_id ==
819 NTB_INTEL_VENDOR_ID);
820
821 if (!hw->queue_size || !hw->queue_pairs) {
822 NTB_LOG(ERR, "No queue size and queue num assigned.");
823 return;
824 }
825
826 hw->hdr_size_per_queue = RTE_ALIGN(sizeof(struct ntb_header) +
827 hw->queue_size * sizeof(struct ntb_desc) +
828 hw->queue_size * sizeof(struct ntb_used),
829 RTE_CACHE_LINE_SIZE);
830 info->ntb_hdr_size = hw->hdr_size_per_queue * hw->queue_pairs;
831}
832
833static int
834ntb_dev_configure(const struct rte_rawdev *dev, rte_rawdev_obj_t config)
835{
836 struct ntb_dev_config *conf = config;
837 struct ntb_hw *hw = dev->dev_private;
838 uint32_t xstats_num;
839 int ret;
840
841 hw->queue_pairs = conf->num_queues;
842 hw->queue_size = conf->queue_size;
843 hw->used_mw_num = conf->mz_num;
844 hw->mz = conf->mz_list;
845 hw->rx_queues = rte_zmalloc("ntb_rx_queues",
846 sizeof(struct ntb_rx_queue *) * hw->queue_pairs, 0);
847 hw->tx_queues = rte_zmalloc("ntb_tx_queues",
848 sizeof(struct ntb_tx_queue *) * hw->queue_pairs, 0);
849 /* First total stats, then per queue stats. */
850 xstats_num = (hw->queue_pairs + 1) * NTB_XSTATS_NUM;
851 hw->ntb_xstats = rte_zmalloc("ntb_xstats", xstats_num *
852 sizeof(uint64_t), 0);
853 hw->ntb_xstats_off = rte_zmalloc("ntb_xstats_off", xstats_num *
854 sizeof(uint64_t), 0);
855
856 /* Start handshake with the peer. */
857 ret = ntb_handshake_work(dev);
858 if (ret < 0) {
859 rte_free(hw->rx_queues);
860 rte_free(hw->tx_queues);
861 hw->rx_queues = NULL;
862 hw->tx_queues = NULL;
863 return ret;
864 }
865
866 return 0;
867}
868
869static int
870ntb_dev_start(struct rte_rawdev *dev)
871{
872 struct ntb_hw *hw = dev->dev_private;
873 uint32_t peer_base_l, peer_val;
874 uint64_t peer_base_h;
875 uint32_t i;
876 int ret;
877
878 if (!hw->link_status || !hw->peer_dev_up)
879 return -EINVAL;
880
881 /* Set total stats. */
882 for (i = 0; i < NTB_XSTATS_NUM; i++) {
883 hw->ntb_xstats[i] = 0;
884 hw->ntb_xstats_off[i] = 0;
885 }
886
887 for (i = 0; i < hw->queue_pairs; i++) {
888 ret = ntb_queue_init(dev, i);
889 if (ret) {
890 NTB_LOG(ERR, "Failed to init queue.");
891 goto err_q_init;
892 }
893 }
894
895 hw->peer_mw_base = rte_zmalloc("ntb_peer_mw_base", hw->mw_cnt *
896 sizeof(uint64_t), 0);
897
898 if (hw->ntb_ops->spad_read == NULL) {
899 ret = -ENOTSUP;
900 goto err_up;
901 }
902
903 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_Q_SZ, 0);
904 if (peer_val != hw->queue_size) {
905 NTB_LOG(ERR, "Inconsistent queue size! (local: %u peer: %u)",
906 hw->queue_size, peer_val);
907 ret = -EINVAL;
908 goto err_up;
909 }
910
911 peer_val = (*hw->ntb_ops->spad_read)(dev, SPAD_NUM_QPS, 0);
912 if (peer_val != hw->queue_pairs) {
913 NTB_LOG(ERR, "Inconsistent number of queues! (local: %u peer:"
914 " %u)", hw->queue_pairs, peer_val);
915 ret = -EINVAL;
916 goto err_up;
917 }
918
919 hw->peer_used_mws = (*hw->ntb_ops->spad_read)(dev, SPAD_USED_MWS, 0);
920
921 for (i = 0; i < hw->peer_used_mws; i++) {
922 peer_base_h = (*hw->ntb_ops->spad_read)(dev,
923 SPAD_MW0_BA_H + 2 * i, 0);
924 peer_base_l = (*hw->ntb_ops->spad_read)(dev,
925 SPAD_MW0_BA_L + 2 * i, 0);
926 hw->peer_mw_base[i] = (peer_base_h << 32) + peer_base_l;
927 }
928
929 dev->started = 1;
930
931 return 0;
932
933err_up:
934 rte_free(hw->peer_mw_base);
935err_q_init:
936 for (i = 0; i < hw->queue_pairs; i++) {
937 ntb_rxq_release_mbufs(hw->rx_queues[i]);
938 ntb_txq_release_mbufs(hw->tx_queues[i]);
939 }
940
941 return ret;
942}
943
944static void
945ntb_dev_stop(struct rte_rawdev *dev)
946{
947 struct ntb_hw *hw = dev->dev_private;
948 uint32_t time_out;
949 int status, i;
950
951 if (!hw->peer_dev_up)
952 goto clean;
953
954 ntb_link_cleanup(dev);
955
956 /* Notify the peer that device will be down. */
957 if (hw->ntb_ops->peer_db_set == NULL) {
958 NTB_LOG(ERR, "Peer doorbell setting is not supported.");
959 return;
960 }
961 status = (*hw->ntb_ops->peer_db_set)(dev, 1);
962 if (status) {
963 NTB_LOG(ERR, "Failed to tell peer device is down.");
964 return;
965 }
966
967 /*
968 * Set time out as 1s in case that the peer is stopped accidently
969 * without any notification.
970 */
971 time_out = 1000000;
972
973 /* Wait for cleanup work down before db mask clear. */
974 while (hw->peer_dev_up && time_out) {
975 time_out -= 10;
976 rte_delay_us(10);
977 }
978
979clean:
980 /* Clear doorbells mask. */
981 if (hw->ntb_ops->db_set_mask == NULL) {
982 NTB_LOG(ERR, "Doorbell mask setting is not supported.");
983 return;
984 }
985 status = (*hw->ntb_ops->db_set_mask)(dev,
986 (((uint64_t)1 << hw->db_cnt) - 1));
987 if (status)
988 NTB_LOG(ERR, "Failed to clear doorbells.");
989
990 for (i = 0; i < hw->queue_pairs; i++) {
991 ntb_rxq_release_mbufs(hw->rx_queues[i]);
992 ntb_txq_release_mbufs(hw->tx_queues[i]);
993 }
994
995 dev->started = 0;
996}
997
998static int
999ntb_dev_close(struct rte_rawdev *dev)
1000{
1001 struct ntb_hw *hw = dev->dev_private;
1002 struct rte_intr_handle *intr_handle;
1003 int i;
1004
1005 if (dev->started)
1006 ntb_dev_stop(dev);
1007
1008 /* free queues */
1009 for (i = 0; i < hw->queue_pairs; i++)
1010 ntb_queue_release(dev, i);
1011 hw->queue_pairs = 0;
1012
1013 intr_handle = &hw->pci_dev->intr_handle;
1014 /* Clean datapath event and vec mapping */
1015 rte_intr_efd_disable(intr_handle);
1016 if (intr_handle->intr_vec) {
1017 rte_free(intr_handle->intr_vec);
1018 intr_handle->intr_vec = NULL;
1019 }
1020 /* Disable uio intr before callback unregister */
1021 rte_intr_disable(intr_handle);
1022
1023 /* Unregister callback func to eal lib */
1024 rte_intr_callback_unregister(intr_handle,
1025 ntb_dev_intr_handler, dev);
1026
1027 return 0;
1028}
1029
1030static int
1031ntb_dev_reset(struct rte_rawdev *rawdev __rte_unused)
1032{
1033 return 0;
1034}
1035
1036static int
1037ntb_attr_set(struct rte_rawdev *dev, const char *attr_name,
1038 uint64_t attr_value)
1039{
1040 struct ntb_hw *hw;
1041 int index;
1042
1043 if (dev == NULL || attr_name == NULL) {
1044 NTB_LOG(ERR, "Invalid arguments for setting attributes");
1045 return -EINVAL;
1046 }
1047
1048 hw = dev->dev_private;
1049
1050 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) {
1051 if (hw->ntb_ops->spad_write == NULL)
1052 return -ENOTSUP;
1053 index = atoi(&attr_name[NTB_SPAD_USER_LEN]);
1054 (*hw->ntb_ops->spad_write)(dev, hw->spad_user_list[index],
1055 1, attr_value);
1056 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
1057 attr_name, attr_value);
1058 return 0;
1059 }
1060
1061 if (!strncmp(attr_name, NTB_QUEUE_SZ_NAME, NTB_ATTR_NAME_LEN)) {
1062 hw->queue_size = attr_value;
1063 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
1064 attr_name, attr_value);
1065 return 0;
1066 }
1067
1068 if (!strncmp(attr_name, NTB_QUEUE_NUM_NAME, NTB_ATTR_NAME_LEN)) {
1069 hw->queue_pairs = attr_value;
1070 NTB_LOG(DEBUG, "Set attribute (%s) Value (%" PRIu64 ")",
1071 attr_name, attr_value);
1072 return 0;
1073 }
1074
1075 /* Attribute not found. */
1076 NTB_LOG(ERR, "Attribute not found.");
1077 return -EINVAL;
1078}
1079
1080static int
1081ntb_attr_get(struct rte_rawdev *dev, const char *attr_name,
1082 uint64_t *attr_value)
1083{
1084 struct ntb_hw *hw;
1085 int index;
1086
1087 if (dev == NULL || attr_name == NULL || attr_value == NULL) {
1088 NTB_LOG(ERR, "Invalid arguments for getting attributes");
1089 return -EINVAL;
1090 }
1091
1092 hw = dev->dev_private;
1093
1094 if (!strncmp(attr_name, NTB_TOPO_NAME, NTB_ATTR_NAME_LEN)) {
1095 *attr_value = hw->topo;
1096 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1097 attr_name, *attr_value);
1098 return 0;
1099 }
1100
1101 if (!strncmp(attr_name, NTB_LINK_STATUS_NAME, NTB_ATTR_NAME_LEN)) {
1102 /* hw->link_status only indicates hw link status. */
1103 *attr_value = hw->link_status && hw->peer_dev_up;
1104 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1105 attr_name, *attr_value);
1106 return 0;
1107 }
1108
1109 if (!strncmp(attr_name, NTB_SPEED_NAME, NTB_ATTR_NAME_LEN)) {
1110 *attr_value = hw->link_speed;
1111 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1112 attr_name, *attr_value);
1113 return 0;
1114 }
1115
1116 if (!strncmp(attr_name, NTB_WIDTH_NAME, NTB_ATTR_NAME_LEN)) {
1117 *attr_value = hw->link_width;
1118 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1119 attr_name, *attr_value);
1120 return 0;
1121 }
1122
1123 if (!strncmp(attr_name, NTB_MW_CNT_NAME, NTB_ATTR_NAME_LEN)) {
1124 *attr_value = hw->mw_cnt;
1125 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1126 attr_name, *attr_value);
1127 return 0;
1128 }
1129
1130 if (!strncmp(attr_name, NTB_DB_CNT_NAME, NTB_ATTR_NAME_LEN)) {
1131 *attr_value = hw->db_cnt;
1132 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1133 attr_name, *attr_value);
1134 return 0;
1135 }
1136
1137 if (!strncmp(attr_name, NTB_SPAD_CNT_NAME, NTB_ATTR_NAME_LEN)) {
1138 *attr_value = hw->spad_cnt;
1139 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1140 attr_name, *attr_value);
1141 return 0;
1142 }
1143
1144 if (!strncmp(attr_name, NTB_SPAD_USER, NTB_SPAD_USER_LEN)) {
1145 if (hw->ntb_ops->spad_read == NULL)
1146 return -ENOTSUP;
1147 index = atoi(&attr_name[NTB_SPAD_USER_LEN]);
1148 *attr_value = (*hw->ntb_ops->spad_read)(dev,
1149 hw->spad_user_list[index], 0);
1150 NTB_LOG(DEBUG, "Attribute (%s) Value (%" PRIu64 ")",
1151 attr_name, *attr_value);
1152 return 0;
1153 }
1154
1155 /* Attribute not found. */
1156 NTB_LOG(ERR, "Attribute not found.");
1157 return -EINVAL;
1158}
1159
1160static inline uint64_t
1161ntb_stats_update(uint64_t offset, uint64_t stat)
1162{
1163 if (stat >= offset)
1164 return (stat - offset);
1165 else
1166 return (uint64_t)(((uint64_t)-1) - offset + stat + 1);
1167}
1168
1169static int
1170ntb_xstats_get(const struct rte_rawdev *dev,
1171 const unsigned int ids[],
1172 uint64_t values[],
1173 unsigned int n)
1174{
1175 struct ntb_hw *hw = dev->dev_private;
1176 uint32_t i, j, off, xstats_num;
1177
1178 /* Calculate total stats of all queues. */
1179 for (i = 0; i < NTB_XSTATS_NUM; i++) {
1180 hw->ntb_xstats[i] = 0;
1181 for (j = 0; j < hw->queue_pairs; j++) {
1182 off = NTB_XSTATS_NUM * (j + 1) + i;
1183 hw->ntb_xstats[i] +=
1184 ntb_stats_update(hw->ntb_xstats_off[off],
1185 hw->ntb_xstats[off]);
1186 }
1187 }
1188
1189 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1190 for (i = 0; i < n && ids[i] < xstats_num; i++) {
1191 if (ids[i] < NTB_XSTATS_NUM)
1192 values[i] = hw->ntb_xstats[ids[i]];
1193 else
1194 values[i] =
1195 ntb_stats_update(hw->ntb_xstats_off[ids[i]],
1196 hw->ntb_xstats[ids[i]]);
1197 }
1198
1199 return i;
1200}
1201
1202static int
1203ntb_xstats_get_names(const struct rte_rawdev *dev,
1204 struct rte_rawdev_xstats_name *xstats_names,
1205 unsigned int size)
1206{
1207 struct ntb_hw *hw = dev->dev_private;
1208 uint32_t xstats_num, i, j, off;
1209
1210 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1211 if (xstats_names == NULL || size < xstats_num)
1212 return xstats_num;
1213
1214 /* Total stats names */
1215 memcpy(xstats_names, ntb_xstats_names, sizeof(ntb_xstats_names));
1216
1217 /* Queue stats names */
1218 for (i = 0; i < hw->queue_pairs; i++) {
1219 for (j = 0; j < NTB_XSTATS_NUM; j++) {
1220 off = j + (i + 1) * NTB_XSTATS_NUM;
1221 snprintf(xstats_names[off].name,
1222 sizeof(xstats_names[0].name),
1223 "%s_q%u", ntb_xstats_names[j].name, i);
1224 }
1225 }
1226
1227 return xstats_num;
1228}
1229
1230static uint64_t
1231ntb_xstats_get_by_name(const struct rte_rawdev *dev,
1232 const char *name, unsigned int *id)
1233{
1234 struct rte_rawdev_xstats_name *xstats_names;
1235 struct ntb_hw *hw = dev->dev_private;
1236 uint32_t xstats_num, i, j, off;
1237
1238 if (name == NULL)
1239 return -EINVAL;
1240
1241 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1242 xstats_names = rte_zmalloc("ntb_stats_name",
1243 sizeof(struct rte_rawdev_xstats_name) *
1244 xstats_num, 0);
1245 ntb_xstats_get_names(dev, xstats_names, xstats_num);
1246
1247 /* Calculate total stats of all queues. */
1248 for (i = 0; i < NTB_XSTATS_NUM; i++) {
1249 for (j = 0; j < hw->queue_pairs; j++) {
1250 off = NTB_XSTATS_NUM * (j + 1) + i;
1251 hw->ntb_xstats[i] +=
1252 ntb_stats_update(hw->ntb_xstats_off[off],
1253 hw->ntb_xstats[off]);
1254 }
1255 }
1256
1257 for (i = 0; i < xstats_num; i++) {
1258 if (!strncmp(name, xstats_names[i].name,
1259 RTE_RAW_DEV_XSTATS_NAME_SIZE)) {
1260 *id = i;
1261 rte_free(xstats_names);
1262 if (i < NTB_XSTATS_NUM)
1263 return hw->ntb_xstats[i];
1264 else
1265 return ntb_stats_update(hw->ntb_xstats_off[i],
1266 hw->ntb_xstats[i]);
1267 }
1268 }
1269
1270 NTB_LOG(ERR, "Cannot find the xstats name.");
1271
1272 return -EINVAL;
1273}
1274
1275static int
1276ntb_xstats_reset(struct rte_rawdev *dev,
1277 const uint32_t ids[],
1278 uint32_t nb_ids)
1279{
1280 struct ntb_hw *hw = dev->dev_private;
1281 uint32_t i, j, off, xstats_num;
1282
1283 xstats_num = NTB_XSTATS_NUM * (hw->queue_pairs + 1);
1284 for (i = 0; i < nb_ids && ids[i] < xstats_num; i++) {
1285 if (ids[i] < NTB_XSTATS_NUM) {
1286 for (j = 0; j < hw->queue_pairs; j++) {
1287 off = NTB_XSTATS_NUM * (j + 1) + ids[i];
1288 hw->ntb_xstats_off[off] = hw->ntb_xstats[off];
1289 }
1290 } else {
1291 hw->ntb_xstats_off[ids[i]] = hw->ntb_xstats[ids[i]];
1292 }
1293 }
1294
1295 return i;
1296}
1297
1298static const struct rte_rawdev_ops ntb_ops = {
1299 .dev_info_get = ntb_dev_info_get,
1300 .dev_configure = ntb_dev_configure,
1301 .dev_start = ntb_dev_start,
1302 .dev_stop = ntb_dev_stop,
1303 .dev_close = ntb_dev_close,
1304 .dev_reset = ntb_dev_reset,
1305
1306 .queue_def_conf = ntb_queue_conf_get,
1307 .queue_setup = ntb_queue_setup,
1308 .queue_release = ntb_queue_release,
1309 .queue_count = ntb_queue_count,
1310
1311 .enqueue_bufs = ntb_enqueue_bufs,
1312 .dequeue_bufs = ntb_dequeue_bufs,
1313
1314 .attr_get = ntb_attr_get,
1315 .attr_set = ntb_attr_set,
1316
1317 .xstats_get = ntb_xstats_get,
1318 .xstats_get_names = ntb_xstats_get_names,
1319 .xstats_get_by_name = ntb_xstats_get_by_name,
1320 .xstats_reset = ntb_xstats_reset,
1321};
1322
1323static int
1324ntb_init_hw(struct rte_rawdev *dev, struct rte_pci_device *pci_dev)
1325{
1326 struct ntb_hw *hw = dev->dev_private;
1327 struct rte_intr_handle *intr_handle;
1328 int ret, i;
1329
1330 hw->pci_dev = pci_dev;
1331 hw->peer_dev_up = 0;
1332 hw->link_status = NTB_LINK_DOWN;
1333 hw->link_speed = NTB_SPEED_NONE;
1334 hw->link_width = NTB_WIDTH_NONE;
1335
1336 switch (pci_dev->id.device_id) {
1337 case NTB_INTEL_DEV_ID_B2B_SKX:
1338 hw->ntb_ops = &intel_ntb_ops;
1339 break;
1340 default:
1341 NTB_LOG(ERR, "Not supported device.");
1342 return -EINVAL;
1343 }
1344
1345 if (hw->ntb_ops->ntb_dev_init == NULL)
1346 return -ENOTSUP;
1347 ret = (*hw->ntb_ops->ntb_dev_init)(dev);
1348 if (ret) {
1349 NTB_LOG(ERR, "Unable to init ntb dev.");
1350 return ret;
1351 }
1352
1353 if (hw->ntb_ops->set_link == NULL)
1354 return -ENOTSUP;
1355 ret = (*hw->ntb_ops->set_link)(dev, 1);
1356 if (ret)
1357 return ret;
1358
1359 /* Init doorbell. */
1360 hw->db_valid_mask = RTE_LEN2MASK(hw->db_cnt, uint64_t);
1361
1362 intr_handle = &pci_dev->intr_handle;
1363 /* Register callback func to eal lib */
1364 rte_intr_callback_register(intr_handle,
1365 ntb_dev_intr_handler, dev);
1366
1367 ret = rte_intr_efd_enable(intr_handle, hw->db_cnt);
1368 if (ret)
1369 return ret;
1370
1371 /* To clarify, the interrupt for each doorbell is already mapped
1372 * by default for intel gen3. They are mapped to msix vec 1-32,
1373 * and hardware intr is mapped to 0. Map all to 0 for uio.
1374 */
1375 if (!rte_intr_cap_multiple(intr_handle)) {
1376 for (i = 0; i < hw->db_cnt; i++) {
1377 if (hw->ntb_ops->vector_bind == NULL)
1378 return -ENOTSUP;
1379 ret = (*hw->ntb_ops->vector_bind)(dev, i, 0);
1380 if (ret)
1381 return ret;
1382 }
1383 }
1384
1385 if (hw->ntb_ops->db_set_mask == NULL ||
1386 hw->ntb_ops->peer_db_set == NULL) {
1387 NTB_LOG(ERR, "Doorbell is not supported.");
1388 return -ENOTSUP;
1389 }
1390 hw->db_mask = 0;
1391 ret = (*hw->ntb_ops->db_set_mask)(dev, hw->db_mask);
1392 if (ret) {
1393 NTB_LOG(ERR, "Unable to enable intr for all dbs.");
1394 return ret;
1395 }
1396
1397 /* enable uio intr after callback register */
1398 rte_intr_enable(intr_handle);
1399
1400 return ret;
1401}
1402
1403static int
1404ntb_create(struct rte_pci_device *pci_dev, int socket_id)
1405{
1406 char name[RTE_RAWDEV_NAME_MAX_LEN];
1407 struct rte_rawdev *rawdev = NULL;
1408 int ret;
1409
1410 if (pci_dev == NULL) {
1411 NTB_LOG(ERR, "Invalid pci_dev.");
1412 return -EINVAL;
1413 }
1414
1415 memset(name, 0, sizeof(name));
1416 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x",
1417 pci_dev->addr.bus, pci_dev->addr.devid,
1418 pci_dev->addr.function);
1419
1420 NTB_LOG(INFO, "Init %s on NUMA node %d", name, socket_id);
1421
1422 /* Allocate device structure. */
1423 rawdev = rte_rawdev_pmd_allocate(name, sizeof(struct ntb_hw),
1424 socket_id);
1425 if (rawdev == NULL) {
1426 NTB_LOG(ERR, "Unable to allocate rawdev.");
1427 return -EINVAL;
1428 }
1429
1430 rawdev->dev_ops = &ntb_ops;
1431 rawdev->device = &pci_dev->device;
1432 rawdev->driver_name = pci_dev->driver->driver.name;
1433
1434 ret = ntb_init_hw(rawdev, pci_dev);
1435 if (ret < 0) {
1436 NTB_LOG(ERR, "Unable to init ntb hw.");
1437 goto fail;
1438 }
1439
1440 return ret;
1441
1442fail:
1443 if (rawdev != NULL)
1444 rte_rawdev_pmd_release(rawdev);
1445
1446 return ret;
1447}
1448
1449static int
1450ntb_destroy(struct rte_pci_device *pci_dev)
1451{
1452 char name[RTE_RAWDEV_NAME_MAX_LEN];
1453 struct rte_rawdev *rawdev;
1454 int ret;
1455
1456 if (pci_dev == NULL) {
1457 NTB_LOG(ERR, "Invalid pci_dev.");
1458 ret = -EINVAL;
1459 return ret;
1460 }
1461
1462 memset(name, 0, sizeof(name));
1463 snprintf(name, RTE_RAWDEV_NAME_MAX_LEN, "NTB:%x:%02x.%x",
1464 pci_dev->addr.bus, pci_dev->addr.devid,
1465 pci_dev->addr.function);
1466
1467 NTB_LOG(INFO, "Closing %s on NUMA node %d", name, rte_socket_id());
1468
1469 rawdev = rte_rawdev_pmd_get_named_dev(name);
1470 if (rawdev == NULL) {
1471 NTB_LOG(ERR, "Invalid device name (%s)", name);
1472 ret = -EINVAL;
1473 return ret;
1474 }
1475
1476 ret = rte_rawdev_pmd_release(rawdev);
1477 if (ret)
1478 NTB_LOG(ERR, "Failed to destroy ntb rawdev.");
1479
1480 return ret;
1481}
1482
1483static int
1484ntb_probe(struct rte_pci_driver *pci_drv __rte_unused,
1485 struct rte_pci_device *pci_dev)
1486{
1487 return ntb_create(pci_dev, rte_socket_id());
1488}
1489
1490static int
1491ntb_remove(struct rte_pci_device *pci_dev)
1492{
1493 return ntb_destroy(pci_dev);
1494}
1495
1496
1497static struct rte_pci_driver rte_ntb_pmd = {
1498 .id_table = pci_id_ntb_map,
1499 .drv_flags = RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_WC_ACTIVATE,
1500 .probe = ntb_probe,
1501 .remove = ntb_remove,
1502};
1503
1504RTE_PMD_REGISTER_PCI(raw_ntb, rte_ntb_pmd);
1505RTE_PMD_REGISTER_PCI_TABLE(raw_ntb, pci_id_ntb_map);
1506RTE_PMD_REGISTER_KMOD_DEP(raw_ntb, "* igb_uio | uio_pci_generic | vfio-pci");
1507
1508RTE_INIT(ntb_init_log)
1509{
1510 ntb_logtype = rte_log_register("pmd.raw.ntb");
1511 if (ntb_logtype >= 0)
1512 rte_log_set_level(ntb_logtype, RTE_LOG_INFO);
1513}
Ceph