1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016 Cavium, Inc
14 #include <netinet/in.h>
15 #include <sys/queue.h>
17 #include <rte_alarm.h>
18 #include <rte_branch_prediction.h>
19 #include <rte_byteorder.h>
20 #include <rte_common.h>
21 #include <rte_cycles.h>
22 #include <rte_debug.h>
25 #include <rte_ether.h>
26 #include <rte_ethdev_driver.h>
27 #include <rte_ethdev_pci.h>
28 #include <rte_interrupts.h>
30 #include <rte_memory.h>
31 #include <rte_memzone.h>
32 #include <rte_malloc.h>
33 #include <rte_random.h>
35 #include <rte_bus_pci.h>
36 #include <rte_tailq.h>
37 #include <rte_devargs.h>
38 #include <rte_kvargs.h>
40 #include "base/nicvf_plat.h"
42 #include "nicvf_ethdev.h"
43 #include "nicvf_rxtx.h"
44 #include "nicvf_svf.h"
45 #include "nicvf_logs.h"
47 int nicvf_logtype_mbox
;
48 int nicvf_logtype_init
;
49 int nicvf_logtype_driver
;
51 static void nicvf_dev_stop(struct rte_eth_dev
*dev
);
52 static void nicvf_dev_stop_cleanup(struct rte_eth_dev
*dev
, bool cleanup
);
53 static void nicvf_vf_stop(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
55 static int nicvf_vlan_offload_config(struct rte_eth_dev
*dev
, int mask
);
56 static int nicvf_vlan_offload_set(struct rte_eth_dev
*dev
, int mask
);
58 RTE_INIT(nicvf_init_log
)
60 nicvf_logtype_mbox
= rte_log_register("pmd.net.thunderx.mbox");
61 if (nicvf_logtype_mbox
>= 0)
62 rte_log_set_level(nicvf_logtype_mbox
, RTE_LOG_NOTICE
);
64 nicvf_logtype_init
= rte_log_register("pmd.net.thunderx.init");
65 if (nicvf_logtype_init
>= 0)
66 rte_log_set_level(nicvf_logtype_init
, RTE_LOG_NOTICE
);
68 nicvf_logtype_driver
= rte_log_register("pmd.net.thunderx.driver");
69 if (nicvf_logtype_driver
>= 0)
70 rte_log_set_level(nicvf_logtype_driver
, RTE_LOG_NOTICE
);
74 nicvf_link_status_update(struct nicvf
*nic
,
75 struct rte_eth_link
*link
)
77 memset(link
, 0, sizeof(*link
));
79 link
->link_status
= nic
->link_up
? ETH_LINK_UP
: ETH_LINK_DOWN
;
81 if (nic
->duplex
== NICVF_HALF_DUPLEX
)
82 link
->link_duplex
= ETH_LINK_HALF_DUPLEX
;
83 else if (nic
->duplex
== NICVF_FULL_DUPLEX
)
84 link
->link_duplex
= ETH_LINK_FULL_DUPLEX
;
85 link
->link_speed
= nic
->speed
;
86 link
->link_autoneg
= ETH_LINK_AUTONEG
;
90 nicvf_interrupt(void *arg
)
92 struct rte_eth_dev
*dev
= arg
;
93 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
94 struct rte_eth_link link
;
96 if (nicvf_reg_poll_interrupts(nic
) == NIC_MBOX_MSG_BGX_LINK_CHANGE
) {
97 if (dev
->data
->dev_conf
.intr_conf
.lsc
) {
98 nicvf_link_status_update(nic
, &link
);
99 rte_eth_linkstatus_set(dev
, &link
);
101 _rte_eth_dev_callback_process(dev
,
102 RTE_ETH_EVENT_INTR_LSC
,
107 rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS
* 1000,
108 nicvf_interrupt
, dev
);
112 nicvf_vf_interrupt(void *arg
)
114 struct nicvf
*nic
= arg
;
116 nicvf_reg_poll_interrupts(nic
);
118 rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS
* 1000,
119 nicvf_vf_interrupt
, nic
);
123 nicvf_periodic_alarm_start(void (fn
)(void *), void *arg
)
125 return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS
* 1000, fn
, arg
);
129 nicvf_periodic_alarm_stop(void (fn
)(void *), void *arg
)
131 return rte_eal_alarm_cancel(fn
, arg
);
135 * Return 0 means link status changed, -1 means not changed
138 nicvf_dev_link_update(struct rte_eth_dev
*dev
, int wait_to_complete
)
140 #define CHECK_INTERVAL 100 /* 100ms */
141 #define MAX_CHECK_TIME 90 /* 9s (90 * 100ms) in total */
142 struct rte_eth_link link
;
143 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
146 PMD_INIT_FUNC_TRACE();
148 if (wait_to_complete
) {
149 /* rte_eth_link_get() might need to wait up to 9 seconds */
150 for (i
= 0; i
< MAX_CHECK_TIME
; i
++) {
151 nicvf_link_status_update(nic
, &link
);
152 if (link
.link_status
== ETH_LINK_UP
)
154 rte_delay_ms(CHECK_INTERVAL
);
157 nicvf_link_status_update(nic
, &link
);
160 return rte_eth_linkstatus_set(dev
, &link
);
164 nicvf_dev_set_mtu(struct rte_eth_dev
*dev
, uint16_t mtu
)
166 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
167 uint32_t buffsz
, frame_size
= mtu
+ NIC_HW_L2_OVERHEAD
;
169 struct rte_eth_rxmode
*rxmode
= &dev
->data
->dev_conf
.rxmode
;
171 PMD_INIT_FUNC_TRACE();
173 if (frame_size
> NIC_HW_MAX_FRS
)
176 if (frame_size
< NIC_HW_MIN_FRS
)
179 buffsz
= dev
->data
->min_rx_buf_size
- RTE_PKTMBUF_HEADROOM
;
182 * Refuse mtu that requires the support of scattered packets
183 * when this feature has not been enabled before.
185 if (dev
->data
->dev_started
&& !dev
->data
->scattered_rx
&&
186 (frame_size
+ 2 * VLAN_TAG_SIZE
> buffsz
))
189 /* check <seg size> * <max_seg> >= max_frame */
190 if (dev
->data
->scattered_rx
&&
191 (frame_size
+ 2 * VLAN_TAG_SIZE
> buffsz
* NIC_HW_MAX_SEGS
))
194 if (frame_size
> ETHER_MAX_LEN
)
195 rxmode
->offloads
|= DEV_RX_OFFLOAD_JUMBO_FRAME
;
197 rxmode
->offloads
&= ~DEV_RX_OFFLOAD_JUMBO_FRAME
;
199 if (nicvf_mbox_update_hw_max_frs(nic
, mtu
))
202 /* Update max_rx_pkt_len */
203 rxmode
->max_rx_pkt_len
= mtu
+ ETHER_HDR_LEN
;
206 for (i
= 0; i
< nic
->sqs_count
; i
++)
207 nic
->snicvf
[i
]->mtu
= mtu
;
213 nicvf_dev_get_regs(struct rte_eth_dev
*dev
, struct rte_dev_reg_info
*regs
)
215 uint64_t *data
= regs
->data
;
216 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
219 regs
->length
= nicvf_reg_get_count();
220 regs
->width
= THUNDERX_REG_BYTES
;
224 /* Support only full register dump */
225 if ((regs
->length
== 0) ||
226 (regs
->length
== (uint32_t)nicvf_reg_get_count())) {
227 regs
->version
= nic
->vendor_id
<< 16 | nic
->device_id
;
228 nicvf_reg_dump(nic
, data
);
235 nicvf_dev_stats_get(struct rte_eth_dev
*dev
, struct rte_eth_stats
*stats
)
238 struct nicvf_hw_rx_qstats rx_qstats
;
239 struct nicvf_hw_tx_qstats tx_qstats
;
240 struct nicvf_hw_stats port_stats
;
241 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
242 uint16_t rx_start
, rx_end
;
243 uint16_t tx_start
, tx_end
;
246 /* RX queue indices for the first VF */
247 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
249 /* Reading per RX ring stats */
250 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
251 if (qidx
>= RTE_ETHDEV_QUEUE_STAT_CNTRS
)
254 nicvf_hw_get_rx_qstats(nic
, &rx_qstats
, qidx
);
255 stats
->q_ibytes
[qidx
] = rx_qstats
.q_rx_bytes
;
256 stats
->q_ipackets
[qidx
] = rx_qstats
.q_rx_packets
;
259 /* TX queue indices for the first VF */
260 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
262 /* Reading per TX ring stats */
263 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++) {
264 if (qidx
>= RTE_ETHDEV_QUEUE_STAT_CNTRS
)
267 nicvf_hw_get_tx_qstats(nic
, &tx_qstats
, qidx
);
268 stats
->q_obytes
[qidx
] = tx_qstats
.q_tx_bytes
;
269 stats
->q_opackets
[qidx
] = tx_qstats
.q_tx_packets
;
272 for (i
= 0; i
< nic
->sqs_count
; i
++) {
273 struct nicvf
*snic
= nic
->snicvf
[i
];
278 /* RX queue indices for a secondary VF */
279 nicvf_rx_range(dev
, snic
, &rx_start
, &rx_end
);
281 /* Reading per RX ring stats */
282 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
283 if (qidx
>= RTE_ETHDEV_QUEUE_STAT_CNTRS
)
286 nicvf_hw_get_rx_qstats(snic
, &rx_qstats
,
287 qidx
% MAX_RCV_QUEUES_PER_QS
);
288 stats
->q_ibytes
[qidx
] = rx_qstats
.q_rx_bytes
;
289 stats
->q_ipackets
[qidx
] = rx_qstats
.q_rx_packets
;
292 /* TX queue indices for a secondary VF */
293 nicvf_tx_range(dev
, snic
, &tx_start
, &tx_end
);
294 /* Reading per TX ring stats */
295 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++) {
296 if (qidx
>= RTE_ETHDEV_QUEUE_STAT_CNTRS
)
299 nicvf_hw_get_tx_qstats(snic
, &tx_qstats
,
300 qidx
% MAX_SND_QUEUES_PER_QS
);
301 stats
->q_obytes
[qidx
] = tx_qstats
.q_tx_bytes
;
302 stats
->q_opackets
[qidx
] = tx_qstats
.q_tx_packets
;
306 nicvf_hw_get_stats(nic
, &port_stats
);
307 stats
->ibytes
= port_stats
.rx_bytes
;
308 stats
->ipackets
= port_stats
.rx_ucast_frames
;
309 stats
->ipackets
+= port_stats
.rx_bcast_frames
;
310 stats
->ipackets
+= port_stats
.rx_mcast_frames
;
311 stats
->ierrors
= port_stats
.rx_l2_errors
;
312 stats
->imissed
= port_stats
.rx_drop_red
;
313 stats
->imissed
+= port_stats
.rx_drop_overrun
;
314 stats
->imissed
+= port_stats
.rx_drop_bcast
;
315 stats
->imissed
+= port_stats
.rx_drop_mcast
;
316 stats
->imissed
+= port_stats
.rx_drop_l3_bcast
;
317 stats
->imissed
+= port_stats
.rx_drop_l3_mcast
;
319 stats
->obytes
= port_stats
.tx_bytes_ok
;
320 stats
->opackets
= port_stats
.tx_ucast_frames_ok
;
321 stats
->opackets
+= port_stats
.tx_bcast_frames_ok
;
322 stats
->opackets
+= port_stats
.tx_mcast_frames_ok
;
323 stats
->oerrors
= port_stats
.tx_drops
;
328 static const uint32_t *
329 nicvf_dev_supported_ptypes_get(struct rte_eth_dev
*dev
)
332 static uint32_t ptypes
[32];
333 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
334 static const uint32_t ptypes_common
[] = {
336 RTE_PTYPE_L3_IPV4_EXT
,
338 RTE_PTYPE_L3_IPV6_EXT
,
343 static const uint32_t ptypes_tunnel
[] = {
344 RTE_PTYPE_TUNNEL_GRE
,
345 RTE_PTYPE_TUNNEL_GENEVE
,
346 RTE_PTYPE_TUNNEL_VXLAN
,
347 RTE_PTYPE_TUNNEL_NVGRE
,
349 static const uint32_t ptypes_end
= RTE_PTYPE_UNKNOWN
;
351 copied
= sizeof(ptypes_common
);
352 memcpy(ptypes
, ptypes_common
, copied
);
353 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
) {
354 memcpy((char *)ptypes
+ copied
, ptypes_tunnel
,
355 sizeof(ptypes_tunnel
));
356 copied
+= sizeof(ptypes_tunnel
);
359 memcpy((char *)ptypes
+ copied
, &ptypes_end
, sizeof(ptypes_end
));
361 /* All Ptypes are supported in all Rx functions. */
366 nicvf_dev_stats_reset(struct rte_eth_dev
*dev
)
369 uint16_t rxqs
= 0, txqs
= 0;
370 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
371 uint16_t rx_start
, rx_end
;
372 uint16_t tx_start
, tx_end
;
374 /* Reset all primary nic counters */
375 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
376 for (i
= rx_start
; i
<= rx_end
; i
++)
377 rxqs
|= (0x3 << (i
* 2));
379 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
380 for (i
= tx_start
; i
<= tx_end
; i
++)
381 txqs
|= (0x3 << (i
* 2));
383 nicvf_mbox_reset_stat_counters(nic
, 0x3FFF, 0x1F, rxqs
, txqs
);
385 /* Reset secondary nic queue counters */
386 for (i
= 0; i
< nic
->sqs_count
; i
++) {
387 struct nicvf
*snic
= nic
->snicvf
[i
];
391 nicvf_rx_range(dev
, snic
, &rx_start
, &rx_end
);
392 for (i
= rx_start
; i
<= rx_end
; i
++)
393 rxqs
|= (0x3 << ((i
% MAX_CMP_QUEUES_PER_QS
) * 2));
395 nicvf_tx_range(dev
, snic
, &tx_start
, &tx_end
);
396 for (i
= tx_start
; i
<= tx_end
; i
++)
397 txqs
|= (0x3 << ((i
% MAX_SND_QUEUES_PER_QS
) * 2));
399 nicvf_mbox_reset_stat_counters(snic
, 0, 0, rxqs
, txqs
);
403 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
405 nicvf_dev_promisc_enable(struct rte_eth_dev
*dev __rte_unused
)
409 static inline uint64_t
410 nicvf_rss_ethdev_to_nic(struct nicvf
*nic
, uint64_t ethdev_rss
)
412 uint64_t nic_rss
= 0;
414 if (ethdev_rss
& ETH_RSS_IPV4
)
415 nic_rss
|= RSS_IP_ENA
;
417 if (ethdev_rss
& ETH_RSS_IPV6
)
418 nic_rss
|= RSS_IP_ENA
;
420 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV4_UDP
)
421 nic_rss
|= (RSS_IP_ENA
| RSS_UDP_ENA
);
423 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV4_TCP
)
424 nic_rss
|= (RSS_IP_ENA
| RSS_TCP_ENA
);
426 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV6_UDP
)
427 nic_rss
|= (RSS_IP_ENA
| RSS_UDP_ENA
);
429 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV6_TCP
)
430 nic_rss
|= (RSS_IP_ENA
| RSS_TCP_ENA
);
432 if (ethdev_rss
& ETH_RSS_PORT
)
433 nic_rss
|= RSS_L2_EXTENDED_HASH_ENA
;
435 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
) {
436 if (ethdev_rss
& ETH_RSS_VXLAN
)
437 nic_rss
|= RSS_TUN_VXLAN_ENA
;
439 if (ethdev_rss
& ETH_RSS_GENEVE
)
440 nic_rss
|= RSS_TUN_GENEVE_ENA
;
442 if (ethdev_rss
& ETH_RSS_NVGRE
)
443 nic_rss
|= RSS_TUN_NVGRE_ENA
;
449 static inline uint64_t
450 nicvf_rss_nic_to_ethdev(struct nicvf
*nic
, uint64_t nic_rss
)
452 uint64_t ethdev_rss
= 0;
454 if (nic_rss
& RSS_IP_ENA
)
455 ethdev_rss
|= (ETH_RSS_IPV4
| ETH_RSS_IPV6
);
457 if ((nic_rss
& RSS_IP_ENA
) && (nic_rss
& RSS_TCP_ENA
))
458 ethdev_rss
|= (ETH_RSS_NONFRAG_IPV4_TCP
|
459 ETH_RSS_NONFRAG_IPV6_TCP
);
461 if ((nic_rss
& RSS_IP_ENA
) && (nic_rss
& RSS_UDP_ENA
))
462 ethdev_rss
|= (ETH_RSS_NONFRAG_IPV4_UDP
|
463 ETH_RSS_NONFRAG_IPV6_UDP
);
465 if (nic_rss
& RSS_L2_EXTENDED_HASH_ENA
)
466 ethdev_rss
|= ETH_RSS_PORT
;
468 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
) {
469 if (nic_rss
& RSS_TUN_VXLAN_ENA
)
470 ethdev_rss
|= ETH_RSS_VXLAN
;
472 if (nic_rss
& RSS_TUN_GENEVE_ENA
)
473 ethdev_rss
|= ETH_RSS_GENEVE
;
475 if (nic_rss
& RSS_TUN_NVGRE_ENA
)
476 ethdev_rss
|= ETH_RSS_NVGRE
;
482 nicvf_dev_reta_query(struct rte_eth_dev
*dev
,
483 struct rte_eth_rss_reta_entry64
*reta_conf
,
486 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
487 uint8_t tbl
[NIC_MAX_RSS_IDR_TBL_SIZE
];
490 if (reta_size
!= NIC_MAX_RSS_IDR_TBL_SIZE
) {
491 RTE_LOG(ERR
, PMD
, "The size of hash lookup table configured "
492 "(%d) doesn't match the number hardware can supported "
493 "(%d)", reta_size
, NIC_MAX_RSS_IDR_TBL_SIZE
);
497 ret
= nicvf_rss_reta_query(nic
, tbl
, NIC_MAX_RSS_IDR_TBL_SIZE
);
501 /* Copy RETA table */
502 for (i
= 0; i
< (NIC_MAX_RSS_IDR_TBL_SIZE
/ RTE_RETA_GROUP_SIZE
); i
++) {
503 for (j
= 0; j
< RTE_RETA_GROUP_SIZE
; j
++)
504 if ((reta_conf
[i
].mask
>> j
) & 0x01)
505 reta_conf
[i
].reta
[j
] = tbl
[j
];
512 nicvf_dev_reta_update(struct rte_eth_dev
*dev
,
513 struct rte_eth_rss_reta_entry64
*reta_conf
,
516 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
517 uint8_t tbl
[NIC_MAX_RSS_IDR_TBL_SIZE
];
520 if (reta_size
!= NIC_MAX_RSS_IDR_TBL_SIZE
) {
521 RTE_LOG(ERR
, PMD
, "The size of hash lookup table configured "
522 "(%d) doesn't match the number hardware can supported "
523 "(%d)", reta_size
, NIC_MAX_RSS_IDR_TBL_SIZE
);
527 ret
= nicvf_rss_reta_query(nic
, tbl
, NIC_MAX_RSS_IDR_TBL_SIZE
);
531 /* Copy RETA table */
532 for (i
= 0; i
< (NIC_MAX_RSS_IDR_TBL_SIZE
/ RTE_RETA_GROUP_SIZE
); i
++) {
533 for (j
= 0; j
< RTE_RETA_GROUP_SIZE
; j
++)
534 if ((reta_conf
[i
].mask
>> j
) & 0x01)
535 tbl
[j
] = reta_conf
[i
].reta
[j
];
538 return nicvf_rss_reta_update(nic
, tbl
, NIC_MAX_RSS_IDR_TBL_SIZE
);
542 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev
*dev
,
543 struct rte_eth_rss_conf
*rss_conf
)
545 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
547 if (rss_conf
->rss_key
)
548 nicvf_rss_get_key(nic
, rss_conf
->rss_key
);
550 rss_conf
->rss_key_len
= RSS_HASH_KEY_BYTE_SIZE
;
551 rss_conf
->rss_hf
= nicvf_rss_nic_to_ethdev(nic
, nicvf_rss_get_cfg(nic
));
556 nicvf_dev_rss_hash_update(struct rte_eth_dev
*dev
,
557 struct rte_eth_rss_conf
*rss_conf
)
559 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
562 if (rss_conf
->rss_key
&&
563 rss_conf
->rss_key_len
!= RSS_HASH_KEY_BYTE_SIZE
) {
564 RTE_LOG(ERR
, PMD
, "Hash key size mismatch %d",
565 rss_conf
->rss_key_len
);
569 if (rss_conf
->rss_key
)
570 nicvf_rss_set_key(nic
, rss_conf
->rss_key
);
572 nic_rss
= nicvf_rss_ethdev_to_nic(nic
, rss_conf
->rss_hf
);
573 nicvf_rss_set_cfg(nic
, nic_rss
);
578 nicvf_qset_cq_alloc(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
579 struct nicvf_rxq
*rxq
, uint16_t qidx
, uint32_t desc_cnt
)
581 const struct rte_memzone
*rz
;
582 uint32_t ring_size
= CMP_QUEUE_SZ_MAX
* sizeof(union cq_entry_t
);
584 rz
= rte_eth_dma_zone_reserve(dev
, "cq_ring",
585 nicvf_netdev_qidx(nic
, qidx
), ring_size
,
586 NICVF_CQ_BASE_ALIGN_BYTES
, nic
->node
);
588 PMD_INIT_LOG(ERR
, "Failed to allocate mem for cq hw ring");
592 memset(rz
->addr
, 0, ring_size
);
594 rxq
->phys
= rz
->iova
;
595 rxq
->desc
= rz
->addr
;
596 rxq
->qlen_mask
= desc_cnt
- 1;
602 nicvf_qset_sq_alloc(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
603 struct nicvf_txq
*sq
, uint16_t qidx
, uint32_t desc_cnt
)
605 const struct rte_memzone
*rz
;
606 uint32_t ring_size
= SND_QUEUE_SZ_MAX
* sizeof(union sq_entry_t
);
608 rz
= rte_eth_dma_zone_reserve(dev
, "sq",
609 nicvf_netdev_qidx(nic
, qidx
), ring_size
,
610 NICVF_SQ_BASE_ALIGN_BYTES
, nic
->node
);
612 PMD_INIT_LOG(ERR
, "Failed allocate mem for sq hw ring");
616 memset(rz
->addr
, 0, ring_size
);
620 sq
->qlen_mask
= desc_cnt
- 1;
626 nicvf_qset_rbdr_alloc(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
627 uint32_t desc_cnt
, uint32_t buffsz
)
629 struct nicvf_rbdr
*rbdr
;
630 const struct rte_memzone
*rz
;
633 assert(nic
->rbdr
== NULL
);
634 rbdr
= rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr
),
635 RTE_CACHE_LINE_SIZE
, nic
->node
);
637 PMD_INIT_LOG(ERR
, "Failed to allocate mem for rbdr");
641 ring_size
= sizeof(struct rbdr_entry_t
) * RBDR_QUEUE_SZ_MAX
;
642 rz
= rte_eth_dma_zone_reserve(dev
, "rbdr",
643 nicvf_netdev_qidx(nic
, 0), ring_size
,
644 NICVF_RBDR_BASE_ALIGN_BYTES
, nic
->node
);
646 PMD_INIT_LOG(ERR
, "Failed to allocate mem for rbdr desc ring");
650 memset(rz
->addr
, 0, ring_size
);
652 rbdr
->phys
= rz
->iova
;
655 rbdr
->desc
= rz
->addr
;
656 rbdr
->buffsz
= buffsz
;
657 rbdr
->qlen_mask
= desc_cnt
- 1;
659 nicvf_qset_base(nic
, 0) + NIC_QSET_RBDR_0_1_STATUS0
;
661 nicvf_qset_base(nic
, 0) + NIC_QSET_RBDR_0_1_DOOR
;
668 nicvf_rbdr_release_mbuf(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
669 nicvf_iova_addr_t phy
)
673 struct nicvf_rxq
*rxq
;
674 uint16_t rx_start
, rx_end
;
676 /* Get queue ranges for this VF */
677 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
679 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
680 rxq
= dev
->data
->rx_queues
[qidx
];
681 if (rxq
->precharge_cnt
) {
682 obj
= (void *)nicvf_mbuff_phy2virt(phy
,
684 rte_mempool_put(rxq
->pool
, obj
);
685 rxq
->precharge_cnt
--;
692 nicvf_rbdr_release_mbufs(struct rte_eth_dev
*dev
, struct nicvf
*nic
)
694 uint32_t qlen_mask
, head
;
695 struct rbdr_entry_t
*entry
;
696 struct nicvf_rbdr
*rbdr
= nic
->rbdr
;
698 qlen_mask
= rbdr
->qlen_mask
;
700 while (head
!= rbdr
->tail
) {
701 entry
= rbdr
->desc
+ head
;
702 nicvf_rbdr_release_mbuf(dev
, nic
, entry
->full_addr
);
704 head
= head
& qlen_mask
;
709 nicvf_tx_queue_release_mbufs(struct nicvf_txq
*txq
)
714 while (head
!= txq
->tail
) {
715 if (txq
->txbuffs
[head
]) {
716 rte_pktmbuf_free_seg(txq
->txbuffs
[head
]);
717 txq
->txbuffs
[head
] = NULL
;
720 head
= head
& txq
->qlen_mask
;
725 nicvf_tx_queue_reset(struct nicvf_txq
*txq
)
727 uint32_t txq_desc_cnt
= txq
->qlen_mask
+ 1;
729 memset(txq
->desc
, 0, sizeof(union sq_entry_t
) * txq_desc_cnt
);
730 memset(txq
->txbuffs
, 0, sizeof(struct rte_mbuf
*) * txq_desc_cnt
);
737 nicvf_vf_start_tx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
740 struct nicvf_txq
*txq
;
743 assert(qidx
< MAX_SND_QUEUES_PER_QS
);
745 if (dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
746 RTE_ETH_QUEUE_STATE_STARTED
)
749 txq
= dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
751 ret
= nicvf_qset_sq_config(nic
, qidx
, txq
);
753 PMD_INIT_LOG(ERR
, "Failed to configure sq VF%d %d %d",
754 nic
->vf_id
, qidx
, ret
);
755 goto config_sq_error
;
758 dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
759 RTE_ETH_QUEUE_STATE_STARTED
;
763 nicvf_qset_sq_reclaim(nic
, qidx
);
768 nicvf_vf_stop_tx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
771 struct nicvf_txq
*txq
;
774 assert(qidx
< MAX_SND_QUEUES_PER_QS
);
776 if (dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
777 RTE_ETH_QUEUE_STATE_STOPPED
)
780 ret
= nicvf_qset_sq_reclaim(nic
, qidx
);
782 PMD_INIT_LOG(ERR
, "Failed to reclaim sq VF%d %d %d",
783 nic
->vf_id
, qidx
, ret
);
785 txq
= dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
786 nicvf_tx_queue_release_mbufs(txq
);
787 nicvf_tx_queue_reset(txq
);
789 dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
790 RTE_ETH_QUEUE_STATE_STOPPED
;
795 nicvf_configure_cpi(struct rte_eth_dev
*dev
)
797 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
801 /* Count started rx queues */
802 for (qidx
= qcnt
= 0; qidx
< dev
->data
->nb_rx_queues
; qidx
++)
803 if (dev
->data
->rx_queue_state
[qidx
] ==
804 RTE_ETH_QUEUE_STATE_STARTED
)
807 nic
->cpi_alg
= CPI_ALG_NONE
;
808 ret
= nicvf_mbox_config_cpi(nic
, qcnt
);
810 PMD_INIT_LOG(ERR
, "Failed to configure CPI %d", ret
);
816 nicvf_configure_rss(struct rte_eth_dev
*dev
)
818 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
822 rsshf
= nicvf_rss_ethdev_to_nic(nic
,
823 dev
->data
->dev_conf
.rx_adv_conf
.rss_conf
.rss_hf
);
824 PMD_DRV_LOG(INFO
, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64
,
825 dev
->data
->dev_conf
.rxmode
.mq_mode
,
826 dev
->data
->nb_rx_queues
,
827 dev
->data
->dev_conf
.lpbk_mode
, rsshf
);
829 if (dev
->data
->dev_conf
.rxmode
.mq_mode
== ETH_MQ_RX_NONE
)
830 ret
= nicvf_rss_term(nic
);
831 else if (dev
->data
->dev_conf
.rxmode
.mq_mode
== ETH_MQ_RX_RSS
)
832 ret
= nicvf_rss_config(nic
, dev
->data
->nb_rx_queues
, rsshf
);
834 PMD_INIT_LOG(ERR
, "Failed to configure RSS %d", ret
);
840 nicvf_configure_rss_reta(struct rte_eth_dev
*dev
)
842 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
843 unsigned int idx
, qmap_size
;
844 uint8_t qmap
[RTE_MAX_QUEUES_PER_PORT
];
845 uint8_t default_reta
[NIC_MAX_RSS_IDR_TBL_SIZE
];
847 if (nic
->cpi_alg
!= CPI_ALG_NONE
)
850 /* Prepare queue map */
851 for (idx
= 0, qmap_size
= 0; idx
< dev
->data
->nb_rx_queues
; idx
++) {
852 if (dev
->data
->rx_queue_state
[idx
] ==
853 RTE_ETH_QUEUE_STATE_STARTED
)
854 qmap
[qmap_size
++] = idx
;
857 /* Update default RSS RETA */
858 for (idx
= 0; idx
< NIC_MAX_RSS_IDR_TBL_SIZE
; idx
++)
859 default_reta
[idx
] = qmap
[idx
% qmap_size
];
861 return nicvf_rss_reta_update(nic
, default_reta
,
862 NIC_MAX_RSS_IDR_TBL_SIZE
);
866 nicvf_dev_tx_queue_release(void *sq
)
868 struct nicvf_txq
*txq
;
870 PMD_INIT_FUNC_TRACE();
872 txq
= (struct nicvf_txq
*)sq
;
874 if (txq
->txbuffs
!= NULL
) {
875 nicvf_tx_queue_release_mbufs(txq
);
876 rte_free(txq
->txbuffs
);
884 nicvf_set_tx_function(struct rte_eth_dev
*dev
)
886 struct nicvf_txq
*txq
= NULL
;
888 bool multiseg
= false;
890 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
891 txq
= dev
->data
->tx_queues
[i
];
892 if (txq
->offloads
& DEV_TX_OFFLOAD_MULTI_SEGS
) {
898 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
900 PMD_DRV_LOG(DEBUG
, "Using multi-segment tx callback");
901 dev
->tx_pkt_burst
= nicvf_xmit_pkts_multiseg
;
903 PMD_DRV_LOG(DEBUG
, "Using single-segment tx callback");
904 dev
->tx_pkt_burst
= nicvf_xmit_pkts
;
910 if (txq
->pool_free
== nicvf_single_pool_free_xmited_buffers
)
911 PMD_DRV_LOG(DEBUG
, "Using single-mempool tx free method");
913 PMD_DRV_LOG(DEBUG
, "Using multi-mempool tx free method");
917 nicvf_set_rx_function(struct rte_eth_dev
*dev
)
919 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
921 const eth_rx_burst_t rx_burst_func
[2][2][2] = {
922 /* [NORMAL/SCATTER] [CKSUM/NO_CKSUM] [VLAN_STRIP/NO_VLAN_STRIP] */
923 [0][0][0] = nicvf_recv_pkts_no_offload
,
924 [0][0][1] = nicvf_recv_pkts_vlan_strip
,
925 [0][1][0] = nicvf_recv_pkts_cksum
,
926 [0][1][1] = nicvf_recv_pkts_cksum_vlan_strip
,
927 [1][0][0] = nicvf_recv_pkts_multiseg_no_offload
,
928 [1][0][1] = nicvf_recv_pkts_multiseg_vlan_strip
,
929 [1][1][0] = nicvf_recv_pkts_multiseg_cksum
,
930 [1][1][1] = nicvf_recv_pkts_multiseg_cksum_vlan_strip
,
934 rx_burst_func
[dev
->data
->scattered_rx
]
935 [nic
->offload_cksum
][nic
->vlan_strip
];
939 nicvf_dev_tx_queue_setup(struct rte_eth_dev
*dev
, uint16_t qidx
,
940 uint16_t nb_desc
, unsigned int socket_id
,
941 const struct rte_eth_txconf
*tx_conf
)
943 uint16_t tx_free_thresh
;
945 struct nicvf_txq
*txq
;
946 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
949 PMD_INIT_FUNC_TRACE();
951 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
952 nic
= nic
->snicvf
[qidx
/ MAX_SND_QUEUES_PER_QS
- 1];
954 qidx
= qidx
% MAX_SND_QUEUES_PER_QS
;
956 /* Socket id check */
957 if (socket_id
!= (unsigned int)SOCKET_ID_ANY
&& socket_id
!= nic
->node
)
958 PMD_DRV_LOG(WARNING
, "socket_id expected %d, configured %d",
959 socket_id
, nic
->node
);
961 /* Tx deferred start is not supported */
962 if (tx_conf
->tx_deferred_start
) {
963 PMD_INIT_LOG(ERR
, "Tx deferred start not supported");
967 /* Roundup nb_desc to available qsize and validate max number of desc */
968 nb_desc
= nicvf_qsize_sq_roundup(nb_desc
);
970 PMD_INIT_LOG(ERR
, "Value of nb_desc beyond available sq qsize");
974 /* Validate tx_free_thresh */
975 tx_free_thresh
= (uint16_t)((tx_conf
->tx_free_thresh
) ?
976 tx_conf
->tx_free_thresh
:
977 NICVF_DEFAULT_TX_FREE_THRESH
);
979 if (tx_free_thresh
> (nb_desc
) ||
980 tx_free_thresh
> NICVF_MAX_TX_FREE_THRESH
) {
982 "tx_free_thresh must be less than the number of TX "
983 "descriptors. (tx_free_thresh=%u port=%d "
984 "queue=%d)", (unsigned int)tx_free_thresh
,
985 (int)dev
->data
->port_id
, (int)qidx
);
989 /* Free memory prior to re-allocation if needed. */
990 if (dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)] != NULL
) {
991 PMD_TX_LOG(DEBUG
, "Freeing memory prior to re-allocation %d",
992 nicvf_netdev_qidx(nic
, qidx
));
993 nicvf_dev_tx_queue_release(
994 dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)]);
995 dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = NULL
;
998 /* Allocating tx queue data structure */
999 txq
= rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq
),
1000 RTE_CACHE_LINE_SIZE
, nic
->node
);
1002 PMD_INIT_LOG(ERR
, "Failed to allocate txq=%d",
1003 nicvf_netdev_qidx(nic
, qidx
));
1008 txq
->queue_id
= qidx
;
1009 txq
->tx_free_thresh
= tx_free_thresh
;
1010 txq
->sq_head
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_SQ_0_7_HEAD
;
1011 txq
->sq_door
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_SQ_0_7_DOOR
;
1012 offloads
= tx_conf
->offloads
| dev
->data
->dev_conf
.txmode
.offloads
;
1013 txq
->offloads
= offloads
;
1015 is_single_pool
= !!(offloads
& DEV_TX_OFFLOAD_MBUF_FAST_FREE
);
1017 /* Choose optimum free threshold value for multipool case */
1018 if (!is_single_pool
) {
1019 txq
->tx_free_thresh
= (uint16_t)
1020 (tx_conf
->tx_free_thresh
== NICVF_DEFAULT_TX_FREE_THRESH
?
1021 NICVF_TX_FREE_MPOOL_THRESH
:
1022 tx_conf
->tx_free_thresh
);
1023 txq
->pool_free
= nicvf_multi_pool_free_xmited_buffers
;
1025 txq
->pool_free
= nicvf_single_pool_free_xmited_buffers
;
1028 /* Allocate software ring */
1029 txq
->txbuffs
= rte_zmalloc_socket("txq->txbuffs",
1030 nb_desc
* sizeof(struct rte_mbuf
*),
1031 RTE_CACHE_LINE_SIZE
, nic
->node
);
1033 if (txq
->txbuffs
== NULL
) {
1034 nicvf_dev_tx_queue_release(txq
);
1038 if (nicvf_qset_sq_alloc(dev
, nic
, txq
, qidx
, nb_desc
)) {
1039 PMD_INIT_LOG(ERR
, "Failed to allocate mem for sq %d", qidx
);
1040 nicvf_dev_tx_queue_release(txq
);
1044 nicvf_tx_queue_reset(txq
);
1046 PMD_INIT_LOG(DEBUG
, "[%d] txq=%p nb_desc=%d desc=%p"
1047 " phys=0x%" PRIx64
" offloads=0x%" PRIx64
,
1048 nicvf_netdev_qidx(nic
, qidx
), txq
, nb_desc
, txq
->desc
,
1049 txq
->phys
, txq
->offloads
);
1051 dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = txq
;
1052 dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1053 RTE_ETH_QUEUE_STATE_STOPPED
;
1058 nicvf_rx_queue_release_mbufs(struct rte_eth_dev
*dev
, struct nicvf_rxq
*rxq
)
1061 uint32_t nb_pkts
, released_pkts
= 0;
1062 uint32_t refill_cnt
= 0;
1063 struct rte_mbuf
*rx_pkts
[NICVF_MAX_RX_FREE_THRESH
];
1065 if (dev
->rx_pkt_burst
== NULL
)
1068 while ((rxq_cnt
= nicvf_dev_rx_queue_count(dev
,
1069 nicvf_netdev_qidx(rxq
->nic
, rxq
->queue_id
)))) {
1070 nb_pkts
= dev
->rx_pkt_burst(rxq
, rx_pkts
,
1071 NICVF_MAX_RX_FREE_THRESH
);
1072 PMD_DRV_LOG(INFO
, "nb_pkts=%d rxq_cnt=%d", nb_pkts
, rxq_cnt
);
1074 rte_pktmbuf_free_seg(rx_pkts
[--nb_pkts
]);
1080 refill_cnt
+= nicvf_dev_rbdr_refill(dev
,
1081 nicvf_netdev_qidx(rxq
->nic
, rxq
->queue_id
));
1083 PMD_DRV_LOG(INFO
, "free_cnt=%d refill_cnt=%d",
1084 released_pkts
, refill_cnt
);
1088 nicvf_rx_queue_reset(struct nicvf_rxq
*rxq
)
1091 rxq
->available_space
= 0;
1092 rxq
->recv_buffers
= 0;
1096 nicvf_vf_start_rx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
1099 struct nicvf_rxq
*rxq
;
1102 assert(qidx
< MAX_RCV_QUEUES_PER_QS
);
1104 if (dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
1105 RTE_ETH_QUEUE_STATE_STARTED
)
1108 /* Update rbdr pointer to all rxq */
1109 rxq
= dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
1110 rxq
->shared_rbdr
= nic
->rbdr
;
1112 ret
= nicvf_qset_rq_config(nic
, qidx
, rxq
);
1114 PMD_INIT_LOG(ERR
, "Failed to configure rq VF%d %d %d",
1115 nic
->vf_id
, qidx
, ret
);
1116 goto config_rq_error
;
1118 ret
= nicvf_qset_cq_config(nic
, qidx
, rxq
);
1120 PMD_INIT_LOG(ERR
, "Failed to configure cq VF%d %d %d",
1121 nic
->vf_id
, qidx
, ret
);
1122 goto config_cq_error
;
1125 dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1126 RTE_ETH_QUEUE_STATE_STARTED
;
1130 nicvf_qset_cq_reclaim(nic
, qidx
);
1132 nicvf_qset_rq_reclaim(nic
, qidx
);
1137 nicvf_vf_stop_rx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
1140 struct nicvf_rxq
*rxq
;
1141 int ret
, other_error
;
1143 if (dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
1144 RTE_ETH_QUEUE_STATE_STOPPED
)
1147 ret
= nicvf_qset_rq_reclaim(nic
, qidx
);
1149 PMD_INIT_LOG(ERR
, "Failed to reclaim rq VF%d %d %d",
1150 nic
->vf_id
, qidx
, ret
);
1153 rxq
= dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
1154 nicvf_rx_queue_release_mbufs(dev
, rxq
);
1155 nicvf_rx_queue_reset(rxq
);
1157 ret
= nicvf_qset_cq_reclaim(nic
, qidx
);
1159 PMD_INIT_LOG(ERR
, "Failed to reclaim cq VF%d %d %d",
1160 nic
->vf_id
, qidx
, ret
);
1163 dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1164 RTE_ETH_QUEUE_STATE_STOPPED
;
1169 nicvf_dev_rx_queue_release(void *rx_queue
)
1171 PMD_INIT_FUNC_TRACE();
1177 nicvf_dev_rx_queue_start(struct rte_eth_dev
*dev
, uint16_t qidx
)
1179 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1182 if (qidx
>= MAX_RCV_QUEUES_PER_QS
)
1183 nic
= nic
->snicvf
[(qidx
/ MAX_RCV_QUEUES_PER_QS
- 1)];
1185 qidx
= qidx
% MAX_RCV_QUEUES_PER_QS
;
1187 ret
= nicvf_vf_start_rx_queue(dev
, nic
, qidx
);
1191 ret
= nicvf_configure_cpi(dev
);
1195 return nicvf_configure_rss_reta(dev
);
1199 nicvf_dev_rx_queue_stop(struct rte_eth_dev
*dev
, uint16_t qidx
)
1202 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1204 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
1205 nic
= nic
->snicvf
[(qidx
/ MAX_SND_QUEUES_PER_QS
- 1)];
1207 qidx
= qidx
% MAX_RCV_QUEUES_PER_QS
;
1209 ret
= nicvf_vf_stop_rx_queue(dev
, nic
, qidx
);
1210 ret
|= nicvf_configure_cpi(dev
);
1211 ret
|= nicvf_configure_rss_reta(dev
);
1216 nicvf_dev_tx_queue_start(struct rte_eth_dev
*dev
, uint16_t qidx
)
1218 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1220 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
1221 nic
= nic
->snicvf
[(qidx
/ MAX_SND_QUEUES_PER_QS
- 1)];
1223 qidx
= qidx
% MAX_SND_QUEUES_PER_QS
;
1225 return nicvf_vf_start_tx_queue(dev
, nic
, qidx
);
1229 nicvf_dev_tx_queue_stop(struct rte_eth_dev
*dev
, uint16_t qidx
)
1231 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1233 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
1234 nic
= nic
->snicvf
[(qidx
/ MAX_SND_QUEUES_PER_QS
- 1)];
1236 qidx
= qidx
% MAX_SND_QUEUES_PER_QS
;
1238 return nicvf_vf_stop_tx_queue(dev
, nic
, qidx
);
1242 nicvf_rxq_mbuf_setup(struct nicvf_rxq
*rxq
)
1245 struct rte_mbuf mb_def
;
1246 struct nicvf
*nic
= rxq
->nic
;
1248 RTE_BUILD_BUG_ON(sizeof(union mbuf_initializer
) != 8);
1249 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, data_off
) % 8 != 0);
1250 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, refcnt
) -
1251 offsetof(struct rte_mbuf
, data_off
) != 2);
1252 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, nb_segs
) -
1253 offsetof(struct rte_mbuf
, data_off
) != 4);
1254 RTE_BUILD_BUG_ON(offsetof(struct rte_mbuf
, port
) -
1255 offsetof(struct rte_mbuf
, data_off
) != 6);
1256 RTE_BUILD_BUG_ON(offsetof(struct nicvf_rxq
, rxq_fastpath_data_end
) -
1257 offsetof(struct nicvf_rxq
,
1258 rxq_fastpath_data_start
) > 128);
1260 mb_def
.data_off
= RTE_PKTMBUF_HEADROOM
+ (nic
->skip_bytes
);
1261 mb_def
.port
= rxq
->port_id
;
1262 rte_mbuf_refcnt_set(&mb_def
, 1);
1264 /* Prevent compiler reordering: rearm_data covers previous fields */
1265 rte_compiler_barrier();
1266 p
= (uintptr_t)&mb_def
.rearm_data
;
1267 rxq
->mbuf_initializer
.value
= *(uint64_t *)p
;
1271 nicvf_dev_rx_queue_setup(struct rte_eth_dev
*dev
, uint16_t qidx
,
1272 uint16_t nb_desc
, unsigned int socket_id
,
1273 const struct rte_eth_rxconf
*rx_conf
,
1274 struct rte_mempool
*mp
)
1276 uint16_t rx_free_thresh
;
1277 struct nicvf_rxq
*rxq
;
1278 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1281 struct rte_pktmbuf_pool_private
*mbp_priv
;
1283 PMD_INIT_FUNC_TRACE();
1285 /* First skip check */
1286 mbp_priv
= rte_mempool_get_priv(mp
);
1287 buffsz
= mbp_priv
->mbuf_data_room_size
- RTE_PKTMBUF_HEADROOM
;
1288 if (buffsz
< (uint32_t)(nic
->skip_bytes
)) {
1289 PMD_INIT_LOG(ERR
, "First skip is more than configured buffer size");
1293 if (qidx
>= MAX_RCV_QUEUES_PER_QS
)
1294 nic
= nic
->snicvf
[qidx
/ MAX_RCV_QUEUES_PER_QS
- 1];
1296 qidx
= qidx
% MAX_RCV_QUEUES_PER_QS
;
1298 /* Socket id check */
1299 if (socket_id
!= (unsigned int)SOCKET_ID_ANY
&& socket_id
!= nic
->node
)
1300 PMD_DRV_LOG(WARNING
, "socket_id expected %d, configured %d",
1301 socket_id
, nic
->node
);
1303 /* Mempool memory must be contiguous, so must be one memory segment*/
1304 if (mp
->nb_mem_chunks
!= 1) {
1305 PMD_INIT_LOG(ERR
, "Non-contiguous mempool, add more huge pages");
1309 /* Mempool memory must be physically contiguous */
1310 if (mp
->flags
& MEMPOOL_F_NO_IOVA_CONTIG
) {
1311 PMD_INIT_LOG(ERR
, "Mempool memory must be physically contiguous");
1315 /* Rx deferred start is not supported */
1316 if (rx_conf
->rx_deferred_start
) {
1317 PMD_INIT_LOG(ERR
, "Rx deferred start not supported");
1321 /* Roundup nb_desc to available qsize and validate max number of desc */
1322 nb_desc
= nicvf_qsize_cq_roundup(nb_desc
);
1324 PMD_INIT_LOG(ERR
, "Value nb_desc beyond available hw cq qsize");
1329 /* Check rx_free_thresh upper bound */
1330 rx_free_thresh
= (uint16_t)((rx_conf
->rx_free_thresh
) ?
1331 rx_conf
->rx_free_thresh
:
1332 NICVF_DEFAULT_RX_FREE_THRESH
);
1333 if (rx_free_thresh
> NICVF_MAX_RX_FREE_THRESH
||
1334 rx_free_thresh
>= nb_desc
* .75) {
1335 PMD_INIT_LOG(ERR
, "rx_free_thresh greater than expected %d",
1340 /* Free memory prior to re-allocation if needed */
1341 if (dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)] != NULL
) {
1342 PMD_RX_LOG(DEBUG
, "Freeing memory prior to re-allocation %d",
1343 nicvf_netdev_qidx(nic
, qidx
));
1344 nicvf_dev_rx_queue_release(
1345 dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)]);
1346 dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = NULL
;
1349 /* Allocate rxq memory */
1350 rxq
= rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq
),
1351 RTE_CACHE_LINE_SIZE
, nic
->node
);
1353 PMD_INIT_LOG(ERR
, "Failed to allocate rxq=%d",
1354 nicvf_netdev_qidx(nic
, qidx
));
1360 rxq
->queue_id
= qidx
;
1361 rxq
->port_id
= dev
->data
->port_id
;
1362 rxq
->rx_free_thresh
= rx_free_thresh
;
1363 rxq
->rx_drop_en
= rx_conf
->rx_drop_en
;
1364 rxq
->cq_status
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_CQ_0_7_STATUS
;
1365 rxq
->cq_door
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_CQ_0_7_DOOR
;
1366 rxq
->precharge_cnt
= 0;
1368 if (nicvf_hw_cap(nic
) & NICVF_CAP_CQE_RX2
)
1369 rxq
->rbptr_offset
= NICVF_CQE_RX2_RBPTR_WORD
;
1371 rxq
->rbptr_offset
= NICVF_CQE_RBPTR_WORD
;
1373 nicvf_rxq_mbuf_setup(rxq
);
1375 /* Alloc completion queue */
1376 if (nicvf_qset_cq_alloc(dev
, nic
, rxq
, rxq
->queue_id
, nb_desc
)) {
1377 PMD_INIT_LOG(ERR
, "failed to allocate cq %u", rxq
->queue_id
);
1378 nicvf_dev_rx_queue_release(rxq
);
1382 nicvf_rx_queue_reset(rxq
);
1384 offloads
= rx_conf
->offloads
| dev
->data
->dev_conf
.rxmode
.offloads
;
1385 PMD_INIT_LOG(DEBUG
, "[%d] rxq=%p pool=%s nb_desc=(%d/%d)"
1386 " phy=0x%" PRIx64
" offloads=0x%" PRIx64
,
1387 nicvf_netdev_qidx(nic
, qidx
), rxq
, mp
->name
, nb_desc
,
1388 rte_mempool_avail_count(mp
), rxq
->phys
, offloads
);
1390 dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = rxq
;
1391 dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1392 RTE_ETH_QUEUE_STATE_STOPPED
;
1397 nicvf_dev_info_get(struct rte_eth_dev
*dev
, struct rte_eth_dev_info
*dev_info
)
1399 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1400 struct rte_pci_device
*pci_dev
= RTE_ETH_DEV_TO_PCI(dev
);
1402 PMD_INIT_FUNC_TRACE();
1404 /* Autonegotiation may be disabled */
1405 dev_info
->speed_capa
= ETH_LINK_SPEED_FIXED
;
1406 dev_info
->speed_capa
|= ETH_LINK_SPEED_10M
| ETH_LINK_SPEED_100M
|
1407 ETH_LINK_SPEED_1G
| ETH_LINK_SPEED_10G
;
1408 if (nicvf_hw_version(nic
) != PCI_SUB_DEVICE_ID_CN81XX_NICVF
)
1409 dev_info
->speed_capa
|= ETH_LINK_SPEED_40G
;
1411 dev_info
->min_rx_bufsize
= ETHER_MIN_MTU
;
1412 dev_info
->max_rx_pktlen
= NIC_HW_MAX_MTU
+ ETHER_HDR_LEN
;
1413 dev_info
->max_rx_queues
=
1414 (uint16_t)MAX_RCV_QUEUES_PER_QS
* (MAX_SQS_PER_VF
+ 1);
1415 dev_info
->max_tx_queues
=
1416 (uint16_t)MAX_SND_QUEUES_PER_QS
* (MAX_SQS_PER_VF
+ 1);
1417 dev_info
->max_mac_addrs
= 1;
1418 dev_info
->max_vfs
= pci_dev
->max_vfs
;
1420 dev_info
->rx_offload_capa
= NICVF_RX_OFFLOAD_CAPA
;
1421 dev_info
->tx_offload_capa
= NICVF_TX_OFFLOAD_CAPA
;
1422 dev_info
->rx_queue_offload_capa
= NICVF_RX_OFFLOAD_CAPA
;
1423 dev_info
->tx_queue_offload_capa
= NICVF_TX_OFFLOAD_CAPA
;
1425 dev_info
->reta_size
= nic
->rss_info
.rss_size
;
1426 dev_info
->hash_key_size
= RSS_HASH_KEY_BYTE_SIZE
;
1427 dev_info
->flow_type_rss_offloads
= NICVF_RSS_OFFLOAD_PASS1
;
1428 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
)
1429 dev_info
->flow_type_rss_offloads
|= NICVF_RSS_OFFLOAD_TUNNEL
;
1431 dev_info
->default_rxconf
= (struct rte_eth_rxconf
) {
1432 .rx_free_thresh
= NICVF_DEFAULT_RX_FREE_THRESH
,
1436 dev_info
->default_txconf
= (struct rte_eth_txconf
) {
1437 .tx_free_thresh
= NICVF_DEFAULT_TX_FREE_THRESH
,
1438 .offloads
= DEV_TX_OFFLOAD_MBUF_FAST_FREE
|
1439 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
|
1440 DEV_TX_OFFLOAD_UDP_CKSUM
|
1441 DEV_TX_OFFLOAD_TCP_CKSUM
,
1445 static nicvf_iova_addr_t
1446 rbdr_rte_mempool_get(void *dev
, void *opaque
)
1450 struct nicvf_rxq
*rxq
;
1451 struct rte_eth_dev
*eth_dev
= (struct rte_eth_dev
*)dev
;
1452 struct nicvf
*nic
= (struct nicvf
*)opaque
;
1453 uint16_t rx_start
, rx_end
;
1455 /* Get queue ranges for this VF */
1456 nicvf_rx_range(eth_dev
, nic
, &rx_start
, &rx_end
);
1458 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1459 rxq
= eth_dev
->data
->rx_queues
[qidx
];
1460 /* Maintain equal buffer count across all pools */
1461 if (rxq
->precharge_cnt
>= rxq
->qlen_mask
)
1463 rxq
->precharge_cnt
++;
1464 mbuf
= (uintptr_t)rte_pktmbuf_alloc(rxq
->pool
);
1466 return nicvf_mbuff_virt2phy(mbuf
, rxq
->mbuf_phys_off
);
1472 nicvf_vf_start(struct rte_eth_dev
*dev
, struct nicvf
*nic
, uint32_t rbdrsz
)
1475 uint16_t qidx
, data_off
;
1476 uint32_t total_rxq_desc
, nb_rbdr_desc
, exp_buffs
;
1477 uint64_t mbuf_phys_off
= 0;
1478 struct nicvf_rxq
*rxq
;
1479 struct rte_mbuf
*mbuf
;
1480 uint16_t rx_start
, rx_end
;
1481 uint16_t tx_start
, tx_end
;
1484 PMD_INIT_FUNC_TRACE();
1486 /* Userspace process exited without proper shutdown in last run */
1487 if (nicvf_qset_rbdr_active(nic
, 0))
1488 nicvf_vf_stop(dev
, nic
, false);
1490 /* Get queue ranges for this VF */
1491 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
1494 * Thunderx nicvf PMD can support more than one pool per port only when
1495 * 1) Data payload size is same across all the pools in given port
1497 * 2) All mbuffs in the pools are from the same hugepage
1499 * 3) Mbuff metadata size is same across all the pools in given port
1501 * This is to support existing application that uses multiple pool/port.
1502 * But, the purpose of using multipool for QoS will not be addressed.
1506 /* Validate mempool attributes */
1507 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1508 rxq
= dev
->data
->rx_queues
[qidx
];
1509 rxq
->mbuf_phys_off
= nicvf_mempool_phy_offset(rxq
->pool
);
1510 mbuf
= rte_pktmbuf_alloc(rxq
->pool
);
1512 PMD_INIT_LOG(ERR
, "Failed allocate mbuf VF%d qid=%d "
1514 nic
->vf_id
, qidx
, rxq
->pool
->name
);
1517 data_off
= nicvf_mbuff_meta_length(mbuf
);
1518 data_off
+= RTE_PKTMBUF_HEADROOM
;
1519 rte_pktmbuf_free(mbuf
);
1521 if (data_off
% RTE_CACHE_LINE_SIZE
) {
1522 PMD_INIT_LOG(ERR
, "%s: unaligned data_off=%d delta=%d",
1523 rxq
->pool
->name
, data_off
,
1524 data_off
% RTE_CACHE_LINE_SIZE
);
1527 rxq
->mbuf_phys_off
-= data_off
;
1528 rxq
->mbuf_phys_off
-= nic
->skip_bytes
;
1530 if (mbuf_phys_off
== 0)
1531 mbuf_phys_off
= rxq
->mbuf_phys_off
;
1532 if (mbuf_phys_off
!= rxq
->mbuf_phys_off
) {
1533 PMD_INIT_LOG(ERR
, "pool params not same,%s VF%d %"
1534 PRIx64
, rxq
->pool
->name
, nic
->vf_id
,
1540 /* Check the level of buffers in the pool */
1542 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1543 rxq
= dev
->data
->rx_queues
[qidx
];
1544 /* Count total numbers of rxq descs */
1545 total_rxq_desc
+= rxq
->qlen_mask
+ 1;
1546 exp_buffs
= RTE_MEMPOOL_CACHE_MAX_SIZE
+ rxq
->rx_free_thresh
;
1547 exp_buffs
*= dev
->data
->nb_rx_queues
;
1548 if (rte_mempool_avail_count(rxq
->pool
) < exp_buffs
) {
1549 PMD_INIT_LOG(ERR
, "Buff shortage in pool=%s (%d/%d)",
1551 rte_mempool_avail_count(rxq
->pool
),
1557 /* Check RBDR desc overflow */
1558 ret
= nicvf_qsize_rbdr_roundup(total_rxq_desc
);
1560 PMD_INIT_LOG(ERR
, "Reached RBDR desc limit, reduce nr desc "
1561 "VF%d", nic
->vf_id
);
1566 ret
= nicvf_qset_config(nic
);
1568 PMD_INIT_LOG(ERR
, "Failed to enable qset %d VF%d", ret
,
1573 /* Allocate RBDR and RBDR ring desc */
1574 nb_rbdr_desc
= nicvf_qsize_rbdr_roundup(total_rxq_desc
);
1575 ret
= nicvf_qset_rbdr_alloc(dev
, nic
, nb_rbdr_desc
, rbdrsz
);
1577 PMD_INIT_LOG(ERR
, "Failed to allocate memory for rbdr alloc "
1578 "VF%d", nic
->vf_id
);
1582 /* Enable and configure RBDR registers */
1583 ret
= nicvf_qset_rbdr_config(nic
, 0);
1585 PMD_INIT_LOG(ERR
, "Failed to configure rbdr %d VF%d", ret
,
1587 goto qset_rbdr_free
;
1590 /* Fill rte_mempool buffers in RBDR pool and precharge it */
1591 ret
= nicvf_qset_rbdr_precharge(dev
, nic
, 0, rbdr_rte_mempool_get
,
1594 PMD_INIT_LOG(ERR
, "Failed to fill rbdr %d VF%d", ret
,
1596 goto qset_rbdr_reclaim
;
1599 PMD_DRV_LOG(INFO
, "Filled %d out of %d entries in RBDR VF%d",
1600 nic
->rbdr
->tail
, nb_rbdr_desc
, nic
->vf_id
);
1602 /* Configure VLAN Strip */
1603 mask
= ETH_VLAN_STRIP_MASK
| ETH_VLAN_FILTER_MASK
|
1604 ETH_VLAN_EXTEND_MASK
;
1605 ret
= nicvf_vlan_offload_config(dev
, mask
);
1607 /* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
1608 * to the 64bit memory address.
1609 * The alignment creates a hole in mbuf(between the end of headroom and
1610 * packet data start). The new revision of the HW provides an option to
1611 * disable the L3 alignment feature and make mbuf layout looks
1612 * more like other NICs. For better application compatibility, disabling
1613 * l3 alignment feature on the hardware revisions it supports
1615 nicvf_apad_config(nic
, false);
1617 /* Get queue ranges for this VF */
1618 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
1620 /* Configure TX queues */
1621 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++) {
1622 ret
= nicvf_vf_start_tx_queue(dev
, nic
,
1623 qidx
% MAX_SND_QUEUES_PER_QS
);
1625 goto start_txq_error
;
1628 /* Configure RX queues */
1629 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1630 ret
= nicvf_vf_start_rx_queue(dev
, nic
,
1631 qidx
% MAX_RCV_QUEUES_PER_QS
);
1633 goto start_rxq_error
;
1636 if (!nic
->sqs_mode
) {
1637 /* Configure CPI algorithm */
1638 ret
= nicvf_configure_cpi(dev
);
1640 goto start_txq_error
;
1642 ret
= nicvf_mbox_get_rss_size(nic
);
1644 PMD_INIT_LOG(ERR
, "Failed to get rss table size");
1645 goto qset_rss_error
;
1649 ret
= nicvf_configure_rss(dev
);
1651 goto qset_rss_error
;
1654 /* Done; Let PF make the BGX's RX and TX switches to ON position */
1655 nicvf_mbox_cfg_done(nic
);
1659 nicvf_rss_term(nic
);
1661 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++)
1662 nicvf_vf_stop_rx_queue(dev
, nic
, qidx
% MAX_RCV_QUEUES_PER_QS
);
1664 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++)
1665 nicvf_vf_stop_tx_queue(dev
, nic
, qidx
% MAX_SND_QUEUES_PER_QS
);
1667 nicvf_qset_rbdr_reclaim(nic
, 0);
1668 nicvf_rbdr_release_mbufs(dev
, nic
);
1671 rte_free(nic
->rbdr
);
1675 nicvf_qset_reclaim(nic
);
1680 nicvf_dev_start(struct rte_eth_dev
*dev
)
1685 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1686 struct rte_eth_rxmode
*rx_conf
= &dev
->data
->dev_conf
.rxmode
;
1688 uint32_t buffsz
= 0, rbdrsz
= 0;
1689 struct rte_pktmbuf_pool_private
*mbp_priv
;
1690 struct nicvf_rxq
*rxq
;
1692 PMD_INIT_FUNC_TRACE();
1694 /* This function must be called for a primary device */
1695 assert_primary(nic
);
1697 /* Validate RBDR buff size */
1698 for (qidx
= 0; qidx
< dev
->data
->nb_rx_queues
; qidx
++) {
1699 rxq
= dev
->data
->rx_queues
[qidx
];
1700 mbp_priv
= rte_mempool_get_priv(rxq
->pool
);
1701 buffsz
= mbp_priv
->mbuf_data_room_size
- RTE_PKTMBUF_HEADROOM
;
1703 PMD_INIT_LOG(ERR
, "rxbuf size must be multiply of 128");
1708 if (rbdrsz
!= buffsz
) {
1709 PMD_INIT_LOG(ERR
, "buffsz not same, qidx=%d (%d/%d)",
1710 qidx
, rbdrsz
, buffsz
);
1715 /* Configure loopback */
1716 ret
= nicvf_loopback_config(nic
, dev
->data
->dev_conf
.lpbk_mode
);
1718 PMD_INIT_LOG(ERR
, "Failed to configure loopback %d", ret
);
1722 /* Reset all statistics counters attached to this port */
1723 ret
= nicvf_mbox_reset_stat_counters(nic
, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1725 PMD_INIT_LOG(ERR
, "Failed to reset stat counters %d", ret
);
1729 /* Setup scatter mode if needed by jumbo */
1730 if (dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
+
1731 2 * VLAN_TAG_SIZE
> buffsz
)
1732 dev
->data
->scattered_rx
= 1;
1733 if ((rx_conf
->offloads
& DEV_RX_OFFLOAD_SCATTER
) != 0)
1734 dev
->data
->scattered_rx
= 1;
1736 /* Setup MTU based on max_rx_pkt_len or default */
1737 mtu
= dev
->data
->dev_conf
.rxmode
.offloads
& DEV_RX_OFFLOAD_JUMBO_FRAME
?
1738 dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
1739 - ETHER_HDR_LEN
: ETHER_MTU
;
1741 if (nicvf_dev_set_mtu(dev
, mtu
)) {
1742 PMD_INIT_LOG(ERR
, "Failed to set default mtu size");
1746 ret
= nicvf_vf_start(dev
, nic
, rbdrsz
);
1750 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1751 assert(nic
->snicvf
[i
]);
1753 ret
= nicvf_vf_start(dev
, nic
->snicvf
[i
], rbdrsz
);
1758 /* Configure callbacks based on offloads */
1759 nicvf_set_tx_function(dev
);
1760 nicvf_set_rx_function(dev
);
1766 nicvf_dev_stop_cleanup(struct rte_eth_dev
*dev
, bool cleanup
)
1770 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1772 PMD_INIT_FUNC_TRACE();
1774 /* Teardown secondary vf first */
1775 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1776 if (!nic
->snicvf
[i
])
1779 nicvf_vf_stop(dev
, nic
->snicvf
[i
], cleanup
);
1782 /* Stop the primary VF now */
1783 nicvf_vf_stop(dev
, nic
, cleanup
);
1785 /* Disable loopback */
1786 ret
= nicvf_loopback_config(nic
, 0);
1788 PMD_INIT_LOG(ERR
, "Failed to disable loopback %d", ret
);
1790 /* Reclaim CPI configuration */
1791 ret
= nicvf_mbox_config_cpi(nic
, 0);
1793 PMD_INIT_LOG(ERR
, "Failed to reclaim CPI config %d", ret
);
1797 nicvf_dev_stop(struct rte_eth_dev
*dev
)
1799 PMD_INIT_FUNC_TRACE();
1801 nicvf_dev_stop_cleanup(dev
, false);
1805 nicvf_vf_stop(struct rte_eth_dev
*dev
, struct nicvf
*nic
, bool cleanup
)
1809 uint16_t tx_start
, tx_end
;
1810 uint16_t rx_start
, rx_end
;
1812 PMD_INIT_FUNC_TRACE();
1815 /* Let PF make the BGX's RX and TX switches to OFF position */
1816 nicvf_mbox_shutdown(nic
);
1819 /* Disable VLAN Strip */
1820 nicvf_vlan_hw_strip(nic
, 0);
1822 /* Get queue ranges for this VF */
1823 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
1825 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++)
1826 nicvf_vf_stop_tx_queue(dev
, nic
, qidx
% MAX_SND_QUEUES_PER_QS
);
1828 /* Get queue ranges for this VF */
1829 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
1832 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++)
1833 nicvf_vf_stop_rx_queue(dev
, nic
, qidx
% MAX_RCV_QUEUES_PER_QS
);
1836 ret
= nicvf_qset_rbdr_reclaim(nic
, 0);
1838 PMD_INIT_LOG(ERR
, "Failed to reclaim RBDR %d", ret
);
1840 /* Move all charged buffers in RBDR back to pool */
1841 if (nic
->rbdr
!= NULL
)
1842 nicvf_rbdr_release_mbufs(dev
, nic
);
1845 ret
= nicvf_qset_reclaim(nic
);
1847 PMD_INIT_LOG(ERR
, "Failed to disable qset %d", ret
);
1849 /* Disable all interrupts */
1850 nicvf_disable_all_interrupts(nic
);
1852 /* Free RBDR SW structure */
1854 rte_free(nic
->rbdr
);
1860 nicvf_dev_close(struct rte_eth_dev
*dev
)
1863 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1865 PMD_INIT_FUNC_TRACE();
1867 nicvf_dev_stop_cleanup(dev
, true);
1868 nicvf_periodic_alarm_stop(nicvf_interrupt
, dev
);
1870 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1871 if (!nic
->snicvf
[i
])
1874 nicvf_periodic_alarm_stop(nicvf_vf_interrupt
, nic
->snicvf
[i
]);
1879 nicvf_request_sqs(struct nicvf
*nic
)
1883 assert_primary(nic
);
1884 assert(nic
->sqs_count
> 0);
1885 assert(nic
->sqs_count
<= MAX_SQS_PER_VF
);
1887 /* Set no of Rx/Tx queues in each of the SQsets */
1888 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1889 if (nicvf_svf_empty())
1890 rte_panic("Cannot assign sufficient number of "
1891 "secondary queues to primary VF%" PRIu8
"\n",
1894 nic
->snicvf
[i
] = nicvf_svf_pop();
1895 nic
->snicvf
[i
]->sqs_id
= i
;
1898 return nicvf_mbox_request_sqs(nic
);
1902 nicvf_dev_configure(struct rte_eth_dev
*dev
)
1904 struct rte_eth_dev_data
*data
= dev
->data
;
1905 struct rte_eth_conf
*conf
= &data
->dev_conf
;
1906 struct rte_eth_rxmode
*rxmode
= &conf
->rxmode
;
1907 struct rte_eth_txmode
*txmode
= &conf
->txmode
;
1908 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1911 PMD_INIT_FUNC_TRACE();
1913 if (!rte_eal_has_hugepages()) {
1914 PMD_INIT_LOG(INFO
, "Huge page is not configured");
1918 if (txmode
->mq_mode
) {
1919 PMD_INIT_LOG(INFO
, "Tx mq_mode DCB or VMDq not supported");
1923 if (rxmode
->mq_mode
!= ETH_MQ_RX_NONE
&&
1924 rxmode
->mq_mode
!= ETH_MQ_RX_RSS
) {
1925 PMD_INIT_LOG(INFO
, "Unsupported rx qmode %d", rxmode
->mq_mode
);
1929 if (rxmode
->split_hdr_size
) {
1930 PMD_INIT_LOG(INFO
, "Rxmode does not support split header");
1934 if (conf
->link_speeds
& ETH_LINK_SPEED_FIXED
) {
1935 PMD_INIT_LOG(INFO
, "Setting link speed/duplex not supported");
1939 if (conf
->dcb_capability_en
) {
1940 PMD_INIT_LOG(INFO
, "DCB enable not supported");
1944 if (conf
->fdir_conf
.mode
!= RTE_FDIR_MODE_NONE
) {
1945 PMD_INIT_LOG(INFO
, "Flow director not supported");
1949 assert_primary(nic
);
1950 NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS
== MAX_SND_QUEUES_PER_QS
);
1951 cqcount
= RTE_MAX(data
->nb_tx_queues
, data
->nb_rx_queues
);
1952 if (cqcount
> MAX_RCV_QUEUES_PER_QS
) {
1953 nic
->sqs_count
= RTE_ALIGN_CEIL(cqcount
, MAX_RCV_QUEUES_PER_QS
);
1954 nic
->sqs_count
= (nic
->sqs_count
/ MAX_RCV_QUEUES_PER_QS
) - 1;
1959 assert(nic
->sqs_count
<= MAX_SQS_PER_VF
);
1961 if (nic
->sqs_count
> 0) {
1962 if (nicvf_request_sqs(nic
)) {
1963 rte_panic("Cannot assign sufficient number of "
1964 "secondary queues to PORT%d VF%" PRIu8
"\n",
1965 dev
->data
->port_id
, nic
->vf_id
);
1969 if (rxmode
->offloads
& DEV_RX_OFFLOAD_CHECKSUM
)
1970 nic
->offload_cksum
= 1;
1972 PMD_INIT_LOG(DEBUG
, "Configured ethdev port%d hwcap=0x%" PRIx64
,
1973 dev
->data
->port_id
, nicvf_hw_cap(nic
));
1978 /* Initialize and register driver with DPDK Application */
1979 static const struct eth_dev_ops nicvf_eth_dev_ops
= {
1980 .dev_configure
= nicvf_dev_configure
,
1981 .dev_start
= nicvf_dev_start
,
1982 .dev_stop
= nicvf_dev_stop
,
1983 .link_update
= nicvf_dev_link_update
,
1984 .dev_close
= nicvf_dev_close
,
1985 .stats_get
= nicvf_dev_stats_get
,
1986 .stats_reset
= nicvf_dev_stats_reset
,
1987 .promiscuous_enable
= nicvf_dev_promisc_enable
,
1988 .dev_infos_get
= nicvf_dev_info_get
,
1989 .dev_supported_ptypes_get
= nicvf_dev_supported_ptypes_get
,
1990 .mtu_set
= nicvf_dev_set_mtu
,
1991 .vlan_offload_set
= nicvf_vlan_offload_set
,
1992 .reta_update
= nicvf_dev_reta_update
,
1993 .reta_query
= nicvf_dev_reta_query
,
1994 .rss_hash_update
= nicvf_dev_rss_hash_update
,
1995 .rss_hash_conf_get
= nicvf_dev_rss_hash_conf_get
,
1996 .rx_queue_start
= nicvf_dev_rx_queue_start
,
1997 .rx_queue_stop
= nicvf_dev_rx_queue_stop
,
1998 .tx_queue_start
= nicvf_dev_tx_queue_start
,
1999 .tx_queue_stop
= nicvf_dev_tx_queue_stop
,
2000 .rx_queue_setup
= nicvf_dev_rx_queue_setup
,
2001 .rx_queue_release
= nicvf_dev_rx_queue_release
,
2002 .rx_queue_count
= nicvf_dev_rx_queue_count
,
2003 .tx_queue_setup
= nicvf_dev_tx_queue_setup
,
2004 .tx_queue_release
= nicvf_dev_tx_queue_release
,
2005 .get_reg
= nicvf_dev_get_regs
,
2009 nicvf_vlan_offload_config(struct rte_eth_dev
*dev
, int mask
)
2011 struct rte_eth_rxmode
*rxmode
;
2012 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
2013 rxmode
= &dev
->data
->dev_conf
.rxmode
;
2014 if (mask
& ETH_VLAN_STRIP_MASK
) {
2015 if (rxmode
->offloads
& DEV_RX_OFFLOAD_VLAN_STRIP
)
2016 nicvf_vlan_hw_strip(nic
, true);
2018 nicvf_vlan_hw_strip(nic
, false);
2025 nicvf_vlan_offload_set(struct rte_eth_dev
*dev
, int mask
)
2027 nicvf_vlan_offload_config(dev
, mask
);
2033 nicvf_set_first_skip(struct rte_eth_dev
*dev
)
2035 int bytes_to_skip
= 0;
2038 struct rte_kvargs
*kvlist
;
2039 static const char *const skip
[] = {
2042 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
2044 if (!dev
->device
->devargs
) {
2045 nicvf_first_skip_config(nic
, 0);
2049 kvlist
= rte_kvargs_parse(dev
->device
->devargs
->args
, skip
);
2053 if (kvlist
->count
== 0)
2056 for (i
= 0; i
!= kvlist
->count
; ++i
) {
2057 const struct rte_kvargs_pair
*pair
= &kvlist
->pairs
[i
];
2059 if (!strcmp(pair
->key
, SKIP_DATA_BYTES
))
2060 bytes_to_skip
= atoi(pair
->value
);
2063 /*128 bytes amounts to one cache line*/
2064 if (bytes_to_skip
>= 0 && bytes_to_skip
< 128) {
2065 if (!(bytes_to_skip
% 8)) {
2066 nicvf_first_skip_config(nic
, (bytes_to_skip
/ 8));
2067 nic
->skip_bytes
= bytes_to_skip
;
2070 PMD_INIT_LOG(ERR
, "skip_data_bytes should be multiple of 8");
2075 PMD_INIT_LOG(ERR
, "skip_data_bytes should be less than 128");
2080 nicvf_first_skip_config(nic
, 0);
2082 rte_kvargs_free(kvlist
);
2086 nicvf_eth_dev_init(struct rte_eth_dev
*eth_dev
)
2089 struct rte_pci_device
*pci_dev
;
2090 struct nicvf
*nic
= nicvf_pmd_priv(eth_dev
);
2092 PMD_INIT_FUNC_TRACE();
2094 eth_dev
->dev_ops
= &nicvf_eth_dev_ops
;
2096 /* For secondary processes, the primary has done all the work */
2097 if (rte_eal_process_type() != RTE_PROC_PRIMARY
) {
2099 /* Setup callbacks for secondary process */
2100 nicvf_set_tx_function(eth_dev
);
2101 nicvf_set_rx_function(eth_dev
);
2104 /* If nic == NULL than it is secondary function
2105 * so ethdev need to be released by caller */
2110 pci_dev
= RTE_ETH_DEV_TO_PCI(eth_dev
);
2111 rte_eth_copy_pci_info(eth_dev
, pci_dev
);
2113 nic
->device_id
= pci_dev
->id
.device_id
;
2114 nic
->vendor_id
= pci_dev
->id
.vendor_id
;
2115 nic
->subsystem_device_id
= pci_dev
->id
.subsystem_device_id
;
2116 nic
->subsystem_vendor_id
= pci_dev
->id
.subsystem_vendor_id
;
2118 PMD_INIT_LOG(DEBUG
, "nicvf: device (%x:%x) %u:%u:%u:%u",
2119 pci_dev
->id
.vendor_id
, pci_dev
->id
.device_id
,
2120 pci_dev
->addr
.domain
, pci_dev
->addr
.bus
,
2121 pci_dev
->addr
.devid
, pci_dev
->addr
.function
);
2123 nic
->reg_base
= (uintptr_t)pci_dev
->mem_resource
[0].addr
;
2124 if (!nic
->reg_base
) {
2125 PMD_INIT_LOG(ERR
, "Failed to map BAR0");
2130 nicvf_disable_all_interrupts(nic
);
2132 ret
= nicvf_periodic_alarm_start(nicvf_interrupt
, eth_dev
);
2134 PMD_INIT_LOG(ERR
, "Failed to start period alarm");
2138 ret
= nicvf_mbox_check_pf_ready(nic
);
2140 PMD_INIT_LOG(ERR
, "Failed to get ready message from PF");
2144 "node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
2145 nic
->node
, nic
->vf_id
,
2146 nic
->tns_mode
== NIC_TNS_MODE
? "tns" : "tns-bypass",
2147 nic
->sqs_mode
? "true" : "false",
2148 nic
->loopback_supported
? "true" : "false"
2152 ret
= nicvf_base_init(nic
);
2154 PMD_INIT_LOG(ERR
, "Failed to execute nicvf_base_init");
2158 if (nic
->sqs_mode
) {
2159 /* Push nic to stack of secondary vfs */
2160 nicvf_svf_push(nic
);
2162 /* Steal nic pointer from the device for further reuse */
2163 eth_dev
->data
->dev_private
= NULL
;
2165 nicvf_periodic_alarm_stop(nicvf_interrupt
, eth_dev
);
2166 ret
= nicvf_periodic_alarm_start(nicvf_vf_interrupt
, nic
);
2168 PMD_INIT_LOG(ERR
, "Failed to start period alarm");
2172 /* Detach port by returning positive error number */
2176 eth_dev
->data
->mac_addrs
= rte_zmalloc("mac_addr", ETHER_ADDR_LEN
, 0);
2177 if (eth_dev
->data
->mac_addrs
== NULL
) {
2178 PMD_INIT_LOG(ERR
, "Failed to allocate memory for mac addr");
2182 if (is_zero_ether_addr((struct ether_addr
*)nic
->mac_addr
))
2183 eth_random_addr(&nic
->mac_addr
[0]);
2185 ether_addr_copy((struct ether_addr
*)nic
->mac_addr
,
2186 ð_dev
->data
->mac_addrs
[0]);
2188 ret
= nicvf_mbox_set_mac_addr(nic
, nic
->mac_addr
);
2190 PMD_INIT_LOG(ERR
, "Failed to set mac addr");
2194 ret
= nicvf_set_first_skip(eth_dev
);
2196 PMD_INIT_LOG(ERR
, "Failed to configure first skip");
2199 PMD_INIT_LOG(INFO
, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
2200 eth_dev
->data
->port_id
, nic
->vendor_id
, nic
->device_id
,
2201 nic
->mac_addr
[0], nic
->mac_addr
[1], nic
->mac_addr
[2],
2202 nic
->mac_addr
[3], nic
->mac_addr
[4], nic
->mac_addr
[5]);
2207 rte_free(eth_dev
->data
->mac_addrs
);
2209 nicvf_periodic_alarm_stop(nicvf_interrupt
, eth_dev
);
2214 static const struct rte_pci_id pci_id_nicvf_map
[] = {
2216 .class_id
= RTE_CLASS_ANY_ID
,
2217 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2218 .device_id
= PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF
,
2219 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2220 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF
,
2223 .class_id
= RTE_CLASS_ANY_ID
,
2224 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2225 .device_id
= PCI_DEVICE_ID_THUNDERX_NICVF
,
2226 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2227 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF
,
2230 .class_id
= RTE_CLASS_ANY_ID
,
2231 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2232 .device_id
= PCI_DEVICE_ID_THUNDERX_NICVF
,
2233 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2234 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN81XX_NICVF
,
2237 .class_id
= RTE_CLASS_ANY_ID
,
2238 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2239 .device_id
= PCI_DEVICE_ID_THUNDERX_NICVF
,
2240 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2241 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN83XX_NICVF
,
2248 static int nicvf_eth_pci_probe(struct rte_pci_driver
*pci_drv __rte_unused
,
2249 struct rte_pci_device
*pci_dev
)
2251 return rte_eth_dev_pci_generic_probe(pci_dev
, sizeof(struct nicvf
),
2252 nicvf_eth_dev_init
);
2255 static int nicvf_eth_pci_remove(struct rte_pci_device
*pci_dev
)
2257 return rte_eth_dev_pci_generic_remove(pci_dev
, NULL
);
2260 static struct rte_pci_driver rte_nicvf_pmd
= {
2261 .id_table
= pci_id_nicvf_map
,
2262 .drv_flags
= RTE_PCI_DRV_NEED_MAPPING
| RTE_PCI_DRV_KEEP_MAPPED_RES
|
2263 RTE_PCI_DRV_INTR_LSC
,
2264 .probe
= nicvf_eth_pci_probe
,
2265 .remove
= nicvf_eth_pci_remove
,
2268 RTE_PMD_REGISTER_PCI(net_thunderx
, rte_nicvf_pmd
);
2269 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx
, pci_id_nicvf_map
);
2270 RTE_PMD_REGISTER_KMOD_DEP(net_thunderx
, "* igb_uio | uio_pci_generic | vfio-pci");
2271 RTE_PMD_REGISTER_PARAM_STRING(net_thunderx
, SKIP_DATA_BYTES
"=<int>");