4 * Copyright (C) Cavium networks Ltd. 2016.
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
10 * * Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * * Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in
14 * the documentation and/or other materials provided with the
16 * * Neither the name of Cavium networks nor the names of its
17 * contributors may be used to endorse or promote products derived
18 * from this software without specific prior written permission.
20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
42 #include <netinet/in.h>
43 #include <sys/queue.h>
44 #include <sys/timerfd.h>
46 #include <rte_alarm.h>
47 #include <rte_atomic.h>
48 #include <rte_branch_prediction.h>
49 #include <rte_byteorder.h>
50 #include <rte_common.h>
51 #include <rte_cycles.h>
52 #include <rte_debug.h>
55 #include <rte_ether.h>
56 #include <rte_ethdev.h>
57 #include <rte_interrupts.h>
59 #include <rte_memory.h>
60 #include <rte_memzone.h>
61 #include <rte_malloc.h>
62 #include <rte_random.h>
64 #include <rte_tailq.h>
66 #include "base/nicvf_plat.h"
68 #include "nicvf_ethdev.h"
69 #include "nicvf_rxtx.h"
70 #include "nicvf_svf.h"
71 #include "nicvf_logs.h"
73 static void nicvf_dev_stop(struct rte_eth_dev
*dev
);
74 static void nicvf_dev_stop_cleanup(struct rte_eth_dev
*dev
, bool cleanup
);
75 static void nicvf_vf_stop(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
79 nicvf_atomic_write_link_status(struct rte_eth_dev
*dev
,
80 struct rte_eth_link
*link
)
82 struct rte_eth_link
*dst
= &dev
->data
->dev_link
;
83 struct rte_eth_link
*src
= link
;
85 if (rte_atomic64_cmpset((uint64_t *)dst
, *(uint64_t *)dst
,
86 *(uint64_t *)src
) == 0)
93 nicvf_set_eth_link_status(struct nicvf
*nic
, struct rte_eth_link
*link
)
95 link
->link_status
= nic
->link_up
;
96 link
->link_duplex
= ETH_LINK_AUTONEG
;
97 if (nic
->duplex
== NICVF_HALF_DUPLEX
)
98 link
->link_duplex
= ETH_LINK_HALF_DUPLEX
;
99 else if (nic
->duplex
== NICVF_FULL_DUPLEX
)
100 link
->link_duplex
= ETH_LINK_FULL_DUPLEX
;
101 link
->link_speed
= nic
->speed
;
102 link
->link_autoneg
= ETH_LINK_SPEED_AUTONEG
;
106 nicvf_interrupt(void *arg
)
108 struct rte_eth_dev
*dev
= arg
;
109 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
111 if (nicvf_reg_poll_interrupts(nic
) == NIC_MBOX_MSG_BGX_LINK_CHANGE
) {
112 if (dev
->data
->dev_conf
.intr_conf
.lsc
)
113 nicvf_set_eth_link_status(nic
, &dev
->data
->dev_link
);
114 _rte_eth_dev_callback_process(dev
, RTE_ETH_EVENT_INTR_LSC
, NULL
);
117 rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS
* 1000,
118 nicvf_interrupt
, dev
);
122 nicvf_vf_interrupt(void *arg
)
124 struct nicvf
*nic
= arg
;
126 nicvf_reg_poll_interrupts(nic
);
128 rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS
* 1000,
129 nicvf_vf_interrupt
, nic
);
133 nicvf_periodic_alarm_start(void (fn
)(void *), void *arg
)
135 return rte_eal_alarm_set(NICVF_INTR_POLL_INTERVAL_MS
* 1000, fn
, arg
);
139 nicvf_periodic_alarm_stop(void (fn
)(void *), void *arg
)
141 return rte_eal_alarm_cancel(fn
, arg
);
145 * Return 0 means link status changed, -1 means not changed
148 nicvf_dev_link_update(struct rte_eth_dev
*dev
,
149 int wait_to_complete __rte_unused
)
151 struct rte_eth_link link
;
152 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
154 PMD_INIT_FUNC_TRACE();
156 memset(&link
, 0, sizeof(link
));
157 nicvf_set_eth_link_status(nic
, &link
);
158 return nicvf_atomic_write_link_status(dev
, &link
);
162 nicvf_dev_set_mtu(struct rte_eth_dev
*dev
, uint16_t mtu
)
164 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
165 uint32_t buffsz
, frame_size
= mtu
+ ETHER_HDR_LEN
+ ETHER_CRC_LEN
;
168 PMD_INIT_FUNC_TRACE();
170 if (frame_size
> NIC_HW_MAX_FRS
)
173 if (frame_size
< NIC_HW_MIN_FRS
)
176 buffsz
= dev
->data
->min_rx_buf_size
- RTE_PKTMBUF_HEADROOM
;
179 * Refuse mtu that requires the support of scattered packets
180 * when this feature has not been enabled before.
182 if (!dev
->data
->scattered_rx
&&
183 (frame_size
+ 2 * VLAN_TAG_SIZE
> buffsz
))
186 /* check <seg size> * <max_seg> >= max_frame */
187 if (dev
->data
->scattered_rx
&&
188 (frame_size
+ 2 * VLAN_TAG_SIZE
> buffsz
* NIC_HW_MAX_SEGS
))
191 if (frame_size
> ETHER_MAX_LEN
)
192 dev
->data
->dev_conf
.rxmode
.jumbo_frame
= 1;
194 dev
->data
->dev_conf
.rxmode
.jumbo_frame
= 0;
196 if (nicvf_mbox_update_hw_max_frs(nic
, frame_size
))
199 /* Update max frame size */
200 dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
= (uint32_t)frame_size
;
203 for (i
= 0; i
< nic
->sqs_count
; i
++)
204 nic
->snicvf
[i
]->mtu
= mtu
;
210 nicvf_dev_get_regs(struct rte_eth_dev
*dev
, struct rte_dev_reg_info
*regs
)
212 uint64_t *data
= regs
->data
;
213 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
216 regs
->length
= nicvf_reg_get_count();
217 regs
->width
= THUNDERX_REG_BYTES
;
221 /* Support only full register dump */
222 if ((regs
->length
== 0) ||
223 (regs
->length
== (uint32_t)nicvf_reg_get_count())) {
224 regs
->version
= nic
->vendor_id
<< 16 | nic
->device_id
;
225 nicvf_reg_dump(nic
, data
);
232 nicvf_dev_stats_get(struct rte_eth_dev
*dev
, struct rte_eth_stats
*stats
)
235 struct nicvf_hw_rx_qstats rx_qstats
;
236 struct nicvf_hw_tx_qstats tx_qstats
;
237 struct nicvf_hw_stats port_stats
;
238 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
239 uint16_t rx_start
, rx_end
;
240 uint16_t tx_start
, tx_end
;
243 /* RX queue indices for the first VF */
244 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
246 /* Reading per RX ring stats */
247 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
248 if (qidx
== RTE_ETHDEV_QUEUE_STAT_CNTRS
)
251 nicvf_hw_get_rx_qstats(nic
, &rx_qstats
, qidx
);
252 stats
->q_ibytes
[qidx
] = rx_qstats
.q_rx_bytes
;
253 stats
->q_ipackets
[qidx
] = rx_qstats
.q_rx_packets
;
256 /* TX queue indices for the first VF */
257 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
259 /* Reading per TX ring stats */
260 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++) {
261 if (qidx
== RTE_ETHDEV_QUEUE_STAT_CNTRS
)
264 nicvf_hw_get_tx_qstats(nic
, &tx_qstats
, qidx
);
265 stats
->q_obytes
[qidx
] = tx_qstats
.q_tx_bytes
;
266 stats
->q_opackets
[qidx
] = tx_qstats
.q_tx_packets
;
269 for (i
= 0; i
< nic
->sqs_count
; i
++) {
270 struct nicvf
*snic
= nic
->snicvf
[i
];
275 /* RX queue indices for a secondary VF */
276 nicvf_rx_range(dev
, snic
, &rx_start
, &rx_end
);
278 /* Reading per RX ring stats */
279 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
280 if (qidx
== RTE_ETHDEV_QUEUE_STAT_CNTRS
)
283 nicvf_hw_get_rx_qstats(snic
, &rx_qstats
,
284 qidx
% MAX_RCV_QUEUES_PER_QS
);
285 stats
->q_ibytes
[qidx
] = rx_qstats
.q_rx_bytes
;
286 stats
->q_ipackets
[qidx
] = rx_qstats
.q_rx_packets
;
289 /* TX queue indices for a secondary VF */
290 nicvf_tx_range(dev
, snic
, &tx_start
, &tx_end
);
291 /* Reading per TX ring stats */
292 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++) {
293 if (qidx
== RTE_ETHDEV_QUEUE_STAT_CNTRS
)
296 nicvf_hw_get_tx_qstats(snic
, &tx_qstats
,
297 qidx
% MAX_SND_QUEUES_PER_QS
);
298 stats
->q_obytes
[qidx
] = tx_qstats
.q_tx_bytes
;
299 stats
->q_opackets
[qidx
] = tx_qstats
.q_tx_packets
;
303 nicvf_hw_get_stats(nic
, &port_stats
);
304 stats
->ibytes
= port_stats
.rx_bytes
;
305 stats
->ipackets
= port_stats
.rx_ucast_frames
;
306 stats
->ipackets
+= port_stats
.rx_bcast_frames
;
307 stats
->ipackets
+= port_stats
.rx_mcast_frames
;
308 stats
->ierrors
= port_stats
.rx_l2_errors
;
309 stats
->imissed
= port_stats
.rx_drop_red
;
310 stats
->imissed
+= port_stats
.rx_drop_overrun
;
311 stats
->imissed
+= port_stats
.rx_drop_bcast
;
312 stats
->imissed
+= port_stats
.rx_drop_mcast
;
313 stats
->imissed
+= port_stats
.rx_drop_l3_bcast
;
314 stats
->imissed
+= port_stats
.rx_drop_l3_mcast
;
316 stats
->obytes
= port_stats
.tx_bytes_ok
;
317 stats
->opackets
= port_stats
.tx_ucast_frames_ok
;
318 stats
->opackets
+= port_stats
.tx_bcast_frames_ok
;
319 stats
->opackets
+= port_stats
.tx_mcast_frames_ok
;
320 stats
->oerrors
= port_stats
.tx_drops
;
323 static const uint32_t *
324 nicvf_dev_supported_ptypes_get(struct rte_eth_dev
*dev
)
327 static uint32_t ptypes
[32];
328 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
329 static const uint32_t ptypes_common
[] = {
331 RTE_PTYPE_L3_IPV4_EXT
,
333 RTE_PTYPE_L3_IPV6_EXT
,
338 static const uint32_t ptypes_tunnel
[] = {
339 RTE_PTYPE_TUNNEL_GRE
,
340 RTE_PTYPE_TUNNEL_GENEVE
,
341 RTE_PTYPE_TUNNEL_VXLAN
,
342 RTE_PTYPE_TUNNEL_NVGRE
,
344 static const uint32_t ptypes_end
= RTE_PTYPE_UNKNOWN
;
346 copied
= sizeof(ptypes_common
);
347 memcpy(ptypes
, ptypes_common
, copied
);
348 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
) {
349 memcpy((char *)ptypes
+ copied
, ptypes_tunnel
,
350 sizeof(ptypes_tunnel
));
351 copied
+= sizeof(ptypes_tunnel
);
354 memcpy((char *)ptypes
+ copied
, &ptypes_end
, sizeof(ptypes_end
));
355 if (dev
->rx_pkt_burst
== nicvf_recv_pkts
||
356 dev
->rx_pkt_burst
== nicvf_recv_pkts_multiseg
)
363 nicvf_dev_stats_reset(struct rte_eth_dev
*dev
)
366 uint16_t rxqs
= 0, txqs
= 0;
367 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
368 uint16_t rx_start
, rx_end
;
369 uint16_t tx_start
, tx_end
;
371 /* Reset all primary nic counters */
372 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
373 for (i
= rx_start
; i
<= rx_end
; i
++)
374 rxqs
|= (0x3 << (i
* 2));
376 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
377 for (i
= tx_start
; i
<= tx_end
; i
++)
378 txqs
|= (0x3 << (i
* 2));
380 nicvf_mbox_reset_stat_counters(nic
, 0x3FFF, 0x1F, rxqs
, txqs
);
382 /* Reset secondary nic queue counters */
383 for (i
= 0; i
< nic
->sqs_count
; i
++) {
384 struct nicvf
*snic
= nic
->snicvf
[i
];
388 nicvf_rx_range(dev
, snic
, &rx_start
, &rx_end
);
389 for (i
= rx_start
; i
<= rx_end
; i
++)
390 rxqs
|= (0x3 << ((i
% MAX_CMP_QUEUES_PER_QS
) * 2));
392 nicvf_tx_range(dev
, snic
, &tx_start
, &tx_end
);
393 for (i
= tx_start
; i
<= tx_end
; i
++)
394 txqs
|= (0x3 << ((i
% MAX_SND_QUEUES_PER_QS
) * 2));
396 nicvf_mbox_reset_stat_counters(snic
, 0, 0, rxqs
, txqs
);
400 /* Promiscuous mode enabled by default in LMAC to VF 1:1 map configuration */
402 nicvf_dev_promisc_enable(struct rte_eth_dev
*dev __rte_unused
)
406 static inline uint64_t
407 nicvf_rss_ethdev_to_nic(struct nicvf
*nic
, uint64_t ethdev_rss
)
409 uint64_t nic_rss
= 0;
411 if (ethdev_rss
& ETH_RSS_IPV4
)
412 nic_rss
|= RSS_IP_ENA
;
414 if (ethdev_rss
& ETH_RSS_IPV6
)
415 nic_rss
|= RSS_IP_ENA
;
417 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV4_UDP
)
418 nic_rss
|= (RSS_IP_ENA
| RSS_UDP_ENA
);
420 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV4_TCP
)
421 nic_rss
|= (RSS_IP_ENA
| RSS_TCP_ENA
);
423 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV6_UDP
)
424 nic_rss
|= (RSS_IP_ENA
| RSS_UDP_ENA
);
426 if (ethdev_rss
& ETH_RSS_NONFRAG_IPV6_TCP
)
427 nic_rss
|= (RSS_IP_ENA
| RSS_TCP_ENA
);
429 if (ethdev_rss
& ETH_RSS_PORT
)
430 nic_rss
|= RSS_L2_EXTENDED_HASH_ENA
;
432 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
) {
433 if (ethdev_rss
& ETH_RSS_VXLAN
)
434 nic_rss
|= RSS_TUN_VXLAN_ENA
;
436 if (ethdev_rss
& ETH_RSS_GENEVE
)
437 nic_rss
|= RSS_TUN_GENEVE_ENA
;
439 if (ethdev_rss
& ETH_RSS_NVGRE
)
440 nic_rss
|= RSS_TUN_NVGRE_ENA
;
446 static inline uint64_t
447 nicvf_rss_nic_to_ethdev(struct nicvf
*nic
, uint64_t nic_rss
)
449 uint64_t ethdev_rss
= 0;
451 if (nic_rss
& RSS_IP_ENA
)
452 ethdev_rss
|= (ETH_RSS_IPV4
| ETH_RSS_IPV6
);
454 if ((nic_rss
& RSS_IP_ENA
) && (nic_rss
& RSS_TCP_ENA
))
455 ethdev_rss
|= (ETH_RSS_NONFRAG_IPV4_TCP
|
456 ETH_RSS_NONFRAG_IPV6_TCP
);
458 if ((nic_rss
& RSS_IP_ENA
) && (nic_rss
& RSS_UDP_ENA
))
459 ethdev_rss
|= (ETH_RSS_NONFRAG_IPV4_UDP
|
460 ETH_RSS_NONFRAG_IPV6_UDP
);
462 if (nic_rss
& RSS_L2_EXTENDED_HASH_ENA
)
463 ethdev_rss
|= ETH_RSS_PORT
;
465 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
) {
466 if (nic_rss
& RSS_TUN_VXLAN_ENA
)
467 ethdev_rss
|= ETH_RSS_VXLAN
;
469 if (nic_rss
& RSS_TUN_GENEVE_ENA
)
470 ethdev_rss
|= ETH_RSS_GENEVE
;
472 if (nic_rss
& RSS_TUN_NVGRE_ENA
)
473 ethdev_rss
|= ETH_RSS_NVGRE
;
479 nicvf_dev_reta_query(struct rte_eth_dev
*dev
,
480 struct rte_eth_rss_reta_entry64
*reta_conf
,
483 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
484 uint8_t tbl
[NIC_MAX_RSS_IDR_TBL_SIZE
];
487 if (reta_size
!= NIC_MAX_RSS_IDR_TBL_SIZE
) {
488 RTE_LOG(ERR
, PMD
, "The size of hash lookup table configured "
489 "(%d) doesn't match the number hardware can supported "
490 "(%d)", reta_size
, NIC_MAX_RSS_IDR_TBL_SIZE
);
494 ret
= nicvf_rss_reta_query(nic
, tbl
, NIC_MAX_RSS_IDR_TBL_SIZE
);
498 /* Copy RETA table */
499 for (i
= 0; i
< (NIC_MAX_RSS_IDR_TBL_SIZE
/ RTE_RETA_GROUP_SIZE
); i
++) {
500 for (j
= 0; j
< RTE_RETA_GROUP_SIZE
; j
++)
501 if ((reta_conf
[i
].mask
>> j
) & 0x01)
502 reta_conf
[i
].reta
[j
] = tbl
[j
];
509 nicvf_dev_reta_update(struct rte_eth_dev
*dev
,
510 struct rte_eth_rss_reta_entry64
*reta_conf
,
513 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
514 uint8_t tbl
[NIC_MAX_RSS_IDR_TBL_SIZE
];
517 if (reta_size
!= NIC_MAX_RSS_IDR_TBL_SIZE
) {
518 RTE_LOG(ERR
, PMD
, "The size of hash lookup table configured "
519 "(%d) doesn't match the number hardware can supported "
520 "(%d)", reta_size
, NIC_MAX_RSS_IDR_TBL_SIZE
);
524 ret
= nicvf_rss_reta_query(nic
, tbl
, NIC_MAX_RSS_IDR_TBL_SIZE
);
528 /* Copy RETA table */
529 for (i
= 0; i
< (NIC_MAX_RSS_IDR_TBL_SIZE
/ RTE_RETA_GROUP_SIZE
); i
++) {
530 for (j
= 0; j
< RTE_RETA_GROUP_SIZE
; j
++)
531 if ((reta_conf
[i
].mask
>> j
) & 0x01)
532 tbl
[j
] = reta_conf
[i
].reta
[j
];
535 return nicvf_rss_reta_update(nic
, tbl
, NIC_MAX_RSS_IDR_TBL_SIZE
);
539 nicvf_dev_rss_hash_conf_get(struct rte_eth_dev
*dev
,
540 struct rte_eth_rss_conf
*rss_conf
)
542 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
544 if (rss_conf
->rss_key
)
545 nicvf_rss_get_key(nic
, rss_conf
->rss_key
);
547 rss_conf
->rss_key_len
= RSS_HASH_KEY_BYTE_SIZE
;
548 rss_conf
->rss_hf
= nicvf_rss_nic_to_ethdev(nic
, nicvf_rss_get_cfg(nic
));
553 nicvf_dev_rss_hash_update(struct rte_eth_dev
*dev
,
554 struct rte_eth_rss_conf
*rss_conf
)
556 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
559 if (rss_conf
->rss_key
&&
560 rss_conf
->rss_key_len
!= RSS_HASH_KEY_BYTE_SIZE
) {
561 RTE_LOG(ERR
, PMD
, "Hash key size mismatch %d",
562 rss_conf
->rss_key_len
);
566 if (rss_conf
->rss_key
)
567 nicvf_rss_set_key(nic
, rss_conf
->rss_key
);
569 nic_rss
= nicvf_rss_ethdev_to_nic(nic
, rss_conf
->rss_hf
);
570 nicvf_rss_set_cfg(nic
, nic_rss
);
575 nicvf_qset_cq_alloc(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
576 struct nicvf_rxq
*rxq
, uint16_t qidx
, uint32_t desc_cnt
)
578 const struct rte_memzone
*rz
;
579 uint32_t ring_size
= CMP_QUEUE_SZ_MAX
* sizeof(union cq_entry_t
);
581 rz
= rte_eth_dma_zone_reserve(dev
, "cq_ring",
582 nicvf_netdev_qidx(nic
, qidx
), ring_size
,
583 NICVF_CQ_BASE_ALIGN_BYTES
, nic
->node
);
585 PMD_INIT_LOG(ERR
, "Failed to allocate mem for cq hw ring");
589 memset(rz
->addr
, 0, ring_size
);
591 rxq
->phys
= rz
->phys_addr
;
592 rxq
->desc
= rz
->addr
;
593 rxq
->qlen_mask
= desc_cnt
- 1;
599 nicvf_qset_sq_alloc(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
600 struct nicvf_txq
*sq
, uint16_t qidx
, uint32_t desc_cnt
)
602 const struct rte_memzone
*rz
;
603 uint32_t ring_size
= SND_QUEUE_SZ_MAX
* sizeof(union sq_entry_t
);
605 rz
= rte_eth_dma_zone_reserve(dev
, "sq",
606 nicvf_netdev_qidx(nic
, qidx
), ring_size
,
607 NICVF_SQ_BASE_ALIGN_BYTES
, nic
->node
);
609 PMD_INIT_LOG(ERR
, "Failed allocate mem for sq hw ring");
613 memset(rz
->addr
, 0, ring_size
);
615 sq
->phys
= rz
->phys_addr
;
617 sq
->qlen_mask
= desc_cnt
- 1;
623 nicvf_qset_rbdr_alloc(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
624 uint32_t desc_cnt
, uint32_t buffsz
)
626 struct nicvf_rbdr
*rbdr
;
627 const struct rte_memzone
*rz
;
630 assert(nic
->rbdr
== NULL
);
631 rbdr
= rte_zmalloc_socket("rbdr", sizeof(struct nicvf_rbdr
),
632 RTE_CACHE_LINE_SIZE
, nic
->node
);
634 PMD_INIT_LOG(ERR
, "Failed to allocate mem for rbdr");
638 ring_size
= sizeof(struct rbdr_entry_t
) * RBDR_QUEUE_SZ_MAX
;
639 rz
= rte_eth_dma_zone_reserve(dev
, "rbdr",
640 nicvf_netdev_qidx(nic
, 0), ring_size
,
641 NICVF_RBDR_BASE_ALIGN_BYTES
, nic
->node
);
643 PMD_INIT_LOG(ERR
, "Failed to allocate mem for rbdr desc ring");
647 memset(rz
->addr
, 0, ring_size
);
649 rbdr
->phys
= rz
->phys_addr
;
652 rbdr
->desc
= rz
->addr
;
653 rbdr
->buffsz
= buffsz
;
654 rbdr
->qlen_mask
= desc_cnt
- 1;
656 nicvf_qset_base(nic
, 0) + NIC_QSET_RBDR_0_1_STATUS0
;
658 nicvf_qset_base(nic
, 0) + NIC_QSET_RBDR_0_1_DOOR
;
665 nicvf_rbdr_release_mbuf(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
666 nicvf_phys_addr_t phy
)
670 struct nicvf_rxq
*rxq
;
671 uint16_t rx_start
, rx_end
;
673 /* Get queue ranges for this VF */
674 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
676 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
677 rxq
= dev
->data
->rx_queues
[qidx
];
678 if (rxq
->precharge_cnt
) {
679 obj
= (void *)nicvf_mbuff_phy2virt(phy
,
681 rte_mempool_put(rxq
->pool
, obj
);
682 rxq
->precharge_cnt
--;
689 nicvf_rbdr_release_mbufs(struct rte_eth_dev
*dev
, struct nicvf
*nic
)
691 uint32_t qlen_mask
, head
;
692 struct rbdr_entry_t
*entry
;
693 struct nicvf_rbdr
*rbdr
= nic
->rbdr
;
695 qlen_mask
= rbdr
->qlen_mask
;
697 while (head
!= rbdr
->tail
) {
698 entry
= rbdr
->desc
+ head
;
699 nicvf_rbdr_release_mbuf(dev
, nic
, entry
->full_addr
);
701 head
= head
& qlen_mask
;
706 nicvf_tx_queue_release_mbufs(struct nicvf_txq
*txq
)
711 while (head
!= txq
->tail
) {
712 if (txq
->txbuffs
[head
]) {
713 rte_pktmbuf_free_seg(txq
->txbuffs
[head
]);
714 txq
->txbuffs
[head
] = NULL
;
717 head
= head
& txq
->qlen_mask
;
722 nicvf_tx_queue_reset(struct nicvf_txq
*txq
)
724 uint32_t txq_desc_cnt
= txq
->qlen_mask
+ 1;
726 memset(txq
->desc
, 0, sizeof(union sq_entry_t
) * txq_desc_cnt
);
727 memset(txq
->txbuffs
, 0, sizeof(struct rte_mbuf
*) * txq_desc_cnt
);
734 nicvf_vf_start_tx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
737 struct nicvf_txq
*txq
;
740 assert(qidx
< MAX_SND_QUEUES_PER_QS
);
742 if (dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
743 RTE_ETH_QUEUE_STATE_STARTED
)
746 txq
= dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
748 ret
= nicvf_qset_sq_config(nic
, qidx
, txq
);
750 PMD_INIT_LOG(ERR
, "Failed to configure sq VF%d %d %d",
751 nic
->vf_id
, qidx
, ret
);
752 goto config_sq_error
;
755 dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
756 RTE_ETH_QUEUE_STATE_STARTED
;
760 nicvf_qset_sq_reclaim(nic
, qidx
);
765 nicvf_vf_stop_tx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
768 struct nicvf_txq
*txq
;
771 assert(qidx
< MAX_SND_QUEUES_PER_QS
);
773 if (dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
774 RTE_ETH_QUEUE_STATE_STOPPED
)
777 ret
= nicvf_qset_sq_reclaim(nic
, qidx
);
779 PMD_INIT_LOG(ERR
, "Failed to reclaim sq VF%d %d %d",
780 nic
->vf_id
, qidx
, ret
);
782 txq
= dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
783 nicvf_tx_queue_release_mbufs(txq
);
784 nicvf_tx_queue_reset(txq
);
786 dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
787 RTE_ETH_QUEUE_STATE_STOPPED
;
792 nicvf_configure_cpi(struct rte_eth_dev
*dev
)
794 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
798 /* Count started rx queues */
799 for (qidx
= qcnt
= 0; qidx
< dev
->data
->nb_rx_queues
; qidx
++)
800 if (dev
->data
->rx_queue_state
[qidx
] ==
801 RTE_ETH_QUEUE_STATE_STARTED
)
804 nic
->cpi_alg
= CPI_ALG_NONE
;
805 ret
= nicvf_mbox_config_cpi(nic
, qcnt
);
807 PMD_INIT_LOG(ERR
, "Failed to configure CPI %d", ret
);
813 nicvf_configure_rss(struct rte_eth_dev
*dev
)
815 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
819 rsshf
= nicvf_rss_ethdev_to_nic(nic
,
820 dev
->data
->dev_conf
.rx_adv_conf
.rss_conf
.rss_hf
);
821 PMD_DRV_LOG(INFO
, "mode=%d rx_queues=%d loopback=%d rsshf=0x%" PRIx64
,
822 dev
->data
->dev_conf
.rxmode
.mq_mode
,
823 dev
->data
->nb_rx_queues
,
824 dev
->data
->dev_conf
.lpbk_mode
, rsshf
);
826 if (dev
->data
->dev_conf
.rxmode
.mq_mode
== ETH_MQ_RX_NONE
)
827 ret
= nicvf_rss_term(nic
);
828 else if (dev
->data
->dev_conf
.rxmode
.mq_mode
== ETH_MQ_RX_RSS
)
829 ret
= nicvf_rss_config(nic
, dev
->data
->nb_rx_queues
, rsshf
);
831 PMD_INIT_LOG(ERR
, "Failed to configure RSS %d", ret
);
837 nicvf_configure_rss_reta(struct rte_eth_dev
*dev
)
839 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
840 unsigned int idx
, qmap_size
;
841 uint8_t qmap
[RTE_MAX_QUEUES_PER_PORT
];
842 uint8_t default_reta
[NIC_MAX_RSS_IDR_TBL_SIZE
];
844 if (nic
->cpi_alg
!= CPI_ALG_NONE
)
847 /* Prepare queue map */
848 for (idx
= 0, qmap_size
= 0; idx
< dev
->data
->nb_rx_queues
; idx
++) {
849 if (dev
->data
->rx_queue_state
[idx
] ==
850 RTE_ETH_QUEUE_STATE_STARTED
)
851 qmap
[qmap_size
++] = idx
;
854 /* Update default RSS RETA */
855 for (idx
= 0; idx
< NIC_MAX_RSS_IDR_TBL_SIZE
; idx
++)
856 default_reta
[idx
] = qmap
[idx
% qmap_size
];
858 return nicvf_rss_reta_update(nic
, default_reta
,
859 NIC_MAX_RSS_IDR_TBL_SIZE
);
863 nicvf_dev_tx_queue_release(void *sq
)
865 struct nicvf_txq
*txq
;
867 PMD_INIT_FUNC_TRACE();
869 txq
= (struct nicvf_txq
*)sq
;
871 if (txq
->txbuffs
!= NULL
) {
872 nicvf_tx_queue_release_mbufs(txq
);
873 rte_free(txq
->txbuffs
);
881 nicvf_set_tx_function(struct rte_eth_dev
*dev
)
883 struct nicvf_txq
*txq
;
885 bool multiseg
= false;
887 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
888 txq
= dev
->data
->tx_queues
[i
];
889 if ((txq
->txq_flags
& ETH_TXQ_FLAGS_NOMULTSEGS
) == 0) {
895 /* Use a simple Tx queue (no offloads, no multi segs) if possible */
897 PMD_DRV_LOG(DEBUG
, "Using multi-segment tx callback");
898 dev
->tx_pkt_burst
= nicvf_xmit_pkts_multiseg
;
900 PMD_DRV_LOG(DEBUG
, "Using single-segment tx callback");
901 dev
->tx_pkt_burst
= nicvf_xmit_pkts
;
904 if (txq
->pool_free
== nicvf_single_pool_free_xmited_buffers
)
905 PMD_DRV_LOG(DEBUG
, "Using single-mempool tx free method");
907 PMD_DRV_LOG(DEBUG
, "Using multi-mempool tx free method");
911 nicvf_set_rx_function(struct rte_eth_dev
*dev
)
913 if (dev
->data
->scattered_rx
) {
914 PMD_DRV_LOG(DEBUG
, "Using multi-segment rx callback");
915 dev
->rx_pkt_burst
= nicvf_recv_pkts_multiseg
;
917 PMD_DRV_LOG(DEBUG
, "Using single-segment rx callback");
918 dev
->rx_pkt_burst
= nicvf_recv_pkts
;
923 nicvf_dev_tx_queue_setup(struct rte_eth_dev
*dev
, uint16_t qidx
,
924 uint16_t nb_desc
, unsigned int socket_id
,
925 const struct rte_eth_txconf
*tx_conf
)
927 uint16_t tx_free_thresh
;
928 uint8_t is_single_pool
;
929 struct nicvf_txq
*txq
;
930 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
932 PMD_INIT_FUNC_TRACE();
934 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
935 nic
= nic
->snicvf
[qidx
/ MAX_SND_QUEUES_PER_QS
- 1];
937 qidx
= qidx
% MAX_SND_QUEUES_PER_QS
;
939 /* Socket id check */
940 if (socket_id
!= (unsigned int)SOCKET_ID_ANY
&& socket_id
!= nic
->node
)
941 PMD_DRV_LOG(WARNING
, "socket_id expected %d, configured %d",
942 socket_id
, nic
->node
);
944 /* Tx deferred start is not supported */
945 if (tx_conf
->tx_deferred_start
) {
946 PMD_INIT_LOG(ERR
, "Tx deferred start not supported");
950 /* Roundup nb_desc to available qsize and validate max number of desc */
951 nb_desc
= nicvf_qsize_sq_roundup(nb_desc
);
953 PMD_INIT_LOG(ERR
, "Value of nb_desc beyond available sq qsize");
957 /* Validate tx_free_thresh */
958 tx_free_thresh
= (uint16_t)((tx_conf
->tx_free_thresh
) ?
959 tx_conf
->tx_free_thresh
:
960 NICVF_DEFAULT_TX_FREE_THRESH
);
962 if (tx_free_thresh
> (nb_desc
) ||
963 tx_free_thresh
> NICVF_MAX_TX_FREE_THRESH
) {
965 "tx_free_thresh must be less than the number of TX "
966 "descriptors. (tx_free_thresh=%u port=%d "
967 "queue=%d)", (unsigned int)tx_free_thresh
,
968 (int)dev
->data
->port_id
, (int)qidx
);
972 /* Free memory prior to re-allocation if needed. */
973 if (dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)] != NULL
) {
974 PMD_TX_LOG(DEBUG
, "Freeing memory prior to re-allocation %d",
975 nicvf_netdev_qidx(nic
, qidx
));
976 nicvf_dev_tx_queue_release(
977 dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)]);
978 dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = NULL
;
981 /* Allocating tx queue data structure */
982 txq
= rte_zmalloc_socket("ethdev TX queue", sizeof(struct nicvf_txq
),
983 RTE_CACHE_LINE_SIZE
, nic
->node
);
985 PMD_INIT_LOG(ERR
, "Failed to allocate txq=%d",
986 nicvf_netdev_qidx(nic
, qidx
));
991 txq
->queue_id
= qidx
;
992 txq
->tx_free_thresh
= tx_free_thresh
;
993 txq
->txq_flags
= tx_conf
->txq_flags
;
994 txq
->sq_head
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_SQ_0_7_HEAD
;
995 txq
->sq_door
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_SQ_0_7_DOOR
;
996 is_single_pool
= (txq
->txq_flags
& ETH_TXQ_FLAGS_NOREFCOUNT
&&
997 txq
->txq_flags
& ETH_TXQ_FLAGS_NOMULTMEMP
);
999 /* Choose optimum free threshold value for multipool case */
1000 if (!is_single_pool
) {
1001 txq
->tx_free_thresh
= (uint16_t)
1002 (tx_conf
->tx_free_thresh
== NICVF_DEFAULT_TX_FREE_THRESH
?
1003 NICVF_TX_FREE_MPOOL_THRESH
:
1004 tx_conf
->tx_free_thresh
);
1005 txq
->pool_free
= nicvf_multi_pool_free_xmited_buffers
;
1007 txq
->pool_free
= nicvf_single_pool_free_xmited_buffers
;
1010 /* Allocate software ring */
1011 txq
->txbuffs
= rte_zmalloc_socket("txq->txbuffs",
1012 nb_desc
* sizeof(struct rte_mbuf
*),
1013 RTE_CACHE_LINE_SIZE
, nic
->node
);
1015 if (txq
->txbuffs
== NULL
) {
1016 nicvf_dev_tx_queue_release(txq
);
1020 if (nicvf_qset_sq_alloc(dev
, nic
, txq
, qidx
, nb_desc
)) {
1021 PMD_INIT_LOG(ERR
, "Failed to allocate mem for sq %d", qidx
);
1022 nicvf_dev_tx_queue_release(txq
);
1026 nicvf_tx_queue_reset(txq
);
1028 PMD_TX_LOG(DEBUG
, "[%d] txq=%p nb_desc=%d desc=%p phys=0x%" PRIx64
,
1029 nicvf_netdev_qidx(nic
, qidx
), txq
, nb_desc
, txq
->desc
,
1032 dev
->data
->tx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = txq
;
1033 dev
->data
->tx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1034 RTE_ETH_QUEUE_STATE_STOPPED
;
1039 nicvf_rx_queue_release_mbufs(struct rte_eth_dev
*dev
, struct nicvf_rxq
*rxq
)
1042 uint32_t nb_pkts
, released_pkts
= 0;
1043 uint32_t refill_cnt
= 0;
1044 struct rte_mbuf
*rx_pkts
[NICVF_MAX_RX_FREE_THRESH
];
1046 if (dev
->rx_pkt_burst
== NULL
)
1049 while ((rxq_cnt
= nicvf_dev_rx_queue_count(dev
,
1050 nicvf_netdev_qidx(rxq
->nic
, rxq
->queue_id
)))) {
1051 nb_pkts
= dev
->rx_pkt_burst(rxq
, rx_pkts
,
1052 NICVF_MAX_RX_FREE_THRESH
);
1053 PMD_DRV_LOG(INFO
, "nb_pkts=%d rxq_cnt=%d", nb_pkts
, rxq_cnt
);
1055 rte_pktmbuf_free_seg(rx_pkts
[--nb_pkts
]);
1061 refill_cnt
+= nicvf_dev_rbdr_refill(dev
,
1062 nicvf_netdev_qidx(rxq
->nic
, rxq
->queue_id
));
1064 PMD_DRV_LOG(INFO
, "free_cnt=%d refill_cnt=%d",
1065 released_pkts
, refill_cnt
);
1069 nicvf_rx_queue_reset(struct nicvf_rxq
*rxq
)
1072 rxq
->available_space
= 0;
1073 rxq
->recv_buffers
= 0;
1077 nicvf_vf_start_rx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
1080 struct nicvf_rxq
*rxq
;
1083 assert(qidx
< MAX_RCV_QUEUES_PER_QS
);
1085 if (dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
1086 RTE_ETH_QUEUE_STATE_STARTED
)
1089 /* Update rbdr pointer to all rxq */
1090 rxq
= dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
1091 rxq
->shared_rbdr
= nic
->rbdr
;
1093 ret
= nicvf_qset_rq_config(nic
, qidx
, rxq
);
1095 PMD_INIT_LOG(ERR
, "Failed to configure rq VF%d %d %d",
1096 nic
->vf_id
, qidx
, ret
);
1097 goto config_rq_error
;
1099 ret
= nicvf_qset_cq_config(nic
, qidx
, rxq
);
1101 PMD_INIT_LOG(ERR
, "Failed to configure cq VF%d %d %d",
1102 nic
->vf_id
, qidx
, ret
);
1103 goto config_cq_error
;
1106 dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1107 RTE_ETH_QUEUE_STATE_STARTED
;
1111 nicvf_qset_cq_reclaim(nic
, qidx
);
1113 nicvf_qset_rq_reclaim(nic
, qidx
);
1118 nicvf_vf_stop_rx_queue(struct rte_eth_dev
*dev
, struct nicvf
*nic
,
1121 struct nicvf_rxq
*rxq
;
1122 int ret
, other_error
;
1124 if (dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] ==
1125 RTE_ETH_QUEUE_STATE_STOPPED
)
1128 ret
= nicvf_qset_rq_reclaim(nic
, qidx
);
1130 PMD_INIT_LOG(ERR
, "Failed to reclaim rq VF%d %d %d",
1131 nic
->vf_id
, qidx
, ret
);
1134 rxq
= dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)];
1135 nicvf_rx_queue_release_mbufs(dev
, rxq
);
1136 nicvf_rx_queue_reset(rxq
);
1138 ret
= nicvf_qset_cq_reclaim(nic
, qidx
);
1140 PMD_INIT_LOG(ERR
, "Failed to reclaim cq VF%d %d %d",
1141 nic
->vf_id
, qidx
, ret
);
1144 dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1145 RTE_ETH_QUEUE_STATE_STOPPED
;
1150 nicvf_dev_rx_queue_release(void *rx_queue
)
1152 PMD_INIT_FUNC_TRACE();
1158 nicvf_dev_rx_queue_start(struct rte_eth_dev
*dev
, uint16_t qidx
)
1160 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1163 if (qidx
>= MAX_RCV_QUEUES_PER_QS
)
1164 nic
= nic
->snicvf
[(qidx
/ MAX_RCV_QUEUES_PER_QS
- 1)];
1166 qidx
= qidx
% MAX_RCV_QUEUES_PER_QS
;
1168 ret
= nicvf_vf_start_rx_queue(dev
, nic
, qidx
);
1172 ret
= nicvf_configure_cpi(dev
);
1176 return nicvf_configure_rss_reta(dev
);
1180 nicvf_dev_rx_queue_stop(struct rte_eth_dev
*dev
, uint16_t qidx
)
1183 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1185 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
1186 nic
= nic
->snicvf
[(qidx
/ MAX_SND_QUEUES_PER_QS
- 1)];
1188 qidx
= qidx
% MAX_RCV_QUEUES_PER_QS
;
1190 ret
= nicvf_vf_stop_rx_queue(dev
, nic
, qidx
);
1191 ret
|= nicvf_configure_cpi(dev
);
1192 ret
|= nicvf_configure_rss_reta(dev
);
1197 nicvf_dev_tx_queue_start(struct rte_eth_dev
*dev
, uint16_t qidx
)
1199 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1201 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
1202 nic
= nic
->snicvf
[(qidx
/ MAX_SND_QUEUES_PER_QS
- 1)];
1204 qidx
= qidx
% MAX_SND_QUEUES_PER_QS
;
1206 return nicvf_vf_start_tx_queue(dev
, nic
, qidx
);
1210 nicvf_dev_tx_queue_stop(struct rte_eth_dev
*dev
, uint16_t qidx
)
1212 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1214 if (qidx
>= MAX_SND_QUEUES_PER_QS
)
1215 nic
= nic
->snicvf
[(qidx
/ MAX_SND_QUEUES_PER_QS
- 1)];
1217 qidx
= qidx
% MAX_SND_QUEUES_PER_QS
;
1219 return nicvf_vf_stop_tx_queue(dev
, nic
, qidx
);
1224 nicvf_dev_rx_queue_setup(struct rte_eth_dev
*dev
, uint16_t qidx
,
1225 uint16_t nb_desc
, unsigned int socket_id
,
1226 const struct rte_eth_rxconf
*rx_conf
,
1227 struct rte_mempool
*mp
)
1229 uint16_t rx_free_thresh
;
1230 struct nicvf_rxq
*rxq
;
1231 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1233 PMD_INIT_FUNC_TRACE();
1235 if (qidx
>= MAX_RCV_QUEUES_PER_QS
)
1236 nic
= nic
->snicvf
[qidx
/ MAX_RCV_QUEUES_PER_QS
- 1];
1238 qidx
= qidx
% MAX_RCV_QUEUES_PER_QS
;
1240 /* Socket id check */
1241 if (socket_id
!= (unsigned int)SOCKET_ID_ANY
&& socket_id
!= nic
->node
)
1242 PMD_DRV_LOG(WARNING
, "socket_id expected %d, configured %d",
1243 socket_id
, nic
->node
);
1245 /* Mempool memory must be contiguous, so must be one memory segment*/
1246 if (mp
->nb_mem_chunks
!= 1) {
1247 PMD_INIT_LOG(ERR
, "Non-contiguous mempool, add more huge pages");
1251 /* Mempool memory must be physically contiguous */
1252 if (mp
->flags
& MEMPOOL_F_NO_PHYS_CONTIG
) {
1253 PMD_INIT_LOG(ERR
, "Mempool memory must be physically contiguous");
1257 /* Rx deferred start is not supported */
1258 if (rx_conf
->rx_deferred_start
) {
1259 PMD_INIT_LOG(ERR
, "Rx deferred start not supported");
1263 /* Roundup nb_desc to available qsize and validate max number of desc */
1264 nb_desc
= nicvf_qsize_cq_roundup(nb_desc
);
1266 PMD_INIT_LOG(ERR
, "Value nb_desc beyond available hw cq qsize");
1270 /* Check rx_free_thresh upper bound */
1271 rx_free_thresh
= (uint16_t)((rx_conf
->rx_free_thresh
) ?
1272 rx_conf
->rx_free_thresh
:
1273 NICVF_DEFAULT_RX_FREE_THRESH
);
1274 if (rx_free_thresh
> NICVF_MAX_RX_FREE_THRESH
||
1275 rx_free_thresh
>= nb_desc
* .75) {
1276 PMD_INIT_LOG(ERR
, "rx_free_thresh greater than expected %d",
1281 /* Free memory prior to re-allocation if needed */
1282 if (dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)] != NULL
) {
1283 PMD_RX_LOG(DEBUG
, "Freeing memory prior to re-allocation %d",
1284 nicvf_netdev_qidx(nic
, qidx
));
1285 nicvf_dev_rx_queue_release(
1286 dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)]);
1287 dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = NULL
;
1290 /* Allocate rxq memory */
1291 rxq
= rte_zmalloc_socket("ethdev rx queue", sizeof(struct nicvf_rxq
),
1292 RTE_CACHE_LINE_SIZE
, nic
->node
);
1294 PMD_INIT_LOG(ERR
, "Failed to allocate rxq=%d",
1295 nicvf_netdev_qidx(nic
, qidx
));
1301 rxq
->queue_id
= qidx
;
1302 rxq
->port_id
= dev
->data
->port_id
;
1303 rxq
->rx_free_thresh
= rx_free_thresh
;
1304 rxq
->rx_drop_en
= rx_conf
->rx_drop_en
;
1305 rxq
->cq_status
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_CQ_0_7_STATUS
;
1306 rxq
->cq_door
= nicvf_qset_base(nic
, qidx
) + NIC_QSET_CQ_0_7_DOOR
;
1307 rxq
->precharge_cnt
= 0;
1309 if (nicvf_hw_cap(nic
) & NICVF_CAP_CQE_RX2
)
1310 rxq
->rbptr_offset
= NICVF_CQE_RX2_RBPTR_WORD
;
1312 rxq
->rbptr_offset
= NICVF_CQE_RBPTR_WORD
;
1315 /* Alloc completion queue */
1316 if (nicvf_qset_cq_alloc(dev
, nic
, rxq
, rxq
->queue_id
, nb_desc
)) {
1317 PMD_INIT_LOG(ERR
, "failed to allocate cq %u", rxq
->queue_id
);
1318 nicvf_dev_rx_queue_release(rxq
);
1322 nicvf_rx_queue_reset(rxq
);
1324 PMD_RX_LOG(DEBUG
, "[%d] rxq=%p pool=%s nb_desc=(%d/%d) phy=%" PRIx64
,
1325 nicvf_netdev_qidx(nic
, qidx
), rxq
, mp
->name
, nb_desc
,
1326 rte_mempool_avail_count(mp
), rxq
->phys
);
1328 dev
->data
->rx_queues
[nicvf_netdev_qidx(nic
, qidx
)] = rxq
;
1329 dev
->data
->rx_queue_state
[nicvf_netdev_qidx(nic
, qidx
)] =
1330 RTE_ETH_QUEUE_STATE_STOPPED
;
1335 nicvf_dev_info_get(struct rte_eth_dev
*dev
, struct rte_eth_dev_info
*dev_info
)
1337 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1339 PMD_INIT_FUNC_TRACE();
1341 dev_info
->min_rx_bufsize
= ETHER_MIN_MTU
;
1342 dev_info
->max_rx_pktlen
= NIC_HW_MAX_FRS
;
1343 dev_info
->max_rx_queues
=
1344 (uint16_t)MAX_RCV_QUEUES_PER_QS
* (MAX_SQS_PER_VF
+ 1);
1345 dev_info
->max_tx_queues
=
1346 (uint16_t)MAX_SND_QUEUES_PER_QS
* (MAX_SQS_PER_VF
+ 1);
1347 dev_info
->max_mac_addrs
= 1;
1348 dev_info
->max_vfs
= dev
->pci_dev
->max_vfs
;
1350 dev_info
->rx_offload_capa
= DEV_RX_OFFLOAD_VLAN_STRIP
;
1351 dev_info
->tx_offload_capa
=
1352 DEV_TX_OFFLOAD_IPV4_CKSUM
|
1353 DEV_TX_OFFLOAD_UDP_CKSUM
|
1354 DEV_TX_OFFLOAD_TCP_CKSUM
|
1355 DEV_TX_OFFLOAD_TCP_TSO
|
1356 DEV_TX_OFFLOAD_OUTER_IPV4_CKSUM
;
1358 dev_info
->reta_size
= nic
->rss_info
.rss_size
;
1359 dev_info
->hash_key_size
= RSS_HASH_KEY_BYTE_SIZE
;
1360 dev_info
->flow_type_rss_offloads
= NICVF_RSS_OFFLOAD_PASS1
;
1361 if (nicvf_hw_cap(nic
) & NICVF_CAP_TUNNEL_PARSING
)
1362 dev_info
->flow_type_rss_offloads
|= NICVF_RSS_OFFLOAD_TUNNEL
;
1364 dev_info
->default_rxconf
= (struct rte_eth_rxconf
) {
1365 .rx_free_thresh
= NICVF_DEFAULT_RX_FREE_THRESH
,
1369 dev_info
->default_txconf
= (struct rte_eth_txconf
) {
1370 .tx_free_thresh
= NICVF_DEFAULT_TX_FREE_THRESH
,
1372 ETH_TXQ_FLAGS_NOMULTSEGS
|
1373 ETH_TXQ_FLAGS_NOREFCOUNT
|
1374 ETH_TXQ_FLAGS_NOMULTMEMP
|
1375 ETH_TXQ_FLAGS_NOVLANOFFL
|
1376 ETH_TXQ_FLAGS_NOXSUMSCTP
,
1380 static nicvf_phys_addr_t
1381 rbdr_rte_mempool_get(void *dev
, void *opaque
)
1385 struct nicvf_rxq
*rxq
;
1386 struct rte_eth_dev
*eth_dev
= (struct rte_eth_dev
*)dev
;
1387 struct nicvf
*nic
= (struct nicvf
*)opaque
;
1388 uint16_t rx_start
, rx_end
;
1390 /* Get queue ranges for this VF */
1391 nicvf_rx_range(eth_dev
, nic
, &rx_start
, &rx_end
);
1393 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1394 rxq
= eth_dev
->data
->rx_queues
[qidx
];
1395 /* Maintain equal buffer count across all pools */
1396 if (rxq
->precharge_cnt
>= rxq
->qlen_mask
)
1398 rxq
->precharge_cnt
++;
1399 mbuf
= (uintptr_t)rte_pktmbuf_alloc(rxq
->pool
);
1401 return nicvf_mbuff_virt2phy(mbuf
, rxq
->mbuf_phys_off
);
1407 nicvf_vf_start(struct rte_eth_dev
*dev
, struct nicvf
*nic
, uint32_t rbdrsz
)
1411 uint32_t total_rxq_desc
, nb_rbdr_desc
, exp_buffs
;
1412 uint64_t mbuf_phys_off
= 0;
1413 struct nicvf_rxq
*rxq
;
1414 struct rte_mbuf
*mbuf
;
1415 uint16_t rx_start
, rx_end
;
1416 uint16_t tx_start
, tx_end
;
1418 PMD_INIT_FUNC_TRACE();
1420 /* Userspace process exited without proper shutdown in last run */
1421 if (nicvf_qset_rbdr_active(nic
, 0))
1422 nicvf_vf_stop(dev
, nic
, false);
1424 /* Get queue ranges for this VF */
1425 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
1428 * Thunderx nicvf PMD can support more than one pool per port only when
1429 * 1) Data payload size is same across all the pools in given port
1431 * 2) All mbuffs in the pools are from the same hugepage
1433 * 3) Mbuff metadata size is same across all the pools in given port
1435 * This is to support existing application that uses multiple pool/port.
1436 * But, the purpose of using multipool for QoS will not be addressed.
1440 /* Validate mempool attributes */
1441 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1442 rxq
= dev
->data
->rx_queues
[qidx
];
1443 rxq
->mbuf_phys_off
= nicvf_mempool_phy_offset(rxq
->pool
);
1444 mbuf
= rte_pktmbuf_alloc(rxq
->pool
);
1446 PMD_INIT_LOG(ERR
, "Failed allocate mbuf VF%d qid=%d "
1448 nic
->vf_id
, qidx
, rxq
->pool
->name
);
1451 rxq
->mbuf_phys_off
-= nicvf_mbuff_meta_length(mbuf
);
1452 rxq
->mbuf_phys_off
-= RTE_PKTMBUF_HEADROOM
;
1453 rte_pktmbuf_free(mbuf
);
1455 if (mbuf_phys_off
== 0)
1456 mbuf_phys_off
= rxq
->mbuf_phys_off
;
1457 if (mbuf_phys_off
!= rxq
->mbuf_phys_off
) {
1458 PMD_INIT_LOG(ERR
, "pool params not same,%s VF%d %"
1459 PRIx64
, rxq
->pool
->name
, nic
->vf_id
,
1465 /* Check the level of buffers in the pool */
1467 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1468 rxq
= dev
->data
->rx_queues
[qidx
];
1469 /* Count total numbers of rxq descs */
1470 total_rxq_desc
+= rxq
->qlen_mask
+ 1;
1471 exp_buffs
= RTE_MEMPOOL_CACHE_MAX_SIZE
+ rxq
->rx_free_thresh
;
1472 exp_buffs
*= dev
->data
->nb_rx_queues
;
1473 if (rte_mempool_avail_count(rxq
->pool
) < exp_buffs
) {
1474 PMD_INIT_LOG(ERR
, "Buff shortage in pool=%s (%d/%d)",
1476 rte_mempool_avail_count(rxq
->pool
),
1482 /* Check RBDR desc overflow */
1483 ret
= nicvf_qsize_rbdr_roundup(total_rxq_desc
);
1485 PMD_INIT_LOG(ERR
, "Reached RBDR desc limit, reduce nr desc "
1486 "VF%d", nic
->vf_id
);
1491 ret
= nicvf_qset_config(nic
);
1493 PMD_INIT_LOG(ERR
, "Failed to enable qset %d VF%d", ret
,
1498 /* Allocate RBDR and RBDR ring desc */
1499 nb_rbdr_desc
= nicvf_qsize_rbdr_roundup(total_rxq_desc
);
1500 ret
= nicvf_qset_rbdr_alloc(dev
, nic
, nb_rbdr_desc
, rbdrsz
);
1502 PMD_INIT_LOG(ERR
, "Failed to allocate memory for rbdr alloc "
1503 "VF%d", nic
->vf_id
);
1507 /* Enable and configure RBDR registers */
1508 ret
= nicvf_qset_rbdr_config(nic
, 0);
1510 PMD_INIT_LOG(ERR
, "Failed to configure rbdr %d VF%d", ret
,
1512 goto qset_rbdr_free
;
1515 /* Fill rte_mempool buffers in RBDR pool and precharge it */
1516 ret
= nicvf_qset_rbdr_precharge(dev
, nic
, 0, rbdr_rte_mempool_get
,
1519 PMD_INIT_LOG(ERR
, "Failed to fill rbdr %d VF%d", ret
,
1521 goto qset_rbdr_reclaim
;
1524 PMD_DRV_LOG(INFO
, "Filled %d out of %d entries in RBDR VF%d",
1525 nic
->rbdr
->tail
, nb_rbdr_desc
, nic
->vf_id
);
1527 /* Configure VLAN Strip */
1528 nicvf_vlan_hw_strip(nic
, dev
->data
->dev_conf
.rxmode
.hw_vlan_strip
);
1530 /* Based on the packet type(IPv4 or IPv6), the nicvf HW aligns L3 data
1531 * to the 64bit memory address.
1532 * The alignment creates a hole in mbuf(between the end of headroom and
1533 * packet data start). The new revision of the HW provides an option to
1534 * disable the L3 alignment feature and make mbuf layout looks
1535 * more like other NICs. For better application compatibility, disabling
1536 * l3 alignment feature on the hardware revisions it supports
1538 nicvf_apad_config(nic
, false);
1540 /* Get queue ranges for this VF */
1541 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
1543 /* Configure TX queues */
1544 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++) {
1545 ret
= nicvf_vf_start_tx_queue(dev
, nic
,
1546 qidx
% MAX_SND_QUEUES_PER_QS
);
1548 goto start_txq_error
;
1551 /* Configure RX queues */
1552 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++) {
1553 ret
= nicvf_vf_start_rx_queue(dev
, nic
,
1554 qidx
% MAX_RCV_QUEUES_PER_QS
);
1556 goto start_rxq_error
;
1559 if (!nic
->sqs_mode
) {
1560 /* Configure CPI algorithm */
1561 ret
= nicvf_configure_cpi(dev
);
1563 goto start_txq_error
;
1565 ret
= nicvf_mbox_get_rss_size(nic
);
1567 PMD_INIT_LOG(ERR
, "Failed to get rss table size");
1568 goto qset_rss_error
;
1572 ret
= nicvf_configure_rss(dev
);
1574 goto qset_rss_error
;
1577 /* Done; Let PF make the BGX's RX and TX switches to ON position */
1578 nicvf_mbox_cfg_done(nic
);
1582 nicvf_rss_term(nic
);
1584 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++)
1585 nicvf_vf_stop_rx_queue(dev
, nic
, qidx
% MAX_RCV_QUEUES_PER_QS
);
1587 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++)
1588 nicvf_vf_stop_tx_queue(dev
, nic
, qidx
% MAX_SND_QUEUES_PER_QS
);
1590 nicvf_qset_rbdr_reclaim(nic
, 0);
1591 nicvf_rbdr_release_mbufs(dev
, nic
);
1594 rte_free(nic
->rbdr
);
1598 nicvf_qset_reclaim(nic
);
1603 nicvf_dev_start(struct rte_eth_dev
*dev
)
1608 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1609 struct rte_eth_rxmode
*rx_conf
= &dev
->data
->dev_conf
.rxmode
;
1611 uint32_t buffsz
= 0, rbdrsz
= 0;
1612 struct rte_pktmbuf_pool_private
*mbp_priv
;
1613 struct nicvf_rxq
*rxq
;
1615 PMD_INIT_FUNC_TRACE();
1617 /* This function must be called for a primary device */
1618 assert_primary(nic
);
1620 /* Validate RBDR buff size */
1621 for (qidx
= 0; qidx
< dev
->data
->nb_rx_queues
; qidx
++) {
1622 rxq
= dev
->data
->rx_queues
[qidx
];
1623 mbp_priv
= rte_mempool_get_priv(rxq
->pool
);
1624 buffsz
= mbp_priv
->mbuf_data_room_size
- RTE_PKTMBUF_HEADROOM
;
1626 PMD_INIT_LOG(ERR
, "rxbuf size must be multiply of 128");
1631 if (rbdrsz
!= buffsz
) {
1632 PMD_INIT_LOG(ERR
, "buffsz not same, qidx=%d (%d/%d)",
1633 qidx
, rbdrsz
, buffsz
);
1638 /* Configure loopback */
1639 ret
= nicvf_loopback_config(nic
, dev
->data
->dev_conf
.lpbk_mode
);
1641 PMD_INIT_LOG(ERR
, "Failed to configure loopback %d", ret
);
1645 /* Reset all statistics counters attached to this port */
1646 ret
= nicvf_mbox_reset_stat_counters(nic
, 0x3FFF, 0x1F, 0xFFFF, 0xFFFF);
1648 PMD_INIT_LOG(ERR
, "Failed to reset stat counters %d", ret
);
1652 /* Setup scatter mode if needed by jumbo */
1653 if (dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
+
1654 2 * VLAN_TAG_SIZE
> buffsz
)
1655 dev
->data
->scattered_rx
= 1;
1656 if (rx_conf
->enable_scatter
)
1657 dev
->data
->scattered_rx
= 1;
1659 /* Setup MTU based on max_rx_pkt_len or default */
1660 mtu
= dev
->data
->dev_conf
.rxmode
.jumbo_frame
?
1661 dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
1662 - ETHER_HDR_LEN
- ETHER_CRC_LEN
1665 if (nicvf_dev_set_mtu(dev
, mtu
)) {
1666 PMD_INIT_LOG(ERR
, "Failed to set default mtu size");
1670 ret
= nicvf_vf_start(dev
, nic
, rbdrsz
);
1674 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1675 assert(nic
->snicvf
[i
]);
1677 ret
= nicvf_vf_start(dev
, nic
->snicvf
[i
], rbdrsz
);
1682 /* Configure callbacks based on scatter mode */
1683 nicvf_set_tx_function(dev
);
1684 nicvf_set_rx_function(dev
);
1690 nicvf_dev_stop_cleanup(struct rte_eth_dev
*dev
, bool cleanup
)
1694 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1696 PMD_INIT_FUNC_TRACE();
1698 /* Teardown secondary vf first */
1699 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1700 if (!nic
->snicvf
[i
])
1703 nicvf_vf_stop(dev
, nic
->snicvf
[i
], cleanup
);
1706 /* Stop the primary VF now */
1707 nicvf_vf_stop(dev
, nic
, cleanup
);
1709 /* Disable loopback */
1710 ret
= nicvf_loopback_config(nic
, 0);
1712 PMD_INIT_LOG(ERR
, "Failed to disable loopback %d", ret
);
1714 /* Reclaim CPI configuration */
1715 ret
= nicvf_mbox_config_cpi(nic
, 0);
1717 PMD_INIT_LOG(ERR
, "Failed to reclaim CPI config %d", ret
);
1721 nicvf_dev_stop(struct rte_eth_dev
*dev
)
1723 PMD_INIT_FUNC_TRACE();
1725 nicvf_dev_stop_cleanup(dev
, false);
1729 nicvf_vf_stop(struct rte_eth_dev
*dev
, struct nicvf
*nic
, bool cleanup
)
1733 uint16_t tx_start
, tx_end
;
1734 uint16_t rx_start
, rx_end
;
1736 PMD_INIT_FUNC_TRACE();
1739 /* Let PF make the BGX's RX and TX switches to OFF position */
1740 nicvf_mbox_shutdown(nic
);
1743 /* Disable VLAN Strip */
1744 nicvf_vlan_hw_strip(nic
, 0);
1746 /* Get queue ranges for this VF */
1747 nicvf_tx_range(dev
, nic
, &tx_start
, &tx_end
);
1749 for (qidx
= tx_start
; qidx
<= tx_end
; qidx
++)
1750 nicvf_vf_stop_tx_queue(dev
, nic
, qidx
% MAX_SND_QUEUES_PER_QS
);
1752 /* Get queue ranges for this VF */
1753 nicvf_rx_range(dev
, nic
, &rx_start
, &rx_end
);
1756 for (qidx
= rx_start
; qidx
<= rx_end
; qidx
++)
1757 nicvf_vf_stop_rx_queue(dev
, nic
, qidx
% MAX_RCV_QUEUES_PER_QS
);
1760 ret
= nicvf_qset_rbdr_reclaim(nic
, 0);
1762 PMD_INIT_LOG(ERR
, "Failed to reclaim RBDR %d", ret
);
1764 /* Move all charged buffers in RBDR back to pool */
1765 if (nic
->rbdr
!= NULL
)
1766 nicvf_rbdr_release_mbufs(dev
, nic
);
1769 ret
= nicvf_qset_reclaim(nic
);
1771 PMD_INIT_LOG(ERR
, "Failed to disable qset %d", ret
);
1773 /* Disable all interrupts */
1774 nicvf_disable_all_interrupts(nic
);
1776 /* Free RBDR SW structure */
1778 rte_free(nic
->rbdr
);
1784 nicvf_dev_close(struct rte_eth_dev
*dev
)
1787 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1789 PMD_INIT_FUNC_TRACE();
1791 nicvf_dev_stop_cleanup(dev
, true);
1792 nicvf_periodic_alarm_stop(nicvf_interrupt
, dev
);
1794 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1795 if (!nic
->snicvf
[i
])
1798 nicvf_periodic_alarm_stop(nicvf_vf_interrupt
, nic
->snicvf
[i
]);
1803 nicvf_request_sqs(struct nicvf
*nic
)
1807 assert_primary(nic
);
1808 assert(nic
->sqs_count
> 0);
1809 assert(nic
->sqs_count
<= MAX_SQS_PER_VF
);
1811 /* Set no of Rx/Tx queues in each of the SQsets */
1812 for (i
= 0; i
< nic
->sqs_count
; i
++) {
1813 if (nicvf_svf_empty())
1814 rte_panic("Cannot assign sufficient number of "
1815 "secondary queues to primary VF%" PRIu8
"\n",
1818 nic
->snicvf
[i
] = nicvf_svf_pop();
1819 nic
->snicvf
[i
]->sqs_id
= i
;
1822 return nicvf_mbox_request_sqs(nic
);
1826 nicvf_dev_configure(struct rte_eth_dev
*dev
)
1828 struct rte_eth_dev_data
*data
= dev
->data
;
1829 struct rte_eth_conf
*conf
= &data
->dev_conf
;
1830 struct rte_eth_rxmode
*rxmode
= &conf
->rxmode
;
1831 struct rte_eth_txmode
*txmode
= &conf
->txmode
;
1832 struct nicvf
*nic
= nicvf_pmd_priv(dev
);
1835 PMD_INIT_FUNC_TRACE();
1837 if (!rte_eal_has_hugepages()) {
1838 PMD_INIT_LOG(INFO
, "Huge page is not configured");
1842 if (txmode
->mq_mode
) {
1843 PMD_INIT_LOG(INFO
, "Tx mq_mode DCB or VMDq not supported");
1847 if (rxmode
->mq_mode
!= ETH_MQ_RX_NONE
&&
1848 rxmode
->mq_mode
!= ETH_MQ_RX_RSS
) {
1849 PMD_INIT_LOG(INFO
, "Unsupported rx qmode %d", rxmode
->mq_mode
);
1853 if (!rxmode
->hw_strip_crc
) {
1854 PMD_INIT_LOG(NOTICE
, "Can't disable hw crc strip");
1855 rxmode
->hw_strip_crc
= 1;
1858 if (rxmode
->hw_ip_checksum
) {
1859 PMD_INIT_LOG(NOTICE
, "Rxcksum not supported");
1860 rxmode
->hw_ip_checksum
= 0;
1863 if (rxmode
->split_hdr_size
) {
1864 PMD_INIT_LOG(INFO
, "Rxmode does not support split header");
1868 if (rxmode
->hw_vlan_filter
) {
1869 PMD_INIT_LOG(INFO
, "VLAN filter not supported");
1873 if (rxmode
->hw_vlan_extend
) {
1874 PMD_INIT_LOG(INFO
, "VLAN extended not supported");
1878 if (rxmode
->enable_lro
) {
1879 PMD_INIT_LOG(INFO
, "LRO not supported");
1883 if (conf
->link_speeds
& ETH_LINK_SPEED_FIXED
) {
1884 PMD_INIT_LOG(INFO
, "Setting link speed/duplex not supported");
1888 if (conf
->dcb_capability_en
) {
1889 PMD_INIT_LOG(INFO
, "DCB enable not supported");
1893 if (conf
->fdir_conf
.mode
!= RTE_FDIR_MODE_NONE
) {
1894 PMD_INIT_LOG(INFO
, "Flow director not supported");
1898 assert_primary(nic
);
1899 NICVF_STATIC_ASSERT(MAX_RCV_QUEUES_PER_QS
== MAX_SND_QUEUES_PER_QS
);
1900 cqcount
= RTE_MAX(data
->nb_tx_queues
, data
->nb_rx_queues
);
1901 if (cqcount
> MAX_RCV_QUEUES_PER_QS
) {
1902 nic
->sqs_count
= RTE_ALIGN_CEIL(cqcount
, MAX_RCV_QUEUES_PER_QS
);
1903 nic
->sqs_count
= (nic
->sqs_count
/ MAX_RCV_QUEUES_PER_QS
) - 1;
1908 assert(nic
->sqs_count
<= MAX_SQS_PER_VF
);
1910 if (nic
->sqs_count
> 0) {
1911 if (nicvf_request_sqs(nic
)) {
1912 rte_panic("Cannot assign sufficient number of "
1913 "secondary queues to PORT%d VF%" PRIu8
"\n",
1914 dev
->data
->port_id
, nic
->vf_id
);
1918 PMD_INIT_LOG(DEBUG
, "Configured ethdev port%d hwcap=0x%" PRIx64
,
1919 dev
->data
->port_id
, nicvf_hw_cap(nic
));
1924 /* Initialize and register driver with DPDK Application */
1925 static const struct eth_dev_ops nicvf_eth_dev_ops
= {
1926 .dev_configure
= nicvf_dev_configure
,
1927 .dev_start
= nicvf_dev_start
,
1928 .dev_stop
= nicvf_dev_stop
,
1929 .link_update
= nicvf_dev_link_update
,
1930 .dev_close
= nicvf_dev_close
,
1931 .stats_get
= nicvf_dev_stats_get
,
1932 .stats_reset
= nicvf_dev_stats_reset
,
1933 .promiscuous_enable
= nicvf_dev_promisc_enable
,
1934 .dev_infos_get
= nicvf_dev_info_get
,
1935 .dev_supported_ptypes_get
= nicvf_dev_supported_ptypes_get
,
1936 .mtu_set
= nicvf_dev_set_mtu
,
1937 .reta_update
= nicvf_dev_reta_update
,
1938 .reta_query
= nicvf_dev_reta_query
,
1939 .rss_hash_update
= nicvf_dev_rss_hash_update
,
1940 .rss_hash_conf_get
= nicvf_dev_rss_hash_conf_get
,
1941 .rx_queue_start
= nicvf_dev_rx_queue_start
,
1942 .rx_queue_stop
= nicvf_dev_rx_queue_stop
,
1943 .tx_queue_start
= nicvf_dev_tx_queue_start
,
1944 .tx_queue_stop
= nicvf_dev_tx_queue_stop
,
1945 .rx_queue_setup
= nicvf_dev_rx_queue_setup
,
1946 .rx_queue_release
= nicvf_dev_rx_queue_release
,
1947 .rx_queue_count
= nicvf_dev_rx_queue_count
,
1948 .tx_queue_setup
= nicvf_dev_tx_queue_setup
,
1949 .tx_queue_release
= nicvf_dev_tx_queue_release
,
1950 .get_reg
= nicvf_dev_get_regs
,
1954 nicvf_eth_dev_init(struct rte_eth_dev
*eth_dev
)
1957 struct rte_pci_device
*pci_dev
;
1958 struct nicvf
*nic
= nicvf_pmd_priv(eth_dev
);
1960 PMD_INIT_FUNC_TRACE();
1962 eth_dev
->dev_ops
= &nicvf_eth_dev_ops
;
1964 /* For secondary processes, the primary has done all the work */
1965 if (rte_eal_process_type() != RTE_PROC_PRIMARY
) {
1967 /* Setup callbacks for secondary process */
1968 nicvf_set_tx_function(eth_dev
);
1969 nicvf_set_rx_function(eth_dev
);
1972 /* If nic == NULL than it is secondary function
1973 * so ethdev need to be released by caller */
1978 pci_dev
= eth_dev
->pci_dev
;
1979 rte_eth_copy_pci_info(eth_dev
, pci_dev
);
1981 nic
->device_id
= pci_dev
->id
.device_id
;
1982 nic
->vendor_id
= pci_dev
->id
.vendor_id
;
1983 nic
->subsystem_device_id
= pci_dev
->id
.subsystem_device_id
;
1984 nic
->subsystem_vendor_id
= pci_dev
->id
.subsystem_vendor_id
;
1986 PMD_INIT_LOG(DEBUG
, "nicvf: device (%x:%x) %u:%u:%u:%u",
1987 pci_dev
->id
.vendor_id
, pci_dev
->id
.device_id
,
1988 pci_dev
->addr
.domain
, pci_dev
->addr
.bus
,
1989 pci_dev
->addr
.devid
, pci_dev
->addr
.function
);
1991 nic
->reg_base
= (uintptr_t)pci_dev
->mem_resource
[0].addr
;
1992 if (!nic
->reg_base
) {
1993 PMD_INIT_LOG(ERR
, "Failed to map BAR0");
1998 nicvf_disable_all_interrupts(nic
);
2000 ret
= nicvf_periodic_alarm_start(nicvf_interrupt
, eth_dev
);
2002 PMD_INIT_LOG(ERR
, "Failed to start period alarm");
2006 ret
= nicvf_mbox_check_pf_ready(nic
);
2008 PMD_INIT_LOG(ERR
, "Failed to get ready message from PF");
2012 "node=%d vf=%d mode=%s sqs=%s loopback_supported=%s",
2013 nic
->node
, nic
->vf_id
,
2014 nic
->tns_mode
== NIC_TNS_MODE
? "tns" : "tns-bypass",
2015 nic
->sqs_mode
? "true" : "false",
2016 nic
->loopback_supported
? "true" : "false"
2020 ret
= nicvf_base_init(nic
);
2022 PMD_INIT_LOG(ERR
, "Failed to execute nicvf_base_init");
2026 if (nic
->sqs_mode
) {
2027 /* Push nic to stack of secondary vfs */
2028 nicvf_svf_push(nic
);
2030 /* Steal nic pointer from the device for further reuse */
2031 eth_dev
->data
->dev_private
= NULL
;
2033 nicvf_periodic_alarm_stop(nicvf_interrupt
, eth_dev
);
2034 ret
= nicvf_periodic_alarm_start(nicvf_vf_interrupt
, nic
);
2036 PMD_INIT_LOG(ERR
, "Failed to start period alarm");
2040 /* Detach port by returning postive error number */
2044 eth_dev
->data
->mac_addrs
= rte_zmalloc("mac_addr", ETHER_ADDR_LEN
, 0);
2045 if (eth_dev
->data
->mac_addrs
== NULL
) {
2046 PMD_INIT_LOG(ERR
, "Failed to allocate memory for mac addr");
2050 if (is_zero_ether_addr((struct ether_addr
*)nic
->mac_addr
))
2051 eth_random_addr(&nic
->mac_addr
[0]);
2053 ether_addr_copy((struct ether_addr
*)nic
->mac_addr
,
2054 ð_dev
->data
->mac_addrs
[0]);
2056 ret
= nicvf_mbox_set_mac_addr(nic
, nic
->mac_addr
);
2058 PMD_INIT_LOG(ERR
, "Failed to set mac addr");
2062 PMD_INIT_LOG(INFO
, "Port %d (%x:%x) mac=%02x:%02x:%02x:%02x:%02x:%02x",
2063 eth_dev
->data
->port_id
, nic
->vendor_id
, nic
->device_id
,
2064 nic
->mac_addr
[0], nic
->mac_addr
[1], nic
->mac_addr
[2],
2065 nic
->mac_addr
[3], nic
->mac_addr
[4], nic
->mac_addr
[5]);
2070 rte_free(eth_dev
->data
->mac_addrs
);
2072 nicvf_periodic_alarm_stop(nicvf_interrupt
, eth_dev
);
2077 static const struct rte_pci_id pci_id_nicvf_map
[] = {
2079 .class_id
= RTE_CLASS_ANY_ID
,
2080 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2081 .device_id
= PCI_DEVICE_ID_THUNDERX_CN88XX_PASS1_NICVF
,
2082 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2083 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN88XX_PASS1_NICVF
,
2086 .class_id
= RTE_CLASS_ANY_ID
,
2087 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2088 .device_id
= PCI_DEVICE_ID_THUNDERX_NICVF
,
2089 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2090 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN88XX_PASS2_NICVF
,
2093 .class_id
= RTE_CLASS_ANY_ID
,
2094 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2095 .device_id
= PCI_DEVICE_ID_THUNDERX_NICVF
,
2096 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2097 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN81XX_NICVF
,
2100 .class_id
= RTE_CLASS_ANY_ID
,
2101 .vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2102 .device_id
= PCI_DEVICE_ID_THUNDERX_NICVF
,
2103 .subsystem_vendor_id
= PCI_VENDOR_ID_CAVIUM
,
2104 .subsystem_device_id
= PCI_SUB_DEVICE_ID_CN83XX_NICVF
,
2111 static struct eth_driver rte_nicvf_pmd
= {
2113 .id_table
= pci_id_nicvf_map
,
2114 .drv_flags
= RTE_PCI_DRV_NEED_MAPPING
| RTE_PCI_DRV_INTR_LSC
,
2115 .probe
= rte_eth_dev_pci_probe
,
2116 .remove
= rte_eth_dev_pci_remove
,
2118 .eth_dev_init
= nicvf_eth_dev_init
,
2119 .dev_private_size
= sizeof(struct nicvf
),
2122 RTE_PMD_REGISTER_PCI(net_thunderx
, rte_nicvf_pmd
.pci_drv
);
2123 RTE_PMD_REGISTER_PCI_TABLE(net_thunderx
, pci_id_nicvf_map
);