1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright(c) 2016-2018 Microsoft Corporation
3 * Copyright(c) 2013-2016 Brocade Communications Systems, Inc.
13 #include <rte_ethdev.h>
14 #include <rte_memcpy.h>
15 #include <rte_string_fns.h>
16 #include <rte_memzone.h>
17 #include <rte_malloc.h>
18 #include <rte_atomic.h>
19 #include <rte_branch_prediction.h>
20 #include <rte_ether.h>
21 #include <rte_ethdev_driver.h>
22 #include <rte_cycles.h>
23 #include <rte_errno.h>
24 #include <rte_memory.h>
27 #include <rte_bus_vmbus.h>
35 #define HN_TX_OFFLOAD_CAPS (DEV_TX_OFFLOAD_IPV4_CKSUM | \
36 DEV_TX_OFFLOAD_TCP_CKSUM | \
37 DEV_TX_OFFLOAD_UDP_CKSUM | \
38 DEV_TX_OFFLOAD_TCP_TSO | \
39 DEV_TX_OFFLOAD_MULTI_SEGS | \
40 DEV_TX_OFFLOAD_VLAN_INSERT)
42 #define HN_RX_OFFLOAD_CAPS (DEV_RX_OFFLOAD_CHECKSUM | \
43 DEV_RX_OFFLOAD_VLAN_STRIP | \
44 DEV_RX_OFFLOAD_CRC_STRIP)
47 int hn_logtype_driver
;
49 struct hn_xstats_name_off
{
50 char name
[RTE_ETH_XSTATS_NAME_SIZE
];
54 static const struct hn_xstats_name_off hn_stat_strings
[] = {
55 { "good_packets", offsetof(struct hn_stats
, packets
) },
56 { "good_bytes", offsetof(struct hn_stats
, bytes
) },
57 { "errors", offsetof(struct hn_stats
, errors
) },
58 { "allocation_failed", offsetof(struct hn_stats
, nomemory
) },
59 { "multicast_packets", offsetof(struct hn_stats
, multicast
) },
60 { "broadcast_packets", offsetof(struct hn_stats
, broadcast
) },
61 { "undersize_packets", offsetof(struct hn_stats
, size_bins
[0]) },
62 { "size_64_packets", offsetof(struct hn_stats
, size_bins
[1]) },
63 { "size_65_127_packets", offsetof(struct hn_stats
, size_bins
[2]) },
64 { "size_128_255_packets", offsetof(struct hn_stats
, size_bins
[3]) },
65 { "size_256_511_packets", offsetof(struct hn_stats
, size_bins
[4]) },
66 { "size_512_1023_packets", offsetof(struct hn_stats
, size_bins
[5]) },
67 { "size_1024_1518_packets", offsetof(struct hn_stats
, size_bins
[6]) },
68 { "size_1519_max_packets", offsetof(struct hn_stats
, size_bins
[7]) },
71 static struct rte_eth_dev
*
72 eth_dev_vmbus_allocate(struct rte_vmbus_device
*dev
, size_t private_data_size
)
74 struct rte_eth_dev
*eth_dev
;
80 name
= dev
->device
.name
;
82 if (rte_eal_process_type() == RTE_PROC_PRIMARY
) {
83 eth_dev
= rte_eth_dev_allocate(name
);
85 PMD_DRV_LOG(NOTICE
, "can not allocate rte ethdev");
89 if (private_data_size
) {
90 eth_dev
->data
->dev_private
=
91 rte_zmalloc_socket(name
, private_data_size
,
92 RTE_CACHE_LINE_SIZE
, dev
->device
.numa_node
);
93 if (!eth_dev
->data
->dev_private
) {
94 PMD_DRV_LOG(NOTICE
, "can not allocate driver data");
95 rte_eth_dev_release_port(eth_dev
);
100 eth_dev
= rte_eth_dev_attach_secondary(name
);
102 PMD_DRV_LOG(NOTICE
, "can not attach secondary");
107 eth_dev
->device
= &dev
->device
;
108 eth_dev
->intr_handle
= &dev
->intr_handle
;
114 eth_dev_vmbus_release(struct rte_eth_dev
*eth_dev
)
116 /* free ether device */
117 rte_eth_dev_release_port(eth_dev
);
119 if (rte_eal_process_type() == RTE_PROC_PRIMARY
)
120 rte_free(eth_dev
->data
->dev_private
);
122 eth_dev
->data
->dev_private
= NULL
;
125 * Secondary process will check the name to attach.
126 * Clear this field to avoid attaching a released ports.
128 eth_dev
->data
->name
[0] = '\0';
130 eth_dev
->device
= NULL
;
131 eth_dev
->intr_handle
= NULL
;
134 /* Update link status.
135 * Note: the DPDK definition of "wait_to_complete"
136 * means block this call until link is up.
137 * which is not worth supporting.
140 hn_dev_link_update(struct rte_eth_dev
*dev
,
141 __rte_unused
int wait_to_complete
)
143 struct hn_data
*hv
= dev
->data
->dev_private
;
144 struct rte_eth_link link
, old
;
147 old
= dev
->data
->dev_link
;
149 error
= hn_rndis_get_linkstatus(hv
);
153 hn_rndis_get_linkspeed(hv
);
155 link
= (struct rte_eth_link
) {
156 .link_duplex
= ETH_LINK_FULL_DUPLEX
,
157 .link_autoneg
= ETH_LINK_SPEED_FIXED
,
158 .link_speed
= hv
->link_speed
/ 10000,
161 if (hv
->link_status
== NDIS_MEDIA_STATE_CONNECTED
)
162 link
.link_status
= ETH_LINK_UP
;
164 link
.link_status
= ETH_LINK_DOWN
;
166 if (old
.link_status
== link
.link_status
)
169 PMD_INIT_LOG(DEBUG
, "Port %d is %s", dev
->data
->port_id
,
170 (link
.link_status
== ETH_LINK_UP
) ? "up" : "down");
172 return rte_eth_linkstatus_set(dev
, &link
);
175 static void hn_dev_info_get(struct rte_eth_dev
*dev
,
176 struct rte_eth_dev_info
*dev_info
)
178 struct hn_data
*hv
= dev
->data
->dev_private
;
180 dev_info
->speed_capa
= ETH_LINK_SPEED_10G
;
181 dev_info
->min_rx_bufsize
= HN_MIN_RX_BUF_SIZE
;
182 dev_info
->max_rx_pktlen
= HN_MAX_XFER_LEN
;
183 dev_info
->max_mac_addrs
= 1;
185 dev_info
->hash_key_size
= NDIS_HASH_KEYSIZE_TOEPLITZ
;
186 dev_info
->flow_type_rss_offloads
=
187 ETH_RSS_IPV4
| ETH_RSS_IPV6
| ETH_RSS_TCP
| ETH_RSS_UDP
;
189 dev_info
->max_rx_queues
= hv
->max_queues
;
190 dev_info
->max_tx_queues
= hv
->max_queues
;
192 hn_rndis_get_offload(hv
, dev_info
);
196 hn_dev_promiscuous_enable(struct rte_eth_dev
*dev
)
198 struct hn_data
*hv
= dev
->data
->dev_private
;
200 hn_rndis_set_rxfilter(hv
, NDIS_PACKET_TYPE_PROMISCUOUS
);
204 hn_dev_promiscuous_disable(struct rte_eth_dev
*dev
)
206 struct hn_data
*hv
= dev
->data
->dev_private
;
209 filter
= NDIS_PACKET_TYPE_DIRECTED
| NDIS_PACKET_TYPE_BROADCAST
;
210 if (dev
->data
->all_multicast
)
211 filter
|= NDIS_PACKET_TYPE_ALL_MULTICAST
;
212 hn_rndis_set_rxfilter(hv
, filter
);
216 hn_dev_allmulticast_enable(struct rte_eth_dev
*dev
)
218 struct hn_data
*hv
= dev
->data
->dev_private
;
220 hn_rndis_set_rxfilter(hv
, NDIS_PACKET_TYPE_DIRECTED
|
221 NDIS_PACKET_TYPE_ALL_MULTICAST
|
222 NDIS_PACKET_TYPE_BROADCAST
);
226 hn_dev_allmulticast_disable(struct rte_eth_dev
*dev
)
228 struct hn_data
*hv
= dev
->data
->dev_private
;
230 hn_rndis_set_rxfilter(hv
, NDIS_PACKET_TYPE_DIRECTED
|
231 NDIS_PACKET_TYPE_BROADCAST
);
234 /* Setup shared rx/tx queue data */
235 static int hn_subchan_configure(struct hn_data
*hv
,
238 struct vmbus_channel
*primary
= hn_primary_chan(hv
);
240 unsigned int retry
= 0;
243 "open %u subchannels", subchan
);
245 /* Send create sub channels command */
246 err
= hn_nvs_alloc_subchans(hv
, &subchan
);
250 while (subchan
> 0) {
251 struct vmbus_channel
*new_sc
;
254 err
= rte_vmbus_subchan_open(primary
, &new_sc
);
255 if (err
== -ENOENT
&& ++retry
< 1000) {
256 /* This can happen if not ready yet */
263 "open subchannel failed: %d", err
);
268 chn_index
= rte_vmbus_sub_channel_index(new_sc
);
269 if (chn_index
== 0 || chn_index
> hv
->max_queues
) {
271 "Invalid subchannel offermsg channel %u",
276 PMD_DRV_LOG(DEBUG
, "new sub channel %u", chn_index
);
277 hv
->channels
[chn_index
] = new_sc
;
284 static int hn_dev_configure(struct rte_eth_dev
*dev
)
286 const struct rte_eth_conf
*dev_conf
= &dev
->data
->dev_conf
;
287 const struct rte_eth_rxmode
*rxmode
= &dev_conf
->rxmode
;
288 const struct rte_eth_txmode
*txmode
= &dev_conf
->txmode
;
290 const struct rte_eth_rss_conf
*rss_conf
=
291 &dev_conf
->rx_adv_conf
.rss_conf
;
292 struct hn_data
*hv
= dev
->data
->dev_private
;
293 uint64_t unsupported
;
296 PMD_INIT_FUNC_TRACE();
298 unsupported
= txmode
->offloads
& ~HN_TX_OFFLOAD_CAPS
;
301 "unsupported TX offload: %#" PRIx64
,
306 unsupported
= rxmode
->offloads
& ~HN_RX_OFFLOAD_CAPS
;
309 "unsupported RX offload: %#" PRIx64
,
314 err
= hn_rndis_conf_offload(hv
, txmode
->offloads
,
318 "offload configure failed");
322 hv
->num_queues
= RTE_MAX(dev
->data
->nb_rx_queues
,
323 dev
->data
->nb_tx_queues
);
324 subchan
= hv
->num_queues
- 1;
326 err
= hn_subchan_configure(hv
, subchan
);
329 "subchannel configuration failed");
333 err
= hn_rndis_conf_rss(hv
, rss_conf
);
336 "rss configuration failed");
344 static int hn_dev_stats_get(struct rte_eth_dev
*dev
,
345 struct rte_eth_stats
*stats
)
349 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
350 const struct hn_tx_queue
*txq
= dev
->data
->tx_queues
[i
];
355 stats
->opackets
+= txq
->stats
.packets
;
356 stats
->obytes
+= txq
->stats
.bytes
;
357 stats
->oerrors
+= txq
->stats
.errors
+ txq
->stats
.nomemory
;
359 if (i
< RTE_ETHDEV_QUEUE_STAT_CNTRS
) {
360 stats
->q_opackets
[i
] = txq
->stats
.packets
;
361 stats
->q_obytes
[i
] = txq
->stats
.bytes
;
365 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
366 const struct hn_rx_queue
*rxq
= dev
->data
->rx_queues
[i
];
371 stats
->ipackets
+= rxq
->stats
.packets
;
372 stats
->ibytes
+= rxq
->stats
.bytes
;
373 stats
->ierrors
+= rxq
->stats
.errors
;
374 stats
->imissed
+= rxq
->ring_full
;
376 if (i
< RTE_ETHDEV_QUEUE_STAT_CNTRS
) {
377 stats
->q_ipackets
[i
] = rxq
->stats
.packets
;
378 stats
->q_ibytes
[i
] = rxq
->stats
.bytes
;
382 stats
->rx_nombuf
= dev
->data
->rx_mbuf_alloc_failed
;
387 hn_dev_stats_reset(struct rte_eth_dev
*dev
)
391 PMD_INIT_FUNC_TRACE();
393 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
394 struct hn_tx_queue
*txq
= dev
->data
->tx_queues
[i
];
398 memset(&txq
->stats
, 0, sizeof(struct hn_stats
));
401 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
402 struct hn_rx_queue
*rxq
= dev
->data
->rx_queues
[i
];
407 memset(&rxq
->stats
, 0, sizeof(struct hn_stats
));
413 hn_dev_xstats_get_names(struct rte_eth_dev
*dev
,
414 struct rte_eth_xstat_name
*xstats_names
,
415 __rte_unused
unsigned int limit
)
417 unsigned int i
, t
, count
= 0;
419 PMD_INIT_FUNC_TRACE();
422 return dev
->data
->nb_tx_queues
* RTE_DIM(hn_stat_strings
)
423 + dev
->data
->nb_rx_queues
* RTE_DIM(hn_stat_strings
);
425 /* Note: limit checked in rte_eth_xstats_names() */
426 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
427 const struct hn_tx_queue
*txq
= dev
->data
->tx_queues
[i
];
432 for (t
= 0; t
< RTE_DIM(hn_stat_strings
); t
++)
433 snprintf(xstats_names
[count
++].name
,
434 RTE_ETH_XSTATS_NAME_SIZE
,
435 "tx_q%u_%s", i
, hn_stat_strings
[t
].name
);
438 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
439 const struct hn_rx_queue
*rxq
= dev
->data
->rx_queues
[i
];
444 for (t
= 0; t
< RTE_DIM(hn_stat_strings
); t
++)
445 snprintf(xstats_names
[count
++].name
,
446 RTE_ETH_XSTATS_NAME_SIZE
,
448 hn_stat_strings
[t
].name
);
455 hn_dev_xstats_get(struct rte_eth_dev
*dev
,
456 struct rte_eth_xstat
*xstats
,
459 unsigned int i
, t
, count
= 0;
461 const unsigned int nstats
=
462 dev
->data
->nb_tx_queues
* RTE_DIM(hn_stat_strings
)
463 + dev
->data
->nb_rx_queues
* RTE_DIM(hn_stat_strings
);
466 PMD_INIT_FUNC_TRACE();
471 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
472 const struct hn_tx_queue
*txq
= dev
->data
->tx_queues
[i
];
477 stats
= (const char *)&txq
->stats
;
478 for (t
= 0; t
< RTE_DIM(hn_stat_strings
); t
++)
479 xstats
[count
++].value
= *(const uint64_t *)
480 (stats
+ hn_stat_strings
[t
].offset
);
483 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
484 const struct hn_rx_queue
*rxq
= dev
->data
->rx_queues
[i
];
489 stats
= (const char *)&rxq
->stats
;
490 for (t
= 0; t
< RTE_DIM(hn_stat_strings
); t
++)
491 xstats
[count
++].value
= *(const uint64_t *)
492 (stats
+ hn_stat_strings
[t
].offset
);
499 hn_dev_start(struct rte_eth_dev
*dev
)
501 struct hn_data
*hv
= dev
->data
->dev_private
;
503 PMD_INIT_FUNC_TRACE();
505 /* check if lsc interrupt feature is enabled */
506 if (dev
->data
->dev_conf
.intr_conf
.lsc
) {
507 PMD_DRV_LOG(ERR
, "link status not supported yet");
511 return hn_rndis_set_rxfilter(hv
,
512 NDIS_PACKET_TYPE_BROADCAST
|
513 NDIS_PACKET_TYPE_ALL_MULTICAST
|
514 NDIS_PACKET_TYPE_DIRECTED
);
518 hn_dev_stop(struct rte_eth_dev
*dev
)
520 struct hn_data
*hv
= dev
->data
->dev_private
;
522 PMD_INIT_FUNC_TRACE();
524 hn_rndis_set_rxfilter(hv
, 0);
528 hn_dev_close(struct rte_eth_dev
*dev __rte_unused
)
530 PMD_INIT_LOG(DEBUG
, "close");
533 static const struct eth_dev_ops hn_eth_dev_ops
= {
534 .dev_configure
= hn_dev_configure
,
535 .dev_start
= hn_dev_start
,
536 .dev_stop
= hn_dev_stop
,
537 .dev_close
= hn_dev_close
,
538 .dev_infos_get
= hn_dev_info_get
,
539 .txq_info_get
= hn_dev_tx_queue_info
,
540 .rxq_info_get
= hn_dev_rx_queue_info
,
541 .promiscuous_enable
= hn_dev_promiscuous_enable
,
542 .promiscuous_disable
= hn_dev_promiscuous_disable
,
543 .allmulticast_enable
= hn_dev_allmulticast_enable
,
544 .allmulticast_disable
= hn_dev_allmulticast_disable
,
545 .tx_queue_setup
= hn_dev_tx_queue_setup
,
546 .tx_queue_release
= hn_dev_tx_queue_release
,
547 .rx_queue_setup
= hn_dev_rx_queue_setup
,
548 .rx_queue_release
= hn_dev_rx_queue_release
,
549 .link_update
= hn_dev_link_update
,
550 .stats_get
= hn_dev_stats_get
,
551 .xstats_get
= hn_dev_xstats_get
,
552 .xstats_get_names
= hn_dev_xstats_get_names
,
553 .stats_reset
= hn_dev_stats_reset
,
554 .xstats_reset
= hn_dev_stats_reset
,
558 * Setup connection between PMD and kernel.
561 hn_attach(struct hn_data
*hv
, unsigned int mtu
)
566 error
= hn_nvs_attach(hv
, mtu
);
571 error
= hn_rndis_attach(hv
);
577 * Under certain conditions on certain versions of Hyper-V,
578 * the RNDIS rxfilter is _not_ zero on the hypervisor side
579 * after the successful RNDIS initialization.
581 hn_rndis_set_rxfilter(hv
, NDIS_PACKET_TYPE_NONE
);
590 hn_detach(struct hn_data
*hv
)
597 eth_hn_dev_init(struct rte_eth_dev
*eth_dev
)
599 struct hn_data
*hv
= eth_dev
->data
->dev_private
;
600 struct rte_device
*device
= eth_dev
->device
;
601 struct rte_vmbus_device
*vmbus
;
602 unsigned int rxr_cnt
;
605 PMD_INIT_FUNC_TRACE();
607 vmbus
= container_of(device
, struct rte_vmbus_device
, device
);
608 eth_dev
->dev_ops
= &hn_eth_dev_ops
;
609 eth_dev
->tx_pkt_burst
= &hn_xmit_pkts
;
610 eth_dev
->rx_pkt_burst
= &hn_recv_pkts
;
613 * for secondary processes, we don't initialize any further as primary
614 * has already done this work.
616 if (rte_eal_process_type() != RTE_PROC_PRIMARY
)
619 /* Since Hyper-V only supports one MAC address, just use local data */
620 eth_dev
->data
->mac_addrs
= &hv
->mac_addr
;
623 hv
->rxbuf_res
= &vmbus
->resource
[HV_RECV_BUF_MAP
];
624 hv
->chim_res
= &vmbus
->resource
[HV_SEND_BUF_MAP
];
625 hv
->port_id
= eth_dev
->data
->port_id
;
627 /* Initialize primary channel input for control operations */
628 err
= rte_vmbus_chan_open(vmbus
, &hv
->channels
[0]);
632 hv
->primary
= hn_rx_queue_alloc(hv
, 0,
633 eth_dev
->device
->numa_node
);
638 err
= hn_attach(hv
, ETHER_MTU
);
642 err
= hn_tx_pool_init(eth_dev
);
646 err
= hn_rndis_get_eaddr(hv
, hv
->mac_addr
.addr_bytes
);
650 max_chan
= rte_vmbus_max_channels(vmbus
);
651 PMD_INIT_LOG(DEBUG
, "VMBus max channels %d", max_chan
);
655 if (hn_rndis_query_rsscaps(hv
, &rxr_cnt
) != 0)
658 hv
->max_queues
= RTE_MIN(rxr_cnt
, (unsigned int)max_chan
);
663 PMD_INIT_LOG(NOTICE
, "device init failed");
670 eth_hn_dev_uninit(struct rte_eth_dev
*eth_dev
)
672 struct hn_data
*hv
= eth_dev
->data
->dev_private
;
674 PMD_INIT_FUNC_TRACE();
676 if (rte_eal_process_type() != RTE_PROC_PRIMARY
)
679 hn_dev_stop(eth_dev
);
680 hn_dev_close(eth_dev
);
682 eth_dev
->dev_ops
= NULL
;
683 eth_dev
->tx_pkt_burst
= NULL
;
684 eth_dev
->rx_pkt_burst
= NULL
;
687 rte_vmbus_chan_close(hv
->primary
->chan
);
688 rte_free(hv
->primary
);
690 eth_dev
->data
->mac_addrs
= NULL
;
695 static int eth_hn_probe(struct rte_vmbus_driver
*drv __rte_unused
,
696 struct rte_vmbus_device
*dev
)
698 struct rte_eth_dev
*eth_dev
;
701 PMD_INIT_FUNC_TRACE();
703 eth_dev
= eth_dev_vmbus_allocate(dev
, sizeof(struct hn_data
));
707 ret
= eth_hn_dev_init(eth_dev
);
709 eth_dev_vmbus_release(eth_dev
);
711 rte_eth_dev_probing_finish(eth_dev
);
716 static int eth_hn_remove(struct rte_vmbus_device
*dev
)
718 struct rte_eth_dev
*eth_dev
;
721 PMD_INIT_FUNC_TRACE();
723 eth_dev
= rte_eth_dev_allocated(dev
->device
.name
);
727 ret
= eth_hn_dev_uninit(eth_dev
);
731 eth_dev_vmbus_release(eth_dev
);
735 /* Network device GUID */
736 static const rte_uuid_t hn_net_ids
[] = {
737 /* f8615163-df3e-46c5-913f-f2d2f965ed0e */
738 RTE_UUID_INIT(0xf8615163, 0xdf3e, 0x46c5, 0x913f, 0xf2d2f965ed0eULL
),
742 static struct rte_vmbus_driver rte_netvsc_pmd
= {
743 .id_table
= hn_net_ids
,
744 .probe
= eth_hn_probe
,
745 .remove
= eth_hn_remove
,
748 RTE_PMD_REGISTER_VMBUS(net_netvsc
, rte_netvsc_pmd
);
749 RTE_PMD_REGISTER_KMOD_DEP(net_netvsc
, "* uio_hv_generic");
751 RTE_INIT(hn_init_log
);
755 hn_logtype_init
= rte_log_register("pmd.net.netvsc.init");
756 if (hn_logtype_init
>= 0)
757 rte_log_set_level(hn_logtype_init
, RTE_LOG_NOTICE
);
758 hn_logtype_driver
= rte_log_register("pmd.net.netvsc.driver");
759 if (hn_logtype_driver
>= 0)
760 rte_log_set_level(hn_logtype_driver
, RTE_LOG_NOTICE
);