4 * Copyright 2015 6WIND S.A.
5 * Copyright 2015 Mellanox.
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
17 * * Neither the name of 6WIND S.A. nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific prior written permission.
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
44 #include <sys/ioctl.h>
45 #include <sys/socket.h>
46 #include <netinet/in.h>
47 #include <linux/ethtool.h>
48 #include <linux/sockios.h>
51 /* DPDK headers don't like -pedantic. */
53 #pragma GCC diagnostic ignored "-Wpedantic"
55 #include <rte_atomic.h>
56 #include <rte_ethdev.h>
58 #include <rte_common.h>
59 #include <rte_interrupts.h>
60 #include <rte_alarm.h>
61 #include <rte_malloc.h>
63 #pragma GCC diagnostic error "-Wpedantic"
67 #include "mlx5_rxtx.h"
68 #include "mlx5_utils.h"
71 * Return private structure associated with an Ethernet device.
74 * Pointer to Ethernet device structure.
77 * Pointer to private structure.
80 mlx5_get_priv(struct rte_eth_dev
*dev
)
82 struct mlx5_secondary_data
*sd
;
84 if (!mlx5_is_secondary())
85 return dev
->data
->dev_private
;
86 sd
= &mlx5_secondary_data
[dev
->data
->port_id
];
87 return sd
->data
.dev_private
;
91 * Check if running as a secondary process.
94 * Nonzero if running as a secondary process.
97 mlx5_is_secondary(void)
99 return rte_eal_process_type() != RTE_PROC_PRIMARY
;
103 * Get interface name from private structure.
106 * Pointer to private structure.
108 * Interface name output buffer.
111 * 0 on success, -1 on failure and errno is set.
114 priv_get_ifname(const struct priv
*priv
, char (*ifname
)[IF_NAMESIZE
])
118 unsigned int dev_type
= 0;
119 unsigned int dev_port_prev
= ~0u;
120 char match
[IF_NAMESIZE
] = "";
123 MKSTR(path
, "%s/device/net", priv
->ctx
->device
->ibdev_path
);
129 while ((dent
= readdir(dir
)) != NULL
) {
130 char *name
= dent
->d_name
;
132 unsigned int dev_port
;
135 if ((name
[0] == '.') &&
136 ((name
[1] == '\0') ||
137 ((name
[1] == '.') && (name
[2] == '\0'))))
140 MKSTR(path
, "%s/device/net/%s/%s",
141 priv
->ctx
->device
->ibdev_path
, name
,
142 (dev_type
? "dev_id" : "dev_port"));
144 file
= fopen(path
, "rb");
149 * Switch to dev_id when dev_port does not exist as
150 * is the case with Linux kernel versions < 3.15.
161 r
= fscanf(file
, (dev_type
? "%x" : "%u"), &dev_port
);
166 * Switch to dev_id when dev_port returns the same value for
167 * all ports. May happen when using a MOFED release older than
168 * 3.0 with a Linux kernel >= 3.15.
170 if (dev_port
== dev_port_prev
)
172 dev_port_prev
= dev_port
;
173 if (dev_port
== (priv
->port
- 1u))
174 snprintf(match
, sizeof(match
), "%s", name
);
177 if (match
[0] == '\0')
179 strncpy(*ifname
, match
, sizeof(*ifname
));
184 * Read from sysfs entry.
187 * Pointer to private structure.
189 * Entry name relative to sysfs path.
191 * Data output buffer.
196 * 0 on success, -1 on failure and errno is set.
199 priv_sysfs_read(const struct priv
*priv
, const char *entry
,
200 char *buf
, size_t size
)
202 char ifname
[IF_NAMESIZE
];
207 if (priv_get_ifname(priv
, &ifname
))
210 MKSTR(path
, "%s/device/net/%s/%s", priv
->ctx
->device
->ibdev_path
,
213 file
= fopen(path
, "rb");
216 ret
= fread(buf
, 1, size
, file
);
218 if (((size_t)ret
< size
) && (ferror(file
)))
228 * Write to sysfs entry.
231 * Pointer to private structure.
233 * Entry name relative to sysfs path.
240 * 0 on success, -1 on failure and errno is set.
243 priv_sysfs_write(const struct priv
*priv
, const char *entry
,
244 char *buf
, size_t size
)
246 char ifname
[IF_NAMESIZE
];
251 if (priv_get_ifname(priv
, &ifname
))
254 MKSTR(path
, "%s/device/net/%s/%s", priv
->ctx
->device
->ibdev_path
,
257 file
= fopen(path
, "wb");
260 ret
= fwrite(buf
, 1, size
, file
);
262 if (((size_t)ret
< size
) || (ferror(file
)))
272 * Get unsigned long sysfs property.
275 * Pointer to private structure.
277 * Entry name relative to sysfs path.
279 * Value output buffer.
282 * 0 on success, -1 on failure and errno is set.
285 priv_get_sysfs_ulong(struct priv
*priv
, const char *name
, unsigned long *value
)
288 unsigned long value_ret
;
291 ret
= priv_sysfs_read(priv
, name
, value_str
, (sizeof(value_str
) - 1));
293 DEBUG("cannot read %s value from sysfs: %s",
294 name
, strerror(errno
));
297 value_str
[ret
] = '\0';
299 value_ret
= strtoul(value_str
, NULL
, 0);
301 DEBUG("invalid %s value `%s': %s", name
, value_str
,
310 * Set unsigned long sysfs property.
313 * Pointer to private structure.
315 * Entry name relative to sysfs path.
320 * 0 on success, -1 on failure and errno is set.
323 priv_set_sysfs_ulong(struct priv
*priv
, const char *name
, unsigned long value
)
326 MKSTR(value_str
, "%lu", value
);
328 ret
= priv_sysfs_write(priv
, name
, value_str
, (sizeof(value_str
) - 1));
330 DEBUG("cannot write %s `%s' (%lu) to sysfs: %s",
331 name
, value_str
, value
, strerror(errno
));
338 * Perform ifreq ioctl() on associated Ethernet device.
341 * Pointer to private structure.
343 * Request number to pass to ioctl().
345 * Interface request structure output buffer.
348 * 0 on success, -1 on failure and errno is set.
351 priv_ifreq(const struct priv
*priv
, int req
, struct ifreq
*ifr
)
353 int sock
= socket(PF_INET
, SOCK_DGRAM
, IPPROTO_IP
);
358 if (priv_get_ifname(priv
, &ifr
->ifr_name
) == 0)
359 ret
= ioctl(sock
, req
, ifr
);
365 * Return the number of active VFs for the current device.
368 * Pointer to private structure.
369 * @param[out] num_vfs
370 * Number of active VFs.
373 * 0 on success, -1 on failure and errno is set.
376 priv_get_num_vfs(struct priv
*priv
, uint16_t *num_vfs
)
378 /* The sysfs entry name depends on the operating system. */
379 const char **name
= (const char *[]){
380 "device/sriov_numvfs",
381 "device/mlx5_num_vfs",
387 unsigned long ulong_num_vfs
;
389 ret
= priv_get_sysfs_ulong(priv
, *name
, &ulong_num_vfs
);
391 *num_vfs
= ulong_num_vfs
;
392 } while (*(++name
) && ret
);
400 * Pointer to private structure.
402 * MTU value output buffer.
405 * 0 on success, -1 on failure and errno is set.
408 priv_get_mtu(struct priv
*priv
, uint16_t *mtu
)
410 unsigned long ulong_mtu
;
412 if (priv_get_sysfs_ulong(priv
, "mtu", &ulong_mtu
) == -1)
422 * Pointer to private structure.
427 * 0 on success, -1 on failure and errno is set.
430 priv_set_mtu(struct priv
*priv
, uint16_t mtu
)
434 if (priv_set_sysfs_ulong(priv
, "mtu", mtu
) ||
435 priv_get_mtu(priv
, &new_mtu
))
447 * Pointer to private structure.
449 * Bitmask for flags that must remain untouched.
451 * Bitmask for flags to modify.
454 * 0 on success, -1 on failure and errno is set.
457 priv_set_flags(struct priv
*priv
, unsigned int keep
, unsigned int flags
)
461 if (priv_get_sysfs_ulong(priv
, "flags", &tmp
) == -1)
464 tmp
|= (flags
& (~keep
));
465 return priv_set_sysfs_ulong(priv
, "flags", tmp
);
469 * Ethernet device configuration.
471 * Prepare the driver for a given number of TX and RX queues.
474 * Pointer to Ethernet device structure.
477 * 0 on success, errno value on failure.
480 dev_configure(struct rte_eth_dev
*dev
)
482 struct priv
*priv
= dev
->data
->dev_private
;
483 unsigned int rxqs_n
= dev
->data
->nb_rx_queues
;
484 unsigned int txqs_n
= dev
->data
->nb_tx_queues
;
487 unsigned int reta_idx_n
;
489 priv
->rss_hf
= dev
->data
->dev_conf
.rx_adv_conf
.rss_conf
.rss_hf
;
490 priv
->rxqs
= (void *)dev
->data
->rx_queues
;
491 priv
->txqs
= (void *)dev
->data
->tx_queues
;
492 if (txqs_n
!= priv
->txqs_n
) {
493 INFO("%p: TX queues number update: %u -> %u",
494 (void *)dev
, priv
->txqs_n
, txqs_n
);
495 priv
->txqs_n
= txqs_n
;
497 if (rxqs_n
> priv
->ind_table_max_size
) {
498 ERROR("cannot handle this many RX queues (%u)", rxqs_n
);
501 if (rxqs_n
== priv
->rxqs_n
)
503 INFO("%p: RX queues number update: %u -> %u",
504 (void *)dev
, priv
->rxqs_n
, rxqs_n
);
505 priv
->rxqs_n
= rxqs_n
;
506 /* If the requested number of RX queues is not a power of two, use the
507 * maximum indirection table size for better balancing.
508 * The result is always rounded to the next power of two. */
509 reta_idx_n
= (1 << log2above((rxqs_n
& (rxqs_n
- 1)) ?
510 priv
->ind_table_max_size
:
512 if (priv_rss_reta_index_resize(priv
, reta_idx_n
))
514 /* When the number of RX queues is not a power of two, the remaining
515 * table entries are padded with reused WQs and hashes are not spread
517 for (i
= 0, j
= 0; (i
!= reta_idx_n
); ++i
) {
518 (*priv
->reta_idx
)[i
] = j
;
526 * DPDK callback for Ethernet device configuration.
529 * Pointer to Ethernet device structure.
532 * 0 on success, negative errno value on failure.
535 mlx5_dev_configure(struct rte_eth_dev
*dev
)
537 struct priv
*priv
= dev
->data
->dev_private
;
540 if (mlx5_is_secondary())
541 return -E_RTE_SECONDARY
;
544 ret
= dev_configure(dev
);
551 * DPDK callback to get information about the device.
554 * Pointer to Ethernet device structure.
556 * Info structure output buffer.
559 mlx5_dev_infos_get(struct rte_eth_dev
*dev
, struct rte_eth_dev_info
*info
)
561 struct priv
*priv
= mlx5_get_priv(dev
);
563 char ifname
[IF_NAMESIZE
];
566 /* FIXME: we should ask the device for these values. */
567 info
->min_rx_bufsize
= 32;
568 info
->max_rx_pktlen
= 65536;
570 * Since we need one CQ per QP, the limit is the minimum number
571 * between the two values.
573 max
= ((priv
->device_attr
.max_cq
> priv
->device_attr
.max_qp
) ?
574 priv
->device_attr
.max_qp
: priv
->device_attr
.max_cq
);
575 /* If max >= 65535 then max = 0, max_rx_queues is uint16_t. */
578 info
->max_rx_queues
= max
;
579 info
->max_tx_queues
= max
;
580 info
->max_mac_addrs
= RTE_DIM(priv
->mac
);
581 info
->rx_offload_capa
=
583 (DEV_RX_OFFLOAD_IPV4_CKSUM
|
584 DEV_RX_OFFLOAD_UDP_CKSUM
|
585 DEV_RX_OFFLOAD_TCP_CKSUM
) :
587 (priv
->hw_vlan_strip
? DEV_RX_OFFLOAD_VLAN_STRIP
: 0);
589 info
->tx_offload_capa
= DEV_TX_OFFLOAD_VLAN_INSERT
;
591 info
->tx_offload_capa
|=
592 (DEV_TX_OFFLOAD_IPV4_CKSUM
|
593 DEV_TX_OFFLOAD_UDP_CKSUM
|
594 DEV_TX_OFFLOAD_TCP_CKSUM
);
595 if (priv_get_ifname(priv
, &ifname
) == 0)
596 info
->if_index
= if_nametoindex(ifname
);
597 /* FIXME: RETA update/query API expects the callee to know the size of
598 * the indirection table, for this PMD the size varies depending on
599 * the number of RX queues, it becomes impossible to find the correct
600 * size if it is not fixed.
601 * The API should be updated to solve this problem. */
602 info
->reta_size
= priv
->ind_table_max_size
;
603 info
->hash_key_size
= ((*priv
->rss_conf
) ?
604 (*priv
->rss_conf
)[0]->rss_key_len
:
606 info
->speed_capa
= priv
->link_speed_capa
;
611 mlx5_dev_supported_ptypes_get(struct rte_eth_dev
*dev
)
613 static const uint32_t ptypes
[] = {
614 /* refers to rxq_cq_to_pkt_type() */
617 RTE_PTYPE_INNER_L3_IPV4
,
618 RTE_PTYPE_INNER_L3_IPV6
,
623 if (dev
->rx_pkt_burst
== mlx5_rx_burst
)
629 * Retrieve physical link information (unlocked version using legacy ioctl).
632 * Pointer to Ethernet device structure.
633 * @param wait_to_complete
634 * Wait for request completion (ignored).
637 mlx5_link_update_unlocked_gset(struct rte_eth_dev
*dev
, int wait_to_complete
)
639 struct priv
*priv
= mlx5_get_priv(dev
);
640 struct ethtool_cmd edata
= {
641 .cmd
= ETHTOOL_GSET
/* Deprecated since Linux v4.5. */
644 struct rte_eth_link dev_link
;
647 (void)wait_to_complete
;
648 if (priv_ifreq(priv
, SIOCGIFFLAGS
, &ifr
)) {
649 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno
));
652 memset(&dev_link
, 0, sizeof(dev_link
));
653 dev_link
.link_status
= ((ifr
.ifr_flags
& IFF_UP
) &&
654 (ifr
.ifr_flags
& IFF_RUNNING
));
655 ifr
.ifr_data
= (void *)&edata
;
656 if (priv_ifreq(priv
, SIOCETHTOOL
, &ifr
)) {
657 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GSET) failed: %s",
661 link_speed
= ethtool_cmd_speed(&edata
);
662 if (link_speed
== -1)
663 dev_link
.link_speed
= 0;
665 dev_link
.link_speed
= link_speed
;
666 priv
->link_speed_capa
= 0;
667 if (edata
.supported
& SUPPORTED_Autoneg
)
668 priv
->link_speed_capa
|= ETH_LINK_SPEED_AUTONEG
;
669 if (edata
.supported
& (SUPPORTED_1000baseT_Full
|
670 SUPPORTED_1000baseKX_Full
))
671 priv
->link_speed_capa
|= ETH_LINK_SPEED_1G
;
672 if (edata
.supported
& SUPPORTED_10000baseKR_Full
)
673 priv
->link_speed_capa
|= ETH_LINK_SPEED_10G
;
674 if (edata
.supported
& (SUPPORTED_40000baseKR4_Full
|
675 SUPPORTED_40000baseCR4_Full
|
676 SUPPORTED_40000baseSR4_Full
|
677 SUPPORTED_40000baseLR4_Full
))
678 priv
->link_speed_capa
|= ETH_LINK_SPEED_40G
;
679 dev_link
.link_duplex
= ((edata
.duplex
== DUPLEX_HALF
) ?
680 ETH_LINK_HALF_DUPLEX
: ETH_LINK_FULL_DUPLEX
);
681 dev_link
.link_autoneg
= !(dev
->data
->dev_conf
.link_speeds
&
682 ETH_LINK_SPEED_FIXED
);
683 if (memcmp(&dev_link
, &dev
->data
->dev_link
, sizeof(dev_link
))) {
684 /* Link status changed. */
685 dev
->data
->dev_link
= dev_link
;
688 /* Link status is still the same. */
693 * Retrieve physical link information (unlocked version using new ioctl from
697 * Pointer to Ethernet device structure.
698 * @param wait_to_complete
699 * Wait for request completion (ignored).
702 mlx5_link_update_unlocked_gs(struct rte_eth_dev
*dev
, int wait_to_complete
)
704 #ifdef ETHTOOL_GLINKSETTINGS
705 struct priv
*priv
= mlx5_get_priv(dev
);
706 struct ethtool_link_settings edata
= {
707 .cmd
= ETHTOOL_GLINKSETTINGS
,
710 struct rte_eth_link dev_link
;
713 (void)wait_to_complete
;
714 if (priv_ifreq(priv
, SIOCGIFFLAGS
, &ifr
)) {
715 WARN("ioctl(SIOCGIFFLAGS) failed: %s", strerror(errno
));
718 memset(&dev_link
, 0, sizeof(dev_link
));
719 dev_link
.link_status
= ((ifr
.ifr_flags
& IFF_UP
) &&
720 (ifr
.ifr_flags
& IFF_RUNNING
));
721 ifr
.ifr_data
= (void *)&edata
;
722 if (priv_ifreq(priv
, SIOCETHTOOL
, &ifr
)) {
723 DEBUG("ioctl(SIOCETHTOOL, ETHTOOL_GLINKSETTINGS) failed: %s",
727 dev_link
.link_speed
= edata
.speed
;
728 sc
= edata
.link_mode_masks
[0] |
729 ((uint64_t)edata
.link_mode_masks
[1] << 32);
730 priv
->link_speed_capa
= 0;
731 /* Link speeds available in kernel v4.5. */
732 if (sc
& ETHTOOL_LINK_MODE_Autoneg_BIT
)
733 priv
->link_speed_capa
|= ETH_LINK_SPEED_AUTONEG
;
734 if (sc
& (ETHTOOL_LINK_MODE_1000baseT_Full_BIT
|
735 ETHTOOL_LINK_MODE_1000baseKX_Full_BIT
))
736 priv
->link_speed_capa
|= ETH_LINK_SPEED_1G
;
737 if (sc
& (ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT
|
738 ETHTOOL_LINK_MODE_10000baseKR_Full_BIT
|
739 ETHTOOL_LINK_MODE_10000baseR_FEC_BIT
))
740 priv
->link_speed_capa
|= ETH_LINK_SPEED_10G
;
741 if (sc
& (ETHTOOL_LINK_MODE_20000baseMLD2_Full_BIT
|
742 ETHTOOL_LINK_MODE_20000baseKR2_Full_BIT
))
743 priv
->link_speed_capa
|= ETH_LINK_SPEED_20G
;
744 if (sc
& (ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT
|
745 ETHTOOL_LINK_MODE_40000baseCR4_Full_BIT
|
746 ETHTOOL_LINK_MODE_40000baseSR4_Full_BIT
|
747 ETHTOOL_LINK_MODE_40000baseLR4_Full_BIT
))
748 priv
->link_speed_capa
|= ETH_LINK_SPEED_40G
;
749 if (sc
& (ETHTOOL_LINK_MODE_56000baseKR4_Full_BIT
|
750 ETHTOOL_LINK_MODE_56000baseCR4_Full_BIT
|
751 ETHTOOL_LINK_MODE_56000baseSR4_Full_BIT
|
752 ETHTOOL_LINK_MODE_56000baseLR4_Full_BIT
))
753 priv
->link_speed_capa
|= ETH_LINK_SPEED_56G
;
754 /* Link speeds available in kernel v4.6. */
755 #ifdef HAVE_ETHTOOL_LINK_MODE_25G
756 if (sc
& (ETHTOOL_LINK_MODE_25000baseCR_Full_BIT
|
757 ETHTOOL_LINK_MODE_25000baseKR_Full_BIT
|
758 ETHTOOL_LINK_MODE_25000baseSR_Full_BIT
))
759 priv
->link_speed_capa
|= ETH_LINK_SPEED_25G
;
761 #ifdef HAVE_ETHTOOL_LINK_MODE_50G
762 if (sc
& (ETHTOOL_LINK_MODE_50000baseCR2_Full_BIT
|
763 ETHTOOL_LINK_MODE_50000baseKR2_Full_BIT
))
764 priv
->link_speed_capa
|= ETH_LINK_SPEED_50G
;
766 #ifdef HAVE_ETHTOOL_LINK_MODE_100G
767 if (sc
& (ETHTOOL_LINK_MODE_100000baseKR4_Full_BIT
|
768 ETHTOOL_LINK_MODE_100000baseSR4_Full_BIT
|
769 ETHTOOL_LINK_MODE_100000baseCR4_Full_BIT
|
770 ETHTOOL_LINK_MODE_100000baseLR4_ER4_Full_BIT
))
771 priv
->link_speed_capa
|= ETH_LINK_SPEED_100G
;
773 dev_link
.link_duplex
= ((edata
.duplex
== DUPLEX_HALF
) ?
774 ETH_LINK_HALF_DUPLEX
: ETH_LINK_FULL_DUPLEX
);
775 dev_link
.link_autoneg
= !(dev
->data
->dev_conf
.link_speeds
&
776 ETH_LINK_SPEED_FIXED
);
777 if (memcmp(&dev_link
, &dev
->data
->dev_link
, sizeof(dev_link
))) {
778 /* Link status changed. */
779 dev
->data
->dev_link
= dev_link
;
784 (void)wait_to_complete
;
786 /* Link status is still the same. */
791 * DPDK callback to retrieve physical link information (unlocked version).
794 * Pointer to Ethernet device structure.
795 * @param wait_to_complete
796 * Wait for request completion (ignored).
799 mlx5_link_update_unlocked(struct rte_eth_dev
*dev
, int wait_to_complete
)
803 ret
= mlx5_link_update_unlocked_gs(dev
, wait_to_complete
);
805 ret
= mlx5_link_update_unlocked_gset(dev
, wait_to_complete
);
810 * DPDK callback to retrieve physical link information.
813 * Pointer to Ethernet device structure.
814 * @param wait_to_complete
815 * Wait for request completion (ignored).
818 mlx5_link_update(struct rte_eth_dev
*dev
, int wait_to_complete
)
820 struct priv
*priv
= mlx5_get_priv(dev
);
824 ret
= mlx5_link_update_unlocked(dev
, wait_to_complete
);
830 * DPDK callback to change the MTU.
832 * Setting the MTU affects hardware MRU (packets larger than the MTU cannot be
833 * received). Use this as a hint to enable/disable scattered packets support
834 * and improve performance when not needed.
835 * Since failure is not an option, reconfiguring queues on the fly is not
839 * Pointer to Ethernet device structure.
844 * 0 on success, negative errno value on failure.
847 mlx5_dev_set_mtu(struct rte_eth_dev
*dev
, uint16_t mtu
)
849 struct priv
*priv
= dev
->data
->dev_private
;
852 uint16_t (*rx_func
)(void *, struct rte_mbuf
**, uint16_t) =
854 unsigned int max_frame_len
;
856 int restart
= priv
->started
;
858 if (mlx5_is_secondary())
859 return -E_RTE_SECONDARY
;
862 /* Set kernel interface MTU first. */
863 if (priv_set_mtu(priv
, mtu
)) {
865 WARN("cannot set port %u MTU to %u: %s", priv
->port
, mtu
,
869 DEBUG("adapter port %u MTU set to %u", priv
->port
, mtu
);
870 /* Temporarily replace RX handler with a fake one, assuming it has not
871 * been copied elsewhere. */
872 dev
->rx_pkt_burst
= removed_rx_burst
;
873 /* Make sure everyone has left mlx5_rx_burst() and uses
874 * removed_rx_burst() instead. */
877 /* MTU does not include header and CRC. */
878 max_frame_len
= ETHER_HDR_LEN
+ mtu
+ ETHER_CRC_LEN
;
879 /* Check if at least one queue is going to need a SGE update. */
880 for (i
= 0; i
!= priv
->rxqs_n
; ++i
) {
881 struct rxq
*rxq
= (*priv
->rxqs
)[i
];
883 unsigned int size
= RTE_PKTMBUF_HEADROOM
+ max_frame_len
;
888 mb_len
= rte_pktmbuf_data_room_size(rxq
->mp
);
889 assert(mb_len
>= RTE_PKTMBUF_HEADROOM
);
891 * Determine the number of SGEs needed for a full packet
892 * and round it to the next power of two.
894 sges_n
= log2above((size
/ mb_len
) + !!(size
% mb_len
));
895 if (sges_n
!= rxq
->sges_n
)
899 * If all queues have the right number of SGEs, a simple rehash
900 * of their buffers is enough, otherwise SGE information can only
901 * be updated in a queue by recreating it. All resources that depend
902 * on queues (flows, indirection tables) must be recreated as well in
905 rehash
= (i
== priv
->rxqs_n
);
907 /* Clean up everything as with mlx5_dev_stop(). */
908 priv_special_flow_disable_all(priv
);
909 priv_mac_addrs_disable(priv
);
910 priv_destroy_hash_rxqs(priv
);
911 priv_fdir_disable(priv
);
912 priv_dev_interrupt_handler_uninstall(priv
, dev
);
915 /* Reconfigure each RX queue. */
916 for (i
= 0; (i
!= priv
->rxqs_n
); ++i
) {
917 struct rxq
*rxq
= (*priv
->rxqs
)[i
];
918 struct rxq_ctrl
*rxq_ctrl
=
919 container_of(rxq
, struct rxq_ctrl
, rxq
);
926 mb_len
= rte_pktmbuf_data_room_size(rxq
->mp
);
927 assert(mb_len
>= RTE_PKTMBUF_HEADROOM
);
928 /* Toggle scattered support (sp) if necessary. */
929 sp
= (max_frame_len
> (mb_len
- RTE_PKTMBUF_HEADROOM
));
930 /* Provide new values to rxq_setup(). */
931 dev
->data
->dev_conf
.rxmode
.jumbo_frame
= sp
;
932 dev
->data
->dev_conf
.rxmode
.max_rx_pkt_len
= max_frame_len
;
934 ret
= rxq_rehash(dev
, rxq_ctrl
);
936 ret
= rxq_ctrl_setup(dev
, rxq_ctrl
, 1 << rxq
->elts_n
,
937 rxq_ctrl
->socket
, NULL
, rxq
->mp
);
940 /* Attempt to roll back in case of error. */
941 tmp
= (mb_len
<< rxq
->sges_n
) - RTE_PKTMBUF_HEADROOM
;
942 if (max_frame_len
!= tmp
) {
946 /* Double fault, disable RX. */
950 * Use a safe RX burst function in case of error, otherwise mimic
954 ERROR("unable to reconfigure RX queues, RX disabled");
955 rx_func
= removed_rx_burst
;
956 } else if (restart
&&
958 !priv_create_hash_rxqs(priv
) &&
959 !priv_rehash_flows(priv
)) {
960 if (dev
->data
->dev_conf
.fdir_conf
.mode
== RTE_FDIR_MODE_NONE
)
961 priv_fdir_enable(priv
);
962 priv_dev_interrupt_handler_install(priv
, dev
);
965 /* Burst functions can now be called again. */
967 dev
->rx_pkt_burst
= rx_func
;
975 * DPDK callback to get flow control status.
978 * Pointer to Ethernet device structure.
979 * @param[out] fc_conf
980 * Flow control output buffer.
983 * 0 on success, negative errno value on failure.
986 mlx5_dev_get_flow_ctrl(struct rte_eth_dev
*dev
, struct rte_eth_fc_conf
*fc_conf
)
988 struct priv
*priv
= dev
->data
->dev_private
;
990 struct ethtool_pauseparam ethpause
= {
991 .cmd
= ETHTOOL_GPAUSEPARAM
995 if (mlx5_is_secondary())
996 return -E_RTE_SECONDARY
;
998 ifr
.ifr_data
= (void *)ðpause
;
1000 if (priv_ifreq(priv
, SIOCETHTOOL
, &ifr
)) {
1002 WARN("ioctl(SIOCETHTOOL, ETHTOOL_GPAUSEPARAM)"
1008 fc_conf
->autoneg
= ethpause
.autoneg
;
1009 if (ethpause
.rx_pause
&& ethpause
.tx_pause
)
1010 fc_conf
->mode
= RTE_FC_FULL
;
1011 else if (ethpause
.rx_pause
)
1012 fc_conf
->mode
= RTE_FC_RX_PAUSE
;
1013 else if (ethpause
.tx_pause
)
1014 fc_conf
->mode
= RTE_FC_TX_PAUSE
;
1016 fc_conf
->mode
= RTE_FC_NONE
;
1026 * DPDK callback to modify flow control parameters.
1029 * Pointer to Ethernet device structure.
1030 * @param[in] fc_conf
1031 * Flow control parameters.
1034 * 0 on success, negative errno value on failure.
1037 mlx5_dev_set_flow_ctrl(struct rte_eth_dev
*dev
, struct rte_eth_fc_conf
*fc_conf
)
1039 struct priv
*priv
= dev
->data
->dev_private
;
1041 struct ethtool_pauseparam ethpause
= {
1042 .cmd
= ETHTOOL_SPAUSEPARAM
1046 if (mlx5_is_secondary())
1047 return -E_RTE_SECONDARY
;
1049 ifr
.ifr_data
= (void *)ðpause
;
1050 ethpause
.autoneg
= fc_conf
->autoneg
;
1051 if (((fc_conf
->mode
& RTE_FC_FULL
) == RTE_FC_FULL
) ||
1052 (fc_conf
->mode
& RTE_FC_RX_PAUSE
))
1053 ethpause
.rx_pause
= 1;
1055 ethpause
.rx_pause
= 0;
1057 if (((fc_conf
->mode
& RTE_FC_FULL
) == RTE_FC_FULL
) ||
1058 (fc_conf
->mode
& RTE_FC_TX_PAUSE
))
1059 ethpause
.tx_pause
= 1;
1061 ethpause
.tx_pause
= 0;
1064 if (priv_ifreq(priv
, SIOCETHTOOL
, &ifr
)) {
1066 WARN("ioctl(SIOCETHTOOL, ETHTOOL_SPAUSEPARAM)"
1080 * Get PCI information from struct ibv_device.
1083 * Pointer to Ethernet device structure.
1084 * @param[out] pci_addr
1085 * PCI bus address output buffer.
1088 * 0 on success, -1 on failure and errno is set.
1091 mlx5_ibv_device_to_pci_addr(const struct ibv_device
*device
,
1092 struct rte_pci_addr
*pci_addr
)
1096 MKSTR(path
, "%s/device/uevent", device
->ibdev_path
);
1098 file
= fopen(path
, "rb");
1101 while (fgets(line
, sizeof(line
), file
) == line
) {
1102 size_t len
= strlen(line
);
1105 /* Truncate long lines. */
1106 if (len
== (sizeof(line
) - 1))
1107 while (line
[(len
- 1)] != '\n') {
1111 line
[(len
- 1)] = ret
;
1113 /* Extract information. */
1116 "%" SCNx16
":%" SCNx8
":%" SCNx8
".%" SCNx8
"\n",
1120 &pci_addr
->function
) == 4) {
1130 * Link status handler.
1133 * Pointer to private structure.
1135 * Pointer to the rte_eth_dev structure.
1138 * Nonzero if the callback process can be called immediately.
1141 priv_dev_link_status_handler(struct priv
*priv
, struct rte_eth_dev
*dev
)
1143 struct ibv_async_event event
;
1144 int port_change
= 0;
1147 /* Read all message and acknowledge them. */
1149 if (ibv_get_async_event(priv
->ctx
, &event
))
1152 if (event
.event_type
== IBV_EVENT_PORT_ACTIVE
||
1153 event
.event_type
== IBV_EVENT_PORT_ERR
)
1156 DEBUG("event type %d on port %d not handled",
1157 event
.event_type
, event
.element
.port_num
);
1158 ibv_ack_async_event(&event
);
1161 if (port_change
^ priv
->pending_alarm
) {
1162 struct rte_eth_link
*link
= &dev
->data
->dev_link
;
1164 priv
->pending_alarm
= 0;
1165 mlx5_link_update_unlocked(dev
, 0);
1166 if (((link
->link_speed
== 0) && link
->link_status
) ||
1167 ((link
->link_speed
!= 0) && !link
->link_status
)) {
1168 /* Inconsistent status, check again later. */
1169 priv
->pending_alarm
= 1;
1170 rte_eal_alarm_set(MLX5_ALARM_TIMEOUT_US
,
1171 mlx5_dev_link_status_handler
,
1180 * Handle delayed link status event.
1183 * Registered argument.
1186 mlx5_dev_link_status_handler(void *arg
)
1188 struct rte_eth_dev
*dev
= arg
;
1189 struct priv
*priv
= dev
->data
->dev_private
;
1193 assert(priv
->pending_alarm
== 1);
1194 ret
= priv_dev_link_status_handler(priv
, dev
);
1197 _rte_eth_dev_callback_process(dev
, RTE_ETH_EVENT_INTR_LSC
, NULL
);
1201 * Handle interrupts from the NIC.
1203 * @param[in] intr_handle
1204 * Interrupt handler.
1206 * Callback argument.
1209 mlx5_dev_interrupt_handler(struct rte_intr_handle
*intr_handle
, void *cb_arg
)
1211 struct rte_eth_dev
*dev
= cb_arg
;
1212 struct priv
*priv
= dev
->data
->dev_private
;
1217 ret
= priv_dev_link_status_handler(priv
, dev
);
1220 _rte_eth_dev_callback_process(dev
, RTE_ETH_EVENT_INTR_LSC
, NULL
);
1224 * Uninstall interrupt handler.
1227 * Pointer to private structure.
1229 * Pointer to the rte_eth_dev structure.
1232 priv_dev_interrupt_handler_uninstall(struct priv
*priv
, struct rte_eth_dev
*dev
)
1234 if (!dev
->data
->dev_conf
.intr_conf
.lsc
)
1236 rte_intr_callback_unregister(&priv
->intr_handle
,
1237 mlx5_dev_interrupt_handler
,
1239 if (priv
->pending_alarm
)
1240 rte_eal_alarm_cancel(mlx5_dev_link_status_handler
, dev
);
1241 priv
->pending_alarm
= 0;
1242 priv
->intr_handle
.fd
= 0;
1243 priv
->intr_handle
.type
= RTE_INTR_HANDLE_UNKNOWN
;
1247 * Install interrupt handler.
1250 * Pointer to private structure.
1252 * Pointer to the rte_eth_dev structure.
1255 priv_dev_interrupt_handler_install(struct priv
*priv
, struct rte_eth_dev
*dev
)
1259 if (!dev
->data
->dev_conf
.intr_conf
.lsc
)
1261 assert(priv
->ctx
->async_fd
> 0);
1262 flags
= fcntl(priv
->ctx
->async_fd
, F_GETFL
);
1263 rc
= fcntl(priv
->ctx
->async_fd
, F_SETFL
, flags
| O_NONBLOCK
);
1265 INFO("failed to change file descriptor async event queue");
1266 dev
->data
->dev_conf
.intr_conf
.lsc
= 0;
1268 priv
->intr_handle
.fd
= priv
->ctx
->async_fd
;
1269 priv
->intr_handle
.type
= RTE_INTR_HANDLE_EXT
;
1270 rte_intr_callback_register(&priv
->intr_handle
,
1271 mlx5_dev_interrupt_handler
,
1277 * Change the link state (UP / DOWN).
1280 * Pointer to Ethernet device structure.
1282 * Nonzero for link up, otherwise link down.
1285 * 0 on success, errno value on failure.
1288 priv_set_link(struct priv
*priv
, int up
)
1290 struct rte_eth_dev
*dev
= priv
->dev
;
1294 err
= priv_set_flags(priv
, ~IFF_UP
, IFF_UP
);
1297 priv_select_tx_function(priv
);
1298 priv_select_rx_function(priv
);
1300 err
= priv_set_flags(priv
, ~IFF_UP
, ~IFF_UP
);
1303 dev
->rx_pkt_burst
= removed_rx_burst
;
1304 dev
->tx_pkt_burst
= removed_tx_burst
;
1310 * DPDK callback to bring the link DOWN.
1313 * Pointer to Ethernet device structure.
1316 * 0 on success, errno value on failure.
1319 mlx5_set_link_down(struct rte_eth_dev
*dev
)
1321 struct priv
*priv
= dev
->data
->dev_private
;
1325 err
= priv_set_link(priv
, 0);
1331 * DPDK callback to bring the link UP.
1334 * Pointer to Ethernet device structure.
1337 * 0 on success, errno value on failure.
1340 mlx5_set_link_up(struct rte_eth_dev
*dev
)
1342 struct priv
*priv
= dev
->data
->dev_private
;
1346 err
= priv_set_link(priv
, 1);
1352 * Configure secondary process queues from a private data pointer (primary
1353 * or secondary) and update burst callbacks. Can take place only once.
1355 * All queues must have been previously created by the primary process to
1356 * avoid undefined behavior.
1359 * Private data pointer from either primary or secondary process.
1362 * Private data pointer from secondary process, NULL in case of error.
1365 mlx5_secondary_data_setup(struct priv
*priv
)
1367 unsigned int port_id
= 0;
1368 struct mlx5_secondary_data
*sd
;
1371 unsigned int nb_tx_queues
;
1372 unsigned int nb_rx_queues
;
1375 /* priv must be valid at this point. */
1376 assert(priv
!= NULL
);
1377 /* priv->dev must also be valid but may point to local memory from
1378 * another process, possibly with the same address and must not
1379 * be dereferenced yet. */
1380 assert(priv
->dev
!= NULL
);
1381 /* Determine port ID by finding out where priv comes from. */
1383 sd
= &mlx5_secondary_data
[port_id
];
1384 rte_spinlock_lock(&sd
->lock
);
1385 /* Primary process? */
1386 if (sd
->primary_priv
== priv
)
1388 /* Secondary process? */
1389 if (sd
->data
.dev_private
== priv
)
1391 rte_spinlock_unlock(&sd
->lock
);
1392 if (++port_id
== RTE_DIM(mlx5_secondary_data
))
1395 /* Switch to secondary private structure. If private data has already
1396 * been updated by another thread, there is nothing else to do. */
1397 priv
= sd
->data
.dev_private
;
1398 if (priv
->dev
->data
== &sd
->data
)
1400 /* Sanity checks. Secondary private structure is supposed to point
1401 * to local eth_dev, itself still pointing to the shared device data
1402 * structure allocated by the primary process. */
1403 assert(sd
->shared_dev_data
!= &sd
->data
);
1404 assert(sd
->data
.nb_tx_queues
== 0);
1405 assert(sd
->data
.tx_queues
== NULL
);
1406 assert(sd
->data
.nb_rx_queues
== 0);
1407 assert(sd
->data
.rx_queues
== NULL
);
1408 assert(priv
!= sd
->primary_priv
);
1409 assert(priv
->dev
->data
== sd
->shared_dev_data
);
1410 assert(priv
->txqs_n
== 0);
1411 assert(priv
->txqs
== NULL
);
1412 assert(priv
->rxqs_n
== 0);
1413 assert(priv
->rxqs
== NULL
);
1414 nb_tx_queues
= sd
->shared_dev_data
->nb_tx_queues
;
1415 nb_rx_queues
= sd
->shared_dev_data
->nb_rx_queues
;
1416 /* Allocate local storage for queues. */
1417 tx_queues
= rte_zmalloc("secondary ethdev->tx_queues",
1418 sizeof(sd
->data
.tx_queues
[0]) * nb_tx_queues
,
1419 RTE_CACHE_LINE_SIZE
);
1420 rx_queues
= rte_zmalloc("secondary ethdev->rx_queues",
1421 sizeof(sd
->data
.rx_queues
[0]) * nb_rx_queues
,
1422 RTE_CACHE_LINE_SIZE
);
1423 if (tx_queues
== NULL
|| rx_queues
== NULL
)
1425 /* Lock to prevent control operations during setup. */
1428 for (i
= 0; i
!= nb_tx_queues
; ++i
) {
1429 struct txq
*primary_txq
= (*sd
->primary_priv
->txqs
)[i
];
1430 struct txq_ctrl
*primary_txq_ctrl
;
1431 struct txq_ctrl
*txq_ctrl
;
1433 if (primary_txq
== NULL
)
1435 primary_txq_ctrl
= container_of(primary_txq
,
1436 struct txq_ctrl
, txq
);
1437 txq_ctrl
= rte_calloc_socket("TXQ", 1, sizeof(*txq_ctrl
) +
1438 (1 << primary_txq
->elts_n
) *
1439 sizeof(struct rte_mbuf
*), 0,
1440 primary_txq_ctrl
->socket
);
1441 if (txq_ctrl
!= NULL
) {
1442 if (txq_ctrl_setup(priv
->dev
,
1444 1 << primary_txq
->elts_n
,
1445 primary_txq_ctrl
->socket
,
1447 txq_ctrl
->txq
.stats
.idx
=
1448 primary_txq
->stats
.idx
;
1449 tx_queues
[i
] = &txq_ctrl
->txq
;
1455 txq_ctrl
= tx_queues
[--i
];
1456 txq_cleanup(txq_ctrl
);
1462 for (i
= 0; i
!= nb_rx_queues
; ++i
) {
1463 struct rxq_ctrl
*primary_rxq
=
1464 container_of((*sd
->primary_priv
->rxqs
)[i
],
1465 struct rxq_ctrl
, rxq
);
1467 if (primary_rxq
== NULL
)
1469 /* Not supported yet. */
1470 rx_queues
[i
] = NULL
;
1472 /* Update everything. */
1473 priv
->txqs
= (void *)tx_queues
;
1474 priv
->txqs_n
= nb_tx_queues
;
1475 priv
->rxqs
= (void *)rx_queues
;
1476 priv
->rxqs_n
= nb_rx_queues
;
1477 sd
->data
.rx_queues
= rx_queues
;
1478 sd
->data
.tx_queues
= tx_queues
;
1479 sd
->data
.nb_rx_queues
= nb_rx_queues
;
1480 sd
->data
.nb_tx_queues
= nb_tx_queues
;
1481 sd
->data
.dev_link
= sd
->shared_dev_data
->dev_link
;
1482 sd
->data
.mtu
= sd
->shared_dev_data
->mtu
;
1483 memcpy(sd
->data
.rx_queue_state
, sd
->shared_dev_data
->rx_queue_state
,
1484 sizeof(sd
->data
.rx_queue_state
));
1485 memcpy(sd
->data
.tx_queue_state
, sd
->shared_dev_data
->tx_queue_state
,
1486 sizeof(sd
->data
.tx_queue_state
));
1487 sd
->data
.dev_flags
= sd
->shared_dev_data
->dev_flags
;
1488 /* Use local data from now on. */
1490 priv
->dev
->data
= &sd
->data
;
1492 priv_select_tx_function(priv
);
1493 priv_select_rx_function(priv
);
1496 /* More sanity checks. */
1497 assert(priv
->dev
->data
== &sd
->data
);
1498 rte_spinlock_unlock(&sd
->lock
);
1502 rte_free(tx_queues
);
1503 rte_free(rx_queues
);
1504 rte_spinlock_unlock(&sd
->lock
);
1509 * Configure the TX function to use.
1512 * Pointer to private structure.
1515 priv_select_tx_function(struct priv
*priv
)
1517 priv
->dev
->tx_pkt_burst
= mlx5_tx_burst
;
1518 /* Display warning for unsupported configurations. */
1519 if (priv
->sriov
&& priv
->mps
)
1520 WARN("multi-packet send WQE cannot be used on a SR-IOV setup");
1521 /* Select appropriate TX function. */
1522 if ((priv
->sriov
== 0) && priv
->mps
&& priv
->txq_inline
) {
1523 priv
->dev
->tx_pkt_burst
= mlx5_tx_burst_mpw_inline
;
1524 DEBUG("selected MPW inline TX function");
1525 } else if ((priv
->sriov
== 0) && priv
->mps
) {
1526 priv
->dev
->tx_pkt_burst
= mlx5_tx_burst_mpw
;
1527 DEBUG("selected MPW TX function");
1532 * Configure the RX function to use.
1535 * Pointer to private structure.
1538 priv_select_rx_function(struct priv
*priv
)
1540 priv
->dev
->rx_pkt_burst
= mlx5_rx_burst
;