1 /* SPDX-License-Identifier: BSD-3-Clause
2 * Copyright 2017 6WIND S.A.
3 * Copyright 2017 Mellanox Technologies, Ltd
10 #include <rte_debug.h>
11 #include <rte_atomic.h>
12 #include <rte_ethdev_driver.h>
13 #include <rte_malloc.h>
15 #include <rte_cycles.h>
16 #include <rte_ethdev.h>
17 #include <rte_string_fns.h>
19 #include "failsafe_private.h"
22 fs_dev_configure(struct rte_eth_dev
*dev
)
24 struct sub_device
*sdev
;
29 FOREACH_SUBDEV(sdev
, i
, dev
) {
30 int rmv_interrupt
= 0;
31 int lsc_interrupt
= 0;
34 if (sdev
->state
!= DEV_PROBED
&&
35 !(PRIV(dev
)->alarm_lock
== 0 && sdev
->state
== DEV_ACTIVE
))
38 rmv_interrupt
= ETH(sdev
)->data
->dev_flags
&
41 DEBUG("Enabling RMV interrupts for sub_device %d", i
);
42 dev
->data
->dev_conf
.intr_conf
.rmv
= 1;
44 DEBUG("sub_device %d does not support RMV event", i
);
46 lsc_enabled
= dev
->data
->dev_conf
.intr_conf
.lsc
;
47 lsc_interrupt
= lsc_enabled
&&
48 (ETH(sdev
)->data
->dev_flags
&
49 RTE_ETH_DEV_INTR_LSC
);
51 DEBUG("Enabling LSC interrupts for sub_device %d", i
);
52 dev
->data
->dev_conf
.intr_conf
.lsc
= 1;
53 } else if (lsc_enabled
&& !lsc_interrupt
) {
54 DEBUG("Disabling LSC interrupts for sub_device %d", i
);
55 dev
->data
->dev_conf
.intr_conf
.lsc
= 0;
57 DEBUG("Configuring sub-device %d", i
);
58 ret
= rte_eth_dev_configure(PORT_ID(sdev
),
59 dev
->data
->nb_rx_queues
,
60 dev
->data
->nb_tx_queues
,
61 &dev
->data
->dev_conf
);
63 if (!fs_err(sdev
, ret
))
65 ERROR("Could not configure sub_device %d", i
);
69 if (rmv_interrupt
&& sdev
->rmv_callback
== 0) {
70 ret
= rte_eth_dev_callback_register(PORT_ID(sdev
),
71 RTE_ETH_EVENT_INTR_RMV
,
72 failsafe_eth_rmv_event_callback
,
75 WARN("Failed to register RMV callback for sub_device %d",
78 sdev
->rmv_callback
= 1;
80 dev
->data
->dev_conf
.intr_conf
.rmv
= 0;
81 if (lsc_interrupt
&& sdev
->lsc_callback
== 0) {
82 ret
= rte_eth_dev_callback_register(PORT_ID(sdev
),
83 RTE_ETH_EVENT_INTR_LSC
,
84 failsafe_eth_lsc_event_callback
,
87 WARN("Failed to register LSC callback for sub_device %d",
90 sdev
->lsc_callback
= 1;
92 dev
->data
->dev_conf
.intr_conf
.lsc
= lsc_enabled
;
93 sdev
->state
= DEV_ACTIVE
;
95 if (PRIV(dev
)->state
< DEV_ACTIVE
)
96 PRIV(dev
)->state
= DEV_ACTIVE
;
102 fs_set_queues_state_start(struct rte_eth_dev
*dev
)
108 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
109 rxq
= dev
->data
->rx_queues
[i
];
110 if (rxq
!= NULL
&& !rxq
->info
.conf
.rx_deferred_start
)
111 dev
->data
->rx_queue_state
[i
] =
112 RTE_ETH_QUEUE_STATE_STARTED
;
114 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
115 txq
= dev
->data
->tx_queues
[i
];
116 if (txq
!= NULL
&& !txq
->info
.conf
.tx_deferred_start
)
117 dev
->data
->tx_queue_state
[i
] =
118 RTE_ETH_QUEUE_STATE_STARTED
;
123 fs_dev_start(struct rte_eth_dev
*dev
)
125 struct sub_device
*sdev
;
130 ret
= failsafe_rx_intr_install(dev
);
135 FOREACH_SUBDEV(sdev
, i
, dev
) {
136 if (sdev
->state
!= DEV_ACTIVE
)
138 DEBUG("Starting sub_device %d", i
);
139 ret
= rte_eth_dev_start(PORT_ID(sdev
));
141 if (!fs_err(sdev
, ret
))
146 ret
= failsafe_rx_intr_install_subdevice(sdev
);
148 if (!fs_err(sdev
, ret
))
150 rte_eth_dev_stop(PORT_ID(sdev
));
154 sdev
->state
= DEV_STARTED
;
156 if (PRIV(dev
)->state
< DEV_STARTED
) {
157 PRIV(dev
)->state
= DEV_STARTED
;
158 fs_set_queues_state_start(dev
);
160 fs_switch_dev(dev
, NULL
);
166 fs_set_queues_state_stop(struct rte_eth_dev
*dev
)
170 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++)
171 if (dev
->data
->rx_queues
[i
] != NULL
)
172 dev
->data
->rx_queue_state
[i
] =
173 RTE_ETH_QUEUE_STATE_STOPPED
;
174 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++)
175 if (dev
->data
->tx_queues
[i
] != NULL
)
176 dev
->data
->tx_queue_state
[i
] =
177 RTE_ETH_QUEUE_STATE_STOPPED
;
181 fs_dev_stop(struct rte_eth_dev
*dev
)
183 struct sub_device
*sdev
;
187 PRIV(dev
)->state
= DEV_STARTED
- 1;
188 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_STARTED
) {
189 rte_eth_dev_stop(PORT_ID(sdev
));
190 failsafe_rx_intr_uninstall_subdevice(sdev
);
191 sdev
->state
= DEV_STARTED
- 1;
193 failsafe_rx_intr_uninstall(dev
);
194 fs_set_queues_state_stop(dev
);
199 fs_dev_set_link_up(struct rte_eth_dev
*dev
)
201 struct sub_device
*sdev
;
206 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
207 DEBUG("Calling rte_eth_dev_set_link_up on sub_device %d", i
);
208 ret
= rte_eth_dev_set_link_up(PORT_ID(sdev
));
209 if ((ret
= fs_err(sdev
, ret
))) {
210 ERROR("Operation rte_eth_dev_set_link_up failed for sub_device %d"
211 " with error %d", i
, ret
);
221 fs_dev_set_link_down(struct rte_eth_dev
*dev
)
223 struct sub_device
*sdev
;
228 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
229 DEBUG("Calling rte_eth_dev_set_link_down on sub_device %d", i
);
230 ret
= rte_eth_dev_set_link_down(PORT_ID(sdev
));
231 if ((ret
= fs_err(sdev
, ret
))) {
232 ERROR("Operation rte_eth_dev_set_link_down failed for sub_device %d"
233 " with error %d", i
, ret
);
242 static void fs_dev_free_queues(struct rte_eth_dev
*dev
);
244 fs_dev_close(struct rte_eth_dev
*dev
)
246 struct sub_device
*sdev
;
250 failsafe_hotplug_alarm_cancel(dev
);
251 if (PRIV(dev
)->state
== DEV_STARTED
)
252 dev
->dev_ops
->dev_stop(dev
);
253 PRIV(dev
)->state
= DEV_ACTIVE
- 1;
254 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
255 DEBUG("Closing sub_device %d", i
);
256 failsafe_eth_dev_unregister_callbacks(sdev
);
257 rte_eth_dev_close(PORT_ID(sdev
));
258 sdev
->state
= DEV_ACTIVE
- 1;
260 fs_dev_free_queues(dev
);
265 fs_rx_queue_stop(struct rte_eth_dev
*dev
, uint16_t rx_queue_id
)
267 struct sub_device
*sdev
;
274 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
275 uint16_t port_id
= ETH(sdev
)->data
->port_id
;
277 ret
= rte_eth_dev_rx_queue_stop(port_id
, rx_queue_id
);
278 ret
= fs_err(sdev
, ret
);
280 ERROR("Rx queue stop failed for subdevice %d", i
);
286 dev
->data
->rx_queue_state
[rx_queue_id
] = RTE_ETH_QUEUE_STATE_STOPPED
;
288 /* Return 0 in case of at least one successful queue stop */
289 return (failure
) ? err
: 0;
293 fs_rx_queue_start(struct rte_eth_dev
*dev
, uint16_t rx_queue_id
)
295 struct sub_device
*sdev
;
300 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
301 uint16_t port_id
= ETH(sdev
)->data
->port_id
;
303 ret
= rte_eth_dev_rx_queue_start(port_id
, rx_queue_id
);
304 ret
= fs_err(sdev
, ret
);
306 ERROR("Rx queue start failed for subdevice %d", i
);
307 fs_rx_queue_stop(dev
, rx_queue_id
);
312 dev
->data
->rx_queue_state
[rx_queue_id
] = RTE_ETH_QUEUE_STATE_STARTED
;
318 fs_tx_queue_stop(struct rte_eth_dev
*dev
, uint16_t tx_queue_id
)
320 struct sub_device
*sdev
;
327 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
328 uint16_t port_id
= ETH(sdev
)->data
->port_id
;
330 ret
= rte_eth_dev_tx_queue_stop(port_id
, tx_queue_id
);
331 ret
= fs_err(sdev
, ret
);
333 ERROR("Tx queue stop failed for subdevice %d", i
);
339 dev
->data
->tx_queue_state
[tx_queue_id
] = RTE_ETH_QUEUE_STATE_STOPPED
;
341 /* Return 0 in case of at least one successful queue stop */
342 return (failure
) ? err
: 0;
346 fs_tx_queue_start(struct rte_eth_dev
*dev
, uint16_t tx_queue_id
)
348 struct sub_device
*sdev
;
353 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
354 uint16_t port_id
= ETH(sdev
)->data
->port_id
;
356 ret
= rte_eth_dev_tx_queue_start(port_id
, tx_queue_id
);
357 ret
= fs_err(sdev
, ret
);
359 ERROR("Tx queue start failed for subdevice %d", i
);
360 fs_tx_queue_stop(dev
, tx_queue_id
);
365 dev
->data
->tx_queue_state
[tx_queue_id
] = RTE_ETH_QUEUE_STATE_STARTED
;
371 fs_rx_queue_release(void *queue
)
373 struct rte_eth_dev
*dev
;
374 struct sub_device
*sdev
;
381 dev
= &rte_eth_devices
[rxq
->priv
->data
->port_id
];
383 if (rxq
->event_fd
>= 0)
384 close(rxq
->event_fd
);
385 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
386 if (ETH(sdev
)->data
->rx_queues
!= NULL
&&
387 ETH(sdev
)->data
->rx_queues
[rxq
->qid
] != NULL
) {
388 SUBOPS(sdev
, rx_queue_release
)
389 (ETH(sdev
)->data
->rx_queues
[rxq
->qid
]);
392 dev
->data
->rx_queues
[rxq
->qid
] = NULL
;
398 fs_rx_queue_setup(struct rte_eth_dev
*dev
,
399 uint16_t rx_queue_id
,
401 unsigned int socket_id
,
402 const struct rte_eth_rxconf
*rx_conf
,
403 struct rte_mempool
*mb_pool
)
406 * FIXME: Add a proper interface in rte_eal_interrupts for
407 * allocating eventfd as an interrupt vector.
408 * For the time being, fake as if we are using MSIX interrupts,
409 * this will cause rte_intr_efd_enable to allocate an eventfd for us.
411 struct rte_intr_handle intr_handle
= {
412 .type
= RTE_INTR_HANDLE_VFIO_MSIX
,
415 struct sub_device
*sdev
;
421 if (rx_conf
->rx_deferred_start
) {
422 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_PROBED
) {
423 if (SUBOPS(sdev
, rx_queue_start
) == NULL
) {
424 ERROR("Rx queue deferred start is not "
425 "supported for subdevice %d", i
);
431 rxq
= dev
->data
->rx_queues
[rx_queue_id
];
433 fs_rx_queue_release(rxq
);
434 dev
->data
->rx_queues
[rx_queue_id
] = NULL
;
436 rxq
= rte_zmalloc(NULL
,
438 sizeof(rte_atomic64_t
) * PRIV(dev
)->subs_tail
,
439 RTE_CACHE_LINE_SIZE
);
444 FOREACH_SUBDEV(sdev
, i
, dev
)
445 rte_atomic64_init(&rxq
->refcnt
[i
]);
446 rxq
->qid
= rx_queue_id
;
447 rxq
->socket_id
= socket_id
;
448 rxq
->info
.mp
= mb_pool
;
449 rxq
->info
.conf
= *rx_conf
;
450 rxq
->info
.nb_desc
= nb_rx_desc
;
451 rxq
->priv
= PRIV(dev
);
452 rxq
->sdev
= PRIV(dev
)->subs
;
453 ret
= rte_intr_efd_enable(&intr_handle
, 1);
458 rxq
->event_fd
= intr_handle
.efds
[0];
459 dev
->data
->rx_queues
[rx_queue_id
] = rxq
;
460 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
461 ret
= rte_eth_rx_queue_setup(PORT_ID(sdev
),
463 nb_rx_desc
, socket_id
,
465 if ((ret
= fs_err(sdev
, ret
))) {
466 ERROR("RX queue setup failed for sub_device %d", i
);
473 fs_rx_queue_release(rxq
);
479 fs_rx_intr_enable(struct rte_eth_dev
*dev
, uint16_t idx
)
482 struct sub_device
*sdev
;
488 if (idx
>= dev
->data
->nb_rx_queues
) {
492 rxq
= dev
->data
->rx_queues
[idx
];
493 if (rxq
== NULL
|| rxq
->event_fd
<= 0) {
497 /* Fail if proxy service is nor running. */
498 if (PRIV(dev
)->rxp
.sstate
!= SS_RUNNING
) {
499 ERROR("failsafe interrupt services are not running");
503 rxq
->enable_events
= 1;
504 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
505 ret
= rte_eth_dev_rx_intr_enable(PORT_ID(sdev
), idx
);
506 ret
= fs_err(sdev
, ret
);
518 fs_rx_intr_disable(struct rte_eth_dev
*dev
, uint16_t idx
)
521 struct sub_device
*sdev
;
528 if (idx
>= dev
->data
->nb_rx_queues
) {
532 rxq
= dev
->data
->rx_queues
[idx
];
533 if (rxq
== NULL
|| rxq
->event_fd
<= 0) {
537 rxq
->enable_events
= 0;
538 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
539 ret
= rte_eth_dev_rx_intr_disable(PORT_ID(sdev
), idx
);
540 ret
= fs_err(sdev
, ret
);
544 /* Clear pending events */
545 while (read(rxq
->event_fd
, &u64
, sizeof(uint64_t)) > 0)
555 fs_tx_queue_release(void *queue
)
557 struct rte_eth_dev
*dev
;
558 struct sub_device
*sdev
;
565 dev
= &rte_eth_devices
[txq
->priv
->data
->port_id
];
567 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
568 if (ETH(sdev
)->data
->tx_queues
!= NULL
&&
569 ETH(sdev
)->data
->tx_queues
[txq
->qid
] != NULL
) {
570 SUBOPS(sdev
, tx_queue_release
)
571 (ETH(sdev
)->data
->tx_queues
[txq
->qid
]);
574 dev
->data
->tx_queues
[txq
->qid
] = NULL
;
580 fs_tx_queue_setup(struct rte_eth_dev
*dev
,
581 uint16_t tx_queue_id
,
583 unsigned int socket_id
,
584 const struct rte_eth_txconf
*tx_conf
)
586 struct sub_device
*sdev
;
592 if (tx_conf
->tx_deferred_start
) {
593 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_PROBED
) {
594 if (SUBOPS(sdev
, tx_queue_start
) == NULL
) {
595 ERROR("Tx queue deferred start is not "
596 "supported for subdevice %d", i
);
602 txq
= dev
->data
->tx_queues
[tx_queue_id
];
604 fs_tx_queue_release(txq
);
605 dev
->data
->tx_queues
[tx_queue_id
] = NULL
;
607 txq
= rte_zmalloc("ethdev TX queue",
609 sizeof(rte_atomic64_t
) * PRIV(dev
)->subs_tail
,
610 RTE_CACHE_LINE_SIZE
);
615 FOREACH_SUBDEV(sdev
, i
, dev
)
616 rte_atomic64_init(&txq
->refcnt
[i
]);
617 txq
->qid
= tx_queue_id
;
618 txq
->socket_id
= socket_id
;
619 txq
->info
.conf
= *tx_conf
;
620 txq
->info
.nb_desc
= nb_tx_desc
;
621 txq
->priv
= PRIV(dev
);
622 dev
->data
->tx_queues
[tx_queue_id
] = txq
;
623 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
624 ret
= rte_eth_tx_queue_setup(PORT_ID(sdev
),
626 nb_tx_desc
, socket_id
,
628 if ((ret
= fs_err(sdev
, ret
))) {
629 ERROR("TX queue setup failed for sub_device %d", i
);
636 fs_tx_queue_release(txq
);
642 fs_dev_free_queues(struct rte_eth_dev
*dev
)
646 for (i
= 0; i
< dev
->data
->nb_rx_queues
; i
++) {
647 fs_rx_queue_release(dev
->data
->rx_queues
[i
]);
648 dev
->data
->rx_queues
[i
] = NULL
;
650 dev
->data
->nb_rx_queues
= 0;
651 for (i
= 0; i
< dev
->data
->nb_tx_queues
; i
++) {
652 fs_tx_queue_release(dev
->data
->tx_queues
[i
]);
653 dev
->data
->tx_queues
[i
] = NULL
;
655 dev
->data
->nb_tx_queues
= 0;
659 fs_promiscuous_enable(struct rte_eth_dev
*dev
)
661 struct sub_device
*sdev
;
666 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
667 ret
= rte_eth_promiscuous_enable(PORT_ID(sdev
));
668 ret
= fs_err(sdev
, ret
);
670 ERROR("Promiscuous mode enable failed for subdevice %d",
676 /* Rollback in the case of failure */
677 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
678 ret
= rte_eth_promiscuous_disable(PORT_ID(sdev
));
679 ret
= fs_err(sdev
, ret
);
681 ERROR("Promiscuous mode disable during rollback failed for subdevice %d",
691 fs_promiscuous_disable(struct rte_eth_dev
*dev
)
693 struct sub_device
*sdev
;
698 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
699 ret
= rte_eth_promiscuous_disable(PORT_ID(sdev
));
700 ret
= fs_err(sdev
, ret
);
702 ERROR("Promiscuous mode disable failed for subdevice %d",
708 /* Rollback in the case of failure */
709 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
710 ret
= rte_eth_promiscuous_enable(PORT_ID(sdev
));
711 ret
= fs_err(sdev
, ret
);
713 ERROR("Promiscuous mode enable during rollback failed for subdevice %d",
723 fs_allmulticast_enable(struct rte_eth_dev
*dev
)
725 struct sub_device
*sdev
;
730 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
731 ret
= rte_eth_allmulticast_enable(PORT_ID(sdev
));
732 ret
= fs_err(sdev
, ret
);
734 ERROR("All-multicast mode enable failed for subdevice %d",
740 /* Rollback in the case of failure */
741 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
742 ret
= rte_eth_allmulticast_disable(PORT_ID(sdev
));
743 ret
= fs_err(sdev
, ret
);
745 ERROR("All-multicast mode disable during rollback failed for subdevice %d",
755 fs_allmulticast_disable(struct rte_eth_dev
*dev
)
757 struct sub_device
*sdev
;
762 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
763 ret
= rte_eth_allmulticast_disable(PORT_ID(sdev
));
764 ret
= fs_err(sdev
, ret
);
766 ERROR("All-multicast mode disable failed for subdevice %d",
772 /* Rollback in the case of failure */
773 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
774 ret
= rte_eth_allmulticast_enable(PORT_ID(sdev
));
775 ret
= fs_err(sdev
, ret
);
777 ERROR("All-multicast mode enable during rollback failed for subdevice %d",
787 fs_link_update(struct rte_eth_dev
*dev
,
788 int wait_to_complete
)
790 struct sub_device
*sdev
;
795 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
796 DEBUG("Calling link_update on sub_device %d", i
);
797 ret
= (SUBOPS(sdev
, link_update
))(ETH(sdev
), wait_to_complete
);
798 if (ret
&& ret
!= -1 && sdev
->remove
== 0 &&
799 rte_eth_dev_is_removed(PORT_ID(sdev
)) == 0) {
800 ERROR("Link update failed for sub_device %d with error %d",
806 if (TX_SUBDEV(dev
)) {
807 struct rte_eth_link
*l1
;
808 struct rte_eth_link
*l2
;
810 l1
= &dev
->data
->dev_link
;
811 l2
= Ð(TX_SUBDEV(dev
))->data
->dev_link
;
812 if (memcmp(l1
, l2
, sizeof(*l1
))) {
823 fs_stats_get(struct rte_eth_dev
*dev
,
824 struct rte_eth_stats
*stats
)
826 struct rte_eth_stats backup
;
827 struct sub_device
*sdev
;
832 rte_memcpy(stats
, &PRIV(dev
)->stats_accumulator
, sizeof(*stats
));
833 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
834 struct rte_eth_stats
*snapshot
= &sdev
->stats_snapshot
.stats
;
835 uint64_t *timestamp
= &sdev
->stats_snapshot
.timestamp
;
837 rte_memcpy(&backup
, snapshot
, sizeof(backup
));
838 ret
= rte_eth_stats_get(PORT_ID(sdev
), snapshot
);
840 if (!fs_err(sdev
, ret
)) {
841 rte_memcpy(snapshot
, &backup
, sizeof(backup
));
844 ERROR("Operation rte_eth_stats_get failed for sub_device %d with error %d",
850 *timestamp
= rte_rdtsc();
852 failsafe_stats_increment(stats
, snapshot
);
859 fs_stats_reset(struct rte_eth_dev
*dev
)
861 struct sub_device
*sdev
;
866 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
867 ret
= rte_eth_stats_reset(PORT_ID(sdev
));
869 if (!fs_err(sdev
, ret
))
872 ERROR("Operation rte_eth_stats_reset failed for sub_device %d with error %d",
877 memset(&sdev
->stats_snapshot
, 0, sizeof(struct rte_eth_stats
));
879 memset(&PRIV(dev
)->stats_accumulator
, 0, sizeof(struct rte_eth_stats
));
886 __fs_xstats_count(struct rte_eth_dev
*dev
)
888 struct sub_device
*sdev
;
893 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
894 ret
= rte_eth_xstats_get_names(PORT_ID(sdev
), NULL
, 0);
904 __fs_xstats_get_names(struct rte_eth_dev
*dev
,
905 struct rte_eth_xstat_name
*xstats_names
,
908 struct sub_device
*sdev
;
909 unsigned int count
= 0;
912 /* Caller only cares about count */
914 return __fs_xstats_count(dev
);
916 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
917 struct rte_eth_xstat_name
*sub_names
= xstats_names
+ count
;
923 r
= rte_eth_xstats_get_names(PORT_ID(sdev
),
924 sub_names
, limit
- count
);
928 /* add subN_ prefix to names */
929 for (j
= 0; j
< r
; j
++) {
930 char *xname
= sub_names
[j
].name
;
931 char tmp
[RTE_ETH_XSTATS_NAME_SIZE
];
933 if ((xname
[0] == 't' || xname
[0] == 'r') &&
934 xname
[1] == 'x' && xname
[2] == '_')
935 snprintf(tmp
, sizeof(tmp
), "%.3ssub%u_%s",
936 xname
, i
, xname
+ 3);
938 snprintf(tmp
, sizeof(tmp
), "sub%u_%s",
941 strlcpy(xname
, tmp
, RTE_ETH_XSTATS_NAME_SIZE
);
949 fs_xstats_get_names(struct rte_eth_dev
*dev
,
950 struct rte_eth_xstat_name
*xstats_names
,
956 ret
= __fs_xstats_get_names(dev
, xstats_names
, limit
);
962 __fs_xstats_get(struct rte_eth_dev
*dev
,
963 struct rte_eth_xstat
*xstats
,
966 unsigned int count
= 0;
967 struct sub_device
*sdev
;
971 ret
= __fs_xstats_count(dev
);
974 * or caller did not give enough space
977 if (ret
< 0 || ret
> (int)n
|| xstats
== NULL
)
980 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
981 ret
= rte_eth_xstats_get(PORT_ID(sdev
), xstats
, n
);
988 /* add offset to id's from sub-device */
989 for (j
= 0; j
< ret
; j
++)
990 xstats
[j
].id
+= count
;
1001 fs_xstats_get(struct rte_eth_dev
*dev
,
1002 struct rte_eth_xstat
*xstats
,
1008 ret
= __fs_xstats_get(dev
, xstats
, n
);
1016 fs_xstats_reset(struct rte_eth_dev
*dev
)
1018 struct sub_device
*sdev
;
1023 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1024 r
= rte_eth_xstats_reset(PORT_ID(sdev
));
1034 fs_dev_merge_desc_lim(struct rte_eth_desc_lim
*to
,
1035 const struct rte_eth_desc_lim
*from
)
1037 to
->nb_max
= RTE_MIN(to
->nb_max
, from
->nb_max
);
1038 to
->nb_min
= RTE_MAX(to
->nb_min
, from
->nb_min
);
1039 to
->nb_align
= RTE_MAX(to
->nb_align
, from
->nb_align
);
1041 to
->nb_seg_max
= RTE_MIN(to
->nb_seg_max
, from
->nb_seg_max
);
1042 to
->nb_mtu_seg_max
= RTE_MIN(to
->nb_mtu_seg_max
, from
->nb_mtu_seg_max
);
1046 * Merge the information from sub-devices.
1048 * The reported values must be the common subset of all sub devices
1051 fs_dev_merge_info(struct rte_eth_dev_info
*info
,
1052 const struct rte_eth_dev_info
*sinfo
)
1054 info
->max_rx_pktlen
= RTE_MIN(info
->max_rx_pktlen
, sinfo
->max_rx_pktlen
);
1055 info
->max_rx_queues
= RTE_MIN(info
->max_rx_queues
, sinfo
->max_rx_queues
);
1056 info
->max_tx_queues
= RTE_MIN(info
->max_tx_queues
, sinfo
->max_tx_queues
);
1057 info
->max_mac_addrs
= RTE_MIN(info
->max_mac_addrs
, sinfo
->max_mac_addrs
);
1058 info
->max_hash_mac_addrs
= RTE_MIN(info
->max_hash_mac_addrs
,
1059 sinfo
->max_hash_mac_addrs
);
1060 info
->max_vmdq_pools
= RTE_MIN(info
->max_vmdq_pools
, sinfo
->max_vmdq_pools
);
1061 info
->max_vfs
= RTE_MIN(info
->max_vfs
, sinfo
->max_vfs
);
1063 fs_dev_merge_desc_lim(&info
->rx_desc_lim
, &sinfo
->rx_desc_lim
);
1064 fs_dev_merge_desc_lim(&info
->tx_desc_lim
, &sinfo
->tx_desc_lim
);
1066 info
->rx_offload_capa
&= sinfo
->rx_offload_capa
;
1067 info
->tx_offload_capa
&= sinfo
->tx_offload_capa
;
1068 info
->rx_queue_offload_capa
&= sinfo
->rx_queue_offload_capa
;
1069 info
->tx_queue_offload_capa
&= sinfo
->tx_queue_offload_capa
;
1070 info
->flow_type_rss_offloads
&= sinfo
->flow_type_rss_offloads
;
1071 info
->hash_key_size
= RTE_MIN(info
->hash_key_size
,
1072 sinfo
->hash_key_size
);
1076 * Fail-safe dev_infos_get rules:
1080 * Use the maximum possible values for any field, so as not
1081 * to impede any further configuration effort.
1083 * Limits capabilities to those that are understood by the
1084 * fail-safe PMD. This understanding stems from the fail-safe
1085 * being capable of verifying that the related capability is
1086 * expressed within the device configuration (struct rte_eth_conf).
1088 * At least one probed sub_device:
1090 * Uses values from the active probed sub_device
1091 * The rationale here is that if any sub_device is less capable
1092 * (for example concerning the number of queues) than the active
1093 * sub_device, then its subsequent configuration will fail.
1094 * It is impossible to foresee this failure when the failing sub_device
1095 * is supposed to be plugged-in later on, so the configuration process
1096 * is the single point of failure and error reporting.
1098 * Uses a logical AND of RX capabilities among
1099 * all sub_devices and the default capabilities.
1100 * Uses a logical AND of TX capabilities among
1101 * the active probed sub_device and the default capabilities.
1102 * Uses a logical AND of device capabilities among
1103 * all sub_devices and the default capabilities.
1107 fs_dev_infos_get(struct rte_eth_dev
*dev
,
1108 struct rte_eth_dev_info
*infos
)
1110 struct sub_device
*sdev
;
1114 /* Use maximum upper bounds by default */
1115 infos
->max_rx_pktlen
= UINT32_MAX
;
1116 infos
->max_rx_queues
= RTE_MAX_QUEUES_PER_PORT
;
1117 infos
->max_tx_queues
= RTE_MAX_QUEUES_PER_PORT
;
1118 infos
->max_mac_addrs
= FAILSAFE_MAX_ETHADDR
;
1119 infos
->max_hash_mac_addrs
= UINT32_MAX
;
1120 infos
->max_vfs
= UINT16_MAX
;
1121 infos
->max_vmdq_pools
= UINT16_MAX
;
1122 infos
->hash_key_size
= UINT8_MAX
;
1125 * Set of capabilities that can be verified upon
1126 * configuring a sub-device.
1128 infos
->rx_offload_capa
=
1129 DEV_RX_OFFLOAD_VLAN_STRIP
|
1130 DEV_RX_OFFLOAD_IPV4_CKSUM
|
1131 DEV_RX_OFFLOAD_UDP_CKSUM
|
1132 DEV_RX_OFFLOAD_TCP_CKSUM
|
1133 DEV_RX_OFFLOAD_TCP_LRO
|
1134 DEV_RX_OFFLOAD_QINQ_STRIP
|
1135 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM
|
1136 DEV_RX_OFFLOAD_MACSEC_STRIP
|
1137 DEV_RX_OFFLOAD_HEADER_SPLIT
|
1138 DEV_RX_OFFLOAD_VLAN_FILTER
|
1139 DEV_RX_OFFLOAD_VLAN_EXTEND
|
1140 DEV_RX_OFFLOAD_JUMBO_FRAME
|
1141 DEV_RX_OFFLOAD_SCATTER
|
1142 DEV_RX_OFFLOAD_TIMESTAMP
|
1143 DEV_RX_OFFLOAD_SECURITY
;
1145 infos
->rx_queue_offload_capa
=
1146 DEV_RX_OFFLOAD_VLAN_STRIP
|
1147 DEV_RX_OFFLOAD_IPV4_CKSUM
|
1148 DEV_RX_OFFLOAD_UDP_CKSUM
|
1149 DEV_RX_OFFLOAD_TCP_CKSUM
|
1150 DEV_RX_OFFLOAD_TCP_LRO
|
1151 DEV_RX_OFFLOAD_QINQ_STRIP
|
1152 DEV_RX_OFFLOAD_OUTER_IPV4_CKSUM
|
1153 DEV_RX_OFFLOAD_MACSEC_STRIP
|
1154 DEV_RX_OFFLOAD_HEADER_SPLIT
|
1155 DEV_RX_OFFLOAD_VLAN_FILTER
|
1156 DEV_RX_OFFLOAD_VLAN_EXTEND
|
1157 DEV_RX_OFFLOAD_JUMBO_FRAME
|
1158 DEV_RX_OFFLOAD_SCATTER
|
1159 DEV_RX_OFFLOAD_TIMESTAMP
|
1160 DEV_RX_OFFLOAD_SECURITY
;
1162 infos
->tx_offload_capa
=
1163 DEV_TX_OFFLOAD_MULTI_SEGS
|
1164 DEV_TX_OFFLOAD_MBUF_FAST_FREE
|
1165 DEV_TX_OFFLOAD_IPV4_CKSUM
|
1166 DEV_TX_OFFLOAD_UDP_CKSUM
|
1167 DEV_TX_OFFLOAD_TCP_CKSUM
|
1168 DEV_TX_OFFLOAD_TCP_TSO
;
1170 infos
->flow_type_rss_offloads
=
1175 RTE_ETH_DEV_CAPA_RUNTIME_RX_QUEUE_SETUP
|
1176 RTE_ETH_DEV_CAPA_RUNTIME_TX_QUEUE_SETUP
;
1178 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_PROBED
) {
1179 struct rte_eth_dev_info sub_info
;
1181 ret
= rte_eth_dev_info_get(PORT_ID(sdev
), &sub_info
);
1182 ret
= fs_err(sdev
, ret
);
1186 fs_dev_merge_info(infos
, &sub_info
);
1192 static const uint32_t *
1193 fs_dev_supported_ptypes_get(struct rte_eth_dev
*dev
)
1195 struct sub_device
*sdev
;
1196 struct rte_eth_dev
*edev
;
1197 const uint32_t *ret
;
1200 sdev
= TX_SUBDEV(dev
);
1206 /* ENOTSUP: counts as no supported ptypes */
1207 if (SUBOPS(sdev
, dev_supported_ptypes_get
) == NULL
) {
1212 * The API does not permit to do a clean AND of all ptypes,
1213 * It is also incomplete by design and we do not really care
1214 * to have a best possible value in this context.
1215 * We just return the ptypes of the device of highest
1216 * priority, usually the PREFERRED device.
1218 ret
= SUBOPS(sdev
, dev_supported_ptypes_get
)(edev
);
1225 fs_mtu_set(struct rte_eth_dev
*dev
, uint16_t mtu
)
1227 struct sub_device
*sdev
;
1232 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1233 DEBUG("Calling rte_eth_dev_set_mtu on sub_device %d", i
);
1234 ret
= rte_eth_dev_set_mtu(PORT_ID(sdev
), mtu
);
1235 if ((ret
= fs_err(sdev
, ret
))) {
1236 ERROR("Operation rte_eth_dev_set_mtu failed for sub_device %d with error %d",
1247 fs_vlan_filter_set(struct rte_eth_dev
*dev
, uint16_t vlan_id
, int on
)
1249 struct sub_device
*sdev
;
1254 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1255 DEBUG("Calling rte_eth_dev_vlan_filter on sub_device %d", i
);
1256 ret
= rte_eth_dev_vlan_filter(PORT_ID(sdev
), vlan_id
, on
);
1257 if ((ret
= fs_err(sdev
, ret
))) {
1258 ERROR("Operation rte_eth_dev_vlan_filter failed for sub_device %d"
1259 " with error %d", i
, ret
);
1269 fs_flow_ctrl_get(struct rte_eth_dev
*dev
,
1270 struct rte_eth_fc_conf
*fc_conf
)
1272 struct sub_device
*sdev
;
1276 sdev
= TX_SUBDEV(dev
);
1281 if (SUBOPS(sdev
, flow_ctrl_get
) == NULL
) {
1285 ret
= SUBOPS(sdev
, flow_ctrl_get
)(ETH(sdev
), fc_conf
);
1292 fs_flow_ctrl_set(struct rte_eth_dev
*dev
,
1293 struct rte_eth_fc_conf
*fc_conf
)
1295 struct sub_device
*sdev
;
1300 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1301 DEBUG("Calling rte_eth_dev_flow_ctrl_set on sub_device %d", i
);
1302 ret
= rte_eth_dev_flow_ctrl_set(PORT_ID(sdev
), fc_conf
);
1303 if ((ret
= fs_err(sdev
, ret
))) {
1304 ERROR("Operation rte_eth_dev_flow_ctrl_set failed for sub_device %d"
1305 " with error %d", i
, ret
);
1315 fs_mac_addr_remove(struct rte_eth_dev
*dev
, uint32_t index
)
1317 struct sub_device
*sdev
;
1321 /* No check: already done within the rte_eth_dev_mac_addr_remove
1322 * call for the fail-safe device.
1324 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
)
1325 rte_eth_dev_mac_addr_remove(PORT_ID(sdev
),
1326 &dev
->data
->mac_addrs
[index
]);
1327 PRIV(dev
)->mac_addr_pool
[index
] = 0;
1332 fs_mac_addr_add(struct rte_eth_dev
*dev
,
1333 struct rte_ether_addr
*mac_addr
,
1337 struct sub_device
*sdev
;
1341 RTE_ASSERT(index
< FAILSAFE_MAX_ETHADDR
);
1343 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1344 ret
= rte_eth_dev_mac_addr_add(PORT_ID(sdev
), mac_addr
, vmdq
);
1345 if ((ret
= fs_err(sdev
, ret
))) {
1346 ERROR("Operation rte_eth_dev_mac_addr_add failed for sub_device %"
1347 PRIu8
" with error %d", i
, ret
);
1352 if (index
>= PRIV(dev
)->nb_mac_addr
) {
1353 DEBUG("Growing mac_addrs array");
1354 PRIV(dev
)->nb_mac_addr
= index
;
1356 PRIV(dev
)->mac_addr_pool
[index
] = vmdq
;
1362 fs_mac_addr_set(struct rte_eth_dev
*dev
, struct rte_ether_addr
*mac_addr
)
1364 struct sub_device
*sdev
;
1369 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1370 ret
= rte_eth_dev_default_mac_addr_set(PORT_ID(sdev
), mac_addr
);
1371 ret
= fs_err(sdev
, ret
);
1373 ERROR("Operation rte_eth_dev_mac_addr_set failed for sub_device %d with error %d",
1385 fs_set_mc_addr_list(struct rte_eth_dev
*dev
,
1386 struct rte_ether_addr
*mc_addr_set
, uint32_t nb_mc_addr
)
1388 struct sub_device
*sdev
;
1395 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1396 ret
= rte_eth_dev_set_mc_addr_list(PORT_ID(sdev
),
1397 mc_addr_set
, nb_mc_addr
);
1399 ERROR("Operation rte_eth_dev_set_mc_addr_list failed for sub_device %d with error %d",
1405 mcast_addrs
= rte_realloc(PRIV(dev
)->mcast_addrs
,
1406 nb_mc_addr
* sizeof(PRIV(dev
)->mcast_addrs
[0]), 0);
1407 if (mcast_addrs
== NULL
&& nb_mc_addr
> 0) {
1411 rte_memcpy(mcast_addrs
, mc_addr_set
,
1412 nb_mc_addr
* sizeof(PRIV(dev
)->mcast_addrs
[0]));
1413 PRIV(dev
)->nb_mcast_addr
= nb_mc_addr
;
1414 PRIV(dev
)->mcast_addrs
= mcast_addrs
;
1420 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1421 int rc
= rte_eth_dev_set_mc_addr_list(PORT_ID(sdev
),
1422 PRIV(dev
)->mcast_addrs
, PRIV(dev
)->nb_mcast_addr
);
1424 ERROR("Multicast MAC address list rollback for sub_device %d failed with error %d",
1434 fs_rss_hash_update(struct rte_eth_dev
*dev
,
1435 struct rte_eth_rss_conf
*rss_conf
)
1437 struct sub_device
*sdev
;
1442 FOREACH_SUBDEV_STATE(sdev
, i
, dev
, DEV_ACTIVE
) {
1443 ret
= rte_eth_dev_rss_hash_update(PORT_ID(sdev
), rss_conf
);
1444 ret
= fs_err(sdev
, ret
);
1446 ERROR("Operation rte_eth_dev_rss_hash_update"
1447 " failed for sub_device %d with error %d",
1459 fs_filter_ctrl(struct rte_eth_dev
*dev __rte_unused
,
1460 enum rte_filter_type type
,
1461 enum rte_filter_op op
,
1464 if (type
== RTE_ETH_FILTER_GENERIC
&&
1465 op
== RTE_ETH_FILTER_GET
) {
1466 *(const void **)arg
= &fs_flow_ops
;
1472 const struct eth_dev_ops failsafe_ops
= {
1473 .dev_configure
= fs_dev_configure
,
1474 .dev_start
= fs_dev_start
,
1475 .dev_stop
= fs_dev_stop
,
1476 .dev_set_link_down
= fs_dev_set_link_down
,
1477 .dev_set_link_up
= fs_dev_set_link_up
,
1478 .dev_close
= fs_dev_close
,
1479 .promiscuous_enable
= fs_promiscuous_enable
,
1480 .promiscuous_disable
= fs_promiscuous_disable
,
1481 .allmulticast_enable
= fs_allmulticast_enable
,
1482 .allmulticast_disable
= fs_allmulticast_disable
,
1483 .link_update
= fs_link_update
,
1484 .stats_get
= fs_stats_get
,
1485 .stats_reset
= fs_stats_reset
,
1486 .xstats_get
= fs_xstats_get
,
1487 .xstats_get_names
= fs_xstats_get_names
,
1488 .xstats_reset
= fs_xstats_reset
,
1489 .dev_infos_get
= fs_dev_infos_get
,
1490 .dev_supported_ptypes_get
= fs_dev_supported_ptypes_get
,
1491 .mtu_set
= fs_mtu_set
,
1492 .vlan_filter_set
= fs_vlan_filter_set
,
1493 .rx_queue_start
= fs_rx_queue_start
,
1494 .rx_queue_stop
= fs_rx_queue_stop
,
1495 .tx_queue_start
= fs_tx_queue_start
,
1496 .tx_queue_stop
= fs_tx_queue_stop
,
1497 .rx_queue_setup
= fs_rx_queue_setup
,
1498 .tx_queue_setup
= fs_tx_queue_setup
,
1499 .rx_queue_release
= fs_rx_queue_release
,
1500 .tx_queue_release
= fs_tx_queue_release
,
1501 .rx_queue_intr_enable
= fs_rx_intr_enable
,
1502 .rx_queue_intr_disable
= fs_rx_intr_disable
,
1503 .flow_ctrl_get
= fs_flow_ctrl_get
,
1504 .flow_ctrl_set
= fs_flow_ctrl_set
,
1505 .mac_addr_remove
= fs_mac_addr_remove
,
1506 .mac_addr_add
= fs_mac_addr_add
,
1507 .mac_addr_set
= fs_mac_addr_set
,
1508 .set_mc_addr_list
= fs_set_mc_addr_list
,
1509 .rss_hash_update
= fs_rss_hash_update
,
1510 .filter_ctrl
= fs_filter_ctrl
,