2 * Copyright (c) 2008, 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "dynamic-string.h"
34 #include "poll-loop.h"
40 VLOG_DEFINE_THIS_MODULE(bond
);
42 /* Bit-mask for hashing a flow down to a bucket.
43 * There are (BOND_MASK + 1) buckets. */
44 #define BOND_MASK 0xff
46 /* A hash bucket for mapping a flow to a slave.
47 * "struct bond" has an array of (BOND_MASK + 1) of these. */
49 struct bond_slave
*slave
; /* Assigned slave, NULL if unassigned. */
50 uint64_t tx_bytes
; /* Count of bytes recently transmitted. */
51 tag_type tag
; /* Tag for entry<->slave association. */
52 struct list list_node
; /* In bond_slave's 'entries' list. */
55 /* A bond slave, that is, one of the links comprising a bond. */
57 struct hmap_node hmap_node
; /* In struct bond's slaves hmap. */
58 struct bond
*bond
; /* The bond that contains this slave. */
59 void *aux
; /* Client-provided handle for this slave. */
61 struct netdev
*netdev
; /* Network device, owned by the client. */
62 char *name
; /* Name (a copy of netdev_get_name(netdev)). */
65 long long delay_expires
; /* Time after which 'enabled' may change. */
66 bool up
; /* Last link status read from netdev. */
67 bool enabled
; /* May be chosen for flows? */
68 bool lacp_may_enable
; /* LACP considers this interface bondable. */
69 tag_type tag
; /* Tag associated with this slave. */
71 /* Rebalancing info. Used only by bond_rebalance(). */
72 struct list bal_node
; /* In bond_rebalance()'s 'bals' list. */
73 struct list entries
; /* 'struct bond_entry's assigned here. */
74 uint64_t tx_bytes
; /* Sum across 'tx_bytes' of entries. */
76 /* BM_STABLE specific bonding info. */
77 uint16_t stb_id
; /* ID used for 'stb_slaves' ordering. */
78 size_t stb_idx
; /* Index in 'bond''s 'stb_slaves' array.
79 Undefined value if participating in a
80 BTM_STABLE bond or not enabled. */
83 /* A bond, that is, a set of network devices grouped to improve performance or
86 struct hmap_node hmap_node
; /* In 'all_bonds' hmap. */
87 char *name
; /* Name provided by client. */
93 enum bond_mode balance
; /* Balancing mode, one of BM_*. */
94 struct bond_slave
*active_slave
;
95 tag_type no_slaves_tag
; /* Tag for flows when all slaves disabled. */
96 int updelay
, downdelay
; /* Delay before slave goes up/down, in ms. */
97 bool lacp_negotiated
; /* LACP negotiations were successful. */
99 /* SLB specific bonding info. */
100 struct bond_entry
*hash
; /* An array of (BOND_MASK + 1) elements. */
101 int rebalance_interval
; /* Interval between rebalances, in ms. */
102 long long int next_rebalance
; /* Next rebalancing time. */
103 bool send_learning_packets
;
105 /* BM_STABLE specific bonding info. */
106 struct bond_slave
**stb_slaves
; /* Ordered list of enabled slaves. */
107 size_t n_stb_slaves
; /* Number of slaves in 'stb_slaves'. */
108 size_t len_stb_slaves
; /* Slaves allocated in 'stb_slaves'. */
109 bool stb_need_sort
; /* True if stb_slaves is not sorted. */
113 enum bond_detect_mode detect
; /* Link status mode, one of BLSM_*. */
114 struct netdev_monitor
*monitor
; /* detect == BLSM_CARRIER only. */
115 long long int miimon_interval
; /* Miimon status refresh interval. */
116 long long int miimon_next_update
; /* Time of next miimon update. */
118 /* Legacy compatibility. */
119 long long int next_fake_iface_update
; /* LLONG_MAX if disabled. */
121 /* Tag set saved for next bond_run(). This tag set is a kluge for cases
122 * where we can't otherwise provide revalidation feedback to the client.
123 * That's only unixctl commands now; I hope no other cases will arise. */
124 struct tag_set unixctl_tags
;
127 static struct hmap all_bonds
= HMAP_INITIALIZER(&all_bonds
);
129 static void bond_entry_reset(struct bond
*);
130 static struct bond_slave
*bond_slave_lookup(struct bond
*, const void *slave_
);
131 static bool bond_is_link_up(struct bond
*, struct netdev
*);
132 static void bond_enable_slave(struct bond_slave
*, bool enable
,
134 static bool bond_stb_sort(struct bond
*);
135 static void bond_stb_enable_slave(struct bond_slave
*);
136 static void bond_link_status_update(struct bond_slave
*, struct tag_set
*);
137 static void bond_choose_active_slave(struct bond
*, struct tag_set
*);
138 static bool bond_is_tcp_hash(const struct bond
*);
139 static unsigned int bond_hash_src(const uint8_t mac
[ETH_ADDR_LEN
],
141 static unsigned int bond_hash_tcp(const struct flow
*, uint16_t vlan
);
142 static struct bond_entry
*lookup_bond_entry(const struct bond
*,
145 static tag_type
bond_get_active_slave_tag(const struct bond
*);
146 static struct bond_slave
*choose_output_slave(const struct bond
*,
149 static void bond_update_fake_slave_stats(struct bond
*);
151 /* Attempts to parse 's' as the name of a bond balancing mode. If successful,
152 * stores the mode in '*balance' and returns true. Otherwise returns false
153 * without modifying '*balance'. */
155 bond_mode_from_string(enum bond_mode
*balance
, const char *s
)
157 if (!strcmp(s
, bond_mode_to_string(BM_TCP
))) {
159 } else if (!strcmp(s
, bond_mode_to_string(BM_SLB
))) {
161 } else if (!strcmp(s
, bond_mode_to_string(BM_STABLE
))) {
162 *balance
= BM_STABLE
;
163 } else if (!strcmp(s
, bond_mode_to_string(BM_AB
))) {
171 /* Returns a string representing 'balance'. */
173 bond_mode_to_string(enum bond_mode balance
) {
176 return "balance-tcp";
178 return "balance-slb";
182 return "active-backup";
187 /* Attempts to parse 's' as the name of a bond link status detection mode. If
188 * successful, stores the mode in '*detect' and returns true. Otherwise
189 * returns false without modifying '*detect'. */
191 bond_detect_mode_from_string(enum bond_detect_mode
*detect
, const char *s
)
193 if (!strcmp(s
, bond_detect_mode_to_string(BLSM_CARRIER
))) {
194 *detect
= BLSM_CARRIER
;
195 } else if (!strcmp(s
, bond_detect_mode_to_string(BLSM_MIIMON
))) {
196 *detect
= BLSM_MIIMON
;
203 /* Returns a string representing 'detect'. */
205 bond_detect_mode_to_string(enum bond_detect_mode detect
)
216 /* Creates and returns a new bond whose configuration is initially taken from
219 * The caller should register each slave on the new bond by calling
220 * bond_slave_register(). */
222 bond_create(const struct bond_settings
*s
)
226 bond
= xzalloc(sizeof *bond
);
227 hmap_init(&bond
->slaves
);
228 bond
->no_slaves_tag
= tag_create_random();
229 bond
->miimon_next_update
= LLONG_MAX
;
230 bond
->next_fake_iface_update
= LLONG_MAX
;
232 bond_reconfigure(bond
, s
);
234 tag_set_init(&bond
->unixctl_tags
);
241 bond_destroy(struct bond
*bond
)
243 struct bond_slave
*slave
, *next_slave
;
249 hmap_remove(&all_bonds
, &bond
->hmap_node
);
251 HMAP_FOR_EACH_SAFE (slave
, next_slave
, hmap_node
, &bond
->slaves
) {
252 hmap_remove(&bond
->slaves
, &slave
->hmap_node
);
253 /* Client owns 'slave->netdev'. */
257 hmap_destroy(&bond
->slaves
);
261 netdev_monitor_destroy(bond
->monitor
);
267 /* Updates 'bond''s overall configuration to 's'.
269 * The caller should register each slave on 'bond' by calling
270 * bond_slave_register(). This is optional if none of the slaves'
271 * configuration has changed. In any case it can't hurt.
273 * Returns true if the configuration has changed in such a way that requires
277 bond_reconfigure(struct bond
*bond
, const struct bond_settings
*s
)
279 bool revalidate
= false;
281 if (!bond
->name
|| strcmp(bond
->name
, s
->name
)) {
283 hmap_remove(&all_bonds
, &bond
->hmap_node
);
286 bond
->name
= xstrdup(s
->name
);
287 hmap_insert(&all_bonds
, &bond
->hmap_node
, hash_string(bond
->name
, 0));
290 bond
->detect
= s
->detect
;
291 bond
->miimon_interval
= s
->miimon_interval
;
292 bond
->updelay
= s
->up_delay
;
293 bond
->downdelay
= s
->down_delay
;
294 bond
->rebalance_interval
= s
->rebalance_interval
;
296 if (bond
->balance
!= s
->balance
) {
297 bond
->balance
= s
->balance
;
301 if (bond
->detect
== BLSM_CARRIER
) {
302 struct bond_slave
*slave
;
304 if (!bond
->monitor
) {
305 bond
->monitor
= netdev_monitor_create();
308 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
309 netdev_monitor_add(bond
->monitor
, slave
->netdev
);
312 netdev_monitor_destroy(bond
->monitor
);
313 bond
->monitor
= NULL
;
315 if (bond
->miimon_next_update
== LLONG_MAX
) {
316 bond
->miimon_next_update
= time_msec() + bond
->miimon_interval
;
321 if (bond
->next_fake_iface_update
== LLONG_MAX
) {
322 bond
->next_fake_iface_update
= time_msec();
325 bond
->next_fake_iface_update
= LLONG_MAX
;
328 if (bond
->balance
!= BM_STABLE
) {
329 free(bond
->stb_slaves
);
330 bond
->stb_slaves
= NULL
;
331 } else if (!bond
->stb_slaves
) {
332 struct bond_slave
*slave
;
334 bond
->n_stb_slaves
= 0;
335 bond
->len_stb_slaves
= 0;
336 bond
->stb_slaves
= NULL
;
338 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
339 if (slave
->enabled
) {
340 bond_stb_enable_slave(slave
);
345 if (bond
->balance
== BM_AB
|| !bond
->hash
|| revalidate
) {
346 bond_entry_reset(bond
);
352 /* Registers 'slave_' as a slave of 'bond'. The 'slave_' pointer is an
353 * arbitrary client-provided pointer that uniquely identifies a slave within a
354 * bond. If 'slave_' already exists within 'bond' then this function
355 * reconfigures the existing slave.
357 * 'stb_id' is used in BM_STABLE bonds to guarantee consistent slave choices
358 * across restarts and distributed vswitch instances. It should be unique per
359 * slave, and preferably consistent across restarts and reconfigurations.
361 * 'netdev' must be the network device that 'slave_' represents. It is owned
362 * by the client, so the client must not close it before either unregistering
363 * 'slave_' or destroying 'bond'.
366 bond_slave_register(struct bond
*bond
, void *slave_
, uint16_t stb_id
,
367 struct netdev
*netdev
)
369 struct bond_slave
*slave
= bond_slave_lookup(bond
, slave_
);
372 slave
= xzalloc(sizeof *slave
);
374 hmap_insert(&bond
->slaves
, &slave
->hmap_node
, hash_pointer(slave_
, 0));
377 slave
->delay_expires
= LLONG_MAX
;
378 slave
->up
= bond_is_link_up(bond
, netdev
);
379 slave
->name
= xstrdup(netdev_get_name(netdev
));
381 slave
->enabled
= false;
382 bond_enable_slave(slave
, slave
->up
, NULL
);
385 if (slave
->stb_id
!= stb_id
) {
386 bond
->stb_need_sort
= true;
387 slave
->stb_id
= stb_id
;
390 slave
->netdev
= netdev
;
392 slave
->name
= xstrdup(netdev_get_name(netdev
));
395 /* Unregisters 'slave_' from 'bond'. If 'bond' does not contain such a slave
396 * then this function has no effect.
398 * Unregistering a slave invalidates all flows. */
400 bond_slave_unregister(struct bond
*bond
, const void *slave_
)
402 struct bond_slave
*slave
= bond_slave_lookup(bond
, slave_
);
409 bond_enable_slave(slave
, false, NULL
);
411 del_active
= bond
->active_slave
== slave
;
413 struct bond_entry
*e
;
414 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
415 if (e
->slave
== slave
) {
423 hmap_remove(&bond
->slaves
, &slave
->hmap_node
);
424 /* Client owns 'slave->netdev'. */
431 bond_choose_active_slave(bond
, &tags
);
432 bond
->send_learning_packets
= true;
436 /* Should be called on each slave in 'bond' before bond_run() to indicate the
437 * results of lacp_slave_may_enable() on 'slave_'. */
439 bond_slave_set_lacp_may_enable(struct bond
*bond
, void *slave_
,
442 bond_slave_lookup(bond
, slave_
)->lacp_may_enable
= may_enable
;
445 /* Performs periodic maintenance on 'bond'. The caller must provide 'tags' to
446 * allow tagged flows to be invalidated.
448 * The caller should check bond_should_send_learning_packets() afterward. */
450 bond_run(struct bond
*bond
, struct tag_set
*tags
, bool lacp_negotiated
)
452 struct bond_slave
*slave
;
453 bool is_tcp_hash
= bond_is_tcp_hash(bond
);
455 bond
->lacp_negotiated
= lacp_negotiated
;
457 /* Update link status. */
458 if (bond
->detect
== BLSM_CARRIER
459 || time_msec() >= bond
->miimon_next_update
)
461 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
462 slave
->up
= bond_is_link_up(bond
, slave
->netdev
);
464 bond
->miimon_next_update
= time_msec() + bond
->miimon_interval
;
467 /* Enable slaves based on link status and LACP feedback. */
468 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
469 bond_link_status_update(slave
, tags
);
471 if (!bond
->active_slave
|| !bond
->active_slave
->enabled
) {
472 bond_choose_active_slave(bond
, tags
);
475 /* Update fake bond interface stats. */
476 if (time_msec() >= bond
->next_fake_iface_update
) {
477 bond_update_fake_slave_stats(bond
);
478 bond
->next_fake_iface_update
= time_msec() + 1000;
481 if (bond_stb_sort(bond
) || is_tcp_hash
!= bond_is_tcp_hash(bond
)) {
482 struct bond_slave
*slave
;
484 bond_entry_reset(bond
);
485 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
486 tag_set_add(tags
, slave
->tag
);
490 /* Invalidate any tags required by */
491 tag_set_union(tags
, &bond
->unixctl_tags
);
492 tag_set_init(&bond
->unixctl_tags
);
495 /* Causes poll_block() to wake up when 'bond' needs something to be done. */
497 bond_wait(struct bond
*bond
)
499 struct bond_slave
*slave
;
501 if (bond
->detect
== BLSM_CARRIER
) {
502 netdev_monitor_poll_wait(bond
->monitor
);
504 poll_timer_wait_until(bond
->miimon_next_update
);
507 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
508 if (slave
->delay_expires
!= LLONG_MAX
) {
509 poll_timer_wait_until(slave
->delay_expires
);
513 if (bond
->next_fake_iface_update
!= LLONG_MAX
) {
514 poll_timer_wait_until(bond
->next_fake_iface_update
);
517 /* Ensure that any saved tags get revalidated right away. */
518 if (!tag_set_is_empty(&bond
->unixctl_tags
)) {
519 poll_immediate_wake();
522 /* We don't wait for bond->next_rebalance because rebalancing can only run
523 * at a flow account checkpoint. ofproto does checkpointing on its own
524 * schedule and bond_rebalance() gets called afterward, so we'd just be
525 * waking up for no purpose. */
528 /* MAC learning table interaction. */
531 may_send_learning_packets(const struct bond
*bond
)
533 return !bond
->lacp_negotiated
&& bond
->balance
!= BM_AB
;
536 /* Returns true if 'bond' needs the client to send out packets to assist with
537 * MAC learning on 'bond'. If this function returns true, then the client
538 * should iterate through its MAC learning table for the bridge on which 'bond'
539 * is located. For each MAC that has been learned on a port other than 'bond',
540 * it should call bond_send_learning_packet().
542 * This function will only return true if 'bond' is in SLB mode and LACP is not
543 * negotiated. Otherwise sending learning packets isn't necessary.
545 * Calling this function resets the state that it checks. */
547 bond_should_send_learning_packets(struct bond
*bond
)
549 bool send
= bond
->send_learning_packets
&& may_send_learning_packets(bond
);
550 bond
->send_learning_packets
= false;
554 /* Sends a gratuitous learning packet on 'bond' from 'eth_src' on 'vlan'.
556 * See bond_should_send_learning_packets() for description of usage. */
558 bond_send_learning_packet(struct bond
*bond
,
559 const uint8_t eth_src
[ETH_ADDR_LEN
],
562 struct bond_slave
*slave
;
563 struct ofpbuf packet
;
567 assert(may_send_learning_packets(bond
));
568 if (!bond
->active_slave
) {
569 /* Nowhere to send the learning packet. */
573 memset(&flow
, 0, sizeof flow
);
574 memcpy(flow
.dl_src
, eth_src
, ETH_ADDR_LEN
);
575 slave
= choose_output_slave(bond
, &flow
, vlan
);
577 ofpbuf_init(&packet
, 0);
578 compose_benign_packet(&packet
, "Open vSwitch Bond Failover", 0xf177,
581 eth_set_vlan_tci(&packet
, htons(vlan
));
583 error
= netdev_send(slave
->netdev
, &packet
);
584 ofpbuf_uninit(&packet
);
589 /* Checks whether a packet that arrived on 'slave_' within 'bond', with an
590 * Ethernet destination address of 'eth_dst', should be admitted.
592 * The return value is one of the following:
594 * - BV_ACCEPT: Admit the packet.
596 * - BV_DROP: Drop the packet.
598 * - BV_DROP_IF_MOVED: Consult the MAC learning table for the packet's
599 * Ethernet source address and VLAN. If there is none, or if the packet
600 * is on the learned port, then admit the packet. If a different port has
601 * been learned, however, drop the packet (and do not use it for MAC
605 bond_check_admissibility(struct bond
*bond
, const void *slave_
,
606 const uint8_t eth_dst
[ETH_ADDR_LEN
], tag_type
*tags
)
608 /* Admit all packets if LACP has been negotiated, because that means that
609 * the remote switch is aware of the bond and will "do the right thing". */
610 if (bond
->lacp_negotiated
) {
614 /* Drop all multicast packets on inactive slaves. */
615 if (eth_addr_is_multicast(eth_dst
)) {
616 *tags
|= bond_get_active_slave_tag(bond
);
617 if (bond
->active_slave
!= bond_slave_lookup(bond
, slave_
)) {
622 /* Drop all packets for which we have learned a different input port,
623 * because we probably sent the packet on one slave and got it back on the
624 * other. Gratuitous ARP packets are an exception to this rule: the host
625 * has moved to another switch. The exception to the exception is if we
626 * locked the learning table to avoid reflections on bond slaves. */
627 return BV_DROP_IF_MOVED
;
630 /* Returns the slave (registered on 'bond' by bond_slave_register()) to which
631 * a packet with the given 'flow' and 'vlan' should be forwarded. Returns
632 * NULL if the packet should be dropped because no slaves are enabled.
634 * 'vlan' is not necessarily the same as 'flow->vlan_tci'. First, 'vlan'
635 * should be a VID only (i.e. excluding the PCP bits). Second,
636 * 'flow->vlan_tci' is the VLAN TCI that appeared on the packet (so it will be
637 * nonzero only for trunk ports), whereas 'vlan' is the logical VLAN that the
638 * packet belongs to (so for an access port it will be the access port's VLAN).
640 * Adds a tag to '*tags' that associates the flow with the returned slave.
643 bond_choose_output_slave(struct bond
*bond
, const struct flow
*flow
,
644 uint16_t vlan
, tag_type
*tags
)
646 struct bond_slave
*slave
= choose_output_slave(bond
, flow
, vlan
);
651 *tags
|= bond
->no_slaves_tag
;
659 bond_is_balanced(const struct bond
*bond
)
661 return bond
->balance
== BM_SLB
|| bond
->balance
== BM_TCP
;
664 /* Notifies 'bond' that 'n_bytes' bytes were sent in 'flow' within 'vlan'. */
666 bond_account(struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
,
670 if (bond_is_balanced(bond
)) {
671 lookup_bond_entry(bond
, flow
, vlan
)->tx_bytes
+= n_bytes
;
675 static struct bond_slave
*
676 bond_slave_from_bal_node(struct list
*bal
)
678 return CONTAINER_OF(bal
, struct bond_slave
, bal_node
);
682 log_bals(struct bond
*bond
, const struct list
*bals
)
684 if (VLOG_IS_DBG_ENABLED()) {
685 struct ds ds
= DS_EMPTY_INITIALIZER
;
686 const struct bond_slave
*slave
;
688 LIST_FOR_EACH (slave
, bal_node
, bals
) {
690 ds_put_char(&ds
, ',');
692 ds_put_format(&ds
, " %s %"PRIu64
"kB",
693 slave
->name
, slave
->tx_bytes
/ 1024);
695 if (!slave
->enabled
) {
696 ds_put_cstr(&ds
, " (disabled)");
698 if (!list_is_empty(&slave
->entries
)) {
699 struct bond_entry
*e
;
701 ds_put_cstr(&ds
, " (");
702 LIST_FOR_EACH (e
, list_node
, &slave
->entries
) {
703 if (&e
->list_node
!= list_front(&slave
->entries
)) {
704 ds_put_cstr(&ds
, " + ");
706 ds_put_format(&ds
, "h%td: %"PRIu64
"kB",
707 e
- bond
->hash
, e
->tx_bytes
/ 1024);
709 ds_put_cstr(&ds
, ")");
712 VLOG_DBG("bond %s:%s", bond
->name
, ds_cstr(&ds
));
717 /* Shifts 'hash' from its current slave to 'to'. */
719 bond_shift_load(struct bond_entry
*hash
, struct bond_slave
*to
,
722 struct bond_slave
*from
= hash
->slave
;
723 struct bond
*bond
= from
->bond
;
724 uint64_t delta
= hash
->tx_bytes
;
726 VLOG_INFO("bond %s: shift %"PRIu64
"kB of load (with hash %td) "
727 "from %s to %s (now carrying %"PRIu64
"kB and "
728 "%"PRIu64
"kB load, respectively)",
729 bond
->name
, delta
/ 1024, hash
- bond
->hash
,
730 from
->name
, to
->name
,
731 (from
->tx_bytes
- delta
) / 1024,
732 (to
->tx_bytes
+ delta
) / 1024);
734 /* Shift load away from 'from' to 'to'. */
735 from
->tx_bytes
-= delta
;
736 to
->tx_bytes
+= delta
;
738 /* Arrange for flows to be revalidated. */
739 tag_set_add(set
, hash
->tag
);
741 hash
->tag
= tag_create_random();
744 /* Pick and returns a bond_entry to migrate to 'to' (the least-loaded slave),
745 * given that doing so must decrease the ratio of the load on the two slaves by
746 * at least 0.1. Returns NULL if there is no appropriate entry.
748 * The list of entries isn't sorted. I don't know of a reason to prefer to
749 * shift away small hashes or large hashes. */
750 static struct bond_entry
*
751 choose_entry_to_migrate(const struct bond_slave
*from
, uint64_t to_tx_bytes
)
753 struct bond_entry
*e
;
755 if (list_is_short(&from
->entries
)) {
756 /* 'from' carries no more than one MAC hash, so shifting load away from
757 * it would be pointless. */
761 LIST_FOR_EACH (e
, list_node
, &from
->entries
) {
762 double old_ratio
, new_ratio
;
765 if (to_tx_bytes
== 0) {
766 /* Nothing on the new slave, move it. */
771 old_ratio
= (double)from
->tx_bytes
/ to_tx_bytes
;
772 new_ratio
= (double)(from
->tx_bytes
- delta
) / (to_tx_bytes
+ delta
);
773 if (old_ratio
- new_ratio
> 0.1) {
774 /* Would decrease the ratio, move it. */
782 /* Inserts 'slave' into 'bals' so that descending order of 'tx_bytes' is
785 insert_bal(struct list
*bals
, struct bond_slave
*slave
)
787 struct bond_slave
*pos
;
789 LIST_FOR_EACH (pos
, bal_node
, bals
) {
790 if (slave
->tx_bytes
> pos
->tx_bytes
) {
794 list_insert(&pos
->bal_node
, &slave
->bal_node
);
797 /* Removes 'slave' from its current list and then inserts it into 'bals' so
798 * that descending order of 'tx_bytes' is maintained. */
800 reinsert_bal(struct list
*bals
, struct bond_slave
*slave
)
802 list_remove(&slave
->bal_node
);
803 insert_bal(bals
, slave
);
806 /* If 'bond' needs rebalancing, does so.
808 * The caller should have called bond_account() for each active flow, to ensure
809 * that flow data is consistently accounted at this point. */
811 bond_rebalance(struct bond
*bond
, struct tag_set
*tags
)
813 struct bond_slave
*slave
;
814 struct bond_entry
*e
;
817 if (!bond_is_balanced(bond
) || time_msec() < bond
->next_rebalance
) {
820 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
822 /* Add each bond_entry to its slave's 'entries' list.
823 * Compute each slave's tx_bytes as the sum of its entries' tx_bytes. */
824 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
826 list_init(&slave
->entries
);
828 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
829 if (e
->slave
&& e
->tx_bytes
) {
830 e
->slave
->tx_bytes
+= e
->tx_bytes
;
831 list_push_back(&e
->slave
->entries
, &e
->list_node
);
835 /* Add enabled slaves to 'bals' in descending order of tx_bytes.
837 * XXX This is O(n**2) in the number of slaves but it could be O(n lg n)
838 * with a proper list sort algorithm. */
840 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
841 if (slave
->enabled
) {
842 insert_bal(&bals
, slave
);
845 log_bals(bond
, &bals
);
847 /* Shift load from the most-loaded slaves to the least-loaded slaves. */
848 while (!list_is_short(&bals
)) {
849 struct bond_slave
*from
= bond_slave_from_bal_node(list_front(&bals
));
850 struct bond_slave
*to
= bond_slave_from_bal_node(list_back(&bals
));
853 overload
= from
->tx_bytes
- to
->tx_bytes
;
854 if (overload
< to
->tx_bytes
>> 5 || overload
< 100000) {
855 /* The extra load on 'from' (and all less-loaded slaves), compared
856 * to that of 'to' (the least-loaded slave), is less than ~3%, or
857 * it is less than ~1Mbps. No point in rebalancing. */
861 /* 'from' is carrying significantly more load than 'to', and that load
862 * is split across at least two different hashes. */
863 e
= choose_entry_to_migrate(from
, to
->tx_bytes
);
865 bond_shift_load(e
, to
, tags
);
867 /* Delete element from from->entries.
869 * We don't add the element to to->hashes. That would only allow
870 * 'e' to be migrated to another slave in this rebalancing run, and
871 * there is no point in doing that. */
872 list_remove(&e
->list_node
);
874 /* Re-sort 'bals'. */
875 reinsert_bal(&bals
, from
);
876 reinsert_bal(&bals
, to
);
878 /* Can't usefully migrate anything away from 'from'.
879 * Don't reconsider it. */
880 list_remove(&from
->bal_node
);
884 /* Implement exponentially weighted moving average. A weight of 1/2 causes
885 * historical data to decay to <1% in 7 rebalancing runs. 1,000,000 bytes
886 * take 20 rebalancing runs to decay to 0 and get deleted entirely. */
887 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
895 /* Bonding unixctl user interface functions. */
898 bond_find(const char *name
)
902 HMAP_FOR_EACH_WITH_HASH (bond
, hmap_node
, hash_string(name
, 0),
904 if (!strcmp(bond
->name
, name
)) {
911 static struct bond_slave
*
912 bond_lookup_slave(struct bond
*bond
, const char *slave_name
)
914 struct bond_slave
*slave
;
916 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
917 if (!strcmp(slave
->name
, slave_name
)) {
925 bond_unixctl_list(struct unixctl_conn
*conn
,
926 const char *args OVS_UNUSED
, void *aux OVS_UNUSED
)
928 struct ds ds
= DS_EMPTY_INITIALIZER
;
929 const struct bond
*bond
;
931 ds_put_cstr(&ds
, "bond\ttype\tslaves\n");
933 HMAP_FOR_EACH (bond
, hmap_node
, &all_bonds
) {
934 const struct bond_slave
*slave
;
937 ds_put_format(&ds
, "%s\t%s\t",
938 bond
->name
, bond_mode_to_string(bond
->balance
));
941 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
943 ds_put_cstr(&ds
, ", ");
945 ds_put_cstr(&ds
, slave
->name
);
947 ds_put_char(&ds
, '\n');
949 unixctl_command_reply(conn
, 200, ds_cstr(&ds
));
954 bond_unixctl_show(struct unixctl_conn
*conn
,
955 const char *args
, void *aux OVS_UNUSED
)
957 struct ds ds
= DS_EMPTY_INITIALIZER
;
958 const struct bond_slave
*slave
;
959 const struct bond
*bond
;
961 bond
= bond_find(args
);
963 unixctl_command_reply(conn
, 501, "no such bond");
967 ds_put_format(&ds
, "bond_mode: %s\n",
968 bond_mode_to_string(bond
->balance
));
970 if (bond
->balance
!= BM_AB
) {
971 ds_put_format(&ds
, "bond-hash-algorithm: %s\n",
972 bond_is_tcp_hash(bond
) ? "balance-tcp" : "balance-slb");
975 ds_put_format(&ds
, "bond-detect-mode: %s\n",
976 bond
->monitor
? "carrier" : "miimon");
978 if (!bond
->monitor
) {
979 ds_put_format(&ds
, "bond-miimon-interval: %lld\n",
980 bond
->miimon_interval
);
983 ds_put_format(&ds
, "updelay: %d ms\n", bond
->updelay
);
984 ds_put_format(&ds
, "downdelay: %d ms\n", bond
->downdelay
);
986 if (bond_is_balanced(bond
)) {
987 ds_put_format(&ds
, "next rebalance: %lld ms\n",
988 bond
->next_rebalance
- time_msec());
991 ds_put_format(&ds
, "lacp_negotiated: %s\n",
992 bond
->lacp_negotiated
? "true" : "false");
994 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
995 struct bond_entry
*be
;
999 ds_put_format(&ds
, "\nslave %s: %s\n",
1000 slave
->name
, slave
->enabled
? "enabled" : "disabled");
1001 if (slave
== bond
->active_slave
) {
1002 ds_put_cstr(&ds
, "\tactive slave\n");
1004 if (slave
->delay_expires
!= LLONG_MAX
) {
1005 ds_put_format(&ds
, "\t%s expires in %lld ms\n",
1006 slave
->enabled
? "downdelay" : "updelay",
1007 slave
->delay_expires
- time_msec());
1010 ds_put_format(&ds
, "\tlacp_may_enable: %s\n",
1011 slave
->lacp_may_enable
? "true" : "false");
1013 if (!bond_is_balanced(bond
)) {
1018 memset(&flow
, 0, sizeof flow
);
1019 for (be
= bond
->hash
; be
<= &bond
->hash
[BOND_MASK
]; be
++) {
1020 int hash
= be
- bond
->hash
;
1022 if (be
->slave
!= slave
) {
1026 ds_put_format(&ds
, "\thash %d: %"PRIu64
" kB load\n",
1027 hash
, be
->tx_bytes
/ 1024);
1029 if (bond
->balance
!= BM_SLB
) {
1033 /* XXX How can we list the MACs assigned to hashes? */
1036 unixctl_command_reply(conn
, 200, ds_cstr(&ds
));
1041 bond_unixctl_migrate(struct unixctl_conn
*conn
, const char *args_
,
1042 void *aux OVS_UNUSED
)
1044 char *args
= (char *) args_
;
1045 char *save_ptr
= NULL
;
1046 char *bond_s
, *hash_s
, *slave_s
;
1048 struct bond_slave
*slave
;
1049 struct bond_entry
*entry
;
1052 bond_s
= strtok_r(args
, " ", &save_ptr
);
1053 hash_s
= strtok_r(NULL
, " ", &save_ptr
);
1054 slave_s
= strtok_r(NULL
, " ", &save_ptr
);
1056 unixctl_command_reply(conn
, 501,
1057 "usage: bond/migrate BOND HASH SLAVE");
1061 bond
= bond_find(bond_s
);
1063 unixctl_command_reply(conn
, 501, "no such bond");
1067 if (bond
->balance
!= BM_SLB
) {
1068 unixctl_command_reply(conn
, 501, "not an SLB bond");
1072 if (strspn(hash_s
, "0123456789") == strlen(hash_s
)) {
1073 hash
= atoi(hash_s
) & BOND_MASK
;
1075 unixctl_command_reply(conn
, 501, "bad hash");
1079 slave
= bond_lookup_slave(bond
, slave_s
);
1081 unixctl_command_reply(conn
, 501, "no such slave");
1085 if (!slave
->enabled
) {
1086 unixctl_command_reply(conn
, 501, "cannot migrate to disabled slave");
1090 entry
= &bond
->hash
[hash
];
1091 tag_set_add(&bond
->unixctl_tags
, entry
->tag
);
1092 entry
->slave
= slave
;
1093 entry
->tag
= tag_create_random();
1094 unixctl_command_reply(conn
, 200, "migrated");
1098 bond_unixctl_set_active_slave(struct unixctl_conn
*conn
, const char *args_
,
1099 void *aux OVS_UNUSED
)
1101 char *args
= (char *) args_
;
1102 char *save_ptr
= NULL
;
1103 char *bond_s
, *slave_s
;
1105 struct bond_slave
*slave
;
1107 bond_s
= strtok_r(args
, " ", &save_ptr
);
1108 slave_s
= strtok_r(NULL
, " ", &save_ptr
);
1110 unixctl_command_reply(conn
, 501,
1111 "usage: bond/set-active-slave BOND SLAVE");
1115 bond
= bond_find(bond_s
);
1117 unixctl_command_reply(conn
, 501, "no such bond");
1121 slave
= bond_lookup_slave(bond
, slave_s
);
1123 unixctl_command_reply(conn
, 501, "no such slave");
1127 if (!slave
->enabled
) {
1128 unixctl_command_reply(conn
, 501, "cannot make disabled slave active");
1132 if (bond
->active_slave
!= slave
) {
1133 tag_set_add(&bond
->unixctl_tags
, bond_get_active_slave_tag(bond
));
1134 bond
->active_slave
= slave
;
1135 bond
->active_slave
->tag
= tag_create_random();
1136 VLOG_INFO("bond %s: active interface is now %s",
1137 bond
->name
, slave
->name
);
1138 bond
->send_learning_packets
= true;
1139 unixctl_command_reply(conn
, 200, "done");
1141 unixctl_command_reply(conn
, 200, "no change");
1146 enable_slave(struct unixctl_conn
*conn
, const char *args_
, bool enable
)
1148 char *args
= (char *) args_
;
1149 char *save_ptr
= NULL
;
1150 char *bond_s
, *slave_s
;
1152 struct bond_slave
*slave
;
1154 bond_s
= strtok_r(args
, " ", &save_ptr
);
1155 slave_s
= strtok_r(NULL
, " ", &save_ptr
);
1157 char *usage
= xasprintf("usage: bond/%s-slave BOND SLAVE",
1158 enable
? "enable" : "disable");
1159 unixctl_command_reply(conn
, 501, usage
);
1164 bond
= bond_find(bond_s
);
1166 unixctl_command_reply(conn
, 501, "no such bond");
1170 slave
= bond_lookup_slave(bond
, slave_s
);
1172 unixctl_command_reply(conn
, 501, "no such slave");
1176 bond_enable_slave(slave
, enable
, &bond
->unixctl_tags
);
1177 unixctl_command_reply(conn
, 501, enable
? "enabled" : "disabled");
1181 bond_unixctl_enable_slave(struct unixctl_conn
*conn
, const char *args
,
1182 void *aux OVS_UNUSED
)
1184 enable_slave(conn
, args
, true);
1188 bond_unixctl_disable_slave(struct unixctl_conn
*conn
, const char *args
,
1189 void *aux OVS_UNUSED
)
1191 enable_slave(conn
, args
, false);
1195 bond_unixctl_hash(struct unixctl_conn
*conn
, const char *args_
,
1196 void *aux OVS_UNUSED
)
1198 char *args
= (char *) args_
;
1199 uint8_t mac
[ETH_ADDR_LEN
];
1203 char *mac_s
, *vlan_s
;
1204 char *save_ptr
= NULL
;
1206 mac_s
= strtok_r(args
, " ", &save_ptr
);
1207 vlan_s
= strtok_r(NULL
, " ", &save_ptr
);
1210 if (sscanf(vlan_s
, "%u", &vlan
) != 1) {
1211 unixctl_command_reply(conn
, 501, "invalid vlan");
1215 vlan
= OFP_VLAN_NONE
;
1218 if (sscanf(mac_s
, ETH_ADDR_SCAN_FMT
, ETH_ADDR_SCAN_ARGS(mac
))
1219 == ETH_ADDR_SCAN_COUNT
) {
1220 hash
= bond_hash_src(mac
, vlan
) & BOND_MASK
;
1222 hash_cstr
= xasprintf("%u", hash
);
1223 unixctl_command_reply(conn
, 200, hash_cstr
);
1226 unixctl_command_reply(conn
, 501, "invalid mac");
1233 unixctl_command_register("bond/list", bond_unixctl_list
, NULL
);
1234 unixctl_command_register("bond/show", bond_unixctl_show
, NULL
);
1235 unixctl_command_register("bond/migrate", bond_unixctl_migrate
, NULL
);
1236 unixctl_command_register("bond/set-active-slave",
1237 bond_unixctl_set_active_slave
, NULL
);
1238 unixctl_command_register("bond/enable-slave", bond_unixctl_enable_slave
,
1240 unixctl_command_register("bond/disable-slave", bond_unixctl_disable_slave
,
1242 unixctl_command_register("bond/hash", bond_unixctl_hash
, NULL
);
1246 bond_entry_reset(struct bond
*bond
)
1248 if (bond
->balance
!= BM_AB
) {
1249 size_t hash_len
= (BOND_MASK
+ 1) * sizeof *bond
->hash
;
1252 bond
->hash
= xmalloc(hash_len
);
1254 memset(bond
->hash
, 0, hash_len
);
1256 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1263 static struct bond_slave
*
1264 bond_slave_lookup(struct bond
*bond
, const void *slave_
)
1266 struct bond_slave
*slave
;
1268 HMAP_FOR_EACH_IN_BUCKET (slave
, hmap_node
, hash_pointer(slave_
, 0),
1270 if (slave
->aux
== slave_
) {
1279 bond_is_link_up(struct bond
*bond
, struct netdev
*netdev
)
1281 return (bond
->detect
== BLSM_CARRIER
1282 ? netdev_get_carrier(netdev
)
1283 : netdev_get_miimon(netdev
));
1287 bond_stb_sort_cmp__(const void *a_
, const void *b_
)
1289 const struct bond_slave
*const *ap
= a_
;
1290 const struct bond_slave
*const *bp
= b_
;
1291 const struct bond_slave
*a
= *ap
;
1292 const struct bond_slave
*b
= *bp
;
1293 uint16_t aid
= a
->stb_id
;
1294 uint16_t bid
= b
->stb_id
;
1296 return aid
< bid
? -1 : aid
> bid
;
1300 bond_stb_sort(struct bond
*bond
)
1304 if (!bond
->stb_slaves
|| !bond
->stb_need_sort
) {
1307 bond
->stb_need_sort
= false;
1309 qsort(bond
->stb_slaves
, bond
->n_stb_slaves
, sizeof *bond
->stb_slaves
,
1310 bond_stb_sort_cmp__
);
1312 for (i
= 0; i
< bond
->n_stb_slaves
; i
++) {
1313 bond
->stb_slaves
[i
]->stb_idx
= i
;
1320 bond_stb_enable_slave(struct bond_slave
*slave
)
1322 struct bond
*bond
= slave
->bond
;
1324 if (bond
->balance
!= BM_STABLE
) {
1328 bond
->stb_need_sort
= true;
1330 if (slave
->enabled
) {
1331 if (bond
->len_stb_slaves
<= bond
->n_stb_slaves
) {
1332 bond
->stb_slaves
= x2nrealloc(bond
->stb_slaves
,
1333 &bond
->len_stb_slaves
,
1334 sizeof *bond
->stb_slaves
);
1337 slave
->stb_idx
= bond
->n_stb_slaves
++;
1338 bond
->stb_slaves
[slave
->stb_idx
] = slave
;
1340 size_t index
= slave
->stb_idx
;
1341 bond
->stb_slaves
[index
] = bond
->stb_slaves
[--bond
->n_stb_slaves
];
1342 bond
->stb_slaves
[index
]->stb_idx
= index
;
1347 bond_enable_slave(struct bond_slave
*slave
, bool enable
, struct tag_set
*tags
)
1349 slave
->delay_expires
= LLONG_MAX
;
1350 if (enable
!= slave
->enabled
) {
1351 slave
->enabled
= enable
;
1352 if (!slave
->enabled
) {
1353 VLOG_WARN("interface %s: disabled", slave
->name
);
1355 tag_set_add(tags
, slave
->tag
);
1358 VLOG_WARN("interface %s: enabled", slave
->name
);
1359 slave
->tag
= tag_create_random();
1361 bond_stb_enable_slave(slave
);
1366 bond_link_status_update(struct bond_slave
*slave
, struct tag_set
*tags
)
1368 struct bond
*bond
= slave
->bond
;
1371 up
= slave
->up
&& slave
->lacp_may_enable
;
1372 if ((up
== slave
->enabled
) != (slave
->delay_expires
== LLONG_MAX
)) {
1373 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1374 VLOG_INFO_RL(&rl
, "interface %s: link state %s",
1375 slave
->name
, up
? "up" : "down");
1376 if (up
== slave
->enabled
) {
1377 slave
->delay_expires
= LLONG_MAX
;
1378 VLOG_INFO_RL(&rl
, "interface %s: will not be %s",
1379 slave
->name
, up
? "disabled" : "enabled");
1381 int delay
= (bond
->lacp_negotiated
? 0
1382 : up
? bond
->updelay
: bond
->downdelay
);
1383 slave
->delay_expires
= time_msec() + delay
;
1385 VLOG_INFO_RL(&rl
, "interface %s: will be %s if it stays %s "
1388 up
? "enabled" : "disabled",
1395 if (time_msec() >= slave
->delay_expires
) {
1396 bond_enable_slave(slave
, up
, tags
);
1401 bond_is_tcp_hash(const struct bond
*bond
)
1403 return (bond
->balance
== BM_TCP
|| bond
->balance
== BM_STABLE
)
1404 && bond
->lacp_negotiated
;
1408 bond_hash_src(const uint8_t mac
[ETH_ADDR_LEN
], uint16_t vlan
)
1410 return hash_bytes(mac
, ETH_ADDR_LEN
, vlan
);
1414 bond_hash_tcp(const struct flow
*flow
, uint16_t vlan
)
1416 struct flow hash_flow
= *flow
;
1417 hash_flow
.vlan_tci
= vlan
;
1419 /* The symmetric quality of this hash function is not required, but
1420 * flow_hash_symmetric_l4 already exists, and is sufficient for our
1421 * purposes, so we use it out of convenience. */
1422 return flow_hash_symmetric_l4(&hash_flow
, 0);
1426 bond_hash(const struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
)
1428 assert(bond
->balance
!= BM_AB
);
1430 return (bond_is_tcp_hash(bond
)
1431 ? bond_hash_tcp(flow
, vlan
)
1432 : bond_hash_src(flow
->dl_src
, vlan
));
1435 static struct bond_entry
*
1436 lookup_bond_entry(const struct bond
*bond
, const struct flow
*flow
,
1439 return &bond
->hash
[bond_hash(bond
, flow
, vlan
) & BOND_MASK
];
1442 static struct bond_slave
*
1443 choose_output_slave(const struct bond
*bond
, const struct flow
*flow
,
1446 struct bond_entry
*e
;
1448 switch (bond
->balance
) {
1450 return bond
->active_slave
;
1453 if (bond
->n_stb_slaves
) {
1454 return bond
->stb_slaves
[bond_hash(bond
, flow
, vlan
)
1455 % bond
->n_stb_slaves
];
1457 return bond
->active_slave
;
1462 e
= lookup_bond_entry(bond
, flow
, vlan
);
1463 if (!e
->slave
|| !e
->slave
->enabled
) {
1464 e
->slave
= CONTAINER_OF(hmap_random_node(&bond
->slaves
),
1465 struct bond_slave
, hmap_node
);
1466 if (!e
->slave
->enabled
) {
1467 e
->slave
= bond
->active_slave
;
1469 e
->tag
= tag_create_random();
1478 static struct bond_slave
*
1479 bond_choose_slave(const struct bond
*bond
)
1481 struct bond_slave
*slave
, *best
;
1483 /* Find an enabled slave. */
1484 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1485 if (slave
->enabled
) {
1490 /* All interfaces are disabled. Find an interface that will be enabled
1491 * after its updelay expires. */
1493 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1494 if (slave
->delay_expires
!= LLONG_MAX
1495 && slave
->lacp_may_enable
1496 && (!best
|| slave
->delay_expires
< best
->delay_expires
)) {
1504 bond_choose_active_slave(struct bond
*bond
, struct tag_set
*tags
)
1506 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1507 struct bond_slave
*old_active_slave
= bond
->active_slave
;
1509 bond
->active_slave
= bond_choose_slave(bond
);
1510 if (bond
->active_slave
) {
1511 if (bond
->active_slave
->enabled
) {
1512 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s",
1513 bond
->name
, bond
->active_slave
->name
);
1515 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s, skipping "
1516 "remaining %lld ms updelay (since no interface was "
1517 "enabled)", bond
->name
, bond
->active_slave
->name
,
1518 bond
->active_slave
->delay_expires
- time_msec());
1519 bond_enable_slave(bond
->active_slave
, true, tags
);
1522 if (!old_active_slave
) {
1523 tag_set_add(tags
, bond
->no_slaves_tag
);
1526 bond
->send_learning_packets
= true;
1527 } else if (old_active_slave
) {
1528 VLOG_WARN_RL(&rl
, "bond %s: all interfaces disabled", bond
->name
);
1532 /* Returns the tag for 'bond''s active slave, or 'bond''s no_slaves_tag if
1533 * there is no active slave. */
1535 bond_get_active_slave_tag(const struct bond
*bond
)
1537 return (bond
->active_slave
1538 ? bond
->active_slave
->tag
1539 : bond
->no_slaves_tag
);
1542 /* Attempts to make the sum of the bond slaves' statistics appear on the fake
1543 * bond interface. */
1545 bond_update_fake_slave_stats(struct bond
*bond
)
1547 struct netdev_stats bond_stats
;
1548 struct bond_slave
*slave
;
1549 struct netdev
*bond_dev
;
1551 memset(&bond_stats
, 0, sizeof bond_stats
);
1553 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1554 struct netdev_stats slave_stats
;
1556 if (!netdev_get_stats(slave
->netdev
, &slave_stats
)) {
1557 /* XXX: We swap the stats here because they are swapped back when
1558 * reported by the internal device. The reason for this is
1559 * internal devices normally represent packets going into the
1560 * system but when used as fake bond device they represent packets
1561 * leaving the system. We really should do this in the internal
1562 * device itself because changing it here reverses the counts from
1563 * the perspective of the switch. However, the internal device
1564 * doesn't know what type of device it represents so we have to do
1565 * it here for now. */
1566 bond_stats
.tx_packets
+= slave_stats
.rx_packets
;
1567 bond_stats
.tx_bytes
+= slave_stats
.rx_bytes
;
1568 bond_stats
.rx_packets
+= slave_stats
.tx_packets
;
1569 bond_stats
.rx_bytes
+= slave_stats
.tx_bytes
;
1573 if (!netdev_open_default(bond
->name
, &bond_dev
)) {
1574 netdev_set_stats(bond_dev
, &bond_stats
);
1575 netdev_close(bond_dev
);