2 * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
27 #include "ofp-actions.h"
28 #include "openvswitch/ofpbuf.h"
29 #include "ofproto/ofproto-provider.h"
30 #include "ofproto/ofproto-dpif.h"
31 #include "ofproto/ofproto-dpif-rid.h"
32 #include "connectivity.h"
34 #include "openvswitch/dynamic-string.h"
38 #include "openvswitch/list.h"
41 #include "openvswitch/ofpbuf.h"
43 #include "dp-packet.h"
44 #include "poll-loop.h"
50 #include "openvswitch/vlog.h"
52 VLOG_DEFINE_THIS_MODULE(bond
);
54 static struct ovs_rwlock rwlock
= OVS_RWLOCK_INITIALIZER
;
55 static struct hmap all_bonds__
= HMAP_INITIALIZER(&all_bonds__
);
56 static struct hmap
*const all_bonds
OVS_GUARDED_BY(rwlock
) = &all_bonds__
;
58 /* Bit-mask for hashing a flow down to a bucket. */
59 #define BOND_MASK 0xff
60 #define BOND_BUCKETS (BOND_MASK + 1)
62 /* A hash bucket for mapping a flow to a slave.
63 * "struct bond" has an array of BOND_BUCKETS of these. */
65 struct bond_slave
*slave
; /* Assigned slave, NULL if unassigned. */
66 uint64_t tx_bytes
/* Count of bytes recently transmitted. */
67 OVS_GUARDED_BY(rwlock
);
68 struct ovs_list list_node
; /* In bond_slave's 'entries' list. */
72 * 'pr_rule' is the post-recirculation rule for this entry.
73 * 'pr_tx_bytes' is the most recently seen statistics for 'pr_rule', which
74 * is used to determine delta (applied to 'tx_bytes' above.) */
76 uint64_t pr_tx_bytes
OVS_GUARDED_BY(rwlock
);
79 /* A bond slave, that is, one of the links comprising a bond. */
81 struct hmap_node hmap_node
; /* In struct bond's slaves hmap. */
82 struct ovs_list list_node
; /* In struct bond's enabled_slaves list. */
83 struct bond
*bond
; /* The bond that contains this slave. */
84 void *aux
; /* Client-provided handle for this slave. */
86 struct netdev
*netdev
; /* Network device, owned by the client. */
87 uint64_t change_seq
; /* Tracks changes in 'netdev'. */
88 ofp_port_t ofp_port
; /* OpenFlow port number. */
89 char *name
; /* Name (a copy of netdev_get_name(netdev)). */
92 long long delay_expires
; /* Time after which 'enabled' may change. */
93 bool enabled
; /* May be chosen for flows? */
94 bool may_enable
; /* Client considers this slave bondable. */
96 /* Rebalancing info. Used only by bond_rebalance(). */
97 struct ovs_list bal_node
; /* In bond_rebalance()'s 'bals' list. */
98 struct ovs_list entries
; /* 'struct bond_entry's assigned here. */
99 uint64_t tx_bytes
; /* Sum across 'tx_bytes' of entries. */
102 /* A bond, that is, a set of network devices grouped to improve performance or
105 struct hmap_node hmap_node
; /* In 'all_bonds' hmap. */
106 char *name
; /* Name provided by client. */
107 struct ofproto_dpif
*ofproto
; /* The bridge this bond belongs to. */
114 * Any reader or writer of 'enabled_slaves' must hold 'mutex'.
115 * (To prevent the bond_slave from disappearing they must also hold
117 struct ovs_mutex mutex
OVS_ACQ_AFTER(rwlock
);
118 struct ovs_list enabled_slaves OVS_GUARDED
; /* Contains struct bond_slaves. */
121 enum bond_mode balance
; /* Balancing mode, one of BM_*. */
122 struct bond_slave
*active_slave
;
123 int updelay
, downdelay
; /* Delay before slave goes up/down, in ms. */
124 enum lacp_status lacp_status
; /* Status of LACP negotiations. */
125 bool bond_revalidate
; /* True if flows need revalidation. */
126 uint32_t basis
; /* Basis for flow hash function. */
128 /* SLB specific bonding info. */
129 struct bond_entry
*hash
; /* An array of BOND_BUCKETS elements. */
130 int rebalance_interval
; /* Interval between rebalances, in ms. */
131 long long int next_rebalance
; /* Next rebalancing time. */
132 bool send_learning_packets
;
133 uint32_t recirc_id
; /* Non zero if recirculation can be used.*/
134 struct hmap pr_rule_ops
; /* Helps to maintain post recirculation rules.*/
136 /* Store active slave to OVSDB. */
137 bool active_slave_changed
; /* Set to true whenever the bond changes
138 active slave. It will be reset to false
139 after it is stored into OVSDB */
141 /* Interface name may not be persistent across an OS reboot, use
142 * MAC address for identifing the active slave */
143 struct eth_addr active_slave_mac
;
144 /* The MAC address of the active interface. */
145 /* Legacy compatibility. */
146 bool lacp_fallback_ab
; /* Fallback to active-backup on LACP failure. */
148 struct ovs_refcount ref_cnt
;
151 /* What to do with an bond_recirc_rule. */
153 ADD
, /* Add the rule to ofproto's flow table. */
154 DEL
, /* Delete the rule from the ofproto's flow table. */
157 /* A rule to add to or delete from ofproto's internal flow table. */
158 struct bond_pr_rule_op
{
159 struct hmap_node hmap_node
;
161 ofp_port_t out_ofport
;
163 struct rule
**pr_rule
;
166 static void bond_entry_reset(struct bond
*) OVS_REQ_WRLOCK(rwlock
);
167 static struct bond_slave
*bond_slave_lookup(struct bond
*, const void *slave_
)
168 OVS_REQ_RDLOCK(rwlock
);
169 static void bond_enable_slave(struct bond_slave
*, bool enable
)
170 OVS_REQ_WRLOCK(rwlock
);
171 static void bond_link_status_update(struct bond_slave
*)
172 OVS_REQ_WRLOCK(rwlock
);
173 static void bond_choose_active_slave(struct bond
*)
174 OVS_REQ_WRLOCK(rwlock
);
175 static unsigned int bond_hash_src(const struct eth_addr mac
,
176 uint16_t vlan
, uint32_t basis
);
177 static unsigned int bond_hash_tcp(const struct flow
*, uint16_t vlan
,
179 static struct bond_entry
*lookup_bond_entry(const struct bond
*,
182 OVS_REQ_RDLOCK(rwlock
);
183 static struct bond_slave
*get_enabled_slave(struct bond
*)
184 OVS_REQ_RDLOCK(rwlock
);
185 static struct bond_slave
*choose_output_slave(const struct bond
*,
187 struct flow_wildcards
*,
189 OVS_REQ_RDLOCK(rwlock
);
191 /* Attempts to parse 's' as the name of a bond balancing mode. If successful,
192 * stores the mode in '*balance' and returns true. Otherwise returns false
193 * without modifying '*balance'. */
195 bond_mode_from_string(enum bond_mode
*balance
, const char *s
)
197 if (!strcmp(s
, bond_mode_to_string(BM_TCP
))) {
199 } else if (!strcmp(s
, bond_mode_to_string(BM_SLB
))) {
201 } else if (!strcmp(s
, bond_mode_to_string(BM_AB
))) {
209 /* Returns a string representing 'balance'. */
211 bond_mode_to_string(enum bond_mode balance
) {
214 return "balance-tcp";
216 return "balance-slb";
218 return "active-backup";
224 /* Creates and returns a new bond whose configuration is initially taken from
227 * The caller should register each slave on the new bond by calling
228 * bond_slave_register(). */
230 bond_create(const struct bond_settings
*s
, struct ofproto_dpif
*ofproto
)
234 bond
= xzalloc(sizeof *bond
);
235 bond
->ofproto
= ofproto
;
236 hmap_init(&bond
->slaves
);
237 ovs_list_init(&bond
->enabled_slaves
);
238 ovs_mutex_init(&bond
->mutex
);
239 ovs_refcount_init(&bond
->ref_cnt
);
240 hmap_init(&bond
->pr_rule_ops
);
242 bond_reconfigure(bond
, s
);
247 bond_ref(const struct bond
*bond_
)
249 struct bond
*bond
= CONST_CAST(struct bond
*, bond_
);
252 ovs_refcount_ref(&bond
->ref_cnt
);
259 bond_unref(struct bond
*bond
)
261 struct bond_slave
*slave
, *next_slave
;
262 struct bond_pr_rule_op
*pr_op
, *next_op
;
264 if (!bond
|| ovs_refcount_unref_relaxed(&bond
->ref_cnt
) != 1) {
268 ovs_rwlock_wrlock(&rwlock
);
269 hmap_remove(all_bonds
, &bond
->hmap_node
);
270 ovs_rwlock_unlock(&rwlock
);
272 HMAP_FOR_EACH_SAFE (slave
, next_slave
, hmap_node
, &bond
->slaves
) {
273 hmap_remove(&bond
->slaves
, &slave
->hmap_node
);
274 /* Client owns 'slave->netdev'. */
278 hmap_destroy(&bond
->slaves
);
280 ovs_mutex_destroy(&bond
->mutex
);
284 HMAP_FOR_EACH_SAFE(pr_op
, next_op
, hmap_node
, &bond
->pr_rule_ops
) {
285 hmap_remove(&bond
->pr_rule_ops
, &pr_op
->hmap_node
);
288 hmap_destroy(&bond
->pr_rule_ops
);
290 if (bond
->recirc_id
) {
291 recirc_free_id(bond
->recirc_id
);
298 add_pr_rule(struct bond
*bond
, const struct match
*match
,
299 ofp_port_t out_ofport
, struct rule
**rule
)
301 uint32_t hash
= match_hash(match
, 0);
302 struct bond_pr_rule_op
*pr_op
;
304 HMAP_FOR_EACH_WITH_HASH(pr_op
, hmap_node
, hash
, &bond
->pr_rule_ops
) {
305 if (match_equal(&pr_op
->match
, match
)) {
307 pr_op
->out_ofport
= out_ofport
;
308 pr_op
->pr_rule
= rule
;
313 pr_op
= xmalloc(sizeof *pr_op
);
314 pr_op
->match
= *match
;
316 pr_op
->out_ofport
= out_ofport
;
317 pr_op
->pr_rule
= rule
;
318 hmap_insert(&bond
->pr_rule_ops
, &pr_op
->hmap_node
, hash
);
322 update_recirc_rules(struct bond
*bond
)
323 OVS_REQ_WRLOCK(rwlock
)
326 struct bond_pr_rule_op
*pr_op
, *next_op
;
327 uint64_t ofpacts_stub
[128 / 8];
328 struct ofpbuf ofpacts
;
331 ofpbuf_use_stub(&ofpacts
, ofpacts_stub
, sizeof ofpacts_stub
);
333 HMAP_FOR_EACH(pr_op
, hmap_node
, &bond
->pr_rule_ops
) {
337 if (bond
->hash
&& bond
->recirc_id
) {
338 for (i
= 0; i
< BOND_BUCKETS
; i
++) {
339 struct bond_slave
*slave
= bond
->hash
[i
].slave
;
342 match_init_catchall(&match
);
343 match_set_recirc_id(&match
, bond
->recirc_id
);
344 match_set_dp_hash_masked(&match
, i
, BOND_MASK
);
346 add_pr_rule(bond
, &match
, slave
->ofp_port
,
347 &bond
->hash
[i
].pr_rule
);
352 HMAP_FOR_EACH_SAFE(pr_op
, next_op
, hmap_node
, &bond
->pr_rule_ops
) {
356 ofpbuf_clear(&ofpacts
);
357 ofpact_put_OUTPUT(&ofpacts
)->port
= pr_op
->out_ofport
;
358 error
= ofproto_dpif_add_internal_flow(bond
->ofproto
,
360 RECIRC_RULE_PRIORITY
, 0,
361 &ofpacts
, pr_op
->pr_rule
);
363 char *err_s
= match_to_string(&pr_op
->match
,
364 RECIRC_RULE_PRIORITY
);
366 VLOG_ERR("failed to add post recirculation flow %s", err_s
);
372 error
= ofproto_dpif_delete_internal_flow(bond
->ofproto
,
374 RECIRC_RULE_PRIORITY
);
376 char *err_s
= match_to_string(&pr_op
->match
,
377 RECIRC_RULE_PRIORITY
);
379 VLOG_ERR("failed to remove post recirculation flow %s", err_s
);
383 hmap_remove(&bond
->pr_rule_ops
, &pr_op
->hmap_node
);
384 *pr_op
->pr_rule
= NULL
;
390 ofpbuf_uninit(&ofpacts
);
394 /* Updates 'bond''s overall configuration to 's'.
396 * The caller should register each slave on 'bond' by calling
397 * bond_slave_register(). This is optional if none of the slaves'
398 * configuration has changed. In any case it can't hurt.
400 * Returns true if the configuration has changed in such a way that requires
404 bond_reconfigure(struct bond
*bond
, const struct bond_settings
*s
)
406 bool revalidate
= false;
408 ovs_rwlock_wrlock(&rwlock
);
409 if (!bond
->name
|| strcmp(bond
->name
, s
->name
)) {
411 hmap_remove(all_bonds
, &bond
->hmap_node
);
414 bond
->name
= xstrdup(s
->name
);
415 hmap_insert(all_bonds
, &bond
->hmap_node
, hash_string(bond
->name
, 0));
418 bond
->updelay
= s
->up_delay
;
419 bond
->downdelay
= s
->down_delay
;
421 if (bond
->lacp_fallback_ab
!= s
->lacp_fallback_ab_cfg
) {
422 bond
->lacp_fallback_ab
= s
->lacp_fallback_ab_cfg
;
426 if (bond
->rebalance_interval
!= s
->rebalance_interval
) {
427 bond
->rebalance_interval
= s
->rebalance_interval
;
431 if (bond
->balance
!= s
->balance
) {
432 bond
->balance
= s
->balance
;
436 if (bond
->basis
!= s
->basis
) {
437 bond
->basis
= s
->basis
;
441 if (bond
->bond_revalidate
) {
443 bond
->bond_revalidate
= false;
446 if (bond
->balance
!= BM_AB
) {
447 if (!bond
->recirc_id
) {
448 bond
->recirc_id
= recirc_alloc_id(bond
->ofproto
);
450 } else if (bond
->recirc_id
) {
451 recirc_free_id(bond
->recirc_id
);
455 if (bond
->balance
== BM_AB
|| !bond
->hash
|| revalidate
) {
456 bond_entry_reset(bond
);
459 bond
->active_slave_mac
= s
->active_slave_mac
;
460 bond
->active_slave_changed
= false;
462 ovs_rwlock_unlock(&rwlock
);
466 static struct bond_slave
*
467 bond_find_slave_by_mac(const struct bond
*bond
, const struct eth_addr mac
)
469 struct bond_slave
*slave
;
471 /* Find the last active slave */
472 HMAP_FOR_EACH(slave
, hmap_node
, &bond
->slaves
) {
473 struct eth_addr slave_mac
;
475 if (netdev_get_etheraddr(slave
->netdev
, &slave_mac
)) {
479 if (eth_addr_equals(slave_mac
, mac
)) {
488 bond_active_slave_changed(struct bond
*bond
)
492 netdev_get_etheraddr(bond
->active_slave
->netdev
, &mac
);
493 bond
->active_slave_mac
= mac
;
494 bond
->active_slave_changed
= true;
495 seq_change(connectivity_seq_get());
499 bond_slave_set_netdev__(struct bond_slave
*slave
, struct netdev
*netdev
)
500 OVS_REQ_WRLOCK(rwlock
)
502 if (slave
->netdev
!= netdev
) {
503 slave
->netdev
= netdev
;
504 slave
->change_seq
= 0;
508 /* Registers 'slave_' as a slave of 'bond'. The 'slave_' pointer is an
509 * arbitrary client-provided pointer that uniquely identifies a slave within a
510 * bond. If 'slave_' already exists within 'bond' then this function
511 * reconfigures the existing slave.
513 * 'netdev' must be the network device that 'slave_' represents. It is owned
514 * by the client, so the client must not close it before either unregistering
515 * 'slave_' or destroying 'bond'.
518 bond_slave_register(struct bond
*bond
, void *slave_
,
519 ofp_port_t ofport
, struct netdev
*netdev
)
521 struct bond_slave
*slave
;
523 ovs_rwlock_wrlock(&rwlock
);
524 slave
= bond_slave_lookup(bond
, slave_
);
526 slave
= xzalloc(sizeof *slave
);
528 hmap_insert(&bond
->slaves
, &slave
->hmap_node
, hash_pointer(slave_
, 0));
531 slave
->ofp_port
= ofport
;
532 slave
->delay_expires
= LLONG_MAX
;
533 slave
->name
= xstrdup(netdev_get_name(netdev
));
534 bond
->bond_revalidate
= true;
536 slave
->enabled
= false;
537 bond_enable_slave(slave
, netdev_get_carrier(netdev
));
540 bond_slave_set_netdev__(slave
, netdev
);
543 slave
->name
= xstrdup(netdev_get_name(netdev
));
544 ovs_rwlock_unlock(&rwlock
);
547 /* Updates the network device to be used with 'slave_' to 'netdev'.
549 * This is useful if the caller closes and re-opens the network device
550 * registered with bond_slave_register() but doesn't need to change anything
553 bond_slave_set_netdev(struct bond
*bond
, void *slave_
, struct netdev
*netdev
)
555 struct bond_slave
*slave
;
557 ovs_rwlock_wrlock(&rwlock
);
558 slave
= bond_slave_lookup(bond
, slave_
);
560 bond_slave_set_netdev__(slave
, netdev
);
562 ovs_rwlock_unlock(&rwlock
);
565 /* Unregisters 'slave_' from 'bond'. If 'bond' does not contain such a slave
566 * then this function has no effect.
568 * Unregistering a slave invalidates all flows. */
570 bond_slave_unregister(struct bond
*bond
, const void *slave_
)
572 struct bond_slave
*slave
;
575 ovs_rwlock_wrlock(&rwlock
);
576 slave
= bond_slave_lookup(bond
, slave_
);
581 bond
->bond_revalidate
= true;
582 bond_enable_slave(slave
, false);
584 del_active
= bond
->active_slave
== slave
;
586 struct bond_entry
*e
;
587 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
588 if (e
->slave
== slave
) {
596 hmap_remove(&bond
->slaves
, &slave
->hmap_node
);
597 /* Client owns 'slave->netdev'. */
601 bond_choose_active_slave(bond
);
602 bond
->send_learning_packets
= true;
605 ovs_rwlock_unlock(&rwlock
);
608 /* Should be called on each slave in 'bond' before bond_run() to indicate
609 * whether or not 'slave_' may be enabled. This function is intended to allow
610 * other protocols to have some impact on bonding decisions. For example LACP
611 * or high level link monitoring protocols may decide that a given slave should
612 * not be able to send traffic. */
614 bond_slave_set_may_enable(struct bond
*bond
, void *slave_
, bool may_enable
)
616 ovs_rwlock_wrlock(&rwlock
);
617 bond_slave_lookup(bond
, slave_
)->may_enable
= may_enable
;
618 ovs_rwlock_unlock(&rwlock
);
621 /* Performs periodic maintenance on 'bond'.
623 * Returns true if the caller should revalidate its flows.
625 * The caller should check bond_should_send_learning_packets() afterward. */
627 bond_run(struct bond
*bond
, enum lacp_status lacp_status
)
629 struct bond_slave
*slave
;
632 ovs_rwlock_wrlock(&rwlock
);
633 if (bond
->lacp_status
!= lacp_status
) {
634 bond
->lacp_status
= lacp_status
;
635 bond
->bond_revalidate
= true;
638 /* Enable slaves based on link status and LACP feedback. */
639 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
640 bond_link_status_update(slave
);
641 slave
->change_seq
= seq_read(connectivity_seq_get());
643 if (!bond
->active_slave
|| !bond
->active_slave
->enabled
) {
644 bond_choose_active_slave(bond
);
647 revalidate
= bond
->bond_revalidate
;
648 bond
->bond_revalidate
= false;
649 ovs_rwlock_unlock(&rwlock
);
654 /* Causes poll_block() to wake up when 'bond' needs something to be done. */
656 bond_wait(struct bond
*bond
)
658 struct bond_slave
*slave
;
660 ovs_rwlock_rdlock(&rwlock
);
661 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
662 if (slave
->delay_expires
!= LLONG_MAX
) {
663 poll_timer_wait_until(slave
->delay_expires
);
666 seq_wait(connectivity_seq_get(), slave
->change_seq
);
669 if (bond
->bond_revalidate
) {
670 poll_immediate_wake();
672 ovs_rwlock_unlock(&rwlock
);
674 /* We don't wait for bond->next_rebalance because rebalancing can only run
675 * at a flow account checkpoint. ofproto does checkpointing on its own
676 * schedule and bond_rebalance() gets called afterward, so we'd just be
677 * waking up for no purpose. */
680 /* MAC learning table interaction. */
683 may_send_learning_packets(const struct bond
*bond
)
685 return ((bond
->lacp_status
== LACP_DISABLED
686 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_AB
))
687 || (bond
->lacp_fallback_ab
&& bond
->lacp_status
== LACP_CONFIGURED
))
688 && bond
->active_slave
;
691 /* Returns true if 'bond' needs the client to send out packets to assist with
692 * MAC learning on 'bond'. If this function returns true, then the client
693 * should iterate through its MAC learning table for the bridge on which 'bond'
694 * is located. For each MAC that has been learned on a port other than 'bond',
695 * it should call bond_compose_learning_packet().
697 * This function will only return true if 'bond' is in SLB or active-backup
698 * mode and LACP is not negotiated. Otherwise sending learning packets isn't
701 * Calling this function resets the state that it checks. */
703 bond_should_send_learning_packets(struct bond
*bond
)
707 ovs_rwlock_wrlock(&rwlock
);
708 send
= bond
->send_learning_packets
&& may_send_learning_packets(bond
);
709 bond
->send_learning_packets
= false;
710 ovs_rwlock_unlock(&rwlock
);
714 /* Sends a gratuitous learning packet on 'bond' from 'eth_src' on 'vlan'.
716 * See bond_should_send_learning_packets() for description of usage. The
717 * caller should send the composed packet on the port associated with
718 * port_aux and takes ownership of the returned ofpbuf. */
720 bond_compose_learning_packet(struct bond
*bond
, const struct eth_addr eth_src
,
721 uint16_t vlan
, void **port_aux
)
723 struct bond_slave
*slave
;
724 struct dp_packet
*packet
;
727 ovs_rwlock_rdlock(&rwlock
);
728 ovs_assert(may_send_learning_packets(bond
));
729 memset(&flow
, 0, sizeof flow
);
730 flow
.dl_src
= eth_src
;
731 slave
= choose_output_slave(bond
, &flow
, NULL
, vlan
);
733 packet
= dp_packet_new(0);
734 compose_rarp(packet
, eth_src
);
736 eth_push_vlan(packet
, htons(ETH_TYPE_VLAN
), htons(vlan
));
739 *port_aux
= slave
->aux
;
740 ovs_rwlock_unlock(&rwlock
);
744 /* Checks whether a packet that arrived on 'slave_' within 'bond', with an
745 * Ethernet destination address of 'eth_dst', should be admitted.
747 * The return value is one of the following:
749 * - BV_ACCEPT: Admit the packet.
751 * - BV_DROP: Drop the packet.
753 * - BV_DROP_IF_MOVED: Consult the MAC learning table for the packet's
754 * Ethernet source address and VLAN. If there is none, or if the packet
755 * is on the learned port, then admit the packet. If a different port has
756 * been learned, however, drop the packet (and do not use it for MAC
760 bond_check_admissibility(struct bond
*bond
, const void *slave_
,
761 const struct eth_addr eth_dst
)
763 enum bond_verdict verdict
= BV_DROP
;
764 struct bond_slave
*slave
;
766 ovs_rwlock_rdlock(&rwlock
);
767 slave
= bond_slave_lookup(bond
, slave_
);
772 /* LACP bonds have very loose admissibility restrictions because we can
773 * assume the remote switch is aware of the bond and will "do the right
774 * thing". However, as a precaution we drop packets on disabled slaves
775 * because no correctly implemented partner switch should be sending
778 * If LACP is configured, but LACP negotiations have been unsuccessful, we
779 * drop all incoming traffic except if lacp_fallback_ab is enabled. */
780 switch (bond
->lacp_status
) {
781 case LACP_NEGOTIATED
:
782 verdict
= slave
->enabled
? BV_ACCEPT
: BV_DROP
;
784 case LACP_CONFIGURED
:
785 if (!bond
->lacp_fallback_ab
) {
792 /* Drop all multicast packets on inactive slaves. */
793 if (eth_addr_is_multicast(eth_dst
)) {
794 if (bond
->active_slave
!= slave
) {
799 switch (bond
->balance
) {
801 /* TCP balanced bonds require successful LACP negotiations. Based on the
802 * above check, LACP is off or lacp_fallback_ab is true on this bond.
803 * If lacp_fallback_ab is true fall through to BM_AB case else, we
804 * drop all incoming traffic. */
805 if (!bond
->lacp_fallback_ab
) {
810 /* Drop all packets which arrive on backup slaves. This is similar to
811 * how Linux bonding handles active-backup bonds. */
812 if (bond
->active_slave
!= slave
) {
813 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
815 VLOG_DBG_RL(&rl
, "active-backup bond received packet on backup"
816 " slave (%s) destined for " ETH_ADDR_FMT
,
817 slave
->name
, ETH_ADDR_ARGS(eth_dst
));
824 /* Drop all packets for which we have learned a different input port,
825 * because we probably sent the packet on one slave and got it back on
826 * the other. Gratuitous ARP packets are an exception to this rule:
827 * the host has moved to another switch. The exception to the
828 * exception is if we locked the learning table to avoid reflections on
830 verdict
= BV_DROP_IF_MOVED
;
836 ovs_rwlock_unlock(&rwlock
);
841 /* Returns the slave (registered on 'bond' by bond_slave_register()) to which
842 * a packet with the given 'flow' and 'vlan' should be forwarded. Returns
843 * NULL if the packet should be dropped because no slaves are enabled.
845 * 'vlan' is not necessarily the same as 'flow->vlan_tci'. First, 'vlan'
846 * should be a VID only (i.e. excluding the PCP bits). Second,
847 * 'flow->vlan_tci' is the VLAN TCI that appeared on the packet (so it will be
848 * nonzero only for trunk ports), whereas 'vlan' is the logical VLAN that the
849 * packet belongs to (so for an access port it will be the access port's VLAN).
851 * If 'wc' is non-NULL, bitwise-OR's 'wc' with the set of bits that were
852 * significant in the selection. At some point earlier, 'wc' should
853 * have been initialized (e.g., by flow_wildcards_init_catchall()).
856 bond_choose_output_slave(struct bond
*bond
, const struct flow
*flow
,
857 struct flow_wildcards
*wc
, uint16_t vlan
)
859 struct bond_slave
*slave
;
862 ovs_rwlock_rdlock(&rwlock
);
863 slave
= choose_output_slave(bond
, flow
, wc
, vlan
);
864 aux
= slave
? slave
->aux
: NULL
;
865 ovs_rwlock_unlock(&rwlock
);
872 bond_entry_account(struct bond_entry
*entry
, uint64_t rule_tx_bytes
)
873 OVS_REQ_WRLOCK(rwlock
)
878 delta
= rule_tx_bytes
- entry
->pr_tx_bytes
;
879 entry
->tx_bytes
+= delta
;
880 entry
->pr_tx_bytes
= rule_tx_bytes
;
884 /* Maintain bond stats using post recirculation rule byte counters.*/
886 bond_recirculation_account(struct bond
*bond
)
887 OVS_REQ_WRLOCK(rwlock
)
891 for (i
=0; i
<=BOND_MASK
; i
++) {
892 struct bond_entry
*entry
= &bond
->hash
[i
];
893 struct rule
*rule
= entry
->pr_rule
;
896 uint64_t n_packets OVS_UNUSED
;
897 long long int used OVS_UNUSED
;
900 rule
->ofproto
->ofproto_class
->rule_get_stats(
901 rule
, &n_packets
, &n_bytes
, &used
);
902 bond_entry_account(entry
, n_bytes
);
908 bond_may_recirc(const struct bond
*bond
, uint32_t *recirc_id
,
911 if (bond
->balance
== BM_TCP
&& bond
->recirc_id
) {
913 *recirc_id
= bond
->recirc_id
;
916 *hash_bias
= bond
->basis
;
925 bond_update_post_recirc_rules__(struct bond
* bond
, const bool force
)
926 OVS_REQ_WRLOCK(rwlock
)
928 struct bond_entry
*e
;
929 bool update_rules
= force
; /* Always update rules if caller forces it. */
931 /* Make sure all bond entries are populated */
932 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
933 if (!e
->slave
|| !e
->slave
->enabled
) {
935 e
->slave
= CONTAINER_OF(hmap_random_node(&bond
->slaves
),
936 struct bond_slave
, hmap_node
);
937 if (!e
->slave
->enabled
) {
938 e
->slave
= bond
->active_slave
;
944 update_recirc_rules(bond
);
949 bond_update_post_recirc_rules(struct bond
* bond
, const bool force
)
951 ovs_rwlock_wrlock(&rwlock
);
952 bond_update_post_recirc_rules__(bond
, force
);
953 ovs_rwlock_unlock(&rwlock
);
959 bond_is_balanced(const struct bond
*bond
) OVS_REQ_RDLOCK(rwlock
)
961 return bond
->rebalance_interval
962 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_TCP
);
965 /* Notifies 'bond' that 'n_bytes' bytes were sent in 'flow' within 'vlan'. */
967 bond_account(struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
,
970 ovs_rwlock_wrlock(&rwlock
);
971 if (bond_is_balanced(bond
)) {
972 lookup_bond_entry(bond
, flow
, vlan
)->tx_bytes
+= n_bytes
;
974 ovs_rwlock_unlock(&rwlock
);
977 static struct bond_slave
*
978 bond_slave_from_bal_node(struct ovs_list
*bal
) OVS_REQ_RDLOCK(rwlock
)
980 return CONTAINER_OF(bal
, struct bond_slave
, bal_node
);
984 log_bals(struct bond
*bond
, const struct ovs_list
*bals
)
985 OVS_REQ_RDLOCK(rwlock
)
987 if (VLOG_IS_DBG_ENABLED()) {
988 struct ds ds
= DS_EMPTY_INITIALIZER
;
989 const struct bond_slave
*slave
;
991 LIST_FOR_EACH (slave
, bal_node
, bals
) {
993 ds_put_char(&ds
, ',');
995 ds_put_format(&ds
, " %s %"PRIu64
"kB",
996 slave
->name
, slave
->tx_bytes
/ 1024);
998 if (!slave
->enabled
) {
999 ds_put_cstr(&ds
, " (disabled)");
1001 if (!ovs_list_is_empty(&slave
->entries
)) {
1002 struct bond_entry
*e
;
1004 ds_put_cstr(&ds
, " (");
1005 LIST_FOR_EACH (e
, list_node
, &slave
->entries
) {
1006 if (&e
->list_node
!= ovs_list_front(&slave
->entries
)) {
1007 ds_put_cstr(&ds
, " + ");
1009 ds_put_format(&ds
, "h%"PRIdPTR
": %"PRIu64
"kB",
1010 e
- bond
->hash
, e
->tx_bytes
/ 1024);
1012 ds_put_cstr(&ds
, ")");
1015 VLOG_DBG("bond %s:%s", bond
->name
, ds_cstr(&ds
));
1020 /* Shifts 'hash' from its current slave to 'to'. */
1022 bond_shift_load(struct bond_entry
*hash
, struct bond_slave
*to
)
1023 OVS_REQ_WRLOCK(rwlock
)
1025 struct bond_slave
*from
= hash
->slave
;
1026 struct bond
*bond
= from
->bond
;
1027 uint64_t delta
= hash
->tx_bytes
;
1029 VLOG_INFO("bond %s: shift %"PRIu64
"kB of load (with hash %"PRIdPTR
") "
1030 "from %s to %s (now carrying %"PRIu64
"kB and "
1031 "%"PRIu64
"kB load, respectively)",
1032 bond
->name
, delta
/ 1024, hash
- bond
->hash
,
1033 from
->name
, to
->name
,
1034 (from
->tx_bytes
- delta
) / 1024,
1035 (to
->tx_bytes
+ delta
) / 1024);
1037 /* Shift load away from 'from' to 'to'. */
1038 from
->tx_bytes
-= delta
;
1039 to
->tx_bytes
+= delta
;
1041 /* Arrange for flows to be revalidated. */
1043 bond
->bond_revalidate
= true;
1046 /* Picks and returns a bond_entry to migrate from 'from' (the most heavily
1047 * loaded bond slave) to a bond slave that has 'to_tx_bytes' bytes of load,
1048 * given that doing so must decrease the ratio of the load on the two slaves by
1049 * at least 0.1. Returns NULL if there is no appropriate entry.
1051 * The list of entries isn't sorted. I don't know of a reason to prefer to
1052 * shift away small hashes or large hashes. */
1053 static struct bond_entry
*
1054 choose_entry_to_migrate(const struct bond_slave
*from
, uint64_t to_tx_bytes
)
1055 OVS_REQ_WRLOCK(rwlock
)
1057 struct bond_entry
*e
;
1059 if (ovs_list_is_short(&from
->entries
)) {
1060 /* 'from' carries no more than one MAC hash, so shifting load away from
1061 * it would be pointless. */
1065 LIST_FOR_EACH (e
, list_node
, &from
->entries
) {
1066 uint64_t delta
= e
->tx_bytes
; /* The amount to rebalance. */
1067 uint64_t ideal_tx_bytes
= (from
->tx_bytes
+ to_tx_bytes
)/2;
1068 /* Note, the ideal traffic is the mid point
1069 * between 'from' and 'to'. This value does
1070 * not change by rebalancing. */
1071 uint64_t new_low
; /* The lower bandwidth between 'to' and 'from'
1072 after rebalancing. */
1074 new_low
= MIN(from
->tx_bytes
- delta
, to_tx_bytes
+ delta
);
1076 if ((new_low
> to_tx_bytes
) &&
1077 (new_low
- to_tx_bytes
>= (ideal_tx_bytes
- to_tx_bytes
) / 10)) {
1078 /* Only rebalance if the new 'low' is closer to to the mid point,
1079 * and the improvement exceeds 10% of current traffic
1080 * deviation from the ideal split.
1082 * The improvement on the 'high' side is always the same as the
1083 * 'low' side. Thus consider 'low' side is sufficient. */
1091 /* Inserts 'slave' into 'bals' so that descending order of 'tx_bytes' is
1094 insert_bal(struct ovs_list
*bals
, struct bond_slave
*slave
)
1096 struct bond_slave
*pos
;
1098 LIST_FOR_EACH (pos
, bal_node
, bals
) {
1099 if (slave
->tx_bytes
> pos
->tx_bytes
) {
1103 ovs_list_insert(&pos
->bal_node
, &slave
->bal_node
);
1106 /* Removes 'slave' from its current list and then inserts it into 'bals' so
1107 * that descending order of 'tx_bytes' is maintained. */
1109 reinsert_bal(struct ovs_list
*bals
, struct bond_slave
*slave
)
1111 ovs_list_remove(&slave
->bal_node
);
1112 insert_bal(bals
, slave
);
1115 /* If 'bond' needs rebalancing, does so.
1117 * The caller should have called bond_account() for each active flow, or in case
1118 * of recirculation is used, have called bond_recirculation_account(bond),
1119 * to ensure that flow data is consistently accounted at this point.
1122 bond_rebalance(struct bond
*bond
)
1124 struct bond_slave
*slave
;
1125 struct bond_entry
*e
;
1126 struct ovs_list bals
;
1127 bool rebalanced
= false;
1130 ovs_rwlock_wrlock(&rwlock
);
1131 if (!bond_is_balanced(bond
) || time_msec() < bond
->next_rebalance
) {
1134 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1136 use_recirc
= ofproto_dpif_get_support(bond
->ofproto
)->odp
.recirc
&&
1137 bond_may_recirc(bond
, NULL
, NULL
);
1140 bond_recirculation_account(bond
);
1143 /* Add each bond_entry to its slave's 'entries' list.
1144 * Compute each slave's tx_bytes as the sum of its entries' tx_bytes. */
1145 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1146 slave
->tx_bytes
= 0;
1147 ovs_list_init(&slave
->entries
);
1149 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
1150 if (e
->slave
&& e
->tx_bytes
) {
1151 e
->slave
->tx_bytes
+= e
->tx_bytes
;
1152 ovs_list_push_back(&e
->slave
->entries
, &e
->list_node
);
1156 /* Add enabled slaves to 'bals' in descending order of tx_bytes.
1158 * XXX This is O(n**2) in the number of slaves but it could be O(n lg n)
1159 * with a proper list sort algorithm. */
1160 ovs_list_init(&bals
);
1161 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1162 if (slave
->enabled
) {
1163 insert_bal(&bals
, slave
);
1166 log_bals(bond
, &bals
);
1168 /* Shift load from the most-loaded slaves to the least-loaded slaves. */
1169 while (!ovs_list_is_short(&bals
)) {
1170 struct bond_slave
*from
= bond_slave_from_bal_node(ovs_list_front(&bals
));
1171 struct bond_slave
*to
= bond_slave_from_bal_node(ovs_list_back(&bals
));
1174 overload
= from
->tx_bytes
- to
->tx_bytes
;
1175 if (overload
< to
->tx_bytes
>> 5 || overload
< 100000) {
1176 /* The extra load on 'from' (and all less-loaded slaves), compared
1177 * to that of 'to' (the least-loaded slave), is less than ~3%, or
1178 * it is less than ~1Mbps. No point in rebalancing. */
1182 /* 'from' is carrying significantly more load than 'to'. Pick a hash
1183 * to move from 'from' to 'to'. */
1184 e
= choose_entry_to_migrate(from
, to
->tx_bytes
);
1186 bond_shift_load(e
, to
);
1188 /* Delete element from from->entries.
1190 * We don't add the element to to->hashes. That would only allow
1191 * 'e' to be migrated to another slave in this rebalancing run, and
1192 * there is no point in doing that. */
1193 ovs_list_remove(&e
->list_node
);
1195 /* Re-sort 'bals'. */
1196 reinsert_bal(&bals
, from
);
1197 reinsert_bal(&bals
, to
);
1200 /* Can't usefully migrate anything away from 'from'.
1201 * Don't reconsider it. */
1202 ovs_list_remove(&from
->bal_node
);
1206 /* Implement exponentially weighted moving average. A weight of 1/2 causes
1207 * historical data to decay to <1% in 7 rebalancing runs. 1,000,000 bytes
1208 * take 20 rebalancing runs to decay to 0 and get deleted entirely. */
1209 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
1213 if (use_recirc
&& rebalanced
) {
1214 bond_update_post_recirc_rules__(bond
,true);
1218 ovs_rwlock_unlock(&rwlock
);
1221 /* Bonding unixctl user interface functions. */
1223 static struct bond
*
1224 bond_find(const char *name
) OVS_REQ_RDLOCK(rwlock
)
1228 HMAP_FOR_EACH_WITH_HASH (bond
, hmap_node
, hash_string(name
, 0),
1230 if (!strcmp(bond
->name
, name
)) {
1237 static struct bond_slave
*
1238 bond_lookup_slave(struct bond
*bond
, const char *slave_name
)
1240 struct bond_slave
*slave
;
1242 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1243 if (!strcmp(slave
->name
, slave_name
)) {
1251 bond_unixctl_list(struct unixctl_conn
*conn
,
1252 int argc OVS_UNUSED
, const char *argv
[] OVS_UNUSED
,
1253 void *aux OVS_UNUSED
)
1255 struct ds ds
= DS_EMPTY_INITIALIZER
;
1256 const struct bond
*bond
;
1258 ds_put_cstr(&ds
, "bond\ttype\trecircID\tslaves\n");
1260 ovs_rwlock_rdlock(&rwlock
);
1261 HMAP_FOR_EACH (bond
, hmap_node
, all_bonds
) {
1262 const struct bond_slave
*slave
;
1265 ds_put_format(&ds
, "%s\t%s\t%d\t", bond
->name
,
1266 bond_mode_to_string(bond
->balance
), bond
->recirc_id
);
1269 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1271 ds_put_cstr(&ds
, ", ");
1273 ds_put_cstr(&ds
, slave
->name
);
1275 ds_put_char(&ds
, '\n');
1277 ovs_rwlock_unlock(&rwlock
);
1278 unixctl_command_reply(conn
, ds_cstr(&ds
));
1283 bond_print_details(struct ds
*ds
, const struct bond
*bond
)
1284 OVS_REQ_RDLOCK(rwlock
)
1286 struct shash slave_shash
= SHASH_INITIALIZER(&slave_shash
);
1287 const struct shash_node
**sorted_slaves
= NULL
;
1288 const struct bond_slave
*slave
;
1293 ds_put_format(ds
, "---- %s ----\n", bond
->name
);
1294 ds_put_format(ds
, "bond_mode: %s\n",
1295 bond_mode_to_string(bond
->balance
));
1297 may_recirc
= bond_may_recirc(bond
, &recirc_id
, NULL
);
1298 ds_put_format(ds
, "bond may use recirculation: %s, Recirc-ID : %d\n",
1299 may_recirc
? "yes" : "no", may_recirc
? recirc_id
: -1);
1301 ds_put_format(ds
, "bond-hash-basis: %"PRIu32
"\n", bond
->basis
);
1303 ds_put_format(ds
, "updelay: %d ms\n", bond
->updelay
);
1304 ds_put_format(ds
, "downdelay: %d ms\n", bond
->downdelay
);
1306 if (bond_is_balanced(bond
)) {
1307 ds_put_format(ds
, "next rebalance: %lld ms\n",
1308 bond
->next_rebalance
- time_msec());
1311 ds_put_cstr(ds
, "lacp_status: ");
1312 switch (bond
->lacp_status
) {
1313 case LACP_NEGOTIATED
:
1314 ds_put_cstr(ds
, "negotiated\n");
1316 case LACP_CONFIGURED
:
1317 ds_put_cstr(ds
, "configured\n");
1320 ds_put_cstr(ds
, "off\n");
1323 ds_put_cstr(ds
, "<unknown>\n");
1327 ds_put_cstr(ds
, "active slave mac: ");
1328 ds_put_format(ds
, ETH_ADDR_FMT
, ETH_ADDR_ARGS(bond
->active_slave_mac
));
1329 slave
= bond_find_slave_by_mac(bond
, bond
->active_slave_mac
);
1330 ds_put_format(ds
,"(%s)\n", slave
? slave
->name
: "none");
1332 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1333 shash_add(&slave_shash
, slave
->name
, slave
);
1335 sorted_slaves
= shash_sort(&slave_shash
);
1337 for (i
= 0; i
< shash_count(&slave_shash
); i
++) {
1338 struct bond_entry
*be
;
1340 slave
= sorted_slaves
[i
]->data
;
1343 ds_put_format(ds
, "\nslave %s: %s\n",
1344 slave
->name
, slave
->enabled
? "enabled" : "disabled");
1345 if (slave
== bond
->active_slave
) {
1346 ds_put_cstr(ds
, "\tactive slave\n");
1348 if (slave
->delay_expires
!= LLONG_MAX
) {
1349 ds_put_format(ds
, "\t%s expires in %lld ms\n",
1350 slave
->enabled
? "downdelay" : "updelay",
1351 slave
->delay_expires
- time_msec());
1354 ds_put_format(ds
, "\tmay_enable: %s\n",
1355 slave
->may_enable
? "true" : "false");
1357 if (!bond_is_balanced(bond
)) {
1362 for (be
= bond
->hash
; be
<= &bond
->hash
[BOND_MASK
]; be
++) {
1363 int hash
= be
- bond
->hash
;
1366 if (be
->slave
!= slave
) {
1370 be_tx_k
= be
->tx_bytes
/ 1024;
1372 ds_put_format(ds
, "\thash %d: %"PRIu64
" kB load\n",
1376 /* XXX How can we list the MACs assigned to hashes of SLB bonds? */
1379 shash_destroy(&slave_shash
);
1380 free(sorted_slaves
);
1381 ds_put_cstr(ds
, "\n");
1385 bond_unixctl_show(struct unixctl_conn
*conn
,
1386 int argc
, const char *argv
[],
1387 void *aux OVS_UNUSED
)
1389 struct ds ds
= DS_EMPTY_INITIALIZER
;
1391 ovs_rwlock_rdlock(&rwlock
);
1393 const struct bond
*bond
= bond_find(argv
[1]);
1396 unixctl_command_reply_error(conn
, "no such bond");
1399 bond_print_details(&ds
, bond
);
1401 const struct bond
*bond
;
1403 HMAP_FOR_EACH (bond
, hmap_node
, all_bonds
) {
1404 bond_print_details(&ds
, bond
);
1408 unixctl_command_reply(conn
, ds_cstr(&ds
));
1412 ovs_rwlock_unlock(&rwlock
);
1416 bond_unixctl_migrate(struct unixctl_conn
*conn
,
1417 int argc OVS_UNUSED
, const char *argv
[],
1418 void *aux OVS_UNUSED
)
1420 const char *bond_s
= argv
[1];
1421 const char *hash_s
= argv
[2];
1422 const char *slave_s
= argv
[3];
1424 struct bond_slave
*slave
;
1425 struct bond_entry
*entry
;
1428 ovs_rwlock_wrlock(&rwlock
);
1429 bond
= bond_find(bond_s
);
1431 unixctl_command_reply_error(conn
, "no such bond");
1435 if (bond
->balance
!= BM_SLB
) {
1436 unixctl_command_reply_error(conn
, "not an SLB bond");
1440 if (strspn(hash_s
, "0123456789") == strlen(hash_s
)) {
1441 hash
= atoi(hash_s
) & BOND_MASK
;
1443 unixctl_command_reply_error(conn
, "bad hash");
1447 slave
= bond_lookup_slave(bond
, slave_s
);
1449 unixctl_command_reply_error(conn
, "no such slave");
1453 if (!slave
->enabled
) {
1454 unixctl_command_reply_error(conn
, "cannot migrate to disabled slave");
1458 entry
= &bond
->hash
[hash
];
1459 bond
->bond_revalidate
= true;
1460 entry
->slave
= slave
;
1461 unixctl_command_reply(conn
, "migrated");
1464 ovs_rwlock_unlock(&rwlock
);
1468 bond_unixctl_set_active_slave(struct unixctl_conn
*conn
,
1469 int argc OVS_UNUSED
, const char *argv
[],
1470 void *aux OVS_UNUSED
)
1472 const char *bond_s
= argv
[1];
1473 const char *slave_s
= argv
[2];
1475 struct bond_slave
*slave
;
1477 ovs_rwlock_wrlock(&rwlock
);
1478 bond
= bond_find(bond_s
);
1480 unixctl_command_reply_error(conn
, "no such bond");
1484 slave
= bond_lookup_slave(bond
, slave_s
);
1486 unixctl_command_reply_error(conn
, "no such slave");
1490 if (!slave
->enabled
) {
1491 unixctl_command_reply_error(conn
, "cannot make disabled slave active");
1495 if (bond
->active_slave
!= slave
) {
1496 bond
->bond_revalidate
= true;
1497 bond
->active_slave
= slave
;
1498 VLOG_INFO("bond %s: active interface is now %s",
1499 bond
->name
, slave
->name
);
1500 bond
->send_learning_packets
= true;
1501 unixctl_command_reply(conn
, "done");
1502 bond_active_slave_changed(bond
);
1504 unixctl_command_reply(conn
, "no change");
1507 ovs_rwlock_unlock(&rwlock
);
1511 enable_slave(struct unixctl_conn
*conn
, const char *argv
[], bool enable
)
1513 const char *bond_s
= argv
[1];
1514 const char *slave_s
= argv
[2];
1516 struct bond_slave
*slave
;
1518 ovs_rwlock_wrlock(&rwlock
);
1519 bond
= bond_find(bond_s
);
1521 unixctl_command_reply_error(conn
, "no such bond");
1525 slave
= bond_lookup_slave(bond
, slave_s
);
1527 unixctl_command_reply_error(conn
, "no such slave");
1531 bond_enable_slave(slave
, enable
);
1532 unixctl_command_reply(conn
, enable
? "enabled" : "disabled");
1535 ovs_rwlock_unlock(&rwlock
);
1539 bond_unixctl_enable_slave(struct unixctl_conn
*conn
,
1540 int argc OVS_UNUSED
, const char *argv
[],
1541 void *aux OVS_UNUSED
)
1543 enable_slave(conn
, argv
, true);
1547 bond_unixctl_disable_slave(struct unixctl_conn
*conn
,
1548 int argc OVS_UNUSED
, const char *argv
[],
1549 void *aux OVS_UNUSED
)
1551 enable_slave(conn
, argv
, false);
1555 bond_unixctl_hash(struct unixctl_conn
*conn
, int argc
, const char *argv
[],
1556 void *aux OVS_UNUSED
)
1558 const char *mac_s
= argv
[1];
1559 const char *vlan_s
= argc
> 2 ? argv
[2] : NULL
;
1560 const char *basis_s
= argc
> 3 ? argv
[3] : NULL
;
1561 struct eth_addr mac
;
1568 if (!ovs_scan(vlan_s
, "%u", &vlan
)) {
1569 unixctl_command_reply_error(conn
, "invalid vlan");
1577 if (!ovs_scan(basis_s
, "%"SCNu32
, &basis
)) {
1578 unixctl_command_reply_error(conn
, "invalid basis");
1585 if (ovs_scan(mac_s
, ETH_ADDR_SCAN_FMT
, ETH_ADDR_SCAN_ARGS(mac
))) {
1586 hash
= bond_hash_src(mac
, vlan
, basis
) & BOND_MASK
;
1588 hash_cstr
= xasprintf("%u", hash
);
1589 unixctl_command_reply(conn
, hash_cstr
);
1592 unixctl_command_reply_error(conn
, "invalid mac");
1599 unixctl_command_register("bond/list", "", 0, 0, bond_unixctl_list
, NULL
);
1600 unixctl_command_register("bond/show", "[port]", 0, 1, bond_unixctl_show
,
1602 unixctl_command_register("bond/migrate", "port hash slave", 3, 3,
1603 bond_unixctl_migrate
, NULL
);
1604 unixctl_command_register("bond/set-active-slave", "port slave", 2, 2,
1605 bond_unixctl_set_active_slave
, NULL
);
1606 unixctl_command_register("bond/enable-slave", "port slave", 2, 2,
1607 bond_unixctl_enable_slave
, NULL
);
1608 unixctl_command_register("bond/disable-slave", "port slave", 2, 2,
1609 bond_unixctl_disable_slave
, NULL
);
1610 unixctl_command_register("bond/hash", "mac [vlan] [basis]", 1, 3,
1611 bond_unixctl_hash
, NULL
);
1615 bond_entry_reset(struct bond
*bond
)
1617 if (bond
->balance
!= BM_AB
) {
1618 size_t hash_len
= BOND_BUCKETS
* sizeof *bond
->hash
;
1621 bond
->hash
= xmalloc(hash_len
);
1623 memset(bond
->hash
, 0, hash_len
);
1625 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1632 static struct bond_slave
*
1633 bond_slave_lookup(struct bond
*bond
, const void *slave_
)
1635 struct bond_slave
*slave
;
1637 HMAP_FOR_EACH_IN_BUCKET (slave
, hmap_node
, hash_pointer(slave_
, 0),
1639 if (slave
->aux
== slave_
) {
1648 bond_enable_slave(struct bond_slave
*slave
, bool enable
)
1650 slave
->delay_expires
= LLONG_MAX
;
1651 if (enable
!= slave
->enabled
) {
1652 slave
->bond
->bond_revalidate
= true;
1653 slave
->enabled
= enable
;
1655 ovs_mutex_lock(&slave
->bond
->mutex
);
1657 ovs_list_insert(&slave
->bond
->enabled_slaves
, &slave
->list_node
);
1659 ovs_list_remove(&slave
->list_node
);
1661 ovs_mutex_unlock(&slave
->bond
->mutex
);
1663 VLOG_INFO("interface %s: %s", slave
->name
,
1664 slave
->enabled
? "enabled" : "disabled");
1669 bond_link_status_update(struct bond_slave
*slave
)
1671 struct bond
*bond
= slave
->bond
;
1674 up
= netdev_get_carrier(slave
->netdev
) && slave
->may_enable
;
1675 if ((up
== slave
->enabled
) != (slave
->delay_expires
== LLONG_MAX
)) {
1676 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1677 VLOG_INFO_RL(&rl
, "interface %s: link state %s",
1678 slave
->name
, up
? "up" : "down");
1679 if (up
== slave
->enabled
) {
1680 slave
->delay_expires
= LLONG_MAX
;
1681 VLOG_INFO_RL(&rl
, "interface %s: will not be %s",
1682 slave
->name
, up
? "disabled" : "enabled");
1684 int delay
= (bond
->lacp_status
!= LACP_DISABLED
? 0
1685 : up
? bond
->updelay
: bond
->downdelay
);
1686 slave
->delay_expires
= time_msec() + delay
;
1688 VLOG_INFO_RL(&rl
, "interface %s: will be %s if it stays %s "
1691 up
? "enabled" : "disabled",
1698 if (time_msec() >= slave
->delay_expires
) {
1699 bond_enable_slave(slave
, up
);
1704 bond_hash_src(const struct eth_addr mac
, uint16_t vlan
, uint32_t basis
)
1706 return hash_mac(mac
, vlan
, basis
);
1710 bond_hash_tcp(const struct flow
*flow
, uint16_t vlan
, uint32_t basis
)
1712 struct flow hash_flow
= *flow
;
1713 hash_flow
.vlan_tci
= htons(vlan
);
1715 /* The symmetric quality of this hash function is not required, but
1716 * flow_hash_symmetric_l4 already exists, and is sufficient for our
1717 * purposes, so we use it out of convenience. */
1718 return flow_hash_symmetric_l4(&hash_flow
, basis
);
1722 bond_hash(const struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
)
1724 ovs_assert(bond
->balance
== BM_TCP
|| bond
->balance
== BM_SLB
);
1726 return (bond
->balance
== BM_TCP
1727 ? bond_hash_tcp(flow
, vlan
, bond
->basis
)
1728 : bond_hash_src(flow
->dl_src
, vlan
, bond
->basis
));
1731 static struct bond_entry
*
1732 lookup_bond_entry(const struct bond
*bond
, const struct flow
*flow
,
1735 return &bond
->hash
[bond_hash(bond
, flow
, vlan
) & BOND_MASK
];
1738 /* Selects and returns an enabled slave from the 'enabled_slaves' list
1739 * in a round-robin fashion. If the 'enabled_slaves' list is empty,
1741 static struct bond_slave
*
1742 get_enabled_slave(struct bond
*bond
)
1744 struct ovs_list
*node
;
1746 ovs_mutex_lock(&bond
->mutex
);
1747 if (ovs_list_is_empty(&bond
->enabled_slaves
)) {
1748 ovs_mutex_unlock(&bond
->mutex
);
1752 node
= ovs_list_pop_front(&bond
->enabled_slaves
);
1753 ovs_list_push_back(&bond
->enabled_slaves
, node
);
1754 ovs_mutex_unlock(&bond
->mutex
);
1756 return CONTAINER_OF(node
, struct bond_slave
, list_node
);
1759 static struct bond_slave
*
1760 choose_output_slave(const struct bond
*bond
, const struct flow
*flow
,
1761 struct flow_wildcards
*wc
, uint16_t vlan
)
1763 struct bond_entry
*e
;
1766 balance
= bond
->balance
;
1767 if (bond
->lacp_status
== LACP_CONFIGURED
) {
1768 /* LACP has been configured on this bond but negotiations were
1769 * unsuccussful. If lacp_fallback_ab is enabled use active-
1770 * backup mode else drop all traffic. */
1771 if (!bond
->lacp_fallback_ab
) {
1779 return bond
->active_slave
;
1782 if (bond
->lacp_status
!= LACP_NEGOTIATED
) {
1783 /* Must have LACP negotiations for TCP balanced bonds. */
1787 flow_mask_hash_fields(flow
, wc
, NX_HASH_FIELDS_SYMMETRIC_L4
);
1792 flow_mask_hash_fields(flow
, wc
, NX_HASH_FIELDS_ETH_SRC
);
1794 e
= lookup_bond_entry(bond
, flow
, vlan
);
1795 if (!e
->slave
|| !e
->slave
->enabled
) {
1796 e
->slave
= get_enabled_slave(CONST_CAST(struct bond
*, bond
));
1805 static struct bond_slave
*
1806 bond_choose_slave(const struct bond
*bond
)
1808 struct bond_slave
*slave
, *best
;
1810 /* Find the last active slave. */
1811 slave
= bond_find_slave_by_mac(bond
, bond
->active_slave_mac
);
1812 if (slave
&& slave
->enabled
) {
1816 /* Find an enabled slave. */
1817 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1818 if (slave
->enabled
) {
1823 /* All interfaces are disabled. Find an interface that will be enabled
1824 * after its updelay expires. */
1826 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1827 if (slave
->delay_expires
!= LLONG_MAX
1828 && slave
->may_enable
1829 && (!best
|| slave
->delay_expires
< best
->delay_expires
)) {
1837 bond_choose_active_slave(struct bond
*bond
)
1839 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1840 struct bond_slave
*old_active_slave
= bond
->active_slave
;
1842 bond
->active_slave
= bond_choose_slave(bond
);
1843 if (bond
->active_slave
) {
1844 if (bond
->active_slave
->enabled
) {
1845 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s",
1846 bond
->name
, bond
->active_slave
->name
);
1848 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s, skipping "
1849 "remaining %lld ms updelay (since no interface was "
1850 "enabled)", bond
->name
, bond
->active_slave
->name
,
1851 bond
->active_slave
->delay_expires
- time_msec());
1852 bond_enable_slave(bond
->active_slave
, true);
1855 bond
->send_learning_packets
= true;
1857 if (bond
->active_slave
!= old_active_slave
) {
1858 bond_active_slave_changed(bond
);
1860 } else if (old_active_slave
) {
1861 VLOG_INFO_RL(&rl
, "bond %s: all interfaces disabled", bond
->name
);
1866 * Return true if bond has unstored active slave change.
1867 * If return true, 'mac' will store the bond's current active slave's
1870 bond_get_changed_active_slave(const char *name
, struct eth_addr
*mac
,
1875 ovs_rwlock_wrlock(&rwlock
);
1876 bond
= bond_find(name
);
1878 if (bond
->active_slave_changed
|| force
) {
1879 *mac
= bond
->active_slave_mac
;
1880 bond
->active_slave_changed
= false;
1881 ovs_rwlock_unlock(&rwlock
);
1885 ovs_rwlock_unlock(&rwlock
);