2 * Copyright (c) 2008-2017 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "connectivity.h"
28 #include "dp-packet.h"
30 #include "openvswitch/hmap.h"
34 #include "ofproto/ofproto-dpif.h"
35 #include "ofproto/ofproto-dpif-rid.h"
36 #include "ofproto/ofproto-provider.h"
37 #include "openvswitch/dynamic-string.h"
38 #include "openvswitch/list.h"
39 #include "openvswitch/match.h"
40 #include "openvswitch/ofp-actions.h"
41 #include "openvswitch/ofp-util.h"
42 #include "openvswitch/ofpbuf.h"
43 #include "openvswitch/vlog.h"
45 #include "poll-loop.h"
47 #include "openvswitch/shash.h"
52 VLOG_DEFINE_THIS_MODULE(bond
);
54 static struct ovs_rwlock rwlock
= OVS_RWLOCK_INITIALIZER
;
55 static struct hmap all_bonds__
= HMAP_INITIALIZER(&all_bonds__
);
56 static struct hmap
*const all_bonds
OVS_GUARDED_BY(rwlock
) = &all_bonds__
;
58 /* Bit-mask for hashing a flow down to a bucket. */
59 #define BOND_MASK 0xff
60 #define BOND_BUCKETS (BOND_MASK + 1)
62 /* Priority for internal rules created to handle recirculation */
63 #define RECIRC_RULE_PRIORITY 20
65 /* A hash bucket for mapping a flow to a slave.
66 * "struct bond" has an array of BOND_BUCKETS of these. */
68 struct bond_slave
*slave
; /* Assigned slave, NULL if unassigned. */
69 uint64_t tx_bytes
/* Count of bytes recently transmitted. */
70 OVS_GUARDED_BY(rwlock
);
71 struct ovs_list list_node
; /* In bond_slave's 'entries' list. */
75 * 'pr_rule' is the post-recirculation rule for this entry.
76 * 'pr_tx_bytes' is the most recently seen statistics for 'pr_rule', which
77 * is used to determine delta (applied to 'tx_bytes' above.) */
79 uint64_t pr_tx_bytes
OVS_GUARDED_BY(rwlock
);
82 /* A bond slave, that is, one of the links comprising a bond. */
84 struct hmap_node hmap_node
; /* In struct bond's slaves hmap. */
85 struct ovs_list list_node
; /* In struct bond's enabled_slaves list. */
86 struct bond
*bond
; /* The bond that contains this slave. */
87 void *aux
; /* Client-provided handle for this slave. */
89 struct netdev
*netdev
; /* Network device, owned by the client. */
90 uint64_t change_seq
; /* Tracks changes in 'netdev'. */
91 ofp_port_t ofp_port
; /* OpenFlow port number. */
92 char *name
; /* Name (a copy of netdev_get_name(netdev)). */
95 long long delay_expires
; /* Time after which 'enabled' may change. */
96 bool enabled
; /* May be chosen for flows? */
97 bool may_enable
; /* Client considers this slave bondable. */
99 /* Rebalancing info. Used only by bond_rebalance(). */
100 struct ovs_list bal_node
; /* In bond_rebalance()'s 'bals' list. */
101 struct ovs_list entries
; /* 'struct bond_entry's assigned here. */
102 uint64_t tx_bytes
; /* Sum across 'tx_bytes' of entries. */
105 /* A bond, that is, a set of network devices grouped to improve performance or
108 struct hmap_node hmap_node
; /* In 'all_bonds' hmap. */
109 char *name
; /* Name provided by client. */
110 struct ofproto_dpif
*ofproto
; /* The bridge this bond belongs to. */
117 * Any reader or writer of 'enabled_slaves' must hold 'mutex'.
118 * (To prevent the bond_slave from disappearing they must also hold
120 struct ovs_mutex mutex
OVS_ACQ_AFTER(rwlock
);
121 struct ovs_list enabled_slaves OVS_GUARDED
; /* Contains struct bond_slaves. */
124 enum bond_mode balance
; /* Balancing mode, one of BM_*. */
125 struct bond_slave
*active_slave
;
126 int updelay
, downdelay
; /* Delay before slave goes up/down, in ms. */
127 enum lacp_status lacp_status
; /* Status of LACP negotiations. */
128 bool bond_revalidate
; /* True if flows need revalidation. */
129 uint32_t basis
; /* Basis for flow hash function. */
131 /* SLB specific bonding info. */
132 struct bond_entry
*hash
; /* An array of BOND_BUCKETS elements. */
133 int rebalance_interval
; /* Interval between rebalances, in ms. */
134 long long int next_rebalance
; /* Next rebalancing time. */
135 bool send_learning_packets
;
136 uint32_t recirc_id
; /* Non zero if recirculation can be used.*/
137 struct hmap pr_rule_ops
; /* Helps to maintain post recirculation rules.*/
139 /* Store active slave to OVSDB. */
140 bool active_slave_changed
; /* Set to true whenever the bond changes
141 active slave. It will be reset to false
142 after it is stored into OVSDB */
144 /* Interface name may not be persistent across an OS reboot, use
145 * MAC address for identifing the active slave */
146 struct eth_addr active_slave_mac
;
147 /* The MAC address of the active interface. */
148 /* Legacy compatibility. */
149 bool lacp_fallback_ab
; /* Fallback to active-backup on LACP failure. */
151 struct ovs_refcount ref_cnt
;
154 /* What to do with an bond_recirc_rule. */
156 ADD
, /* Add the rule to ofproto's flow table. */
157 DEL
, /* Delete the rule from the ofproto's flow table. */
160 /* A rule to add to or delete from ofproto's internal flow table. */
161 struct bond_pr_rule_op
{
162 struct hmap_node hmap_node
;
164 ofp_port_t out_ofport
;
166 struct rule
**pr_rule
;
169 static void bond_entry_reset(struct bond
*) OVS_REQ_WRLOCK(rwlock
);
170 static struct bond_slave
*bond_slave_lookup(struct bond
*, const void *slave_
)
171 OVS_REQ_RDLOCK(rwlock
);
172 static void bond_enable_slave(struct bond_slave
*, bool enable
)
173 OVS_REQ_WRLOCK(rwlock
);
174 static void bond_link_status_update(struct bond_slave
*)
175 OVS_REQ_WRLOCK(rwlock
);
176 static void bond_choose_active_slave(struct bond
*)
177 OVS_REQ_WRLOCK(rwlock
);
178 static unsigned int bond_hash_src(const struct eth_addr mac
,
179 uint16_t vlan
, uint32_t basis
);
180 static unsigned int bond_hash_tcp(const struct flow
*, uint16_t vlan
,
182 static struct bond_entry
*lookup_bond_entry(const struct bond
*,
185 OVS_REQ_RDLOCK(rwlock
);
186 static struct bond_slave
*get_enabled_slave(struct bond
*)
187 OVS_REQ_RDLOCK(rwlock
);
188 static struct bond_slave
*choose_output_slave(const struct bond
*,
190 struct flow_wildcards
*,
192 OVS_REQ_RDLOCK(rwlock
);
193 static void update_recirc_rules__(struct bond
*bond
);
195 /* Attempts to parse 's' as the name of a bond balancing mode. If successful,
196 * stores the mode in '*balance' and returns true. Otherwise returns false
197 * without modifying '*balance'. */
199 bond_mode_from_string(enum bond_mode
*balance
, const char *s
)
201 if (!strcmp(s
, bond_mode_to_string(BM_TCP
))) {
203 } else if (!strcmp(s
, bond_mode_to_string(BM_SLB
))) {
205 } else if (!strcmp(s
, bond_mode_to_string(BM_AB
))) {
213 /* Returns a string representing 'balance'. */
215 bond_mode_to_string(enum bond_mode balance
) {
218 return "balance-tcp";
220 return "balance-slb";
222 return "active-backup";
228 /* Creates and returns a new bond whose configuration is initially taken from
231 * The caller should register each slave on the new bond by calling
232 * bond_slave_register(). */
234 bond_create(const struct bond_settings
*s
, struct ofproto_dpif
*ofproto
)
238 bond
= xzalloc(sizeof *bond
);
239 bond
->ofproto
= ofproto
;
240 hmap_init(&bond
->slaves
);
241 ovs_list_init(&bond
->enabled_slaves
);
242 ovs_mutex_init(&bond
->mutex
);
243 ovs_refcount_init(&bond
->ref_cnt
);
244 hmap_init(&bond
->pr_rule_ops
);
246 bond
->active_slave_mac
= eth_addr_zero
;
247 bond
->active_slave_changed
= false;
249 bond_reconfigure(bond
, s
);
254 bond_ref(const struct bond
*bond_
)
256 struct bond
*bond
= CONST_CAST(struct bond
*, bond_
);
259 ovs_refcount_ref(&bond
->ref_cnt
);
266 bond_unref(struct bond
*bond
)
268 struct bond_slave
*slave
;
270 if (!bond
|| ovs_refcount_unref_relaxed(&bond
->ref_cnt
) != 1) {
274 ovs_rwlock_wrlock(&rwlock
);
275 hmap_remove(all_bonds
, &bond
->hmap_node
);
276 ovs_rwlock_unlock(&rwlock
);
278 HMAP_FOR_EACH_POP (slave
, hmap_node
, &bond
->slaves
) {
279 /* Client owns 'slave->netdev'. */
283 hmap_destroy(&bond
->slaves
);
285 ovs_mutex_destroy(&bond
->mutex
);
287 /* Free bond resources. Remove existing post recirc rules. */
288 if (bond
->recirc_id
) {
289 recirc_free_id(bond
->recirc_id
);
294 update_recirc_rules__(bond
);
296 hmap_destroy(&bond
->pr_rule_ops
);
302 add_pr_rule(struct bond
*bond
, const struct match
*match
,
303 ofp_port_t out_ofport
, struct rule
**rule
)
305 uint32_t hash
= match_hash(match
, 0);
306 struct bond_pr_rule_op
*pr_op
;
308 HMAP_FOR_EACH_WITH_HASH(pr_op
, hmap_node
, hash
, &bond
->pr_rule_ops
) {
309 if (match_equal(&pr_op
->match
, match
)) {
311 pr_op
->out_ofport
= out_ofport
;
312 pr_op
->pr_rule
= rule
;
317 pr_op
= xmalloc(sizeof *pr_op
);
318 pr_op
->match
= *match
;
320 pr_op
->out_ofport
= out_ofport
;
321 pr_op
->pr_rule
= rule
;
322 hmap_insert(&bond
->pr_rule_ops
, &pr_op
->hmap_node
, hash
);
325 /* This function should almost never be called directly.
326 * 'update_recirc_rules()' should be called instead. Since
327 * this function modifies 'bond->pr_rule_ops', it is only
328 * safe when 'rwlock' is held.
330 * However, when the 'bond' is the only reference in the system,
331 * calling this function avoid acquiring lock only to satisfy
332 * lock annotation. Currently, only 'bond_unref()' calls
333 * this function directly. */
335 update_recirc_rules__(struct bond
*bond
)
338 struct bond_pr_rule_op
*pr_op
, *next_op
;
339 uint64_t ofpacts_stub
[128 / 8];
340 struct ofpbuf ofpacts
;
343 ofpbuf_use_stub(&ofpacts
, ofpacts_stub
, sizeof ofpacts_stub
);
345 HMAP_FOR_EACH(pr_op
, hmap_node
, &bond
->pr_rule_ops
) {
349 if (bond
->hash
&& bond
->recirc_id
) {
350 for (i
= 0; i
< BOND_BUCKETS
; i
++) {
351 struct bond_slave
*slave
= bond
->hash
[i
].slave
;
354 match_init_catchall(&match
);
355 match_set_recirc_id(&match
, bond
->recirc_id
);
356 match_set_dp_hash_masked(&match
, i
, BOND_MASK
);
358 add_pr_rule(bond
, &match
, slave
->ofp_port
,
359 &bond
->hash
[i
].pr_rule
);
364 HMAP_FOR_EACH_SAFE(pr_op
, next_op
, hmap_node
, &bond
->pr_rule_ops
) {
368 ofpbuf_clear(&ofpacts
);
369 ofpact_put_OUTPUT(&ofpacts
)->port
= pr_op
->out_ofport
;
370 error
= ofproto_dpif_add_internal_flow(bond
->ofproto
,
372 RECIRC_RULE_PRIORITY
, 0,
373 &ofpacts
, pr_op
->pr_rule
);
375 char *err_s
= match_to_string(&pr_op
->match
, NULL
,
376 RECIRC_RULE_PRIORITY
);
378 VLOG_ERR("failed to add post recirculation flow %s", err_s
);
384 error
= ofproto_dpif_delete_internal_flow(bond
->ofproto
,
386 RECIRC_RULE_PRIORITY
);
388 char *err_s
= match_to_string(&pr_op
->match
, NULL
,
389 RECIRC_RULE_PRIORITY
);
391 VLOG_ERR("failed to remove post recirculation flow %s", err_s
);
395 hmap_remove(&bond
->pr_rule_ops
, &pr_op
->hmap_node
);
396 *pr_op
->pr_rule
= NULL
;
402 ofpbuf_uninit(&ofpacts
);
406 update_recirc_rules(struct bond
*bond
)
407 OVS_REQ_RDLOCK(rwlock
)
409 update_recirc_rules__(bond
);
412 /* Updates 'bond''s overall configuration to 's'.
414 * The caller should register each slave on 'bond' by calling
415 * bond_slave_register(). This is optional if none of the slaves'
416 * configuration has changed. In any case it can't hurt.
418 * Returns true if the configuration has changed in such a way that requires
422 bond_reconfigure(struct bond
*bond
, const struct bond_settings
*s
)
424 bool revalidate
= false;
426 ovs_rwlock_wrlock(&rwlock
);
427 if (!bond
->name
|| strcmp(bond
->name
, s
->name
)) {
429 hmap_remove(all_bonds
, &bond
->hmap_node
);
432 bond
->name
= xstrdup(s
->name
);
433 hmap_insert(all_bonds
, &bond
->hmap_node
, hash_string(bond
->name
, 0));
436 bond
->updelay
= s
->up_delay
;
437 bond
->downdelay
= s
->down_delay
;
439 if (bond
->lacp_fallback_ab
!= s
->lacp_fallback_ab_cfg
) {
440 bond
->lacp_fallback_ab
= s
->lacp_fallback_ab_cfg
;
444 if (bond
->rebalance_interval
!= s
->rebalance_interval
) {
445 bond
->rebalance_interval
= s
->rebalance_interval
;
449 if (bond
->balance
!= s
->balance
) {
450 bond
->balance
= s
->balance
;
454 if (bond
->basis
!= s
->basis
) {
455 bond
->basis
= s
->basis
;
459 if (bond
->bond_revalidate
) {
461 bond
->bond_revalidate
= false;
464 if (bond
->balance
!= BM_AB
) {
465 if (!bond
->recirc_id
) {
466 bond
->recirc_id
= recirc_alloc_id(bond
->ofproto
);
468 } else if (bond
->recirc_id
) {
469 recirc_free_id(bond
->recirc_id
);
473 if (bond
->balance
== BM_AB
|| !bond
->hash
|| revalidate
) {
474 bond_entry_reset(bond
);
477 ovs_rwlock_unlock(&rwlock
);
481 static struct bond_slave
*
482 bond_find_slave_by_mac(const struct bond
*bond
, const struct eth_addr mac
)
484 struct bond_slave
*slave
;
486 /* Find the last active slave */
487 HMAP_FOR_EACH(slave
, hmap_node
, &bond
->slaves
) {
488 struct eth_addr slave_mac
;
490 if (netdev_get_etheraddr(slave
->netdev
, &slave_mac
)) {
494 if (eth_addr_equals(slave_mac
, mac
)) {
503 bond_active_slave_changed(struct bond
*bond
)
505 if (bond
->active_slave
) {
507 netdev_get_etheraddr(bond
->active_slave
->netdev
, &mac
);
508 bond
->active_slave_mac
= mac
;
510 bond
->active_slave_mac
= eth_addr_zero
;
512 bond
->active_slave_changed
= true;
513 seq_change(connectivity_seq_get());
517 bond_slave_set_netdev__(struct bond_slave
*slave
, struct netdev
*netdev
)
518 OVS_REQ_WRLOCK(rwlock
)
520 if (slave
->netdev
!= netdev
) {
521 slave
->netdev
= netdev
;
522 slave
->change_seq
= 0;
526 /* Registers 'slave_' as a slave of 'bond'. The 'slave_' pointer is an
527 * arbitrary client-provided pointer that uniquely identifies a slave within a
528 * bond. If 'slave_' already exists within 'bond' then this function
529 * reconfigures the existing slave.
531 * 'netdev' must be the network device that 'slave_' represents. It is owned
532 * by the client, so the client must not close it before either unregistering
533 * 'slave_' or destroying 'bond'.
536 bond_slave_register(struct bond
*bond
, void *slave_
,
537 ofp_port_t ofport
, struct netdev
*netdev
)
539 struct bond_slave
*slave
;
541 ovs_rwlock_wrlock(&rwlock
);
542 slave
= bond_slave_lookup(bond
, slave_
);
544 slave
= xzalloc(sizeof *slave
);
546 hmap_insert(&bond
->slaves
, &slave
->hmap_node
, hash_pointer(slave_
, 0));
549 slave
->ofp_port
= ofport
;
550 slave
->delay_expires
= LLONG_MAX
;
551 slave
->name
= xstrdup(netdev_get_name(netdev
));
552 bond
->bond_revalidate
= true;
554 slave
->enabled
= false;
555 bond_enable_slave(slave
, netdev_get_carrier(netdev
));
558 bond_slave_set_netdev__(slave
, netdev
);
561 slave
->name
= xstrdup(netdev_get_name(netdev
));
562 ovs_rwlock_unlock(&rwlock
);
565 /* Updates the network device to be used with 'slave_' to 'netdev'.
567 * This is useful if the caller closes and re-opens the network device
568 * registered with bond_slave_register() but doesn't need to change anything
571 bond_slave_set_netdev(struct bond
*bond
, void *slave_
, struct netdev
*netdev
)
573 struct bond_slave
*slave
;
575 ovs_rwlock_wrlock(&rwlock
);
576 slave
= bond_slave_lookup(bond
, slave_
);
578 bond_slave_set_netdev__(slave
, netdev
);
580 ovs_rwlock_unlock(&rwlock
);
583 /* Unregisters 'slave_' from 'bond'. If 'bond' does not contain such a slave
584 * then this function has no effect.
586 * Unregistering a slave invalidates all flows. */
588 bond_slave_unregister(struct bond
*bond
, const void *slave_
)
590 struct bond_slave
*slave
;
593 ovs_rwlock_wrlock(&rwlock
);
594 slave
= bond_slave_lookup(bond
, slave_
);
599 bond
->bond_revalidate
= true;
600 bond_enable_slave(slave
, false);
602 del_active
= bond
->active_slave
== slave
;
604 struct bond_entry
*e
;
605 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
606 if (e
->slave
== slave
) {
614 hmap_remove(&bond
->slaves
, &slave
->hmap_node
);
615 /* Client owns 'slave->netdev'. */
619 bond_choose_active_slave(bond
);
620 bond
->send_learning_packets
= true;
623 ovs_rwlock_unlock(&rwlock
);
626 /* Should be called on each slave in 'bond' before bond_run() to indicate
627 * whether or not 'slave_' may be enabled. This function is intended to allow
628 * other protocols to have some impact on bonding decisions. For example LACP
629 * or high level link monitoring protocols may decide that a given slave should
630 * not be able to send traffic. */
632 bond_slave_set_may_enable(struct bond
*bond
, void *slave_
, bool may_enable
)
634 ovs_rwlock_wrlock(&rwlock
);
635 bond_slave_lookup(bond
, slave_
)->may_enable
= may_enable
;
636 ovs_rwlock_unlock(&rwlock
);
639 /* Performs periodic maintenance on 'bond'.
641 * Returns true if the caller should revalidate its flows.
643 * The caller should check bond_should_send_learning_packets() afterward. */
645 bond_run(struct bond
*bond
, enum lacp_status lacp_status
)
647 struct bond_slave
*slave
;
650 ovs_rwlock_wrlock(&rwlock
);
651 if (bond
->lacp_status
!= lacp_status
) {
652 bond
->lacp_status
= lacp_status
;
653 bond
->bond_revalidate
= true;
656 /* Enable slaves based on link status and LACP feedback. */
657 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
658 bond_link_status_update(slave
);
659 slave
->change_seq
= seq_read(connectivity_seq_get());
661 if (!bond
->active_slave
|| !bond
->active_slave
->enabled
) {
662 bond_choose_active_slave(bond
);
665 revalidate
= bond
->bond_revalidate
;
666 bond
->bond_revalidate
= false;
667 ovs_rwlock_unlock(&rwlock
);
672 /* Causes poll_block() to wake up when 'bond' needs something to be done. */
674 bond_wait(struct bond
*bond
)
676 struct bond_slave
*slave
;
678 ovs_rwlock_rdlock(&rwlock
);
679 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
680 if (slave
->delay_expires
!= LLONG_MAX
) {
681 poll_timer_wait_until(slave
->delay_expires
);
684 seq_wait(connectivity_seq_get(), slave
->change_seq
);
687 if (bond
->bond_revalidate
) {
688 poll_immediate_wake();
690 ovs_rwlock_unlock(&rwlock
);
692 /* We don't wait for bond->next_rebalance because rebalancing can only run
693 * at a flow account checkpoint. ofproto does checkpointing on its own
694 * schedule and bond_rebalance() gets called afterward, so we'd just be
695 * waking up for no purpose. */
698 /* MAC learning table interaction. */
701 may_send_learning_packets(const struct bond
*bond
)
703 return ((bond
->lacp_status
== LACP_DISABLED
704 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_AB
))
705 || (bond
->lacp_fallback_ab
&& bond
->lacp_status
== LACP_CONFIGURED
))
706 && bond
->active_slave
;
709 /* Returns true if 'bond' needs the client to send out packets to assist with
710 * MAC learning on 'bond'. If this function returns true, then the client
711 * should iterate through its MAC learning table for the bridge on which 'bond'
712 * is located. For each MAC that has been learned on a port other than 'bond',
713 * it should call bond_compose_learning_packet().
715 * This function will only return true if 'bond' is in SLB or active-backup
716 * mode and LACP is not negotiated. Otherwise sending learning packets isn't
719 * Calling this function resets the state that it checks. */
721 bond_should_send_learning_packets(struct bond
*bond
)
725 ovs_rwlock_wrlock(&rwlock
);
726 send
= bond
->send_learning_packets
&& may_send_learning_packets(bond
);
727 bond
->send_learning_packets
= false;
728 ovs_rwlock_unlock(&rwlock
);
732 /* Sends a gratuitous learning packet on 'bond' from 'eth_src' on 'vlan'.
734 * See bond_should_send_learning_packets() for description of usage. The
735 * caller should send the composed packet on the port associated with
736 * port_aux and takes ownership of the returned ofpbuf. */
738 bond_compose_learning_packet(struct bond
*bond
, const struct eth_addr eth_src
,
739 uint16_t vlan
, void **port_aux
)
741 struct bond_slave
*slave
;
742 struct dp_packet
*packet
;
745 ovs_rwlock_rdlock(&rwlock
);
746 ovs_assert(may_send_learning_packets(bond
));
747 memset(&flow
, 0, sizeof flow
);
748 flow
.dl_src
= eth_src
;
749 slave
= choose_output_slave(bond
, &flow
, NULL
, vlan
);
751 packet
= dp_packet_new(0);
752 compose_rarp(packet
, eth_src
);
754 eth_push_vlan(packet
, htons(ETH_TYPE_VLAN
), htons(vlan
));
757 *port_aux
= slave
->aux
;
758 ovs_rwlock_unlock(&rwlock
);
762 /* Checks whether a packet that arrived on 'slave_' within 'bond', with an
763 * Ethernet destination address of 'eth_dst', should be admitted.
765 * The return value is one of the following:
767 * - BV_ACCEPT: Admit the packet.
769 * - BV_DROP: Drop the packet.
771 * - BV_DROP_IF_MOVED: Consult the MAC learning table for the packet's
772 * Ethernet source address and VLAN. If there is none, or if the packet
773 * is on the learned port, then admit the packet. If a different port has
774 * been learned, however, drop the packet (and do not use it for MAC
778 bond_check_admissibility(struct bond
*bond
, const void *slave_
,
779 const struct eth_addr eth_dst
)
781 enum bond_verdict verdict
= BV_DROP
;
782 struct bond_slave
*slave
;
784 ovs_rwlock_rdlock(&rwlock
);
785 slave
= bond_slave_lookup(bond
, slave_
);
790 /* LACP bonds have very loose admissibility restrictions because we can
791 * assume the remote switch is aware of the bond and will "do the right
792 * thing". However, as a precaution we drop packets on disabled slaves
793 * because no correctly implemented partner switch should be sending
796 * If LACP is configured, but LACP negotiations have been unsuccessful, we
797 * drop all incoming traffic except if lacp_fallback_ab is enabled. */
798 switch (bond
->lacp_status
) {
799 case LACP_NEGOTIATED
:
800 verdict
= slave
->enabled
? BV_ACCEPT
: BV_DROP
;
802 case LACP_CONFIGURED
:
803 if (!bond
->lacp_fallback_ab
) {
808 if (bond
->balance
== BM_TCP
) {
814 /* Drop all multicast packets on inactive slaves. */
815 if (eth_addr_is_multicast(eth_dst
)) {
816 if (bond
->active_slave
!= slave
) {
821 switch (bond
->balance
) {
823 /* TCP balanced bonds require successful LACP negotiations. Based on the
824 * above check, LACP is off or lacp_fallback_ab is true on this bond.
825 * If lacp_fallback_ab is true fall through to BM_AB case else, we
826 * drop all incoming traffic. */
827 if (!bond
->lacp_fallback_ab
) {
833 /* Drop all packets which arrive on backup slaves. This is similar to
834 * how Linux bonding handles active-backup bonds. */
835 if (bond
->active_slave
!= slave
) {
836 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
838 VLOG_DBG_RL(&rl
, "active-backup bond received packet on backup"
839 " slave (%s) destined for " ETH_ADDR_FMT
,
840 slave
->name
, ETH_ADDR_ARGS(eth_dst
));
847 /* Drop all packets for which we have learned a different input port,
848 * because we probably sent the packet on one slave and got it back on
849 * the other. Gratuitous ARP packets are an exception to this rule:
850 * the host has moved to another switch. The exception to the
851 * exception is if we locked the learning table to avoid reflections on
853 verdict
= BV_DROP_IF_MOVED
;
859 ovs_rwlock_unlock(&rwlock
);
864 /* Returns the slave (registered on 'bond' by bond_slave_register()) to which
865 * a packet with the given 'flow' and 'vlan' should be forwarded. Returns
866 * NULL if the packet should be dropped because no slaves are enabled.
868 * 'vlan' is not necessarily the same as 'flow->vlan_tci'. First, 'vlan'
869 * should be a VID only (i.e. excluding the PCP bits). Second,
870 * 'flow->vlan_tci' is the VLAN TCI that appeared on the packet (so it will be
871 * nonzero only for trunk ports), whereas 'vlan' is the logical VLAN that the
872 * packet belongs to (so for an access port it will be the access port's VLAN).
874 * If 'wc' is non-NULL, bitwise-OR's 'wc' with the set of bits that were
875 * significant in the selection. At some point earlier, 'wc' should
876 * have been initialized (e.g., by flow_wildcards_init_catchall()).
879 bond_choose_output_slave(struct bond
*bond
, const struct flow
*flow
,
880 struct flow_wildcards
*wc
, uint16_t vlan
)
882 struct bond_slave
*slave
;
885 ovs_rwlock_rdlock(&rwlock
);
886 slave
= choose_output_slave(bond
, flow
, wc
, vlan
);
887 aux
= slave
? slave
->aux
: NULL
;
888 ovs_rwlock_unlock(&rwlock
);
895 bond_entry_account(struct bond_entry
*entry
, uint64_t rule_tx_bytes
)
896 OVS_REQ_WRLOCK(rwlock
)
901 delta
= rule_tx_bytes
- entry
->pr_tx_bytes
;
902 entry
->tx_bytes
+= delta
;
903 entry
->pr_tx_bytes
= rule_tx_bytes
;
907 /* Maintain bond stats using post recirculation rule byte counters.*/
909 bond_recirculation_account(struct bond
*bond
)
910 OVS_REQ_WRLOCK(rwlock
)
914 for (i
=0; i
<=BOND_MASK
; i
++) {
915 struct bond_entry
*entry
= &bond
->hash
[i
];
916 struct rule
*rule
= entry
->pr_rule
;
919 uint64_t n_packets OVS_UNUSED
;
920 long long int used OVS_UNUSED
;
923 rule
->ofproto
->ofproto_class
->rule_get_stats(
924 rule
, &n_packets
, &n_bytes
, &used
);
925 bond_entry_account(entry
, n_bytes
);
931 bond_may_recirc(const struct bond
*bond
)
933 return bond
->balance
== BM_TCP
&& bond
->recirc_id
;
937 bond_update_post_recirc_rules__(struct bond
* bond
, const bool force
)
938 OVS_REQ_WRLOCK(rwlock
)
940 struct bond_entry
*e
;
941 bool update_rules
= force
; /* Always update rules if caller forces it. */
943 /* Make sure all bond entries are populated */
944 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
945 if (!e
->slave
|| !e
->slave
->enabled
) {
947 e
->slave
= CONTAINER_OF(hmap_random_node(&bond
->slaves
),
948 struct bond_slave
, hmap_node
);
949 if (!e
->slave
->enabled
) {
950 e
->slave
= bond
->active_slave
;
956 update_recirc_rules(bond
);
961 bond_update_post_recirc_rules(struct bond
*bond
, uint32_t *recirc_id
,
962 uint32_t *hash_basis
)
964 bool may_recirc
= bond_may_recirc(bond
);
967 /* To avoid unnecessary locking, bond_may_recirc() is first
968 * called outside of the 'rwlock'. After acquiring the lock,
969 * check again to make sure bond configuration has not been changed. */
970 ovs_rwlock_wrlock(&rwlock
);
971 may_recirc
= bond_may_recirc(bond
);
973 *recirc_id
= bond
->recirc_id
;
974 *hash_basis
= bond
->basis
;
975 bond_update_post_recirc_rules__(bond
, false);
977 ovs_rwlock_unlock(&rwlock
);
981 *recirc_id
= *hash_basis
= 0;
989 bond_is_balanced(const struct bond
*bond
) OVS_REQ_RDLOCK(rwlock
)
991 return bond
->rebalance_interval
992 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_TCP
);
995 /* Notifies 'bond' that 'n_bytes' bytes were sent in 'flow' within 'vlan'. */
997 bond_account(struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
,
1000 ovs_rwlock_wrlock(&rwlock
);
1001 if (bond_is_balanced(bond
)) {
1002 lookup_bond_entry(bond
, flow
, vlan
)->tx_bytes
+= n_bytes
;
1004 ovs_rwlock_unlock(&rwlock
);
1007 static struct bond_slave
*
1008 bond_slave_from_bal_node(struct ovs_list
*bal
) OVS_REQ_RDLOCK(rwlock
)
1010 return CONTAINER_OF(bal
, struct bond_slave
, bal_node
);
1014 log_bals(struct bond
*bond
, const struct ovs_list
*bals
)
1015 OVS_REQ_RDLOCK(rwlock
)
1017 if (VLOG_IS_DBG_ENABLED()) {
1018 struct ds ds
= DS_EMPTY_INITIALIZER
;
1019 const struct bond_slave
*slave
;
1021 LIST_FOR_EACH (slave
, bal_node
, bals
) {
1023 ds_put_char(&ds
, ',');
1025 ds_put_format(&ds
, " %s %"PRIu64
"kB",
1026 slave
->name
, slave
->tx_bytes
/ 1024);
1028 if (!slave
->enabled
) {
1029 ds_put_cstr(&ds
, " (disabled)");
1031 if (!ovs_list_is_empty(&slave
->entries
)) {
1032 struct bond_entry
*e
;
1034 ds_put_cstr(&ds
, " (");
1035 LIST_FOR_EACH (e
, list_node
, &slave
->entries
) {
1036 if (&e
->list_node
!= ovs_list_front(&slave
->entries
)) {
1037 ds_put_cstr(&ds
, " + ");
1039 ds_put_format(&ds
, "h%"PRIdPTR
": %"PRIu64
"kB",
1040 e
- bond
->hash
, e
->tx_bytes
/ 1024);
1042 ds_put_cstr(&ds
, ")");
1045 VLOG_DBG("bond %s:%s", bond
->name
, ds_cstr(&ds
));
1050 /* Shifts 'hash' from its current slave to 'to'. */
1052 bond_shift_load(struct bond_entry
*hash
, struct bond_slave
*to
)
1053 OVS_REQ_WRLOCK(rwlock
)
1055 struct bond_slave
*from
= hash
->slave
;
1056 struct bond
*bond
= from
->bond
;
1057 uint64_t delta
= hash
->tx_bytes
;
1059 VLOG_INFO("bond %s: shift %"PRIu64
"kB of load (with hash %"PRIdPTR
") "
1060 "from %s to %s (now carrying %"PRIu64
"kB and "
1061 "%"PRIu64
"kB load, respectively)",
1062 bond
->name
, delta
/ 1024, hash
- bond
->hash
,
1063 from
->name
, to
->name
,
1064 (from
->tx_bytes
- delta
) / 1024,
1065 (to
->tx_bytes
+ delta
) / 1024);
1067 /* Shift load away from 'from' to 'to'. */
1068 from
->tx_bytes
-= delta
;
1069 to
->tx_bytes
+= delta
;
1071 /* Arrange for flows to be revalidated. */
1073 bond
->bond_revalidate
= true;
1076 /* Picks and returns a bond_entry to migrate from 'from' (the most heavily
1077 * loaded bond slave) to a bond slave that has 'to_tx_bytes' bytes of load,
1078 * given that doing so must decrease the ratio of the load on the two slaves by
1079 * at least 0.1. Returns NULL if there is no appropriate entry.
1081 * The list of entries isn't sorted. I don't know of a reason to prefer to
1082 * shift away small hashes or large hashes. */
1083 static struct bond_entry
*
1084 choose_entry_to_migrate(const struct bond_slave
*from
, uint64_t to_tx_bytes
)
1085 OVS_REQ_WRLOCK(rwlock
)
1087 struct bond_entry
*e
;
1089 if (ovs_list_is_short(&from
->entries
)) {
1090 /* 'from' carries no more than one MAC hash, so shifting load away from
1091 * it would be pointless. */
1095 LIST_FOR_EACH (e
, list_node
, &from
->entries
) {
1096 uint64_t delta
= e
->tx_bytes
; /* The amount to rebalance. */
1097 uint64_t ideal_tx_bytes
= (from
->tx_bytes
+ to_tx_bytes
)/2;
1098 /* Note, the ideal traffic is the mid point
1099 * between 'from' and 'to'. This value does
1100 * not change by rebalancing. */
1101 uint64_t new_low
; /* The lower bandwidth between 'to' and 'from'
1102 after rebalancing. */
1104 new_low
= MIN(from
->tx_bytes
- delta
, to_tx_bytes
+ delta
);
1106 if ((new_low
> to_tx_bytes
) &&
1107 (new_low
- to_tx_bytes
>= (ideal_tx_bytes
- to_tx_bytes
) / 10)) {
1108 /* Only rebalance if the new 'low' is closer to to the mid point,
1109 * and the improvement exceeds 10% of current traffic
1110 * deviation from the ideal split.
1112 * The improvement on the 'high' side is always the same as the
1113 * 'low' side. Thus consider 'low' side is sufficient. */
1121 /* Inserts 'slave' into 'bals' so that descending order of 'tx_bytes' is
1124 insert_bal(struct ovs_list
*bals
, struct bond_slave
*slave
)
1126 struct bond_slave
*pos
;
1128 LIST_FOR_EACH (pos
, bal_node
, bals
) {
1129 if (slave
->tx_bytes
> pos
->tx_bytes
) {
1133 ovs_list_insert(&pos
->bal_node
, &slave
->bal_node
);
1136 /* Removes 'slave' from its current list and then inserts it into 'bals' so
1137 * that descending order of 'tx_bytes' is maintained. */
1139 reinsert_bal(struct ovs_list
*bals
, struct bond_slave
*slave
)
1141 ovs_list_remove(&slave
->bal_node
);
1142 insert_bal(bals
, slave
);
1145 /* If 'bond' needs rebalancing, does so.
1147 * The caller should have called bond_account() for each active flow, or in case
1148 * of recirculation is used, have called bond_recirculation_account(bond),
1149 * to ensure that flow data is consistently accounted at this point.
1152 bond_rebalance(struct bond
*bond
)
1154 struct bond_slave
*slave
;
1155 struct bond_entry
*e
;
1156 struct ovs_list bals
;
1157 bool rebalanced
= false;
1160 ovs_rwlock_wrlock(&rwlock
);
1161 if (!bond_is_balanced(bond
) || time_msec() < bond
->next_rebalance
) {
1164 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1166 use_recirc
= bond
->ofproto
->backer
->support
.odp
.recirc
&&
1167 bond_may_recirc(bond
);
1170 bond_recirculation_account(bond
);
1173 /* Add each bond_entry to its slave's 'entries' list.
1174 * Compute each slave's tx_bytes as the sum of its entries' tx_bytes. */
1175 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1176 slave
->tx_bytes
= 0;
1177 ovs_list_init(&slave
->entries
);
1179 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
1180 if (e
->slave
&& e
->tx_bytes
) {
1181 e
->slave
->tx_bytes
+= e
->tx_bytes
;
1182 ovs_list_push_back(&e
->slave
->entries
, &e
->list_node
);
1186 /* Add enabled slaves to 'bals' in descending order of tx_bytes.
1188 * XXX This is O(n**2) in the number of slaves but it could be O(n lg n)
1189 * with a proper list sort algorithm. */
1190 ovs_list_init(&bals
);
1191 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1192 if (slave
->enabled
) {
1193 insert_bal(&bals
, slave
);
1196 log_bals(bond
, &bals
);
1198 /* Shift load from the most-loaded slaves to the least-loaded slaves. */
1199 while (!ovs_list_is_short(&bals
)) {
1200 struct bond_slave
*from
= bond_slave_from_bal_node(ovs_list_front(&bals
));
1201 struct bond_slave
*to
= bond_slave_from_bal_node(ovs_list_back(&bals
));
1204 overload
= from
->tx_bytes
- to
->tx_bytes
;
1205 if (overload
< to
->tx_bytes
>> 5 || overload
< 100000) {
1206 /* The extra load on 'from' (and all less-loaded slaves), compared
1207 * to that of 'to' (the least-loaded slave), is less than ~3%, or
1208 * it is less than ~1Mbps. No point in rebalancing. */
1212 /* 'from' is carrying significantly more load than 'to'. Pick a hash
1213 * to move from 'from' to 'to'. */
1214 e
= choose_entry_to_migrate(from
, to
->tx_bytes
);
1216 bond_shift_load(e
, to
);
1218 /* Delete element from from->entries.
1220 * We don't add the element to to->hashes. That would only allow
1221 * 'e' to be migrated to another slave in this rebalancing run, and
1222 * there is no point in doing that. */
1223 ovs_list_remove(&e
->list_node
);
1225 /* Re-sort 'bals'. */
1226 reinsert_bal(&bals
, from
);
1227 reinsert_bal(&bals
, to
);
1230 /* Can't usefully migrate anything away from 'from'.
1231 * Don't reconsider it. */
1232 ovs_list_remove(&from
->bal_node
);
1236 /* Implement exponentially weighted moving average. A weight of 1/2 causes
1237 * historical data to decay to <1% in 7 rebalancing runs. 1,000,000 bytes
1238 * take 20 rebalancing runs to decay to 0 and get deleted entirely. */
1239 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
1243 if (use_recirc
&& rebalanced
) {
1244 bond_update_post_recirc_rules__(bond
,true);
1248 ovs_rwlock_unlock(&rwlock
);
1251 /* Bonding unixctl user interface functions. */
1253 static struct bond
*
1254 bond_find(const char *name
) OVS_REQ_RDLOCK(rwlock
)
1258 HMAP_FOR_EACH_WITH_HASH (bond
, hmap_node
, hash_string(name
, 0),
1260 if (!strcmp(bond
->name
, name
)) {
1267 static struct bond_slave
*
1268 bond_lookup_slave(struct bond
*bond
, const char *slave_name
)
1270 struct bond_slave
*slave
;
1272 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1273 if (!strcmp(slave
->name
, slave_name
)) {
1281 bond_unixctl_list(struct unixctl_conn
*conn
,
1282 int argc OVS_UNUSED
, const char *argv
[] OVS_UNUSED
,
1283 void *aux OVS_UNUSED
)
1285 struct ds ds
= DS_EMPTY_INITIALIZER
;
1286 const struct bond
*bond
;
1288 ds_put_cstr(&ds
, "bond\ttype\trecircID\tslaves\n");
1290 ovs_rwlock_rdlock(&rwlock
);
1291 HMAP_FOR_EACH (bond
, hmap_node
, all_bonds
) {
1292 const struct bond_slave
*slave
;
1295 ds_put_format(&ds
, "%s\t%s\t%d\t", bond
->name
,
1296 bond_mode_to_string(bond
->balance
), bond
->recirc_id
);
1299 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1301 ds_put_cstr(&ds
, ", ");
1303 ds_put_cstr(&ds
, slave
->name
);
1305 ds_put_char(&ds
, '\n');
1307 ovs_rwlock_unlock(&rwlock
);
1308 unixctl_command_reply(conn
, ds_cstr(&ds
));
1313 bond_print_details(struct ds
*ds
, const struct bond
*bond
)
1314 OVS_REQ_RDLOCK(rwlock
)
1316 struct shash slave_shash
= SHASH_INITIALIZER(&slave_shash
);
1317 const struct shash_node
**sorted_slaves
= NULL
;
1318 const struct bond_slave
*slave
;
1323 ds_put_format(ds
, "---- %s ----\n", bond
->name
);
1324 ds_put_format(ds
, "bond_mode: %s\n",
1325 bond_mode_to_string(bond
->balance
));
1327 may_recirc
= bond_may_recirc(bond
);
1328 recirc_id
= bond
->recirc_id
;
1329 ds_put_format(ds
, "bond may use recirculation: %s, Recirc-ID : %d\n",
1330 may_recirc
? "yes" : "no", may_recirc
? recirc_id
: -1);
1332 ds_put_format(ds
, "bond-hash-basis: %"PRIu32
"\n", bond
->basis
);
1334 ds_put_format(ds
, "updelay: %d ms\n", bond
->updelay
);
1335 ds_put_format(ds
, "downdelay: %d ms\n", bond
->downdelay
);
1337 if (bond_is_balanced(bond
)) {
1338 ds_put_format(ds
, "next rebalance: %lld ms\n",
1339 bond
->next_rebalance
- time_msec());
1342 ds_put_cstr(ds
, "lacp_status: ");
1343 switch (bond
->lacp_status
) {
1344 case LACP_NEGOTIATED
:
1345 ds_put_cstr(ds
, "negotiated\n");
1347 case LACP_CONFIGURED
:
1348 ds_put_cstr(ds
, "configured\n");
1351 ds_put_cstr(ds
, "off\n");
1354 ds_put_cstr(ds
, "<unknown>\n");
1358 ds_put_format(ds
, "lacp_fallback_ab: %s\n",
1359 bond
->lacp_fallback_ab
? "true" : "false");
1361 ds_put_cstr(ds
, "active slave mac: ");
1362 ds_put_format(ds
, ETH_ADDR_FMT
, ETH_ADDR_ARGS(bond
->active_slave_mac
));
1363 slave
= bond_find_slave_by_mac(bond
, bond
->active_slave_mac
);
1364 ds_put_format(ds
,"(%s)\n", slave
? slave
->name
: "none");
1366 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1367 shash_add(&slave_shash
, slave
->name
, slave
);
1369 sorted_slaves
= shash_sort(&slave_shash
);
1371 for (i
= 0; i
< shash_count(&slave_shash
); i
++) {
1372 struct bond_entry
*be
;
1374 slave
= sorted_slaves
[i
]->data
;
1377 ds_put_format(ds
, "\nslave %s: %s\n",
1378 slave
->name
, slave
->enabled
? "enabled" : "disabled");
1379 if (slave
== bond
->active_slave
) {
1380 ds_put_cstr(ds
, "\tactive slave\n");
1382 if (slave
->delay_expires
!= LLONG_MAX
) {
1383 ds_put_format(ds
, "\t%s expires in %lld ms\n",
1384 slave
->enabled
? "downdelay" : "updelay",
1385 slave
->delay_expires
- time_msec());
1388 ds_put_format(ds
, "\tmay_enable: %s\n",
1389 slave
->may_enable
? "true" : "false");
1391 if (!bond_is_balanced(bond
)) {
1396 for (be
= bond
->hash
; be
<= &bond
->hash
[BOND_MASK
]; be
++) {
1397 int hash
= be
- bond
->hash
;
1400 if (be
->slave
!= slave
) {
1404 be_tx_k
= be
->tx_bytes
/ 1024;
1406 ds_put_format(ds
, "\thash %d: %"PRIu64
" kB load\n",
1410 /* XXX How can we list the MACs assigned to hashes of SLB bonds? */
1413 shash_destroy(&slave_shash
);
1414 free(sorted_slaves
);
1415 ds_put_cstr(ds
, "\n");
1419 bond_unixctl_show(struct unixctl_conn
*conn
,
1420 int argc
, const char *argv
[],
1421 void *aux OVS_UNUSED
)
1423 struct ds ds
= DS_EMPTY_INITIALIZER
;
1425 ovs_rwlock_rdlock(&rwlock
);
1427 const struct bond
*bond
= bond_find(argv
[1]);
1430 unixctl_command_reply_error(conn
, "no such bond");
1433 bond_print_details(&ds
, bond
);
1435 const struct bond
*bond
;
1437 HMAP_FOR_EACH (bond
, hmap_node
, all_bonds
) {
1438 bond_print_details(&ds
, bond
);
1442 unixctl_command_reply(conn
, ds_cstr(&ds
));
1446 ovs_rwlock_unlock(&rwlock
);
1450 bond_unixctl_migrate(struct unixctl_conn
*conn
,
1451 int argc OVS_UNUSED
, const char *argv
[],
1452 void *aux OVS_UNUSED
)
1454 const char *bond_s
= argv
[1];
1455 const char *hash_s
= argv
[2];
1456 const char *slave_s
= argv
[3];
1458 struct bond_slave
*slave
;
1459 struct bond_entry
*entry
;
1462 ovs_rwlock_wrlock(&rwlock
);
1463 bond
= bond_find(bond_s
);
1465 unixctl_command_reply_error(conn
, "no such bond");
1469 if (bond
->balance
!= BM_SLB
) {
1470 unixctl_command_reply_error(conn
, "not an SLB bond");
1474 if (strspn(hash_s
, "0123456789") == strlen(hash_s
)) {
1475 hash
= atoi(hash_s
) & BOND_MASK
;
1477 unixctl_command_reply_error(conn
, "bad hash");
1481 slave
= bond_lookup_slave(bond
, slave_s
);
1483 unixctl_command_reply_error(conn
, "no such slave");
1487 if (!slave
->enabled
) {
1488 unixctl_command_reply_error(conn
, "cannot migrate to disabled slave");
1492 entry
= &bond
->hash
[hash
];
1493 bond
->bond_revalidate
= true;
1494 entry
->slave
= slave
;
1495 unixctl_command_reply(conn
, "migrated");
1498 ovs_rwlock_unlock(&rwlock
);
1502 bond_unixctl_set_active_slave(struct unixctl_conn
*conn
,
1503 int argc OVS_UNUSED
, const char *argv
[],
1504 void *aux OVS_UNUSED
)
1506 const char *bond_s
= argv
[1];
1507 const char *slave_s
= argv
[2];
1509 struct bond_slave
*slave
;
1511 ovs_rwlock_wrlock(&rwlock
);
1512 bond
= bond_find(bond_s
);
1514 unixctl_command_reply_error(conn
, "no such bond");
1518 slave
= bond_lookup_slave(bond
, slave_s
);
1520 unixctl_command_reply_error(conn
, "no such slave");
1524 if (!slave
->enabled
) {
1525 unixctl_command_reply_error(conn
, "cannot make disabled slave active");
1529 if (bond
->active_slave
!= slave
) {
1530 bond
->bond_revalidate
= true;
1531 bond
->active_slave
= slave
;
1532 VLOG_INFO("bond %s: active interface is now %s",
1533 bond
->name
, slave
->name
);
1534 bond
->send_learning_packets
= true;
1535 unixctl_command_reply(conn
, "done");
1536 bond_active_slave_changed(bond
);
1538 unixctl_command_reply(conn
, "no change");
1541 ovs_rwlock_unlock(&rwlock
);
1545 enable_slave(struct unixctl_conn
*conn
, const char *argv
[], bool enable
)
1547 const char *bond_s
= argv
[1];
1548 const char *slave_s
= argv
[2];
1550 struct bond_slave
*slave
;
1552 ovs_rwlock_wrlock(&rwlock
);
1553 bond
= bond_find(bond_s
);
1555 unixctl_command_reply_error(conn
, "no such bond");
1559 slave
= bond_lookup_slave(bond
, slave_s
);
1561 unixctl_command_reply_error(conn
, "no such slave");
1565 bond_enable_slave(slave
, enable
);
1566 unixctl_command_reply(conn
, enable
? "enabled" : "disabled");
1569 ovs_rwlock_unlock(&rwlock
);
1573 bond_unixctl_enable_slave(struct unixctl_conn
*conn
,
1574 int argc OVS_UNUSED
, const char *argv
[],
1575 void *aux OVS_UNUSED
)
1577 enable_slave(conn
, argv
, true);
1581 bond_unixctl_disable_slave(struct unixctl_conn
*conn
,
1582 int argc OVS_UNUSED
, const char *argv
[],
1583 void *aux OVS_UNUSED
)
1585 enable_slave(conn
, argv
, false);
1589 bond_unixctl_hash(struct unixctl_conn
*conn
, int argc
, const char *argv
[],
1590 void *aux OVS_UNUSED
)
1592 const char *mac_s
= argv
[1];
1593 const char *vlan_s
= argc
> 2 ? argv
[2] : NULL
;
1594 const char *basis_s
= argc
> 3 ? argv
[3] : NULL
;
1595 struct eth_addr mac
;
1602 if (!ovs_scan(vlan_s
, "%u", &vlan
)) {
1603 unixctl_command_reply_error(conn
, "invalid vlan");
1611 if (!ovs_scan(basis_s
, "%"SCNu32
, &basis
)) {
1612 unixctl_command_reply_error(conn
, "invalid basis");
1619 if (ovs_scan(mac_s
, ETH_ADDR_SCAN_FMT
, ETH_ADDR_SCAN_ARGS(mac
))) {
1620 hash
= bond_hash_src(mac
, vlan
, basis
) & BOND_MASK
;
1622 hash_cstr
= xasprintf("%u", hash
);
1623 unixctl_command_reply(conn
, hash_cstr
);
1626 unixctl_command_reply_error(conn
, "invalid mac");
1633 unixctl_command_register("bond/list", "", 0, 0, bond_unixctl_list
, NULL
);
1634 unixctl_command_register("bond/show", "[port]", 0, 1, bond_unixctl_show
,
1636 unixctl_command_register("bond/migrate", "port hash slave", 3, 3,
1637 bond_unixctl_migrate
, NULL
);
1638 unixctl_command_register("bond/set-active-slave", "port slave", 2, 2,
1639 bond_unixctl_set_active_slave
, NULL
);
1640 unixctl_command_register("bond/enable-slave", "port slave", 2, 2,
1641 bond_unixctl_enable_slave
, NULL
);
1642 unixctl_command_register("bond/disable-slave", "port slave", 2, 2,
1643 bond_unixctl_disable_slave
, NULL
);
1644 unixctl_command_register("bond/hash", "mac [vlan] [basis]", 1, 3,
1645 bond_unixctl_hash
, NULL
);
1649 bond_entry_reset(struct bond
*bond
)
1651 if (bond
->balance
!= BM_AB
) {
1652 size_t hash_len
= BOND_BUCKETS
* sizeof *bond
->hash
;
1655 bond
->hash
= xmalloc(hash_len
);
1657 memset(bond
->hash
, 0, hash_len
);
1659 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1663 /* Remove existing post recirc rules. */
1664 update_recirc_rules(bond
);
1668 static struct bond_slave
*
1669 bond_slave_lookup(struct bond
*bond
, const void *slave_
)
1671 struct bond_slave
*slave
;
1673 HMAP_FOR_EACH_IN_BUCKET (slave
, hmap_node
, hash_pointer(slave_
, 0),
1675 if (slave
->aux
== slave_
) {
1684 bond_enable_slave(struct bond_slave
*slave
, bool enable
)
1686 slave
->delay_expires
= LLONG_MAX
;
1687 if (enable
!= slave
->enabled
) {
1688 slave
->bond
->bond_revalidate
= true;
1689 slave
->enabled
= enable
;
1691 ovs_mutex_lock(&slave
->bond
->mutex
);
1693 ovs_list_insert(&slave
->bond
->enabled_slaves
, &slave
->list_node
);
1695 ovs_list_remove(&slave
->list_node
);
1697 ovs_mutex_unlock(&slave
->bond
->mutex
);
1699 VLOG_INFO("interface %s: %s", slave
->name
,
1700 slave
->enabled
? "enabled" : "disabled");
1705 bond_link_status_update(struct bond_slave
*slave
)
1707 struct bond
*bond
= slave
->bond
;
1710 up
= netdev_get_carrier(slave
->netdev
) && slave
->may_enable
;
1711 if ((up
== slave
->enabled
) != (slave
->delay_expires
== LLONG_MAX
)) {
1712 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1713 VLOG_INFO_RL(&rl
, "interface %s: link state %s",
1714 slave
->name
, up
? "up" : "down");
1715 if (up
== slave
->enabled
) {
1716 slave
->delay_expires
= LLONG_MAX
;
1717 VLOG_INFO_RL(&rl
, "interface %s: will not be %s",
1718 slave
->name
, up
? "disabled" : "enabled");
1720 int delay
= (bond
->lacp_status
!= LACP_DISABLED
? 0
1721 : up
? bond
->updelay
: bond
->downdelay
);
1722 slave
->delay_expires
= time_msec() + delay
;
1724 VLOG_INFO_RL(&rl
, "interface %s: will be %s if it stays %s "
1727 up
? "enabled" : "disabled",
1734 if (time_msec() >= slave
->delay_expires
) {
1735 bond_enable_slave(slave
, up
);
1740 bond_hash_src(const struct eth_addr mac
, uint16_t vlan
, uint32_t basis
)
1742 return hash_mac(mac
, vlan
, basis
);
1746 bond_hash_tcp(const struct flow
*flow
, uint16_t vlan
, uint32_t basis
)
1748 struct flow hash_flow
= *flow
;
1749 hash_flow
.vlans
[0].tci
= htons(vlan
);
1751 /* The symmetric quality of this hash function is not required, but
1752 * flow_hash_symmetric_l4 already exists, and is sufficient for our
1753 * purposes, so we use it out of convenience. */
1754 return flow_hash_symmetric_l4(&hash_flow
, basis
);
1758 bond_hash(const struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
)
1760 ovs_assert(bond
->balance
== BM_TCP
|| bond
->balance
== BM_SLB
);
1762 return (bond
->balance
== BM_TCP
1763 ? bond_hash_tcp(flow
, vlan
, bond
->basis
)
1764 : bond_hash_src(flow
->dl_src
, vlan
, bond
->basis
));
1767 static struct bond_entry
*
1768 lookup_bond_entry(const struct bond
*bond
, const struct flow
*flow
,
1771 return &bond
->hash
[bond_hash(bond
, flow
, vlan
) & BOND_MASK
];
1774 /* Selects and returns an enabled slave from the 'enabled_slaves' list
1775 * in a round-robin fashion. If the 'enabled_slaves' list is empty,
1777 static struct bond_slave
*
1778 get_enabled_slave(struct bond
*bond
)
1780 struct ovs_list
*node
;
1782 ovs_mutex_lock(&bond
->mutex
);
1783 if (ovs_list_is_empty(&bond
->enabled_slaves
)) {
1784 ovs_mutex_unlock(&bond
->mutex
);
1788 node
= ovs_list_pop_front(&bond
->enabled_slaves
);
1789 ovs_list_push_back(&bond
->enabled_slaves
, node
);
1790 ovs_mutex_unlock(&bond
->mutex
);
1792 return CONTAINER_OF(node
, struct bond_slave
, list_node
);
1795 static struct bond_slave
*
1796 choose_output_slave(const struct bond
*bond
, const struct flow
*flow
,
1797 struct flow_wildcards
*wc
, uint16_t vlan
)
1799 struct bond_entry
*e
;
1802 balance
= bond
->balance
;
1803 if (bond
->lacp_status
== LACP_CONFIGURED
) {
1804 /* LACP has been configured on this bond but negotiations were
1805 * unsuccussful. If lacp_fallback_ab is enabled use active-
1806 * backup mode else drop all traffic. */
1807 if (!bond
->lacp_fallback_ab
) {
1815 return bond
->active_slave
;
1818 if (bond
->lacp_status
!= LACP_NEGOTIATED
) {
1819 /* Must have LACP negotiations for TCP balanced bonds. */
1823 flow_mask_hash_fields(flow
, wc
, NX_HASH_FIELDS_SYMMETRIC_L4
);
1828 flow_mask_hash_fields(flow
, wc
, NX_HASH_FIELDS_ETH_SRC
);
1830 e
= lookup_bond_entry(bond
, flow
, vlan
);
1831 if (!e
->slave
|| !e
->slave
->enabled
) {
1832 e
->slave
= get_enabled_slave(CONST_CAST(struct bond
*, bond
));
1841 static struct bond_slave
*
1842 bond_choose_slave(const struct bond
*bond
)
1844 struct bond_slave
*slave
, *best
;
1846 /* Find the last active slave. */
1847 slave
= bond_find_slave_by_mac(bond
, bond
->active_slave_mac
);
1848 if (slave
&& slave
->enabled
) {
1852 /* Find an enabled slave. */
1853 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1854 if (slave
->enabled
) {
1859 /* All interfaces are disabled. Find an interface that will be enabled
1860 * after its updelay expires. */
1862 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1863 if (slave
->delay_expires
!= LLONG_MAX
1864 && slave
->may_enable
1865 && (!best
|| slave
->delay_expires
< best
->delay_expires
)) {
1873 bond_choose_active_slave(struct bond
*bond
)
1875 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1876 struct bond_slave
*old_active_slave
= bond
->active_slave
;
1878 bond
->active_slave
= bond_choose_slave(bond
);
1879 if (bond
->active_slave
) {
1880 if (bond
->active_slave
->enabled
) {
1881 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s",
1882 bond
->name
, bond
->active_slave
->name
);
1884 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s, skipping "
1885 "remaining %lld ms updelay (since no interface was "
1886 "enabled)", bond
->name
, bond
->active_slave
->name
,
1887 bond
->active_slave
->delay_expires
- time_msec());
1888 bond_enable_slave(bond
->active_slave
, true);
1891 bond
->send_learning_packets
= true;
1893 if (bond
->active_slave
!= old_active_slave
) {
1894 bond_active_slave_changed(bond
);
1896 } else if (old_active_slave
) {
1897 bond_active_slave_changed(bond
);
1898 VLOG_INFO_RL(&rl
, "bond %s: all interfaces disabled", bond
->name
);
1903 * Return true if bond has unstored active slave change.
1904 * If return true, 'mac' will store the bond's current active slave's
1907 bond_get_changed_active_slave(const char *name
, struct eth_addr
*mac
,
1912 ovs_rwlock_wrlock(&rwlock
);
1913 bond
= bond_find(name
);
1915 if (bond
->active_slave_changed
|| force
) {
1916 *mac
= bond
->active_slave_mac
;
1917 bond
->active_slave_changed
= false;
1918 ovs_rwlock_unlock(&rwlock
);
1922 ovs_rwlock_unlock(&rwlock
);