2 * Copyright (c) 2008-2017 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "connectivity.h"
28 #include "dp-packet.h"
30 #include "openvswitch/hmap.h"
34 #include "ofproto/ofproto-dpif.h"
35 #include "ofproto/ofproto-dpif-rid.h"
36 #include "ofproto/ofproto-provider.h"
37 #include "openvswitch/dynamic-string.h"
38 #include "openvswitch/list.h"
39 #include "openvswitch/match.h"
40 #include "openvswitch/ofp-actions.h"
41 #include "openvswitch/ofpbuf.h"
42 #include "openvswitch/vlog.h"
44 #include "openvswitch/poll-loop.h"
46 #include "openvswitch/shash.h"
51 VLOG_DEFINE_THIS_MODULE(bond
);
53 static struct ovs_rwlock rwlock
= OVS_RWLOCK_INITIALIZER
;
54 static struct hmap all_bonds__
= HMAP_INITIALIZER(&all_bonds__
);
55 static struct hmap
*const all_bonds
OVS_GUARDED_BY(rwlock
) = &all_bonds__
;
57 /* Bit-mask for hashing a flow down to a bucket. */
58 #define BOND_MASK 0xff
59 #define BOND_BUCKETS (BOND_MASK + 1)
61 /* Priority for internal rules created to handle recirculation */
62 #define RECIRC_RULE_PRIORITY 20
64 /* A hash bucket for mapping a flow to a slave.
65 * "struct bond" has an array of BOND_BUCKETS of these. */
67 struct bond_slave
*slave
; /* Assigned slave, NULL if unassigned. */
68 uint64_t tx_bytes
/* Count of bytes recently transmitted. */
69 OVS_GUARDED_BY(rwlock
);
70 struct ovs_list list_node
; /* In bond_slave's 'entries' list. */
74 * 'pr_rule' is the post-recirculation rule for this entry.
75 * 'pr_tx_bytes' is the most recently seen statistics for 'pr_rule', which
76 * is used to determine delta (applied to 'tx_bytes' above.) */
78 uint64_t pr_tx_bytes
OVS_GUARDED_BY(rwlock
);
81 /* A bond slave, that is, one of the links comprising a bond. */
83 struct hmap_node hmap_node
; /* In struct bond's slaves hmap. */
84 struct ovs_list list_node
; /* In struct bond's enabled_slaves list. */
85 struct bond
*bond
; /* The bond that contains this slave. */
86 void *aux
; /* Client-provided handle for this slave. */
88 struct netdev
*netdev
; /* Network device, owned by the client. */
89 uint64_t change_seq
; /* Tracks changes in 'netdev'. */
90 char *name
; /* Name (a copy of netdev_get_name(netdev)). */
91 ofp_port_t ofp_port
; /* OpenFlow port number. */
94 bool enabled
; /* May be chosen for flows? */
95 bool may_enable
; /* Client considers this slave bondable. */
96 long long delay_expires
; /* Time after which 'enabled' may change. */
98 /* Rebalancing info. Used only by bond_rebalance(). */
99 struct ovs_list bal_node
; /* In bond_rebalance()'s 'bals' list. */
100 struct ovs_list entries
; /* 'struct bond_entry's assigned here. */
101 uint64_t tx_bytes
; /* Sum across 'tx_bytes' of entries. */
104 /* A bond, that is, a set of network devices grouped to improve performance or
107 struct hmap_node hmap_node
; /* In 'all_bonds' hmap. */
108 char *name
; /* Name provided by client. */
109 struct ofproto_dpif
*ofproto
; /* The bridge this bond belongs to. */
116 * Any reader or writer of 'enabled_slaves' must hold 'mutex'.
117 * (To prevent the bond_slave from disappearing they must also hold
119 struct ovs_mutex mutex
OVS_ACQ_AFTER(rwlock
);
120 struct ovs_list enabled_slaves OVS_GUARDED
; /* Contains struct bond_slaves. */
123 enum bond_mode balance
; /* Balancing mode, one of BM_*. */
124 struct bond_slave
*active_slave
;
125 int updelay
, downdelay
; /* Delay before slave goes up/down, in ms. */
126 enum lacp_status lacp_status
; /* Status of LACP negotiations. */
127 bool bond_revalidate
; /* True if flows need revalidation. */
128 uint32_t basis
; /* Basis for flow hash function. */
130 /* SLB specific bonding info. */
131 struct bond_entry
*hash
; /* An array of BOND_BUCKETS elements. */
132 int rebalance_interval
; /* Interval between rebalances, in ms. */
133 long long int next_rebalance
; /* Next rebalancing time. */
134 bool send_learning_packets
;
135 uint32_t recirc_id
; /* Non zero if recirculation can be used.*/
136 struct hmap pr_rule_ops
; /* Helps to maintain post recirculation rules.*/
138 /* Store active slave to OVSDB. */
139 bool active_slave_changed
; /* Set to true whenever the bond changes
140 active slave. It will be reset to false
141 after it is stored into OVSDB */
143 /* Interface name may not be persistent across an OS reboot, use
144 * MAC address for identifing the active slave */
145 struct eth_addr active_slave_mac
;
146 /* The MAC address of the active interface. */
147 /* Legacy compatibility. */
148 bool lacp_fallback_ab
; /* Fallback to active-backup on LACP failure. */
150 struct ovs_refcount ref_cnt
;
153 /* What to do with an bond_recirc_rule. */
155 ADD
, /* Add the rule to ofproto's flow table. */
156 DEL
, /* Delete the rule from the ofproto's flow table. */
159 /* A rule to add to or delete from ofproto's internal flow table. */
160 struct bond_pr_rule_op
{
161 struct hmap_node hmap_node
;
163 ofp_port_t out_ofport
;
165 struct rule
**pr_rule
;
168 static void bond_entry_reset(struct bond
*) OVS_REQ_WRLOCK(rwlock
);
169 static struct bond_slave
*bond_slave_lookup(struct bond
*, const void *slave_
)
170 OVS_REQ_RDLOCK(rwlock
);
171 static void bond_enable_slave(struct bond_slave
*, bool enable
)
172 OVS_REQ_WRLOCK(rwlock
);
173 static void bond_link_status_update(struct bond_slave
*)
174 OVS_REQ_WRLOCK(rwlock
);
175 static void bond_choose_active_slave(struct bond
*)
176 OVS_REQ_WRLOCK(rwlock
);
177 static struct bond_entry
*lookup_bond_entry(const struct bond
*,
180 OVS_REQ_RDLOCK(rwlock
);
181 static struct bond_slave
*get_enabled_slave(struct bond
*)
182 OVS_REQ_RDLOCK(rwlock
);
183 static struct bond_slave
*choose_output_slave(const struct bond
*,
185 struct flow_wildcards
*,
187 OVS_REQ_RDLOCK(rwlock
);
188 static void update_recirc_rules__(struct bond
*bond
);
189 static bool bond_is_falling_back_to_ab(const struct bond
*);
191 /* Attempts to parse 's' as the name of a bond balancing mode. If successful,
192 * stores the mode in '*balance' and returns true. Otherwise returns false
193 * without modifying '*balance'. */
195 bond_mode_from_string(enum bond_mode
*balance
, const char *s
)
197 if (!strcmp(s
, bond_mode_to_string(BM_TCP
))) {
199 } else if (!strcmp(s
, bond_mode_to_string(BM_SLB
))) {
201 } else if (!strcmp(s
, bond_mode_to_string(BM_AB
))) {
209 /* Returns a string representing 'balance'. */
211 bond_mode_to_string(enum bond_mode balance
) {
214 return "balance-tcp";
216 return "balance-slb";
218 return "active-backup";
224 /* Creates and returns a new bond whose configuration is initially taken from
227 * The caller should register each slave on the new bond by calling
228 * bond_slave_register(). */
230 bond_create(const struct bond_settings
*s
, struct ofproto_dpif
*ofproto
)
234 bond
= xzalloc(sizeof *bond
);
235 bond
->ofproto
= ofproto
;
236 hmap_init(&bond
->slaves
);
237 ovs_list_init(&bond
->enabled_slaves
);
238 ovs_mutex_init(&bond
->mutex
);
239 ovs_refcount_init(&bond
->ref_cnt
);
240 hmap_init(&bond
->pr_rule_ops
);
242 bond
->active_slave_mac
= eth_addr_zero
;
243 bond
->active_slave_changed
= false;
245 bond_reconfigure(bond
, s
);
250 bond_ref(const struct bond
*bond_
)
252 struct bond
*bond
= CONST_CAST(struct bond
*, bond_
);
255 ovs_refcount_ref(&bond
->ref_cnt
);
262 bond_unref(struct bond
*bond
)
264 struct bond_slave
*slave
;
266 if (!bond
|| ovs_refcount_unref_relaxed(&bond
->ref_cnt
) != 1) {
270 ovs_rwlock_wrlock(&rwlock
);
271 hmap_remove(all_bonds
, &bond
->hmap_node
);
272 ovs_rwlock_unlock(&rwlock
);
274 HMAP_FOR_EACH_POP (slave
, hmap_node
, &bond
->slaves
) {
275 /* Client owns 'slave->netdev'. */
279 hmap_destroy(&bond
->slaves
);
281 ovs_mutex_destroy(&bond
->mutex
);
283 /* Free bond resources. Remove existing post recirc rules. */
284 if (bond
->recirc_id
) {
285 recirc_free_id(bond
->recirc_id
);
290 update_recirc_rules__(bond
);
292 hmap_destroy(&bond
->pr_rule_ops
);
298 add_pr_rule(struct bond
*bond
, const struct match
*match
,
299 ofp_port_t out_ofport
, struct rule
**rule
)
301 uint32_t hash
= match_hash(match
, 0);
302 struct bond_pr_rule_op
*pr_op
;
304 HMAP_FOR_EACH_WITH_HASH(pr_op
, hmap_node
, hash
, &bond
->pr_rule_ops
) {
305 if (match_equal(&pr_op
->match
, match
)) {
307 pr_op
->out_ofport
= out_ofport
;
308 pr_op
->pr_rule
= rule
;
313 pr_op
= xmalloc(sizeof *pr_op
);
314 pr_op
->match
= *match
;
316 pr_op
->out_ofport
= out_ofport
;
317 pr_op
->pr_rule
= rule
;
318 hmap_insert(&bond
->pr_rule_ops
, &pr_op
->hmap_node
, hash
);
321 /* This function should almost never be called directly.
322 * 'update_recirc_rules()' should be called instead. Since
323 * this function modifies 'bond->pr_rule_ops', it is only
324 * safe when 'rwlock' is held.
326 * However, when the 'bond' is the only reference in the system,
327 * calling this function avoid acquiring lock only to satisfy
328 * lock annotation. Currently, only 'bond_unref()' calls
329 * this function directly. */
331 update_recirc_rules__(struct bond
*bond
)
334 struct bond_pr_rule_op
*pr_op
, *next_op
;
335 uint64_t ofpacts_stub
[128 / 8];
336 struct ofpbuf ofpacts
;
339 ofpbuf_use_stub(&ofpacts
, ofpacts_stub
, sizeof ofpacts_stub
);
341 HMAP_FOR_EACH(pr_op
, hmap_node
, &bond
->pr_rule_ops
) {
345 if (bond
->hash
&& bond
->recirc_id
) {
346 for (i
= 0; i
< BOND_BUCKETS
; i
++) {
347 struct bond_slave
*slave
= bond
->hash
[i
].slave
;
350 match_init_catchall(&match
);
351 match_set_recirc_id(&match
, bond
->recirc_id
);
352 match_set_dp_hash_masked(&match
, i
, BOND_MASK
);
354 add_pr_rule(bond
, &match
, slave
->ofp_port
,
355 &bond
->hash
[i
].pr_rule
);
360 HMAP_FOR_EACH_SAFE(pr_op
, next_op
, hmap_node
, &bond
->pr_rule_ops
) {
364 ofpbuf_clear(&ofpacts
);
365 ofpact_put_OUTPUT(&ofpacts
)->port
= pr_op
->out_ofport
;
366 error
= ofproto_dpif_add_internal_flow(bond
->ofproto
,
368 RECIRC_RULE_PRIORITY
, 0,
369 &ofpacts
, pr_op
->pr_rule
);
371 char *err_s
= match_to_string(&pr_op
->match
, NULL
,
372 RECIRC_RULE_PRIORITY
);
374 VLOG_ERR("failed to add post recirculation flow %s", err_s
);
380 error
= ofproto_dpif_delete_internal_flow(bond
->ofproto
,
382 RECIRC_RULE_PRIORITY
);
384 char *err_s
= match_to_string(&pr_op
->match
, NULL
,
385 RECIRC_RULE_PRIORITY
);
387 VLOG_ERR("failed to remove post recirculation flow %s", err_s
);
391 hmap_remove(&bond
->pr_rule_ops
, &pr_op
->hmap_node
);
393 *pr_op
->pr_rule
= NULL
;
400 ofpbuf_uninit(&ofpacts
);
404 update_recirc_rules(struct bond
*bond
)
405 OVS_REQ_RDLOCK(rwlock
)
407 update_recirc_rules__(bond
);
410 /* Updates 'bond''s overall configuration to 's'.
412 * The caller should register each slave on 'bond' by calling
413 * bond_slave_register(). This is optional if none of the slaves'
414 * configuration has changed. In any case it can't hurt.
416 * Returns true if the configuration has changed in such a way that requires
420 bond_reconfigure(struct bond
*bond
, const struct bond_settings
*s
)
422 bool revalidate
= false;
424 ovs_rwlock_wrlock(&rwlock
);
425 if (!bond
->name
|| strcmp(bond
->name
, s
->name
)) {
427 hmap_remove(all_bonds
, &bond
->hmap_node
);
430 bond
->name
= xstrdup(s
->name
);
431 hmap_insert(all_bonds
, &bond
->hmap_node
, hash_string(bond
->name
, 0));
434 bond
->updelay
= s
->up_delay
;
435 bond
->downdelay
= s
->down_delay
;
437 if (bond
->lacp_fallback_ab
!= s
->lacp_fallback_ab_cfg
) {
438 bond
->lacp_fallback_ab
= s
->lacp_fallback_ab_cfg
;
442 if (bond
->rebalance_interval
!= s
->rebalance_interval
) {
443 bond
->rebalance_interval
= s
->rebalance_interval
;
447 if (bond
->balance
!= s
->balance
) {
448 bond
->balance
= s
->balance
;
452 if (bond
->basis
!= s
->basis
) {
453 bond
->basis
= s
->basis
;
457 if (bond
->bond_revalidate
) {
459 bond
->bond_revalidate
= false;
462 if (bond
->balance
!= BM_AB
) {
463 if (!bond
->recirc_id
) {
464 bond
->recirc_id
= recirc_alloc_id(bond
->ofproto
);
466 } else if (bond
->recirc_id
) {
467 recirc_free_id(bond
->recirc_id
);
471 if (bond
->balance
== BM_AB
|| !bond
->hash
|| revalidate
) {
472 bond_entry_reset(bond
);
475 ovs_rwlock_unlock(&rwlock
);
479 static struct bond_slave
*
480 bond_find_slave_by_mac(const struct bond
*bond
, const struct eth_addr mac
)
482 struct bond_slave
*slave
;
484 /* Find the last active slave */
485 HMAP_FOR_EACH(slave
, hmap_node
, &bond
->slaves
) {
486 struct eth_addr slave_mac
;
488 if (netdev_get_etheraddr(slave
->netdev
, &slave_mac
)) {
492 if (eth_addr_equals(slave_mac
, mac
)) {
501 bond_active_slave_changed(struct bond
*bond
)
503 if (bond
->active_slave
) {
505 netdev_get_etheraddr(bond
->active_slave
->netdev
, &mac
);
506 bond
->active_slave_mac
= mac
;
508 bond
->active_slave_mac
= eth_addr_zero
;
510 bond
->active_slave_changed
= true;
511 seq_change(connectivity_seq_get());
515 bond_slave_set_netdev__(struct bond_slave
*slave
, struct netdev
*netdev
)
516 OVS_REQ_WRLOCK(rwlock
)
518 if (slave
->netdev
!= netdev
) {
519 slave
->netdev
= netdev
;
520 slave
->change_seq
= 0;
524 /* Registers 'slave_' as a slave of 'bond'. The 'slave_' pointer is an
525 * arbitrary client-provided pointer that uniquely identifies a slave within a
526 * bond. If 'slave_' already exists within 'bond' then this function
527 * reconfigures the existing slave.
529 * 'netdev' must be the network device that 'slave_' represents. It is owned
530 * by the client, so the client must not close it before either unregistering
531 * 'slave_' or destroying 'bond'.
534 bond_slave_register(struct bond
*bond
, void *slave_
,
535 ofp_port_t ofport
, struct netdev
*netdev
)
537 struct bond_slave
*slave
;
539 ovs_rwlock_wrlock(&rwlock
);
540 slave
= bond_slave_lookup(bond
, slave_
);
542 slave
= xzalloc(sizeof *slave
);
544 hmap_insert(&bond
->slaves
, &slave
->hmap_node
, hash_pointer(slave_
, 0));
547 slave
->ofp_port
= ofport
;
548 slave
->delay_expires
= LLONG_MAX
;
549 slave
->name
= xstrdup(netdev_get_name(netdev
));
550 bond
->bond_revalidate
= true;
552 slave
->enabled
= false;
553 bond_enable_slave(slave
, netdev_get_carrier(netdev
));
556 bond_slave_set_netdev__(slave
, netdev
);
559 slave
->name
= xstrdup(netdev_get_name(netdev
));
560 ovs_rwlock_unlock(&rwlock
);
563 /* Updates the network device to be used with 'slave_' to 'netdev'.
565 * This is useful if the caller closes and re-opens the network device
566 * registered with bond_slave_register() but doesn't need to change anything
569 bond_slave_set_netdev(struct bond
*bond
, void *slave_
, struct netdev
*netdev
)
571 struct bond_slave
*slave
;
573 ovs_rwlock_wrlock(&rwlock
);
574 slave
= bond_slave_lookup(bond
, slave_
);
576 bond_slave_set_netdev__(slave
, netdev
);
578 ovs_rwlock_unlock(&rwlock
);
581 /* Unregisters 'slave_' from 'bond'. If 'bond' does not contain such a slave
582 * then this function has no effect.
584 * Unregistering a slave invalidates all flows. */
586 bond_slave_unregister(struct bond
*bond
, const void *slave_
)
588 struct bond_slave
*slave
;
591 ovs_rwlock_wrlock(&rwlock
);
592 slave
= bond_slave_lookup(bond
, slave_
);
597 bond
->bond_revalidate
= true;
598 bond_enable_slave(slave
, false);
600 del_active
= bond
->active_slave
== slave
;
602 struct bond_entry
*e
;
603 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
604 if (e
->slave
== slave
) {
612 hmap_remove(&bond
->slaves
, &slave
->hmap_node
);
613 /* Client owns 'slave->netdev'. */
617 bond_choose_active_slave(bond
);
618 bond
->send_learning_packets
= true;
621 ovs_rwlock_unlock(&rwlock
);
624 /* Should be called on each slave in 'bond' before bond_run() to indicate
625 * whether or not 'slave_' may be enabled. This function is intended to allow
626 * other protocols to have some impact on bonding decisions. For example LACP
627 * or high level link monitoring protocols may decide that a given slave should
628 * not be able to send traffic. */
630 bond_slave_set_may_enable(struct bond
*bond
, void *slave_
, bool may_enable
)
632 ovs_rwlock_wrlock(&rwlock
);
633 bond_slave_lookup(bond
, slave_
)->may_enable
= may_enable
;
634 ovs_rwlock_unlock(&rwlock
);
637 /* Performs periodic maintenance on 'bond'.
639 * Returns true if the caller should revalidate its flows.
641 * The caller should check bond_should_send_learning_packets() afterward. */
643 bond_run(struct bond
*bond
, enum lacp_status lacp_status
)
645 struct bond_slave
*slave
;
648 ovs_rwlock_wrlock(&rwlock
);
649 if (bond
->lacp_status
!= lacp_status
) {
650 bond
->lacp_status
= lacp_status
;
651 bond
->bond_revalidate
= true;
653 /* Change in LACP status can affect whether the bond is falling back to
654 * active-backup. Make sure to create or destroy buckets if
656 if (bond_is_falling_back_to_ab(bond
) || !bond
->hash
) {
657 bond_entry_reset(bond
);
661 /* Enable slaves based on link status and LACP feedback. */
662 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
663 bond_link_status_update(slave
);
664 slave
->change_seq
= seq_read(connectivity_seq_get());
666 if (!bond
->active_slave
|| !bond
->active_slave
->enabled
) {
667 bond_choose_active_slave(bond
);
670 revalidate
= bond
->bond_revalidate
;
671 bond
->bond_revalidate
= false;
672 ovs_rwlock_unlock(&rwlock
);
677 /* Causes poll_block() to wake up when 'bond' needs something to be done. */
679 bond_wait(struct bond
*bond
)
681 struct bond_slave
*slave
;
683 ovs_rwlock_rdlock(&rwlock
);
684 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
685 if (slave
->delay_expires
!= LLONG_MAX
) {
686 poll_timer_wait_until(slave
->delay_expires
);
689 seq_wait(connectivity_seq_get(), slave
->change_seq
);
692 if (bond
->bond_revalidate
) {
693 poll_immediate_wake();
695 ovs_rwlock_unlock(&rwlock
);
697 /* We don't wait for bond->next_rebalance because rebalancing can only run
698 * at a flow account checkpoint. ofproto does checkpointing on its own
699 * schedule and bond_rebalance() gets called afterward, so we'd just be
700 * waking up for no purpose. */
703 /* MAC learning table interaction. */
706 may_send_learning_packets(const struct bond
*bond
)
708 return ((bond
->lacp_status
== LACP_DISABLED
709 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_AB
))
710 || (bond
->lacp_fallback_ab
&& bond
->lacp_status
== LACP_CONFIGURED
))
711 && bond
->active_slave
;
714 /* Returns true if 'bond' needs the client to send out packets to assist with
715 * MAC learning on 'bond'. If this function returns true, then the client
716 * should iterate through its MAC learning table for the bridge on which 'bond'
717 * is located. For each MAC that has been learned on a port other than 'bond',
718 * it should call bond_compose_learning_packet().
720 * This function will only return true if 'bond' is in SLB or active-backup
721 * mode and LACP is not negotiated. Otherwise sending learning packets isn't
724 * Calling this function resets the state that it checks. */
726 bond_should_send_learning_packets(struct bond
*bond
)
730 ovs_rwlock_wrlock(&rwlock
);
731 send
= bond
->send_learning_packets
&& may_send_learning_packets(bond
);
732 bond
->send_learning_packets
= false;
733 ovs_rwlock_unlock(&rwlock
);
737 /* Sends a gratuitous learning packet on 'bond' from 'eth_src' on 'vlan'.
739 * See bond_should_send_learning_packets() for description of usage. The
740 * caller should send the composed packet on the port associated with
741 * port_aux and takes ownership of the returned ofpbuf. */
743 bond_compose_learning_packet(struct bond
*bond
, const struct eth_addr eth_src
,
744 uint16_t vlan
, void **port_aux
)
746 struct bond_slave
*slave
;
747 struct dp_packet
*packet
;
750 ovs_rwlock_rdlock(&rwlock
);
751 ovs_assert(may_send_learning_packets(bond
));
752 memset(&flow
, 0, sizeof flow
);
753 flow
.dl_src
= eth_src
;
754 slave
= choose_output_slave(bond
, &flow
, NULL
, vlan
);
756 packet
= dp_packet_new(0);
757 compose_rarp(packet
, eth_src
);
759 eth_push_vlan(packet
, htons(ETH_TYPE_VLAN
), htons(vlan
));
762 *port_aux
= slave
->aux
;
763 ovs_rwlock_unlock(&rwlock
);
769 bond_is_falling_back_to_ab(const struct bond
*bond
)
771 return (bond
->lacp_fallback_ab
772 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_TCP
)
773 && bond
->lacp_status
== LACP_CONFIGURED
);
776 /* Checks whether a packet that arrived on 'slave_' within 'bond', with an
777 * Ethernet destination address of 'eth_dst', should be admitted.
779 * The return value is one of the following:
781 * - BV_ACCEPT: Admit the packet.
783 * - BV_DROP: Drop the packet.
785 * - BV_DROP_IF_MOVED: Consult the MAC learning table for the packet's
786 * Ethernet source address and VLAN. If there is none, or if the packet
787 * is on the learned port, then admit the packet. If a different port has
788 * been learned, however, drop the packet (and do not use it for MAC
792 bond_check_admissibility(struct bond
*bond
, const void *slave_
,
793 const struct eth_addr eth_dst
)
795 enum bond_verdict verdict
= BV_DROP
;
796 struct bond_slave
*slave
;
797 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
799 ovs_rwlock_rdlock(&rwlock
);
800 slave
= bond_slave_lookup(bond
, slave_
);
805 /* LACP bonds have very loose admissibility restrictions because we can
806 * assume the remote switch is aware of the bond and will "do the right
807 * thing". However, as a precaution we drop packets on disabled slaves
808 * because no correctly implemented partner switch should be sending
811 * If LACP is configured, but LACP negotiations have been unsuccessful, we
812 * drop all incoming traffic except if lacp_fallback_ab is enabled. */
813 switch (bond
->lacp_status
) {
814 case LACP_NEGOTIATED
:
815 /* To reduce packet-drops due to delay in enabling of slave (post
816 * LACP-SYNC), from main thread, check for may_enable as well.
817 * When may_enable is TRUE, it means LACP is UP and waiting for the
818 * main thread to run LACP state machine and enable the slave. */
819 verdict
= (slave
->enabled
|| slave
->may_enable
) ? BV_ACCEPT
: BV_DROP
;
821 case LACP_CONFIGURED
:
822 if (!bond
->lacp_fallback_ab
) {
827 if (bond
->balance
== BM_TCP
) {
833 /* Drop all multicast packets on inactive slaves. */
834 if (eth_addr_is_multicast(eth_dst
)) {
835 if (bond
->active_slave
!= slave
) {
840 switch (bond
->balance
) {
842 /* TCP balanced bonds require successful LACP negotiations. Based on the
843 * above check, LACP is off or lacp_fallback_ab is true on this bond.
844 * If lacp_fallback_ab is true fall through to BM_AB case else, we
845 * drop all incoming traffic. */
846 if (!bond
->lacp_fallback_ab
) {
852 /* Drop all packets which arrive on backup slaves. This is similar to
853 * how Linux bonding handles active-backup bonds. */
854 if (bond
->active_slave
!= slave
) {
855 VLOG_DBG_RL(&rl
, "active-backup bond received packet on backup"
856 " slave (%s) destined for " ETH_ADDR_FMT
,
857 slave
->name
, ETH_ADDR_ARGS(eth_dst
));
864 /* Drop all packets for which we have learned a different input port,
865 * because we probably sent the packet on one slave and got it back on
866 * the other. Gratuitous ARP packets are an exception to this rule:
867 * the host has moved to another switch. The exception to the
868 * exception is if we locked the learning table to avoid reflections on
870 verdict
= BV_DROP_IF_MOVED
;
876 if (slave
&& (verdict
!= BV_ACCEPT
)) {
877 VLOG_DBG_RL(&rl
, "slave (%s): Admissibility verdict is to drop pkt %s."
878 "active slave: %s, may_enable: %s enable: %s "
881 (verdict
== BV_DROP_IF_MOVED
) ?
882 "as different port is learned" : "",
883 (bond
->active_slave
== slave
) ? "true" : "false",
884 slave
->may_enable
? "true" : "false",
885 slave
->enabled
? "true" : "false",
889 ovs_rwlock_unlock(&rwlock
);
894 /* Returns the slave (registered on 'bond' by bond_slave_register()) to which
895 * a packet with the given 'flow' and 'vlan' should be forwarded. Returns
896 * NULL if the packet should be dropped because no slaves are enabled.
898 * 'vlan' is not necessarily the same as 'flow->vlan_tci'. First, 'vlan'
899 * should be a VID only (i.e. excluding the PCP bits). Second,
900 * 'flow->vlan_tci' is the VLAN TCI that appeared on the packet (so it will be
901 * nonzero only for trunk ports), whereas 'vlan' is the logical VLAN that the
902 * packet belongs to (so for an access port it will be the access port's VLAN).
904 * If 'wc' is non-NULL, bitwise-OR's 'wc' with the set of bits that were
905 * significant in the selection. At some point earlier, 'wc' should
906 * have been initialized (e.g., by flow_wildcards_init_catchall()).
909 bond_choose_output_slave(struct bond
*bond
, const struct flow
*flow
,
910 struct flow_wildcards
*wc
, uint16_t vlan
)
912 struct bond_slave
*slave
;
915 ovs_rwlock_rdlock(&rwlock
);
916 slave
= choose_output_slave(bond
, flow
, wc
, vlan
);
917 aux
= slave
? slave
->aux
: NULL
;
918 ovs_rwlock_unlock(&rwlock
);
925 bond_entry_account(struct bond_entry
*entry
, uint64_t rule_tx_bytes
)
926 OVS_REQ_WRLOCK(rwlock
)
931 delta
= rule_tx_bytes
- entry
->pr_tx_bytes
;
932 entry
->tx_bytes
+= delta
;
933 entry
->pr_tx_bytes
= rule_tx_bytes
;
937 /* Maintain bond stats using post recirculation rule byte counters.*/
939 bond_recirculation_account(struct bond
*bond
)
940 OVS_REQ_WRLOCK(rwlock
)
944 for (i
=0; i
<=BOND_MASK
; i
++) {
945 struct bond_entry
*entry
= &bond
->hash
[i
];
946 struct rule
*rule
= entry
->pr_rule
;
949 uint64_t n_packets OVS_UNUSED
;
950 long long int used OVS_UNUSED
;
953 rule
->ofproto
->ofproto_class
->rule_get_stats(
954 rule
, &n_packets
, &n_bytes
, &used
);
955 bond_entry_account(entry
, n_bytes
);
961 bond_may_recirc(const struct bond
*bond
)
963 return (bond
->balance
== BM_TCP
&& bond
->recirc_id
964 && !bond_is_falling_back_to_ab(bond
));
968 bond_update_post_recirc_rules__(struct bond
* bond
, const bool force
)
969 OVS_REQ_WRLOCK(rwlock
)
971 struct bond_entry
*e
;
972 bool update_rules
= force
; /* Always update rules if caller forces it. */
974 /* Make sure all bond entries are populated */
975 for (e
= bond
->hash
; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
976 if (!e
->slave
|| !e
->slave
->enabled
) {
978 e
->slave
= CONTAINER_OF(hmap_random_node(&bond
->slaves
),
979 struct bond_slave
, hmap_node
);
980 if (!e
->slave
->enabled
) {
981 e
->slave
= bond
->active_slave
;
987 update_recirc_rules(bond
);
992 bond_update_post_recirc_rules(struct bond
*bond
, uint32_t *recirc_id
,
993 uint32_t *hash_basis
)
995 bool may_recirc
= bond_may_recirc(bond
);
998 /* To avoid unnecessary locking, bond_may_recirc() is first
999 * called outside of the 'rwlock'. After acquiring the lock,
1000 * check again to make sure bond configuration has not been changed. */
1001 ovs_rwlock_wrlock(&rwlock
);
1002 may_recirc
= bond_may_recirc(bond
);
1004 *recirc_id
= bond
->recirc_id
;
1005 *hash_basis
= bond
->basis
;
1006 bond_update_post_recirc_rules__(bond
, false);
1008 ovs_rwlock_unlock(&rwlock
);
1012 *recirc_id
= *hash_basis
= 0;
1020 bond_is_balanced(const struct bond
*bond
) OVS_REQ_RDLOCK(rwlock
)
1022 return bond
->rebalance_interval
1023 && (bond
->balance
== BM_SLB
|| bond
->balance
== BM_TCP
)
1024 && !(bond
->lacp_fallback_ab
&& bond
->lacp_status
== LACP_CONFIGURED
);
1027 /* Notifies 'bond' that 'n_bytes' bytes were sent in 'flow' within 'vlan'. */
1029 bond_account(struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
,
1032 ovs_rwlock_wrlock(&rwlock
);
1033 if (bond_is_balanced(bond
)) {
1034 lookup_bond_entry(bond
, flow
, vlan
)->tx_bytes
+= n_bytes
;
1036 ovs_rwlock_unlock(&rwlock
);
1039 static struct bond_slave
*
1040 bond_slave_from_bal_node(struct ovs_list
*bal
) OVS_REQ_RDLOCK(rwlock
)
1042 return CONTAINER_OF(bal
, struct bond_slave
, bal_node
);
1046 log_bals(struct bond
*bond
, const struct ovs_list
*bals
)
1047 OVS_REQ_RDLOCK(rwlock
)
1049 if (VLOG_IS_DBG_ENABLED()) {
1050 struct ds ds
= DS_EMPTY_INITIALIZER
;
1051 const struct bond_slave
*slave
;
1053 LIST_FOR_EACH (slave
, bal_node
, bals
) {
1055 ds_put_char(&ds
, ',');
1057 ds_put_format(&ds
, " %s %"PRIu64
"kB",
1058 slave
->name
, slave
->tx_bytes
/ 1024);
1060 if (!slave
->enabled
) {
1061 ds_put_cstr(&ds
, " (disabled)");
1063 if (!ovs_list_is_empty(&slave
->entries
)) {
1064 struct bond_entry
*e
;
1066 ds_put_cstr(&ds
, " (");
1067 LIST_FOR_EACH (e
, list_node
, &slave
->entries
) {
1068 if (&e
->list_node
!= ovs_list_front(&slave
->entries
)) {
1069 ds_put_cstr(&ds
, " + ");
1071 ds_put_format(&ds
, "h%"PRIdPTR
": %"PRIu64
"kB",
1072 e
- bond
->hash
, e
->tx_bytes
/ 1024);
1074 ds_put_cstr(&ds
, ")");
1077 VLOG_DBG("bond %s:%s", bond
->name
, ds_cstr(&ds
));
1082 /* Shifts 'hash' from its current slave to 'to'. */
1084 bond_shift_load(struct bond_entry
*hash
, struct bond_slave
*to
)
1085 OVS_REQ_WRLOCK(rwlock
)
1087 struct bond_slave
*from
= hash
->slave
;
1088 struct bond
*bond
= from
->bond
;
1089 uint64_t delta
= hash
->tx_bytes
;
1091 VLOG_INFO("bond %s: shift %"PRIu64
"kB of load (with hash %"PRIdPTR
") "
1092 "from %s to %s (now carrying %"PRIu64
"kB and "
1093 "%"PRIu64
"kB load, respectively)",
1094 bond
->name
, delta
/ 1024, hash
- bond
->hash
,
1095 from
->name
, to
->name
,
1096 (from
->tx_bytes
- delta
) / 1024,
1097 (to
->tx_bytes
+ delta
) / 1024);
1099 /* Shift load away from 'from' to 'to'. */
1100 from
->tx_bytes
-= delta
;
1101 to
->tx_bytes
+= delta
;
1103 /* Arrange for flows to be revalidated. */
1105 bond
->bond_revalidate
= true;
1108 /* Picks and returns a bond_entry to migrate from 'from' (the most heavily
1109 * loaded bond slave) to a bond slave that has 'to_tx_bytes' bytes of load,
1110 * given that doing so must decrease the ratio of the load on the two slaves by
1111 * at least 0.1. Returns NULL if there is no appropriate entry.
1113 * The list of entries isn't sorted. I don't know of a reason to prefer to
1114 * shift away small hashes or large hashes. */
1115 static struct bond_entry
*
1116 choose_entry_to_migrate(const struct bond_slave
*from
, uint64_t to_tx_bytes
)
1117 OVS_REQ_WRLOCK(rwlock
)
1119 struct bond_entry
*e
;
1121 if (ovs_list_is_short(&from
->entries
)) {
1122 /* 'from' carries no more than one MAC hash, so shifting load away from
1123 * it would be pointless. */
1127 LIST_FOR_EACH (e
, list_node
, &from
->entries
) {
1128 uint64_t delta
= e
->tx_bytes
; /* The amount to rebalance. */
1129 uint64_t ideal_tx_bytes
= (from
->tx_bytes
+ to_tx_bytes
)/2;
1130 /* Note, the ideal traffic is the mid point
1131 * between 'from' and 'to'. This value does
1132 * not change by rebalancing. */
1133 uint64_t new_low
; /* The lower bandwidth between 'to' and 'from'
1134 after rebalancing. */
1136 new_low
= MIN(from
->tx_bytes
- delta
, to_tx_bytes
+ delta
);
1138 if ((new_low
> to_tx_bytes
) &&
1139 (new_low
- to_tx_bytes
>= (ideal_tx_bytes
- to_tx_bytes
) / 10)) {
1140 /* Only rebalance if the new 'low' is closer to to the mid point,
1141 * and the improvement exceeds 10% of current traffic
1142 * deviation from the ideal split.
1144 * The improvement on the 'high' side is always the same as the
1145 * 'low' side. Thus consider 'low' side is sufficient. */
1153 /* Inserts 'slave' into 'bals' so that descending order of 'tx_bytes' is
1156 insert_bal(struct ovs_list
*bals
, struct bond_slave
*slave
)
1158 struct bond_slave
*pos
;
1160 LIST_FOR_EACH (pos
, bal_node
, bals
) {
1161 if (slave
->tx_bytes
> pos
->tx_bytes
) {
1165 ovs_list_insert(&pos
->bal_node
, &slave
->bal_node
);
1168 /* Removes 'slave' from its current list and then inserts it into 'bals' so
1169 * that descending order of 'tx_bytes' is maintained. */
1171 reinsert_bal(struct ovs_list
*bals
, struct bond_slave
*slave
)
1173 ovs_list_remove(&slave
->bal_node
);
1174 insert_bal(bals
, slave
);
1177 /* If 'bond' needs rebalancing, does so.
1179 * The caller should have called bond_account() for each active flow, or in case
1180 * of recirculation is used, have called bond_recirculation_account(bond),
1181 * to ensure that flow data is consistently accounted at this point.
1184 bond_rebalance(struct bond
*bond
)
1186 struct bond_slave
*slave
;
1187 struct bond_entry
*e
;
1188 struct ovs_list bals
;
1189 bool rebalanced
= false;
1192 ovs_rwlock_wrlock(&rwlock
);
1193 if (!bond_is_balanced(bond
) || time_msec() < bond
->next_rebalance
) {
1196 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1198 use_recirc
= bond
->ofproto
->backer
->rt_support
.odp
.recirc
&&
1199 bond_may_recirc(bond
);
1202 bond_recirculation_account(bond
);
1205 /* Add each bond_entry to its slave's 'entries' list.
1206 * Compute each slave's tx_bytes as the sum of its entries' tx_bytes. */
1207 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1208 slave
->tx_bytes
= 0;
1209 ovs_list_init(&slave
->entries
);
1211 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
1212 if (e
->slave
&& e
->tx_bytes
) {
1213 e
->slave
->tx_bytes
+= e
->tx_bytes
;
1214 ovs_list_push_back(&e
->slave
->entries
, &e
->list_node
);
1218 /* Add enabled slaves to 'bals' in descending order of tx_bytes.
1220 * XXX This is O(n**2) in the number of slaves but it could be O(n lg n)
1221 * with a proper list sort algorithm. */
1222 ovs_list_init(&bals
);
1223 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1224 if (slave
->enabled
) {
1225 insert_bal(&bals
, slave
);
1228 log_bals(bond
, &bals
);
1230 /* Shift load from the most-loaded slaves to the least-loaded slaves. */
1231 while (!ovs_list_is_short(&bals
)) {
1232 struct bond_slave
*from
= bond_slave_from_bal_node(ovs_list_front(&bals
));
1233 struct bond_slave
*to
= bond_slave_from_bal_node(ovs_list_back(&bals
));
1236 overload
= from
->tx_bytes
- to
->tx_bytes
;
1237 if (overload
< to
->tx_bytes
>> 5 || overload
< 100000) {
1238 /* The extra load on 'from' (and all less-loaded slaves), compared
1239 * to that of 'to' (the least-loaded slave), is less than ~3%, or
1240 * it is less than ~1Mbps. No point in rebalancing. */
1244 /* 'from' is carrying significantly more load than 'to'. Pick a hash
1245 * to move from 'from' to 'to'. */
1246 e
= choose_entry_to_migrate(from
, to
->tx_bytes
);
1248 bond_shift_load(e
, to
);
1250 /* Delete element from from->entries.
1252 * We don't add the element to to->hashes. That would only allow
1253 * 'e' to be migrated to another slave in this rebalancing run, and
1254 * there is no point in doing that. */
1255 ovs_list_remove(&e
->list_node
);
1257 /* Re-sort 'bals'. */
1258 reinsert_bal(&bals
, from
);
1259 reinsert_bal(&bals
, to
);
1262 /* Can't usefully migrate anything away from 'from'.
1263 * Don't reconsider it. */
1264 ovs_list_remove(&from
->bal_node
);
1268 /* Implement exponentially weighted moving average. A weight of 1/2 causes
1269 * historical data to decay to <1% in 7 rebalancing runs. 1,000,000 bytes
1270 * take 20 rebalancing runs to decay to 0 and get deleted entirely. */
1271 for (e
= &bond
->hash
[0]; e
<= &bond
->hash
[BOND_MASK
]; e
++) {
1275 if (use_recirc
&& rebalanced
) {
1276 bond_update_post_recirc_rules__(bond
,true);
1280 ovs_rwlock_unlock(&rwlock
);
1283 /* Bonding unixctl user interface functions. */
1285 static struct bond
*
1286 bond_find(const char *name
) OVS_REQ_RDLOCK(rwlock
)
1290 HMAP_FOR_EACH_WITH_HASH (bond
, hmap_node
, hash_string(name
, 0),
1292 if (!strcmp(bond
->name
, name
)) {
1299 static struct bond_slave
*
1300 bond_lookup_slave(struct bond
*bond
, const char *slave_name
)
1302 struct bond_slave
*slave
;
1304 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1305 if (!strcmp(slave
->name
, slave_name
)) {
1313 bond_unixctl_list(struct unixctl_conn
*conn
,
1314 int argc OVS_UNUSED
, const char *argv
[] OVS_UNUSED
,
1315 void *aux OVS_UNUSED
)
1317 struct ds ds
= DS_EMPTY_INITIALIZER
;
1318 const struct bond
*bond
;
1320 ds_put_cstr(&ds
, "bond\ttype\trecircID\tslaves\n");
1322 ovs_rwlock_rdlock(&rwlock
);
1323 HMAP_FOR_EACH (bond
, hmap_node
, all_bonds
) {
1324 const struct bond_slave
*slave
;
1327 ds_put_format(&ds
, "%s\t%s\t%d\t", bond
->name
,
1328 bond_mode_to_string(bond
->balance
), bond
->recirc_id
);
1331 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1333 ds_put_cstr(&ds
, ", ");
1335 ds_put_cstr(&ds
, slave
->name
);
1337 ds_put_char(&ds
, '\n');
1339 ovs_rwlock_unlock(&rwlock
);
1340 unixctl_command_reply(conn
, ds_cstr(&ds
));
1345 bond_print_details(struct ds
*ds
, const struct bond
*bond
)
1346 OVS_REQ_RDLOCK(rwlock
)
1348 struct shash slave_shash
= SHASH_INITIALIZER(&slave_shash
);
1349 const struct shash_node
**sorted_slaves
= NULL
;
1350 const struct bond_slave
*slave
;
1355 ds_put_format(ds
, "---- %s ----\n", bond
->name
);
1356 ds_put_format(ds
, "bond_mode: %s\n",
1357 bond_mode_to_string(bond
->balance
));
1359 may_recirc
= bond_may_recirc(bond
);
1360 recirc_id
= bond
->recirc_id
;
1361 ds_put_format(ds
, "bond may use recirculation: %s, Recirc-ID : %d\n",
1362 may_recirc
? "yes" : "no", may_recirc
? recirc_id
: -1);
1364 ds_put_format(ds
, "bond-hash-basis: %"PRIu32
"\n", bond
->basis
);
1366 ds_put_format(ds
, "updelay: %d ms\n", bond
->updelay
);
1367 ds_put_format(ds
, "downdelay: %d ms\n", bond
->downdelay
);
1369 if (bond_is_balanced(bond
)) {
1370 ds_put_format(ds
, "next rebalance: %lld ms\n",
1371 bond
->next_rebalance
- time_msec());
1374 ds_put_cstr(ds
, "lacp_status: ");
1375 switch (bond
->lacp_status
) {
1376 case LACP_NEGOTIATED
:
1377 ds_put_cstr(ds
, "negotiated\n");
1379 case LACP_CONFIGURED
:
1380 ds_put_cstr(ds
, "configured\n");
1383 ds_put_cstr(ds
, "off\n");
1386 ds_put_cstr(ds
, "<unknown>\n");
1390 ds_put_format(ds
, "lacp_fallback_ab: %s\n",
1391 bond
->lacp_fallback_ab
? "true" : "false");
1393 ds_put_cstr(ds
, "active slave mac: ");
1394 ds_put_format(ds
, ETH_ADDR_FMT
, ETH_ADDR_ARGS(bond
->active_slave_mac
));
1395 slave
= bond_find_slave_by_mac(bond
, bond
->active_slave_mac
);
1396 ds_put_format(ds
,"(%s)\n", slave
? slave
->name
: "none");
1398 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1399 shash_add(&slave_shash
, slave
->name
, slave
);
1401 sorted_slaves
= shash_sort(&slave_shash
);
1403 for (i
= 0; i
< shash_count(&slave_shash
); i
++) {
1404 struct bond_entry
*be
;
1406 slave
= sorted_slaves
[i
]->data
;
1409 ds_put_format(ds
, "\nslave %s: %s\n",
1410 slave
->name
, slave
->enabled
? "enabled" : "disabled");
1411 if (slave
== bond
->active_slave
) {
1412 ds_put_cstr(ds
, " active slave\n");
1414 if (slave
->delay_expires
!= LLONG_MAX
) {
1415 ds_put_format(ds
, " %s expires in %lld ms\n",
1416 slave
->enabled
? "downdelay" : "updelay",
1417 slave
->delay_expires
- time_msec());
1420 ds_put_format(ds
, " may_enable: %s\n",
1421 slave
->may_enable
? "true" : "false");
1423 if (!bond_is_balanced(bond
)) {
1428 for (be
= bond
->hash
; be
<= &bond
->hash
[BOND_MASK
]; be
++) {
1429 int hash
= be
- bond
->hash
;
1432 if (be
->slave
!= slave
) {
1436 be_tx_k
= be
->tx_bytes
/ 1024;
1438 ds_put_format(ds
, " hash %d: %"PRIu64
" kB load\n",
1442 /* XXX How can we list the MACs assigned to hashes of SLB bonds? */
1445 shash_destroy(&slave_shash
);
1446 free(sorted_slaves
);
1447 ds_put_cstr(ds
, "\n");
1451 bond_unixctl_show(struct unixctl_conn
*conn
,
1452 int argc
, const char *argv
[],
1453 void *aux OVS_UNUSED
)
1455 struct ds ds
= DS_EMPTY_INITIALIZER
;
1457 ovs_rwlock_rdlock(&rwlock
);
1459 const struct bond
*bond
= bond_find(argv
[1]);
1462 unixctl_command_reply_error(conn
, "no such bond");
1465 bond_print_details(&ds
, bond
);
1467 const struct bond
*bond
;
1469 HMAP_FOR_EACH (bond
, hmap_node
, all_bonds
) {
1470 bond_print_details(&ds
, bond
);
1474 unixctl_command_reply(conn
, ds_cstr(&ds
));
1478 ovs_rwlock_unlock(&rwlock
);
1482 bond_unixctl_migrate(struct unixctl_conn
*conn
,
1483 int argc OVS_UNUSED
, const char *argv
[],
1484 void *aux OVS_UNUSED
)
1486 const char *bond_s
= argv
[1];
1487 const char *hash_s
= argv
[2];
1488 const char *slave_s
= argv
[3];
1490 struct bond_slave
*slave
;
1491 struct bond_entry
*entry
;
1494 ovs_rwlock_wrlock(&rwlock
);
1495 bond
= bond_find(bond_s
);
1497 unixctl_command_reply_error(conn
, "no such bond");
1501 if (bond
->balance
!= BM_SLB
) {
1502 unixctl_command_reply_error(conn
, "not an SLB bond");
1506 if (strspn(hash_s
, "0123456789") == strlen(hash_s
)) {
1507 hash
= atoi(hash_s
) & BOND_MASK
;
1509 unixctl_command_reply_error(conn
, "bad hash");
1513 slave
= bond_lookup_slave(bond
, slave_s
);
1515 unixctl_command_reply_error(conn
, "no such slave");
1519 if (!slave
->enabled
) {
1520 unixctl_command_reply_error(conn
, "cannot migrate to disabled slave");
1524 entry
= &bond
->hash
[hash
];
1525 bond
->bond_revalidate
= true;
1526 entry
->slave
= slave
;
1527 unixctl_command_reply(conn
, "migrated");
1530 ovs_rwlock_unlock(&rwlock
);
1534 bond_unixctl_set_active_slave(struct unixctl_conn
*conn
,
1535 int argc OVS_UNUSED
, const char *argv
[],
1536 void *aux OVS_UNUSED
)
1538 const char *bond_s
= argv
[1];
1539 const char *slave_s
= argv
[2];
1541 struct bond_slave
*slave
;
1543 ovs_rwlock_wrlock(&rwlock
);
1544 bond
= bond_find(bond_s
);
1546 unixctl_command_reply_error(conn
, "no such bond");
1550 slave
= bond_lookup_slave(bond
, slave_s
);
1552 unixctl_command_reply_error(conn
, "no such slave");
1556 if (!slave
->enabled
) {
1557 unixctl_command_reply_error(conn
, "cannot make disabled slave active");
1561 if (bond
->active_slave
!= slave
) {
1562 bond
->bond_revalidate
= true;
1563 bond
->active_slave
= slave
;
1564 VLOG_INFO("bond %s: active interface is now %s",
1565 bond
->name
, slave
->name
);
1566 bond
->send_learning_packets
= true;
1567 unixctl_command_reply(conn
, "done");
1568 bond_active_slave_changed(bond
);
1570 unixctl_command_reply(conn
, "no change");
1573 ovs_rwlock_unlock(&rwlock
);
1577 enable_slave(struct unixctl_conn
*conn
, const char *argv
[], bool enable
)
1579 const char *bond_s
= argv
[1];
1580 const char *slave_s
= argv
[2];
1582 struct bond_slave
*slave
;
1584 ovs_rwlock_wrlock(&rwlock
);
1585 bond
= bond_find(bond_s
);
1587 unixctl_command_reply_error(conn
, "no such bond");
1591 slave
= bond_lookup_slave(bond
, slave_s
);
1593 unixctl_command_reply_error(conn
, "no such slave");
1597 bond_enable_slave(slave
, enable
);
1598 unixctl_command_reply(conn
, enable
? "enabled" : "disabled");
1601 ovs_rwlock_unlock(&rwlock
);
1605 bond_unixctl_enable_slave(struct unixctl_conn
*conn
,
1606 int argc OVS_UNUSED
, const char *argv
[],
1607 void *aux OVS_UNUSED
)
1609 enable_slave(conn
, argv
, true);
1613 bond_unixctl_disable_slave(struct unixctl_conn
*conn
,
1614 int argc OVS_UNUSED
, const char *argv
[],
1615 void *aux OVS_UNUSED
)
1617 enable_slave(conn
, argv
, false);
1621 bond_unixctl_hash(struct unixctl_conn
*conn
, int argc
, const char *argv
[],
1622 void *aux OVS_UNUSED
)
1624 const char *mac_s
= argv
[1];
1625 const char *vlan_s
= argc
> 2 ? argv
[2] : NULL
;
1626 const char *basis_s
= argc
> 3 ? argv
[3] : NULL
;
1627 struct eth_addr mac
;
1634 if (!ovs_scan(vlan_s
, "%u", &vlan
)) {
1635 unixctl_command_reply_error(conn
, "invalid vlan");
1643 if (!ovs_scan(basis_s
, "%"SCNu32
, &basis
)) {
1644 unixctl_command_reply_error(conn
, "invalid basis");
1651 if (ovs_scan(mac_s
, ETH_ADDR_SCAN_FMT
, ETH_ADDR_SCAN_ARGS(mac
))) {
1652 hash
= hash_mac(mac
, vlan
, basis
) & BOND_MASK
;
1654 hash_cstr
= xasprintf("%u", hash
);
1655 unixctl_command_reply(conn
, hash_cstr
);
1658 unixctl_command_reply_error(conn
, "invalid mac");
1665 unixctl_command_register("bond/list", "", 0, 0, bond_unixctl_list
, NULL
);
1666 unixctl_command_register("bond/show", "[port]", 0, 1, bond_unixctl_show
,
1668 unixctl_command_register("bond/migrate", "port hash slave", 3, 3,
1669 bond_unixctl_migrate
, NULL
);
1670 unixctl_command_register("bond/set-active-slave", "port slave", 2, 2,
1671 bond_unixctl_set_active_slave
, NULL
);
1672 unixctl_command_register("bond/enable-slave", "port slave", 2, 2,
1673 bond_unixctl_enable_slave
, NULL
);
1674 unixctl_command_register("bond/disable-slave", "port slave", 2, 2,
1675 bond_unixctl_disable_slave
, NULL
);
1676 unixctl_command_register("bond/hash", "mac [vlan] [basis]", 1, 3,
1677 bond_unixctl_hash
, NULL
);
1681 bond_entry_reset(struct bond
*bond
)
1683 if (bond
->balance
!= BM_AB
&& !bond_is_falling_back_to_ab(bond
)) {
1684 size_t hash_len
= BOND_BUCKETS
* sizeof *bond
->hash
;
1687 bond
->hash
= xmalloc(hash_len
);
1689 memset(bond
->hash
, 0, hash_len
);
1691 bond
->next_rebalance
= time_msec() + bond
->rebalance_interval
;
1695 /* Remove existing post recirc rules. */
1696 update_recirc_rules(bond
);
1700 static struct bond_slave
*
1701 bond_slave_lookup(struct bond
*bond
, const void *slave_
)
1703 struct bond_slave
*slave
;
1705 HMAP_FOR_EACH_IN_BUCKET (slave
, hmap_node
, hash_pointer(slave_
, 0),
1707 if (slave
->aux
== slave_
) {
1716 bond_enable_slave(struct bond_slave
*slave
, bool enable
)
1718 struct bond
*bond
= slave
->bond
;
1720 slave
->delay_expires
= LLONG_MAX
;
1721 if (enable
!= slave
->enabled
) {
1722 slave
->bond
->bond_revalidate
= true;
1723 slave
->enabled
= enable
;
1725 ovs_mutex_lock(&slave
->bond
->mutex
);
1727 ovs_list_insert(&slave
->bond
->enabled_slaves
, &slave
->list_node
);
1729 bond
->send_learning_packets
= true;
1730 ovs_list_remove(&slave
->list_node
);
1732 ovs_mutex_unlock(&slave
->bond
->mutex
);
1734 VLOG_INFO("interface %s: %s", slave
->name
,
1735 slave
->enabled
? "enabled" : "disabled");
1740 bond_link_status_update(struct bond_slave
*slave
)
1742 struct bond
*bond
= slave
->bond
;
1745 up
= netdev_get_carrier(slave
->netdev
) && slave
->may_enable
;
1746 if ((up
== slave
->enabled
) != (slave
->delay_expires
== LLONG_MAX
)) {
1747 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1748 VLOG_INFO_RL(&rl
, "interface %s: link state %s",
1749 slave
->name
, up
? "up" : "down");
1750 if (up
== slave
->enabled
) {
1751 slave
->delay_expires
= LLONG_MAX
;
1752 VLOG_INFO_RL(&rl
, "interface %s: will not be %s",
1753 slave
->name
, up
? "disabled" : "enabled");
1755 int delay
= up
? bond
->updelay
: bond
->downdelay
;
1756 slave
->delay_expires
= time_msec() + delay
;
1758 VLOG_INFO_RL(&rl
, "interface %s: will be %s if it stays %s "
1761 up
? "enabled" : "disabled",
1768 if (time_msec() >= slave
->delay_expires
) {
1769 bond_enable_slave(slave
, up
);
1774 bond_hash(const struct bond
*bond
, const struct flow
*flow
, uint16_t vlan
)
1776 ovs_assert(bond
->balance
== BM_TCP
|| bond
->balance
== BM_SLB
);
1778 return (bond
->balance
== BM_TCP
1779 ? flow_hash_5tuple(flow
, bond
->basis
)
1780 : hash_mac(flow
->dl_src
, vlan
, bond
->basis
));
1783 static struct bond_entry
*
1784 lookup_bond_entry(const struct bond
*bond
, const struct flow
*flow
,
1787 return &bond
->hash
[bond_hash(bond
, flow
, vlan
) & BOND_MASK
];
1790 /* Selects and returns an enabled slave from the 'enabled_slaves' list
1791 * in a round-robin fashion. If the 'enabled_slaves' list is empty,
1793 static struct bond_slave
*
1794 get_enabled_slave(struct bond
*bond
)
1796 struct ovs_list
*node
;
1798 ovs_mutex_lock(&bond
->mutex
);
1799 if (ovs_list_is_empty(&bond
->enabled_slaves
)) {
1800 ovs_mutex_unlock(&bond
->mutex
);
1804 node
= ovs_list_pop_front(&bond
->enabled_slaves
);
1805 ovs_list_push_back(&bond
->enabled_slaves
, node
);
1806 ovs_mutex_unlock(&bond
->mutex
);
1808 return CONTAINER_OF(node
, struct bond_slave
, list_node
);
1811 static struct bond_slave
*
1812 choose_output_slave(const struct bond
*bond
, const struct flow
*flow
,
1813 struct flow_wildcards
*wc
, uint16_t vlan
)
1815 struct bond_entry
*e
;
1818 balance
= bond
->balance
;
1819 if (bond
->lacp_status
== LACP_CONFIGURED
) {
1820 /* LACP has been configured on this bond but negotiations were
1821 * unsuccussful. If lacp_fallback_ab is enabled use active-
1822 * backup mode else drop all traffic. */
1823 if (!bond
->lacp_fallback_ab
) {
1831 return bond
->active_slave
;
1834 if (bond
->lacp_status
!= LACP_NEGOTIATED
) {
1835 /* Must have LACP negotiations for TCP balanced bonds. */
1839 flow_mask_hash_fields(flow
, wc
, NX_HASH_FIELDS_SYMMETRIC_L3L4_UDP
);
1843 if (wc
&& balance
== BM_SLB
) {
1844 flow_mask_hash_fields(flow
, wc
, NX_HASH_FIELDS_ETH_SRC
);
1846 e
= lookup_bond_entry(bond
, flow
, vlan
);
1847 if (!e
->slave
|| !e
->slave
->enabled
) {
1848 e
->slave
= get_enabled_slave(CONST_CAST(struct bond
*, bond
));
1857 static struct bond_slave
*
1858 bond_choose_slave(const struct bond
*bond
)
1860 struct bond_slave
*slave
, *best
;
1862 /* Find the last active slave. */
1863 slave
= bond_find_slave_by_mac(bond
, bond
->active_slave_mac
);
1864 if (slave
&& slave
->enabled
) {
1868 /* Find an enabled slave. */
1869 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1870 if (slave
->enabled
) {
1875 /* All interfaces are disabled. Find an interface that will be enabled
1876 * after its updelay expires. */
1878 HMAP_FOR_EACH (slave
, hmap_node
, &bond
->slaves
) {
1879 if (slave
->delay_expires
!= LLONG_MAX
1880 && slave
->may_enable
1881 && (!best
|| slave
->delay_expires
< best
->delay_expires
)) {
1889 bond_choose_active_slave(struct bond
*bond
)
1891 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(5, 20);
1892 struct bond_slave
*old_active_slave
= bond
->active_slave
;
1894 bond
->active_slave
= bond_choose_slave(bond
);
1895 if (bond
->active_slave
) {
1896 if (bond
->active_slave
->enabled
) {
1897 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s",
1898 bond
->name
, bond
->active_slave
->name
);
1900 VLOG_INFO_RL(&rl
, "bond %s: active interface is now %s, skipping "
1901 "remaining %lld ms updelay (since no interface was "
1902 "enabled)", bond
->name
, bond
->active_slave
->name
,
1903 bond
->active_slave
->delay_expires
- time_msec());
1904 bond_enable_slave(bond
->active_slave
, true);
1907 bond
->send_learning_packets
= true;
1909 if (bond
->active_slave
!= old_active_slave
) {
1910 bond_active_slave_changed(bond
);
1912 } else if (old_active_slave
) {
1913 bond_active_slave_changed(bond
);
1914 VLOG_INFO_RL(&rl
, "bond %s: all interfaces disabled", bond
->name
);
1919 * Return true if bond has unstored active slave change.
1920 * If return true, 'mac' will store the bond's current active slave's
1923 bond_get_changed_active_slave(const char *name
, struct eth_addr
*mac
,
1928 ovs_rwlock_wrlock(&rwlock
);
1929 bond
= bond_find(name
);
1931 if (bond
->active_slave_changed
|| force
) {
1932 *mac
= bond
->active_slave_mac
;
1933 bond
->active_slave_changed
= false;
1934 ovs_rwlock_unlock(&rwlock
);
1938 ovs_rwlock_unlock(&rwlock
);