2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "fail-open.h"
32 #include "poll-loop.h"
40 VLOG_DEFINE_THIS_MODULE(connmgr
);
41 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
43 COVERAGE_DEFINE(ofconn_stuck
);
45 /* An OpenFlow connection. */
47 struct connmgr
*connmgr
; /* Connection's manager. */
48 struct list node
; /* In struct connmgr's "all_conns" list. */
49 struct rconn
*rconn
; /* OpenFlow connection. */
50 enum ofconn_type type
; /* Type. */
51 enum nx_flow_format flow_format
; /* Currently selected flow format. */
53 /* OFPT_PACKET_IN related data. */
54 struct rconn_packet_counter
*packet_in_counter
; /* # queued on 'rconn'. */
55 #define N_SCHEDULERS 2
56 struct pinsched
*schedulers
[N_SCHEDULERS
];
57 struct pktbuf
*pktbuf
; /* OpenFlow packet buffers. */
58 int miss_send_len
; /* Bytes to send of buffered packets. */
60 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
61 * requests, and the maximum number before we stop reading OpenFlow
63 #define OFCONN_REPLY_MAX 100
64 struct rconn_packet_counter
*reply_counter
;
66 /* type == OFCONN_PRIMARY only. */
67 enum nx_role role
; /* Role. */
68 struct hmap_node hmap_node
; /* In struct connmgr's "controllers" map. */
69 enum ofproto_band band
; /* In-band or out-of-band? */
72 static struct ofconn
*ofconn_create(struct connmgr
*, struct rconn
*,
74 static void ofconn_destroy(struct ofconn
*);
76 static void ofconn_reconfigure(struct ofconn
*,
77 const struct ofproto_controller
*);
79 static void ofconn_run(struct ofconn
*,
80 void (*handle_openflow
)(struct ofconn
*,
81 struct ofpbuf
*ofp_msg
));
82 static void ofconn_wait(struct ofconn
*);
84 static const char *ofconn_get_target(const struct ofconn
*);
85 static char *ofconn_make_name(const struct connmgr
*, const char *target
);
87 static void ofconn_set_rate_limit(struct ofconn
*, int rate
, int burst
);
89 static bool ofconn_receives_async_msgs(const struct ofconn
*);
91 static void ofconn_send(const struct ofconn
*, struct ofpbuf
*,
92 struct rconn_packet_counter
*);
94 static void do_send_packet_in(struct ofpbuf
*, void *ofconn_
);
96 /* A listener for incoming OpenFlow "service" connections. */
98 struct hmap_node node
; /* In struct connmgr's "services" hmap. */
99 struct pvconn
*pvconn
; /* OpenFlow connection listener. */
101 /* These are not used by ofservice directly. They are settings for
102 * accepted "struct ofconn"s from the pvconn. */
103 int probe_interval
; /* Max idle time before probing, in seconds. */
104 int rate_limit
; /* Max packet-in rate in packets per second. */
105 int burst_limit
; /* Limit on accumulating packet credits. */
108 static void ofservice_reconfigure(struct ofservice
*,
109 const struct ofproto_controller
*);
110 static int ofservice_create(struct connmgr
*, const char *target
);
111 static void ofservice_destroy(struct connmgr
*, struct ofservice
*);
112 static struct ofservice
*ofservice_lookup(struct connmgr
*,
115 /* Connection manager for an OpenFlow switch. */
117 struct ofproto
*ofproto
;
119 char *local_port_name
;
121 /* OpenFlow connections. */
122 struct hmap controllers
; /* Controller "struct ofconn"s. */
123 struct list all_conns
; /* Contains "struct ofconn"s. */
125 /* OpenFlow listeners. */
126 struct hmap services
; /* Contains "struct ofservice"s. */
127 struct pvconn
**snoops
;
131 struct fail_open
*fail_open
;
132 enum ofproto_fail_mode fail_mode
;
134 /* In-band control. */
135 struct in_band
*in_band
;
136 long long int next_in_band_update
;
137 struct sockaddr_in
*extra_in_band_remotes
;
138 size_t n_extra_remotes
;
142 static void update_in_band_remotes(struct connmgr
*);
143 static void add_snooper(struct connmgr
*, struct vconn
*);
145 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
146 * a name for the ofproto suitable for using in log messages.
147 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
150 connmgr_create(struct ofproto
*ofproto
,
151 const char *name
, const char *local_port_name
)
155 mgr
= xmalloc(sizeof *mgr
);
156 mgr
->ofproto
= ofproto
;
157 mgr
->name
= xstrdup(name
);
158 mgr
->local_port_name
= xstrdup(local_port_name
);
160 hmap_init(&mgr
->controllers
);
161 list_init(&mgr
->all_conns
);
163 hmap_init(&mgr
->services
);
167 mgr
->fail_open
= NULL
;
168 mgr
->fail_mode
= OFPROTO_FAIL_SECURE
;
171 mgr
->next_in_band_update
= LLONG_MAX
;
172 mgr
->extra_in_band_remotes
= NULL
;
173 mgr
->n_extra_remotes
= 0;
174 mgr
->in_band_queue
= -1;
179 /* Frees 'mgr' and all of its resources. */
181 connmgr_destroy(struct connmgr
*mgr
)
183 struct ofservice
*ofservice
, *next_ofservice
;
184 struct ofconn
*ofconn
, *next_ofconn
;
191 LIST_FOR_EACH_SAFE (ofconn
, next_ofconn
, node
, &mgr
->all_conns
) {
192 ofconn_destroy(ofconn
);
194 hmap_destroy(&mgr
->controllers
);
196 HMAP_FOR_EACH_SAFE (ofservice
, next_ofservice
, node
, &mgr
->services
) {
197 ofservice_destroy(mgr
, ofservice
);
199 hmap_destroy(&mgr
->services
);
201 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
202 pvconn_close(mgr
->snoops
[i
]);
206 fail_open_destroy(mgr
->fail_open
);
207 mgr
->fail_open
= NULL
;
209 in_band_destroy(mgr
->in_band
);
211 free(mgr
->extra_in_band_remotes
);
213 free(mgr
->local_port_name
);
218 /* Does all of the periodic maintenance required by 'mgr'. Calls
219 * 'handle_openflow' for each message received on an OpenFlow connection,
220 * passing along the OpenFlow connection itself and the message that was sent.
221 * The 'handle_openflow' callback must not free the message. */
223 connmgr_run(struct connmgr
*mgr
,
224 void (*handle_openflow
)(struct ofconn
*, struct ofpbuf
*ofp_msg
))
226 struct ofconn
*ofconn
, *next_ofconn
;
227 struct ofservice
*ofservice
;
231 if (time_msec() >= mgr
->next_in_band_update
) {
232 update_in_band_remotes(mgr
);
234 in_band_run(mgr
->in_band
);
237 LIST_FOR_EACH_SAFE (ofconn
, next_ofconn
, node
, &mgr
->all_conns
) {
238 ofconn_run(ofconn
, handle_openflow
);
241 /* Fail-open maintenance. Do this after processing the ofconns since
242 * fail-open checks the status of the controller rconn. */
243 if (mgr
->fail_open
) {
244 fail_open_run(mgr
->fail_open
);
247 HMAP_FOR_EACH (ofservice
, node
, &mgr
->services
) {
251 retval
= pvconn_accept(ofservice
->pvconn
, OFP_VERSION
, &vconn
);
256 rconn
= rconn_create(ofservice
->probe_interval
, 0);
257 name
= ofconn_make_name(mgr
, vconn_get_name(vconn
));
258 rconn_connect_unreliably(rconn
, vconn
, name
);
261 ofconn
= ofconn_create(mgr
, rconn
, OFCONN_SERVICE
);
262 ofconn_set_rate_limit(ofconn
, ofservice
->rate_limit
,
263 ofservice
->burst_limit
);
264 } else if (retval
!= EAGAIN
) {
265 VLOG_WARN_RL(&rl
, "accept failed (%s)", strerror(retval
));
269 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
273 retval
= pvconn_accept(mgr
->snoops
[i
], OFP_VERSION
, &vconn
);
275 add_snooper(mgr
, vconn
);
276 } else if (retval
!= EAGAIN
) {
277 VLOG_WARN_RL(&rl
, "accept failed (%s)", strerror(retval
));
282 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
284 connmgr_wait(struct connmgr
*mgr
)
286 struct ofservice
*ofservice
;
287 struct ofconn
*ofconn
;
290 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
294 poll_timer_wait_until(mgr
->next_in_band_update
);
295 in_band_wait(mgr
->in_band
);
297 if (mgr
->fail_open
) {
298 fail_open_wait(mgr
->fail_open
);
300 HMAP_FOR_EACH (ofservice
, node
, &mgr
->services
) {
301 pvconn_wait(ofservice
->pvconn
);
303 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
304 pvconn_wait(mgr
->snoops
[i
]);
308 /* Returns the ofproto that owns 'ofconn''s connmgr. */
310 ofconn_get_ofproto(const struct ofconn
*ofconn
)
312 return ofconn
->connmgr
->ofproto
;
315 /* OpenFlow configuration. */
317 static void add_controller(struct connmgr
*, const char *target
);
318 static struct ofconn
*find_controller_by_target(struct connmgr
*,
320 static void update_fail_open(struct connmgr
*);
321 static int set_pvconns(struct pvconn
***pvconnsp
, size_t *n_pvconnsp
,
322 const struct svec
*);
324 /* Returns true if 'mgr' has any configured primary controllers.
326 * Service controllers do not count, but configured primary controllers do
327 * count whether or not they are currently connected. */
329 connmgr_has_controllers(const struct connmgr
*mgr
)
331 return !hmap_is_empty(&mgr
->controllers
);
334 /* Initializes 'info' and populates it with information about each configured
335 * primary controller. The keys in 'info' are the controllers' targets; the
336 * data values are corresponding "struct ofproto_controller_info".
338 * The caller owns 'info' and everything in it and should free it when it is no
341 connmgr_get_controller_info(struct connmgr
*mgr
, struct shash
*info
)
343 const struct ofconn
*ofconn
;
347 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
348 const struct rconn
*rconn
= ofconn
->rconn
;
349 time_t now
= time_now();
350 time_t last_connection
= rconn_get_last_connection(rconn
);
351 time_t last_disconnect
= rconn_get_last_disconnect(rconn
);
352 int last_error
= rconn_get_last_error(rconn
);
353 struct ofproto_controller_info
*cinfo
= xmalloc(sizeof *cinfo
);
355 shash_add(info
, rconn_get_target(rconn
), cinfo
);
357 cinfo
->is_connected
= rconn_is_connected(rconn
);
358 cinfo
->role
= ofconn
->role
;
363 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "last_error";
364 cinfo
->pairs
.values
[cinfo
->pairs
.n
++] =
365 xstrdup(ovs_retval_to_string(last_error
));
368 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "state";
369 cinfo
->pairs
.values
[cinfo
->pairs
.n
++] =
370 xstrdup(rconn_get_state(rconn
));
372 if (last_connection
!= TIME_MIN
) {
373 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "sec_since_connect";
374 cinfo
->pairs
.values
[cinfo
->pairs
.n
++]
375 = xasprintf("%ld", (long int) (now
- last_connection
));
378 if (last_disconnect
!= TIME_MIN
) {
379 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "sec_since_disconnect";
380 cinfo
->pairs
.values
[cinfo
->pairs
.n
++]
381 = xasprintf("%ld", (long int) (now
- last_disconnect
));
386 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
389 connmgr_set_controllers(struct connmgr
*mgr
,
390 const struct ofproto_controller
*controllers
,
391 size_t n_controllers
)
393 struct shash new_controllers
;
394 struct ofconn
*ofconn
, *next_ofconn
;
395 struct ofservice
*ofservice
, *next_ofservice
;
399 /* Create newly configured controllers and services.
400 * Create a name to ofproto_controller mapping in 'new_controllers'. */
401 shash_init(&new_controllers
);
402 for (i
= 0; i
< n_controllers
; i
++) {
403 const struct ofproto_controller
*c
= &controllers
[i
];
405 if (!vconn_verify_name(c
->target
)) {
406 if (!find_controller_by_target(mgr
, c
->target
)) {
407 add_controller(mgr
, c
->target
);
409 } else if (!pvconn_verify_name(c
->target
)) {
410 if (!ofservice_lookup(mgr
, c
->target
)) {
411 ofservice_create(mgr
, c
->target
);
414 VLOG_WARN_RL(&rl
, "%s: unsupported controller \"%s\"",
415 mgr
->name
, c
->target
);
419 shash_add_once(&new_controllers
, c
->target
, &controllers
[i
]);
422 /* Delete controllers that are no longer configured.
423 * Update configuration of all now-existing controllers. */
425 HMAP_FOR_EACH_SAFE (ofconn
, next_ofconn
, hmap_node
, &mgr
->controllers
) {
426 struct ofproto_controller
*c
;
428 c
= shash_find_data(&new_controllers
, ofconn_get_target(ofconn
));
430 ofconn_destroy(ofconn
);
432 ofconn_reconfigure(ofconn
, c
);
436 /* Delete services that are no longer configured.
437 * Update configuration of all now-existing services. */
438 HMAP_FOR_EACH_SAFE (ofservice
, next_ofservice
, node
, &mgr
->services
) {
439 struct ofproto_controller
*c
;
441 c
= shash_find_data(&new_controllers
,
442 pvconn_get_name(ofservice
->pvconn
));
444 ofservice_destroy(mgr
, ofservice
);
446 ofservice_reconfigure(ofservice
, c
);
450 shash_destroy(&new_controllers
);
452 update_in_band_remotes(mgr
);
453 update_fail_open(mgr
);
456 /* Drops the connections between 'mgr' and all of its primary and secondary
457 * controllers, forcing them to reconnect. */
459 connmgr_reconnect(const struct connmgr
*mgr
)
461 struct ofconn
*ofconn
;
463 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
464 rconn_reconnect(ofconn
->rconn
);
468 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
470 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
471 * important controller on 'mgr' is mirrored. */
473 connmgr_set_snoops(struct connmgr
*mgr
, const struct svec
*snoops
)
475 return set_pvconns(&mgr
->snoops
, &mgr
->n_snoops
, snoops
);
478 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
480 connmgr_get_snoops(const struct connmgr
*mgr
, struct svec
*snoops
)
484 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
485 svec_add(snoops
, pvconn_get_name(mgr
->snoops
[i
]));
489 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
490 * to be called later to finish the new ofconn's configuration. */
492 add_controller(struct connmgr
*mgr
, const char *target
)
494 char *name
= ofconn_make_name(mgr
, target
);
495 struct ofconn
*ofconn
;
497 ofconn
= ofconn_create(mgr
, rconn_create(5, 8), OFCONN_PRIMARY
);
498 ofconn
->pktbuf
= pktbuf_create();
499 ofconn
->miss_send_len
= OFP_DEFAULT_MISS_SEND_LEN
;
500 rconn_connect(ofconn
->rconn
, target
, name
);
501 hmap_insert(&mgr
->controllers
, &ofconn
->hmap_node
, hash_string(target
, 0));
506 static struct ofconn
*
507 find_controller_by_target(struct connmgr
*mgr
, const char *target
)
509 struct ofconn
*ofconn
;
511 HMAP_FOR_EACH_WITH_HASH (ofconn
, hmap_node
,
512 hash_string(target
, 0), &mgr
->controllers
) {
513 if (!strcmp(ofconn_get_target(ofconn
), target
)) {
521 update_in_band_remotes(struct connmgr
*mgr
)
523 struct sockaddr_in
*addrs
;
524 size_t max_addrs
, n_addrs
;
525 struct ofconn
*ofconn
;
528 /* Allocate enough memory for as many remotes as we could possibly have. */
529 max_addrs
= mgr
->n_extra_remotes
+ hmap_count(&mgr
->controllers
);
530 addrs
= xmalloc(max_addrs
* sizeof *addrs
);
533 /* Add all the remotes. */
534 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
535 struct sockaddr_in
*sin
= &addrs
[n_addrs
];
537 if (ofconn
->band
== OFPROTO_OUT_OF_BAND
) {
541 sin
->sin_addr
.s_addr
= rconn_get_remote_ip(ofconn
->rconn
);
542 if (sin
->sin_addr
.s_addr
) {
543 sin
->sin_port
= rconn_get_remote_port(ofconn
->rconn
);
547 for (i
= 0; i
< mgr
->n_extra_remotes
; i
++) {
548 addrs
[n_addrs
++] = mgr
->extra_in_band_remotes
[i
];
551 /* Create or update or destroy in-band. */
554 in_band_create(mgr
->ofproto
, mgr
->local_port_name
, &mgr
->in_band
);
557 in_band_set_remotes(mgr
->in_band
, addrs
, n_addrs
);
559 in_band_set_queue(mgr
->in_band
, mgr
->in_band_queue
);
560 mgr
->next_in_band_update
= time_msec() + 1000;
562 in_band_destroy(mgr
->in_band
);
571 update_fail_open(struct connmgr
*mgr
)
573 if (connmgr_has_controllers(mgr
)
574 && mgr
->fail_mode
== OFPROTO_FAIL_STANDALONE
) {
575 if (!mgr
->fail_open
) {
576 mgr
->fail_open
= fail_open_create(mgr
->ofproto
, mgr
);
579 fail_open_destroy(mgr
->fail_open
);
580 mgr
->fail_open
= NULL
;
585 set_pvconns(struct pvconn
***pvconnsp
, size_t *n_pvconnsp
,
586 const struct svec
*svec
)
588 struct pvconn
**pvconns
= *pvconnsp
;
589 size_t n_pvconns
= *n_pvconnsp
;
593 for (i
= 0; i
< n_pvconns
; i
++) {
594 pvconn_close(pvconns
[i
]);
598 pvconns
= xmalloc(svec
->n
* sizeof *pvconns
);
600 for (i
= 0; i
< svec
->n
; i
++) {
601 const char *name
= svec
->names
[i
];
602 struct pvconn
*pvconn
;
605 error
= pvconn_open(name
, &pvconn
);
607 pvconns
[n_pvconns
++] = pvconn
;
609 VLOG_ERR("failed to listen on %s: %s", name
, strerror(error
));
617 *n_pvconnsp
= n_pvconns
;
622 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
623 * means that 'ofconn' is more interesting for monitoring than a lower return
626 snoop_preference(const struct ofconn
*ofconn
)
628 switch (ofconn
->role
) {
636 /* Shouldn't happen. */
641 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
642 * Connects this vconn to a controller. */
644 add_snooper(struct connmgr
*mgr
, struct vconn
*vconn
)
646 struct ofconn
*ofconn
, *best
;
648 /* Pick a controller for monitoring. */
650 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
651 if (ofconn
->type
== OFCONN_PRIMARY
652 && (!best
|| snoop_preference(ofconn
) > snoop_preference(best
))) {
658 rconn_add_monitor(best
->rconn
, vconn
);
660 VLOG_INFO_RL(&rl
, "no controller connection to snoop");
665 /* Public ofconn functions. */
667 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
669 ofconn_get_type(const struct ofconn
*ofconn
)
674 /* Returns the role configured for 'ofconn'.
676 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
678 ofconn_get_role(const struct ofconn
*ofconn
)
683 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
684 * existing master is demoted to a slave. */
686 ofconn_set_role(struct ofconn
*ofconn
, enum nx_role role
)
688 if (role
== NX_ROLE_MASTER
) {
689 struct ofconn
*other
;
691 HMAP_FOR_EACH (other
, hmap_node
, &ofconn
->connmgr
->controllers
) {
692 if (other
->role
== NX_ROLE_MASTER
) {
693 other
->role
= NX_ROLE_SLAVE
;
700 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
702 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
704 ofconn_get_flow_format(struct ofconn
*ofconn
)
706 return ofconn
->flow_format
;
709 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
711 ofconn_set_flow_format(struct ofconn
*ofconn
, enum nx_flow_format flow_format
)
713 ofconn
->flow_format
= flow_format
;
716 /* Returns the default miss send length for 'ofconn'. */
718 ofconn_get_miss_send_len(const struct ofconn
*ofconn
)
720 return ofconn
->miss_send_len
;
723 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
725 ofconn_set_miss_send_len(struct ofconn
*ofconn
, int miss_send_len
)
727 ofconn
->miss_send_len
= miss_send_len
;
730 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
731 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
732 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
733 * controller has accepted some of the replies.) */
735 ofconn_send_reply(const struct ofconn
*ofconn
, struct ofpbuf
*msg
)
737 ofconn_send(ofconn
, msg
, ofconn
->reply_counter
);
740 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
742 ofconn_pktbuf_retrieve(struct ofconn
*ofconn
, uint32_t id
,
743 struct ofpbuf
**bufferp
, uint16_t *in_port
)
745 return pktbuf_retrieve(ofconn
->pktbuf
, id
, bufferp
, in_port
);
748 /* Private ofconn functions. */
751 ofconn_get_target(const struct ofconn
*ofconn
)
753 return rconn_get_target(ofconn
->rconn
);
756 static struct ofconn
*
757 ofconn_create(struct connmgr
*mgr
, struct rconn
*rconn
, enum ofconn_type type
)
759 struct ofconn
*ofconn
= xzalloc(sizeof *ofconn
);
760 ofconn
->connmgr
= mgr
;
761 list_push_back(&mgr
->all_conns
, &ofconn
->node
);
762 ofconn
->rconn
= rconn
;
764 ofconn
->flow_format
= NXFF_OPENFLOW10
;
765 ofconn
->role
= NX_ROLE_OTHER
;
766 ofconn
->packet_in_counter
= rconn_packet_counter_create ();
767 ofconn
->pktbuf
= NULL
;
768 ofconn
->miss_send_len
= 0;
769 ofconn
->reply_counter
= rconn_packet_counter_create ();
774 ofconn_destroy(struct ofconn
*ofconn
)
776 if (ofconn
->type
== OFCONN_PRIMARY
) {
777 hmap_remove(&ofconn
->connmgr
->controllers
, &ofconn
->hmap_node
);
780 list_remove(&ofconn
->node
);
781 rconn_destroy(ofconn
->rconn
);
782 rconn_packet_counter_destroy(ofconn
->packet_in_counter
);
783 rconn_packet_counter_destroy(ofconn
->reply_counter
);
784 pktbuf_destroy(ofconn
->pktbuf
);
788 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
791 ofconn_reconfigure(struct ofconn
*ofconn
, const struct ofproto_controller
*c
)
795 ofconn
->band
= c
->band
;
797 rconn_set_max_backoff(ofconn
->rconn
, c
->max_backoff
);
799 probe_interval
= c
->probe_interval
? MAX(c
->probe_interval
, 5) : 0;
800 rconn_set_probe_interval(ofconn
->rconn
, probe_interval
);
802 ofconn_set_rate_limit(ofconn
, c
->rate_limit
, c
->burst_limit
);
806 ofconn_run(struct ofconn
*ofconn
,
807 void (*handle_openflow
)(struct ofconn
*, struct ofpbuf
*ofp_msg
))
809 struct connmgr
*mgr
= ofconn
->connmgr
;
813 for (i
= 0; i
< N_SCHEDULERS
; i
++) {
814 pinsched_run(ofconn
->schedulers
[i
], do_send_packet_in
, ofconn
);
817 rconn_run(ofconn
->rconn
);
819 if (rconn_packet_counter_read (ofconn
->reply_counter
) < OFCONN_REPLY_MAX
) {
820 /* Limit the number of iterations to prevent other tasks from
822 for (iteration
= 0; iteration
< 50; iteration
++) {
823 struct ofpbuf
*of_msg
= rconn_recv(ofconn
->rconn
);
827 if (mgr
->fail_open
) {
828 fail_open_maybe_recover(mgr
->fail_open
);
830 handle_openflow(ofconn
, of_msg
);
831 ofpbuf_delete(of_msg
);
835 if (!rconn_is_alive(ofconn
->rconn
)) {
836 ofconn_destroy(ofconn
);
841 ofconn_wait(struct ofconn
*ofconn
)
845 for (i
= 0; i
< N_SCHEDULERS
; i
++) {
846 pinsched_wait(ofconn
->schedulers
[i
]);
848 rconn_run_wait(ofconn
->rconn
);
849 if (rconn_packet_counter_read (ofconn
->reply_counter
) < OFCONN_REPLY_MAX
) {
850 rconn_recv_wait(ofconn
->rconn
);
852 COVERAGE_INC(ofconn_stuck
);
856 /* Returns true if 'ofconn' should receive asynchronous messages. */
858 ofconn_receives_async_msgs(const struct ofconn
*ofconn
)
860 if (!rconn_is_connected(ofconn
->rconn
)) {
862 } else if (ofconn
->type
== OFCONN_PRIMARY
) {
863 /* Primary controllers always get asynchronous messages unless they
864 * have configured themselves as "slaves". */
865 return ofconn
->role
!= NX_ROLE_SLAVE
;
867 /* Service connections don't get asynchronous messages unless they have
868 * explicitly asked for them by setting a nonzero miss send length. */
869 return ofconn
->miss_send_len
> 0;
873 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
874 * 'target', suitable for use in log messages for identifying the connection.
876 * The name is dynamically allocated. The caller should free it (with free())
877 * when it is no longer needed. */
879 ofconn_make_name(const struct connmgr
*mgr
, const char *target
)
881 return xasprintf("%s<->%s", mgr
->name
, target
);
885 ofconn_set_rate_limit(struct ofconn
*ofconn
, int rate
, int burst
)
889 for (i
= 0; i
< N_SCHEDULERS
; i
++) {
890 struct pinsched
**s
= &ofconn
->schedulers
[i
];
894 *s
= pinsched_create(rate
, burst
);
896 pinsched_set_limits(*s
, rate
, burst
);
899 pinsched_destroy(*s
);
906 ofconn_send(const struct ofconn
*ofconn
, struct ofpbuf
*msg
,
907 struct rconn_packet_counter
*counter
)
909 update_openflow_length(msg
);
910 if (rconn_send(ofconn
->rconn
, msg
, counter
)) {
915 /* Sending asynchronous messages. */
917 static void schedule_packet_in(struct ofconn
*, const struct dpif_upcall
*,
918 const struct flow
*, struct ofpbuf
*rw_packet
);
920 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
921 * controllers managed by 'mgr'.
923 * 'opp' is in *HOST* byte order. */
925 connmgr_send_port_status(struct connmgr
*mgr
, const struct ofp_phy_port
*opp
,
928 /* XXX Should limit the number of queued port status change messages. */
929 struct ofconn
*ofconn
;
931 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
932 struct ofp_port_status
*ops
;
935 /* Primary controllers, even slaves, should always get port status
936 updates. Otherwise obey ofconn_receives_async_msgs(). */
937 if (ofconn
->type
!= OFCONN_PRIMARY
938 && !ofconn_receives_async_msgs(ofconn
)) {
942 ops
= make_openflow_xid(sizeof *ops
, OFPT_PORT_STATUS
, 0, &b
);
943 ops
->reason
= reason
;
945 hton_ofp_phy_port(&ops
->desc
);
946 ofconn_send(ofconn
, b
, NULL
);
950 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
951 * appropriate controllers managed by 'mgr'. */
953 connmgr_send_flow_removed(struct connmgr
*mgr
,
954 const struct ofputil_flow_removed
*fr
)
956 struct ofconn
*ofconn
;
958 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
961 if (!ofconn_receives_async_msgs(ofconn
)) {
965 /* Account flow expirations as replies to OpenFlow requests. That
966 * works because preventing OpenFlow requests from being processed also
967 * prevents new flows from being added (and expiring). (It also
968 * prevents processing OpenFlow requests that would not add new flows,
969 * so it is imperfect.) */
970 msg
= ofputil_encode_flow_removed(fr
, ofconn
->flow_format
);
971 ofconn_send_reply(ofconn
, msg
);
975 /* Given 'upcall', of type DPIF_UC_ACTION or DPIF_UC_MISS, sends an
976 * OFPT_PACKET_IN message to each OpenFlow controller as necessary according to
977 * their individual configurations.
979 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
980 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
981 * of 'rw_packet' is transferred to this function. */
983 connmgr_send_packet_in(struct connmgr
*mgr
, const struct dpif_upcall
*upcall
,
984 const struct flow
*flow
, struct ofpbuf
*rw_packet
)
986 struct ofconn
*ofconn
, *prev
;
989 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
990 if (ofconn_receives_async_msgs(ofconn
)) {
992 schedule_packet_in(prev
, upcall
, flow
, NULL
);
998 schedule_packet_in(prev
, upcall
, flow
, rw_packet
);
1000 ofpbuf_delete(rw_packet
);
1004 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1006 do_send_packet_in(struct ofpbuf
*ofp_packet_in
, void *ofconn_
)
1008 struct ofconn
*ofconn
= ofconn_
;
1010 rconn_send_with_limit(ofconn
->rconn
, ofp_packet_in
,
1011 ofconn
->packet_in_counter
, 100);
1014 /* Takes 'upcall', whose packet has the flow specified by 'flow', composes an
1015 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1016 * scheduler for sending.
1018 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1019 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
1020 * of 'rw_packet' is transferred to this function. */
1022 schedule_packet_in(struct ofconn
*ofconn
, const struct dpif_upcall
*upcall
,
1023 const struct flow
*flow
, struct ofpbuf
*rw_packet
)
1025 struct connmgr
*mgr
= ofconn
->connmgr
;
1026 struct ofputil_packet_in pin
;
1028 /* Figure out the easy parts. */
1029 pin
.packet
= upcall
->packet
;
1030 pin
.in_port
= odp_port_to_ofp_port(flow
->in_port
);
1031 pin
.reason
= upcall
->type
== DPIF_UC_MISS
? OFPR_NO_MATCH
: OFPR_ACTION
;
1033 /* Get OpenFlow buffer_id. */
1034 if (upcall
->type
== DPIF_UC_ACTION
) {
1035 pin
.buffer_id
= UINT32_MAX
;
1036 } else if (mgr
->fail_open
&& fail_open_is_active(mgr
->fail_open
)) {
1037 pin
.buffer_id
= pktbuf_get_null();
1038 } else if (!ofconn
->pktbuf
) {
1039 pin
.buffer_id
= UINT32_MAX
;
1041 pin
.buffer_id
= pktbuf_save(ofconn
->pktbuf
, upcall
->packet
,
1045 /* Figure out how much of the packet to send. */
1046 pin
.send_len
= upcall
->packet
->size
;
1047 if (pin
.buffer_id
!= UINT32_MAX
) {
1048 pin
.send_len
= MIN(pin
.send_len
, ofconn
->miss_send_len
);
1050 if (upcall
->type
== DPIF_UC_ACTION
) {
1051 pin
.send_len
= MIN(pin
.send_len
, upcall
->userdata
);
1054 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1055 * immediately call into do_send_packet_in() or it might buffer it for a
1056 * while (until a later call to pinsched_run()). */
1057 pinsched_send(ofconn
->schedulers
[upcall
->type
== DPIF_UC_MISS
? 0 : 1],
1058 flow
->in_port
, ofputil_encode_packet_in(&pin
, rw_packet
),
1059 do_send_packet_in
, ofconn
);
1062 /* Fail-open settings. */
1064 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1065 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1066 enum ofproto_fail_mode
1067 connmgr_get_fail_mode(const struct connmgr
*mgr
)
1069 return mgr
->fail_mode
;
1072 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1073 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1075 connmgr_set_fail_mode(struct connmgr
*mgr
, enum ofproto_fail_mode fail_mode
)
1077 mgr
->fail_mode
= fail_mode
;
1078 update_fail_open(mgr
);
1081 /* Fail-open implementation. */
1083 /* Returns the longest probe interval among the primary controllers configured
1084 * on 'mgr'. Returns 0 if there are no primary controllers. */
1086 connmgr_get_max_probe_interval(const struct connmgr
*mgr
)
1088 const struct ofconn
*ofconn
;
1089 int max_probe_interval
;
1091 max_probe_interval
= 0;
1092 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1093 int probe_interval
= rconn_get_probe_interval(ofconn
->rconn
);
1094 max_probe_interval
= MAX(max_probe_interval
, probe_interval
);
1096 return max_probe_interval
;
1099 /* Returns the number of seconds for which all of 'mgr's primary controllers
1100 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1102 connmgr_failure_duration(const struct connmgr
*mgr
)
1104 const struct ofconn
*ofconn
;
1105 int min_failure_duration
;
1107 if (!connmgr_has_controllers(mgr
)) {
1111 min_failure_duration
= INT_MAX
;
1112 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1113 int failure_duration
= rconn_failure_duration(ofconn
->rconn
);
1114 min_failure_duration
= MIN(min_failure_duration
, failure_duration
);
1116 return min_failure_duration
;
1119 /* Returns true if at least one primary controller is connected (regardless of
1120 * whether those controllers are believed to have authenticated and accepted
1121 * this switch), false if none of them are connected. */
1123 connmgr_is_any_controller_connected(const struct connmgr
*mgr
)
1125 const struct ofconn
*ofconn
;
1127 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1128 if (rconn_is_connected(ofconn
->rconn
)) {
1135 /* Returns true if at least one primary controller is believed to have
1136 * authenticated and accepted this switch, false otherwise. */
1138 connmgr_is_any_controller_admitted(const struct connmgr
*mgr
)
1140 const struct ofconn
*ofconn
;
1142 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1143 if (rconn_is_admitted(ofconn
->rconn
)) {
1150 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1153 connmgr_broadcast(struct connmgr
*mgr
, struct ofpbuf
*packet
)
1155 struct ofconn
*ofconn
, *prev
;
1158 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
1160 ofconn_send_reply(ofconn
, ofpbuf_clone(packet
));
1162 if (rconn_is_connected(ofconn
->rconn
)) {
1167 ofconn_send_reply(prev
, packet
);
1169 ofpbuf_delete(packet
);
1173 /* In-band configuration. */
1175 static bool any_extras_changed(const struct connmgr
*,
1176 const struct sockaddr_in
*extras
, size_t n
);
1178 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1179 * in-band control should guarantee access, in the same way that in-band
1180 * control guarantees access to OpenFlow controllers. */
1182 connmgr_set_extra_in_band_remotes(struct connmgr
*mgr
,
1183 const struct sockaddr_in
*extras
, size_t n
)
1185 if (!any_extras_changed(mgr
, extras
, n
)) {
1189 free(mgr
->extra_in_band_remotes
);
1190 mgr
->n_extra_remotes
= n
;
1191 mgr
->extra_in_band_remotes
= xmemdup(extras
, n
* sizeof *extras
);
1193 update_in_band_remotes(mgr
);
1196 /* Sets the OpenFlow queue used by flows set up by in-band control on
1197 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1198 * flows will use the default queue. */
1200 connmgr_set_in_band_queue(struct connmgr
*mgr
, int queue_id
)
1202 if (queue_id
!= mgr
->in_band_queue
) {
1203 mgr
->in_band_queue
= queue_id
;
1204 update_in_band_remotes(mgr
);
1209 any_extras_changed(const struct connmgr
*mgr
,
1210 const struct sockaddr_in
*extras
, size_t n
)
1214 if (n
!= mgr
->n_extra_remotes
) {
1218 for (i
= 0; i
< n
; i
++) {
1219 const struct sockaddr_in
*old
= &mgr
->extra_in_band_remotes
[i
];
1220 const struct sockaddr_in
*new = &extras
[i
];
1222 if (old
->sin_addr
.s_addr
!= new->sin_addr
.s_addr
||
1223 old
->sin_port
!= new->sin_port
) {
1231 /* In-band implementation. */
1234 connmgr_msg_in_hook(struct connmgr
*mgr
, const struct flow
*flow
,
1235 const struct ofpbuf
*packet
)
1237 return mgr
->in_band
&& in_band_msg_in_hook(mgr
->in_band
, flow
, packet
);
1241 connmgr_may_set_up_flow(struct connmgr
*mgr
, const struct flow
*flow
,
1242 const struct nlattr
*odp_actions
,
1245 return !mgr
->in_band
|| in_band_rule_check(flow
, odp_actions
, actions_len
);
1248 /* Fail-open and in-band implementation. */
1250 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1251 * and in-band control to re-create their flows. */
1253 connmgr_flushed(struct connmgr
*mgr
)
1256 in_band_flushed(mgr
->in_band
);
1258 if (mgr
->fail_open
) {
1259 fail_open_flushed(mgr
->fail_open
);
1263 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1264 * otherwise a positive errno value.
1266 * ofservice_reconfigure() must be called to fully configure the new
1269 ofservice_create(struct connmgr
*mgr
, const char *target
)
1271 struct ofservice
*ofservice
;
1272 struct pvconn
*pvconn
;
1275 error
= pvconn_open(target
, &pvconn
);
1280 ofservice
= xzalloc(sizeof *ofservice
);
1281 hmap_insert(&mgr
->services
, &ofservice
->node
, hash_string(target
, 0));
1282 ofservice
->pvconn
= pvconn
;
1288 ofservice_destroy(struct connmgr
*mgr
, struct ofservice
*ofservice
)
1290 hmap_remove(&mgr
->services
, &ofservice
->node
);
1291 pvconn_close(ofservice
->pvconn
);
1296 ofservice_reconfigure(struct ofservice
*ofservice
,
1297 const struct ofproto_controller
*c
)
1299 ofservice
->probe_interval
= c
->probe_interval
;
1300 ofservice
->rate_limit
= c
->rate_limit
;
1301 ofservice
->burst_limit
= c
->burst_limit
;
1304 /* Finds and returns the ofservice within 'mgr' that has the given
1305 * 'target', or a null pointer if none exists. */
1306 static struct ofservice
*
1307 ofservice_lookup(struct connmgr
*mgr
, const char *target
)
1309 struct ofservice
*ofservice
;
1311 HMAP_FOR_EACH_WITH_HASH (ofservice
, node
, hash_string(target
, 0),
1313 if (!strcmp(pvconn_get_name(ofservice
->pvconn
), target
)) {