2 * Copyright (c) 2009, 2010, 2011 Nicira Networks.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
26 #include "fail-open.h"
32 #include "poll-loop.h"
41 VLOG_DEFINE_THIS_MODULE(connmgr
);
42 static struct vlog_rate_limit rl
= VLOG_RATE_LIMIT_INIT(1, 5);
44 COVERAGE_DEFINE(ofconn_stuck
);
46 /* An OpenFlow connection. */
48 struct connmgr
*connmgr
; /* Connection's manager. */
49 struct list node
; /* In struct connmgr's "all_conns" list. */
50 struct rconn
*rconn
; /* OpenFlow connection. */
51 enum ofconn_type type
; /* Type. */
52 enum nx_flow_format flow_format
; /* Currently selected flow format. */
54 /* OFPT_PACKET_IN related data. */
55 struct rconn_packet_counter
*packet_in_counter
; /* # queued on 'rconn'. */
56 #define N_SCHEDULERS 2
57 struct pinsched
*schedulers
[N_SCHEDULERS
];
58 struct pktbuf
*pktbuf
; /* OpenFlow packet buffers. */
59 int miss_send_len
; /* Bytes to send of buffered packets. */
61 /* Number of OpenFlow messages queued on 'rconn' as replies to OpenFlow
62 * requests, and the maximum number before we stop reading OpenFlow
64 #define OFCONN_REPLY_MAX 100
65 struct rconn_packet_counter
*reply_counter
;
67 /* type == OFCONN_PRIMARY only. */
68 enum nx_role role
; /* Role. */
69 struct hmap_node hmap_node
; /* In struct connmgr's "controllers" map. */
70 enum ofproto_band band
; /* In-band or out-of-band? */
73 static struct ofconn
*ofconn_create(struct connmgr
*, struct rconn
*,
75 static void ofconn_destroy(struct ofconn
*);
77 static void ofconn_reconfigure(struct ofconn
*,
78 const struct ofproto_controller
*);
80 static void ofconn_run(struct ofconn
*,
81 void (*handle_openflow
)(struct ofconn
*,
82 struct ofpbuf
*ofp_msg
));
83 static void ofconn_wait(struct ofconn
*);
85 static const char *ofconn_get_target(const struct ofconn
*);
86 static char *ofconn_make_name(const struct connmgr
*, const char *target
);
88 static void ofconn_set_rate_limit(struct ofconn
*, int rate
, int burst
);
90 static bool ofconn_receives_async_msgs(const struct ofconn
*);
92 static void ofconn_send(const struct ofconn
*, struct ofpbuf
*,
93 struct rconn_packet_counter
*);
95 static void do_send_packet_in(struct ofpbuf
*, void *ofconn_
);
97 /* A listener for incoming OpenFlow "service" connections. */
99 struct hmap_node node
; /* In struct connmgr's "services" hmap. */
100 struct pvconn
*pvconn
; /* OpenFlow connection listener. */
102 /* These are not used by ofservice directly. They are settings for
103 * accepted "struct ofconn"s from the pvconn. */
104 int probe_interval
; /* Max idle time before probing, in seconds. */
105 int rate_limit
; /* Max packet-in rate in packets per second. */
106 int burst_limit
; /* Limit on accumulating packet credits. */
109 static void ofservice_reconfigure(struct ofservice
*,
110 const struct ofproto_controller
*);
111 static int ofservice_create(struct connmgr
*, const char *target
);
112 static void ofservice_destroy(struct connmgr
*, struct ofservice
*);
113 static struct ofservice
*ofservice_lookup(struct connmgr
*,
116 /* Connection manager for an OpenFlow switch. */
118 struct ofproto
*ofproto
;
120 char *local_port_name
;
122 /* OpenFlow connections. */
123 struct hmap controllers
; /* Controller "struct ofconn"s. */
124 struct list all_conns
; /* Contains "struct ofconn"s. */
126 /* OpenFlow listeners. */
127 struct hmap services
; /* Contains "struct ofservice"s. */
128 struct pvconn
**snoops
;
132 struct fail_open
*fail_open
;
133 enum ofproto_fail_mode fail_mode
;
135 /* In-band control. */
136 struct in_band
*in_band
;
137 long long int next_in_band_update
;
138 struct sockaddr_in
*extra_in_band_remotes
;
139 size_t n_extra_remotes
;
143 static void update_in_band_remotes(struct connmgr
*);
144 static void add_snooper(struct connmgr
*, struct vconn
*);
146 /* Creates and returns a new connection manager owned by 'ofproto'. 'name' is
147 * a name for the ofproto suitable for using in log messages.
148 * 'local_port_name' is the name of the local port (OFPP_LOCAL) within
151 connmgr_create(struct ofproto
*ofproto
,
152 const char *name
, const char *local_port_name
)
156 mgr
= xmalloc(sizeof *mgr
);
157 mgr
->ofproto
= ofproto
;
158 mgr
->name
= xstrdup(name
);
159 mgr
->local_port_name
= xstrdup(local_port_name
);
161 hmap_init(&mgr
->controllers
);
162 list_init(&mgr
->all_conns
);
164 hmap_init(&mgr
->services
);
168 mgr
->fail_open
= NULL
;
169 mgr
->fail_mode
= OFPROTO_FAIL_SECURE
;
172 mgr
->next_in_band_update
= LLONG_MAX
;
173 mgr
->extra_in_band_remotes
= NULL
;
174 mgr
->n_extra_remotes
= 0;
175 mgr
->in_band_queue
= -1;
180 /* Frees 'mgr' and all of its resources. */
182 connmgr_destroy(struct connmgr
*mgr
)
184 struct ofservice
*ofservice
, *next_ofservice
;
185 struct ofconn
*ofconn
, *next_ofconn
;
192 LIST_FOR_EACH_SAFE (ofconn
, next_ofconn
, node
, &mgr
->all_conns
) {
193 ofconn_destroy(ofconn
);
195 hmap_destroy(&mgr
->controllers
);
197 HMAP_FOR_EACH_SAFE (ofservice
, next_ofservice
, node
, &mgr
->services
) {
198 ofservice_destroy(mgr
, ofservice
);
200 hmap_destroy(&mgr
->services
);
202 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
203 pvconn_close(mgr
->snoops
[i
]);
207 fail_open_destroy(mgr
->fail_open
);
208 mgr
->fail_open
= NULL
;
210 in_band_destroy(mgr
->in_band
);
212 free(mgr
->extra_in_band_remotes
);
214 free(mgr
->local_port_name
);
219 /* Does all of the periodic maintenance required by 'mgr'. Calls
220 * 'handle_openflow' for each message received on an OpenFlow connection,
221 * passing along the OpenFlow connection itself and the message that was sent.
222 * The 'handle_openflow' callback must not free the message. */
224 connmgr_run(struct connmgr
*mgr
,
225 void (*handle_openflow
)(struct ofconn
*, struct ofpbuf
*ofp_msg
))
227 struct ofconn
*ofconn
, *next_ofconn
;
228 struct ofservice
*ofservice
;
232 if (time_msec() >= mgr
->next_in_band_update
) {
233 update_in_band_remotes(mgr
);
235 in_band_run(mgr
->in_band
);
238 LIST_FOR_EACH_SAFE (ofconn
, next_ofconn
, node
, &mgr
->all_conns
) {
239 ofconn_run(ofconn
, handle_openflow
);
242 /* Fail-open maintenance. Do this after processing the ofconns since
243 * fail-open checks the status of the controller rconn. */
244 if (mgr
->fail_open
) {
245 fail_open_run(mgr
->fail_open
);
248 HMAP_FOR_EACH (ofservice
, node
, &mgr
->services
) {
252 retval
= pvconn_accept(ofservice
->pvconn
, OFP_VERSION
, &vconn
);
257 rconn
= rconn_create(ofservice
->probe_interval
, 0);
258 name
= ofconn_make_name(mgr
, vconn_get_name(vconn
));
259 rconn_connect_unreliably(rconn
, vconn
, name
);
262 ofconn
= ofconn_create(mgr
, rconn
, OFCONN_SERVICE
);
263 ofconn_set_rate_limit(ofconn
, ofservice
->rate_limit
,
264 ofservice
->burst_limit
);
265 } else if (retval
!= EAGAIN
) {
266 VLOG_WARN_RL(&rl
, "accept failed (%s)", strerror(retval
));
270 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
274 retval
= pvconn_accept(mgr
->snoops
[i
], OFP_VERSION
, &vconn
);
276 add_snooper(mgr
, vconn
);
277 } else if (retval
!= EAGAIN
) {
278 VLOG_WARN_RL(&rl
, "accept failed (%s)", strerror(retval
));
283 /* Causes the poll loop to wake up when connmgr_run() needs to run. */
285 connmgr_wait(struct connmgr
*mgr
)
287 struct ofservice
*ofservice
;
288 struct ofconn
*ofconn
;
291 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
295 poll_timer_wait_until(mgr
->next_in_band_update
);
296 in_band_wait(mgr
->in_band
);
298 if (mgr
->fail_open
) {
299 fail_open_wait(mgr
->fail_open
);
301 HMAP_FOR_EACH (ofservice
, node
, &mgr
->services
) {
302 pvconn_wait(ofservice
->pvconn
);
304 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
305 pvconn_wait(mgr
->snoops
[i
]);
309 /* Returns the ofproto that owns 'ofconn''s connmgr. */
311 ofconn_get_ofproto(const struct ofconn
*ofconn
)
313 return ofconn
->connmgr
->ofproto
;
316 /* OpenFlow configuration. */
318 static void add_controller(struct connmgr
*, const char *target
);
319 static struct ofconn
*find_controller_by_target(struct connmgr
*,
321 static void update_fail_open(struct connmgr
*);
322 static int set_pvconns(struct pvconn
***pvconnsp
, size_t *n_pvconnsp
,
323 const struct sset
*);
325 /* Returns true if 'mgr' has any configured primary controllers.
327 * Service controllers do not count, but configured primary controllers do
328 * count whether or not they are currently connected. */
330 connmgr_has_controllers(const struct connmgr
*mgr
)
332 return !hmap_is_empty(&mgr
->controllers
);
335 /* Initializes 'info' and populates it with information about each configured
336 * primary controller. The keys in 'info' are the controllers' targets; the
337 * data values are corresponding "struct ofproto_controller_info".
339 * The caller owns 'info' and everything in it and should free it when it is no
342 connmgr_get_controller_info(struct connmgr
*mgr
, struct shash
*info
)
344 const struct ofconn
*ofconn
;
348 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
349 const struct rconn
*rconn
= ofconn
->rconn
;
350 time_t now
= time_now();
351 time_t last_connection
= rconn_get_last_connection(rconn
);
352 time_t last_disconnect
= rconn_get_last_disconnect(rconn
);
353 int last_error
= rconn_get_last_error(rconn
);
354 struct ofproto_controller_info
*cinfo
= xmalloc(sizeof *cinfo
);
356 shash_add(info
, rconn_get_target(rconn
), cinfo
);
358 cinfo
->is_connected
= rconn_is_connected(rconn
);
359 cinfo
->role
= ofconn
->role
;
364 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "last_error";
365 cinfo
->pairs
.values
[cinfo
->pairs
.n
++] =
366 xstrdup(ovs_retval_to_string(last_error
));
369 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "state";
370 cinfo
->pairs
.values
[cinfo
->pairs
.n
++] =
371 xstrdup(rconn_get_state(rconn
));
373 if (last_connection
!= TIME_MIN
) {
374 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "sec_since_connect";
375 cinfo
->pairs
.values
[cinfo
->pairs
.n
++]
376 = xasprintf("%ld", (long int) (now
- last_connection
));
379 if (last_disconnect
!= TIME_MIN
) {
380 cinfo
->pairs
.keys
[cinfo
->pairs
.n
] = "sec_since_disconnect";
381 cinfo
->pairs
.values
[cinfo
->pairs
.n
++]
382 = xasprintf("%ld", (long int) (now
- last_disconnect
));
387 /* Changes 'mgr''s set of controllers to the 'n_controllers' controllers in
390 connmgr_set_controllers(struct connmgr
*mgr
,
391 const struct ofproto_controller
*controllers
,
392 size_t n_controllers
)
394 bool had_controllers
= connmgr_has_controllers(mgr
);
395 struct shash new_controllers
;
396 struct ofconn
*ofconn
, *next_ofconn
;
397 struct ofservice
*ofservice
, *next_ofservice
;
401 /* Create newly configured controllers and services.
402 * Create a name to ofproto_controller mapping in 'new_controllers'. */
403 shash_init(&new_controllers
);
404 for (i
= 0; i
< n_controllers
; i
++) {
405 const struct ofproto_controller
*c
= &controllers
[i
];
407 if (!vconn_verify_name(c
->target
)) {
408 if (!find_controller_by_target(mgr
, c
->target
)) {
409 add_controller(mgr
, c
->target
);
411 } else if (!pvconn_verify_name(c
->target
)) {
412 if (!ofservice_lookup(mgr
, c
->target
)) {
413 ofservice_create(mgr
, c
->target
);
416 VLOG_WARN_RL(&rl
, "%s: unsupported controller \"%s\"",
417 mgr
->name
, c
->target
);
421 shash_add_once(&new_controllers
, c
->target
, &controllers
[i
]);
424 /* Delete controllers that are no longer configured.
425 * Update configuration of all now-existing controllers. */
427 HMAP_FOR_EACH_SAFE (ofconn
, next_ofconn
, hmap_node
, &mgr
->controllers
) {
428 struct ofproto_controller
*c
;
430 c
= shash_find_data(&new_controllers
, ofconn_get_target(ofconn
));
432 ofconn_destroy(ofconn
);
434 ofconn_reconfigure(ofconn
, c
);
438 /* Delete services that are no longer configured.
439 * Update configuration of all now-existing services. */
440 HMAP_FOR_EACH_SAFE (ofservice
, next_ofservice
, node
, &mgr
->services
) {
441 struct ofproto_controller
*c
;
443 c
= shash_find_data(&new_controllers
,
444 pvconn_get_name(ofservice
->pvconn
));
446 ofservice_destroy(mgr
, ofservice
);
448 ofservice_reconfigure(ofservice
, c
);
452 shash_destroy(&new_controllers
);
454 update_in_band_remotes(mgr
);
455 update_fail_open(mgr
);
456 if (had_controllers
!= connmgr_has_controllers(mgr
)) {
457 ofproto_flush_flows(mgr
->ofproto
);
461 /* Drops the connections between 'mgr' and all of its primary and secondary
462 * controllers, forcing them to reconnect. */
464 connmgr_reconnect(const struct connmgr
*mgr
)
466 struct ofconn
*ofconn
;
468 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
469 rconn_reconnect(ofconn
->rconn
);
473 /* Sets the "snoops" for 'mgr' to the pvconn targets listed in 'snoops'.
475 * A "snoop" is a pvconn to which every OpenFlow message to or from the most
476 * important controller on 'mgr' is mirrored. */
478 connmgr_set_snoops(struct connmgr
*mgr
, const struct sset
*snoops
)
480 return set_pvconns(&mgr
->snoops
, &mgr
->n_snoops
, snoops
);
483 /* Adds each of the snoops currently configured on 'mgr' to 'snoops'. */
485 connmgr_get_snoops(const struct connmgr
*mgr
, struct sset
*snoops
)
489 for (i
= 0; i
< mgr
->n_snoops
; i
++) {
490 sset_add(snoops
, pvconn_get_name(mgr
->snoops
[i
]));
494 /* Returns true if 'mgr' has at least one snoop, false if it has none. */
496 connmgr_has_snoops(const struct connmgr
*mgr
)
498 return mgr
->n_snoops
> 0;
501 /* Creates a new controller for 'target' in 'mgr'. update_controller() needs
502 * to be called later to finish the new ofconn's configuration. */
504 add_controller(struct connmgr
*mgr
, const char *target
)
506 char *name
= ofconn_make_name(mgr
, target
);
507 struct ofconn
*ofconn
;
509 ofconn
= ofconn_create(mgr
, rconn_create(5, 8), OFCONN_PRIMARY
);
510 ofconn
->pktbuf
= pktbuf_create();
511 ofconn
->miss_send_len
= OFP_DEFAULT_MISS_SEND_LEN
;
512 rconn_connect(ofconn
->rconn
, target
, name
);
513 hmap_insert(&mgr
->controllers
, &ofconn
->hmap_node
, hash_string(target
, 0));
518 static struct ofconn
*
519 find_controller_by_target(struct connmgr
*mgr
, const char *target
)
521 struct ofconn
*ofconn
;
523 HMAP_FOR_EACH_WITH_HASH (ofconn
, hmap_node
,
524 hash_string(target
, 0), &mgr
->controllers
) {
525 if (!strcmp(ofconn_get_target(ofconn
), target
)) {
533 update_in_band_remotes(struct connmgr
*mgr
)
535 struct sockaddr_in
*addrs
;
536 size_t max_addrs
, n_addrs
;
537 struct ofconn
*ofconn
;
540 /* Allocate enough memory for as many remotes as we could possibly have. */
541 max_addrs
= mgr
->n_extra_remotes
+ hmap_count(&mgr
->controllers
);
542 addrs
= xmalloc(max_addrs
* sizeof *addrs
);
545 /* Add all the remotes. */
546 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
547 struct sockaddr_in
*sin
= &addrs
[n_addrs
];
549 if (ofconn
->band
== OFPROTO_OUT_OF_BAND
) {
553 sin
->sin_addr
.s_addr
= rconn_get_remote_ip(ofconn
->rconn
);
554 if (sin
->sin_addr
.s_addr
) {
555 sin
->sin_port
= rconn_get_remote_port(ofconn
->rconn
);
559 for (i
= 0; i
< mgr
->n_extra_remotes
; i
++) {
560 addrs
[n_addrs
++] = mgr
->extra_in_band_remotes
[i
];
563 /* Create or update or destroy in-band. */
566 in_band_create(mgr
->ofproto
, mgr
->local_port_name
, &mgr
->in_band
);
569 in_band_set_remotes(mgr
->in_band
, addrs
, n_addrs
);
571 in_band_set_queue(mgr
->in_band
, mgr
->in_band_queue
);
572 mgr
->next_in_band_update
= time_msec() + 1000;
574 in_band_destroy(mgr
->in_band
);
583 update_fail_open(struct connmgr
*mgr
)
585 if (connmgr_has_controllers(mgr
)
586 && mgr
->fail_mode
== OFPROTO_FAIL_STANDALONE
) {
587 if (!mgr
->fail_open
) {
588 mgr
->fail_open
= fail_open_create(mgr
->ofproto
, mgr
);
591 fail_open_destroy(mgr
->fail_open
);
592 mgr
->fail_open
= NULL
;
597 set_pvconns(struct pvconn
***pvconnsp
, size_t *n_pvconnsp
,
598 const struct sset
*sset
)
600 struct pvconn
**pvconns
= *pvconnsp
;
601 size_t n_pvconns
= *n_pvconnsp
;
606 for (i
= 0; i
< n_pvconns
; i
++) {
607 pvconn_close(pvconns
[i
]);
611 pvconns
= xmalloc(sset_count(sset
) * sizeof *pvconns
);
613 SSET_FOR_EACH (name
, sset
) {
614 struct pvconn
*pvconn
;
617 error
= pvconn_open(name
, &pvconn
);
619 pvconns
[n_pvconns
++] = pvconn
;
621 VLOG_ERR("failed to listen on %s: %s", name
, strerror(error
));
629 *n_pvconnsp
= n_pvconns
;
634 /* Returns a "preference level" for snooping 'ofconn'. A higher return value
635 * means that 'ofconn' is more interesting for monitoring than a lower return
638 snoop_preference(const struct ofconn
*ofconn
)
640 switch (ofconn
->role
) {
648 /* Shouldn't happen. */
653 /* One of 'mgr''s "snoop" pvconns has accepted a new connection on 'vconn'.
654 * Connects this vconn to a controller. */
656 add_snooper(struct connmgr
*mgr
, struct vconn
*vconn
)
658 struct ofconn
*ofconn
, *best
;
660 /* Pick a controller for monitoring. */
662 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
663 if (ofconn
->type
== OFCONN_PRIMARY
664 && (!best
|| snoop_preference(ofconn
) > snoop_preference(best
))) {
670 rconn_add_monitor(best
->rconn
, vconn
);
672 VLOG_INFO_RL(&rl
, "no controller connection to snoop");
677 /* Public ofconn functions. */
679 /* Returns the connection type, either OFCONN_PRIMARY or OFCONN_SERVICE. */
681 ofconn_get_type(const struct ofconn
*ofconn
)
686 /* Returns the role configured for 'ofconn'.
688 * The default role, if no other role has been set, is NX_ROLE_OTHER. */
690 ofconn_get_role(const struct ofconn
*ofconn
)
695 /* Changes 'ofconn''s role to 'role'. If 'role' is NX_ROLE_MASTER then any
696 * existing master is demoted to a slave. */
698 ofconn_set_role(struct ofconn
*ofconn
, enum nx_role role
)
700 if (role
== NX_ROLE_MASTER
) {
701 struct ofconn
*other
;
703 HMAP_FOR_EACH (other
, hmap_node
, &ofconn
->connmgr
->controllers
) {
704 if (other
->role
== NX_ROLE_MASTER
) {
705 other
->role
= NX_ROLE_SLAVE
;
712 /* Returns the currently configured flow format for 'ofconn', one of NXFF_*.
714 * The default, if no other format has been set, is NXFF_OPENFLOW10. */
716 ofconn_get_flow_format(struct ofconn
*ofconn
)
718 return ofconn
->flow_format
;
721 /* Sets the flow format for 'ofconn' to 'flow_format' (one of NXFF_*). */
723 ofconn_set_flow_format(struct ofconn
*ofconn
, enum nx_flow_format flow_format
)
725 ofconn
->flow_format
= flow_format
;
728 /* Returns the default miss send length for 'ofconn'. */
730 ofconn_get_miss_send_len(const struct ofconn
*ofconn
)
732 return ofconn
->miss_send_len
;
735 /* Sets the default miss send length for 'ofconn' to 'miss_send_len'. */
737 ofconn_set_miss_send_len(struct ofconn
*ofconn
, int miss_send_len
)
739 ofconn
->miss_send_len
= miss_send_len
;
742 /* Sends 'msg' on 'ofconn', accounting it as a reply. (If there is a
743 * sufficient number of OpenFlow replies in-flight on a single ofconn, then the
744 * connmgr will stop accepting new OpenFlow requests on that ofconn until the
745 * controller has accepted some of the replies.) */
747 ofconn_send_reply(const struct ofconn
*ofconn
, struct ofpbuf
*msg
)
749 ofconn_send(ofconn
, msg
, ofconn
->reply_counter
);
752 /* Same as pktbuf_retrieve(), using the pktbuf owned by 'ofconn'. */
754 ofconn_pktbuf_retrieve(struct ofconn
*ofconn
, uint32_t id
,
755 struct ofpbuf
**bufferp
, uint16_t *in_port
)
757 return pktbuf_retrieve(ofconn
->pktbuf
, id
, bufferp
, in_port
);
760 /* Private ofconn functions. */
763 ofconn_get_target(const struct ofconn
*ofconn
)
765 return rconn_get_target(ofconn
->rconn
);
768 static struct ofconn
*
769 ofconn_create(struct connmgr
*mgr
, struct rconn
*rconn
, enum ofconn_type type
)
771 struct ofconn
*ofconn
= xzalloc(sizeof *ofconn
);
772 ofconn
->connmgr
= mgr
;
773 list_push_back(&mgr
->all_conns
, &ofconn
->node
);
774 ofconn
->rconn
= rconn
;
776 ofconn
->flow_format
= NXFF_OPENFLOW10
;
777 ofconn
->role
= NX_ROLE_OTHER
;
778 ofconn
->packet_in_counter
= rconn_packet_counter_create ();
779 ofconn
->pktbuf
= NULL
;
780 ofconn
->miss_send_len
= 0;
781 ofconn
->reply_counter
= rconn_packet_counter_create ();
786 ofconn_destroy(struct ofconn
*ofconn
)
788 if (ofconn
->type
== OFCONN_PRIMARY
) {
789 hmap_remove(&ofconn
->connmgr
->controllers
, &ofconn
->hmap_node
);
792 list_remove(&ofconn
->node
);
793 rconn_destroy(ofconn
->rconn
);
794 rconn_packet_counter_destroy(ofconn
->packet_in_counter
);
795 rconn_packet_counter_destroy(ofconn
->reply_counter
);
796 pktbuf_destroy(ofconn
->pktbuf
);
800 /* Reconfigures 'ofconn' to match 'c'. 'ofconn' and 'c' must have the same
803 ofconn_reconfigure(struct ofconn
*ofconn
, const struct ofproto_controller
*c
)
807 ofconn
->band
= c
->band
;
809 rconn_set_max_backoff(ofconn
->rconn
, c
->max_backoff
);
811 probe_interval
= c
->probe_interval
? MAX(c
->probe_interval
, 5) : 0;
812 rconn_set_probe_interval(ofconn
->rconn
, probe_interval
);
814 ofconn_set_rate_limit(ofconn
, c
->rate_limit
, c
->burst_limit
);
818 ofconn_run(struct ofconn
*ofconn
,
819 void (*handle_openflow
)(struct ofconn
*, struct ofpbuf
*ofp_msg
))
821 struct connmgr
*mgr
= ofconn
->connmgr
;
825 for (i
= 0; i
< N_SCHEDULERS
; i
++) {
826 pinsched_run(ofconn
->schedulers
[i
], do_send_packet_in
, ofconn
);
829 rconn_run(ofconn
->rconn
);
831 if (rconn_packet_counter_read (ofconn
->reply_counter
) < OFCONN_REPLY_MAX
) {
832 /* Limit the number of iterations to prevent other tasks from
834 for (iteration
= 0; iteration
< 50; iteration
++) {
835 struct ofpbuf
*of_msg
= rconn_recv(ofconn
->rconn
);
839 if (mgr
->fail_open
) {
840 fail_open_maybe_recover(mgr
->fail_open
);
842 handle_openflow(ofconn
, of_msg
);
843 ofpbuf_delete(of_msg
);
847 if (!rconn_is_alive(ofconn
->rconn
)) {
848 ofconn_destroy(ofconn
);
853 ofconn_wait(struct ofconn
*ofconn
)
857 for (i
= 0; i
< N_SCHEDULERS
; i
++) {
858 pinsched_wait(ofconn
->schedulers
[i
]);
860 rconn_run_wait(ofconn
->rconn
);
861 if (rconn_packet_counter_read (ofconn
->reply_counter
) < OFCONN_REPLY_MAX
) {
862 rconn_recv_wait(ofconn
->rconn
);
864 COVERAGE_INC(ofconn_stuck
);
868 /* Returns true if 'ofconn' should receive asynchronous messages. */
870 ofconn_receives_async_msgs(const struct ofconn
*ofconn
)
872 if (!rconn_is_connected(ofconn
->rconn
)) {
874 } else if (ofconn
->type
== OFCONN_PRIMARY
) {
875 /* Primary controllers always get asynchronous messages unless they
876 * have configured themselves as "slaves". */
877 return ofconn
->role
!= NX_ROLE_SLAVE
;
879 /* Service connections don't get asynchronous messages unless they have
880 * explicitly asked for them by setting a nonzero miss send length. */
881 return ofconn
->miss_send_len
> 0;
885 /* Returns a human-readable name for an OpenFlow connection between 'mgr' and
886 * 'target', suitable for use in log messages for identifying the connection.
888 * The name is dynamically allocated. The caller should free it (with free())
889 * when it is no longer needed. */
891 ofconn_make_name(const struct connmgr
*mgr
, const char *target
)
893 return xasprintf("%s<->%s", mgr
->name
, target
);
897 ofconn_set_rate_limit(struct ofconn
*ofconn
, int rate
, int burst
)
901 for (i
= 0; i
< N_SCHEDULERS
; i
++) {
902 struct pinsched
**s
= &ofconn
->schedulers
[i
];
906 *s
= pinsched_create(rate
, burst
);
908 pinsched_set_limits(*s
, rate
, burst
);
911 pinsched_destroy(*s
);
918 ofconn_send(const struct ofconn
*ofconn
, struct ofpbuf
*msg
,
919 struct rconn_packet_counter
*counter
)
921 update_openflow_length(msg
);
922 if (rconn_send(ofconn
->rconn
, msg
, counter
)) {
927 /* Sending asynchronous messages. */
929 static void schedule_packet_in(struct ofconn
*, const struct dpif_upcall
*,
930 const struct flow
*, struct ofpbuf
*rw_packet
);
932 /* Sends an OFPT_PORT_STATUS message with 'opp' and 'reason' to appropriate
933 * controllers managed by 'mgr'. */
935 connmgr_send_port_status(struct connmgr
*mgr
, const struct ofp_phy_port
*opp
,
938 /* XXX Should limit the number of queued port status change messages. */
939 struct ofconn
*ofconn
;
941 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
942 struct ofp_port_status
*ops
;
945 /* Primary controllers, even slaves, should always get port status
946 updates. Otherwise obey ofconn_receives_async_msgs(). */
947 if (ofconn
->type
!= OFCONN_PRIMARY
948 && !ofconn_receives_async_msgs(ofconn
)) {
952 ops
= make_openflow_xid(sizeof *ops
, OFPT_PORT_STATUS
, 0, &b
);
953 ops
->reason
= reason
;
955 ofconn_send(ofconn
, b
, NULL
);
959 /* Sends an OFPT_FLOW_REMOVED or NXT_FLOW_REMOVED message based on 'fr' to
960 * appropriate controllers managed by 'mgr'. */
962 connmgr_send_flow_removed(struct connmgr
*mgr
,
963 const struct ofputil_flow_removed
*fr
)
965 struct ofconn
*ofconn
;
967 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
970 if (!ofconn_receives_async_msgs(ofconn
)) {
974 /* Account flow expirations as replies to OpenFlow requests. That
975 * works because preventing OpenFlow requests from being processed also
976 * prevents new flows from being added (and expiring). (It also
977 * prevents processing OpenFlow requests that would not add new flows,
978 * so it is imperfect.) */
979 msg
= ofputil_encode_flow_removed(fr
, ofconn
->flow_format
);
980 ofconn_send_reply(ofconn
, msg
);
984 /* Given 'upcall', of type DPIF_UC_ACTION or DPIF_UC_MISS, sends an
985 * OFPT_PACKET_IN message to each OpenFlow controller as necessary according to
986 * their individual configurations.
988 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
989 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
990 * of 'rw_packet' is transferred to this function. */
992 connmgr_send_packet_in(struct connmgr
*mgr
, const struct dpif_upcall
*upcall
,
993 const struct flow
*flow
, struct ofpbuf
*rw_packet
)
995 struct ofconn
*ofconn
, *prev
;
998 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
999 if (ofconn_receives_async_msgs(ofconn
)) {
1001 schedule_packet_in(prev
, upcall
, flow
, NULL
);
1007 schedule_packet_in(prev
, upcall
, flow
, rw_packet
);
1009 ofpbuf_delete(rw_packet
);
1013 /* pinsched callback for sending 'ofp_packet_in' on 'ofconn'. */
1015 do_send_packet_in(struct ofpbuf
*ofp_packet_in
, void *ofconn_
)
1017 struct ofconn
*ofconn
= ofconn_
;
1019 rconn_send_with_limit(ofconn
->rconn
, ofp_packet_in
,
1020 ofconn
->packet_in_counter
, 100);
1023 /* Takes 'upcall', whose packet has the flow specified by 'flow', composes an
1024 * OpenFlow packet-in message from it, and passes it to 'ofconn''s packet
1025 * scheduler for sending.
1027 * 'rw_packet' may be NULL. Otherwise, 'rw_packet' must contain the same data
1028 * as upcall->packet. (rw_packet == upcall->packet is also valid.) Ownership
1029 * of 'rw_packet' is transferred to this function. */
1031 schedule_packet_in(struct ofconn
*ofconn
, const struct dpif_upcall
*upcall
,
1032 const struct flow
*flow
, struct ofpbuf
*rw_packet
)
1034 struct connmgr
*mgr
= ofconn
->connmgr
;
1035 struct ofputil_packet_in pin
;
1037 /* Figure out the easy parts. */
1038 pin
.packet
= upcall
->packet
;
1039 pin
.in_port
= flow
->in_port
;
1040 pin
.reason
= upcall
->type
== DPIF_UC_MISS
? OFPR_NO_MATCH
: OFPR_ACTION
;
1042 /* Get OpenFlow buffer_id. */
1043 if (upcall
->type
== DPIF_UC_ACTION
) {
1044 pin
.buffer_id
= UINT32_MAX
;
1045 } else if (mgr
->fail_open
&& fail_open_is_active(mgr
->fail_open
)) {
1046 pin
.buffer_id
= pktbuf_get_null();
1047 } else if (!ofconn
->pktbuf
) {
1048 pin
.buffer_id
= UINT32_MAX
;
1050 pin
.buffer_id
= pktbuf_save(ofconn
->pktbuf
, upcall
->packet
,
1054 /* Figure out how much of the packet to send. */
1055 pin
.send_len
= upcall
->packet
->size
;
1056 if (pin
.buffer_id
!= UINT32_MAX
) {
1057 pin
.send_len
= MIN(pin
.send_len
, ofconn
->miss_send_len
);
1059 if (upcall
->type
== DPIF_UC_ACTION
) {
1060 pin
.send_len
= MIN(pin
.send_len
, upcall
->userdata
);
1063 /* Make OFPT_PACKET_IN and hand over to packet scheduler. It might
1064 * immediately call into do_send_packet_in() or it might buffer it for a
1065 * while (until a later call to pinsched_run()). */
1066 pinsched_send(ofconn
->schedulers
[upcall
->type
== DPIF_UC_MISS
? 0 : 1],
1067 flow
->in_port
, ofputil_encode_packet_in(&pin
, rw_packet
),
1068 do_send_packet_in
, ofconn
);
1071 /* Fail-open settings. */
1073 /* Returns the failure handling mode (OFPROTO_FAIL_SECURE or
1074 * OFPROTO_FAIL_STANDALONE) for 'mgr'. */
1075 enum ofproto_fail_mode
1076 connmgr_get_fail_mode(const struct connmgr
*mgr
)
1078 return mgr
->fail_mode
;
1081 /* Sets the failure handling mode for 'mgr' to 'fail_mode' (either
1082 * OFPROTO_FAIL_SECURE or OFPROTO_FAIL_STANDALONE). */
1084 connmgr_set_fail_mode(struct connmgr
*mgr
, enum ofproto_fail_mode fail_mode
)
1086 if (mgr
->fail_mode
!= fail_mode
) {
1087 mgr
->fail_mode
= fail_mode
;
1088 update_fail_open(mgr
);
1089 if (!connmgr_has_controllers(mgr
)) {
1090 ofproto_flush_flows(mgr
->ofproto
);
1095 /* Fail-open implementation. */
1097 /* Returns the longest probe interval among the primary controllers configured
1098 * on 'mgr'. Returns 0 if there are no primary controllers. */
1100 connmgr_get_max_probe_interval(const struct connmgr
*mgr
)
1102 const struct ofconn
*ofconn
;
1103 int max_probe_interval
;
1105 max_probe_interval
= 0;
1106 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1107 int probe_interval
= rconn_get_probe_interval(ofconn
->rconn
);
1108 max_probe_interval
= MAX(max_probe_interval
, probe_interval
);
1110 return max_probe_interval
;
1113 /* Returns the number of seconds for which all of 'mgr's primary controllers
1114 * have been disconnected. Returns 0 if 'mgr' has no primary controllers. */
1116 connmgr_failure_duration(const struct connmgr
*mgr
)
1118 const struct ofconn
*ofconn
;
1119 int min_failure_duration
;
1121 if (!connmgr_has_controllers(mgr
)) {
1125 min_failure_duration
= INT_MAX
;
1126 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1127 int failure_duration
= rconn_failure_duration(ofconn
->rconn
);
1128 min_failure_duration
= MIN(min_failure_duration
, failure_duration
);
1130 return min_failure_duration
;
1133 /* Returns true if at least one primary controller is connected (regardless of
1134 * whether those controllers are believed to have authenticated and accepted
1135 * this switch), false if none of them are connected. */
1137 connmgr_is_any_controller_connected(const struct connmgr
*mgr
)
1139 const struct ofconn
*ofconn
;
1141 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1142 if (rconn_is_connected(ofconn
->rconn
)) {
1149 /* Returns true if at least one primary controller is believed to have
1150 * authenticated and accepted this switch, false otherwise. */
1152 connmgr_is_any_controller_admitted(const struct connmgr
*mgr
)
1154 const struct ofconn
*ofconn
;
1156 HMAP_FOR_EACH (ofconn
, hmap_node
, &mgr
->controllers
) {
1157 if (rconn_is_admitted(ofconn
->rconn
)) {
1164 /* Sends 'packet' to each controller connected to 'mgr'. Takes ownership of
1167 connmgr_broadcast(struct connmgr
*mgr
, struct ofpbuf
*packet
)
1169 struct ofconn
*ofconn
, *prev
;
1172 LIST_FOR_EACH (ofconn
, node
, &mgr
->all_conns
) {
1174 ofconn_send_reply(ofconn
, ofpbuf_clone(packet
));
1176 if (rconn_is_connected(ofconn
->rconn
)) {
1181 ofconn_send_reply(prev
, packet
);
1183 ofpbuf_delete(packet
);
1187 /* In-band configuration. */
1189 static bool any_extras_changed(const struct connmgr
*,
1190 const struct sockaddr_in
*extras
, size_t n
);
1192 /* Sets the 'n' TCP port addresses in 'extras' as ones to which 'mgr''s
1193 * in-band control should guarantee access, in the same way that in-band
1194 * control guarantees access to OpenFlow controllers. */
1196 connmgr_set_extra_in_band_remotes(struct connmgr
*mgr
,
1197 const struct sockaddr_in
*extras
, size_t n
)
1199 if (!any_extras_changed(mgr
, extras
, n
)) {
1203 free(mgr
->extra_in_band_remotes
);
1204 mgr
->n_extra_remotes
= n
;
1205 mgr
->extra_in_band_remotes
= xmemdup(extras
, n
* sizeof *extras
);
1207 update_in_band_remotes(mgr
);
1210 /* Sets the OpenFlow queue used by flows set up by in-band control on
1211 * 'mgr' to 'queue_id'. If 'queue_id' is negative, then in-band control
1212 * flows will use the default queue. */
1214 connmgr_set_in_band_queue(struct connmgr
*mgr
, int queue_id
)
1216 if (queue_id
!= mgr
->in_band_queue
) {
1217 mgr
->in_band_queue
= queue_id
;
1218 update_in_band_remotes(mgr
);
1223 any_extras_changed(const struct connmgr
*mgr
,
1224 const struct sockaddr_in
*extras
, size_t n
)
1228 if (n
!= mgr
->n_extra_remotes
) {
1232 for (i
= 0; i
< n
; i
++) {
1233 const struct sockaddr_in
*old
= &mgr
->extra_in_band_remotes
[i
];
1234 const struct sockaddr_in
*new = &extras
[i
];
1236 if (old
->sin_addr
.s_addr
!= new->sin_addr
.s_addr
||
1237 old
->sin_port
!= new->sin_port
) {
1245 /* In-band implementation. */
1248 connmgr_msg_in_hook(struct connmgr
*mgr
, const struct flow
*flow
,
1249 const struct ofpbuf
*packet
)
1251 return mgr
->in_band
&& in_band_msg_in_hook(mgr
->in_band
, flow
, packet
);
1255 connmgr_may_set_up_flow(struct connmgr
*mgr
, const struct flow
*flow
,
1256 const struct nlattr
*odp_actions
,
1259 return !mgr
->in_band
|| in_band_rule_check(flow
, odp_actions
, actions_len
);
1262 /* Fail-open and in-band implementation. */
1264 /* Called by 'ofproto' after all flows have been flushed, to allow fail-open
1265 * and in-band control to re-create their flows. */
1267 connmgr_flushed(struct connmgr
*mgr
)
1270 in_band_flushed(mgr
->in_band
);
1272 if (mgr
->fail_open
) {
1273 fail_open_flushed(mgr
->fail_open
);
1276 /* If there are no controllers and we're in standalone mode, set up a flow
1277 * that matches every packet and directs them to OFPP_NORMAL (which goes to
1278 * us). Otherwise, the switch is in secure mode and we won't pass any
1279 * traffic until a controller has been defined and it tells us to do so. */
1280 if (!connmgr_has_controllers(mgr
)
1281 && mgr
->fail_mode
== OFPROTO_FAIL_STANDALONE
) {
1282 union ofp_action action
;
1283 struct cls_rule rule
;
1285 memset(&action
, 0, sizeof action
);
1286 action
.type
= htons(OFPAT_OUTPUT
);
1287 action
.output
.len
= htons(sizeof action
);
1288 action
.output
.port
= htons(OFPP_NORMAL
);
1289 cls_rule_init_catchall(&rule
, 0);
1290 ofproto_add_flow(mgr
->ofproto
, &rule
, &action
, 1);
1294 /* Creates a new ofservice for 'target' in 'mgr'. Returns 0 if successful,
1295 * otherwise a positive errno value.
1297 * ofservice_reconfigure() must be called to fully configure the new
1300 ofservice_create(struct connmgr
*mgr
, const char *target
)
1302 struct ofservice
*ofservice
;
1303 struct pvconn
*pvconn
;
1306 error
= pvconn_open(target
, &pvconn
);
1311 ofservice
= xzalloc(sizeof *ofservice
);
1312 hmap_insert(&mgr
->services
, &ofservice
->node
, hash_string(target
, 0));
1313 ofservice
->pvconn
= pvconn
;
1319 ofservice_destroy(struct connmgr
*mgr
, struct ofservice
*ofservice
)
1321 hmap_remove(&mgr
->services
, &ofservice
->node
);
1322 pvconn_close(ofservice
->pvconn
);
1327 ofservice_reconfigure(struct ofservice
*ofservice
,
1328 const struct ofproto_controller
*c
)
1330 ofservice
->probe_interval
= c
->probe_interval
;
1331 ofservice
->rate_limit
= c
->rate_limit
;
1332 ofservice
->burst_limit
= c
->burst_limit
;
1335 /* Finds and returns the ofservice within 'mgr' that has the given
1336 * 'target', or a null pointer if none exists. */
1337 static struct ofservice
*
1338 ofservice_lookup(struct connmgr
*mgr
, const char *target
)
1340 struct ofservice
*ofservice
;
1342 HMAP_FOR_EACH_WITH_HASH (ofservice
, node
, hash_string(target
, 0),
1344 if (!strcmp(pvconn_get_name(ofservice
->pvconn
), target
)) {