]>
Commit | Line | Data |
---|---|---|
064af421 | 1 | /* |
77ab5fd2 | 2 | * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016 Nicira, Inc. |
064af421 | 3 | * |
a14bc59f BP |
4 | * Licensed under the Apache License, Version 2.0 (the "License"); |
5 | * you may not use this file except in compliance with the License. | |
6 | * You may obtain a copy of the License at: | |
064af421 | 7 | * |
a14bc59f BP |
8 | * http://www.apache.org/licenses/LICENSE-2.0 |
9 | * | |
10 | * Unless required by applicable law or agreed to in writing, software | |
11 | * distributed under the License is distributed on an "AS IS" BASIS, | |
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
13 | * See the License for the specific language governing permissions and | |
14 | * limitations under the License. | |
064af421 BP |
15 | */ |
16 | ||
17 | #include <config.h> | |
18 | #include "rconn.h" | |
064af421 BP |
19 | #include <errno.h> |
20 | #include <limits.h> | |
21 | #include <stdlib.h> | |
22 | #include <string.h> | |
23 | #include "coverage.h" | |
d271907f BW |
24 | #include "openflow/openflow.h" |
25 | #include "openvswitch/ofp-msgs.h" | |
f4248336 | 26 | #include "openvswitch/ofp-util.h" |
64c96779 | 27 | #include "openvswitch/ofpbuf.h" |
d271907f BW |
28 | #include "openvswitch/vconn.h" |
29 | #include "openvswitch/vlog.h" | |
064af421 BP |
30 | #include "poll-loop.h" |
31 | #include "sat-math.h" | |
c1bfdd9d | 32 | #include "stream.h" |
064af421 BP |
33 | #include "timeval.h" |
34 | #include "util.h" | |
064af421 | 35 | |
d98e6007 | 36 | VLOG_DEFINE_THIS_MODULE(rconn); |
5136ce49 | 37 | |
d76f09ea BP |
38 | COVERAGE_DEFINE(rconn_discarded); |
39 | COVERAGE_DEFINE(rconn_overflow); | |
40 | COVERAGE_DEFINE(rconn_queued); | |
41 | COVERAGE_DEFINE(rconn_sent); | |
42 | ||
46e2b6c8 BP |
43 | /* The connection states have the following meanings: |
44 | * | |
45 | * - S_VOID: No connection information is configured. | |
46 | * | |
47 | * - S_BACKOFF: Waiting for a period of time before reconnecting. | |
48 | * | |
49 | * - S_CONNECTING: A connection attempt is in progress and has not yet | |
50 | * succeeded or failed. | |
51 | * | |
52 | * - S_ACTIVE: A connection has been established and appears to be healthy. | |
53 | * | |
54 | * - S_IDLE: A connection has been established but has been idle for some | |
55 | * time. An echo request has been sent, but no reply has yet been | |
56 | * received. | |
57 | * | |
58 | * - S_DISCONNECTED: An unreliable connection has disconnected and cannot be | |
59 | * automatically retried. | |
60 | */ | |
064af421 BP |
61 | #define STATES \ |
62 | STATE(VOID, 1 << 0) \ | |
63 | STATE(BACKOFF, 1 << 1) \ | |
64 | STATE(CONNECTING, 1 << 2) \ | |
65 | STATE(ACTIVE, 1 << 3) \ | |
46e2b6c8 BP |
66 | STATE(IDLE, 1 << 4) \ |
67 | STATE(DISCONNECTED, 1 << 5) | |
064af421 BP |
68 | enum state { |
69 | #define STATE(NAME, VALUE) S_##NAME = VALUE, | |
70 | STATES | |
71 | #undef STATE | |
72 | }; | |
73 | ||
74 | static const char * | |
75 | state_name(enum state state) | |
76 | { | |
77 | switch (state) { | |
78 | #define STATE(NAME, VALUE) case S_##NAME: return #NAME; | |
79 | STATES | |
80 | #undef STATE | |
81 | } | |
82 | return "***ERROR***"; | |
83 | } | |
84 | ||
85 | /* A reliable connection to an OpenFlow switch or controller. | |
86 | * | |
87 | * See the large comment in rconn.h for more information. */ | |
88 | struct rconn { | |
9f5e8906 BP |
89 | struct ovs_mutex mutex; |
90 | ||
064af421 BP |
91 | enum state state; |
92 | time_t state_entered; | |
93 | ||
94 | struct vconn *vconn; | |
eb15cdbb BP |
95 | char *name; /* Human-readable descriptive name. */ |
96 | char *target; /* vconn name, passed to vconn_open(). */ | |
064af421 BP |
97 | bool reliable; |
98 | ||
ca6ba700 | 99 | struct ovs_list txq; /* Contains "struct ofpbuf"s. */ |
064af421 BP |
100 | |
101 | int backoff; | |
102 | int max_backoff; | |
103 | time_t backoff_deadline; | |
064af421 | 104 | time_t last_connected; |
2cdcb898 | 105 | time_t last_disconnected; |
064af421 | 106 | unsigned int seqno; |
88a20d6e | 107 | int last_error; |
064af421 BP |
108 | |
109 | /* In S_ACTIVE and S_IDLE, probably_admitted reports whether we believe | |
110 | * that the peer has made a (positive) admission control decision on our | |
111 | * connection. If we have not yet been (probably) admitted, then the | |
112 | * connection does not reset the timer used for deciding whether the switch | |
113 | * should go into fail-open mode. | |
114 | * | |
115 | * last_admitted reports the last time we believe such a positive admission | |
116 | * control decision was made. */ | |
117 | bool probably_admitted; | |
118 | time_t last_admitted; | |
119 | ||
120 | /* These values are simply for statistics reporting, not used directly by | |
8cd4882f | 121 | * anything internal to the rconn (or ofproto for that matter). */ |
064af421 BP |
122 | unsigned int n_attempted_connections, n_successful_connections; |
123 | time_t creation_time; | |
124 | unsigned long int total_time_connected; | |
125 | ||
133f2dc9 BP |
126 | /* Throughout this file, "probe" is shorthand for "inactivity probe". When |
127 | * no activity has been observed from the peer for a while, we send out an | |
128 | * echo request as an inactivity probe packet. We should receive back a | |
129 | * response. | |
130 | * | |
131 | * "Activity" is defined as either receiving an OpenFlow message from the | |
132 | * peer or successfully sending a message that had been in 'txq'. */ | |
064af421 | 133 | int probe_interval; /* Secs of inactivity before sending probe. */ |
133f2dc9 | 134 | time_t last_activity; /* Last time we saw some activity. */ |
064af421 | 135 | |
f125905c | 136 | uint8_t dscp; |
19d1ab55 | 137 | |
064af421 | 138 | /* Messages sent or received are copied to the monitor connections. */ |
4f35691d GS |
139 | #define MAXIMUM_MONITORS 8 |
140 | struct vconn *monitors[MAXIMUM_MONITORS]; | |
064af421 | 141 | size_t n_monitors; |
6042457b SH |
142 | |
143 | uint32_t allowed_versions; | |
064af421 BP |
144 | }; |
145 | ||
e182670b SH |
146 | uint32_t rconn_get_allowed_versions(const struct rconn *rconn) |
147 | { | |
148 | return rconn->allowed_versions; | |
149 | } | |
150 | ||
9f5e8906 BP |
151 | static unsigned int elapsed_in_this_state(const struct rconn *rc) |
152 | OVS_REQUIRES(rc->mutex); | |
153 | static unsigned int timeout(const struct rconn *rc) OVS_REQUIRES(rc->mutex); | |
154 | static bool timed_out(const struct rconn *rc) OVS_REQUIRES(rc->mutex); | |
155 | static void state_transition(struct rconn *rc, enum state) | |
156 | OVS_REQUIRES(rc->mutex); | |
157 | static void rconn_set_target__(struct rconn *rc, | |
158 | const char *target, const char *name) | |
159 | OVS_REQUIRES(rc->mutex); | |
160 | static int rconn_send__(struct rconn *rc, struct ofpbuf *, | |
161 | struct rconn_packet_counter *) | |
162 | OVS_REQUIRES(rc->mutex); | |
163 | static int try_send(struct rconn *rc) OVS_REQUIRES(rc->mutex); | |
164 | static void reconnect(struct rconn *rc) OVS_REQUIRES(rc->mutex); | |
165 | static void report_error(struct rconn *rc, int error) OVS_REQUIRES(rc->mutex); | |
166 | static void rconn_disconnect__(struct rconn *rc) OVS_REQUIRES(rc->mutex); | |
167 | static void disconnect(struct rconn *rc, int error) OVS_REQUIRES(rc->mutex); | |
168 | static void flush_queue(struct rconn *rc) OVS_REQUIRES(rc->mutex); | |
169 | static void close_monitor(struct rconn *rc, size_t idx, int retval) | |
170 | OVS_REQUIRES(rc->mutex); | |
064af421 BP |
171 | static void copy_to_monitor(struct rconn *, const struct ofpbuf *); |
172 | static bool is_connected_state(enum state); | |
173 | static bool is_admitted_msg(const struct ofpbuf *); | |
9f5e8906 BP |
174 | static bool rconn_logging_connection_attempts__(const struct rconn *rc) |
175 | OVS_REQUIRES(rc->mutex); | |
176 | static int rconn_get_version__(const struct rconn *rconn) | |
177 | OVS_REQUIRES(rconn->mutex); | |
178 | ||
179 | /* The following prototypes duplicate those in rconn.h, but there we weren't | |
180 | * able to add the OVS_EXCLUDED annotations because the definition of struct | |
181 | * rconn was not visible. */ | |
182 | ||
183 | void rconn_set_max_backoff(struct rconn *rc, int max_backoff) | |
184 | OVS_EXCLUDED(rc->mutex); | |
185 | void rconn_connect(struct rconn *rc, const char *target, const char *name) | |
186 | OVS_EXCLUDED(rc->mutex); | |
187 | void rconn_connect_unreliably(struct rconn *rc, | |
188 | struct vconn *vconn, const char *name) | |
189 | OVS_EXCLUDED(rc->mutex); | |
190 | void rconn_reconnect(struct rconn *rc) OVS_EXCLUDED(rc->mutex); | |
191 | void rconn_disconnect(struct rconn *rc) OVS_EXCLUDED(rc->mutex); | |
192 | void rconn_run(struct rconn *rc) OVS_EXCLUDED(rc->mutex); | |
193 | void rconn_run_wait(struct rconn *rc) OVS_EXCLUDED(rc->mutex); | |
194 | struct ofpbuf *rconn_recv(struct rconn *rc) OVS_EXCLUDED(rc->mutex); | |
195 | void rconn_recv_wait(struct rconn *rc) OVS_EXCLUDED(rc->mutex); | |
196 | int rconn_send(struct rconn *rc, struct ofpbuf *b, | |
197 | struct rconn_packet_counter *counter) | |
198 | OVS_EXCLUDED(rc->mutex); | |
199 | int rconn_send_with_limit(struct rconn *rc, struct ofpbuf *b, | |
200 | struct rconn_packet_counter *counter, | |
201 | int queue_limit) | |
202 | OVS_EXCLUDED(rc->mutex); | |
203 | void rconn_add_monitor(struct rconn *rc, struct vconn *vconn) | |
204 | OVS_EXCLUDED(rc->mutex); | |
205 | void rconn_set_name(struct rconn *rc, const char *new_name) | |
206 | OVS_EXCLUDED(rc->mutex); | |
207 | bool rconn_is_admitted(const struct rconn *rconn) OVS_EXCLUDED(rconn->mutex); | |
208 | int rconn_failure_duration(const struct rconn *rconn) | |
209 | OVS_EXCLUDED(rconn->mutex); | |
210 | ovs_be16 rconn_get_local_port(const struct rconn *rconn) | |
211 | OVS_EXCLUDED(rconn->mutex); | |
212 | int rconn_get_version(const struct rconn *rconn) OVS_EXCLUDED(rconn->mutex); | |
213 | unsigned int rconn_count_txqlen(const struct rconn *rc) | |
214 | OVS_EXCLUDED(rc->mutex); | |
215 | ||
064af421 | 216 | |
064af421 BP |
217 | /* Creates and returns a new rconn. |
218 | * | |
219 | * 'probe_interval' is a number of seconds. If the interval passes once | |
220 | * without an OpenFlow message being received from the peer, the rconn sends | |
221 | * out an "echo request" message. If the interval passes again without a | |
222 | * message being received, the rconn disconnects and re-connects to the peer. | |
223 | * Setting 'probe_interval' to 0 disables this behavior. | |
224 | * | |
225 | * 'max_backoff' is the maximum number of seconds between attempts to connect | |
226 | * to the peer. The actual interval starts at 1 second and doubles on each | |
227 | * failure until it reaches 'max_backoff'. If 0 is specified, the default of | |
9794e806 BP |
228 | * 8 seconds is used. |
229 | * | |
230 | * The new rconn is initially unconnected. Use rconn_connect() or | |
6042457b SH |
231 | * rconn_connect_unreliably() to connect it. |
232 | * | |
233 | * Connections made by the rconn will automatically negotiate an OpenFlow | |
234 | * protocol version acceptable to both peers on the connection. The version | |
5b8ab80c BP |
235 | * negotiated will be one of those in the 'allowed_versions' bitmap: version |
236 | * 'x' is allowed if allowed_versions & (1 << x) is nonzero. (The underlying | |
237 | * vconn will treat an 'allowed_versions' of 0 as OFPUTIL_DEFAULT_VERSIONS.) | |
238 | */ | |
064af421 | 239 | struct rconn * |
6042457b SH |
240 | rconn_create(int probe_interval, int max_backoff, uint8_t dscp, |
241 | uint32_t allowed_versions) | |
064af421 | 242 | { |
ec6fde61 | 243 | struct rconn *rc = xzalloc(sizeof *rc); |
064af421 | 244 | |
9f5e8906 BP |
245 | ovs_mutex_init(&rc->mutex); |
246 | ||
064af421 BP |
247 | rc->state = S_VOID; |
248 | rc->state_entered = time_now(); | |
249 | ||
250 | rc->vconn = NULL; | |
251 | rc->name = xstrdup("void"); | |
eb15cdbb | 252 | rc->target = xstrdup("void"); |
064af421 BP |
253 | rc->reliable = false; |
254 | ||
417e7e66 | 255 | ovs_list_init(&rc->txq); |
064af421 BP |
256 | |
257 | rc->backoff = 0; | |
c9aaa877 | 258 | rc->max_backoff = max_backoff ? max_backoff : 8; |
064af421 | 259 | rc->backoff_deadline = TIME_MIN; |
2cdcb898 AE |
260 | rc->last_connected = TIME_MIN; |
261 | rc->last_disconnected = TIME_MIN; | |
064af421 BP |
262 | rc->seqno = 0; |
263 | ||
064af421 BP |
264 | rc->probably_admitted = false; |
265 | rc->last_admitted = time_now(); | |
266 | ||
064af421 BP |
267 | rc->n_attempted_connections = 0; |
268 | rc->n_successful_connections = 0; | |
269 | rc->creation_time = time_now(); | |
270 | rc->total_time_connected = 0; | |
271 | ||
133f2dc9 BP |
272 | rc->last_activity = time_now(); |
273 | ||
064af421 | 274 | rconn_set_probe_interval(rc, probe_interval); |
f125905c | 275 | rconn_set_dscp(rc, dscp); |
064af421 BP |
276 | |
277 | rc->n_monitors = 0; | |
5b8ab80c | 278 | rc->allowed_versions = allowed_versions; |
064af421 BP |
279 | |
280 | return rc; | |
281 | } | |
282 | ||
283 | void | |
284 | rconn_set_max_backoff(struct rconn *rc, int max_backoff) | |
9f5e8906 | 285 | OVS_EXCLUDED(rc->mutex) |
064af421 | 286 | { |
9f5e8906 | 287 | ovs_mutex_lock(&rc->mutex); |
064af421 BP |
288 | rc->max_backoff = MAX(1, max_backoff); |
289 | if (rc->state == S_BACKOFF && rc->backoff > max_backoff) { | |
290 | rc->backoff = max_backoff; | |
291 | if (rc->backoff_deadline > time_now() + max_backoff) { | |
292 | rc->backoff_deadline = time_now() + max_backoff; | |
293 | } | |
294 | } | |
9f5e8906 | 295 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
296 | } |
297 | ||
298 | int | |
299 | rconn_get_max_backoff(const struct rconn *rc) | |
300 | { | |
301 | return rc->max_backoff; | |
302 | } | |
303 | ||
f125905c MM |
304 | void |
305 | rconn_set_dscp(struct rconn *rc, uint8_t dscp) | |
306 | { | |
307 | rc->dscp = dscp; | |
308 | } | |
309 | ||
0442efd9 MM |
310 | uint8_t |
311 | rconn_get_dscp(const struct rconn *rc) | |
312 | { | |
313 | return rc->dscp; | |
314 | } | |
315 | ||
064af421 BP |
316 | void |
317 | rconn_set_probe_interval(struct rconn *rc, int probe_interval) | |
318 | { | |
319 | rc->probe_interval = probe_interval ? MAX(5, probe_interval) : 0; | |
320 | } | |
321 | ||
322 | int | |
323 | rconn_get_probe_interval(const struct rconn *rc) | |
324 | { | |
325 | return rc->probe_interval; | |
326 | } | |
327 | ||
eb15cdbb BP |
328 | /* Drops any existing connection on 'rc', then sets up 'rc' to connect to |
329 | * 'target' and reconnect as needed. 'target' should be a remote OpenFlow | |
330 | * target in a form acceptable to vconn_open(). | |
331 | * | |
332 | * If 'name' is nonnull, then it is used in log messages in place of 'target'. | |
333 | * It should presumably give more information to a human reader than 'target', | |
334 | * but it need not be acceptable to vconn_open(). */ | |
d4cbfb19 | 335 | void |
eb15cdbb | 336 | rconn_connect(struct rconn *rc, const char *target, const char *name) |
9f5e8906 | 337 | OVS_EXCLUDED(rc->mutex) |
064af421 | 338 | { |
9f5e8906 BP |
339 | ovs_mutex_lock(&rc->mutex); |
340 | rconn_disconnect__(rc); | |
eb15cdbb | 341 | rconn_set_target__(rc, target, name); |
064af421 | 342 | rc->reliable = true; |
c1bfdd9d | 343 | if (!stream_or_pstream_needs_probes(target)) { |
344 | rc->probe_interval = 0; | |
345 | } | |
d4cbfb19 | 346 | reconnect(rc); |
9f5e8906 | 347 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
348 | } |
349 | ||
eb15cdbb BP |
350 | /* Drops any existing connection on 'rc', then configures 'rc' to use |
351 | * 'vconn'. If the connection on 'vconn' drops, 'rc' will not reconnect on it | |
352 | * own. | |
353 | * | |
354 | * By default, the target obtained from vconn_get_name(vconn) is used in log | |
355 | * messages. If 'name' is nonnull, then it is used instead. It should | |
356 | * presumably give more information to a human reader than the target, but it | |
357 | * need not be acceptable to vconn_open(). */ | |
064af421 | 358 | void |
eb15cdbb BP |
359 | rconn_connect_unreliably(struct rconn *rc, |
360 | struct vconn *vconn, const char *name) | |
9f5e8906 | 361 | OVS_EXCLUDED(rc->mutex) |
064af421 | 362 | { |
cb22974d | 363 | ovs_assert(vconn != NULL); |
9f5e8906 BP |
364 | |
365 | ovs_mutex_lock(&rc->mutex); | |
366 | rconn_disconnect__(rc); | |
eb15cdbb | 367 | rconn_set_target__(rc, vconn_get_name(vconn), name); |
064af421 BP |
368 | rc->reliable = false; |
369 | rc->vconn = vconn; | |
370 | rc->last_connected = time_now(); | |
371 | state_transition(rc, S_ACTIVE); | |
9f5e8906 | 372 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
373 | } |
374 | ||
375 | /* If 'rc' is connected, forces it to drop the connection and reconnect. */ | |
376 | void | |
377 | rconn_reconnect(struct rconn *rc) | |
9f5e8906 | 378 | OVS_EXCLUDED(rc->mutex) |
064af421 | 379 | { |
9f5e8906 | 380 | ovs_mutex_lock(&rc->mutex); |
064af421 | 381 | if (rc->state & (S_ACTIVE | S_IDLE)) { |
b97ba90b | 382 | VLOG_INFO("%s: disconnecting", rc->name); |
064af421 BP |
383 | disconnect(rc, 0); |
384 | } | |
9f5e8906 | 385 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
386 | } |
387 | ||
9f5e8906 BP |
388 | static void |
389 | rconn_disconnect__(struct rconn *rc) | |
390 | OVS_REQUIRES(rc->mutex) | |
064af421 BP |
391 | { |
392 | if (rc->state != S_VOID) { | |
393 | if (rc->vconn) { | |
394 | vconn_close(rc->vconn); | |
395 | rc->vconn = NULL; | |
396 | } | |
eb15cdbb | 397 | rconn_set_target__(rc, "void", NULL); |
064af421 BP |
398 | rc->reliable = false; |
399 | ||
400 | rc->backoff = 0; | |
401 | rc->backoff_deadline = TIME_MIN; | |
402 | ||
403 | state_transition(rc, S_VOID); | |
404 | } | |
405 | } | |
406 | ||
9f5e8906 BP |
407 | void |
408 | rconn_disconnect(struct rconn *rc) | |
409 | OVS_EXCLUDED(rc->mutex) | |
410 | { | |
411 | ovs_mutex_lock(&rc->mutex); | |
412 | rconn_disconnect__(rc); | |
413 | ovs_mutex_unlock(&rc->mutex); | |
414 | } | |
415 | ||
064af421 BP |
416 | /* Disconnects 'rc' and frees the underlying storage. */ |
417 | void | |
418 | rconn_destroy(struct rconn *rc) | |
419 | { | |
420 | if (rc) { | |
421 | size_t i; | |
422 | ||
9f5e8906 | 423 | ovs_mutex_lock(&rc->mutex); |
064af421 | 424 | free(rc->name); |
eb15cdbb | 425 | free(rc->target); |
064af421 BP |
426 | vconn_close(rc->vconn); |
427 | flush_queue(rc); | |
b3907fbc | 428 | ofpbuf_list_delete(&rc->txq); |
064af421 BP |
429 | for (i = 0; i < rc->n_monitors; i++) { |
430 | vconn_close(rc->monitors[i]); | |
431 | } | |
9f5e8906 BP |
432 | ovs_mutex_unlock(&rc->mutex); |
433 | ovs_mutex_destroy(&rc->mutex); | |
434 | ||
064af421 BP |
435 | free(rc); |
436 | } | |
437 | } | |
438 | ||
439 | static unsigned int | |
67a4917b | 440 | timeout_VOID(const struct rconn *rc OVS_UNUSED) |
9f5e8906 | 441 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
442 | { |
443 | return UINT_MAX; | |
444 | } | |
445 | ||
446 | static void | |
67a4917b | 447 | run_VOID(struct rconn *rc OVS_UNUSED) |
9f5e8906 | 448 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
449 | { |
450 | /* Nothing to do. */ | |
451 | } | |
452 | ||
d4cbfb19 | 453 | static void |
064af421 | 454 | reconnect(struct rconn *rc) |
9f5e8906 | 455 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
456 | { |
457 | int retval; | |
458 | ||
07c8c80d BP |
459 | if (rconn_logging_connection_attempts__(rc)) { |
460 | VLOG_INFO("%s: connecting...", rc->name); | |
461 | } | |
064af421 | 462 | rc->n_attempted_connections++; |
82c8c53c BP |
463 | retval = vconn_open(rc->target, rc->allowed_versions, rc->dscp, |
464 | &rc->vconn); | |
064af421 BP |
465 | if (!retval) { |
466 | rc->backoff_deadline = time_now() + rc->backoff; | |
467 | state_transition(rc, S_CONNECTING); | |
468 | } else { | |
10a89ef0 BP |
469 | VLOG_WARN("%s: connection failed (%s)", |
470 | rc->name, ovs_strerror(retval)); | |
064af421 | 471 | rc->backoff_deadline = TIME_MAX; /* Prevent resetting backoff. */ |
88a20d6e | 472 | disconnect(rc, retval); |
064af421 | 473 | } |
064af421 BP |
474 | } |
475 | ||
476 | static unsigned int | |
477 | timeout_BACKOFF(const struct rconn *rc) | |
9f5e8906 | 478 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
479 | { |
480 | return rc->backoff; | |
481 | } | |
482 | ||
483 | static void | |
484 | run_BACKOFF(struct rconn *rc) | |
9f5e8906 | 485 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
486 | { |
487 | if (timed_out(rc)) { | |
488 | reconnect(rc); | |
489 | } | |
490 | } | |
491 | ||
492 | static unsigned int | |
493 | timeout_CONNECTING(const struct rconn *rc) | |
9f5e8906 | 494 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
495 | { |
496 | return MAX(1, rc->backoff); | |
497 | } | |
498 | ||
499 | static void | |
500 | run_CONNECTING(struct rconn *rc) | |
9f5e8906 | 501 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
502 | { |
503 | int retval = vconn_connect(rc->vconn); | |
504 | if (!retval) { | |
505 | VLOG_INFO("%s: connected", rc->name); | |
506 | rc->n_successful_connections++; | |
507 | state_transition(rc, S_ACTIVE); | |
508 | rc->last_connected = rc->state_entered; | |
509 | } else if (retval != EAGAIN) { | |
07c8c80d BP |
510 | if (rconn_logging_connection_attempts__(rc)) { |
511 | VLOG_INFO("%s: connection failed (%s)", | |
10a89ef0 | 512 | rc->name, ovs_strerror(retval)); |
07c8c80d | 513 | } |
064af421 BP |
514 | disconnect(rc, retval); |
515 | } else if (timed_out(rc)) { | |
07c8c80d BP |
516 | if (rconn_logging_connection_attempts__(rc)) { |
517 | VLOG_INFO("%s: connection timed out", rc->name); | |
518 | } | |
064af421 | 519 | rc->backoff_deadline = TIME_MAX; /* Prevent resetting backoff. */ |
88a20d6e | 520 | disconnect(rc, ETIMEDOUT); |
064af421 BP |
521 | } |
522 | } | |
523 | ||
524 | static void | |
525 | do_tx_work(struct rconn *rc) | |
9f5e8906 | 526 | OVS_REQUIRES(rc->mutex) |
064af421 | 527 | { |
417e7e66 | 528 | if (ovs_list_is_empty(&rc->txq)) { |
064af421 BP |
529 | return; |
530 | } | |
417e7e66 | 531 | while (!ovs_list_is_empty(&rc->txq)) { |
064af421 BP |
532 | int error = try_send(rc); |
533 | if (error) { | |
534 | break; | |
535 | } | |
133f2dc9 | 536 | rc->last_activity = time_now(); |
064af421 | 537 | } |
417e7e66 | 538 | if (ovs_list_is_empty(&rc->txq)) { |
064af421 BP |
539 | poll_immediate_wake(); |
540 | } | |
541 | } | |
542 | ||
543 | static unsigned int | |
544 | timeout_ACTIVE(const struct rconn *rc) | |
9f5e8906 | 545 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
546 | { |
547 | if (rc->probe_interval) { | |
133f2dc9 | 548 | unsigned int base = MAX(rc->last_activity, rc->state_entered); |
064af421 BP |
549 | unsigned int arg = base + rc->probe_interval - rc->state_entered; |
550 | return arg; | |
551 | } | |
552 | return UINT_MAX; | |
553 | } | |
554 | ||
555 | static void | |
556 | run_ACTIVE(struct rconn *rc) | |
9f5e8906 | 557 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
558 | { |
559 | if (timed_out(rc)) { | |
133f2dc9 | 560 | unsigned int base = MAX(rc->last_activity, rc->state_entered); |
064af421 BP |
561 | VLOG_DBG("%s: idle %u seconds, sending inactivity probe", |
562 | rc->name, (unsigned int) (time_now() - base)); | |
563 | ||
564 | /* Ordering is important here: rconn_send() can transition to BACKOFF, | |
565 | * and we don't want to transition back to IDLE if so, because then we | |
566 | * can end up queuing a packet with vconn == NULL and then *boom*. */ | |
567 | state_transition(rc, S_IDLE); | |
a203ce69 BP |
568 | |
569 | /* Send an echo request if we can. (If version negotiation is not | |
570 | * complete, that is, if we did not yet receive a "hello" message from | |
571 | * the peer, we do not know the version to use, so we don't send | |
572 | * anything.) */ | |
573 | int version = rconn_get_version__(rc); | |
574 | if (version >= 0 && version <= 0xff) { | |
575 | rconn_send__(rc, make_echo_request(version), NULL); | |
576 | } | |
577 | ||
064af421 BP |
578 | return; |
579 | } | |
580 | ||
581 | do_tx_work(rc); | |
582 | } | |
583 | ||
584 | static unsigned int | |
585 | timeout_IDLE(const struct rconn *rc) | |
9f5e8906 | 586 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
587 | { |
588 | return rc->probe_interval; | |
589 | } | |
590 | ||
591 | static void | |
592 | run_IDLE(struct rconn *rc) | |
9f5e8906 | 593 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
594 | { |
595 | if (timed_out(rc)) { | |
064af421 BP |
596 | VLOG_ERR("%s: no response to inactivity probe after %u " |
597 | "seconds, disconnecting", | |
598 | rc->name, elapsed_in_this_state(rc)); | |
88a20d6e | 599 | disconnect(rc, ETIMEDOUT); |
064af421 BP |
600 | } else { |
601 | do_tx_work(rc); | |
602 | } | |
603 | } | |
604 | ||
46e2b6c8 BP |
605 | static unsigned int |
606 | timeout_DISCONNECTED(const struct rconn *rc OVS_UNUSED) | |
607 | OVS_REQUIRES(rc->mutex) | |
608 | { | |
609 | return UINT_MAX; | |
610 | } | |
611 | ||
612 | static void | |
613 | run_DISCONNECTED(struct rconn *rc OVS_UNUSED) | |
614 | OVS_REQUIRES(rc->mutex) | |
615 | { | |
616 | /* Nothing to do. */ | |
617 | } | |
618 | ||
064af421 BP |
619 | /* Performs whatever activities are necessary to maintain 'rc': if 'rc' is |
620 | * disconnected, attempts to (re)connect, backing off as necessary; if 'rc' is | |
621 | * connected, attempts to send packets in the send queue, if any. */ | |
622 | void | |
623 | rconn_run(struct rconn *rc) | |
9f5e8906 | 624 | OVS_EXCLUDED(rc->mutex) |
064af421 BP |
625 | { |
626 | int old_state; | |
60cb3eb8 BP |
627 | size_t i; |
628 | ||
9f5e8906 | 629 | ovs_mutex_lock(&rc->mutex); |
60cb3eb8 | 630 | if (rc->vconn) { |
accaecc4 BP |
631 | int error; |
632 | ||
60cb3eb8 | 633 | vconn_run(rc->vconn); |
accaecc4 BP |
634 | |
635 | error = vconn_get_status(rc->vconn); | |
636 | if (error) { | |
637 | report_error(rc, error); | |
638 | disconnect(rc, error); | |
639 | } | |
60cb3eb8 | 640 | } |
f44c5146 BP |
641 | for (i = 0; i < rc->n_monitors; ) { |
642 | struct ofpbuf *msg; | |
643 | int retval; | |
644 | ||
60cb3eb8 | 645 | vconn_run(rc->monitors[i]); |
f44c5146 BP |
646 | |
647 | /* Drain any stray message that came in on the monitor connection. */ | |
648 | retval = vconn_recv(rc->monitors[i], &msg); | |
649 | if (!retval) { | |
650 | ofpbuf_delete(msg); | |
651 | } else if (retval != EAGAIN) { | |
652 | close_monitor(rc, i, retval); | |
653 | continue; | |
654 | } | |
655 | i++; | |
60cb3eb8 BP |
656 | } |
657 | ||
064af421 BP |
658 | do { |
659 | old_state = rc->state; | |
660 | switch (rc->state) { | |
661 | #define STATE(NAME, VALUE) case S_##NAME: run_##NAME(rc); break; | |
662 | STATES | |
663 | #undef STATE | |
664 | default: | |
428b2edd | 665 | OVS_NOT_REACHED(); |
064af421 BP |
666 | } |
667 | } while (rc->state != old_state); | |
9f5e8906 | 668 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
669 | } |
670 | ||
671 | /* Causes the next call to poll_block() to wake up when rconn_run() should be | |
672 | * called on 'rc'. */ | |
673 | void | |
674 | rconn_run_wait(struct rconn *rc) | |
9f5e8906 | 675 | OVS_EXCLUDED(rc->mutex) |
064af421 | 676 | { |
60cb3eb8 BP |
677 | unsigned int timeo; |
678 | size_t i; | |
679 | ||
9f5e8906 | 680 | ovs_mutex_lock(&rc->mutex); |
60cb3eb8 BP |
681 | if (rc->vconn) { |
682 | vconn_run_wait(rc->vconn); | |
417e7e66 | 683 | if ((rc->state & (S_ACTIVE | S_IDLE)) && !ovs_list_is_empty(&rc->txq)) { |
82c2b79d BP |
684 | vconn_wait(rc->vconn, WAIT_SEND); |
685 | } | |
60cb3eb8 BP |
686 | } |
687 | for (i = 0; i < rc->n_monitors; i++) { | |
688 | vconn_run_wait(rc->monitors[i]); | |
f44c5146 | 689 | vconn_recv_wait(rc->monitors[i]); |
60cb3eb8 BP |
690 | } |
691 | ||
692 | timeo = timeout(rc); | |
064af421 | 693 | if (timeo != UINT_MAX) { |
7cf8b266 BP |
694 | long long int expires = sat_add(rc->state_entered, timeo); |
695 | poll_timer_wait_until(expires * 1000); | |
064af421 | 696 | } |
9f5e8906 | 697 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
698 | } |
699 | ||
700 | /* Attempts to receive a packet from 'rc'. If successful, returns the packet; | |
701 | * otherwise, returns a null pointer. The caller is responsible for freeing | |
702 | * the packet (with ofpbuf_delete()). */ | |
703 | struct ofpbuf * | |
704 | rconn_recv(struct rconn *rc) | |
9f5e8906 | 705 | OVS_EXCLUDED(rc->mutex) |
064af421 | 706 | { |
9f5e8906 BP |
707 | struct ofpbuf *buffer = NULL; |
708 | ||
709 | ovs_mutex_lock(&rc->mutex); | |
064af421 | 710 | if (rc->state & (S_ACTIVE | S_IDLE)) { |
064af421 BP |
711 | int error = vconn_recv(rc->vconn, &buffer); |
712 | if (!error) { | |
713 | copy_to_monitor(rc, buffer); | |
7778bd15 | 714 | if (rc->probably_admitted || is_admitted_msg(buffer) |
064af421 BP |
715 | || time_now() - rc->last_connected >= 30) { |
716 | rc->probably_admitted = true; | |
717 | rc->last_admitted = time_now(); | |
718 | } | |
133f2dc9 | 719 | rc->last_activity = time_now(); |
064af421 BP |
720 | if (rc->state == S_IDLE) { |
721 | state_transition(rc, S_ACTIVE); | |
722 | } | |
064af421 | 723 | } else if (error != EAGAIN) { |
b97ba90b | 724 | report_error(rc, error); |
064af421 BP |
725 | disconnect(rc, error); |
726 | } | |
727 | } | |
9f5e8906 BP |
728 | ovs_mutex_unlock(&rc->mutex); |
729 | ||
730 | return buffer; | |
064af421 BP |
731 | } |
732 | ||
733 | /* Causes the next call to poll_block() to wake up when a packet may be ready | |
734 | * to be received by vconn_recv() on 'rc'. */ | |
735 | void | |
736 | rconn_recv_wait(struct rconn *rc) | |
9f5e8906 | 737 | OVS_EXCLUDED(rc->mutex) |
064af421 | 738 | { |
9f5e8906 | 739 | ovs_mutex_lock(&rc->mutex); |
064af421 BP |
740 | if (rc->vconn) { |
741 | vconn_wait(rc->vconn, WAIT_RECV); | |
742 | } | |
9f5e8906 | 743 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
744 | } |
745 | ||
9f5e8906 BP |
746 | static int |
747 | rconn_send__(struct rconn *rc, struct ofpbuf *b, | |
064af421 | 748 | struct rconn_packet_counter *counter) |
9f5e8906 | 749 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
750 | { |
751 | if (rconn_is_connected(rc)) { | |
752 | COVERAGE_INC(rconn_queued); | |
753 | copy_to_monitor(rc, b); | |
437d0d22 | 754 | |
064af421 | 755 | if (counter) { |
6fd6ed71 | 756 | rconn_packet_counter_inc(counter, b->size); |
064af421 | 757 | } |
437d0d22 | 758 | |
cf3b7538 | 759 | /* Reuse 'frame' as a private pointer while 'b' is in txq. */ |
6fd6ed71 | 760 | b->header = counter; |
437d0d22 | 761 | |
417e7e66 | 762 | ovs_list_push_back(&rc->txq, &b->list_node); |
064af421 BP |
763 | |
764 | /* If the queue was empty before we added 'b', try to send some | |
765 | * packets. (But if the queue had packets in it, it's because the | |
766 | * vconn is backlogged and there's no point in stuffing more into it | |
767 | * now. We'll get back to that in rconn_run().) */ | |
b3907fbc | 768 | if (rc->txq.next == &b->list_node) { |
064af421 BP |
769 | try_send(rc); |
770 | } | |
771 | return 0; | |
772 | } else { | |
acb9da40 | 773 | ofpbuf_delete(b); |
064af421 BP |
774 | return ENOTCONN; |
775 | } | |
776 | } | |
777 | ||
9f5e8906 BP |
778 | /* Sends 'b' on 'rc'. Returns 0 if successful, or ENOTCONN if 'rc' is not |
779 | * currently connected. Takes ownership of 'b'. | |
780 | * | |
781 | * If 'counter' is non-null, then 'counter' will be incremented while the | |
782 | * packet is in flight, then decremented when it has been sent (or discarded | |
783 | * due to disconnection). Because 'b' may be sent (or discarded) before this | |
784 | * function returns, the caller may not be able to observe any change in | |
785 | * 'counter'. | |
786 | * | |
787 | * There is no rconn_send_wait() function: an rconn has a send queue that it | |
788 | * takes care of sending if you call rconn_run(), which will have the side | |
789 | * effect of waking up poll_block(). */ | |
790 | int | |
791 | rconn_send(struct rconn *rc, struct ofpbuf *b, | |
792 | struct rconn_packet_counter *counter) | |
793 | OVS_EXCLUDED(rc->mutex) | |
794 | { | |
795 | int error; | |
796 | ||
797 | ovs_mutex_lock(&rc->mutex); | |
798 | error = rconn_send__(rc, b, counter); | |
799 | ovs_mutex_unlock(&rc->mutex); | |
800 | ||
801 | return error; | |
802 | } | |
803 | ||
064af421 BP |
804 | /* Sends 'b' on 'rc'. Increments 'counter' while the packet is in flight; it |
805 | * will be decremented when it has been sent (or discarded due to | |
806 | * disconnection). Returns 0 if successful, EAGAIN if 'counter->n' is already | |
807 | * at least as large as 'queue_limit', or ENOTCONN if 'rc' is not currently | |
808 | * connected. Regardless of return value, 'b' is destroyed. | |
809 | * | |
810 | * Because 'b' may be sent (or discarded) before this function returns, the | |
811 | * caller may not be able to observe any change in 'counter'. | |
812 | * | |
813 | * There is no rconn_send_wait() function: an rconn has a send queue that it | |
814 | * takes care of sending if you call rconn_run(), which will have the side | |
815 | * effect of waking up poll_block(). */ | |
816 | int | |
817 | rconn_send_with_limit(struct rconn *rc, struct ofpbuf *b, | |
818 | struct rconn_packet_counter *counter, int queue_limit) | |
9f5e8906 | 819 | OVS_EXCLUDED(rc->mutex) |
064af421 | 820 | { |
9f5e8906 BP |
821 | int error; |
822 | ||
823 | ovs_mutex_lock(&rc->mutex); | |
a3d1ff00 | 824 | if (rconn_packet_counter_n_packets(counter) < queue_limit) { |
9f5e8906 | 825 | error = rconn_send__(rc, b, counter); |
b8dddecf | 826 | } else { |
064af421 | 827 | COVERAGE_INC(rconn_overflow); |
b8dddecf | 828 | ofpbuf_delete(b); |
9f5e8906 | 829 | error = EAGAIN; |
064af421 | 830 | } |
9f5e8906 BP |
831 | ovs_mutex_unlock(&rc->mutex); |
832 | ||
833 | return error; | |
064af421 BP |
834 | } |
835 | ||
064af421 BP |
836 | /* Adds 'vconn' to 'rc' as a monitoring connection, to which all messages sent |
837 | * and received on 'rconn' will be copied. 'rc' takes ownership of 'vconn'. */ | |
838 | void | |
839 | rconn_add_monitor(struct rconn *rc, struct vconn *vconn) | |
9f5e8906 | 840 | OVS_EXCLUDED(rc->mutex) |
064af421 | 841 | { |
9f5e8906 | 842 | ovs_mutex_lock(&rc->mutex); |
064af421 BP |
843 | if (rc->n_monitors < ARRAY_SIZE(rc->monitors)) { |
844 | VLOG_INFO("new monitor connection from %s", vconn_get_name(vconn)); | |
845 | rc->monitors[rc->n_monitors++] = vconn; | |
846 | } else { | |
847 | VLOG_DBG("too many monitor connections, discarding %s", | |
848 | vconn_get_name(vconn)); | |
849 | vconn_close(vconn); | |
850 | } | |
9f5e8906 | 851 | ovs_mutex_unlock(&rc->mutex); |
064af421 BP |
852 | } |
853 | ||
eb15cdbb BP |
854 | /* Returns 'rc''s name. This is a name for human consumption, appropriate for |
855 | * use in log messages. It is not necessarily a name that may be passed | |
856 | * directly to, e.g., vconn_open(). */ | |
064af421 BP |
857 | const char * |
858 | rconn_get_name(const struct rconn *rc) | |
859 | { | |
860 | return rc->name; | |
861 | } | |
862 | ||
eb15cdbb BP |
863 | /* Sets 'rc''s name to 'new_name'. */ |
864 | void | |
865 | rconn_set_name(struct rconn *rc, const char *new_name) | |
9f5e8906 | 866 | OVS_EXCLUDED(rc->mutex) |
eb15cdbb | 867 | { |
9f5e8906 | 868 | ovs_mutex_lock(&rc->mutex); |
eb15cdbb BP |
869 | free(rc->name); |
870 | rc->name = xstrdup(new_name); | |
9f5e8906 | 871 | ovs_mutex_unlock(&rc->mutex); |
eb15cdbb BP |
872 | } |
873 | ||
874 | /* Returns 'rc''s target. This is intended to be a string that may be passed | |
875 | * directly to, e.g., vconn_open(). */ | |
876 | const char * | |
877 | rconn_get_target(const struct rconn *rc) | |
878 | { | |
879 | return rc->target; | |
880 | } | |
881 | ||
064af421 BP |
882 | /* Returns true if 'rconn' is connected or in the process of reconnecting, |
883 | * false if 'rconn' is disconnected and will not reconnect on its own. */ | |
884 | bool | |
885 | rconn_is_alive(const struct rconn *rconn) | |
886 | { | |
46e2b6c8 | 887 | return rconn->state != S_VOID && rconn->state != S_DISCONNECTED; |
064af421 BP |
888 | } |
889 | ||
890 | /* Returns true if 'rconn' is connected, false otherwise. */ | |
891 | bool | |
892 | rconn_is_connected(const struct rconn *rconn) | |
893 | { | |
894 | return is_connected_state(rconn->state); | |
895 | } | |
896 | ||
9f5e8906 BP |
897 | static bool |
898 | rconn_is_admitted__(const struct rconn *rconn) | |
899 | OVS_REQUIRES(rconn->mutex) | |
900 | { | |
901 | return (rconn_is_connected(rconn) | |
902 | && rconn->last_admitted >= rconn->last_connected); | |
903 | } | |
904 | ||
7778bd15 BP |
905 | /* Returns true if 'rconn' is connected and thought to have been accepted by |
906 | * the peer's admission-control policy. */ | |
907 | bool | |
908 | rconn_is_admitted(const struct rconn *rconn) | |
9f5e8906 | 909 | OVS_EXCLUDED(rconn->mutex) |
7778bd15 | 910 | { |
9f5e8906 BP |
911 | bool admitted; |
912 | ||
913 | ovs_mutex_lock(&rconn->mutex); | |
914 | admitted = rconn_is_admitted__(rconn); | |
915 | ovs_mutex_unlock(&rconn->mutex); | |
916 | ||
917 | return admitted; | |
7778bd15 BP |
918 | } |
919 | ||
920 | /* Returns 0 if 'rconn' is currently connected and considered to have been | |
921 | * accepted by the peer's admission-control policy, otherwise the number of | |
922 | * seconds since 'rconn' was last in such a state. */ | |
064af421 BP |
923 | int |
924 | rconn_failure_duration(const struct rconn *rconn) | |
9f5e8906 | 925 | OVS_EXCLUDED(rconn->mutex) |
064af421 | 926 | { |
9f5e8906 BP |
927 | int duration; |
928 | ||
929 | ovs_mutex_lock(&rconn->mutex); | |
930 | duration = (rconn_is_admitted__(rconn) | |
931 | ? 0 | |
932 | : time_now() - rconn->last_admitted); | |
933 | ovs_mutex_unlock(&rconn->mutex); | |
934 | ||
935 | return duration; | |
064af421 BP |
936 | } |
937 | ||
9f5e8906 BP |
938 | static int |
939 | rconn_get_version__(const struct rconn *rconn) | |
940 | OVS_REQUIRES(rconn->mutex) | |
941 | { | |
942 | return rconn->vconn ? vconn_get_version(rconn->vconn) : -1; | |
064af421 BP |
943 | } |
944 | ||
27527aa0 BP |
945 | /* Returns the OpenFlow version negotiated with the peer, or -1 if there is |
946 | * currently no connection or if version negotiation is not yet complete. */ | |
947 | int | |
948 | rconn_get_version(const struct rconn *rconn) | |
9f5e8906 | 949 | OVS_EXCLUDED(rconn->mutex) |
27527aa0 | 950 | { |
9f5e8906 BP |
951 | int version; |
952 | ||
953 | ovs_mutex_lock(&rconn->mutex); | |
954 | version = rconn_get_version__(rconn); | |
955 | ovs_mutex_unlock(&rconn->mutex); | |
956 | ||
957 | return version; | |
27527aa0 BP |
958 | } |
959 | ||
064af421 BP |
960 | /* Returns a string representing the internal state of 'rc'. The caller must |
961 | * not modify or free the string. */ | |
962 | const char * | |
963 | rconn_get_state(const struct rconn *rc) | |
964 | { | |
965 | return state_name(rc->state); | |
966 | } | |
967 | ||
064af421 | 968 | /* Returns the time at which the last successful connection was made by |
2cdcb898 | 969 | * 'rc'. Returns TIME_MIN if never connected. */ |
064af421 BP |
970 | time_t |
971 | rconn_get_last_connection(const struct rconn *rc) | |
972 | { | |
973 | return rc->last_connected; | |
974 | } | |
975 | ||
2cdcb898 AE |
976 | /* Returns the time at which 'rc' was last disconnected. Returns TIME_MIN |
977 | * if never disconnected. */ | |
978 | time_t | |
979 | rconn_get_last_disconnect(const struct rconn *rc) | |
980 | { | |
981 | return rc->last_disconnected; | |
982 | } | |
983 | ||
064af421 BP |
984 | /* Returns 'rc''s current connection sequence number, a number that changes |
985 | * every time that 'rconn' connects or disconnects. */ | |
986 | unsigned int | |
987 | rconn_get_connection_seqno(const struct rconn *rc) | |
988 | { | |
989 | return rc->seqno; | |
990 | } | |
88a20d6e BP |
991 | |
992 | /* Returns a value that explains why 'rc' last disconnected: | |
993 | * | |
994 | * - 0 means that the last disconnection was caused by a call to | |
995 | * rconn_disconnect(), or that 'rc' is new and has not yet completed its | |
996 | * initial connection or connection attempt. | |
997 | * | |
998 | * - EOF means that the connection was closed in the normal way by the peer. | |
999 | * | |
1000 | * - A positive integer is an errno value that represents the error. | |
1001 | */ | |
1002 | int | |
1003 | rconn_get_last_error(const struct rconn *rc) | |
1004 | { | |
1005 | return rc->last_error; | |
1006 | } | |
0d085684 BP |
1007 | |
1008 | /* Returns the number of messages queued for transmission on 'rc'. */ | |
1009 | unsigned int | |
1010 | rconn_count_txqlen(const struct rconn *rc) | |
9f5e8906 | 1011 | OVS_EXCLUDED(rc->mutex) |
0d085684 | 1012 | { |
9f5e8906 BP |
1013 | unsigned int len; |
1014 | ||
1015 | ovs_mutex_lock(&rc->mutex); | |
417e7e66 | 1016 | len = ovs_list_size(&rc->txq); |
9f5e8906 BP |
1017 | ovs_mutex_unlock(&rc->mutex); |
1018 | ||
1019 | return len; | |
0d085684 | 1020 | } |
064af421 BP |
1021 | \f |
1022 | struct rconn_packet_counter * | |
1023 | rconn_packet_counter_create(void) | |
1024 | { | |
a6441685 | 1025 | struct rconn_packet_counter *c = xzalloc(sizeof *c); |
a3d1ff00 BP |
1026 | ovs_mutex_init(&c->mutex); |
1027 | ovs_mutex_lock(&c->mutex); | |
064af421 | 1028 | c->ref_cnt = 1; |
a3d1ff00 | 1029 | ovs_mutex_unlock(&c->mutex); |
064af421 BP |
1030 | return c; |
1031 | } | |
1032 | ||
1033 | void | |
1034 | rconn_packet_counter_destroy(struct rconn_packet_counter *c) | |
1035 | { | |
1036 | if (c) { | |
a3d1ff00 BP |
1037 | bool dead; |
1038 | ||
1039 | ovs_mutex_lock(&c->mutex); | |
cb22974d | 1040 | ovs_assert(c->ref_cnt > 0); |
a3d1ff00 BP |
1041 | dead = !--c->ref_cnt && !c->n_packets; |
1042 | ovs_mutex_unlock(&c->mutex); | |
1043 | ||
1044 | if (dead) { | |
1045 | ovs_mutex_destroy(&c->mutex); | |
064af421 BP |
1046 | free(c); |
1047 | } | |
1048 | } | |
1049 | } | |
1050 | ||
1051 | void | |
a6441685 | 1052 | rconn_packet_counter_inc(struct rconn_packet_counter *c, unsigned int n_bytes) |
064af421 | 1053 | { |
a3d1ff00 | 1054 | ovs_mutex_lock(&c->mutex); |
a6441685 BP |
1055 | c->n_packets++; |
1056 | c->n_bytes += n_bytes; | |
a3d1ff00 | 1057 | ovs_mutex_unlock(&c->mutex); |
064af421 BP |
1058 | } |
1059 | ||
1060 | void | |
a6441685 | 1061 | rconn_packet_counter_dec(struct rconn_packet_counter *c, unsigned int n_bytes) |
064af421 | 1062 | { |
a3d1ff00 | 1063 | bool dead = false; |
a6441685 | 1064 | |
a3d1ff00 BP |
1065 | ovs_mutex_lock(&c->mutex); |
1066 | ovs_assert(c->n_packets > 0); | |
1067 | ovs_assert(c->n_packets == 1 | |
1068 | ? c->n_bytes == n_bytes | |
1069 | : c->n_bytes > n_bytes); | |
a6441685 | 1070 | c->n_packets--; |
a3d1ff00 BP |
1071 | c->n_bytes -= n_bytes; |
1072 | dead = !c->n_packets && !c->ref_cnt; | |
1073 | ovs_mutex_unlock(&c->mutex); | |
1074 | ||
1075 | if (dead) { | |
1076 | ovs_mutex_destroy(&c->mutex); | |
1077 | free(c); | |
064af421 BP |
1078 | } |
1079 | } | |
a3d1ff00 BP |
1080 | |
1081 | unsigned int | |
1082 | rconn_packet_counter_n_packets(const struct rconn_packet_counter *c) | |
1083 | { | |
1084 | unsigned int n; | |
1085 | ||
1086 | ovs_mutex_lock(&c->mutex); | |
1087 | n = c->n_packets; | |
1088 | ovs_mutex_unlock(&c->mutex); | |
1089 | ||
1090 | return n; | |
1091 | } | |
1092 | ||
1093 | unsigned int | |
1094 | rconn_packet_counter_n_bytes(const struct rconn_packet_counter *c) | |
1095 | { | |
1096 | unsigned int n; | |
1097 | ||
1098 | ovs_mutex_lock(&c->mutex); | |
1099 | n = c->n_bytes; | |
1100 | ovs_mutex_unlock(&c->mutex); | |
1101 | ||
1102 | return n; | |
1103 | } | |
064af421 | 1104 | \f |
eb15cdbb | 1105 | /* Set rc->target and rc->name to 'target' and 'name', respectively. If 'name' |
9e7b1b11 | 1106 | * is null, 'target' is used. */ |
19d1ab55 | 1107 | static void |
eb15cdbb | 1108 | rconn_set_target__(struct rconn *rc, const char *target, const char *name) |
9f5e8906 | 1109 | OVS_REQUIRES(rc->mutex) |
19d1ab55 BP |
1110 | { |
1111 | free(rc->name); | |
eb15cdbb BP |
1112 | rc->name = xstrdup(name ? name : target); |
1113 | free(rc->target); | |
1114 | rc->target = xstrdup(target); | |
19d1ab55 BP |
1115 | } |
1116 | ||
064af421 BP |
1117 | /* Tries to send a packet from 'rc''s send buffer. Returns 0 if successful, |
1118 | * otherwise a positive errno value. */ | |
1119 | static int | |
1120 | try_send(struct rconn *rc) | |
9f5e8906 | 1121 | OVS_REQUIRES(rc->mutex) |
064af421 | 1122 | { |
b3907fbc | 1123 | struct ofpbuf *msg = ofpbuf_from_list(rc->txq.next); |
6fd6ed71 PS |
1124 | unsigned int n_bytes = msg->size; |
1125 | struct rconn_packet_counter *counter = msg->header; | |
b3907fbc BP |
1126 | int retval; |
1127 | ||
1128 | /* Eagerly remove 'msg' from the txq. We can't remove it from the list | |
1129 | * after sending, if sending is successful, because it is then owned by the | |
1130 | * vconn, which might have freed it already. */ | |
417e7e66 | 1131 | ovs_list_remove(&msg->list_node); |
6fd6ed71 | 1132 | msg->header = NULL; |
b3907fbc BP |
1133 | |
1134 | retval = vconn_send(rc->vconn, msg); | |
064af421 | 1135 | if (retval) { |
6fd6ed71 | 1136 | msg->header = counter; |
417e7e66 | 1137 | ovs_list_push_front(&rc->txq, &msg->list_node); |
064af421 | 1138 | if (retval != EAGAIN) { |
b97ba90b | 1139 | report_error(rc, retval); |
064af421 BP |
1140 | disconnect(rc, retval); |
1141 | } | |
1142 | return retval; | |
1143 | } | |
1144 | COVERAGE_INC(rconn_sent); | |
064af421 | 1145 | if (counter) { |
a6441685 | 1146 | rconn_packet_counter_dec(counter, n_bytes); |
064af421 | 1147 | } |
064af421 BP |
1148 | return 0; |
1149 | } | |
1150 | ||
b97ba90b BP |
1151 | /* Reports that 'error' caused 'rc' to disconnect. 'error' may be a positive |
1152 | * errno value, or it may be EOF to indicate that the connection was closed | |
1153 | * normally. */ | |
064af421 | 1154 | static void |
b97ba90b | 1155 | report_error(struct rconn *rc, int error) |
9f5e8906 | 1156 | OVS_REQUIRES(rc->mutex) |
b97ba90b | 1157 | { |
055b1668 GS |
1158 | /* On Windows, when a peer terminates without calling a closesocket() |
1159 | * on socket fd, we get WSAECONNRESET. Don't print warning messages | |
1160 | * for that case. */ | |
1161 | if (error == EOF | |
1162 | #ifdef _WIN32 | |
1163 | || error == WSAECONNRESET | |
1164 | #endif | |
1165 | ) { | |
b97ba90b BP |
1166 | /* If 'rc' isn't reliable, then we don't really expect this connection |
1167 | * to last forever anyway (probably it's a connection that we received | |
1168 | * via accept()), so use DBG level to avoid cluttering the logs. */ | |
1169 | enum vlog_level level = rc->reliable ? VLL_INFO : VLL_DBG; | |
1170 | VLOG(level, "%s: connection closed by peer", rc->name); | |
1171 | } else { | |
10a89ef0 BP |
1172 | VLOG_WARN("%s: connection dropped (%s)", |
1173 | rc->name, ovs_strerror(error)); | |
b97ba90b BP |
1174 | } |
1175 | } | |
1176 | ||
88a20d6e BP |
1177 | /* Disconnects 'rc' and records 'error' as the error that caused 'rc''s last |
1178 | * disconnection: | |
1179 | * | |
1180 | * - 0 means that this disconnection is due to a request by 'rc''s client, | |
1181 | * not due to any kind of network error. | |
1182 | * | |
1183 | * - EOF means that the connection was closed in the normal way by the peer. | |
1184 | * | |
1185 | * - A positive integer is an errno value that represents the error. | |
1186 | */ | |
064af421 BP |
1187 | static void |
1188 | disconnect(struct rconn *rc, int error) | |
9f5e8906 | 1189 | OVS_REQUIRES(rc->mutex) |
064af421 | 1190 | { |
88a20d6e | 1191 | rc->last_error = error; |
46e2b6c8 BP |
1192 | if (rc->vconn) { |
1193 | vconn_close(rc->vconn); | |
1194 | rc->vconn = NULL; | |
1195 | } | |
064af421 BP |
1196 | if (rc->reliable) { |
1197 | time_t now = time_now(); | |
1198 | ||
1199 | if (rc->state & (S_CONNECTING | S_ACTIVE | S_IDLE)) { | |
2cdcb898 | 1200 | rc->last_disconnected = now; |
064af421 BP |
1201 | flush_queue(rc); |
1202 | } | |
1203 | ||
1204 | if (now >= rc->backoff_deadline) { | |
1205 | rc->backoff = 1; | |
07c8c80d BP |
1206 | } else if (rc->backoff < rc->max_backoff / 2) { |
1207 | rc->backoff = MAX(1, 2 * rc->backoff); | |
46816c34 | 1208 | VLOG_INFO("%s: waiting %d seconds before reconnect", |
064af421 | 1209 | rc->name, rc->backoff); |
07c8c80d BP |
1210 | } else { |
1211 | if (rconn_logging_connection_attempts__(rc)) { | |
1212 | VLOG_INFO("%s: continuing to retry connections in the " | |
1213 | "background but suppressing further logging", | |
1214 | rc->name); | |
1215 | } | |
1216 | rc->backoff = rc->max_backoff; | |
064af421 BP |
1217 | } |
1218 | rc->backoff_deadline = now + rc->backoff; | |
1219 | state_transition(rc, S_BACKOFF); | |
064af421 | 1220 | } else { |
2cdcb898 | 1221 | rc->last_disconnected = time_now(); |
46e2b6c8 | 1222 | state_transition(rc, S_DISCONNECTED); |
064af421 BP |
1223 | } |
1224 | } | |
1225 | ||
1226 | /* Drops all the packets from 'rc''s send queue and decrements their queue | |
1227 | * counts. */ | |
1228 | static void | |
1229 | flush_queue(struct rconn *rc) | |
9f5e8906 | 1230 | OVS_REQUIRES(rc->mutex) |
064af421 | 1231 | { |
417e7e66 | 1232 | if (ovs_list_is_empty(&rc->txq)) { |
064af421 BP |
1233 | return; |
1234 | } | |
417e7e66 BW |
1235 | while (!ovs_list_is_empty(&rc->txq)) { |
1236 | struct ofpbuf *b = ofpbuf_from_list(ovs_list_pop_front(&rc->txq)); | |
6fd6ed71 | 1237 | struct rconn_packet_counter *counter = b->header; |
064af421 | 1238 | if (counter) { |
6fd6ed71 | 1239 | rconn_packet_counter_dec(counter, b->size); |
064af421 BP |
1240 | } |
1241 | COVERAGE_INC(rconn_discarded); | |
1242 | ofpbuf_delete(b); | |
1243 | } | |
1244 | poll_immediate_wake(); | |
1245 | } | |
1246 | ||
1247 | static unsigned int | |
1248 | elapsed_in_this_state(const struct rconn *rc) | |
9f5e8906 | 1249 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
1250 | { |
1251 | return time_now() - rc->state_entered; | |
1252 | } | |
1253 | ||
1254 | static unsigned int | |
1255 | timeout(const struct rconn *rc) | |
9f5e8906 | 1256 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
1257 | { |
1258 | switch (rc->state) { | |
1259 | #define STATE(NAME, VALUE) case S_##NAME: return timeout_##NAME(rc); | |
1260 | STATES | |
1261 | #undef STATE | |
1262 | default: | |
428b2edd | 1263 | OVS_NOT_REACHED(); |
064af421 BP |
1264 | } |
1265 | } | |
1266 | ||
1267 | static bool | |
1268 | timed_out(const struct rconn *rc) | |
9f5e8906 | 1269 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
1270 | { |
1271 | return time_now() >= sat_add(rc->state_entered, timeout(rc)); | |
1272 | } | |
1273 | ||
1274 | static void | |
1275 | state_transition(struct rconn *rc, enum state state) | |
9f5e8906 | 1276 | OVS_REQUIRES(rc->mutex) |
064af421 | 1277 | { |
8f2bc8d7 | 1278 | rc->seqno += is_connected_state(rc->state) != is_connected_state(state); |
064af421 BP |
1279 | if (is_connected_state(state) && !is_connected_state(rc->state)) { |
1280 | rc->probably_admitted = false; | |
1281 | } | |
1282 | if (rconn_is_connected(rc)) { | |
1283 | rc->total_time_connected += elapsed_in_this_state(rc); | |
1284 | } | |
1285 | VLOG_DBG("%s: entering %s", rc->name, state_name(state)); | |
1286 | rc->state = state; | |
1287 | rc->state_entered = time_now(); | |
1288 | } | |
1289 | ||
5ac7c4dc BP |
1290 | static void |
1291 | close_monitor(struct rconn *rc, size_t idx, int retval) | |
9f5e8906 | 1292 | OVS_REQUIRES(rc->mutex) |
5ac7c4dc BP |
1293 | { |
1294 | VLOG_DBG("%s: closing monitor connection to %s: %s", | |
1295 | rconn_get_name(rc), vconn_get_name(rc->monitors[idx]), | |
1296 | ovs_retval_to_string(retval)); | |
1297 | rc->monitors[idx] = rc->monitors[--rc->n_monitors]; | |
1298 | } | |
1299 | ||
064af421 BP |
1300 | static void |
1301 | copy_to_monitor(struct rconn *rc, const struct ofpbuf *b) | |
9f5e8906 | 1302 | OVS_REQUIRES(rc->mutex) |
064af421 BP |
1303 | { |
1304 | struct ofpbuf *clone = NULL; | |
1305 | int retval; | |
1306 | size_t i; | |
1307 | ||
1308 | for (i = 0; i < rc->n_monitors; ) { | |
1309 | struct vconn *vconn = rc->monitors[i]; | |
1310 | ||
1311 | if (!clone) { | |
1312 | clone = ofpbuf_clone(b); | |
1313 | } | |
1314 | retval = vconn_send(vconn, clone); | |
1315 | if (!retval) { | |
1316 | clone = NULL; | |
1317 | } else if (retval != EAGAIN) { | |
5ac7c4dc | 1318 | close_monitor(rc, i, retval); |
064af421 BP |
1319 | continue; |
1320 | } | |
1321 | i++; | |
1322 | } | |
1323 | ofpbuf_delete(clone); | |
1324 | } | |
1325 | ||
1326 | static bool | |
d295e8e9 | 1327 | is_connected_state(enum state state) |
064af421 BP |
1328 | { |
1329 | return (state & (S_ACTIVE | S_IDLE)) != 0; | |
1330 | } | |
1331 | ||
81196254 BP |
1332 | /* When a switch initially connects to a controller, the controller may spend a |
1333 | * little time examining the switch, looking at, for example, its datapath ID, | |
1334 | * before it decides whether it is willing to control that switch. At that | |
1335 | * point, it either disconnects or starts controlling the switch. | |
1336 | * | |
1337 | * This function returns a guess to its caller about whether 'b' is OpenFlow | |
1338 | * message that indicates that the controller has decided to control the | |
1339 | * switch. It returns false if the message is one that a controller typically | |
1340 | * uses to determine whether a switch is admissible, true if the message is one | |
1341 | * that would typically be used only after the controller has admitted the | |
1342 | * switch. */ | |
064af421 BP |
1343 | static bool |
1344 | is_admitted_msg(const struct ofpbuf *b) | |
1345 | { | |
982697a4 BP |
1346 | enum ofptype type; |
1347 | enum ofperr error; | |
1348 | ||
6fd6ed71 | 1349 | error = ofptype_decode(&type, b->data); |
982697a4 BP |
1350 | if (error) { |
1351 | return false; | |
1352 | } | |
1353 | ||
1354 | switch (type) { | |
1355 | case OFPTYPE_HELLO: | |
1356 | case OFPTYPE_ERROR: | |
1357 | case OFPTYPE_ECHO_REQUEST: | |
1358 | case OFPTYPE_ECHO_REPLY: | |
1359 | case OFPTYPE_FEATURES_REQUEST: | |
1360 | case OFPTYPE_FEATURES_REPLY: | |
1361 | case OFPTYPE_GET_CONFIG_REQUEST: | |
1362 | case OFPTYPE_GET_CONFIG_REPLY: | |
1363 | case OFPTYPE_SET_CONFIG: | |
c545d38d JR |
1364 | case OFPTYPE_QUEUE_GET_CONFIG_REQUEST: |
1365 | case OFPTYPE_QUEUE_GET_CONFIG_REPLY: | |
2e1ae200 JR |
1366 | case OFPTYPE_GET_ASYNC_REQUEST: |
1367 | case OFPTYPE_GET_ASYNC_REPLY: | |
261bd854 BP |
1368 | case OFPTYPE_GROUP_STATS_REQUEST: |
1369 | case OFPTYPE_GROUP_STATS_REPLY: | |
1370 | case OFPTYPE_GROUP_DESC_STATS_REQUEST: | |
1371 | case OFPTYPE_GROUP_DESC_STATS_REPLY: | |
1372 | case OFPTYPE_GROUP_FEATURES_STATS_REQUEST: | |
1373 | case OFPTYPE_GROUP_FEATURES_STATS_REPLY: | |
1374 | case OFPTYPE_TABLE_FEATURES_STATS_REQUEST: | |
1375 | case OFPTYPE_TABLE_FEATURES_STATS_REPLY: | |
03c72922 BP |
1376 | case OFPTYPE_TABLE_DESC_REQUEST: |
1377 | case OFPTYPE_TABLE_DESC_REPLY: | |
982697a4 BP |
1378 | return false; |
1379 | ||
1380 | case OFPTYPE_PACKET_IN: | |
1381 | case OFPTYPE_FLOW_REMOVED: | |
1382 | case OFPTYPE_PORT_STATUS: | |
1383 | case OFPTYPE_PACKET_OUT: | |
1384 | case OFPTYPE_FLOW_MOD: | |
7395c052 | 1385 | case OFPTYPE_GROUP_MOD: |
982697a4 | 1386 | case OFPTYPE_PORT_MOD: |
918f2b82 | 1387 | case OFPTYPE_TABLE_MOD: |
9cae45dc | 1388 | case OFPTYPE_METER_MOD: |
982697a4 BP |
1389 | case OFPTYPE_BARRIER_REQUEST: |
1390 | case OFPTYPE_BARRIER_REPLY: | |
1391 | case OFPTYPE_DESC_STATS_REQUEST: | |
1392 | case OFPTYPE_DESC_STATS_REPLY: | |
1393 | case OFPTYPE_FLOW_STATS_REQUEST: | |
1394 | case OFPTYPE_FLOW_STATS_REPLY: | |
1395 | case OFPTYPE_AGGREGATE_STATS_REQUEST: | |
1396 | case OFPTYPE_AGGREGATE_STATS_REPLY: | |
1397 | case OFPTYPE_TABLE_STATS_REQUEST: | |
1398 | case OFPTYPE_TABLE_STATS_REPLY: | |
1399 | case OFPTYPE_PORT_STATS_REQUEST: | |
1400 | case OFPTYPE_PORT_STATS_REPLY: | |
1401 | case OFPTYPE_QUEUE_STATS_REQUEST: | |
1402 | case OFPTYPE_QUEUE_STATS_REPLY: | |
1403 | case OFPTYPE_PORT_DESC_STATS_REQUEST: | |
1404 | case OFPTYPE_PORT_DESC_STATS_REPLY: | |
261bd854 BP |
1405 | case OFPTYPE_METER_STATS_REQUEST: |
1406 | case OFPTYPE_METER_STATS_REPLY: | |
1407 | case OFPTYPE_METER_CONFIG_STATS_REQUEST: | |
1408 | case OFPTYPE_METER_CONFIG_STATS_REPLY: | |
1409 | case OFPTYPE_METER_FEATURES_STATS_REQUEST: | |
1410 | case OFPTYPE_METER_FEATURES_STATS_REPLY: | |
982697a4 BP |
1411 | case OFPTYPE_ROLE_REQUEST: |
1412 | case OFPTYPE_ROLE_REPLY: | |
252f3411 | 1413 | case OFPTYPE_ROLE_STATUS: |
3c35db62 | 1414 | case OFPTYPE_REQUESTFORWARD: |
6c6eedc5 | 1415 | case OFPTYPE_TABLE_STATUS: |
982697a4 BP |
1416 | case OFPTYPE_SET_FLOW_FORMAT: |
1417 | case OFPTYPE_FLOW_MOD_TABLE_ID: | |
1418 | case OFPTYPE_SET_PACKET_IN_FORMAT: | |
1419 | case OFPTYPE_FLOW_AGE: | |
1420 | case OFPTYPE_SET_ASYNC_CONFIG: | |
1421 | case OFPTYPE_SET_CONTROLLER_ID: | |
1422 | case OFPTYPE_FLOW_MONITOR_STATS_REQUEST: | |
1423 | case OFPTYPE_FLOW_MONITOR_STATS_REPLY: | |
1424 | case OFPTYPE_FLOW_MONITOR_CANCEL: | |
1425 | case OFPTYPE_FLOW_MONITOR_PAUSED: | |
1426 | case OFPTYPE_FLOW_MONITOR_RESUMED: | |
b58990a6 JR |
1427 | case OFPTYPE_BUNDLE_CONTROL: |
1428 | case OFPTYPE_BUNDLE_ADD_MESSAGE: | |
4e548ad9 ML |
1429 | case OFPTYPE_NXT_TLV_TABLE_MOD: |
1430 | case OFPTYPE_NXT_TLV_TABLE_REQUEST: | |
1431 | case OFPTYPE_NXT_TLV_TABLE_REPLY: | |
77ab5fd2 | 1432 | case OFPTYPE_NXT_RESUME: |
fb8f22c1 BY |
1433 | case OFPTYPE_IPFIX_BRIDGE_STATS_REQUEST: |
1434 | case OFPTYPE_IPFIX_BRIDGE_STATS_REPLY: | |
1435 | case OFPTYPE_IPFIX_FLOW_STATS_REQUEST: | |
1436 | case OFPTYPE_IPFIX_FLOW_STATS_REPLY: | |
2a7c4805 | 1437 | case OFPTYPE_CT_FLUSH_ZONE: |
982697a4 BP |
1438 | default: |
1439 | return true; | |
1440 | } | |
064af421 | 1441 | } |
07c8c80d BP |
1442 | |
1443 | /* Returns true if 'rc' is currently logging information about connection | |
1444 | * attempts, false if logging should be suppressed because 'rc' hasn't | |
1445 | * successuflly connected in too long. */ | |
1446 | static bool | |
1447 | rconn_logging_connection_attempts__(const struct rconn *rc) | |
9f5e8906 | 1448 | OVS_REQUIRES(rc->mutex) |
07c8c80d BP |
1449 | { |
1450 | return rc->backoff < rc->max_backoff; | |
1451 | } |