]>
Commit | Line | Data |
---|---|---|
73e141f9 | 1 | /* Copyright (c) 2009, 2010, 2011, 2012, 2013, 2014 Nicira, Inc. |
e1ec7dd4 EJ |
2 | * |
3 | * Licensed under the Apache License, Version 2.0 (the "License"); | |
4 | * you may not use this file except in compliance with the License. | |
5 | * You may obtain a copy of the License at: | |
6 | * | |
7 | * http://www.apache.org/licenses/LICENSE-2.0 | |
8 | * | |
9 | * Unless required by applicable law or agreed to in writing, software | |
10 | * distributed under the License is distributed on an "AS IS" BASIS, | |
11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
12 | * See the License for the specific language governing permissions and | |
13 | * limitations under the License. */ | |
14 | ||
15 | #include <config.h> | |
16 | #include "ofproto-dpif-upcall.h" | |
17 | ||
18 | #include <errno.h> | |
19 | #include <stdbool.h> | |
20 | #include <inttypes.h> | |
21 | ||
0fb7792a | 22 | #include "connmgr.h" |
e1ec7dd4 | 23 | #include "coverage.h" |
9fce0584 | 24 | #include "cmap.h" |
e1ec7dd4 | 25 | #include "dpif.h" |
e22d52ee | 26 | #include "dynamic-string.h" |
e1ec7dd4 | 27 | #include "fail-open.h" |
05067881 | 28 | #include "guarded-list.h" |
e1ec7dd4 | 29 | #include "latch.h" |
e1ec7dd4 EJ |
30 | #include "list.h" |
31 | #include "netlink.h" | |
32 | #include "ofpbuf.h" | |
10e57640 EJ |
33 | #include "ofproto-dpif-ipfix.h" |
34 | #include "ofproto-dpif-sflow.h" | |
e79a6c83 | 35 | #include "ofproto-dpif-xlate.h" |
0f2ea848 | 36 | #include "ovs-rcu.h" |
e1ec7dd4 EJ |
37 | #include "packets.h" |
38 | #include "poll-loop.h" | |
e22d52ee EJ |
39 | #include "seq.h" |
40 | #include "unixctl.h" | |
e1ec7dd4 EJ |
41 | #include "vlog.h" |
42 | ||
43 | #define MAX_QUEUE_LENGTH 512 | |
6b31e073 | 44 | #define UPCALL_MAX_BATCH 64 |
e79a6c83 | 45 | #define REVALIDATE_MAX_BATCH 50 |
e1ec7dd4 EJ |
46 | |
47 | VLOG_DEFINE_THIS_MODULE(ofproto_dpif_upcall); | |
48 | ||
ec47af51 JS |
49 | COVERAGE_DEFINE(dumped_duplicate_flow); |
50 | COVERAGE_DEFINE(dumped_new_flow); | |
3b62a9d3 | 51 | COVERAGE_DEFINE(revalidate_missed_dp_flow); |
73a3c475 | 52 | |
9a159f74 AW |
53 | /* A thread that reads upcalls from dpif, forwards each upcall's packet, |
54 | * and possibly sets up a kernel flow as a cache. */ | |
e1ec7dd4 EJ |
55 | struct handler { |
56 | struct udpif *udpif; /* Parent udpif. */ | |
57 | pthread_t thread; /* Thread ID. */ | |
9a159f74 | 58 | uint32_t handler_id; /* Handler id. */ |
e1ec7dd4 EJ |
59 | }; |
60 | ||
b8d3daeb JS |
61 | /* In the absence of a multiple-writer multiple-reader datastructure for |
62 | * storing ukeys, we use a large number of cmaps, each with its own lock for | |
63 | * writing. */ | |
64 | #define N_UMAPS 512 /* per udpif. */ | |
65 | struct umap { | |
66 | struct ovs_mutex mutex; /* Take for writing to the following. */ | |
67 | struct cmap cmap; /* Datapath flow keys. */ | |
68 | }; | |
69 | ||
7d170098 EJ |
70 | /* A thread that processes datapath flows, updates OpenFlow statistics, and |
71 | * updates or removes them if necessary. */ | |
e79a6c83 EJ |
72 | struct revalidator { |
73 | struct udpif *udpif; /* Parent udpif. */ | |
e79a6c83 | 74 | pthread_t thread; /* Thread ID. */ |
8ba0a522 | 75 | unsigned int id; /* ovsthread_id_self(). */ |
e79a6c83 EJ |
76 | }; |
77 | ||
e1ec7dd4 EJ |
78 | /* An upcall handler for ofproto_dpif. |
79 | * | |
9a159f74 AW |
80 | * udpif keeps records of two kind of logically separate units: |
81 | * | |
82 | * upcall handling | |
83 | * --------------- | |
84 | * | |
85 | * - An array of 'struct handler's for upcall handling and flow | |
86 | * installation. | |
e79a6c83 | 87 | * |
9a159f74 AW |
88 | * flow revalidation |
89 | * ----------------- | |
90 | * | |
7d170098 EJ |
91 | * - Revalidation threads which read the datapath flow table and maintains |
92 | * them. | |
93 | */ | |
e1ec7dd4 | 94 | struct udpif { |
e22d52ee EJ |
95 | struct list list_node; /* In all_udpifs list. */ |
96 | ||
e1ec7dd4 EJ |
97 | struct dpif *dpif; /* Datapath handle. */ |
98 | struct dpif_backer *backer; /* Opaque dpif_backer pointer. */ | |
99 | ||
100 | uint32_t secret; /* Random seed for upcall hash. */ | |
101 | ||
10e57640 | 102 | struct handler *handlers; /* Upcall handlers. */ |
e1ec7dd4 EJ |
103 | size_t n_handlers; |
104 | ||
e79a6c83 EJ |
105 | struct revalidator *revalidators; /* Flow revalidators. */ |
106 | size_t n_revalidators; | |
107 | ||
e79a6c83 EJ |
108 | struct latch exit_latch; /* Tells child threads to exit. */ |
109 | ||
7d170098 EJ |
110 | /* Revalidation. */ |
111 | struct seq *reval_seq; /* Incremented to force revalidation. */ | |
112 | bool need_revalidate; /* As indicated by 'reval_seq'. */ | |
113 | bool reval_exit; /* Set by leader on 'exit_latch. */ | |
d8043da7 | 114 | struct ovs_barrier reval_barrier; /* Barrier used by revalidators. */ |
ac64794a | 115 | struct dpif_flow_dump *dump; /* DPIF flow dump state. */ |
e79a6c83 | 116 | long long int dump_duration; /* Duration of the last flow dump. */ |
7d170098 EJ |
117 | struct seq *dump_seq; /* Increments each dump iteration. */ |
118 | ||
b8d3daeb | 119 | /* There are 'N_UMAPS' maps containing 'struct udpif_key' elements. |
7d170098 EJ |
120 | * |
121 | * During the flow dump phase, revalidators insert into these with a random | |
122 | * distribution. During the garbage collection phase, each revalidator | |
b8d3daeb JS |
123 | * takes care of garbage collecting a slice of these maps. */ |
124 | struct umap *ukeys; | |
e1ec7dd4 | 125 | |
e79a6c83 EJ |
126 | /* Datapath flow statistics. */ |
127 | unsigned int max_n_flows; | |
128 | unsigned int avg_n_flows; | |
e1ec7dd4 | 129 | |
e79a6c83 | 130 | /* Following fields are accessed and modified by different threads. */ |
e79a6c83 | 131 | atomic_uint flow_limit; /* Datapath flow hard limit. */ |
64ca9472 JS |
132 | |
133 | /* n_flows_mutex prevents multiple threads updating these concurrently. */ | |
b482e960 | 134 | atomic_uint n_flows; /* Number of flows in the datapath. */ |
64ca9472 JS |
135 | atomic_llong n_flows_timestamp; /* Last time n_flows was updated. */ |
136 | struct ovs_mutex n_flows_mutex; | |
27f57736 JS |
137 | |
138 | /* Following fields are accessed and modified only from the main thread. */ | |
139 | struct unixctl_conn **conns; /* Connections waiting on dump_seq. */ | |
140 | uint64_t conn_seq; /* Corresponds to 'dump_seq' when | |
141 | conns[n_conns-1] was stored. */ | |
142 | size_t n_conns; /* Number of connections waiting. */ | |
e1ec7dd4 EJ |
143 | }; |
144 | ||
10e57640 EJ |
145 | enum upcall_type { |
146 | BAD_UPCALL, /* Some kind of bug somewhere. */ | |
147 | MISS_UPCALL, /* A flow miss. */ | |
148 | SFLOW_UPCALL, /* sFlow sample. */ | |
149 | FLOW_SAMPLE_UPCALL, /* Per-flow sampling. */ | |
150 | IPFIX_UPCALL /* Per-bridge sampling. */ | |
151 | }; | |
152 | ||
153 | struct upcall { | |
cc377352 | 154 | struct ofproto_dpif *ofproto; /* Parent ofproto. */ |
a0bab870 | 155 | |
cc377352 EJ |
156 | /* The flow and packet are only required to be constant when using |
157 | * dpif-netdev. If a modification is absolutely necessary, a const cast | |
158 | * may be used with other datapaths. */ | |
159 | const struct flow *flow; /* Parsed representation of the packet. */ | |
160 | const struct ofpbuf *packet; /* Packet associated with this upcall. */ | |
161 | ofp_port_t in_port; /* OpenFlow in port, or OFPP_NONE. */ | |
a0bab870 | 162 | |
cc377352 EJ |
163 | enum dpif_upcall_type type; /* Datapath type of the upcall. */ |
164 | const struct nlattr *userdata; /* Userdata for DPIF_UC_ACTION Upcalls. */ | |
165 | ||
166 | bool xout_initialized; /* True if 'xout' must be uninitialized. */ | |
167 | struct xlate_out xout; /* Result of xlate_actions(). */ | |
168 | struct ofpbuf put_actions; /* Actions 'put' in the fastapath. */ | |
169 | ||
dcc2c6cd JR |
170 | struct dpif_ipfix *ipfix; /* IPFIX pointer or NULL. */ |
171 | struct dpif_sflow *sflow; /* SFlow pointer or NULL. */ | |
a0bab870 | 172 | |
cc377352 EJ |
173 | bool vsp_adjusted; /* 'packet' and 'flow' were adjusted for |
174 | VLAN splinters if true. */ | |
10e57640 | 175 | |
cc377352 EJ |
176 | /* Not used by the upcall callback interface. */ |
177 | const struct nlattr *key; /* Datapath flow key. */ | |
178 | size_t key_len; /* Datapath flow key length. */ | |
8b7ea2d4 | 179 | const struct nlattr *out_tun_key; /* Datapath output tunnel key. */ |
10e57640 EJ |
180 | }; |
181 | ||
e79a6c83 EJ |
182 | /* 'udpif_key's are responsible for tracking the little bit of state udpif |
183 | * needs to do flow expiration which can't be pulled directly from the | |
7d170098 EJ |
184 | * datapath. They may be created or maintained by any revalidator during |
185 | * the dump phase, but are owned by a single revalidator, and are destroyed | |
186 | * by that revalidator during the garbage-collection phase. | |
187 | * | |
b8d3daeb JS |
188 | * The mutex within the ukey protects some members of the ukey. The ukey |
189 | * itself is protected by RCU and is held within a umap in the parent udpif. | |
190 | * Adding or removing a ukey from a umap is only safe when holding the | |
191 | * corresponding umap lock. */ | |
e79a6c83 | 192 | struct udpif_key { |
9fce0584 | 193 | struct cmap_node cmap_node; /* In parent revalidator 'ukeys' map. */ |
e79a6c83 | 194 | |
7d170098 EJ |
195 | /* These elements are read only once created, and therefore aren't |
196 | * protected by a mutex. */ | |
197 | const struct nlattr *key; /* Datapath flow key. */ | |
e79a6c83 | 198 | size_t key_len; /* Length of 'key'. */ |
9fce0584 | 199 | uint32_t hash; /* Pre-computed hash for 'key'. */ |
e79a6c83 | 200 | |
7d170098 EJ |
201 | struct ovs_mutex mutex; /* Guards the following. */ |
202 | struct dpif_flow_stats stats OVS_GUARDED; /* Last known stats.*/ | |
203 | long long int created OVS_GUARDED; /* Estimate of creation time. */ | |
efa08531 | 204 | uint64_t dump_seq OVS_GUARDED; /* Tracks udpif->dump_seq. */ |
7d170098 EJ |
205 | bool flow_exists OVS_GUARDED; /* Ensures flows are only deleted |
206 | once. */ | |
207 | ||
208 | struct xlate_cache *xcache OVS_GUARDED; /* Cache for xlate entries that | |
209 | * are affected by this ukey. | |
210 | * Used for stats and learning.*/ | |
02334943 JR |
211 | union { |
212 | struct odputil_keybuf key_buf; /* Memory for 'key'. */ | |
213 | struct nlattr key_buf_nla; | |
214 | }; | |
e79a6c83 EJ |
215 | }; |
216 | ||
6dad4d44 JS |
217 | /* Datapath operation with optional ukey attached. */ |
218 | struct ukey_op { | |
219 | struct udpif_key *ukey; | |
220 | struct dpif_flow_stats stats; /* Stats for 'op'. */ | |
221 | struct dpif_op dop; /* Flow operation. */ | |
222 | }; | |
223 | ||
e1ec7dd4 | 224 | static struct vlog_rate_limit rl = VLOG_RATE_LIMIT_INIT(1, 5); |
e22d52ee | 225 | static struct list all_udpifs = LIST_INITIALIZER(&all_udpifs); |
e1ec7dd4 | 226 | |
cc377352 EJ |
227 | static size_t recv_upcalls(struct handler *); |
228 | static int process_upcall(struct udpif *, struct upcall *, | |
229 | struct ofpbuf *odp_actions); | |
6b31e073 | 230 | static void handle_upcalls(struct udpif *, struct upcall *, size_t n_upcalls); |
1f867548 AW |
231 | static void udpif_stop_threads(struct udpif *); |
232 | static void udpif_start_threads(struct udpif *, size_t n_handlers, | |
233 | size_t n_revalidators); | |
10e57640 | 234 | static void *udpif_upcall_handler(void *); |
e79a6c83 | 235 | static void *udpif_revalidator(void *); |
0e2a9f6f | 236 | static unsigned long udpif_get_n_flows(struct udpif *); |
7d170098 | 237 | static void revalidate(struct revalidator *); |
e79a6c83 | 238 | static void revalidator_sweep(struct revalidator *); |
e96a5c24 | 239 | static void revalidator_purge(struct revalidator *); |
e22d52ee EJ |
240 | static void upcall_unixctl_show(struct unixctl_conn *conn, int argc, |
241 | const char *argv[], void *aux); | |
e79a6c83 EJ |
242 | static void upcall_unixctl_disable_megaflows(struct unixctl_conn *, int argc, |
243 | const char *argv[], void *aux); | |
244 | static void upcall_unixctl_enable_megaflows(struct unixctl_conn *, int argc, | |
245 | const char *argv[], void *aux); | |
94b8c324 JS |
246 | static void upcall_unixctl_set_flow_limit(struct unixctl_conn *conn, int argc, |
247 | const char *argv[], void *aux); | |
27f57736 JS |
248 | static void upcall_unixctl_dump_wait(struct unixctl_conn *conn, int argc, |
249 | const char *argv[], void *aux); | |
98bb4286 JS |
250 | static void upcall_unixctl_purge(struct unixctl_conn *conn, int argc, |
251 | const char *argv[], void *aux); | |
7d170098 EJ |
252 | |
253 | static struct udpif_key *ukey_create(const struct nlattr *key, size_t key_len, | |
9fce0584 | 254 | long long int used, uint32_t hash); |
feca8bd7 JS |
255 | static struct udpif_key *ukey_lookup(struct udpif *udpif, |
256 | const struct nlattr *key, size_t key_len, | |
257 | uint32_t hash); | |
258 | static bool ukey_acquire(struct udpif *udpif, const struct nlattr *key, | |
259 | size_t key_len, long long int used, | |
260 | struct udpif_key **result); | |
9fce0584 | 261 | static void ukey_delete__(struct udpif_key *); |
b8d3daeb | 262 | static void ukey_delete(struct umap *, struct udpif_key *); |
cc377352 EJ |
263 | static enum upcall_type classify_upcall(enum dpif_upcall_type type, |
264 | const struct nlattr *userdata); | |
265 | ||
cc377352 EJ |
266 | static int upcall_receive(struct upcall *, const struct dpif_backer *, |
267 | const struct ofpbuf *packet, enum dpif_upcall_type, | |
268 | const struct nlattr *userdata, const struct flow *); | |
269 | static void upcall_uninit(struct upcall *); | |
e79a6c83 | 270 | |
623540e4 EJ |
271 | static upcall_callback upcall_cb; |
272 | ||
e79a6c83 | 273 | static atomic_bool enable_megaflows = ATOMIC_VAR_INIT(true); |
e1ec7dd4 EJ |
274 | |
275 | struct udpif * | |
276 | udpif_create(struct dpif_backer *backer, struct dpif *dpif) | |
277 | { | |
e22d52ee | 278 | static struct ovsthread_once once = OVSTHREAD_ONCE_INITIALIZER; |
e1ec7dd4 EJ |
279 | struct udpif *udpif = xzalloc(sizeof *udpif); |
280 | ||
e22d52ee EJ |
281 | if (ovsthread_once_start(&once)) { |
282 | unixctl_command_register("upcall/show", "", 0, 0, upcall_unixctl_show, | |
283 | NULL); | |
e79a6c83 EJ |
284 | unixctl_command_register("upcall/disable-megaflows", "", 0, 0, |
285 | upcall_unixctl_disable_megaflows, NULL); | |
286 | unixctl_command_register("upcall/enable-megaflows", "", 0, 0, | |
287 | upcall_unixctl_enable_megaflows, NULL); | |
94b8c324 JS |
288 | unixctl_command_register("upcall/set-flow-limit", "", 1, 1, |
289 | upcall_unixctl_set_flow_limit, NULL); | |
27f57736 JS |
290 | unixctl_command_register("revalidator/wait", "", 0, 0, |
291 | upcall_unixctl_dump_wait, NULL); | |
98bb4286 JS |
292 | unixctl_command_register("revalidator/purge", "", 0, 0, |
293 | upcall_unixctl_purge, NULL); | |
e22d52ee EJ |
294 | ovsthread_once_done(&once); |
295 | } | |
296 | ||
e1ec7dd4 EJ |
297 | udpif->dpif = dpif; |
298 | udpif->backer = backer; | |
e79a6c83 | 299 | atomic_init(&udpif->flow_limit, MIN(ofproto_flow_limit, 10000)); |
e1ec7dd4 | 300 | udpif->secret = random_uint32(); |
d7285d74 | 301 | udpif->reval_seq = seq_create(); |
e79a6c83 | 302 | udpif->dump_seq = seq_create(); |
e1ec7dd4 | 303 | latch_init(&udpif->exit_latch); |
e22d52ee | 304 | list_push_back(&all_udpifs, &udpif->list_node); |
64ca9472 JS |
305 | atomic_init(&udpif->n_flows, 0); |
306 | atomic_init(&udpif->n_flows_timestamp, LLONG_MIN); | |
307 | ovs_mutex_init(&udpif->n_flows_mutex); | |
b8d3daeb JS |
308 | udpif->ukeys = xmalloc(N_UMAPS * sizeof *udpif->ukeys); |
309 | for (int i = 0; i < N_UMAPS; i++) { | |
310 | cmap_init(&udpif->ukeys[i].cmap); | |
311 | ovs_mutex_init(&udpif->ukeys[i].mutex); | |
312 | } | |
e1ec7dd4 | 313 | |
623540e4 | 314 | dpif_register_upcall_cb(dpif, upcall_cb, udpif); |
6b31e073 | 315 | |
e1ec7dd4 EJ |
316 | return udpif; |
317 | } | |
318 | ||
27f57736 JS |
319 | void |
320 | udpif_run(struct udpif *udpif) | |
321 | { | |
322 | if (udpif->conns && udpif->conn_seq != seq_read(udpif->dump_seq)) { | |
323 | int i; | |
324 | ||
325 | for (i = 0; i < udpif->n_conns; i++) { | |
326 | unixctl_command_reply(udpif->conns[i], NULL); | |
327 | } | |
328 | free(udpif->conns); | |
329 | udpif->conns = NULL; | |
330 | udpif->n_conns = 0; | |
331 | } | |
332 | } | |
333 | ||
e1ec7dd4 EJ |
334 | void |
335 | udpif_destroy(struct udpif *udpif) | |
336 | { | |
1f867548 | 337 | udpif_stop_threads(udpif); |
e1ec7dd4 | 338 | |
b8d3daeb JS |
339 | for (int i = 0; i < N_UMAPS; i++) { |
340 | cmap_destroy(&udpif->ukeys[i].cmap); | |
341 | ovs_mutex_destroy(&udpif->ukeys[i].mutex); | |
342 | } | |
343 | free(udpif->ukeys); | |
344 | udpif->ukeys = NULL; | |
345 | ||
e22d52ee | 346 | list_remove(&udpif->list_node); |
e1ec7dd4 | 347 | latch_destroy(&udpif->exit_latch); |
d7285d74 | 348 | seq_destroy(udpif->reval_seq); |
e79a6c83 | 349 | seq_destroy(udpif->dump_seq); |
64ca9472 | 350 | ovs_mutex_destroy(&udpif->n_flows_mutex); |
e1ec7dd4 EJ |
351 | free(udpif); |
352 | } | |
353 | ||
1f867548 AW |
354 | /* Stops the handler and revalidator threads, must be enclosed in |
355 | * ovsrcu quiescent state unless when destroying udpif. */ | |
356 | static void | |
357 | udpif_stop_threads(struct udpif *udpif) | |
e1ec7dd4 | 358 | { |
3aadc5bb | 359 | if (udpif && (udpif->n_handlers != 0 || udpif->n_revalidators != 0)) { |
e1ec7dd4 EJ |
360 | size_t i; |
361 | ||
362 | latch_set(&udpif->exit_latch); | |
363 | ||
e1ec7dd4 EJ |
364 | for (i = 0; i < udpif->n_handlers; i++) { |
365 | struct handler *handler = &udpif->handlers[i]; | |
366 | ||
e79a6c83 EJ |
367 | xpthread_join(handler->thread, NULL); |
368 | } | |
369 | ||
370 | for (i = 0; i < udpif->n_revalidators; i++) { | |
7d170098 | 371 | xpthread_join(udpif->revalidators[i].thread, NULL); |
e1ec7dd4 EJ |
372 | } |
373 | ||
6b31e073 RW |
374 | dpif_disable_upcall(udpif->dpif); |
375 | ||
e79a6c83 EJ |
376 | for (i = 0; i < udpif->n_revalidators; i++) { |
377 | struct revalidator *revalidator = &udpif->revalidators[i]; | |
e79a6c83 | 378 | |
e96a5c24 JS |
379 | /* Delete ukeys, and delete all flows from the datapath to prevent |
380 | * double-counting stats. */ | |
381 | revalidator_purge(revalidator); | |
e79a6c83 EJ |
382 | } |
383 | ||
e1ec7dd4 EJ |
384 | latch_poll(&udpif->exit_latch); |
385 | ||
d8043da7 | 386 | ovs_barrier_destroy(&udpif->reval_barrier); |
7d170098 | 387 | |
e79a6c83 EJ |
388 | free(udpif->revalidators); |
389 | udpif->revalidators = NULL; | |
390 | udpif->n_revalidators = 0; | |
391 | ||
e1ec7dd4 EJ |
392 | free(udpif->handlers); |
393 | udpif->handlers = NULL; | |
394 | udpif->n_handlers = 0; | |
395 | } | |
1f867548 | 396 | } |
e1ec7dd4 | 397 | |
1f867548 AW |
398 | /* Starts the handler and revalidator threads, must be enclosed in |
399 | * ovsrcu quiescent state. */ | |
400 | static void | |
401 | udpif_start_threads(struct udpif *udpif, size_t n_handlers, | |
402 | size_t n_revalidators) | |
403 | { | |
6f12bda3 | 404 | if (udpif && n_handlers && n_revalidators) { |
e1ec7dd4 EJ |
405 | size_t i; |
406 | ||
407 | udpif->n_handlers = n_handlers; | |
e79a6c83 EJ |
408 | udpif->n_revalidators = n_revalidators; |
409 | ||
e1ec7dd4 EJ |
410 | udpif->handlers = xzalloc(udpif->n_handlers * sizeof *udpif->handlers); |
411 | for (i = 0; i < udpif->n_handlers; i++) { | |
412 | struct handler *handler = &udpif->handlers[i]; | |
413 | ||
414 | handler->udpif = udpif; | |
9a159f74 | 415 | handler->handler_id = i; |
8ba0a522 BP |
416 | handler->thread = ovs_thread_create( |
417 | "handler", udpif_upcall_handler, handler); | |
e1ec7dd4 | 418 | } |
e1ec7dd4 | 419 | |
6b31e073 RW |
420 | dpif_enable_upcall(udpif->dpif); |
421 | ||
d8043da7 | 422 | ovs_barrier_init(&udpif->reval_barrier, udpif->n_revalidators); |
7d170098 | 423 | udpif->reval_exit = false; |
e79a6c83 EJ |
424 | udpif->revalidators = xzalloc(udpif->n_revalidators |
425 | * sizeof *udpif->revalidators); | |
426 | for (i = 0; i < udpif->n_revalidators; i++) { | |
427 | struct revalidator *revalidator = &udpif->revalidators[i]; | |
428 | ||
429 | revalidator->udpif = udpif; | |
8ba0a522 BP |
430 | revalidator->thread = ovs_thread_create( |
431 | "revalidator", udpif_revalidator, revalidator); | |
e79a6c83 | 432 | } |
e1ec7dd4 | 433 | } |
1f867548 | 434 | } |
0f2ea848 | 435 | |
1f867548 AW |
436 | /* Tells 'udpif' how many threads it should use to handle upcalls. |
437 | * 'n_handlers' and 'n_revalidators' can never be zero. 'udpif''s | |
438 | * datapath handle must have packet reception enabled before starting | |
439 | * threads. */ | |
440 | void | |
441 | udpif_set_threads(struct udpif *udpif, size_t n_handlers, | |
442 | size_t n_revalidators) | |
443 | { | |
3aadc5bb | 444 | ovs_assert(udpif); |
1f867548 AW |
445 | ovs_assert(n_handlers && n_revalidators); |
446 | ||
447 | ovsrcu_quiesce_start(); | |
3aadc5bb AW |
448 | if (udpif->n_handlers != n_handlers |
449 | || udpif->n_revalidators != n_revalidators) { | |
450 | udpif_stop_threads(udpif); | |
451 | } | |
1f867548 | 452 | |
3aadc5bb | 453 | if (!udpif->handlers && !udpif->revalidators) { |
380fffec AW |
454 | int error; |
455 | ||
456 | error = dpif_handlers_set(udpif->dpif, n_handlers); | |
457 | if (error) { | |
458 | VLOG_ERR("failed to configure handlers in dpif %s: %s", | |
459 | dpif_name(udpif->dpif), ovs_strerror(error)); | |
460 | return; | |
461 | } | |
462 | ||
3aadc5bb AW |
463 | udpif_start_threads(udpif, n_handlers, n_revalidators); |
464 | } | |
0f2ea848 | 465 | ovsrcu_quiesce_end(); |
e1ec7dd4 EJ |
466 | } |
467 | ||
3f142f59 BP |
468 | /* Waits for all ongoing upcall translations to complete. This ensures that |
469 | * there are no transient references to any removed ofprotos (or other | |
470 | * objects). In particular, this should be called after an ofproto is removed | |
471 | * (e.g. via xlate_remove_ofproto()) but before it is destroyed. */ | |
472 | void | |
473 | udpif_synchronize(struct udpif *udpif) | |
474 | { | |
475 | /* This is stronger than necessary. It would be sufficient to ensure | |
476 | * (somehow) that each handler and revalidator thread had passed through | |
477 | * its main loop once. */ | |
478 | size_t n_handlers = udpif->n_handlers; | |
479 | size_t n_revalidators = udpif->n_revalidators; | |
1f867548 AW |
480 | |
481 | ovsrcu_quiesce_start(); | |
482 | udpif_stop_threads(udpif); | |
483 | udpif_start_threads(udpif, n_handlers, n_revalidators); | |
484 | ovsrcu_quiesce_end(); | |
3f142f59 BP |
485 | } |
486 | ||
e1ec7dd4 EJ |
487 | /* Notifies 'udpif' that something changed which may render previous |
488 | * xlate_actions() results invalid. */ | |
489 | void | |
490 | udpif_revalidate(struct udpif *udpif) | |
491 | { | |
d7285d74 | 492 | seq_change(udpif->reval_seq); |
e79a6c83 | 493 | } |
05067881 | 494 | |
e79a6c83 EJ |
495 | /* Returns a seq which increments every time 'udpif' pulls stats from the |
496 | * datapath. Callers can use this to get a sense of when might be a good time | |
497 | * to do periodic work which relies on relatively up to date statistics. */ | |
498 | struct seq * | |
499 | udpif_dump_seq(struct udpif *udpif) | |
500 | { | |
501 | return udpif->dump_seq; | |
e1ec7dd4 EJ |
502 | } |
503 | ||
1c030aa5 EJ |
504 | void |
505 | udpif_get_memory_usage(struct udpif *udpif, struct simap *usage) | |
506 | { | |
507 | size_t i; | |
508 | ||
1c030aa5 | 509 | simap_increase(usage, "handlers", udpif->n_handlers); |
e79a6c83 EJ |
510 | |
511 | simap_increase(usage, "revalidators", udpif->n_revalidators); | |
b8d3daeb | 512 | for (i = 0; i < N_UMAPS; i++) { |
9fce0584 | 513 | simap_increase(usage, "udpif keys", cmap_count(&udpif->ukeys[i].cmap)); |
e79a6c83 | 514 | } |
1c030aa5 EJ |
515 | } |
516 | ||
1b5b5071 | 517 | /* Remove flows from a single datapath. */ |
e79a6c83 | 518 | void |
1b5b5071 AZ |
519 | udpif_flush(struct udpif *udpif) |
520 | { | |
521 | size_t n_handlers, n_revalidators; | |
522 | ||
523 | n_handlers = udpif->n_handlers; | |
524 | n_revalidators = udpif->n_revalidators; | |
525 | ||
1f867548 AW |
526 | ovsrcu_quiesce_start(); |
527 | ||
528 | udpif_stop_threads(udpif); | |
1b5b5071 | 529 | dpif_flow_flush(udpif->dpif); |
1f867548 AW |
530 | udpif_start_threads(udpif, n_handlers, n_revalidators); |
531 | ||
532 | ovsrcu_quiesce_end(); | |
1b5b5071 AZ |
533 | } |
534 | ||
535 | /* Removes all flows from all datapaths. */ | |
536 | static void | |
537 | udpif_flush_all_datapaths(void) | |
e79a6c83 EJ |
538 | { |
539 | struct udpif *udpif; | |
540 | ||
541 | LIST_FOR_EACH (udpif, list_node, &all_udpifs) { | |
1b5b5071 | 542 | udpif_flush(udpif); |
e79a6c83 EJ |
543 | } |
544 | } | |
1b5b5071 | 545 | |
e79a6c83 | 546 | \f |
0e2a9f6f | 547 | static unsigned long |
64ca9472 | 548 | udpif_get_n_flows(struct udpif *udpif) |
e1ec7dd4 | 549 | { |
64ca9472 | 550 | long long int time, now; |
0e2a9f6f | 551 | unsigned long flow_count; |
64ca9472 JS |
552 | |
553 | now = time_msec(); | |
b482e960 | 554 | atomic_read_relaxed(&udpif->n_flows_timestamp, &time); |
64ca9472 JS |
555 | if (time < now - 100 && !ovs_mutex_trylock(&udpif->n_flows_mutex)) { |
556 | struct dpif_dp_stats stats; | |
557 | ||
b482e960 | 558 | atomic_store_relaxed(&udpif->n_flows_timestamp, now); |
64ca9472 JS |
559 | dpif_get_dp_stats(udpif->dpif, &stats); |
560 | flow_count = stats.n_flows; | |
b482e960 | 561 | atomic_store_relaxed(&udpif->n_flows, flow_count); |
64ca9472 JS |
562 | ovs_mutex_unlock(&udpif->n_flows_mutex); |
563 | } else { | |
b482e960 | 564 | atomic_read_relaxed(&udpif->n_flows, &flow_count); |
64ca9472 JS |
565 | } |
566 | return flow_count; | |
e79a6c83 | 567 | } |
e1ec7dd4 | 568 | |
a0bab870 | 569 | /* The upcall handler thread tries to read a batch of UPCALL_MAX_BATCH |
9a159f74 AW |
570 | * upcalls from dpif, processes the batch and installs corresponding flows |
571 | * in dpif. */ | |
e1ec7dd4 | 572 | static void * |
10e57640 | 573 | udpif_upcall_handler(void *arg) |
e1ec7dd4 | 574 | { |
e1ec7dd4 | 575 | struct handler *handler = arg; |
9a159f74 | 576 | struct udpif *udpif = handler->udpif; |
e1ec7dd4 | 577 | |
61057e88 | 578 | while (!latch_is_set(&handler->udpif->exit_latch)) { |
cc377352 | 579 | if (!recv_upcalls(handler)) { |
9a159f74 AW |
580 | dpif_recv_wait(udpif->dpif, handler->handler_id); |
581 | latch_wait(&udpif->exit_latch); | |
582 | poll_block(); | |
e1ec7dd4 | 583 | } |
de80e4b6 | 584 | coverage_clear(); |
e1ec7dd4 | 585 | } |
61057e88 BP |
586 | |
587 | return NULL; | |
e1ec7dd4 | 588 | } |
e79a6c83 | 589 | |
cc377352 EJ |
590 | static size_t |
591 | recv_upcalls(struct handler *handler) | |
592 | { | |
593 | struct udpif *udpif = handler->udpif; | |
594 | uint64_t recv_stubs[UPCALL_MAX_BATCH][512 / 8]; | |
595 | struct ofpbuf recv_bufs[UPCALL_MAX_BATCH]; | |
a6f4ad08 | 596 | struct dpif_upcall dupcalls[UPCALL_MAX_BATCH]; |
cc377352 | 597 | struct upcall upcalls[UPCALL_MAX_BATCH]; |
ff601a08 | 598 | struct flow flows[UPCALL_MAX_BATCH]; |
cc377352 EJ |
599 | size_t n_upcalls, i; |
600 | ||
601 | n_upcalls = 0; | |
602 | while (n_upcalls < UPCALL_MAX_BATCH) { | |
603 | struct ofpbuf *recv_buf = &recv_bufs[n_upcalls]; | |
a6f4ad08 | 604 | struct dpif_upcall *dupcall = &dupcalls[n_upcalls]; |
cc377352 | 605 | struct upcall *upcall = &upcalls[n_upcalls]; |
ff601a08 | 606 | struct flow *flow = &flows[n_upcalls]; |
cc377352 | 607 | struct pkt_metadata md; |
cc377352 EJ |
608 | int error; |
609 | ||
7174c145 | 610 | ofpbuf_use_stub(recv_buf, recv_stubs[n_upcalls], |
cc377352 | 611 | sizeof recv_stubs[n_upcalls]); |
a6f4ad08 | 612 | if (dpif_recv(udpif->dpif, handler->handler_id, dupcall, recv_buf)) { |
cc377352 EJ |
613 | ofpbuf_uninit(recv_buf); |
614 | break; | |
615 | } | |
616 | ||
ff601a08 | 617 | if (odp_flow_key_to_flow(dupcall->key, dupcall->key_len, flow) |
cc377352 EJ |
618 | == ODP_FIT_ERROR) { |
619 | goto free_dupcall; | |
620 | } | |
621 | ||
a6f4ad08 | 622 | error = upcall_receive(upcall, udpif->backer, &dupcall->packet, |
ff601a08 | 623 | dupcall->type, dupcall->userdata, flow); |
cc377352 EJ |
624 | if (error) { |
625 | if (error == ENODEV) { | |
626 | /* Received packet on datapath port for which we couldn't | |
627 | * associate an ofproto. This can happen if a port is removed | |
628 | * while traffic is being received. Print a rate-limited | |
629 | * message in case it happens frequently. */ | |
a6f4ad08 AW |
630 | dpif_flow_put(udpif->dpif, DPIF_FP_CREATE, dupcall->key, |
631 | dupcall->key_len, NULL, 0, NULL, 0, NULL); | |
cc377352 | 632 | VLOG_INFO_RL(&rl, "received packet on unassociated datapath " |
ff601a08 | 633 | "port %"PRIu32, flow->in_port.odp_port); |
cc377352 EJ |
634 | } |
635 | goto free_dupcall; | |
636 | } | |
637 | ||
a6f4ad08 AW |
638 | upcall->key = dupcall->key; |
639 | upcall->key_len = dupcall->key_len; | |
cc377352 | 640 | |
8b7ea2d4 WZ |
641 | upcall->out_tun_key = dupcall->out_tun_key; |
642 | ||
ff601a08 | 643 | if (vsp_adjust_flow(upcall->ofproto, flow, &dupcall->packet)) { |
cc377352 EJ |
644 | upcall->vsp_adjusted = true; |
645 | } | |
646 | ||
ff601a08 AW |
647 | md = pkt_metadata_from_flow(flow); |
648 | flow_extract(&dupcall->packet, &md, flow); | |
cc377352 EJ |
649 | |
650 | error = process_upcall(udpif, upcall, NULL); | |
651 | if (error) { | |
652 | goto cleanup; | |
653 | } | |
654 | ||
655 | n_upcalls++; | |
656 | continue; | |
657 | ||
658 | cleanup: | |
659 | upcall_uninit(upcall); | |
660 | free_dupcall: | |
a6f4ad08 | 661 | ofpbuf_uninit(&dupcall->packet); |
cc377352 EJ |
662 | ofpbuf_uninit(recv_buf); |
663 | } | |
664 | ||
665 | if (n_upcalls) { | |
666 | handle_upcalls(handler->udpif, upcalls, n_upcalls); | |
667 | for (i = 0; i < n_upcalls; i++) { | |
a6f4ad08 | 668 | ofpbuf_uninit(&dupcalls[i].packet); |
cc377352 EJ |
669 | ofpbuf_uninit(&recv_bufs[i]); |
670 | upcall_uninit(&upcalls[i]); | |
671 | } | |
672 | } | |
673 | ||
674 | return n_upcalls; | |
675 | } | |
676 | ||
e79a6c83 EJ |
677 | static void * |
678 | udpif_revalidator(void *arg) | |
e1ec7dd4 | 679 | { |
7d170098 | 680 | /* Used by all revalidators. */ |
e79a6c83 | 681 | struct revalidator *revalidator = arg; |
7d170098 EJ |
682 | struct udpif *udpif = revalidator->udpif; |
683 | bool leader = revalidator == &udpif->revalidators[0]; | |
684 | ||
685 | /* Used only by the leader. */ | |
686 | long long int start_time = 0; | |
687 | uint64_t last_reval_seq = 0; | |
7d170098 | 688 | size_t n_flows = 0; |
e1ec7dd4 | 689 | |
8ba0a522 | 690 | revalidator->id = ovsthread_id_self(); |
e79a6c83 | 691 | for (;;) { |
7d170098 EJ |
692 | if (leader) { |
693 | uint64_t reval_seq; | |
e79a6c83 | 694 | |
7d170098 EJ |
695 | reval_seq = seq_read(udpif->reval_seq); |
696 | udpif->need_revalidate = last_reval_seq != reval_seq; | |
697 | last_reval_seq = reval_seq; | |
e79a6c83 | 698 | |
7d170098 EJ |
699 | n_flows = udpif_get_n_flows(udpif); |
700 | udpif->max_n_flows = MAX(n_flows, udpif->max_n_flows); | |
701 | udpif->avg_n_flows = (udpif->avg_n_flows + n_flows) / 2; | |
702 | ||
703 | /* Only the leader checks the exit latch to prevent a race where | |
704 | * some threads think it's true and exit and others think it's | |
705 | * false and block indefinitely on the reval_barrier */ | |
706 | udpif->reval_exit = latch_is_set(&udpif->exit_latch); | |
707 | ||
708 | start_time = time_msec(); | |
709 | if (!udpif->reval_exit) { | |
ac64794a | 710 | udpif->dump = dpif_flow_dump_create(udpif->dpif); |
e79a6c83 EJ |
711 | } |
712 | } | |
713 | ||
7d170098 | 714 | /* Wait for the leader to start the flow dump. */ |
d8043da7 | 715 | ovs_barrier_block(&udpif->reval_barrier); |
7d170098 EJ |
716 | if (udpif->reval_exit) { |
717 | break; | |
e79a6c83 | 718 | } |
7d170098 EJ |
719 | revalidate(revalidator); |
720 | ||
721 | /* Wait for all flows to have been dumped before we garbage collect. */ | |
d8043da7 | 722 | ovs_barrier_block(&udpif->reval_barrier); |
7d170098 EJ |
723 | revalidator_sweep(revalidator); |
724 | ||
725 | /* Wait for all revalidators to finish garbage collection. */ | |
d8043da7 | 726 | ovs_barrier_block(&udpif->reval_barrier); |
7d170098 EJ |
727 | |
728 | if (leader) { | |
b482e960 | 729 | unsigned int flow_limit; |
7d170098 EJ |
730 | long long int duration; |
731 | ||
b482e960 JR |
732 | atomic_read_relaxed(&udpif->flow_limit, &flow_limit); |
733 | ||
ac64794a | 734 | dpif_flow_dump_destroy(udpif->dump); |
7d170098 EJ |
735 | seq_change(udpif->dump_seq); |
736 | ||
737 | duration = MAX(time_msec() - start_time, 1); | |
7d170098 EJ |
738 | udpif->dump_duration = duration; |
739 | if (duration > 2000) { | |
740 | flow_limit /= duration / 1000; | |
741 | } else if (duration > 1300) { | |
742 | flow_limit = flow_limit * 3 / 4; | |
743 | } else if (duration < 1000 && n_flows > 2000 | |
744 | && flow_limit < n_flows * 1000 / duration) { | |
745 | flow_limit += 1000; | |
746 | } | |
747 | flow_limit = MIN(ofproto_flow_limit, MAX(flow_limit, 1000)); | |
b482e960 | 748 | atomic_store_relaxed(&udpif->flow_limit, flow_limit); |
e79a6c83 | 749 | |
7d170098 EJ |
750 | if (duration > 2000) { |
751 | VLOG_INFO("Spent an unreasonably long %lldms dumping flows", | |
752 | duration); | |
753 | } | |
e79a6c83 | 754 | |
7d170098 EJ |
755 | poll_timer_wait_until(start_time + MIN(ofproto_max_idle, 500)); |
756 | seq_wait(udpif->reval_seq, last_reval_seq); | |
757 | latch_wait(&udpif->exit_latch); | |
758 | poll_block(); | |
e79a6c83 EJ |
759 | } |
760 | } | |
761 | ||
762 | return NULL; | |
763 | } | |
764 | \f | |
e1ec7dd4 | 765 | static enum upcall_type |
cc377352 | 766 | classify_upcall(enum dpif_upcall_type type, const struct nlattr *userdata) |
e1ec7dd4 | 767 | { |
e1ec7dd4 EJ |
768 | union user_action_cookie cookie; |
769 | size_t userdata_len; | |
770 | ||
771 | /* First look at the upcall type. */ | |
cc377352 | 772 | switch (type) { |
e1ec7dd4 EJ |
773 | case DPIF_UC_ACTION: |
774 | break; | |
775 | ||
776 | case DPIF_UC_MISS: | |
777 | return MISS_UPCALL; | |
778 | ||
779 | case DPIF_N_UC_TYPES: | |
780 | default: | |
cc377352 | 781 | VLOG_WARN_RL(&rl, "upcall has unexpected type %"PRIu32, type); |
e1ec7dd4 EJ |
782 | return BAD_UPCALL; |
783 | } | |
784 | ||
785 | /* "action" upcalls need a closer look. */ | |
cc377352 | 786 | if (!userdata) { |
e1ec7dd4 EJ |
787 | VLOG_WARN_RL(&rl, "action upcall missing cookie"); |
788 | return BAD_UPCALL; | |
789 | } | |
cc377352 | 790 | userdata_len = nl_attr_get_size(userdata); |
e1ec7dd4 EJ |
791 | if (userdata_len < sizeof cookie.type |
792 | || userdata_len > sizeof cookie) { | |
34582733 | 793 | VLOG_WARN_RL(&rl, "action upcall cookie has unexpected size %"PRIuSIZE, |
e1ec7dd4 EJ |
794 | userdata_len); |
795 | return BAD_UPCALL; | |
796 | } | |
797 | memset(&cookie, 0, sizeof cookie); | |
cc377352 | 798 | memcpy(&cookie, nl_attr_get(userdata), userdata_len); |
f5790bf6 | 799 | if (userdata_len == MAX(8, sizeof cookie.sflow) |
e1ec7dd4 EJ |
800 | && cookie.type == USER_ACTION_COOKIE_SFLOW) { |
801 | return SFLOW_UPCALL; | |
f5790bf6 | 802 | } else if (userdata_len == MAX(8, sizeof cookie.slow_path) |
e1ec7dd4 EJ |
803 | && cookie.type == USER_ACTION_COOKIE_SLOW_PATH) { |
804 | return MISS_UPCALL; | |
f5790bf6 | 805 | } else if (userdata_len == MAX(8, sizeof cookie.flow_sample) |
e1ec7dd4 EJ |
806 | && cookie.type == USER_ACTION_COOKIE_FLOW_SAMPLE) { |
807 | return FLOW_SAMPLE_UPCALL; | |
f5790bf6 | 808 | } else if (userdata_len == MAX(8, sizeof cookie.ipfix) |
e1ec7dd4 EJ |
809 | && cookie.type == USER_ACTION_COOKIE_IPFIX) { |
810 | return IPFIX_UPCALL; | |
811 | } else { | |
812 | VLOG_WARN_RL(&rl, "invalid user cookie of type %"PRIu16 | |
34582733 | 813 | " and size %"PRIuSIZE, cookie.type, userdata_len); |
e1ec7dd4 EJ |
814 | return BAD_UPCALL; |
815 | } | |
816 | } | |
817 | ||
e79a6c83 EJ |
818 | /* Calculates slow path actions for 'xout'. 'buf' must statically be |
819 | * initialized with at least 128 bytes of space. */ | |
820 | static void | |
821 | compose_slow_path(struct udpif *udpif, struct xlate_out *xout, | |
cc377352 | 822 | const struct flow *flow, odp_port_t odp_in_port, |
9a159f74 | 823 | struct ofpbuf *buf) |
e79a6c83 EJ |
824 | { |
825 | union user_action_cookie cookie; | |
826 | odp_port_t port; | |
827 | uint32_t pid; | |
828 | ||
829 | cookie.type = USER_ACTION_COOKIE_SLOW_PATH; | |
830 | cookie.slow_path.unused = 0; | |
831 | cookie.slow_path.reason = xout->slow; | |
832 | ||
833 | port = xout->slow & (SLOW_CFM | SLOW_BFD | SLOW_LACP | SLOW_STP) | |
834 | ? ODPP_NONE | |
835 | : odp_in_port; | |
9a159f74 | 836 | pid = dpif_port_get_pid(udpif->dpif, port, flow_hash_5tuple(flow, 0)); |
8b7ea2d4 WZ |
837 | odp_put_userspace_action(pid, &cookie, sizeof cookie.slow_path, ODPP_NONE, |
838 | buf); | |
e79a6c83 EJ |
839 | } |
840 | ||
3d76b86c AW |
841 | /* If there is no error, the upcall must be destroyed with upcall_uninit() |
842 | * before quiescing, as the referred objects are guaranteed to exist only | |
843 | * until the calling thread quiesces. Otherwise, do not call upcall_uninit() | |
844 | * since the 'upcall->put_actions' remains uninitialized. */ | |
cc377352 EJ |
845 | static int |
846 | upcall_receive(struct upcall *upcall, const struct dpif_backer *backer, | |
847 | const struct ofpbuf *packet, enum dpif_upcall_type type, | |
848 | const struct nlattr *userdata, const struct flow *flow) | |
849 | { | |
850 | int error; | |
851 | ||
5c476ea3 JR |
852 | error = xlate_lookup(backer, flow, &upcall->ofproto, &upcall->ipfix, |
853 | &upcall->sflow, NULL, &upcall->in_port); | |
cc377352 EJ |
854 | if (error) { |
855 | return error; | |
856 | } | |
857 | ||
858 | upcall->flow = flow; | |
859 | upcall->packet = packet; | |
860 | upcall->type = type; | |
861 | upcall->userdata = userdata; | |
862 | ofpbuf_init(&upcall->put_actions, 0); | |
863 | ||
864 | upcall->xout_initialized = false; | |
865 | upcall->vsp_adjusted = false; | |
866 | ||
867 | upcall->key = NULL; | |
868 | upcall->key_len = 0; | |
869 | ||
8b7ea2d4 WZ |
870 | upcall->out_tun_key = NULL; |
871 | ||
cc377352 EJ |
872 | return 0; |
873 | } | |
874 | ||
a0bab870 | 875 | static void |
cc377352 EJ |
876 | upcall_xlate(struct udpif *udpif, struct upcall *upcall, |
877 | struct ofpbuf *odp_actions) | |
e1ec7dd4 | 878 | { |
cc377352 | 879 | struct dpif_flow_stats stats; |
691d39b2 | 880 | struct xlate_in xin; |
a0bab870 | 881 | |
cc377352 EJ |
882 | stats.n_packets = 1; |
883 | stats.n_bytes = ofpbuf_size(upcall->packet); | |
884 | stats.used = time_msec(); | |
885 | stats.tcp_flags = ntohs(upcall->flow->tcp_flags); | |
a0bab870 | 886 | |
cc377352 EJ |
887 | xlate_in_init(&xin, upcall->ofproto, upcall->flow, upcall->in_port, NULL, |
888 | stats.tcp_flags, upcall->packet); | |
889 | xin.odp_actions = odp_actions; | |
a0bab870 | 890 | |
cc377352 EJ |
891 | if (upcall->type == DPIF_UC_MISS) { |
892 | xin.resubmit_stats = &stats; | |
a0bab870 RW |
893 | } else { |
894 | /* For non-miss upcalls, there's a flow in the datapath which this | |
895 | * packet was accounted to. Presumably the revalidators will deal | |
896 | * with pushing its stats eventually. */ | |
e1ec7dd4 EJ |
897 | } |
898 | ||
a0bab870 | 899 | xlate_actions(&xin, &upcall->xout); |
cc377352 EJ |
900 | upcall->xout_initialized = true; |
901 | ||
902 | /* Special case for fail-open mode. | |
903 | * | |
904 | * If we are in fail-open mode, but we are connected to a controller too, | |
905 | * then we should send the packet up to the controller in the hope that it | |
906 | * will try to set up a flow and thereby allow us to exit fail-open. | |
907 | * | |
908 | * See the top-level comment in fail-open.c for more information. | |
909 | * | |
910 | * Copy packets before they are modified by execution. */ | |
911 | if (upcall->xout.fail_open) { | |
912 | const struct ofpbuf *packet = upcall->packet; | |
913 | struct ofproto_packet_in *pin; | |
914 | ||
915 | pin = xmalloc(sizeof *pin); | |
916 | pin->up.packet = xmemdup(ofpbuf_data(packet), ofpbuf_size(packet)); | |
917 | pin->up.packet_len = ofpbuf_size(packet); | |
918 | pin->up.reason = OFPR_NO_MATCH; | |
919 | pin->up.table_id = 0; | |
920 | pin->up.cookie = OVS_BE64_MAX; | |
921 | flow_get_metadata(upcall->flow, &pin->up.fmd); | |
922 | pin->send_len = 0; /* Not used for flow table misses. */ | |
923 | pin->miss_type = OFPROTO_PACKET_IN_NO_MISS; | |
924 | ofproto_dpif_send_packet_in(upcall->ofproto, pin); | |
925 | } | |
926 | ||
927 | if (!upcall->xout.slow) { | |
928 | ofpbuf_use_const(&upcall->put_actions, | |
929 | ofpbuf_data(upcall->xout.odp_actions), | |
930 | ofpbuf_size(upcall->xout.odp_actions)); | |
931 | } else { | |
932 | ofpbuf_init(&upcall->put_actions, 0); | |
933 | compose_slow_path(udpif, &upcall->xout, upcall->flow, | |
934 | upcall->flow->in_port.odp_port, | |
935 | &upcall->put_actions); | |
936 | } | |
e1ec7dd4 EJ |
937 | } |
938 | ||
3eed53e9 | 939 | static void |
cc377352 | 940 | upcall_uninit(struct upcall *upcall) |
6b31e073 | 941 | { |
cc377352 EJ |
942 | if (upcall) { |
943 | if (upcall->xout_initialized) { | |
944 | xlate_out_uninit(&upcall->xout); | |
945 | } | |
946 | ofpbuf_uninit(&upcall->put_actions); | |
cc377352 | 947 | } |
6b31e073 RW |
948 | } |
949 | ||
623540e4 EJ |
950 | static int |
951 | upcall_cb(const struct ofpbuf *packet, const struct flow *flow, | |
952 | enum dpif_upcall_type type, const struct nlattr *userdata, | |
953 | struct ofpbuf *actions, struct flow_wildcards *wc, | |
954 | struct ofpbuf *put_actions, void *aux) | |
6b31e073 | 955 | { |
623540e4 EJ |
956 | struct udpif *udpif = aux; |
957 | unsigned int flow_limit; | |
958 | struct upcall upcall; | |
959 | bool megaflow; | |
960 | int error; | |
6b31e073 | 961 | |
b482e960 JR |
962 | atomic_read_relaxed(&enable_megaflows, &megaflow); |
963 | atomic_read_relaxed(&udpif->flow_limit, &flow_limit); | |
964 | ||
623540e4 EJ |
965 | error = upcall_receive(&upcall, udpif->backer, packet, type, userdata, |
966 | flow); | |
967 | if (error) { | |
3d76b86c | 968 | return error; |
6b31e073 | 969 | } |
6b31e073 | 970 | |
623540e4 EJ |
971 | error = process_upcall(udpif, &upcall, actions); |
972 | if (error) { | |
973 | goto out; | |
974 | } | |
cc377352 | 975 | |
623540e4 EJ |
976 | if (upcall.xout.slow && put_actions) { |
977 | ofpbuf_put(put_actions, ofpbuf_data(&upcall.put_actions), | |
978 | ofpbuf_size(&upcall.put_actions)); | |
979 | } | |
cc377352 | 980 | |
b482e960 | 981 | if (OVS_LIKELY(wc)) { |
623540e4 EJ |
982 | if (megaflow) { |
983 | /* XXX: This could be avoided with sufficient API changes. */ | |
984 | *wc = upcall.xout.wc; | |
985 | } else { | |
78c9486d | 986 | flow_wildcards_init_for_packet(wc, flow); |
9a159f74 | 987 | } |
623540e4 | 988 | } |
9a159f74 | 989 | |
623540e4 EJ |
990 | if (udpif_get_n_flows(udpif) >= flow_limit) { |
991 | error = ENOSPC; | |
6b31e073 | 992 | } |
623540e4 EJ |
993 | |
994 | out: | |
995 | upcall_uninit(&upcall); | |
996 | return error; | |
6b31e073 | 997 | } |
10e57640 | 998 | |
3eed53e9 | 999 | static int |
cc377352 EJ |
1000 | process_upcall(struct udpif *udpif, struct upcall *upcall, |
1001 | struct ofpbuf *odp_actions) | |
6b31e073 | 1002 | { |
cc377352 EJ |
1003 | const struct nlattr *userdata = upcall->userdata; |
1004 | const struct ofpbuf *packet = upcall->packet; | |
1005 | const struct flow *flow = upcall->flow; | |
04a19fb8 | 1006 | |
cc377352 EJ |
1007 | switch (classify_upcall(upcall->type, userdata)) { |
1008 | case MISS_UPCALL: | |
1009 | upcall_xlate(udpif, upcall, odp_actions); | |
1010 | return 0; | |
10e57640 | 1011 | |
6b31e073 | 1012 | case SFLOW_UPCALL: |
cc377352 | 1013 | if (upcall->sflow) { |
6b31e073 RW |
1014 | union user_action_cookie cookie; |
1015 | ||
1016 | memset(&cookie, 0, sizeof cookie); | |
cc377352 EJ |
1017 | memcpy(&cookie, nl_attr_get(userdata), sizeof cookie.sflow); |
1018 | dpif_sflow_received(upcall->sflow, packet, flow, | |
1019 | flow->in_port.odp_port, &cookie); | |
6b31e073 RW |
1020 | } |
1021 | break; | |
cc377352 | 1022 | |
6b31e073 | 1023 | case IPFIX_UPCALL: |
cc377352 | 1024 | if (upcall->ipfix) { |
8b7ea2d4 WZ |
1025 | union user_action_cookie cookie; |
1026 | struct flow_tnl output_tunnel_key; | |
1027 | ||
1028 | memset(&cookie, 0, sizeof cookie); | |
1029 | memcpy(&cookie, nl_attr_get(userdata), sizeof cookie.ipfix); | |
1030 | ||
1031 | if (upcall->out_tun_key) { | |
1032 | memset(&output_tunnel_key, 0, sizeof output_tunnel_key); | |
1033 | odp_tun_key_from_attr(upcall->out_tun_key, | |
1034 | &output_tunnel_key); | |
1035 | } | |
1036 | dpif_ipfix_bridge_sample(upcall->ipfix, packet, flow, | |
1037 | flow->in_port.odp_port, | |
1038 | cookie.ipfix.output_odp_port, | |
1039 | upcall->out_tun_key ? | |
1040 | &output_tunnel_key : NULL); | |
6b31e073 RW |
1041 | } |
1042 | break; | |
cc377352 | 1043 | |
6b31e073 | 1044 | case FLOW_SAMPLE_UPCALL: |
cc377352 | 1045 | if (upcall->ipfix) { |
6b31e073 RW |
1046 | union user_action_cookie cookie; |
1047 | ||
1048 | memset(&cookie, 0, sizeof cookie); | |
cc377352 | 1049 | memcpy(&cookie, nl_attr_get(userdata), sizeof cookie.flow_sample); |
6b31e073 RW |
1050 | |
1051 | /* The flow reflects exactly the contents of the packet. | |
1052 | * Sample the packet using it. */ | |
cc377352 | 1053 | dpif_ipfix_flow_sample(upcall->ipfix, packet, flow, |
6b31e073 RW |
1054 | cookie.flow_sample.collector_set_id, |
1055 | cookie.flow_sample.probability, | |
1056 | cookie.flow_sample.obs_domain_id, | |
1057 | cookie.flow_sample.obs_point_id); | |
e1ec7dd4 | 1058 | } |
6b31e073 | 1059 | break; |
cc377352 | 1060 | |
6b31e073 RW |
1061 | case BAD_UPCALL: |
1062 | break; | |
6b31e073 | 1063 | } |
10e57640 | 1064 | |
cc377352 | 1065 | return EAGAIN; |
9a159f74 AW |
1066 | } |
1067 | ||
1068 | static void | |
6b31e073 | 1069 | handle_upcalls(struct udpif *udpif, struct upcall *upcalls, |
a0bab870 | 1070 | size_t n_upcalls) |
9a159f74 | 1071 | { |
cc377352 | 1072 | struct odputil_keybuf mask_bufs[UPCALL_MAX_BATCH]; |
a0bab870 | 1073 | struct dpif_op *opsp[UPCALL_MAX_BATCH * 2]; |
6dad4d44 | 1074 | struct ukey_op ops[UPCALL_MAX_BATCH * 2]; |
9a159f74 | 1075 | unsigned int flow_limit; |
cc377352 EJ |
1076 | size_t n_ops, i; |
1077 | bool may_put; | |
b482e960 JR |
1078 | bool megaflow; |
1079 | ||
1080 | atomic_read_relaxed(&udpif->flow_limit, &flow_limit); | |
1081 | atomic_read_relaxed(&enable_megaflows, &megaflow); | |
9a159f74 | 1082 | |
9a159f74 AW |
1083 | may_put = udpif_get_n_flows(udpif) < flow_limit; |
1084 | ||
a0bab870 | 1085 | /* Handle the packets individually in order of arrival. |
04a19fb8 BP |
1086 | * |
1087 | * - For SLOW_CFM, SLOW_LACP, SLOW_STP, and SLOW_BFD, translation is what | |
1088 | * processes received packets for these protocols. | |
1089 | * | |
1090 | * - For SLOW_CONTROLLER, translation sends the packet to the OpenFlow | |
1091 | * controller. | |
1092 | * | |
1093 | * The loop fills 'ops' with an array of operations to execute in the | |
1094 | * datapath. */ | |
1095 | n_ops = 0; | |
9a159f74 AW |
1096 | for (i = 0; i < n_upcalls; i++) { |
1097 | struct upcall *upcall = &upcalls[i]; | |
cc377352 | 1098 | const struct ofpbuf *packet = upcall->packet; |
6dad4d44 | 1099 | struct ukey_op *op; |
d02c42bf | 1100 | |
cc377352 EJ |
1101 | if (upcall->vsp_adjusted) { |
1102 | /* This packet was received on a VLAN splinter port. We added a | |
1103 | * VLAN to the packet to make the packet resemble the flow, but the | |
1104 | * actions were composed assuming that the packet contained no | |
1105 | * VLAN. So, we must remove the VLAN header from the packet before | |
1106 | * trying to execute the actions. */ | |
1107 | if (ofpbuf_size(upcall->xout.odp_actions)) { | |
1108 | eth_pop_vlan(CONST_CAST(struct ofpbuf *, upcall->packet)); | |
d02c42bf AZ |
1109 | } |
1110 | ||
1111 | /* Remove the flow vlan tags inserted by vlan splinter logic | |
1112 | * to ensure megaflow masks generated match the data path flow. */ | |
cc377352 | 1113 | CONST_CAST(struct flow *, upcall->flow)->vlan_tci = 0; |
e79a6c83 | 1114 | } |
04a19fb8 | 1115 | |
73e141f9 BP |
1116 | /* Do not install a flow into the datapath if: |
1117 | * | |
1118 | * - The datapath already has too many flows. | |
1119 | * | |
73e141f9 BP |
1120 | * - We received this packet via some flow installed in the kernel |
1121 | * already. */ | |
cc377352 | 1122 | if (may_put && upcall->type == DPIF_UC_MISS) { |
d02c42bf | 1123 | struct ofpbuf mask; |
d02c42bf | 1124 | |
cc377352 | 1125 | ofpbuf_use_stack(&mask, &mask_bufs[i], sizeof mask_bufs[i]); |
b482e960 | 1126 | |
d02c42bf | 1127 | if (megaflow) { |
8bfd0fda | 1128 | size_t max_mpls; |
7ce2769e | 1129 | bool recirc; |
8bfd0fda | 1130 | |
a0bab870 RW |
1131 | recirc = ofproto_dpif_get_enable_recirc(upcall->ofproto); |
1132 | max_mpls = ofproto_dpif_get_max_mpls_depth(upcall->ofproto); | |
1133 | odp_flow_key_from_mask(&mask, &upcall->xout.wc.masks, | |
cc377352 | 1134 | upcall->flow, UINT32_MAX, max_mpls, |
7ce2769e | 1135 | recirc); |
d02c42bf AZ |
1136 | } |
1137 | ||
e79a6c83 | 1138 | op = &ops[n_ops++]; |
6dad4d44 JS |
1139 | op->dop.type = DPIF_OP_FLOW_PUT; |
1140 | op->dop.u.flow_put.flags = DPIF_FP_CREATE; | |
1141 | op->dop.u.flow_put.key = upcall->key; | |
1142 | op->dop.u.flow_put.key_len = upcall->key_len; | |
1143 | op->dop.u.flow_put.mask = ofpbuf_data(&mask); | |
1144 | op->dop.u.flow_put.mask_len = ofpbuf_size(&mask); | |
1145 | op->dop.u.flow_put.stats = NULL; | |
1146 | op->dop.u.flow_put.actions = ofpbuf_data(&upcall->put_actions); | |
1147 | op->dop.u.flow_put.actions_len = ofpbuf_size(&upcall->put_actions); | |
e79a6c83 EJ |
1148 | } |
1149 | ||
cc377352 | 1150 | if (ofpbuf_size(upcall->xout.odp_actions)) { |
04a19fb8 | 1151 | op = &ops[n_ops++]; |
6dad4d44 JS |
1152 | op->dop.type = DPIF_OP_EXECUTE; |
1153 | op->dop.u.execute.packet = CONST_CAST(struct ofpbuf *, packet); | |
a0bab870 | 1154 | odp_key_to_pkt_metadata(upcall->key, upcall->key_len, |
6dad4d44 JS |
1155 | &op->dop.u.execute.md); |
1156 | op->dop.u.execute.actions = ofpbuf_data(upcall->xout.odp_actions); | |
1157 | op->dop.u.execute.actions_len = ofpbuf_size(upcall->xout.odp_actions); | |
1158 | op->dop.u.execute.needs_help = (upcall->xout.slow & SLOW_ACTION) != 0; | |
1159 | op->dop.u.execute.probe = false; | |
04a19fb8 | 1160 | } |
e1ec7dd4 | 1161 | } |
e1ec7dd4 | 1162 | |
da546e07 JR |
1163 | /* Execute batch. */ |
1164 | for (i = 0; i < n_ops; i++) { | |
6dad4d44 | 1165 | opsp[i] = &ops[i].dop; |
da546e07 JR |
1166 | } |
1167 | dpif_operate(udpif->dpif, opsp, n_ops); | |
e79a6c83 EJ |
1168 | } |
1169 | ||
1170 | static struct udpif_key * | |
feca8bd7 JS |
1171 | ukey_lookup(struct udpif *udpif, const struct nlattr *key, size_t key_len, |
1172 | uint32_t hash) | |
e79a6c83 EJ |
1173 | { |
1174 | struct udpif_key *ukey; | |
b8d3daeb | 1175 | struct cmap *cmap = &udpif->ukeys[hash % N_UMAPS].cmap; |
e79a6c83 | 1176 | |
9fce0584 | 1177 | CMAP_FOR_EACH_WITH_HASH (ukey, cmap_node, hash, cmap) { |
7d170098 | 1178 | if (ukey->key_len == key_len && !memcmp(ukey->key, key, key_len)) { |
e79a6c83 EJ |
1179 | return ukey; |
1180 | } | |
1181 | } | |
1182 | return NULL; | |
1183 | } | |
1184 | ||
feca8bd7 JS |
1185 | /* Creates a ukey for 'key' and 'key_len', returning it with ukey->mutex in |
1186 | * a locked state. */ | |
13bb6ed0 | 1187 | static struct udpif_key * |
9fce0584 JS |
1188 | ukey_create(const struct nlattr *key, size_t key_len, long long int used, |
1189 | uint32_t hash) | |
feca8bd7 | 1190 | OVS_NO_THREAD_SAFETY_ANALYSIS |
13bb6ed0 JS |
1191 | { |
1192 | struct udpif_key *ukey = xmalloc(sizeof *ukey); | |
1193 | ||
feca8bd7 | 1194 | ovs_mutex_init(&ukey->mutex); |
02334943 | 1195 | ukey->key = &ukey->key_buf_nla; |
13bb6ed0 JS |
1196 | memcpy(&ukey->key_buf, key, key_len); |
1197 | ukey->key_len = key_len; | |
1198 | ||
7d170098 | 1199 | ovs_mutex_lock(&ukey->mutex); |
9fce0584 | 1200 | ukey->hash = hash; |
efa08531 | 1201 | ukey->dump_seq = 0; |
73a3c475 | 1202 | ukey->flow_exists = true; |
13bb6ed0 JS |
1203 | ukey->created = used ? used : time_msec(); |
1204 | memset(&ukey->stats, 0, sizeof ukey->stats); | |
b256dc52 | 1205 | ukey->xcache = NULL; |
13bb6ed0 JS |
1206 | |
1207 | return ukey; | |
1208 | } | |
1209 | ||
feca8bd7 JS |
1210 | /* Searches for a ukey in 'udpif->ukeys' that matches 'key' and 'key_len' and |
1211 | * attempts to lock the ukey. If the ukey does not exist, create it. | |
7d170098 | 1212 | * |
feca8bd7 JS |
1213 | * Returns true on success, setting *result to the matching ukey and returning |
1214 | * it in a locked state. Otherwise, returns false and clears *result. */ | |
7d170098 | 1215 | static bool |
feca8bd7 JS |
1216 | ukey_acquire(struct udpif *udpif, const struct nlattr *key, size_t key_len, |
1217 | long long int used, struct udpif_key **result) | |
1218 | OVS_TRY_LOCK(true, (*result)->mutex) | |
7d170098 | 1219 | { |
feca8bd7 JS |
1220 | struct udpif_key *ukey; |
1221 | uint32_t hash, idx; | |
1222 | bool locked = false; | |
1223 | ||
1224 | hash = hash_bytes(key, key_len, udpif->secret); | |
b8d3daeb | 1225 | idx = hash % N_UMAPS; |
7d170098 EJ |
1226 | |
1227 | ovs_mutex_lock(&udpif->ukeys[idx].mutex); | |
feca8bd7 JS |
1228 | ukey = ukey_lookup(udpif, key, key_len, hash); |
1229 | if (!ukey) { | |
9fce0584 JS |
1230 | ukey = ukey_create(key, key_len, used, hash); |
1231 | cmap_insert(&udpif->ukeys[idx].cmap, &ukey->cmap_node, hash); | |
feca8bd7 JS |
1232 | locked = true; |
1233 | } else if (!ovs_mutex_trylock(&ukey->mutex)) { | |
1234 | locked = true; | |
7d170098 EJ |
1235 | } |
1236 | ovs_mutex_unlock(&udpif->ukeys[idx].mutex); | |
1237 | ||
feca8bd7 JS |
1238 | if (locked) { |
1239 | *result = ukey; | |
1240 | } else { | |
1241 | *result = NULL; | |
1242 | } | |
1243 | return locked; | |
7d170098 EJ |
1244 | } |
1245 | ||
e79a6c83 | 1246 | static void |
9fce0584 | 1247 | ukey_delete__(struct udpif_key *ukey) |
7d170098 | 1248 | OVS_NO_THREAD_SAFETY_ANALYSIS |
e79a6c83 | 1249 | { |
b256dc52 | 1250 | xlate_cache_delete(ukey->xcache); |
7d170098 | 1251 | ovs_mutex_destroy(&ukey->mutex); |
e79a6c83 EJ |
1252 | free(ukey); |
1253 | } | |
1254 | ||
9fce0584 | 1255 | static void |
b8d3daeb JS |
1256 | ukey_delete(struct umap *umap, struct udpif_key *ukey) |
1257 | OVS_REQUIRES(umap->mutex) | |
9fce0584 | 1258 | { |
b8d3daeb | 1259 | cmap_remove(&umap->cmap, &ukey->cmap_node, ukey->hash); |
9fce0584 JS |
1260 | ovsrcu_postpone(ukey_delete__, ukey); |
1261 | } | |
1262 | ||
698ffe36 | 1263 | static bool |
49fae772 JS |
1264 | should_revalidate(const struct udpif *udpif, uint64_t packets, |
1265 | long long int used) | |
698ffe36 JS |
1266 | { |
1267 | long long int metric, now, duration; | |
1268 | ||
49fae772 JS |
1269 | if (udpif->dump_duration < 200) { |
1270 | /* We are likely to handle full revalidation for the flows. */ | |
1271 | return true; | |
1272 | } | |
1273 | ||
698ffe36 JS |
1274 | /* Calculate the mean time between seeing these packets. If this |
1275 | * exceeds the threshold, then delete the flow rather than performing | |
1276 | * costly revalidation for flows that aren't being hit frequently. | |
1277 | * | |
1278 | * This is targeted at situations where the dump_duration is high (~1s), | |
1279 | * and revalidation is triggered by a call to udpif_revalidate(). In | |
1280 | * these situations, revalidation of all flows causes fluctuations in the | |
1281 | * flow_limit due to the interaction with the dump_duration and max_idle. | |
1282 | * This tends to result in deletion of low-throughput flows anyway, so | |
1283 | * skip the revalidation and just delete those flows. */ | |
1284 | packets = MAX(packets, 1); | |
1285 | now = MAX(used, time_msec()); | |
1286 | duration = now - used; | |
1287 | metric = duration / packets; | |
1288 | ||
49fae772 JS |
1289 | if (metric < 200) { |
1290 | /* The flow is receiving more than ~5pps, so keep it. */ | |
1291 | return true; | |
698ffe36 | 1292 | } |
49fae772 | 1293 | return false; |
698ffe36 JS |
1294 | } |
1295 | ||
e79a6c83 | 1296 | static bool |
7d170098 | 1297 | revalidate_ukey(struct udpif *udpif, struct udpif_key *ukey, |
ac64794a | 1298 | const struct dpif_flow *f) |
acaa8dac | 1299 | OVS_REQUIRES(ukey->mutex) |
e79a6c83 | 1300 | { |
e79a6c83 EJ |
1301 | uint64_t slow_path_buf[128 / 8]; |
1302 | struct xlate_out xout, *xoutp; | |
42f3baca | 1303 | struct netflow *netflow; |
e79a6c83 EJ |
1304 | struct ofproto_dpif *ofproto; |
1305 | struct dpif_flow_stats push; | |
7d170098 EJ |
1306 | struct ofpbuf xout_actions; |
1307 | struct flow flow, dp_mask; | |
1308 | uint32_t *dp32, *xout32; | |
cc377352 | 1309 | ofp_port_t ofp_in_port; |
e79a6c83 | 1310 | struct xlate_in xin; |
698ffe36 | 1311 | long long int last_used; |
e79a6c83 EJ |
1312 | int error; |
1313 | size_t i; | |
0725e747 | 1314 | bool ok; |
e79a6c83 EJ |
1315 | |
1316 | ok = false; | |
1317 | xoutp = NULL; | |
42f3baca | 1318 | netflow = NULL; |
e79a6c83 | 1319 | |
698ffe36 | 1320 | last_used = ukey->stats.used; |
ac64794a BP |
1321 | push.used = f->stats.used; |
1322 | push.tcp_flags = f->stats.tcp_flags; | |
1323 | push.n_packets = (f->stats.n_packets > ukey->stats.n_packets | |
1324 | ? f->stats.n_packets - ukey->stats.n_packets | |
1325 | : 0); | |
1326 | push.n_bytes = (f->stats.n_bytes > ukey->stats.n_bytes | |
1327 | ? f->stats.n_bytes - ukey->stats.n_bytes | |
1328 | : 0); | |
e79a6c83 | 1329 | |
7d170098 | 1330 | if (udpif->need_revalidate && last_used |
49fae772 | 1331 | && !should_revalidate(udpif, push.n_packets, last_used)) { |
698ffe36 JS |
1332 | ok = false; |
1333 | goto exit; | |
1334 | } | |
1335 | ||
28c5588e | 1336 | /* We will push the stats, so update the ukey stats cache. */ |
ac64794a | 1337 | ukey->stats = f->stats; |
7d170098 | 1338 | if (!push.n_packets && !udpif->need_revalidate) { |
e79a6c83 EJ |
1339 | ok = true; |
1340 | goto exit; | |
1341 | } | |
1342 | ||
df1a9a49 | 1343 | if (ukey->xcache && !udpif->need_revalidate) { |
0725e747 | 1344 | xlate_push_stats(ukey->xcache, &push); |
df1a9a49 JS |
1345 | ok = true; |
1346 | goto exit; | |
b256dc52 JS |
1347 | } |
1348 | ||
cc377352 EJ |
1349 | if (odp_flow_key_to_flow(ukey->key, ukey->key_len, &flow) |
1350 | == ODP_FIT_ERROR) { | |
1351 | goto exit; | |
1352 | } | |
1353 | ||
5c476ea3 JR |
1354 | error = xlate_lookup(udpif->backer, &flow, &ofproto, NULL, NULL, &netflow, |
1355 | &ofp_in_port); | |
e79a6c83 EJ |
1356 | if (error) { |
1357 | goto exit; | |
1358 | } | |
1359 | ||
df1a9a49 JS |
1360 | if (udpif->need_revalidate) { |
1361 | xlate_cache_clear(ukey->xcache); | |
1362 | } | |
b256dc52 JS |
1363 | if (!ukey->xcache) { |
1364 | ukey->xcache = xlate_cache_new(); | |
1365 | } | |
1366 | ||
cc377352 EJ |
1367 | xlate_in_init(&xin, ofproto, &flow, ofp_in_port, NULL, push.tcp_flags, |
1368 | NULL); | |
0725e747 BP |
1369 | if (push.n_packets) { |
1370 | xin.resubmit_stats = &push; | |
1371 | xin.may_learn = true; | |
1372 | } | |
b256dc52 | 1373 | xin.xcache = ukey->xcache; |
7d170098 | 1374 | xin.skip_wildcards = !udpif->need_revalidate; |
e79a6c83 EJ |
1375 | xlate_actions(&xin, &xout); |
1376 | xoutp = &xout; | |
ddeca9a4 | 1377 | |
7d170098 | 1378 | if (!udpif->need_revalidate) { |
e79a6c83 EJ |
1379 | ok = true; |
1380 | goto exit; | |
1381 | } | |
1382 | ||
1383 | if (!xout.slow) { | |
cc377352 EJ |
1384 | ofpbuf_use_const(&xout_actions, ofpbuf_data(xout.odp_actions), |
1385 | ofpbuf_size(xout.odp_actions)); | |
05067881 | 1386 | } else { |
e79a6c83 | 1387 | ofpbuf_use_stack(&xout_actions, slow_path_buf, sizeof slow_path_buf); |
cc377352 EJ |
1388 | compose_slow_path(udpif, &xout, &flow, flow.in_port.odp_port, |
1389 | &xout_actions); | |
e79a6c83 EJ |
1390 | } |
1391 | ||
ac64794a BP |
1392 | if (f->actions_len != ofpbuf_size(&xout_actions) |
1393 | || memcmp(ofpbuf_data(&xout_actions), f->actions, f->actions_len)) { | |
e79a6c83 EJ |
1394 | goto exit; |
1395 | } | |
1396 | ||
ac64794a | 1397 | if (odp_flow_key_to_mask(f->mask, f->mask_len, &dp_mask, &flow) |
e79a6c83 EJ |
1398 | == ODP_FIT_ERROR) { |
1399 | goto exit; | |
1400 | } | |
1401 | ||
1402 | /* Since the kernel is free to ignore wildcarded bits in the mask, we can't | |
1403 | * directly check that the masks are the same. Instead we check that the | |
1404 | * mask in the kernel is more specific i.e. less wildcarded, than what | |
1405 | * we've calculated here. This guarantees we don't catch any packets we | |
1406 | * shouldn't with the megaflow. */ | |
7d170098 | 1407 | dp32 = (uint32_t *) &dp_mask; |
e79a6c83 EJ |
1408 | xout32 = (uint32_t *) &xout.wc.masks; |
1409 | for (i = 0; i < FLOW_U32S; i++) { | |
7d170098 | 1410 | if ((dp32[i] | xout32[i]) != dp32[i]) { |
e79a6c83 EJ |
1411 | goto exit; |
1412 | } | |
1413 | } | |
1414 | ok = true; | |
1415 | ||
1416 | exit: | |
dcc2c6cd JR |
1417 | if (netflow && !ok) { |
1418 | netflow_flow_clear(netflow, &flow); | |
42f3baca | 1419 | } |
e79a6c83 EJ |
1420 | xlate_out_uninit(xoutp); |
1421 | return ok; | |
1422 | } | |
1423 | ||
1424 | static void | |
6dad4d44 JS |
1425 | delete_op_init(struct ukey_op *op, const struct nlattr *key, size_t key_len, |
1426 | struct udpif_key *ukey) | |
13bb6ed0 JS |
1427 | { |
1428 | op->ukey = ukey; | |
6dad4d44 JS |
1429 | op->dop.type = DPIF_OP_FLOW_DEL; |
1430 | op->dop.u.flow_del.key = key; | |
1431 | op->dop.u.flow_del.key_len = key_len; | |
1432 | op->dop.u.flow_del.stats = &op->stats; | |
13bb6ed0 JS |
1433 | } |
1434 | ||
1435 | static void | |
6dad4d44 | 1436 | push_ukey_ops__(struct udpif *udpif, struct ukey_op *ops, size_t n_ops) |
e79a6c83 | 1437 | { |
13bb6ed0 JS |
1438 | struct dpif_op *opsp[REVALIDATE_MAX_BATCH]; |
1439 | size_t i; | |
e79a6c83 | 1440 | |
13bb6ed0 JS |
1441 | ovs_assert(n_ops <= REVALIDATE_MAX_BATCH); |
1442 | for (i = 0; i < n_ops; i++) { | |
6dad4d44 | 1443 | opsp[i] = &ops[i].dop; |
13bb6ed0 JS |
1444 | } |
1445 | dpif_operate(udpif->dpif, opsp, n_ops); | |
1446 | ||
1447 | for (i = 0; i < n_ops; i++) { | |
6dad4d44 | 1448 | struct ukey_op *op = &ops[i]; |
13bb6ed0 JS |
1449 | struct dpif_flow_stats *push, *stats, push_buf; |
1450 | ||
6dad4d44 | 1451 | stats = op->dop.u.flow_del.stats; |
5e73c322 JS |
1452 | push = &push_buf; |
1453 | ||
1454 | ovs_mutex_lock(&op->ukey->mutex); | |
1455 | push->used = MAX(stats->used, op->ukey->stats.used); | |
1456 | push->tcp_flags = stats->tcp_flags | op->ukey->stats.tcp_flags; | |
1457 | push->n_packets = stats->n_packets - op->ukey->stats.n_packets; | |
1458 | push->n_bytes = stats->n_bytes - op->ukey->stats.n_bytes; | |
1459 | ovs_mutex_unlock(&op->ukey->mutex); | |
13bb6ed0 JS |
1460 | |
1461 | if (push->n_packets || netflow_exists()) { | |
1462 | struct ofproto_dpif *ofproto; | |
1463 | struct netflow *netflow; | |
cc377352 | 1464 | ofp_port_t ofp_in_port; |
13bb6ed0 | 1465 | struct flow flow; |
5e73c322 | 1466 | int error; |
b256dc52 | 1467 | |
5e73c322 JS |
1468 | ovs_mutex_lock(&op->ukey->mutex); |
1469 | if (op->ukey->xcache) { | |
0725e747 | 1470 | xlate_push_stats(op->ukey->xcache, push); |
7d170098 | 1471 | ovs_mutex_unlock(&op->ukey->mutex); |
5e73c322 | 1472 | continue; |
b256dc52 | 1473 | } |
5e73c322 | 1474 | ovs_mutex_unlock(&op->ukey->mutex); |
13bb6ed0 | 1475 | |
6dad4d44 JS |
1476 | if (odp_flow_key_to_flow(op->dop.u.flow_del.key, |
1477 | op->dop.u.flow_del.key_len, &flow) | |
cc377352 EJ |
1478 | == ODP_FIT_ERROR) { |
1479 | continue; | |
1480 | } | |
1481 | ||
5c476ea3 JR |
1482 | error = xlate_lookup(udpif->backer, &flow, &ofproto, |
1483 | NULL, NULL, &netflow, &ofp_in_port); | |
5e73c322 | 1484 | if (!error) { |
13bb6ed0 JS |
1485 | struct xlate_in xin; |
1486 | ||
cc377352 EJ |
1487 | xlate_in_init(&xin, ofproto, &flow, ofp_in_port, NULL, |
1488 | push->tcp_flags, NULL); | |
13bb6ed0 | 1489 | xin.resubmit_stats = push->n_packets ? push : NULL; |
0725e747 | 1490 | xin.may_learn = push->n_packets > 0; |
13bb6ed0 JS |
1491 | xin.skip_wildcards = true; |
1492 | xlate_actions_for_side_effects(&xin); | |
1493 | ||
1494 | if (netflow) { | |
13bb6ed0 | 1495 | netflow_flow_clear(netflow, &flow); |
13bb6ed0 JS |
1496 | } |
1497 | } | |
1498 | } | |
1499 | } | |
7d170098 | 1500 | } |
13bb6ed0 | 1501 | |
7d170098 | 1502 | static void |
6dad4d44 JS |
1503 | push_ukey_ops(struct udpif *udpif, struct umap *umap, |
1504 | struct ukey_op *ops, size_t n_ops) | |
7d170098 EJ |
1505 | { |
1506 | int i; | |
13bb6ed0 | 1507 | |
6dad4d44 | 1508 | push_ukey_ops__(udpif, ops, n_ops); |
b8d3daeb | 1509 | ovs_mutex_lock(&umap->mutex); |
7d170098 | 1510 | for (i = 0; i < n_ops; i++) { |
b8d3daeb | 1511 | ukey_delete(umap, ops[i].ukey); |
13bb6ed0 | 1512 | } |
b8d3daeb | 1513 | ovs_mutex_unlock(&umap->mutex); |
13bb6ed0 JS |
1514 | } |
1515 | ||
1516 | static void | |
7d170098 | 1517 | revalidate(struct revalidator *revalidator) |
13bb6ed0 JS |
1518 | { |
1519 | struct udpif *udpif = revalidator->udpif; | |
ac64794a | 1520 | struct dpif_flow_dump_thread *dump_thread; |
efa08531 | 1521 | uint64_t dump_seq; |
e79a6c83 | 1522 | unsigned int flow_limit; |
e79a6c83 | 1523 | |
efa08531 | 1524 | dump_seq = seq_read(udpif->dump_seq); |
b482e960 | 1525 | atomic_read_relaxed(&udpif->flow_limit, &flow_limit); |
ac64794a BP |
1526 | dump_thread = dpif_flow_dump_thread_create(udpif->dump); |
1527 | for (;;) { | |
6dad4d44 | 1528 | struct ukey_op ops[REVALIDATE_MAX_BATCH]; |
ac64794a | 1529 | int n_ops = 0; |
e79a6c83 | 1530 | |
ac64794a BP |
1531 | struct dpif_flow flows[REVALIDATE_MAX_BATCH]; |
1532 | const struct dpif_flow *f; | |
1533 | int n_dumped; | |
7d170098 | 1534 | |
ac64794a BP |
1535 | long long int max_idle; |
1536 | long long int now; | |
1537 | size_t n_dp_flows; | |
1538 | bool kill_them_all; | |
e79a6c83 | 1539 | |
ac64794a BP |
1540 | n_dumped = dpif_flow_dump_next(dump_thread, flows, ARRAY_SIZE(flows)); |
1541 | if (!n_dumped) { | |
1542 | break; | |
73a3c475 JS |
1543 | } |
1544 | ||
ac64794a BP |
1545 | now = time_msec(); |
1546 | ||
1547 | /* In normal operation we want to keep flows around until they have | |
1548 | * been idle for 'ofproto_max_idle' milliseconds. However: | |
1549 | * | |
1550 | * - If the number of datapath flows climbs above 'flow_limit', | |
1551 | * drop that down to 100 ms to try to bring the flows down to | |
1552 | * the limit. | |
1553 | * | |
1554 | * - If the number of datapath flows climbs above twice | |
1555 | * 'flow_limit', delete all the datapath flows as an emergency | |
1556 | * measure. (We reassess this condition for the next batch of | |
1557 | * datapath flows, so we will recover before all the flows are | |
1558 | * gone.) */ | |
1559 | n_dp_flows = udpif_get_n_flows(udpif); | |
1560 | kill_them_all = n_dp_flows > flow_limit * 2; | |
1561 | max_idle = n_dp_flows > flow_limit ? 100 : ofproto_max_idle; | |
1562 | ||
1563 | for (f = flows; f < &flows[n_dumped]; f++) { | |
1564 | long long int used = f->stats.used; | |
feca8bd7 | 1565 | struct udpif_key *ukey; |
efa08531 | 1566 | bool already_dumped, keep; |
acaa8dac | 1567 | |
feca8bd7 JS |
1568 | if (!ukey_acquire(udpif, f->key, f->key_len, used, &ukey)) { |
1569 | /* We couldn't acquire the ukey. This means that | |
1570 | * another revalidator is processing this flow | |
1571 | * concurrently, so don't bother processing it. */ | |
ec47af51 | 1572 | COVERAGE_INC(dumped_duplicate_flow); |
acaa8dac JS |
1573 | continue; |
1574 | } | |
1575 | ||
efa08531 | 1576 | already_dumped = ukey->dump_seq == dump_seq; |
acaa8dac | 1577 | if (already_dumped) { |
ec47af51 JS |
1578 | /* The flow has already been handled during this flow dump |
1579 | * operation. Skip it. */ | |
1580 | if (ukey->xcache) { | |
1581 | COVERAGE_INC(dumped_duplicate_flow); | |
1582 | } else { | |
1583 | COVERAGE_INC(dumped_new_flow); | |
1584 | } | |
acaa8dac JS |
1585 | ovs_mutex_unlock(&ukey->mutex); |
1586 | continue; | |
1587 | } | |
1588 | ||
1589 | if (!used) { | |
1590 | used = ukey->created; | |
1591 | } | |
ac64794a | 1592 | if (kill_them_all || (used && used < now - max_idle)) { |
efa08531 | 1593 | keep = false; |
ac64794a | 1594 | } else { |
efa08531 | 1595 | keep = revalidate_ukey(udpif, ukey, f); |
ac64794a | 1596 | } |
efa08531 JS |
1597 | ukey->dump_seq = dump_seq; |
1598 | ukey->flow_exists = keep; | |
e79a6c83 | 1599 | |
efa08531 | 1600 | if (!keep) { |
6dad4d44 | 1601 | delete_op_init(&ops[n_ops++], f->key, f->key_len, ukey); |
ac64794a | 1602 | } |
acaa8dac | 1603 | ovs_mutex_unlock(&ukey->mutex); |
7d170098 | 1604 | } |
ad3415c0 | 1605 | |
ac64794a | 1606 | if (n_ops) { |
6dad4d44 | 1607 | push_ukey_ops__(udpif, ops, n_ops); |
7d170098 | 1608 | } |
9fce0584 | 1609 | ovsrcu_quiesce(); |
e79a6c83 | 1610 | } |
ac64794a | 1611 | dpif_flow_dump_thread_destroy(dump_thread); |
e79a6c83 EJ |
1612 | } |
1613 | ||
3b62a9d3 JS |
1614 | /* Called with exclusive access to 'revalidator' and 'ukey'. */ |
1615 | static bool | |
1616 | handle_missed_revalidation(struct revalidator *revalidator, | |
1617 | struct udpif_key *ukey) | |
3b62a9d3 JS |
1618 | { |
1619 | struct udpif *udpif = revalidator->udpif; | |
1620 | struct dpif_flow flow; | |
6fe09f8c JS |
1621 | struct ofpbuf buf; |
1622 | uint64_t stub[DPIF_FLOW_BUFSIZE / 8]; | |
3b62a9d3 JS |
1623 | bool keep = false; |
1624 | ||
1625 | COVERAGE_INC(revalidate_missed_dp_flow); | |
1626 | ||
6fe09f8c | 1627 | ofpbuf_use_stub(&buf, &stub, sizeof stub); |
3b62a9d3 | 1628 | if (!dpif_flow_get(udpif->dpif, ukey->key, ukey->key_len, &buf, &flow)) { |
a2606936 | 1629 | ovs_mutex_lock(&ukey->mutex); |
3b62a9d3 | 1630 | keep = revalidate_ukey(udpif, ukey, &flow); |
a2606936 | 1631 | ovs_mutex_unlock(&ukey->mutex); |
3b62a9d3 | 1632 | } |
6fe09f8c | 1633 | ofpbuf_uninit(&buf); |
3b62a9d3 JS |
1634 | |
1635 | return keep; | |
1636 | } | |
1637 | ||
e79a6c83 | 1638 | static void |
e96a5c24 | 1639 | revalidator_sweep__(struct revalidator *revalidator, bool purge) |
e79a6c83 | 1640 | { |
b8d3daeb | 1641 | struct udpif *udpif; |
efa08531 | 1642 | uint64_t dump_seq; |
b8d3daeb | 1643 | int slice; |
e4b79342 | 1644 | |
b8d3daeb JS |
1645 | udpif = revalidator->udpif; |
1646 | dump_seq = seq_read(udpif->dump_seq); | |
1647 | slice = revalidator - udpif->revalidators; | |
1648 | ovs_assert(slice < udpif->n_revalidators); | |
1649 | ||
1650 | for (int i = slice; i < N_UMAPS; i += udpif->n_revalidators) { | |
6dad4d44 | 1651 | struct ukey_op ops[REVALIDATE_MAX_BATCH]; |
b8d3daeb JS |
1652 | struct udpif_key *ukey; |
1653 | struct umap *umap = &udpif->ukeys[i]; | |
1654 | size_t n_ops = 0; | |
e79a6c83 | 1655 | |
b8d3daeb JS |
1656 | CMAP_FOR_EACH(ukey, cmap_node, &umap->cmap) { |
1657 | bool flow_exists, seq_mismatch; | |
a2606936 | 1658 | |
b8d3daeb JS |
1659 | ovs_mutex_lock(&ukey->mutex); |
1660 | flow_exists = ukey->flow_exists; | |
1661 | seq_mismatch = (ukey->dump_seq != dump_seq | |
1662 | && revalidator->udpif->need_revalidate); | |
1663 | ovs_mutex_unlock(&ukey->mutex); | |
a2606936 | 1664 | |
b8d3daeb JS |
1665 | if (flow_exists |
1666 | && (purge | |
1667 | || (seq_mismatch | |
1668 | && !handle_missed_revalidation(revalidator, ukey)))) { | |
6dad4d44 | 1669 | struct ukey_op *op = &ops[n_ops++]; |
e4b79342 | 1670 | |
6dad4d44 | 1671 | delete_op_init(op, ukey->key, ukey->key_len, ukey); |
b8d3daeb | 1672 | if (n_ops == REVALIDATE_MAX_BATCH) { |
6dad4d44 | 1673 | push_ukey_ops(udpif, umap, ops, n_ops); |
b8d3daeb JS |
1674 | n_ops = 0; |
1675 | } | |
1676 | } else if (!flow_exists) { | |
1677 | ovs_mutex_lock(&umap->mutex); | |
1678 | ukey_delete(umap, ukey); | |
1679 | ovs_mutex_unlock(&umap->mutex); | |
e4b79342 | 1680 | } |
e79a6c83 | 1681 | } |
e4b79342 | 1682 | |
b8d3daeb | 1683 | if (n_ops) { |
6dad4d44 | 1684 | push_ukey_ops(udpif, umap, ops, n_ops); |
b8d3daeb JS |
1685 | } |
1686 | ovsrcu_quiesce(); | |
e4b79342 | 1687 | } |
e1ec7dd4 | 1688 | } |
e96a5c24 JS |
1689 | |
1690 | static void | |
1691 | revalidator_sweep(struct revalidator *revalidator) | |
1692 | { | |
1693 | revalidator_sweep__(revalidator, false); | |
1694 | } | |
1695 | ||
1696 | static void | |
1697 | revalidator_purge(struct revalidator *revalidator) | |
1698 | { | |
1699 | revalidator_sweep__(revalidator, true); | |
1700 | } | |
e22d52ee EJ |
1701 | \f |
1702 | static void | |
1703 | upcall_unixctl_show(struct unixctl_conn *conn, int argc OVS_UNUSED, | |
1704 | const char *argv[] OVS_UNUSED, void *aux OVS_UNUSED) | |
1705 | { | |
1706 | struct ds ds = DS_EMPTY_INITIALIZER; | |
1707 | struct udpif *udpif; | |
1708 | ||
1709 | LIST_FOR_EACH (udpif, list_node, &all_udpifs) { | |
e79a6c83 | 1710 | unsigned int flow_limit; |
e22d52ee EJ |
1711 | size_t i; |
1712 | ||
b482e960 | 1713 | atomic_read_relaxed(&udpif->flow_limit, &flow_limit); |
e79a6c83 | 1714 | |
e22d52ee | 1715 | ds_put_format(&ds, "%s:\n", dpif_name(udpif->dpif)); |
0e2a9f6f | 1716 | ds_put_format(&ds, "\tflows : (current %lu)" |
e79a6c83 EJ |
1717 | " (avg %u) (max %u) (limit %u)\n", udpif_get_n_flows(udpif), |
1718 | udpif->avg_n_flows, udpif->max_n_flows, flow_limit); | |
e79a6c83 | 1719 | ds_put_format(&ds, "\tdump duration : %lldms\n", udpif->dump_duration); |
e79a6c83 | 1720 | ds_put_char(&ds, '\n'); |
b8d3daeb | 1721 | |
e79a6c83 EJ |
1722 | for (i = 0; i < n_revalidators; i++) { |
1723 | struct revalidator *revalidator = &udpif->revalidators[i]; | |
b8d3daeb | 1724 | int j, elements = 0; |
e79a6c83 | 1725 | |
b8d3daeb JS |
1726 | for (j = i; j < N_UMAPS; j += n_revalidators) { |
1727 | elements += cmap_count(&udpif->ukeys[j].cmap); | |
1728 | } | |
1729 | ds_put_format(&ds, "\t%u: (keys %d)\n", revalidator->id, elements); | |
e79a6c83 | 1730 | } |
e22d52ee EJ |
1731 | } |
1732 | ||
1733 | unixctl_command_reply(conn, ds_cstr(&ds)); | |
1734 | ds_destroy(&ds); | |
1735 | } | |
e79a6c83 EJ |
1736 | |
1737 | /* Disable using the megaflows. | |
1738 | * | |
1739 | * This command is only needed for advanced debugging, so it's not | |
1740 | * documented in the man page. */ | |
1741 | static void | |
1742 | upcall_unixctl_disable_megaflows(struct unixctl_conn *conn, | |
1743 | int argc OVS_UNUSED, | |
1744 | const char *argv[] OVS_UNUSED, | |
1745 | void *aux OVS_UNUSED) | |
1746 | { | |
b482e960 | 1747 | atomic_store_relaxed(&enable_megaflows, false); |
1b5b5071 | 1748 | udpif_flush_all_datapaths(); |
e79a6c83 EJ |
1749 | unixctl_command_reply(conn, "megaflows disabled"); |
1750 | } | |
1751 | ||
1752 | /* Re-enable using megaflows. | |
1753 | * | |
1754 | * This command is only needed for advanced debugging, so it's not | |
1755 | * documented in the man page. */ | |
1756 | static void | |
1757 | upcall_unixctl_enable_megaflows(struct unixctl_conn *conn, | |
1758 | int argc OVS_UNUSED, | |
1759 | const char *argv[] OVS_UNUSED, | |
1760 | void *aux OVS_UNUSED) | |
1761 | { | |
b482e960 | 1762 | atomic_store_relaxed(&enable_megaflows, true); |
1b5b5071 | 1763 | udpif_flush_all_datapaths(); |
e79a6c83 EJ |
1764 | unixctl_command_reply(conn, "megaflows enabled"); |
1765 | } | |
94b8c324 JS |
1766 | |
1767 | /* Set the flow limit. | |
1768 | * | |
1769 | * This command is only needed for advanced debugging, so it's not | |
1770 | * documented in the man page. */ | |
1771 | static void | |
1772 | upcall_unixctl_set_flow_limit(struct unixctl_conn *conn, | |
1773 | int argc OVS_UNUSED, | |
1774 | const char *argv[] OVS_UNUSED, | |
1775 | void *aux OVS_UNUSED) | |
1776 | { | |
1777 | struct ds ds = DS_EMPTY_INITIALIZER; | |
1778 | struct udpif *udpif; | |
1779 | unsigned int flow_limit = atoi(argv[1]); | |
1780 | ||
1781 | LIST_FOR_EACH (udpif, list_node, &all_udpifs) { | |
b482e960 | 1782 | atomic_store_relaxed(&udpif->flow_limit, flow_limit); |
94b8c324 JS |
1783 | } |
1784 | ds_put_format(&ds, "set flow_limit to %u\n", flow_limit); | |
1785 | unixctl_command_reply(conn, ds_cstr(&ds)); | |
1786 | ds_destroy(&ds); | |
1787 | } | |
27f57736 JS |
1788 | |
1789 | static void | |
1790 | upcall_unixctl_dump_wait(struct unixctl_conn *conn, | |
1791 | int argc OVS_UNUSED, | |
1792 | const char *argv[] OVS_UNUSED, | |
1793 | void *aux OVS_UNUSED) | |
1794 | { | |
1795 | if (list_is_singleton(&all_udpifs)) { | |
d72eff6c | 1796 | struct udpif *udpif = NULL; |
27f57736 JS |
1797 | size_t len; |
1798 | ||
1799 | udpif = OBJECT_CONTAINING(list_front(&all_udpifs), udpif, list_node); | |
1800 | len = (udpif->n_conns + 1) * sizeof *udpif->conns; | |
1801 | udpif->conn_seq = seq_read(udpif->dump_seq); | |
1802 | udpif->conns = xrealloc(udpif->conns, len); | |
1803 | udpif->conns[udpif->n_conns++] = conn; | |
1804 | } else { | |
1805 | unixctl_command_reply_error(conn, "can't wait on multiple udpifs."); | |
1806 | } | |
1807 | } | |
98bb4286 JS |
1808 | |
1809 | static void | |
1810 | upcall_unixctl_purge(struct unixctl_conn *conn, int argc OVS_UNUSED, | |
1811 | const char *argv[] OVS_UNUSED, void *aux OVS_UNUSED) | |
1812 | { | |
1813 | struct udpif *udpif; | |
1814 | ||
1815 | LIST_FOR_EACH (udpif, list_node, &all_udpifs) { | |
1816 | int n; | |
1817 | ||
1818 | for (n = 0; n < udpif->n_revalidators; n++) { | |
1819 | revalidator_purge(&udpif->revalidators[n]); | |
1820 | } | |
1821 | } | |
1822 | unixctl_command_reply(conn, ""); | |
1823 | } |