2 * Copyright (c) 2008, 2009, 2010, 2011, 2012, 2013 Nicira, Inc.
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at:
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
18 * dpif, the DataPath InterFace.
20 * In Open vSwitch terminology, a "datapath" is a flow-based software switch.
21 * A datapath has no intelligence of its own. Rather, it relies entirely on
22 * its client to set up flows. The datapath layer is core to the Open vSwitch
23 * software switch: one could say, without much exaggeration, that everything
24 * in ovs-vswitchd above dpif exists only to make the correct decisions
25 * interacting with dpif.
27 * Typically, the client of a datapath is the software switch module in
28 * "ovs-vswitchd", but other clients can be written. The "ovs-dpctl" utility
29 * is also a (simple) client.
35 * The terms written in quotes below are defined in later sections.
37 * When a datapath "port" receives a packet, it extracts the headers (the
38 * "flow"). If the datapath's "flow table" contains a "flow entry" whose flow
39 * is the same as the packet's, then it executes the "actions" in the flow
40 * entry and increments the flow's statistics. If there is no matching flow
41 * entry, the datapath instead appends the packet to an "upcall" queue.
47 * A datapath has a set of ports that are analogous to the ports on an Ethernet
48 * switch. At the datapath level, each port has the following information
51 * - A name, a short string that must be unique within the host. This is
52 * typically a name that would be familiar to the system administrator,
53 * e.g. "eth0" or "vif1.1", but it is otherwise arbitrary.
55 * - A 32-bit port number that must be unique within the datapath but is
56 * otherwise arbitrary. The port number is the most important identifier
57 * for a port in the datapath interface.
59 * - A type, a short string that identifies the kind of port. On a Linux
60 * host, typical types are "system" (for a network device such as eth0),
61 * "internal" (for a simulated port used to connect to the TCP/IP stack),
62 * and "gre" (for a GRE tunnel).
64 * - A Netlink PID (see "Upcall Queuing and Ordering" below).
66 * The dpif interface has functions for adding and deleting ports. When a
67 * datapath implements these (e.g. as the Linux and netdev datapaths do), then
68 * Open vSwitch's ovs-vswitchd daemon can directly control what ports are used
69 * for switching. Some datapaths might not implement them, or implement them
70 * with restrictions on the types of ports that can be added or removed
71 * (e.g. on ESX), on systems where port membership can only be changed by some
74 * Each datapath must have a port, sometimes called the "local port", whose
75 * name is the same as the datapath itself, with port number 0. The local port
78 * Ports are available as "struct netdev"s. To obtain a "struct netdev *" for
79 * a port named 'name' with type 'port_type', in a datapath of type
80 * 'datapath_type', call netdev_open(name, dpif_port_open_type(datapath_type,
81 * port_type). The netdev can be used to get and set important data related to
84 * - MTU (netdev_get_mtu(), netdev_set_mtu()).
86 * - Ethernet address (netdev_get_etheraddr(), netdev_set_etheraddr()).
88 * - Statistics such as the number of packets and bytes transmitted and
89 * received (netdev_get_stats()).
91 * - Carrier status (netdev_get_carrier()).
93 * - Speed (netdev_get_features()).
95 * - QoS queue configuration (netdev_get_queue(), netdev_set_queue() and
98 * - Arbitrary port-specific configuration parameters (netdev_get_config(),
99 * netdev_set_config()). An example of such a parameter is the IP
100 * endpoint for a GRE tunnel.
106 * The flow table is a hash table of "flow entries". Each flow entry contains:
108 * - A "flow", that is, a summary of the headers in an Ethernet packet. The
109 * flow is the hash key and thus must be unique within the flow table.
110 * Flows are fine-grained entities that include L2, L3, and L4 headers. A
111 * single TCP connection consists of two flows, one in each direction.
113 * In Open vSwitch userspace, "struct flow" is the typical way to describe
114 * a flow, but the datapath interface uses a different data format to
115 * allow ABI forward- and backward-compatibility. datapath/README
116 * describes the rationale and design. Refer to OVS_KEY_ATTR_* and
117 * "struct ovs_key_*" in include/linux/openvswitch.h for details.
118 * lib/odp-util.h defines several functions for working with these flows.
120 * (In case you are familiar with OpenFlow, datapath flows are analogous
121 * to OpenFlow flow matches. The most important difference is that
122 * OpenFlow allows fields to be wildcarded and prioritized, whereas a
123 * datapath's flow table is a hash table so every flow must be
124 * exact-match, thus without priorities.)
126 * - A list of "actions" that tell the datapath what to do with packets
127 * within a flow. Some examples of actions are OVS_ACTION_ATTR_OUTPUT,
128 * which transmits the packet out a port, and OVS_ACTION_ATTR_SET, which
129 * modifies packet headers. Refer to OVS_ACTION_ATTR_* and "struct
130 * ovs_action_*" in include/linux/openvswitch.h for details.
131 * lib/odp-util.h defines several functions for working with datapath
134 * The actions list may be empty. This indicates that nothing should be
135 * done to matching packets, that is, they should be dropped.
137 * (In case you are familiar with OpenFlow, datapath actions are analogous
138 * to OpenFlow actions.)
140 * - Statistics: the number of packets and bytes that the flow has
141 * processed, the last time that the flow processed a packet, and the
142 * union of all the TCP flags in packets processed by the flow. (The
143 * latter is 0 if the flow is not a TCP flow.)
145 * The datapath's client manages the flow table, primarily in reaction to
146 * "upcalls" (see below).
152 * A datapath sometimes needs to notify its client that a packet was received.
153 * The datapath mechanism to do this is called an "upcall".
155 * Upcalls are used in two situations:
157 * - When a packet is received, but there is no matching flow entry in its
158 * flow table (a flow table "miss"), this causes an upcall of type
159 * DPIF_UC_MISS. These are called "miss" upcalls.
161 * - A datapath action of type OVS_ACTION_ATTR_USERSPACE causes an upcall of
162 * type DPIF_UC_ACTION. These are called "action" upcalls.
164 * An upcall contains an entire packet. There is no attempt to, e.g., copy
165 * only as much of the packet as normally needed to make a forwarding decision.
166 * Such an optimization is doable, but experimental prototypes showed it to be
167 * of little benefit because an upcall typically contains the first packet of a
168 * flow, which is usually short (e.g. a TCP SYN). Also, the entire packet can
169 * sometimes really be needed.
171 * After a client reads a given upcall, the datapath is finished with it, that
172 * is, the datapath doesn't maintain any lingering state past that point.
174 * The latency from the time that a packet arrives at a port to the time that
175 * it is received from dpif_recv() is critical in some benchmarks. For
176 * example, if this latency is 1 ms, then a netperf TCP_CRR test, which opens
177 * and closes TCP connections one at a time as quickly as it can, cannot
178 * possibly achieve more than 500 transactions per second, since every
179 * connection consists of two flows with 1-ms latency to set up each one.
181 * To receive upcalls, a client has to enable them with dpif_recv_set(). A
182 * datapath should generally support multiple clients at once (e.g. so that one
183 * may run "ovs-dpctl show" or "ovs-dpctl dump-flows" while "ovs-vswitchd" is
184 * also running) but need not support multiple clients enabling upcalls at
188 * Upcall Queuing and Ordering
189 * ---------------------------
191 * The datapath's client reads upcalls one at a time by calling dpif_recv().
192 * When more than one upcall is pending, the order in which the datapath
193 * presents upcalls to its client is important. The datapath's client does not
194 * directly control this order, so the datapath implementer must take care
197 * The minimal behavior, suitable for initial testing of a datapath
198 * implementation, is that all upcalls are appended to a single queue, which is
199 * delivered to the client in order.
201 * The datapath should ensure that a high rate of upcalls from one particular
202 * port cannot cause upcalls from other sources to be dropped or unreasonably
203 * delayed. Otherwise, one port conducting a port scan or otherwise initiating
204 * high-rate traffic spanning many flows could suppress other traffic.
205 * Ideally, the datapath should present upcalls from each port in a "round
206 * robin" manner, to ensure fairness.
208 * The client has no control over "miss" upcalls and no insight into the
209 * datapath's implementation, so the datapath is entirely responsible for
210 * queuing and delivering them. On the other hand, the datapath has
211 * considerable freedom of implementation. One good approach is to maintain a
212 * separate queue for each port, to prevent any given port's upcalls from
213 * interfering with other ports' upcalls. If this is impractical, then another
214 * reasonable choice is to maintain some fixed number of queues and assign each
215 * port to one of them. Ports assigned to the same queue can then interfere
216 * with each other, but not with ports assigned to different queues. Other
217 * approaches are also possible.
219 * The client has some control over "action" upcalls: it can specify a 32-bit
220 * "Netlink PID" as part of the action. This terminology comes from the Linux
221 * datapath implementation, which uses a protocol called Netlink in which a PID
222 * designates a particular socket and the upcall data is delivered to the
223 * socket's receive queue. Generically, though, a Netlink PID identifies a
224 * queue for upcalls. The basic requirements on the datapath are:
226 * - The datapath must provide a Netlink PID associated with each port. The
227 * client can retrieve the PID with dpif_port_get_pid().
229 * - The datapath must provide a "special" Netlink PID not associated with
230 * any port. dpif_port_get_pid() also provides this PID. (ovs-vswitchd
231 * uses this PID to queue special packets that must not be lost even if a
232 * port is otherwise busy, such as packets used for tunnel monitoring.)
234 * The minimal behavior of dpif_port_get_pid() and the treatment of the Netlink
235 * PID in "action" upcalls is that dpif_port_get_pid() returns a constant value
236 * and all upcalls are appended to a single queue.
238 * The ideal behavior is:
240 * - Each port has a PID that identifies the queue used for "miss" upcalls
241 * on that port. (Thus, if each port has its own queue for "miss"
242 * upcalls, then each port has a different Netlink PID.)
244 * - "miss" upcalls for a given port and "action" upcalls that specify that
245 * port's Netlink PID add their upcalls to the same queue. The upcalls
246 * are delivered to the datapath's client in the order that the packets
247 * were received, regardless of whether the upcalls are "miss" or "action"
250 * - Upcalls that specify the "special" Netlink PID are queued separately.
256 * The datapath interface works with packets in a particular form. This is the
257 * form taken by packets received via upcalls (i.e. by dpif_recv()). Packets
258 * supplied to the datapath for processing (i.e. to dpif_execute()) also take
261 * A VLAN tag is represented by an 802.1Q header. If the layer below the
262 * datapath interface uses another representation, then the datapath interface
263 * must perform conversion.
265 * The datapath interface requires all packets to fit within the MTU. Some
266 * operating systems internally process packets larger than MTU, with features
267 * such as TSO and UFO. When such a packet passes through the datapath
268 * interface, it must be broken into multiple MTU or smaller sized packets for
269 * presentation as upcalls. (This does not happen often, because an upcall
270 * typically contains the first packet of a flow, which is usually short.)
272 * Some operating system TCP/IP stacks maintain packets in an unchecksummed or
273 * partially checksummed state until transmission. The datapath interface
274 * requires all host-generated packets to be fully checksummed (e.g. IP and TCP
275 * checksums must be correct). On such an OS, the datapath interface must fill
276 * in these checksums.
278 * Packets passed through the datapath interface must be at least 14 bytes
279 * long, that is, they must have a complete Ethernet header. They are not
280 * required to be padded to the minimum Ethernet length.
286 * Typically, the client of a datapath begins by configuring the datapath with
287 * a set of ports. Afterward, the client runs in a loop polling for upcalls to
290 * For each upcall received, the client examines the enclosed packet and
291 * figures out what should be done with it. For example, if the client
292 * implements a MAC-learning switch, then it searches the forwarding database
293 * for the packet's destination MAC and VLAN and determines the set of ports to
294 * which it should be sent. In any case, the client composes a set of datapath
295 * actions to properly dispatch the packet and then directs the datapath to
296 * execute those actions on the packet (e.g. with dpif_execute()).
298 * Most of the time, the actions that the client executed on the packet apply
299 * to every packet with the same flow. For example, the flow includes both
300 * destination MAC and VLAN ID (and much more), so this is true for the
301 * MAC-learning switch example above. In such a case, the client can also
302 * direct the datapath to treat any further packets in the flow in the same
303 * way, using dpif_flow_put() to add a new flow entry.
305 * Other tasks the client might need to perform, in addition to reacting to
308 * - Periodically polling flow statistics, perhaps to supply to its own
311 * - Deleting flow entries from the datapath that haven't been used
312 * recently, to save memory.
314 * - Updating flow entries whose actions should change. For example, if a
315 * MAC learning switch learns that a MAC has moved, then it must update
316 * the actions of flow entries that sent packets to the MAC at its old
319 * - Adding and removing ports to achieve a new configuration.
325 * Most of the dpif functions are fully thread-safe: they may be called from
326 * any number of threads on the same or different dpif objects. The exceptions
329 * - dpif_port_poll() and dpif_port_poll_wait() are conditionally
330 * thread-safe: they may be called from different threads only on
331 * different dpif objects.
333 * - Functions that operate on struct dpif_port_dump or struct
334 * dpif_flow_dump are conditionally thread-safe with respect to those
335 * objects. That is, one may dump ports or flows from any number of
336 * threads at once, but each thread must use its own struct dpif_port_dump
345 #include "openflow/openflow.h"
361 int dp_register_provider(const struct dpif_class
*);
362 int dp_unregister_provider(const char *type
);
363 void dp_blacklist_provider(const char *type
);
364 void dp_enumerate_types(struct sset
*types
);
365 const char *dpif_normalize_type(const char *);
367 int dp_enumerate_names(const char *type
, struct sset
*names
);
368 void dp_parse_name(const char *datapath_name
, char **name
, char **type
);
370 int dpif_open(const char *name
, const char *type
, struct dpif
**);
371 int dpif_create(const char *name
, const char *type
, struct dpif
**);
372 int dpif_create_and_open(const char *name
, const char *type
, struct dpif
**);
373 void dpif_close(struct dpif
*);
375 void dpif_run(struct dpif
*);
376 void dpif_wait(struct dpif
*);
378 const char *dpif_name(const struct dpif
*);
379 const char *dpif_base_name(const struct dpif
*);
380 const char *dpif_type(const struct dpif
*);
382 int dpif_delete(struct dpif
*);
384 /* Statistics for a dpif as a whole. */
385 struct dpif_dp_stats
{
386 uint64_t n_hit
; /* Number of flow table matches. */
387 uint64_t n_missed
; /* Number of flow table misses. */
388 uint64_t n_lost
; /* Number of misses not sent to userspace. */
389 uint64_t n_flows
; /* Number of flows present. */
390 uint64_t n_masks
; /* Number of mega flow masks. */
391 uint64_t n_mask_hit
; /* Number of mega flow masks visited for
392 flow table matches. */
394 int dpif_get_dp_stats(const struct dpif
*, struct dpif_dp_stats
*);
397 /* Port operations. */
399 const char *dpif_port_open_type(const char *datapath_type
,
400 const char *port_type
);
401 int dpif_port_add(struct dpif
*, struct netdev
*, odp_port_t
*port_nop
);
402 int dpif_port_del(struct dpif
*, odp_port_t port_no
);
404 /* A port within a datapath.
406 * 'name' and 'type' are suitable for passing to netdev_open(). */
408 char *name
; /* Network device name, e.g. "eth0". */
409 char *type
; /* Network device type, e.g. "system". */
410 odp_port_t port_no
; /* Port number within datapath. */
412 void dpif_port_clone(struct dpif_port
*, const struct dpif_port
*);
413 void dpif_port_destroy(struct dpif_port
*);
414 bool dpif_port_exists(const struct dpif
*dpif
, const char *devname
);
415 int dpif_port_query_by_number(const struct dpif
*, odp_port_t port_no
,
417 int dpif_port_query_by_name(const struct dpif
*, const char *devname
,
419 int dpif_port_get_name(struct dpif
*, odp_port_t port_no
,
420 char *name
, size_t name_size
);
421 uint32_t dpif_get_max_ports(const struct dpif
*);
422 uint32_t dpif_port_get_pid(const struct dpif
*, odp_port_t port_no
);
424 struct dpif_port_dump
{
425 const struct dpif
*dpif
;
429 void dpif_port_dump_start(struct dpif_port_dump
*, const struct dpif
*);
430 bool dpif_port_dump_next(struct dpif_port_dump
*, struct dpif_port
*);
431 int dpif_port_dump_done(struct dpif_port_dump
*);
433 /* Iterates through each DPIF_PORT in DPIF, using DUMP as state.
435 * Arguments all have pointer type.
437 * If you break out of the loop, then you need to free the dump structure by
438 * hand using dpif_port_dump_done(). */
439 #define DPIF_PORT_FOR_EACH(DPIF_PORT, DUMP, DPIF) \
440 for (dpif_port_dump_start(DUMP, DPIF); \
441 (dpif_port_dump_next(DUMP, DPIF_PORT) \
443 : (dpif_port_dump_done(DUMP), false)); \
446 int dpif_port_poll(const struct dpif
*, char **devnamep
);
447 void dpif_port_poll_wait(const struct dpif
*);
449 /* Flow table operations. */
451 struct dpif_flow_stats
{
458 void dpif_flow_stats_extract(const struct flow
*, const struct ofpbuf
*packet
,
459 long long int used
, struct dpif_flow_stats
*);
460 void dpif_flow_stats_format(const struct dpif_flow_stats
*, struct ds
*);
462 enum dpif_flow_put_flags
{
463 DPIF_FP_CREATE
= 1 << 0, /* Allow creating a new flow. */
464 DPIF_FP_MODIFY
= 1 << 1, /* Allow modifying an existing flow. */
465 DPIF_FP_ZERO_STATS
= 1 << 2 /* Zero the stats of an existing flow. */
468 int dpif_flow_flush(struct dpif
*);
469 int dpif_flow_put(struct dpif
*, enum dpif_flow_put_flags
,
470 const struct nlattr
*key
, size_t key_len
,
471 const struct nlattr
*mask
, size_t mask_len
,
472 const struct nlattr
*actions
, size_t actions_len
,
473 struct dpif_flow_stats
*);
474 int dpif_flow_del(struct dpif
*,
475 const struct nlattr
*key
, size_t key_len
,
476 struct dpif_flow_stats
*);
477 int dpif_flow_get(const struct dpif
*,
478 const struct nlattr
*key
, size_t key_len
,
479 struct ofpbuf
**actionsp
, struct dpif_flow_stats
*);
481 struct dpif_flow_dump
{
482 const struct dpif
*dpif
;
486 void dpif_flow_dump_start(struct dpif_flow_dump
*, const struct dpif
*);
487 bool dpif_flow_dump_next(struct dpif_flow_dump
*,
488 const struct nlattr
**key
, size_t *key_len
,
489 const struct nlattr
**mask
, size_t *mask_len
,
490 const struct nlattr
**actions
, size_t *actions_len
,
491 const struct dpif_flow_stats
**);
492 int dpif_flow_dump_done(struct dpif_flow_dump
*);
494 /* Packet operations. */
496 int dpif_execute(struct dpif
*,
497 const struct nlattr
*key
, size_t key_len
,
498 const struct nlattr
*actions
, size_t actions_len
,
499 const struct ofpbuf
*,
502 /* Operation batching interface.
504 * Some datapaths are faster at performing N operations together than the same
505 * N operations individually, hence an interface for batching.
509 DPIF_OP_FLOW_PUT
= 1,
514 struct dpif_flow_put
{
516 enum dpif_flow_put_flags flags
; /* DPIF_FP_*. */
517 const struct nlattr
*key
; /* Flow to put. */
518 size_t key_len
; /* Length of 'key' in bytes. */
519 const struct nlattr
*mask
; /* Mask to put. */
520 size_t mask_len
; /* Length of 'mask' in bytes. */
521 const struct nlattr
*actions
; /* Actions to perform on flow. */
522 size_t actions_len
; /* Length of 'actions' in bytes. */
525 struct dpif_flow_stats
*stats
; /* Optional flow statistics. */
528 struct dpif_flow_del
{
530 const struct nlattr
*key
; /* Flow to delete. */
531 size_t key_len
; /* Length of 'key' in bytes. */
534 struct dpif_flow_stats
*stats
; /* Optional flow statistics. */
537 struct dpif_execute
{
538 /* Raw support for execute passed along to the provider. */
539 const struct nlattr
*key
; /* Partial flow key (only for metadata). */
540 size_t key_len
; /* Length of 'key' in bytes. */
541 const struct nlattr
*actions
; /* Actions to execute on packet. */
542 size_t actions_len
; /* Length of 'actions' in bytes. */
543 const struct ofpbuf
*packet
; /* Packet to execute. */
545 /* Some dpif providers do not implement every action. The Linux kernel
546 * datapath, in particular, does not implement ARP field modification.
548 * If this member is set to true, the dpif layer executes in userspace all
549 * of the actions that it can, and for OVS_ACTION_ATTR_OUTPUT and
550 * OVS_ACTION_ATTR_USERSPACE actions it passes the packet through to the
551 * dpif implementation. */
556 enum dpif_op_type type
;
559 struct dpif_flow_put flow_put
;
560 struct dpif_flow_del flow_del
;
561 struct dpif_execute execute
;
565 void dpif_operate(struct dpif
*, struct dpif_op
**ops
, size_t n_ops
);
569 enum dpif_upcall_type
{
570 DPIF_UC_MISS
, /* Miss in flow table. */
571 DPIF_UC_ACTION
, /* OVS_ACTION_ATTR_USERSPACE action. */
575 const char *dpif_upcall_type_to_string(enum dpif_upcall_type
);
577 /* A packet passed up from the datapath to userspace.
579 * If 'key', 'actions', or 'userdata' is nonnull, then it points into data
580 * owned by 'packet', so their memory cannot be freed separately. (This is
581 * hardly a great way to do things but it works out OK for the dpif providers
582 * and clients that exist so far.)
586 enum dpif_upcall_type type
;
587 struct ofpbuf
*packet
; /* Packet data. */
588 struct nlattr
*key
; /* Flow key. */
589 size_t key_len
; /* Length of 'key' in bytes. */
591 /* DPIF_UC_ACTION only. */
592 struct nlattr
*userdata
; /* Argument to OVS_ACTION_ATTR_USERSPACE. */
595 int dpif_recv_set(struct dpif
*, bool enable
);
596 int dpif_recv(struct dpif
*, struct dpif_upcall
*, struct ofpbuf
*);
597 void dpif_recv_purge(struct dpif
*);
598 void dpif_recv_wait(struct dpif
*);
602 void dpif_get_netflow_ids(const struct dpif
*,
603 uint8_t *engine_type
, uint8_t *engine_id
);
605 int dpif_queue_to_priority(const struct dpif
*, uint32_t queue_id
,