]> git.proxmox.com Git - mirror_ovs.git/blame - datapath/datapath.c
projects / mirror_ovs.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
datapath: hide clang frame-overflow warnings
[mirror_ovs.git] / datapath / datapath.c
CommitLineData
064af421 1/*
e23775f2 2 * Copyright (c) 2007-2015 Nicira, Inc.
a14bc59f 3 *
a9a29d22
JG		 4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of version 2 of the GNU General Public
6 * License as published by the Free Software Foundation.
7 *
8 * This program is distributed in the hope that it will be useful, but
9 * WITHOUT ANY WARRANTY; without even the implied warranty of
10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
11 * General Public License for more details.
12 *
13 * You should have received a copy of the GNU General Public License
14 * along with this program; if not, write to the Free Software
15 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
16 * 02110-1301, USA
064af421
BP		 17 */
18
dfffaef1
JP		 19#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20
064af421
BP		 21#include <linux/init.h>
22#include <linux/module.h>
064af421 23#include <linux/if_arp.h>
064af421
BP		 24#include <linux/if_vlan.h>
25#include <linux/in.h>
26#include <linux/ip.h>
982b8810 27#include <linux/jhash.h>
064af421
BP		 28#include <linux/delay.h>
29#include <linux/time.h>
30#include <linux/etherdevice.h>
ed099e92 31#include <linux/genetlink.h>
064af421
BP		 32#include <linux/kernel.h>
33#include <linux/kthread.h>
064af421
BP		 34#include <linux/mutex.h>
35#include <linux/percpu.h>
36#include <linux/rcupdate.h>
37#include <linux/tcp.h>
38#include <linux/udp.h>
39#include <linux/version.h>
40#include <linux/ethtool.h>
064af421 41#include <linux/wait.h>
064af421 42#include <asm/div64.h>
656a0e37 43#include <linux/highmem.h>
064af421
BP		 44#include <linux/netfilter_bridge.h>
45#include <linux/netfilter_ipv4.h>
46#include <linux/inetdevice.h>
47#include <linux/list.h>
077257b8 48#include <linux/openvswitch.h>
064af421 49#include <linux/rculist.h>
064af421 50#include <linux/dmi.h>
36956a7d 51#include <net/genetlink.h>
2a4999f3
PS		 52#include <net/net_namespace.h>
53#include <net/netns/generic.h>
907c26a8 54#include <net/nsh.h>
064af421 55
064af421 56#include "datapath.h"
038e34ab 57#include "conntrack.h"
064af421 58#include "flow.h"
d103f479 59#include "flow_table.h"
a097c0b2 60#include "flow_netlink.h"
1cb57039 61#include "meter.h"
e23775f2 62#include "gso.h"
f2459fe7 63#include "vport-internal_dev.h"
d5de5b0d 64#include "vport-netdev.h"
064af421 65
f56f0b73 66unsigned int ovs_net_id __read_mostly;
2a4999f3 67
cb25142c
PS		 68static struct genl_family dp_packet_genl_family;
69static struct genl_family dp_flow_genl_family;
70static struct genl_family dp_datapath_genl_family;
71
bc619e29
JS		 72static const struct nla_policy flow_policy[];
73
18fd3a52
PS		 74static struct genl_multicast_group ovs_dp_flow_multicast_group = {
75 .name = OVS_FLOW_MCGROUP
cb25142c
PS		 76};
77
18fd3a52
PS		 78static struct genl_multicast_group ovs_dp_datapath_multicast_group = {
79 .name = OVS_DATAPATH_MCGROUP
cb25142c
PS		 80};
81
18fd3a52
PS		 82struct genl_multicast_group ovs_dp_vport_multicast_group = {
83 .name = OVS_VPORT_MCGROUP
cb25142c
PS		 84};
85
afad3556 86/* Check if need to build a reply message.
af465b67
PS		 87 * OVS userspace sets the NLM_F_ECHO flag if it needs the reply.
88 */
114fce23
SG		 89static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
90 unsigned int group)
afad3556
JR		 91{
92 return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
6233a1bd 93 genl_has_listeners(family, genl_info_net(info), group);
afad3556
JR		 94}
95
18fd3a52 96static void ovs_notify(struct genl_family *family, struct genl_multicast_group *grp,
cb25142c 97 struct sk_buff *skb, struct genl_info *info)
e297c6b7 98{
0643a78b 99 genl_notify(family, skb, info, GROUP_ID(grp), GFP_KERNEL);
e297c6b7
TG		 100}
101
ed099e92
BP		 102/**
103 * DOC: Locking:
064af421 104 *
cd2a59e9
PS		 105 * All writes e.g. Writes to device state (add/remove datapath, port, set
106 * operations on vports, etc.), Writes to other state (flow table
107 * modifications, set miscellaneous datapath parameters, etc.) are protected
108 * by ovs_lock.
ed099e92
BP		 109 *
110 * Reads are protected by RCU.
111 *
112 * There are a few special cases (mostly stats) that have their own
113 * synchronization but they nest under all of above and don't interact with
114 * each other.
cd2a59e9
PS		 115 *
116 * The RTNL lock nests inside ovs_mutex.
064af421 117 */
ed099e92 118
cd2a59e9
PS		 119static DEFINE_MUTEX(ovs_mutex);
120
121void ovs_lock(void)
122{
123 mutex_lock(&ovs_mutex);
124}
125
126void ovs_unlock(void)
127{
128 mutex_unlock(&ovs_mutex);
129}
130
131#ifdef CONFIG_LOCKDEP
132int lockdep_ovsl_is_held(void)
133{
134 if (debug_locks)
135 return lockdep_is_held(&ovs_mutex);
136 else
137 return 1;
138}
139#endif
140
5ae440c3 141static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
f1f60b85 142 const struct sw_flow_key *,
4c7804f1
WT		 143 const struct dp_upcall_info *,
144 uint32_t cutlen);
5ae440c3 145static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
7d16c847 146 const struct sw_flow_key *,
4c7804f1
WT		 147 const struct dp_upcall_info *,
148 uint32_t cutlen);
064af421 149
cd2a59e9 150/* Must be called with rcu_read_lock or ovs_mutex. */
850b6b3b 151const char *ovs_dp_name(const struct datapath *dp)
f2459fe7 152{
cd2a59e9 153 struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
e23775f2 154 return ovs_vport_name(vport);
f2459fe7
JG		 155}
156
f1f60b85 157static int get_dpifindex(const struct datapath *dp)
99769a40
JG		 158{
159 struct vport *local;
160 int ifindex;
161
162 rcu_read_lock();
163
95b1d73a 164 local = ovs_vport_rcu(dp, OVSP_LOCAL);
99769a40 165 if (local)
e23775f2 166 ifindex = local->dev->ifindex;
99769a40
JG		 167 else
168 ifindex = 0;
169
170 rcu_read_unlock();
171
172 return ifindex;
173}
174
46c6a11d
JG		 175static void destroy_dp_rcu(struct rcu_head *rcu)
176{
177 struct datapath *dp = container_of(rcu, struct datapath, rcu);
46c6a11d 178
e379e4d1 179 ovs_flow_tbl_destroy(&dp->table);
46c6a11d 180 free_percpu(dp->stats_percpu);
95b1d73a 181 kfree(dp->ports);
1cb57039 182 ovs_meters_exit(dp);
5ca1ba48 183 kfree(dp);
46c6a11d
JG		 184}
185
95b1d73a
PS		 186static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
187 u16 port_no)
188{
189 return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
190}
191
aa917006 192/* Called with ovs_mutex or RCU read lock. */
95b1d73a
PS		 193struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
194{
195 struct vport *vport;
95b1d73a
PS		 196 struct hlist_head *head;
197
198 head = vport_hash_bucket(dp, port_no);
f8dfbcb7 199 hlist_for_each_entry_rcu(vport, head, dp_hash_node) {
95b1d73a
PS		 200 if (vport->port_no == port_no)
201 return vport;
202 }
203 return NULL;
204}
205
cd2a59e9 206/* Called with ovs_mutex. */
c19e6535 207static struct vport *new_vport(const struct vport_parms *parms)
064af421 208{
f2459fe7 209 struct vport *vport;
f2459fe7 210
850b6b3b 211 vport = ovs_vport_add(parms);
c19e6535
BP		 212 if (!IS_ERR(vport)) {
213 struct datapath *dp = parms->dp;
95b1d73a 214 struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
064af421 215
95b1d73a 216 hlist_add_head_rcu(&vport->dp_hash_node, head);
c19e6535 217 }
c19e6535 218 return vport;
064af421
BP		 219}
220
850b6b3b 221void ovs_dp_detach_port(struct vport *p)
064af421 222{
cd2a59e9 223 ASSERT_OVSL();
064af421 224
064af421 225 /* First drop references to device. */
95b1d73a 226 hlist_del_rcu(&p->dp_hash_node);
f2459fe7 227
7237e4f4 228 /* Then destroy it. */
850b6b3b 229 ovs_vport_del(p);
064af421
BP		 230}
231
fb66fbd1 232/* Must be called with rcu_read_lock. */
e74d4817 233void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
064af421 234{
a6059080 235 const struct vport *p = OVS_CB(skb)->input_vport;
064af421 236 struct datapath *dp = p->dp;
3544358a 237 struct sw_flow *flow;
ad50cb60 238 struct sw_flow_actions *sf_acts;
064af421 239 struct dp_stats_percpu *stats;
e9141eec 240 u64 *stats_counter;
4fa72a95 241 u32 n_mask_hit;
064af421 242
70dbc259 243 stats = this_cpu_ptr(dp->stats_percpu);
a063b0df 244
52a23d92 245 /* Look up flow. */
e74d4817 246 flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
5604935e 247 &n_mask_hit);
52a23d92
JG		 248 if (unlikely(!flow)) {
249 struct dp_upcall_info upcall;
a7d607c5 250 int error;
52a23d92 251
0e469d3b 252 memset(&upcall, 0, sizeof(upcall));
52a23d92 253 upcall.cmd = OVS_PACKET_CMD_MISS;
beb1c69a 254 upcall.portid = ovs_vport_find_upcall_portid(p, skb);
a94ebc39 255 upcall.mru = OVS_CB(skb)->mru;
4c7804f1 256 error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
a7d607c5
LR		 257 if (unlikely(error))
258 kfree_skb(skb);
259 else
260 consume_skb(skb);
52a23d92
JG		 261 stats_counter = &stats->n_missed;
262 goto out;
263 }
264
e74d4817 265 ovs_flow_stats_update(flow, key->tp.flags, skb);
ad50cb60 266 sf_acts = rcu_dereference(flow->sf_acts);
7d16c847
PS		 267 ovs_execute_actions(dp, skb, sf_acts, key);
268
b0b906cc 269 stats_counter = &stats->n_hit;
55574bb0 270
8819fac7 271out:
55574bb0 272 /* Update datapath statistics. */
b81deb15 273 u64_stats_update_begin(&stats->syncp);
e9141eec 274 (*stats_counter)++;
4fa72a95 275 stats->n_mask_hit += n_mask_hit;
b81deb15 276 u64_stats_update_end(&stats->syncp);
064af421
BP		 277}
278
850b6b3b 279int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
f1f60b85 280 const struct sw_flow_key *key,
4c7804f1
WT		 281 const struct dp_upcall_info *upcall_info,
282 uint32_t cutlen)
aa5a8fdc
JG		 283{
284 struct dp_stats_percpu *stats;
285 int err;
286
28aea917 287 if (upcall_info->portid == 0) {
b063d9f0 288 err = -ENOTCONN;
b063d9f0
JG		 289 goto err;
290 }
291
7257b535 292 if (!skb_is_gso(skb))
4c7804f1 293 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
7257b535 294 else
4c7804f1 295 err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
d76195db
JG		 296 if (err)
297 goto err;
298
299 return 0;
aa5a8fdc 300
aa5a8fdc 301err:
70dbc259 302 stats = this_cpu_ptr(dp->stats_percpu);
aa5a8fdc 303
b81deb15 304 u64_stats_update_begin(&stats->syncp);
aa5a8fdc 305 stats->n_lost++;
b81deb15 306 u64_stats_update_end(&stats->syncp);
aa5a8fdc 307
aa5a8fdc 308 return err;
982b8810
BP		 309}
310
5ae440c3 311static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
f1f60b85 312 const struct sw_flow_key *key,
4c7804f1
WT		 313 const struct dp_upcall_info *upcall_info,
314 uint32_t cutlen)
cb5087ca 315{
969b8e6b 316#ifdef HAVE_SKB_GSO_UDP
f54a7a5d 317 unsigned int gso_type = skb_shinfo(skb)->gso_type;
7257b535 318 struct sw_flow_key later_key;
969b8e6b 319#endif
7257b535 320 struct sk_buff *segs, *nskb;
b2a23c4e 321 struct ovs_skb_cb ovs_cb;
7257b535 322 int err;
cb5087ca 323
b2a23c4e 324 ovs_cb = *OVS_CB(skb);
1d04cd4e 325 segs = __skb_gso_segment(skb, NETIF_F_SG, false);
b2a23c4e 326 *OVS_CB(skb) = ovs_cb;
79089764
PS		 327 if (IS_ERR(segs))
328 return PTR_ERR(segs);
d1da7669
PS		 329 if (segs == NULL)
330 return -EINVAL;
969b8e6b 331#ifdef HAVE_SKB_GSO_UDP
9b277b39 332 if (gso_type & SKB_GSO_UDP) {
c135bba1 333 /* The initial flow key extracted by ovs_flow_key_extract()
9b277b39
PS		 334 * in this case is for a first fragment, so we need to
335 * properly mark later fragments.
336 */
e74d4817 337 later_key = *key;
9b277b39
PS		 338 later_key.ip.frag = OVS_FRAG_TYPE_LATER;
339 }
969b8e6b 340#endif
7257b535
BP		 341 /* Queue all of the segments. */
342 skb = segs;
cb5087ca 343 do {
b2a23c4e 344 *OVS_CB(skb) = ovs_cb;
969b8e6b 345#ifdef HAVE_SKB_GSO_UDP
9b277b39 346 if (gso_type & SKB_GSO_UDP && skb != segs)
e74d4817 347 key = &later_key;
969b8e6b 348#endif
4c7804f1 349 err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
982b8810 350 if (err)
7257b535 351 break;
856081f6 352
36ce148c 353 } while ((skb = skb->next));
cb5087ca 354
7257b535
BP		 355 /* Free all of the segments. */
356 skb = segs;
357 do {
358 nskb = skb->next;
359 if (err)
360 kfree_skb(skb);
361 else
362 consume_skb(skb);
363 } while ((skb = nskb));
364 return err;
365}
366
8b7ea2d4 367static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
6b330a60 368 unsigned int hdrlen, int actions_attrlen)
0afa2373
TG		 369{
370 size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
533bea51 371 + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
039fb36c
WT		 372 + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
373 + nla_total_size(sizeof(unsigned int)); /* OVS_PACKET_ATTR_LEN */
0afa2373
TG		 374
375 /* OVS_PACKET_ATTR_USERDATA */
8b7ea2d4
WZ		 376 if (upcall_info->userdata)
377 size += NLA_ALIGN(upcall_info->userdata->nla_len);
378
379 /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
380 if (upcall_info->egress_tun_info)
381 size += nla_total_size(ovs_tun_key_attr_size());
0afa2373 382
0e469d3b
NM		 383 /* OVS_PACKET_ATTR_ACTIONS */
384 if (upcall_info->actions_len)
6b330a60 385 size += nla_total_size(actions_attrlen);
0e469d3b 386
a94ebc39
JS		 387 /* OVS_PACKET_ATTR_MRU */
388 if (upcall_info->mru)
389 size += nla_total_size(sizeof(upcall_info->mru));
390
0afa2373
TG		 391 return size;
392}
393
a94ebc39
JS		 394static void pad_packet(struct datapath *dp, struct sk_buff *skb)
395{
396 if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
397 size_t plen = NLA_ALIGN(skb->len) - skb->len;
398
399 if (plen > 0)
0ace0a26 400 skb_put_zero(skb, plen);
a94ebc39
JS		 401 }
402}
403
5ae440c3 404static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
f1f60b85 405 const struct sw_flow_key *key,
4c7804f1
WT		 406 const struct dp_upcall_info *upcall_info,
407 uint32_t cutlen)
7257b535
BP		 408{
409 struct ovs_header *upcall;
6161d3fd 410 struct sk_buff *nskb = NULL;
82706a6f 411 struct sk_buff *user_skb = NULL; /* to be queued to userspace */
7257b535 412 struct nlattr *nla;
978188b2 413 size_t len;
533bea51 414 unsigned int hlen;
5ae440c3
TG		 415 int err, dp_ifindex;
416
417 dp_ifindex = get_dpifindex(dp);
418 if (!dp_ifindex)
419 return -ENODEV;
7257b535 420
efd8a18e 421 if (skb_vlan_tag_present(skb)) {
6161d3fd
JG		 422 nskb = skb_clone(skb, GFP_ATOMIC);
423 if (!nskb)
424 return -ENOMEM;
07ac71ea 425
8063e095 426 nskb = __vlan_hwaccel_push_inside(nskb);
07ac71ea
PS		 427 if (!nskb)
428 return -ENOMEM;
429
6161d3fd
JG		 430 skb = nskb;
431 }
432
433 if (nla_attr_size(skb->len) > USHRT_MAX) {
434 err = -EFBIG;
435 goto out;
436 }
7257b535 437
533bea51
TG		 438 /* Complete checksum if needed */
439 if (skb->ip_summed == CHECKSUM_PARTIAL &&
a0c9fedc 440 (err = skb_csum_hwoffload_help(skb, 0)))
533bea51
TG		 441 goto out;
442
443 /* Older versions of OVS user space enforce alignment of the last
444 * Netlink attribute to NLA_ALIGNTO which would require extensive
445 * padding logic. Only perform zerocopy if padding is not required.
446 */
447 if (dp->user_features & OVS_DP_F_UNALIGNED)
448 hlen = skb_zerocopy_headlen(skb);
449 else
450 hlen = skb->len;
451
6b330a60
GR		 452 len = upcall_msg_size(upcall_info, hlen - cutlen,
453 OVS_CB(skb)->acts_origlen);
40c08cda 454 user_skb = genlmsg_new(len, GFP_ATOMIC);
6161d3fd
JG		 455 if (!user_skb) {
456 err = -ENOMEM;
457 goto out;
458 }
7257b535
BP		 459
460 upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
461 0, upcall_info->cmd);
f463bdc9
KL		 462 if (!upcall) {
463 err = -EINVAL;
464 goto out;
465 }
7257b535
BP		 466 upcall->dp_ifindex = dp_ifindex;
467
db7f2238 468 err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
9a621f82 469 BUG_ON(err);
7257b535
BP		 470
471 if (upcall_info->userdata)
e995e3df 472 __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
462a988b 473 nla_len(upcall_info->userdata),
e995e3df 474 nla_data(upcall_info->userdata));
7257b535 475
e23775f2 476
8b7ea2d4
WZ		 477 if (upcall_info->egress_tun_info) {
478 nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_EGRESS_TUN_KEY);
eee4c8eb
KL		 479 if (!nla) {
480 err = -EMSGSIZE;
481 goto out;
482 }
aad7cb91
PS		 483 err = ovs_nla_put_tunnel_info(user_skb,
484 upcall_info->egress_tun_info);
8b7ea2d4
WZ		 485 BUG_ON(err);
486 nla_nest_end(user_skb, nla);
487 }
488
0e469d3b
NM		 489 if (upcall_info->actions_len) {
490 nla = nla_nest_start(user_skb, OVS_PACKET_ATTR_ACTIONS);
eee4c8eb
KL		 491 if (!nla) {
492 err = -EMSGSIZE;
493 goto out;
494 }
0e469d3b
NM		 495 err = ovs_nla_put_actions(upcall_info->actions,
496 upcall_info->actions_len,
497 user_skb);
498 if (!err)
499 nla_nest_end(user_skb, nla);
500 else
501 nla_nest_cancel(user_skb, nla);
502 }
503
a94ebc39
JS		 504 /* Add OVS_PACKET_ATTR_MRU */
505 if (upcall_info->mru) {
506 if (nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU,
507 upcall_info->mru)) {
508 err = -ENOBUFS;
509 goto out;
510 }
511 pad_packet(dp, user_skb);
512 }
513
039fb36c
WT		 514 /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
515 if (cutlen > 0) {
516 if (nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN,
517 skb->len)) {
518 err = -ENOBUFS;
519 goto out;
520 }
521 pad_packet(dp, user_skb);
522 }
523
533bea51 524 /* Only reserve room for attribute header, packet data is added
af465b67
PS		 525 * in skb_zerocopy()
526 */
533bea51
TG		 527 if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
528 err = -ENOBUFS;
529 goto out;
530 }
4c7804f1 531 nla->nla_len = nla_attr_size(skb->len - cutlen);
bed53bd1 532
4c7804f1 533 err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
2c272bd9
ZK		 534 if (err)
535 goto out;
7257b535 536
ef507cec 537 /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
a94ebc39 538 pad_packet(dp, user_skb);
ef507cec 539
533bea51 540 ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
6161d3fd 541
533bea51 542 err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
82706a6f 543 user_skb = NULL;
6161d3fd 544out:
2c272bd9
ZK		 545 if (err)
546 skb_tx_error(skb);
82706a6f 547 kfree_skb(user_skb);
6161d3fd
JG		 548 kfree_skb(nskb);
549 return err;
cb5087ca
BP		 550}
551
df2c07f4 552static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
064af421 553{
df2c07f4 554 struct ovs_header *ovs_header = info->userhdr;
a94ebc39 555 struct net *net = sock_net(skb->sk);
982b8810 556 struct nlattr **a = info->attrs;
e0e57990 557 struct sw_flow_actions *acts;
982b8810 558 struct sk_buff *packet;
e0e57990 559 struct sw_flow *flow;
ad50cb60 560 struct sw_flow_actions *sf_acts;
f7cd0081 561 struct datapath *dp;
a6059080 562 struct vport *input_vport;
a94ebc39 563 u16 mru = 0;
3f19d399 564 int len;
d6569377 565 int err;
2e460098 566 bool log = !a[OVS_PACKET_ATTR_PROBE];
064af421 567
f7cd0081 568 err = -EINVAL;
df2c07f4 569 if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
7c3072cc 570 !a[OVS_PACKET_ATTR_ACTIONS])
e5cad958 571 goto err;
064af421 572
df2c07f4 573 len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
3f19d399 574 packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
f7cd0081
BP		 575 err = -ENOMEM;
576 if (!packet)
e5cad958 577 goto err;
3f19d399
BP		 578 skb_reserve(packet, NET_IP_ALIGN);
579
bf3d6fce 580 nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
8d5ebd83 581
a94ebc39
JS		 582 /* Set packet's mru */
583 if (a[OVS_PACKET_ATTR_MRU]) {
584 mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
585 packet->ignore_df = 1;
586 }
587 OVS_CB(packet)->mru = mru;
588
e0e57990 589 /* Build an sw_flow for sending this packet. */
df65fec1 590 flow = ovs_flow_alloc();
e0e57990
BP		 591 err = PTR_ERR(flow);
592 if (IS_ERR(flow))
e5cad958 593 goto err_kfree_skb;
064af421 594
038e34ab
JS		 595 err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
596 packet, &flow->key, log);
e0e57990 597 if (err)
9321954a 598 goto err_flow_free;
e0e57990 599
a94ebc39 600 err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
9233cef7 601 &flow->key, &acts, log);
9b405f1a
PS		 602 if (err)
603 goto err_flow_free;
e0e57990 604
ff27161e 605 rcu_assign_pointer(flow->sf_acts, acts);
abff858b 606 packet->priority = flow->key.phy.priority;
3025a772 607 packet->mark = flow->key.phy.skb_mark;
e0e57990 608
d6569377 609 rcu_read_lock();
a94ebc39 610 dp = get_dp_rcu(net, ovs_header->dp_ifindex);
f7cd0081 611 err = -ENODEV;
e5cad958
BP		 612 if (!dp)
613 goto err_unlock;
cc4015df 614
a6059080
AZ		 615 input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
616 if (!input_vport)
617 input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
618
619 if (!input_vport)
620 goto err_unlock;
621
e23775f2 622 packet->dev = input_vport->dev;
a6059080 623 OVS_CB(packet)->input_vport = input_vport;
ad50cb60 624 sf_acts = rcu_dereference(flow->sf_acts);
a6059080 625
e9141eec 626 local_bh_disable();
7d16c847 627 err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
e9141eec 628 local_bh_enable();
d6569377 629 rcu_read_unlock();
e0e57990 630
a1c564be 631 ovs_flow_free(flow, false);
e5cad958 632 return err;
064af421 633
e5cad958
BP		 634err_unlock:
635 rcu_read_unlock();
9321954a 636err_flow_free:
a1c564be 637 ovs_flow_free(flow, false);
e5cad958
BP		 638err_kfree_skb:
639 kfree_skb(packet);
640err:
d6569377 641 return err;
064af421
BP		 642}
643
df2c07f4 644static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
7c3072cc 645 [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
df2c07f4
JP		 646 [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
647 [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
2e460098 648 [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
a94ebc39 649 [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
982b8810
BP		 650};
651
18fd3a52 652static struct genl_ops dp_packet_genl_ops[] = {
df2c07f4 653 { .cmd = OVS_PACKET_CMD_EXECUTE,
a6a8674d 654 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
982b8810 655 .policy = packet_policy,
df2c07f4 656 .doit = ovs_packet_cmd_execute
982b8810
BP		 657 }
658};
659
ba63fe26 660static struct genl_family dp_packet_genl_family __ro_after_init = {
cb25142c
PS		 661 .hdrsize = sizeof(struct ovs_header),
662 .name = OVS_PACKET_FAMILY,
663 .version = OVS_PACKET_VERSION,
664 .maxattr = OVS_PACKET_ATTR_MAX,
665 .netnsok = true,
666 .parallel_ops = true,
667 .ops = dp_packet_genl_ops,
668 .n_ops = ARRAY_SIZE(dp_packet_genl_ops),
ba63fe26 669 .module = THIS_MODULE,
cb25142c
PS		 670};
671
f1f60b85 672static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
4fa72a95 673 struct ovs_dp_megaflow_stats *mega_stats)
064af421 674{
d6569377 675 int i;
f180c2e2 676
4fa72a95
AZ		 677 memset(mega_stats, 0, sizeof(*mega_stats));
678
994dc286 679 stats->n_flows = ovs_flow_tbl_count(&dp->table);
4fa72a95 680 mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
064af421 681
7257b535 682 stats->n_hit = stats->n_missed = stats->n_lost = 0;
4fa72a95 683
d6569377
BP		 684 for_each_possible_cpu(i) {
685 const struct dp_stats_percpu *percpu_stats;
686 struct dp_stats_percpu local_stats;
821cb9fa 687 unsigned int start;
44e05eca 688
d6569377 689 percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
064af421 690
d6569377 691 do {
b81deb15 692 start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
d6569377 693 local_stats = *percpu_stats;
b81deb15 694 } while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
064af421 695
d6569377
BP		 696 stats->n_hit += local_stats.n_hit;
697 stats->n_missed += local_stats.n_missed;
698 stats->n_lost += local_stats.n_lost;
4fa72a95 699 mega_stats->n_mask_hit += local_stats.n_mask_hit;
d6569377
BP		 700 }
701}
064af421 702
bc619e29
JS		 703static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
704{
705 return ovs_identifier_is_ufid(sfid) &&
706 !(ufid_flags & OVS_UFID_F_OMIT_KEY);
707}
708
709static bool should_fill_mask(uint32_t ufid_flags)
710{
711 return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
712}
713
714static bool should_fill_actions(uint32_t ufid_flags)
0afa2373 715{
bc619e29
JS		 716 return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
717}
718
719static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
720 const struct sw_flow_id *sfid,
721 uint32_t ufid_flags)
722{
723 size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
724
725 /* OVS_FLOW_ATTR_UFID */
726 if (sfid && ovs_identifier_is_ufid(sfid))
727 len += nla_total_size(sfid->ufid_len);
728
729 /* OVS_FLOW_ATTR_KEY */
730 if (!sfid || should_fill_key(sfid, ufid_flags))
731 len += nla_total_size(ovs_key_attr_size());
732
733 /* OVS_FLOW_ATTR_MASK */
734 if (should_fill_mask(ufid_flags))
735 len += nla_total_size(ovs_key_attr_size());
736
737 /* OVS_FLOW_ATTR_ACTIONS */
738 if (should_fill_actions(ufid_flags))
c3bb15b3 739 len += nla_total_size(acts->orig_len);
bc619e29
JS		 740
741 return len
91b37647 742 + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
0afa2373 743 + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
91b37647 744 + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
0afa2373
TG		 745}
746
f1948bb9
JS		 747/* Called with ovs_mutex or RCU read lock. */
748static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
749 struct sk_buff *skb)
750{
751 struct ovs_flow_stats stats;
752 __be16 tcp_flags;
753 unsigned long used;
754
b0f3a2fe 755 ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
f71db6b1 756
b0f3a2fe 757 if (used &&
89be7da8
PS		 758 nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
759 OVS_FLOW_ATTR_PAD))
f1948bb9 760 return -EMSGSIZE;
d6569377 761
b0f3a2fe 762 if (stats.n_packets &&
91b37647
PS		 763 nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
764 sizeof(struct ovs_flow_stats), &stats,
765 OVS_FLOW_ATTR_PAD))
f1948bb9 766 return -EMSGSIZE;
b0b906cc 767
b0f3a2fe
PS		 768 if ((u8)ntohs(tcp_flags) &&
769 nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
f1948bb9
JS		 770 return -EMSGSIZE;
771
772 return 0;
773}
774
775/* Called with ovs_mutex or RCU read lock. */
776static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
777 struct sk_buff *skb, int skb_orig_len)
778{
779 struct nlattr *start;
780 int err;
d6569377 781
df2c07f4 782 /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
30053024
BP		 783 * this is the first flow to be dumped into 'skb'. This is unusual for
784 * Netlink but individual action lists can be longer than
785 * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
786 * The userspace caller can always fetch the actions separately if it
787 * really wants them. (Most userspace callers in fact don't care.)
788 *
789 * This can only fail for dump operations because the skb is always
790 * properly sized for single flows.
791 */
9b405f1a 792 start = nla_nest_start(skb, OVS_FLOW_ATTR_ACTIONS);
f6f481ee 793 if (start) {
f44ccce1
PS		 794 const struct sw_flow_actions *sf_acts;
795
780ec6ae 796 sf_acts = rcu_dereference_ovsl(flow->sf_acts);
a097c0b2
PS		 797 err = ovs_nla_put_actions(sf_acts->actions,
798 sf_acts->actions_len, skb);
f71db6b1 799
0a25b039
BP		 800 if (!err)
801 nla_nest_end(skb, start);
802 else {
803 if (skb_orig_len)
f1948bb9 804 return err;
0a25b039
BP		 805
806 nla_nest_cancel(skb, start);
807 }
f1948bb9
JS		 808 } else if (skb_orig_len) {
809 return -EMSGSIZE;
810 }
811
812 return 0;
813}
814
815/* Called with ovs_mutex or RCU read lock. */
2c622e5a 816static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
f1948bb9 817 struct sk_buff *skb, u32 portid,
bc619e29 818 u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
f1948bb9
JS		 819{
820 const int skb_orig_len = skb->len;
821 struct ovs_header *ovs_header;
822 int err;
823
7d16c847
PS		 824 ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
825 flags, cmd);
f1948bb9
JS		 826 if (!ovs_header)
827 return -EMSGSIZE;
7d16c847 828
f1948bb9
JS		 829 ovs_header->dp_ifindex = dp_ifindex;
830
bc619e29 831 err = ovs_nla_put_identifier(flow, skb);
db7f2238
JS		 832 if (err)
833 goto error;
834
bc619e29
JS		 835 if (should_fill_key(&flow->id, ufid_flags)) {
836 err = ovs_nla_put_masked_key(flow, skb);
837 if (err)
838 goto error;
839 }
840
841 if (should_fill_mask(ufid_flags)) {
842 err = ovs_nla_put_mask(flow, skb);
843 if (err)
844 goto error;
845 }
f1948bb9
JS		 846
847 err = ovs_flow_cmd_fill_stats(flow, skb);
848 if (err)
849 goto error;
850
bc619e29
JS		 851 if (should_fill_actions(ufid_flags)) {
852 err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
853 if (err)
854 goto error;
855 }
37a1300c 856
23b48dc1
TG		 857 genlmsg_end(skb, ovs_header);
858 return 0;
d6569377 859
37a1300c 860error:
df2c07f4 861 genlmsg_cancel(skb, ovs_header);
d6569377 862 return err;
44e05eca
BP		 863}
864
f71db6b1
JR		 865/* May not be called with RCU read lock. */
866static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
bc619e29 867 const struct sw_flow_id *sfid,
afad3556 868 struct genl_info *info,
bc619e29
JS		 869 bool always,
870 uint32_t ufid_flags)
44e05eca 871{
afad3556 872 struct sk_buff *skb;
bc619e29 873 size_t len;
d6569377 874
114fce23
SG		 875 if (!always && !ovs_must_notify(&dp_flow_genl_family, info,
876 GROUP_ID(&ovs_dp_flow_multicast_group)))
afad3556
JR		 877 return NULL;
878
bc619e29 879 len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
40c08cda 880 skb = genlmsg_new(len, GFP_KERNEL);
afad3556
JR		 881 if (!skb)
882 return ERR_PTR(-ENOMEM);
883
884 return skb;
37a1300c 885}
8d5ebd83 886
f71db6b1 887/* Called with ovs_mutex. */
7d16c847 888static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
f71db6b1
JR		 889 int dp_ifindex,
890 struct genl_info *info, u8 cmd,
bc619e29 891 bool always, u32 ufid_flags)
37a1300c
BP		 892{
893 struct sk_buff *skb;
894 int retval;
d6569377 895
bc619e29
JS		 896 skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
897 &flow->id, info, always, ufid_flags);
a6ddcc9a 898 if (IS_ERR_OR_NULL(skb))
afad3556 899 return skb;
d6569377 900
2c622e5a 901 retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
f71db6b1 902 info->snd_portid, info->snd_seq, 0,
bc619e29 903 cmd, ufid_flags);
37a1300c 904 BUG_ON(retval < 0);
d6569377 905 return skb;
064af421
BP		 906}
907
0c9fd022 908static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
064af421 909{
a94ebc39 910 struct net *net = sock_net(skb->sk);
37a1300c 911 struct nlattr **a = info->attrs;
df2c07f4 912 struct ovs_header *ovs_header = info->userhdr;
bc619e29 913 struct sw_flow *flow = NULL, *new_flow;
a1c564be 914 struct sw_flow_mask mask;
37a1300c 915 struct sk_buff *reply;
9c52546b 916 struct datapath *dp;
0c9fd022 917 struct sw_flow_actions *acts;
a1c564be 918 struct sw_flow_match match;
bc619e29 919 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
bc4a05c6 920 int error;
9233cef7 921 bool log = !a[OVS_FLOW_ATTR_PROBE];
064af421 922
6740b721 923 /* Must have key and actions. */
37a1300c 924 error = -EINVAL;
a473df5b 925 if (!a[OVS_FLOW_ATTR_KEY]) {
7d16c847 926 OVS_NLERR(log, "Flow key attr not present in new flow.");
37a1300c 927 goto error;
a473df5b
JG		 928 }
929 if (!a[OVS_FLOW_ATTR_ACTIONS]) {
7d16c847 930 OVS_NLERR(log, "Flow actions attr not present in new flow.");
6740b721 931 goto error;
a473df5b 932 }
a1c564be 933
6740b721 934 /* Most of the time we need to allocate a new flow, do it before
af465b67
PS		 935 * locking.
936 */
6740b721
JR		 937 new_flow = ovs_flow_alloc();
938 if (IS_ERR(new_flow)) {
939 error = PTR_ERR(new_flow);
940 goto error;
941 }
942
943 /* Extract key. */
9b94fa6c 944 ovs_match_init(&match, &new_flow->key, false, &mask);
038e34ab 945 error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
9233cef7 946 a[OVS_FLOW_ATTR_MASK], log);
37a1300c 947 if (error)
6740b721 948 goto err_kfree_flow;
064af421 949
bc619e29
JS		 950 /* Extract flow identifier. */
951 error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
1d334d4f 952 &new_flow->key, log);
bc619e29
JS		 953 if (error)
954 goto err_kfree_flow;
9b405f1a 955
1d334d4f 956 /* unmasked key is needed to match when ufid is not used. */
957 if (ovs_identifier_is_key(&new_flow->id))
958 match.key = new_flow->id.unmasked_key;
959
960 ovs_flow_mask_key(&new_flow->key, &new_flow->key, true, &mask);
961
6740b721 962 /* Validate actions. */
a94ebc39
JS		 963 error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
964 &new_flow->key, &acts, log);
0c9fd022 965 if (error) {
7d16c847 966 OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
4f67b12a 967 goto err_kfree_flow;
6740b721
JR		 968 }
969
bc619e29
JS		 970 reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
971 ufid_flags);
6740b721
JR		 972 if (IS_ERR(reply)) {
973 error = PTR_ERR(reply);
974 goto err_kfree_acts;
37a1300c
BP		 975 }
976
cd2a59e9 977 ovs_lock();
a94ebc39 978 dp = get_dp(net, ovs_header->dp_ifindex);
6740b721
JR		 979 if (unlikely(!dp)) {
980 error = -ENODEV;
cd2a59e9 981 goto err_unlock_ovs;
6740b721 982 }
bc619e29 983
a1c564be 984 /* Check if this is a duplicate flow */
bc619e29
JS		 985 if (ovs_identifier_is_ufid(&new_flow->id))
986 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
987 if (!flow)
1d334d4f 988 flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->key);
6740b721
JR		 989 if (likely(!flow)) {
990 rcu_assign_pointer(new_flow->sf_acts, acts);
d6569377 991
d6569377 992 /* Put flow in bucket. */
6740b721
JR		 993 error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
994 if (unlikely(error)) {
0585f7a8 995 acts = NULL;
6740b721
JR		 996 goto err_unlock_ovs;
997 }
998
999 if (unlikely(reply)) {
2c622e5a 1000 error = ovs_flow_cmd_fill_info(new_flow,
6740b721
JR		 1001 ovs_header->dp_ifindex,
1002 reply, info->snd_portid,
1003 info->snd_seq, 0,
bc619e29
JS		 1004 OVS_FLOW_CMD_NEW,
1005 ufid_flags);
6740b721 1006 BUG_ON(error < 0);
0585f7a8 1007 }
6740b721 1008 ovs_unlock();
d6569377 1009 } else {
0c9fd022
JR		 1010 struct sw_flow_actions *old_acts;
1011
d6569377
BP		 1012 /* Bail out if we're not allowed to modify an existing flow.
1013 * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
1014 * because Generic Netlink treats the latter as a dump
1015 * request. We also accept NLM_F_EXCL in case that bug ever
1016 * gets fixed.
1017 */
6740b721
JR		 1018 if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
1019 | NLM_F_EXCL))) {
1020 error = -EEXIST;
cd2a59e9 1021 goto err_unlock_ovs;
6740b721 1022 }
bc619e29
JS		 1023 /* The flow identifier has to be the same for flow updates.
1024 * Look for any overlapping flow.
1025 */
1026 if (unlikely(!ovs_flow_cmp(flow, &match))) {
1027 if (ovs_identifier_is_key(&flow->id))
1028 flow = ovs_flow_tbl_lookup_exact(&dp->table,
1029 &match);
1030 else /* UFID matches but key is different */
1031 flow = NULL;
3440e4bc
AW		 1032 if (!flow) {
1033 error = -ENOENT;
1034 goto err_unlock_ovs;
1035 }
6740b721 1036 }
0c9fd022
JR		 1037 /* Update actions. */
1038 old_acts = ovsl_dereference(flow->sf_acts);
1039 rcu_assign_pointer(flow->sf_acts, acts);
0c9fd022 1040
6740b721 1041 if (unlikely(reply)) {
2c622e5a 1042 error = ovs_flow_cmd_fill_info(flow,
6740b721
JR		 1043 ovs_header->dp_ifindex,
1044 reply, info->snd_portid,
1045 info->snd_seq, 0,
bc619e29
JS		 1046 OVS_FLOW_CMD_NEW,
1047 ufid_flags);
6740b721
JR		 1048 BUG_ON(error < 0);
1049 }
1050 ovs_unlock();
0c9fd022 1051
e23775f2 1052 ovs_nla_free_flow_actions_rcu(old_acts);
6740b721 1053 ovs_flow_free(new_flow, false);
0c9fd022 1054 }
6740b721
JR		 1055
1056 if (reply)
cb25142c 1057 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
0c9fd022
JR		 1058 return 0;
1059
0c9fd022
JR		 1060err_unlock_ovs:
1061 ovs_unlock();
6740b721
JR		 1062 kfree_skb(reply);
1063err_kfree_acts:
e23775f2 1064 ovs_nla_free_flow_actions(acts);
6740b721
JR		 1065err_kfree_flow:
1066 ovs_flow_free(new_flow, false);
0c9fd022
JR		 1067error:
1068 return error;
1069}
1070
cc561abf 1071/* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
19b6110d 1072static noinline_for_stack struct sw_flow_actions *get_flow_actions(struct net *net,
a94ebc39 1073 const struct nlattr *a,
cc561abf 1074 const struct sw_flow_key *key,
9233cef7
JR		 1075 const struct sw_flow_mask *mask,
1076 bool log)
cc561abf
PS		 1077{
1078 struct sw_flow_actions *acts;
1079 struct sw_flow_key masked_key;
1080 int error;
1081
ad4adec2 1082 ovs_flow_mask_key(&masked_key, key, true, mask);
a94ebc39 1083 error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
cc561abf 1084 if (error) {
9233cef7 1085 OVS_NLERR(log,
7d16c847 1086 "Actions may not be safe on all matching packets");
cc561abf
PS		 1087 return ERR_PTR(error);
1088 }
1089
1090 return acts;
1091}
1092
850c2a4d
TZ		 1093/* Factor out match-init and action-copy to avoid
1094 * "Wframe-larger-than=1024" warning. Because mask is only
1095 * used to get actions, we new a function to save some
1096 * stack space.
1097 *
1098 * If there are not key and action attrs, we return 0
1099 * directly. In the case, the caller will also not use the
1100 * match as before. If there is action attr, we try to get
1101 * actions and save them to *acts. Before returning from
1102 * the function, we reset the match->mask pointer. Because
1103 * we should not to return match object with dangling reference
1104 * to mask.
1105 * */
19b6110d
AB		 1106static noinline_for_stack int
1107ovs_nla_init_match_and_action(struct net *net,
1108 struct sw_flow_match *match,
1109 struct sw_flow_key *key,
1110 struct nlattr **a,
1111 struct sw_flow_actions **acts,
1112 bool log)
850c2a4d
TZ		 1113{
1114 struct sw_flow_mask mask;
1115 int error = 0;
1116
1117 if (a[OVS_FLOW_ATTR_KEY]) {
1118 ovs_match_init(match, key, true, &mask);
1119 error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
1120 a[OVS_FLOW_ATTR_MASK], log);
1121 if (error)
1122 goto error;
1123 }
1124
1125 if (a[OVS_FLOW_ATTR_ACTIONS]) {
1126 if (!a[OVS_FLOW_ATTR_KEY]) {
1127 OVS_NLERR(log,
1128 "Flow key attribute not present in set flow.");
497895ca
CJ		 1129 error = -EINVAL;
1130 goto error;
850c2a4d
TZ		 1131 }
1132
1133 *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
1134 &mask, log);
1135 if (IS_ERR(*acts)) {
1136 error = PTR_ERR(*acts);
1137 goto error;
1138 }
1139 }
1140
1141 /* On success, error is 0. */
1142error:
1143 match->mask = NULL;
1144 return error;
1145}
1146
0c9fd022
JR		 1147static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
1148{
a94ebc39 1149 struct net *net = sock_net(skb->sk);
0c9fd022
JR		 1150 struct nlattr **a = info->attrs;
1151 struct ovs_header *ovs_header = info->userhdr;
1d2a1b5f 1152 struct sw_flow_key key;
0c9fd022 1153 struct sw_flow *flow;
0c9fd022
JR		 1154 struct sk_buff *reply = NULL;
1155 struct datapath *dp;
6740b721 1156 struct sw_flow_actions *old_acts = NULL, *acts = NULL;
0c9fd022 1157 struct sw_flow_match match;
bc619e29
JS		 1158 struct sw_flow_id sfid;
1159 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
b24baa1a 1160 int error = 0;
9233cef7 1161 bool log = !a[OVS_FLOW_ATTR_PROBE];
bc619e29 1162 bool ufid_present;
0c9fd022 1163
bc619e29 1164 ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
850c2a4d 1165 if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
b24baa1a
PS		 1166 OVS_NLERR(log,
1167 "Flow set message rejected, Key attribute missing.");
850c2a4d 1168 return -EINVAL;
b24baa1a 1169 }
850c2a4d
TZ		 1170
1171 error = ovs_nla_init_match_and_action(net, &match, &key, a,
1172 &acts, log);
0c9fd022
JR		 1173 if (error)
1174 goto error;
d6569377 1175
850c2a4d 1176 if (acts) {
ff27161e 1177 /* Can allocate before locking if have acts. */
bc619e29
JS		 1178 reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
1179 ufid_flags);
6740b721
JR		 1180 if (IS_ERR(reply)) {
1181 error = PTR_ERR(reply);
1182 goto err_kfree_acts;
90b8c2f7 1183 }
0c9fd022
JR		 1184 }
1185
1186 ovs_lock();
a94ebc39 1187 dp = get_dp(net, ovs_header->dp_ifindex);
6740b721
JR		 1188 if (unlikely(!dp)) {
1189 error = -ENODEV;
0c9fd022 1190 goto err_unlock_ovs;
6740b721 1191 }
0c9fd022 1192 /* Check that the flow exists. */
bc619e29
JS		 1193 if (ufid_present)
1194 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
1195 else
1196 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
6740b721
JR		 1197 if (unlikely(!flow)) {
1198 error = -ENOENT;
0c9fd022 1199 goto err_unlock_ovs;
6740b721 1200 }
3440e4bc 1201
0c9fd022 1202 /* Update actions, if present. */
6740b721 1203 if (likely(acts)) {
0c9fd022
JR		 1204 old_acts = ovsl_dereference(flow->sf_acts);
1205 rcu_assign_pointer(flow->sf_acts, acts);
6740b721
JR		 1206
1207 if (unlikely(reply)) {
2c622e5a 1208 error = ovs_flow_cmd_fill_info(flow,
6740b721
JR		 1209 ovs_header->dp_ifindex,
1210 reply, info->snd_portid,
1211 info->snd_seq, 0,
229486c2 1212 OVS_FLOW_CMD_SET,
bc619e29 1213 ufid_flags);
6740b721
JR		 1214 BUG_ON(error < 0);
1215 }
1216 } else {
1217 /* Could not alloc without acts before locking. */
7d16c847 1218 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
229486c2 1219 info, OVS_FLOW_CMD_SET, false,
bc619e29
JS		 1220 ufid_flags);
1221
6740b721
JR		 1222 if (unlikely(IS_ERR(reply))) {
1223 error = PTR_ERR(reply);
1224 goto err_unlock_ovs;
1225 }
9c52546b 1226 }
0c9fd022 1227
0c9fd022
JR		 1228 /* Clear stats. */
1229 if (a[OVS_FLOW_ATTR_CLEAR])
1230 ovs_flow_stats_clear(flow);
cd2a59e9 1231 ovs_unlock();
37a1300c 1232
6740b721 1233 if (reply)
cb25142c 1234 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
6740b721 1235 if (old_acts)
e23775f2 1236 ovs_nla_free_flow_actions_rcu(old_acts);
7d16c847 1237
d6569377 1238 return 0;
704a1e09 1239
cd2a59e9
PS		 1240err_unlock_ovs:
1241 ovs_unlock();
6740b721
JR		 1242 kfree_skb(reply);
1243err_kfree_acts:
e23775f2 1244 ovs_nla_free_flow_actions(acts);
37a1300c 1245error:
9c52546b 1246 return error;
704a1e09
BP		 1247}
1248
df2c07f4 1249static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
704a1e09 1250{
37a1300c 1251 struct nlattr **a = info->attrs;
df2c07f4 1252 struct ovs_header *ovs_header = info->userhdr;
038e34ab 1253 struct net *net = sock_net(skb->sk);
37a1300c 1254 struct sw_flow_key key;
37a1300c 1255 struct sk_buff *reply;
704a1e09 1256 struct sw_flow *flow;
9c52546b 1257 struct datapath *dp;
a1c564be 1258 struct sw_flow_match match;
bc619e29
JS		 1259 struct sw_flow_id ufid;
1260 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
1261 int err = 0;
9233cef7 1262 bool log = !a[OVS_FLOW_ATTR_PROBE];
bc619e29 1263 bool ufid_present;
704a1e09 1264
bc619e29
JS		 1265 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1266 if (a[OVS_FLOW_ATTR_KEY]) {
9b94fa6c 1267 ovs_match_init(&match, &key, true, NULL);
038e34ab 1268 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
bc619e29
JS		 1269 log);
1270 } else if (!ufid_present) {
9233cef7
JR		 1271 OVS_NLERR(log,
1272 "Flow get message rejected, Key attribute missing.");
bc619e29 1273 err = -EINVAL;
1b936472 1274 }
37a1300c
BP		 1275 if (err)
1276 return err;
704a1e09 1277
cd2a59e9 1278 ovs_lock();
2a4999f3 1279 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
cd2a59e9
PS		 1280 if (!dp) {
1281 err = -ENODEV;
1282 goto unlock;
1283 }
704a1e09 1284
bc619e29
JS		 1285 if (ufid_present)
1286 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1287 else
1288 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
3440e4bc 1289 if (!flow) {
cd2a59e9
PS		 1290 err = -ENOENT;
1291 goto unlock;
1292 }
d6569377 1293
7d16c847 1294 reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
229486c2 1295 OVS_FLOW_CMD_GET, true, ufid_flags);
cd2a59e9
PS		 1296 if (IS_ERR(reply)) {
1297 err = PTR_ERR(reply);
1298 goto unlock;
1299 }
36956a7d 1300
cd2a59e9 1301 ovs_unlock();
37a1300c 1302 return genlmsg_reply(reply, info);
cd2a59e9
PS		 1303unlock:
1304 ovs_unlock();
1305 return err;
d6569377 1306}
9c52546b 1307
df2c07f4 1308static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
d6569377 1309{
37a1300c 1310 struct nlattr **a = info->attrs;
df2c07f4 1311 struct ovs_header *ovs_header = info->userhdr;
038e34ab 1312 struct net *net = sock_net(skb->sk);
37a1300c 1313 struct sw_flow_key key;
37a1300c 1314 struct sk_buff *reply;
bc619e29 1315 struct sw_flow *flow = NULL;
d6569377 1316 struct datapath *dp;
a1c564be 1317 struct sw_flow_match match;
bc619e29
JS		 1318 struct sw_flow_id ufid;
1319 u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
d6569377 1320 int err;
9233cef7 1321 bool log = !a[OVS_FLOW_ATTR_PROBE];
bc619e29 1322 bool ufid_present;
36956a7d 1323
bc619e29
JS		 1324 ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
1325 if (a[OVS_FLOW_ATTR_KEY]) {
9b94fa6c 1326 ovs_match_init(&match, &key, true, NULL);
038e34ab
JS		 1327 err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
1328 NULL, log);
cde7f3ba
JR		 1329 if (unlikely(err))
1330 return err;
1331 }
1332
cd2a59e9 1333 ovs_lock();
2a4999f3 1334 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
cde7f3ba 1335 if (unlikely(!dp)) {
cd2a59e9
PS		 1336 err = -ENODEV;
1337 goto unlock;
1338 }
7d16c847 1339
bc619e29 1340 if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
994dc286 1341 err = ovs_flow_tbl_flush(&dp->table);
cd2a59e9
PS		 1342 goto unlock;
1343 }
7d16c847 1344
bc619e29
JS		 1345 if (ufid_present)
1346 flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
1347 else
1348 flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
3440e4bc 1349 if (unlikely(!flow)) {
cd2a59e9
PS		 1350 err = -ENOENT;
1351 goto unlock;
1352 }
d6569377 1353
994dc286 1354 ovs_flow_tbl_remove(&dp->table, flow);
cde7f3ba 1355 ovs_unlock();
37a1300c 1356
46051cf8 1357 reply = ovs_flow_cmd_alloc_info(rcu_dereference_raw(flow->sf_acts),
bc619e29 1358 &flow->id, info, false, ufid_flags);
cde7f3ba
JR		 1359
1360 if (likely(reply)) {
1361 if (likely(!IS_ERR(reply))) {
7d16c847
PS		 1362 rcu_read_lock(); /*To keep RCU checker happy. */
1363 err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
cde7f3ba
JR		 1364 reply, info->snd_portid,
1365 info->snd_seq, 0,
bc619e29
JS		 1366 OVS_FLOW_CMD_DEL,
1367 ufid_flags);
cde7f3ba
JR		 1368 rcu_read_unlock();
1369 BUG_ON(err < 0);
cb25142c 1370 ovs_notify(&dp_flow_genl_family, &ovs_dp_flow_multicast_group, reply, info);
cde7f3ba 1371 } else {
cb25142c
PS		 1372 genl_set_err(&dp_flow_genl_family, sock_net(skb->sk), 0,
1373 GROUP_ID(&ovs_dp_flow_multicast_group), PTR_ERR(reply));
1374
cde7f3ba 1375 }
afad3556 1376 }
37a1300c 1377
a1c564be 1378 ovs_flow_free(flow, true);
37a1300c 1379 return 0;
cd2a59e9
PS		 1380unlock:
1381 ovs_unlock();
1382 return err;
37a1300c
BP		 1383}
1384
df2c07f4 1385static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
37a1300c 1386{
bc619e29 1387 struct nlattr *a[__OVS_FLOW_ATTR_MAX];
df2c07f4 1388 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
994dc286 1389 struct table_instance *ti;
37a1300c 1390 struct datapath *dp;
bc619e29
JS		 1391 u32 ufid_flags;
1392 int err;
1393
1394 err = genlmsg_parse(cb->nlh, &dp_flow_genl_family, a,
15702dc9 1395 OVS_FLOW_ATTR_MAX, flow_policy, NULL);
bc619e29
JS		 1396 if (err)
1397 return err;
1398 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
37a1300c 1399
f44ccce1 1400 rcu_read_lock();
01ac0970 1401 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
cd2a59e9 1402 if (!dp) {
f44ccce1 1403 rcu_read_unlock();
37a1300c 1404 return -ENODEV;
cd2a59e9 1405 }
37a1300c 1406
994dc286 1407 ti = rcu_dereference(dp->table.ti);
37a1300c 1408 for (;;) {
37a1300c
BP		 1409 struct sw_flow *flow;
1410 u32 bucket, obj;
1411
1412 bucket = cb->args[0];
1413 obj = cb->args[1];
994dc286 1414 flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
3544358a 1415 if (!flow)
37a1300c
BP		 1416 break;
1417
2c622e5a 1418 if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
28aea917 1419 NETLINK_CB(cb->skb).portid,
37a1300c 1420 cb->nlh->nlmsg_seq, NLM_F_MULTI,
229486c2 1421 OVS_FLOW_CMD_GET, ufid_flags) < 0)
37a1300c
BP		 1422 break;
1423
1424 cb->args[0] = bucket;
1425 cb->args[1] = obj;
1426 }
f44ccce1 1427 rcu_read_unlock();
37a1300c 1428 return skb->len;
704a1e09
BP		 1429}
1430
cb25142c
PS		 1431static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
1432 [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
9233cef7 1433 [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
cb25142c
PS		 1434 [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
1435 [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
9233cef7 1436 [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
bc619e29
JS		 1437 [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
1438 [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
cb25142c
PS		 1439};
1440
18fd3a52 1441static struct genl_ops dp_flow_genl_ops[] = {
df2c07f4 1442 { .cmd = OVS_FLOW_CMD_NEW,
a6a8674d 1443 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
37a1300c 1444 .policy = flow_policy,
0c9fd022 1445 .doit = ovs_flow_cmd_new
37a1300c 1446 },
df2c07f4 1447 { .cmd = OVS_FLOW_CMD_DEL,
a6a8674d 1448 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
37a1300c 1449 .policy = flow_policy,
df2c07f4 1450 .doit = ovs_flow_cmd_del
37a1300c 1451 },
df2c07f4 1452 { .cmd = OVS_FLOW_CMD_GET,
37a1300c
BP		 1453 .flags = 0, /* OK for unprivileged users. */
1454 .policy = flow_policy,
df2c07f4
JP		 1455 .doit = ovs_flow_cmd_get,
1456 .dumpit = ovs_flow_cmd_dump
37a1300c 1457 },
df2c07f4 1458 { .cmd = OVS_FLOW_CMD_SET,
a6a8674d 1459 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
37a1300c 1460 .policy = flow_policy,
0c9fd022 1461 .doit = ovs_flow_cmd_set,
37a1300c
BP		 1462 },
1463};
1464
ba63fe26 1465static struct genl_family dp_flow_genl_family __ro_after_init = {
df2c07f4 1466 .hdrsize = sizeof(struct ovs_header),
cb25142c
PS		 1467 .name = OVS_FLOW_FAMILY,
1468 .version = OVS_FLOW_VERSION,
1469 .maxattr = OVS_FLOW_ATTR_MAX,
b3dcb73c 1470 .netnsok = true,
cb25142c
PS		 1471 .parallel_ops = true,
1472 .ops = dp_flow_genl_ops,
1473 .n_ops = ARRAY_SIZE(dp_flow_genl_ops),
1474 .mcgrps = &ovs_dp_flow_multicast_group,
1475 .n_mcgrps = 1,
ba63fe26 1476 .module = THIS_MODULE,
aaff4b55
BP		 1477};
1478
0afa2373
TG		 1479static size_t ovs_dp_cmd_msg_size(void)
1480{
1481 size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
1482
1483 msgsize += nla_total_size(IFNAMSIZ);
91b37647
PS		 1484 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
1485 msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
300af20a 1486 msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
0afa2373
TG		 1487
1488 return msgsize;
1489}
1490
d637497c 1491/* Called with ovs_mutex. */
df2c07f4 1492static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
28aea917 1493 u32 portid, u32 seq, u32 flags, u8 cmd)
064af421 1494{
df2c07f4 1495 struct ovs_header *ovs_header;
e926dfe3 1496 struct ovs_dp_stats dp_stats;
4fa72a95 1497 struct ovs_dp_megaflow_stats dp_megaflow_stats;
064af421
BP		 1498 int err;
1499
28aea917 1500 ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
aaff4b55 1501 flags, cmd);
df2c07f4 1502 if (!ovs_header)
aaff4b55 1503 goto error;
064af421 1504
b063d9f0 1505 ovs_header->dp_ifindex = get_dpifindex(dp);
064af421 1506
850b6b3b 1507 err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
064af421 1508 if (err)
d6569377 1509 goto nla_put_failure;
064af421 1510
4fa72a95 1511 get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
91b37647
PS		 1512 if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
1513 &dp_stats, OVS_DP_ATTR_PAD))
4fa72a95
AZ		 1514 goto nla_put_failure;
1515
91b37647
PS		 1516 if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
1517 sizeof(struct ovs_dp_megaflow_stats),
1518 &dp_megaflow_stats, OVS_DP_ATTR_PAD))
c3cc8c03 1519 goto nla_put_failure;
d6569377 1520
c58cc9a4
TG		 1521 if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
1522 goto nla_put_failure;
1523
23b48dc1
TG		 1524 genlmsg_end(skb, ovs_header);
1525 return 0;
d6569377
BP		 1526
1527nla_put_failure:
df2c07f4 1528 genlmsg_cancel(skb, ovs_header);
aaff4b55
BP		 1529error:
1530 return -EMSGSIZE;
d6569377
BP		 1531}
1532
40c08cda 1533static struct sk_buff *ovs_dp_cmd_alloc_info(void)
d6569377 1534{
40c08cda 1535 return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
aaff4b55 1536}
9dca7bd5 1537
aa917006 1538/* Called with rcu_read_lock or ovs_mutex. */
2a4999f3 1539static struct datapath *lookup_datapath(struct net *net,
f1f60b85 1540 const struct ovs_header *ovs_header,
6455100f 1541 struct nlattr *a[OVS_DP_ATTR_MAX + 1])
d6569377 1542{
254f2dc8
BP		 1543 struct datapath *dp;
1544
df2c07f4 1545 if (!a[OVS_DP_ATTR_NAME])
2a4999f3 1546 dp = get_dp(net, ovs_header->dp_ifindex);
254f2dc8 1547 else {
d6569377 1548 struct vport *vport;
d6569377 1549
2a4999f3 1550 vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
df2c07f4 1551 dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
d6569377 1552 }
254f2dc8 1553 return dp ? dp : ERR_PTR(-ENODEV);
d6569377
BP		 1554}
1555
94358dcf
TG		 1556static void ovs_dp_reset_user_features(struct sk_buff *skb, struct genl_info *info)
1557{
1558 struct datapath *dp;
1559
1560 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
09350a3d 1561 if (IS_ERR(dp))
94358dcf
TG		 1562 return;
1563
1564 WARN(dp->user_features, "Dropping previously announced user features\n");
1565 dp->user_features = 0;
1566}
1567
f1f60b85 1568static void ovs_dp_change(struct datapath *dp, struct nlattr *a[])
c58cc9a4
TG		 1569{
1570 if (a[OVS_DP_ATTR_USER_FEATURES])
1571 dp->user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
1572}
1573
df2c07f4 1574static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
d6569377 1575{
aaff4b55 1576 struct nlattr **a = info->attrs;
d6569377 1577 struct vport_parms parms;
aaff4b55 1578 struct sk_buff *reply;
d6569377
BP		 1579 struct datapath *dp;
1580 struct vport *vport;
2a4999f3 1581 struct ovs_net *ovs_net;
95b1d73a 1582 int err, i;
d6569377 1583
d6569377 1584 err = -EINVAL;
ea36840f 1585 if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
aaff4b55
BP		 1586 goto err;
1587
40c08cda 1588 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1589 if (!reply)
1590 return -ENOMEM;
d6569377 1591
d6569377
BP		 1592 err = -ENOMEM;
1593 dp = kzalloc(sizeof(*dp), GFP_KERNEL);
1594 if (dp == NULL)
d81eef1b 1595 goto err_free_reply;
2a4999f3 1596
c0cddcec 1597 ovs_dp_set_net(dp, sock_net(skb->sk));
0ceaa66c 1598
d6569377 1599 /* Allocate table. */
994dc286
PS		 1600 err = ovs_flow_tbl_init(&dp->table);
1601 if (err)
d6569377
BP		 1602 goto err_free_dp;
1603
08fb1bbd 1604 dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
99769a40
JG		 1605 if (!dp->stats_percpu) {
1606 err = -ENOMEM;
1607 goto err_destroy_table;
1608 }
1609
8401bb6a
KC		 1610 dp->ports = kmalloc_array(DP_VPORT_HASH_BUCKETS,
1611 sizeof(struct hlist_head),
1612 GFP_KERNEL);
95b1d73a
PS		 1613 if (!dp->ports) {
1614 err = -ENOMEM;
1615 goto err_destroy_percpu;
1616 }
1617
1618 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
1619 INIT_HLIST_HEAD(&dp->ports[i]);
1620
1cb57039
AZ		 1621 err = ovs_meters_init(dp);
1622 if (err)
1623 goto err_destroy_ports_array;
1624
d6569377 1625 /* Set up our datapath device. */
df2c07f4
JP		 1626 parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
1627 parms.type = OVS_VPORT_TYPE_INTERNAL;
d6569377
BP		 1628 parms.options = NULL;
1629 parms.dp = dp;
df2c07f4 1630 parms.port_no = OVSP_LOCAL;
beb1c69a 1631 parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
b063d9f0 1632
c58cc9a4
TG		 1633 ovs_dp_change(dp, a);
1634
d81eef1b
JR		 1635 /* So far only local changes have been made, now need the lock. */
1636 ovs_lock();
1637
d6569377
BP		 1638 vport = new_vport(&parms);
1639 if (IS_ERR(vport)) {
1640 err = PTR_ERR(vport);
1641 if (err == -EBUSY)
1642 err = -EEXIST;
1643
94358dcf
TG		 1644 if (err == -EEXIST) {
1645 /* An outdated user space instance that does not understand
1646 * the concept of user_features has attempted to create a new
1647 * datapath and is likely to reuse it. Drop all user features.
1648 */
1649 if (info->genlhdr->version < OVS_DP_VER_FEATURES)
1650 ovs_dp_reset_user_features(skb, info);
1651 }
1652
1cb57039 1653 goto err_destroy_meters;
d6569377 1654 }
d6569377 1655
d81eef1b
JR		 1656 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1657 info->snd_seq, 0, OVS_DP_CMD_NEW);
1658 BUG_ON(err < 0);
aaff4b55 1659
2a4999f3 1660 ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
fb93e9aa 1661 list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
a0fb56c1 1662
cd2a59e9 1663 ovs_unlock();
d6569377 1664
cb25142c 1665 ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
d6569377
BP		 1666 return 0;
1667
1cb57039 1668err_destroy_meters:
d81eef1b 1669 ovs_unlock();
1cb57039
AZ		 1670 ovs_meters_exit(dp);
1671err_destroy_ports_array:
95b1d73a 1672 kfree(dp->ports);
99769a40
JG		 1673err_destroy_percpu:
1674 free_percpu(dp->stats_percpu);
d6569377 1675err_destroy_table:
e379e4d1 1676 ovs_flow_tbl_destroy(&dp->table);
d6569377 1677err_free_dp:
d6569377 1678 kfree(dp);
d81eef1b
JR		 1679err_free_reply:
1680 kfree_skb(reply);
d6569377 1681err:
064af421
BP		 1682 return err;
1683}
1684
cd2a59e9 1685/* Called with ovs_mutex. */
2a4999f3 1686static void __dp_destroy(struct datapath *dp)
44e05eca 1687{
95b1d73a 1688 int i;
44e05eca 1689
95b1d73a
PS		 1690 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1691 struct vport *vport;
f8dfbcb7 1692 struct hlist_node *n;
95b1d73a 1693
f8dfbcb7 1694 hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
95b1d73a
PS		 1695 if (vport->port_no != OVSP_LOCAL)
1696 ovs_dp_detach_port(vport);
1697 }
ed099e92 1698
fb93e9aa 1699 list_del_rcu(&dp->list_node);
ed099e92 1700
cd2a59e9 1701 /* OVSP_LOCAL is datapath internal port. We need to make sure that
d103f479
AZ		 1702 * all ports in datapath are destroyed first before freeing datapath.
1703 */
cd2a59e9 1704 ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
99620d2c 1705
d103f479 1706 /* RCU destroy the flow table */
ed099e92 1707 call_rcu(&dp->rcu, destroy_dp_rcu);
2a4999f3
PS		 1708}
1709
1710static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
1711{
1712 struct sk_buff *reply;
1713 struct datapath *dp;
1714 int err;
1715
40c08cda 1716 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1717 if (!reply)
1718 return -ENOMEM;
1719
cd2a59e9 1720 ovs_lock();
2a4999f3
PS		 1721 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
1722 err = PTR_ERR(dp);
1723 if (IS_ERR(dp))
d81eef1b 1724 goto err_unlock_free;
2a4999f3 1725
d81eef1b
JR		 1726 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
1727 info->snd_seq, 0, OVS_DP_CMD_DEL);
1728 BUG_ON(err < 0);
2a4999f3
PS		 1729
1730 __dp_destroy(dp);
d81eef1b 1731 ovs_unlock();
7d16c847 1732
cb25142c 1733 ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
99620d2c 1734 return 0;
d81eef1b
JR		 1735
1736err_unlock_free:
cd2a59e9 1737 ovs_unlock();
d81eef1b 1738 kfree_skb(reply);
cd2a59e9 1739 return err;
44e05eca
BP		 1740}
1741
df2c07f4 1742static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
064af421 1743{
aaff4b55 1744 struct sk_buff *reply;
d6569377 1745 struct datapath *dp;
d6569377 1746 int err;
064af421 1747
40c08cda 1748 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1749 if (!reply)
1750 return -ENOMEM;
1751
cd2a59e9 1752 ovs_lock();
2a4999f3 1753 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
cd2a59e9 1754 err = PTR_ERR(dp);
d6569377 1755 if (IS_ERR(dp))
d81eef1b 1756 goto err_unlock_free;
38c6ecbc 1757
c58cc9a4
TG		 1758 ovs_dp_change(dp, info->attrs);
1759
d81eef1b 1760 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
229486c2 1761 info->snd_seq, 0, OVS_DP_CMD_GET);
d81eef1b 1762 BUG_ON(err < 0);
a0fb56c1 1763
cd2a59e9 1764 ovs_unlock();
7d16c847 1765
cb25142c 1766 ovs_notify(&dp_datapath_genl_family, &ovs_dp_datapath_multicast_group, reply, info);
aaff4b55 1767 return 0;
d81eef1b
JR		 1768
1769err_unlock_free:
cd2a59e9 1770 ovs_unlock();
d81eef1b 1771 kfree_skb(reply);
cd2a59e9 1772 return err;
064af421
BP		 1773}
1774
df2c07f4 1775static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
1dcf111b 1776{
aaff4b55 1777 struct sk_buff *reply;
d6569377 1778 struct datapath *dp;
d6569377 1779 int err;
1dcf111b 1780
40c08cda 1781 reply = ovs_dp_cmd_alloc_info();
d81eef1b
JR		 1782 if (!reply)
1783 return -ENOMEM;
1784
d637497c 1785 ovs_lock();
2a4999f3 1786 dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
cd2a59e9
PS		 1787 if (IS_ERR(dp)) {
1788 err = PTR_ERR(dp);
d81eef1b 1789 goto err_unlock_free;
cd2a59e9 1790 }
d81eef1b 1791 err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
229486c2 1792 info->snd_seq, 0, OVS_DP_CMD_GET);
d81eef1b 1793 BUG_ON(err < 0);
d637497c 1794 ovs_unlock();
aaff4b55
BP		 1795
1796 return genlmsg_reply(reply, info);
cd2a59e9 1797
d81eef1b 1798err_unlock_free:
d637497c 1799 ovs_unlock();
d81eef1b 1800 kfree_skb(reply);
cd2a59e9 1801 return err;
1dcf111b
JP		 1802}
1803
df2c07f4 1804static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
a7786963 1805{
2a4999f3 1806 struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
254f2dc8
BP		 1807 struct datapath *dp;
1808 int skip = cb->args[0];
1809 int i = 0;
a7786963 1810
d637497c
PS		 1811 ovs_lock();
1812 list_for_each_entry(dp, &ovs_net->dps, list_node) {
a2bab2f0 1813 if (i >= skip &&
28aea917 1814 ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
aaff4b55 1815 cb->nlh->nlmsg_seq, NLM_F_MULTI,
229486c2 1816 OVS_DP_CMD_GET) < 0)
aaff4b55 1817 break;
254f2dc8 1818 i++;
a7786963 1819 }
d637497c 1820 ovs_unlock();
aaff4b55 1821
254f2dc8
BP		 1822 cb->args[0] = i;
1823
aaff4b55 1824 return skb->len;
c19e6535
BP		 1825}
1826
cb25142c
PS		 1827static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
1828 [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
1829 [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
1830 [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
1831};
1832
18fd3a52 1833static struct genl_ops dp_datapath_genl_ops[] = {
df2c07f4 1834 { .cmd = OVS_DP_CMD_NEW,
a6a8674d 1835 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
aaff4b55 1836 .policy = datapath_policy,
df2c07f4 1837 .doit = ovs_dp_cmd_new
aaff4b55 1838 },
df2c07f4 1839 { .cmd = OVS_DP_CMD_DEL,
a6a8674d 1840 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
aaff4b55 1841 .policy = datapath_policy,
df2c07f4 1842 .doit = ovs_dp_cmd_del
aaff4b55 1843 },
df2c07f4 1844 { .cmd = OVS_DP_CMD_GET,
aaff4b55
BP		 1845 .flags = 0, /* OK for unprivileged users. */
1846 .policy = datapath_policy,
df2c07f4
JP		 1847 .doit = ovs_dp_cmd_get,
1848 .dumpit = ovs_dp_cmd_dump
aaff4b55 1849 },
df2c07f4 1850 { .cmd = OVS_DP_CMD_SET,
a6a8674d 1851 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
aaff4b55 1852 .policy = datapath_policy,
df2c07f4 1853 .doit = ovs_dp_cmd_set,
aaff4b55
BP		 1854 },
1855};
1856
ba63fe26 1857static struct genl_family dp_datapath_genl_family __ro_after_init = {
df2c07f4 1858 .hdrsize = sizeof(struct ovs_header),
cb25142c
PS		 1859 .name = OVS_DATAPATH_FAMILY,
1860 .version = OVS_DATAPATH_VERSION,
1861 .maxattr = OVS_DP_ATTR_MAX,
b3dcb73c 1862 .netnsok = true,
cb25142c
PS		 1863 .parallel_ops = true,
1864 .ops = dp_datapath_genl_ops,
1865 .n_ops = ARRAY_SIZE(dp_datapath_genl_ops),
1866 .mcgrps = &ovs_dp_datapath_multicast_group,
1867 .n_mcgrps = 1,
ba63fe26 1868 .module = THIS_MODULE,
f0fef760
BP		 1869};
1870
cd2a59e9 1871/* Called with ovs_mutex or RCU read lock. */
df2c07f4 1872static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
b147f2e9
JB		 1873 struct net *net, u32 portid, u32 seq,
1874 u32 flags, u8 cmd)
064af421 1875{
df2c07f4 1876 struct ovs_header *ovs_header;
e926dfe3 1877 struct ovs_vport_stats vport_stats;
c19e6535
BP		 1878 int err;
1879
28aea917 1880 ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
f0fef760 1881 flags, cmd);
df2c07f4 1882 if (!ovs_header)
f0fef760 1883 return -EMSGSIZE;
c19e6535 1884
99769a40 1885 ovs_header->dp_ifindex = get_dpifindex(vport->dp);
c19e6535 1886
c3cc8c03
DM		 1887 if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
1888 nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
e23775f2 1889 nla_put_string(skb, OVS_VPORT_ATTR_NAME,
b147f2e9
JB		 1890 ovs_vport_name(vport)) ||
1891 nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
c3cc8c03 1892 goto nla_put_failure;
c19e6535 1893
b147f2e9
JB		 1894#ifdef HAVE_PEERNET2ID_ALLOC
1895 if (!net_eq(net, dev_net(vport->dev))) {
1896 int id = peernet2id_alloc(net, dev_net(vport->dev));
1897
1898 if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
1899 goto nla_put_failure;
1900 }
1901
1902#endif
850b6b3b 1903 ovs_vport_get_stats(vport, &vport_stats);
91b37647
PS		 1904 if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
1905 sizeof(struct ovs_vport_stats), &vport_stats,
1906 OVS_VPORT_ATTR_PAD))
c3cc8c03 1907 goto nla_put_failure;
c19e6535 1908
beb1c69a
AW		 1909 if (ovs_vport_get_upcall_portids(vport, skb))
1910 goto nla_put_failure;
1911
850b6b3b 1912 err = ovs_vport_get_options(vport, skb);
f0fef760
BP		 1913 if (err == -EMSGSIZE)
1914 goto error;
c19e6535 1915
23b48dc1
TG		 1916 genlmsg_end(skb, ovs_header);
1917 return 0;
c19e6535
BP		 1918
1919nla_put_failure:
1920 err = -EMSGSIZE;
f0fef760 1921error:
df2c07f4 1922 genlmsg_cancel(skb, ovs_header);
f0fef760 1923 return err;
064af421
BP		 1924}
1925
d81eef1b
JR		 1926static struct sk_buff *ovs_vport_cmd_alloc_info(void)
1927{
1928 return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
1929}
1930
1931/* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
b147f2e9
JB		 1932struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
1933 u32 portid, u32 seq, u8 cmd)
064af421 1934{
c19e6535 1935 struct sk_buff *skb;
f0fef760 1936 int retval;
c19e6535 1937
f0fef760 1938 skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_ATOMIC);
c19e6535
BP		 1939 if (!skb)
1940 return ERR_PTR(-ENOMEM);
1941
b147f2e9 1942 retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd);
c25ea534
JG		 1943 BUG_ON(retval < 0);
1944
c19e6535 1945 return skb;
f0fef760 1946}
c19e6535 1947
cd2a59e9 1948/* Called with ovs_mutex or RCU read lock. */
2a4999f3 1949static struct vport *lookup_vport(struct net *net,
f1f60b85 1950 const struct ovs_header *ovs_header,
df2c07f4 1951 struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
c19e6535
BP		 1952{
1953 struct datapath *dp;
1954 struct vport *vport;
1955
b147f2e9
JB		 1956 if (a[OVS_VPORT_ATTR_IFINDEX])
1957 return ERR_PTR(-EOPNOTSUPP);
df2c07f4 1958 if (a[OVS_VPORT_ATTR_NAME]) {
2a4999f3 1959 vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
ed099e92 1960 if (!vport)
c19e6535 1961 return ERR_PTR(-ENODEV);
24ce832d
BP		 1962 if (ovs_header->dp_ifindex &&
1963 ovs_header->dp_ifindex != get_dpifindex(vport->dp))
1964 return ERR_PTR(-ENODEV);
c19e6535 1965 return vport;
df2c07f4
JP		 1966 } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
1967 u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
c19e6535
BP		 1968
1969 if (port_no >= DP_MAX_PORTS)
f0fef760 1970 return ERR_PTR(-EFBIG);
c19e6535 1971
2a4999f3 1972 dp = get_dp(net, ovs_header->dp_ifindex);
c19e6535
BP		 1973 if (!dp)
1974 return ERR_PTR(-ENODEV);
f2459fe7 1975
cd2a59e9 1976 vport = ovs_vport_ovsl_rcu(dp, port_no);
ed099e92 1977 if (!vport)
17535c57 1978 return ERR_PTR(-ENODEV);
c19e6535
BP		 1979 return vport;
1980 } else
1981 return ERR_PTR(-EINVAL);
b147f2e9 1982
064af421
BP		 1983}
1984
8ce37339
PS		 1985/* Called with ovs_mutex */
1986static void update_headroom(struct datapath *dp)
1987{
1988 unsigned dev_headroom, max_headroom = 0;
1989 struct net_device *dev;
1990 struct vport *vport;
1991 int i;
1992
1993 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
1994 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
1995 dev = vport->dev;
1996 dev_headroom = netdev_get_fwd_headroom(dev);
1997 if (dev_headroom > max_headroom)
1998 max_headroom = dev_headroom;
1999 }
2000 }
2001
2002 dp->max_headroom = max_headroom;
2003 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
2004 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node)
2005 netdev_set_rx_headroom(vport->dev, max_headroom);
2006}
2007
df2c07f4 2008static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
c19e6535 2009{
f0fef760 2010 struct nlattr **a = info->attrs;
df2c07f4 2011 struct ovs_header *ovs_header = info->userhdr;
c19e6535 2012 struct vport_parms parms;
ed099e92 2013 struct sk_buff *reply;
c19e6535 2014 struct vport *vport;
c19e6535 2015 struct datapath *dp;
b0ec0f27 2016 u32 port_no;
c19e6535 2017 int err;
b0ec0f27 2018
ea36840f
BP		 2019 if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
2020 !a[OVS_VPORT_ATTR_UPCALL_PID])
d81eef1b 2021 return -EINVAL;
b147f2e9
JB		 2022 if (a[OVS_VPORT_ATTR_IFINDEX])
2023 return -EOPNOTSUPP;
d81eef1b
JR		 2024
2025 port_no = a[OVS_VPORT_ATTR_PORT_NO]
2026 ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
2027 if (port_no >= DP_MAX_PORTS)
2028 return -EFBIG;
2029
2030 reply = ovs_vport_cmd_alloc_info();
2031 if (!reply)
2032 return -ENOMEM;
f0fef760 2033
cd2a59e9 2034 ovs_lock();
5a38795f 2035restart:
2a4999f3 2036 dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
c19e6535
BP		 2037 err = -ENODEV;
2038 if (!dp)
d81eef1b 2039 goto exit_unlock_free;
c19e6535 2040
d81eef1b 2041 if (port_no) {
cd2a59e9 2042 vport = ovs_vport_ovsl(dp, port_no);
c19e6535
BP		 2043 err = -EBUSY;
2044 if (vport)
d81eef1b 2045 goto exit_unlock_free;
c19e6535
BP		 2046 } else {
2047 for (port_no = 1; ; port_no++) {
2048 if (port_no >= DP_MAX_PORTS) {
2049 err = -EFBIG;
d81eef1b 2050 goto exit_unlock_free;
c19e6535 2051 }
cd2a59e9 2052 vport = ovs_vport_ovsl(dp, port_no);
c19e6535
BP		 2053 if (!vport)
2054 break;
51d4d598 2055 }
064af421 2056 }
b0ec0f27 2057
df2c07f4
JP		 2058 parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
2059 parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
2060 parms.options = a[OVS_VPORT_ATTR_OPTIONS];
c19e6535
BP		 2061 parms.dp = dp;
2062 parms.port_no = port_no;
beb1c69a 2063 parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
c19e6535
BP		 2064
2065 vport = new_vport(&parms);
2066 err = PTR_ERR(vport);
5a38795f
TG		 2067 if (IS_ERR(vport)) {
2068 if (err == -EAGAIN)
2069 goto restart;
d81eef1b 2070 goto exit_unlock_free;
5a38795f 2071 }
c19e6535 2072
b147f2e9
JB		 2073 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2074 info->snd_portid, info->snd_seq, 0,
2075 OVS_VPORT_CMD_NEW);
d81eef1b 2076 BUG_ON(err < 0);
8ce37339
PS		 2077
2078 if (netdev_get_fwd_headroom(vport->dev) > dp->max_headroom)
2079 update_headroom(dp);
2080 else
2081 netdev_set_rx_headroom(vport->dev, dp->max_headroom);
2082
d81eef1b 2083 ovs_unlock();
e297c6b7 2084
cb25142c 2085 ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
d81eef1b 2086 return 0;
c19e6535 2087
d81eef1b 2088exit_unlock_free:
cd2a59e9 2089 ovs_unlock();
d81eef1b 2090 kfree_skb(reply);
c19e6535 2091 return err;
44e05eca
BP		 2092}
2093
df2c07f4 2094static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
44e05eca 2095{
f0fef760
BP		 2096 struct nlattr **a = info->attrs;
2097 struct sk_buff *reply;
c19e6535 2098 struct vport *vport;
c19e6535 2099 int err;
44e05eca 2100
d81eef1b
JR		 2101 reply = ovs_vport_cmd_alloc_info();
2102 if (!reply)
2103 return -ENOMEM;
2104
cd2a59e9 2105 ovs_lock();
2a4999f3 2106 vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
c19e6535
BP		 2107 err = PTR_ERR(vport);
2108 if (IS_ERR(vport))
d81eef1b 2109 goto exit_unlock_free;
44e05eca 2110
6455100f 2111 if (a[OVS_VPORT_ATTR_TYPE] &&
17ec1d04 2112 nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
4879d4c7 2113 err = -EINVAL;
d81eef1b 2114 goto exit_unlock_free;
c25ea534
JG		 2115 }
2116
17ec1d04 2117 if (a[OVS_VPORT_ATTR_OPTIONS]) {
850b6b3b 2118 err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
17ec1d04 2119 if (err)
d81eef1b 2120 goto exit_unlock_free;
17ec1d04 2121 }
1fc7083d 2122
beb1c69a 2123 if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
7d16c847
PS		 2124 struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
2125
2126 err = ovs_vport_set_upcall_portids(vport, ids);
beb1c69a
AW		 2127 if (err)
2128 goto exit_unlock_free;
2129 }
c19e6535 2130
b147f2e9
JB		 2131 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2132 info->snd_portid, info->snd_seq, 0,
229486c2 2133 OVS_VPORT_CMD_SET);
c25ea534 2134 BUG_ON(err < 0);
cd2a59e9 2135 ovs_unlock();
d81eef1b 2136
cb25142c 2137 ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
c25ea534
JG		 2138 return 0;
2139
d81eef1b 2140exit_unlock_free:
cd2a59e9 2141 ovs_unlock();
d81eef1b 2142 kfree_skb(reply);
c19e6535 2143 return err;
064af421
BP		 2144}
2145
df2c07f4 2146static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
7c40efc9 2147{
8ce37339 2148 bool must_update_headroom = false;
f0fef760
BP		 2149 struct nlattr **a = info->attrs;
2150 struct sk_buff *reply;
8ce37339 2151 struct datapath *dp;
c19e6535 2152 struct vport *vport;
c19e6535
BP		 2153 int err;
2154
d81eef1b
JR		 2155 reply = ovs_vport_cmd_alloc_info();
2156 if (!reply)
2157 return -ENOMEM;
2158
cd2a59e9 2159 ovs_lock();
2a4999f3 2160 vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
c19e6535 2161 err = PTR_ERR(vport);
f0fef760 2162 if (IS_ERR(vport))
d81eef1b 2163 goto exit_unlock_free;
c19e6535 2164
df2c07f4 2165 if (vport->port_no == OVSP_LOCAL) {
f0fef760 2166 err = -EINVAL;
d81eef1b 2167 goto exit_unlock_free;
f0fef760
BP		 2168 }
2169
b147f2e9
JB		 2170 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2171 info->snd_portid, info->snd_seq, 0,
2172 OVS_VPORT_CMD_DEL);
d81eef1b 2173 BUG_ON(err < 0);
8ce37339
PS		 2174
2175 /* the vport deletion may trigger dp headroom update */
2176 dp = vport->dp;
2177 if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
2178 must_update_headroom = true;
2179 netdev_reset_rx_headroom(vport->dev);
850b6b3b 2180 ovs_dp_detach_port(vport);
8ce37339
PS		 2181
2182 if (must_update_headroom)
2183 update_headroom(dp);
2184
d81eef1b 2185 ovs_unlock();
f0fef760 2186
cb25142c 2187 ovs_notify(&dp_vport_genl_family, &ovs_dp_vport_multicast_group, reply, info);
d81eef1b 2188 return 0;
f0fef760 2189
d81eef1b 2190exit_unlock_free:
cd2a59e9 2191 ovs_unlock();
d81eef1b 2192 kfree_skb(reply);
c19e6535 2193 return err;
7c40efc9
BP		 2194}
2195
df2c07f4 2196static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
7c40efc9 2197{
f0fef760 2198 struct nlattr **a = info->attrs;
df2c07f4 2199 struct ovs_header *ovs_header = info->userhdr;
ed099e92 2200 struct sk_buff *reply;
c19e6535 2201 struct vport *vport;
c19e6535
BP		 2202 int err;
2203
d81eef1b
JR		 2204 reply = ovs_vport_cmd_alloc_info();
2205 if (!reply)
2206 return -ENOMEM;
2207
ed099e92 2208 rcu_read_lock();
2a4999f3 2209 vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
c19e6535
BP		 2210 err = PTR_ERR(vport);
2211 if (IS_ERR(vport))
d81eef1b 2212 goto exit_unlock_free;
b147f2e9
JB		 2213 err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
2214 info->snd_portid, info->snd_seq, 0,
229486c2 2215 OVS_VPORT_CMD_GET);
d81eef1b 2216 BUG_ON(err < 0);
df2fa9b5
JG		 2217 rcu_read_unlock();
2218
2219 return genlmsg_reply(reply, info);
ed099e92 2220
d81eef1b 2221exit_unlock_free:
ed099e92 2222 rcu_read_unlock();
d81eef1b 2223 kfree_skb(reply);
c19e6535
BP		 2224 return err;
2225}
2226
df2c07f4 2227static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
c19e6535 2228{
df2c07f4 2229 struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
c19e6535 2230 struct datapath *dp;
95b1d73a
PS		 2231 int bucket = cb->args[0], skip = cb->args[1];
2232 int i, j = 0;
c19e6535 2233
03fc2881 2234 rcu_read_lock();
01ac0970 2235 dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
03fc2881
JR		 2236 if (!dp) {
2237 rcu_read_unlock();
f0fef760 2238 return -ENODEV;
03fc2881 2239 }
95b1d73a 2240 for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
ed099e92 2241 struct vport *vport;
95b1d73a
PS		 2242
2243 j = 0;
f8dfbcb7 2244 hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
95b1d73a
PS		 2245 if (j >= skip &&
2246 ovs_vport_cmd_fill_info(vport, skb,
b147f2e9 2247 sock_net(skb->sk),
28aea917 2248 NETLINK_CB(cb->skb).portid,
95b1d73a
PS		 2249 cb->nlh->nlmsg_seq,
2250 NLM_F_MULTI,
229486c2 2251 OVS_VPORT_CMD_GET) < 0)
95b1d73a
PS		 2252 goto out;
2253
2254 j++;
2255 }
2256 skip = 0;
c19e6535 2257 }
95b1d73a 2258out:
ed099e92 2259 rcu_read_unlock();
c19e6535 2260
95b1d73a
PS		 2261 cb->args[0] = i;
2262 cb->args[1] = j;
f0fef760 2263
95b1d73a 2264 return skb->len;
7c40efc9
BP		 2265}
2266
cb25142c
PS		 2267static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
2268 [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
2269 [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
2270 [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
2271 [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
2272 [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_U32 },
2273 [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
b147f2e9
JB		 2274 [OVS_VPORT_ATTR_IFINDEX] = { .type = NLA_U32 },
2275 [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
cb25142c
PS		 2276};
2277
18fd3a52 2278static struct genl_ops dp_vport_genl_ops[] = {
df2c07f4 2279 { .cmd = OVS_VPORT_CMD_NEW,
a6a8674d 2280 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
f0fef760 2281 .policy = vport_policy,
df2c07f4 2282 .doit = ovs_vport_cmd_new
f0fef760 2283 },
df2c07f4 2284 { .cmd = OVS_VPORT_CMD_DEL,
a6a8674d 2285 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
f0fef760 2286 .policy = vport_policy,
df2c07f4 2287 .doit = ovs_vport_cmd_del
f0fef760 2288 },
df2c07f4 2289 { .cmd = OVS_VPORT_CMD_GET,
f0fef760
BP		 2290 .flags = 0, /* OK for unprivileged users. */
2291 .policy = vport_policy,
df2c07f4
JP		 2292 .doit = ovs_vport_cmd_get,
2293 .dumpit = ovs_vport_cmd_dump
f0fef760 2294 },
df2c07f4 2295 { .cmd = OVS_VPORT_CMD_SET,
a6a8674d 2296 .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
f0fef760 2297 .policy = vport_policy,
df2c07f4 2298 .doit = ovs_vport_cmd_set,
f0fef760
BP		 2299 },
2300};
2301
ba63fe26 2302struct genl_family dp_vport_genl_family __ro_after_init = {
cb25142c
PS		 2303 .hdrsize = sizeof(struct ovs_header),
2304 .name = OVS_VPORT_FAMILY,
2305 .version = OVS_VPORT_VERSION,
2306 .maxattr = OVS_VPORT_ATTR_MAX,
2307 .netnsok = true,
2308 .parallel_ops = true,
2309 .ops = dp_vport_genl_ops,
2310 .n_ops = ARRAY_SIZE(dp_vport_genl_ops),
2311 .mcgrps = &ovs_dp_vport_multicast_group,
2312 .n_mcgrps = 1,
ba63fe26 2313 .module = THIS_MODULE,
982b8810 2314};
ed099e92 2315
18fd3a52 2316static struct genl_family *dp_genl_families[] = {
cb25142c
PS		 2317 &dp_datapath_genl_family,
2318 &dp_vport_genl_family,
2319 &dp_flow_genl_family,
2320 &dp_packet_genl_family,
1cb57039 2321 &dp_meter_genl_family,
cb2a5486
YHW		 2322#if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
2323 &dp_ct_limit_genl_family,
2324#endif
982b8810 2325};
ed099e92 2326
982b8810
BP		 2327static void dp_unregister_genl(int n_families)
2328{
2329 int i;
ed099e92 2330
b867ca75 2331 for (i = 0; i < n_families; i++)
cb25142c 2332 genl_unregister_family(dp_genl_families[i]);
ed099e92
BP		 2333}
2334
ba63fe26 2335static int __init dp_register_genl(void)
064af421 2336{
982b8810
BP		 2337 int err;
2338 int i;
064af421 2339
982b8810 2340 for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
064af421 2341
cb25142c 2342 err = genl_register_family(dp_genl_families[i]);
982b8810
BP		 2343 if (err)
2344 goto error;
982b8810 2345 }
9cc8b4e4 2346
982b8810 2347 return 0;
064af421
BP		 2348
2349error:
cb25142c 2350 dp_unregister_genl(i);
982b8810 2351 return err;
064af421
BP		 2352}
2353
2a4999f3
PS		 2354static int __net_init ovs_init_net(struct net *net)
2355{
2356 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
2357
2358 INIT_LIST_HEAD(&ovs_net->dps);
cd2a59e9 2359 INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
7f4a5d68 2360 ovs_netns_frags_init(net);
2361 ovs_netns_frags6_init(net);
cb2a5486 2362 return ovs_ct_init(net);
2a4999f3
PS		 2363}
2364
cabd5516
PS		 2365static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
2366 struct list_head *head)
2a4999f3
PS		 2367{
2368 struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
cabd5516
PS		 2369 struct datapath *dp;
2370
2371 list_for_each_entry(dp, &ovs_net->dps, list_node) {
2372 int i;
2373
2374 for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
2375 struct vport *vport;
2376
2377 hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
cabd5516
PS		 2378
2379 if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
2380 continue;
2381
e23775f2 2382 if (dev_net(vport->dev) == dnet)
cabd5516
PS		 2383 list_add(&vport->detach_list, head);
2384 }
2385 }
2386 }
2387}
2388
2389static void __net_exit ovs_exit_net(struct net *dnet)
2390{
2391 struct datapath *dp, *dp_next;
2392 struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
2393 struct vport *vport, *vport_next;
2394 struct net *net;
2395 LIST_HEAD(head);
2a4999f3 2396
7f4a5d68 2397 ovs_netns_frags6_exit(dnet);
2398 ovs_netns_frags_exit(dnet);
038e34ab 2399 ovs_ct_exit(dnet);
cd2a59e9
PS		 2400 ovs_lock();
2401 list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
2402 __dp_destroy(dp);
cabd5516 2403
3d10a0c8
YHW		 2404#ifdef HAVE_NET_RWSEM
2405 down_read(&net_rwsem);
2406#else
cabd5516 2407 rtnl_lock();
3d10a0c8 2408#endif
cabd5516
PS		 2409 for_each_net(net)
2410 list_vports_from_net(net, dnet, &head);
3d10a0c8
YHW		 2411#ifdef HAVE_NET_RWSEM
2412 up_read(&net_rwsem);
2413#else
cabd5516 2414 rtnl_unlock();
3d10a0c8 2415#endif
cabd5516
PS		 2416
2417 /* Detach all vports from given namespace. */
2418 list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
2419 list_del(&vport->detach_list);
2420 ovs_dp_detach_port(vport);
2421 }
2422
cd2a59e9
PS		 2423 ovs_unlock();
2424
2425 cancel_work_sync(&ovs_net->dp_notify_work);
2a4999f3
PS		 2426}
2427
2428static struct pernet_operations ovs_net_ops = {
2429 .init = ovs_init_net,
2430 .exit = ovs_exit_net,
2431 .id = &ovs_net_id,
2432 .size = sizeof(struct ovs_net),
2433};
2434
22d24ebf
BP		 2435static int __init dp_init(void)
2436{
2437 int err;
2438
f3d85db3 2439 BUILD_BUG_ON(sizeof(struct ovs_skb_cb) > FIELD_SIZEOF(struct sk_buff, cb));
22d24ebf 2440
26bfaeaa 2441 pr_info("Open vSwitch switching datapath %s\n", VERSION);
064af421 2442
907c26a8 2443 ovs_nsh_init();
595e069a
JS		 2444 err = action_fifos_init();
2445 if (err)
7f4a5d68 2446 goto error;
595e069a 2447
5282e284 2448 err = ovs_internal_dev_rtnl_link_register();
2c8c4fb7
AZ		 2449 if (err)
2450 goto error_action_fifos_exit;
2451
5282e284
TG		 2452 err = ovs_flow_init();
2453 if (err)
2454 goto error_unreg_rtnl_link;
2455
850b6b3b 2456 err = ovs_vport_init();
064af421
BP		 2457 if (err)
2458 goto error_flow_exit;
2459
2a4999f3 2460 err = register_pernet_device(&ovs_net_ops);
f2459fe7
JG		 2461 if (err)
2462 goto error_vport_exit;
2463
7f4a5d68 2464 err = compat_init();
2a4999f3
PS		 2465 if (err)
2466 goto error_netns_exit;
2467
7f4a5d68 2468 err = register_netdevice_notifier(&ovs_dp_device_notifier);
2469 if (err)
2470 goto error_compat_exit;
2471
5a38795f
TG		 2472 err = ovs_netdev_init();
2473 if (err)
2474 goto error_unreg_notifier;
2475
982b8810
BP		 2476 err = dp_register_genl();
2477 if (err < 0)
5a38795f 2478 goto error_unreg_netdev;
982b8810 2479
064af421
BP		 2480 return 0;
2481
5a38795f
TG		 2482error_unreg_netdev:
2483 ovs_netdev_exit();
064af421 2484error_unreg_notifier:
850b6b3b 2485 unregister_netdevice_notifier(&ovs_dp_device_notifier);
7f4a5d68 2486error_compat_exit:
2487 compat_exit();
2a4999f3
PS		 2488error_netns_exit:
2489 unregister_pernet_device(&ovs_net_ops);
f2459fe7 2490error_vport_exit:
850b6b3b 2491 ovs_vport_exit();
064af421 2492error_flow_exit:
850b6b3b 2493 ovs_flow_exit();
5282e284
TG		 2494error_unreg_rtnl_link:
2495 ovs_internal_dev_rtnl_link_unregister();
2c8c4fb7
AZ		 2496error_action_fifos_exit:
2497 action_fifos_exit();
064af421 2498error:
907c26a8 2499 ovs_nsh_cleanup();
064af421
BP		 2500 return err;
2501}
2502
2503static void dp_cleanup(void)
2504{
982b8810 2505 dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
5a38795f 2506 ovs_netdev_exit();
850b6b3b 2507 unregister_netdevice_notifier(&ovs_dp_device_notifier);
7f4a5d68 2508 compat_exit();
2a4999f3
PS		 2509 unregister_pernet_device(&ovs_net_ops);
2510 rcu_barrier();
850b6b3b
JG		 2511 ovs_vport_exit();
2512 ovs_flow_exit();
5282e284 2513 ovs_internal_dev_rtnl_link_unregister();
2c8c4fb7 2514 action_fifos_exit();
907c26a8 2515 ovs_nsh_cleanup();
064af421
BP		 2516}
2517
2518module_init(dp_init);
2519module_exit(dp_cleanup);
2520
2521MODULE_DESCRIPTION("Open vSwitch switching datapath");
2522MODULE_LICENSE("GPL");
3d0666d2 2523MODULE_VERSION(VERSION);
75e2077e
TLSC		 2524MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
2525MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
2526MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
2527MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
1cb57039 2528MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
cb2a5486 2529MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);
openvswitch mirror